From 38d404aa2bf09e8014dde818eb0ee67185c7681d Mon Sep 17 00:00:00 2001 From: Ben Sima Date: Mon, 18 May 2026 16:27:20 -0400 Subject: pin models: parasail-qwen3-235b on Parasail, qwen2.5:14b locally --- main.py | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/main.py b/main.py index b6d57dc..d5c73ec 100644 --- a/main.py +++ b/main.py @@ -65,35 +65,22 @@ log.info("Connected as %s", client.email) # LLM client setup: Parasail primary, Ollama fallback # --------------------------------------------------------------------------- +# Model config: big Qwen3 on Parasail, smaller local Qwen2.5 as fallback +PARASAIL_MODEL = "parasail-qwen3-235b-a22b-instruct-2507" +OLLAMA_MODEL = "qwen2.5:14b-instruct-q4_K_M" + + def _make_llm_client() -> tuple[OpenAI, str]: """Return (openai_client, model_name) for the best available backend.""" parasail_key = os.environ.get("PARASAIL_API_KEY", "") if parasail_key: - log.info("Calling Parasail for LLM response") - # Prefer a small/fast model; fall back to Llama if unavailable - try: - r = requests.get( - "https://api.parasail.io/v1/models", - headers={"Authorization": f"Bearer {parasail_key}"}, - timeout=8, - ) - model_ids = [m["id"] for m in r.json().get("data", [])] - log.info("Parasail available models: %s", model_ids) - preferred = ["qwen3-4b", "meta-llama/Llama-3.1-8B-Instruct"] - model = next((m for m in preferred if m in model_ids), None) - if model is None and model_ids: - model = model_ids[0] - model = model or "meta-llama/Llama-3.1-8B-Instruct" - except Exception as e: - log.warning("Could not fetch Parasail models (%s); using default", e) - model = "meta-llama/Llama-3.1-8B-Instruct" - + log.info("Using Parasail model: %s", PARASAIL_MODEL) c = OpenAI(api_key=parasail_key, base_url="https://api.parasail.io/v1") - return c, model + return c, PARASAIL_MODEL - log.info("Calling Ollama for LLM response") + log.info("Using Ollama model: %s", OLLAMA_MODEL) c = OpenAI(api_key="ollama", base_url="http://localhost:11434/v1") - return c, "qwen2.5:3b" + return c, OLLAMA_MODEL # --------------------------------------------------------------------------- -- cgit v1.2.3