Add locked-down research tools

7 files changed, 137 insertions, 14 deletions

diff --git a/.env.example b/.env.example
new file mode 100644
index 0000000..6362d2f
--- /dev/null
+++ b/.env.example
@@ -0,0 +1,9 @@
+# Required by slop-search.
+KAGI_API_KEY=
+
+SLOPBOT_AGENT_MODEL=parasail/kimi-k26
+SLOPBOT_WORKERS=4
+SLOPBOT_AGENT_WORKERS=1
+SLOPBOT_AGENT_TIMEOUT=120
+SLOPBOT_HISTORY_LIMIT=20
+SLOPBOT_SYSTEM_PROMPT=prompts/system.md
diff --git a/README.md b/README.md
index 273fb83..20a0bb4 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,9 @@ Post `$NVDA` or `$AAPL is mooning` in any stream and slopbot replies with the cu
 ```bash
 pip install -r requirements.txt
 cp .zuliprc.example .zuliprc
+cp .env.example .env
 # edit .zuliprc with your bot credentials
+# edit .env with agent/provider settings if needed
 python main.py
 ```
 
@@ -22,6 +24,10 @@ Get your `.zuliprc` from Zulip: Settings → Bots → your bot → download conf
 - `main.py` — single-file bot, raw `zulip` event loop (no framework)
 - `fetch_price(ticker)` — isolated data layer, easy to swap yfinance for a real API
 - `ask_agent(msg, content)` — runs the Rust `agent` binary once per mention
+- `prompts/system.md` — bot persona, research workflow, and response behavior
+- `slop-search QUERY` — Kagi search helper available to the agent shell
+- `slop-fetch URL` — trafilatura page extraction helper available to the agent shell
+- `.env.example` — operational defaults, including `KAGI_API_KEY`
 - Regex: `\$([A-Z]{1,5})` — matches 1-5 uppercase letters after `$`
 - Deduplicates tickers per message, replies in-place (same stream + topic)
 - `flake.nix` — packages slopbot with `agent` from `/home/ben/src/bsima/agentd` as a Nix input
diff --git a/deploy.sh b/deploy.sh
index 6dfe14c..5517f94 100755
--- a/deploy.sh
+++ b/deploy.sh
@@ -15,10 +15,15 @@ fi
 
 if [[ ! -f .env ]]; then
   cat > .env <<'EOF'
+# Required by slop-search.
+KAGI_API_KEY=
+
 SLOPBOT_AGENT_MODEL=parasail/kimi-k26
 SLOPBOT_WORKERS=4
 SLOPBOT_AGENT_WORKERS=1
 SLOPBOT_AGENT_TIMEOUT=120
+SLOPBOT_HISTORY_LIMIT=20
+SLOPBOT_SYSTEM_PROMPT=prompts/system.md
 EOF
   echo "created .env with slopbot defaults"
   echo "edit .env if agent needs provider keys or config before restarting again"
diff --git a/flake.nix b/flake.nix
index 843f9c5..a70e64e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -29,10 +29,78 @@
         cargoBuildFlags = [ "-p" "agent" ];
         doCheck = false;
       };
+
+      researchPython = pkgs.python3.withPackages (ps: [
+        ps.trafilatura
+      ]);
+
+      slopSearch = pkgs.writeShellApplication {
+        name = "slop-search";
+        runtimeInputs = [ pkgs.curl pkgs.jq pkgs.python3 ];
+        text = ''
+          if [ "$#" -eq 0 ]; then
+            echo "usage: slop-search QUERY" >&2
+            exit 2
+          fi
+          if [ -z "''${KAGI_API_KEY:-}" ]; then
+            echo "error: KAGI_API_KEY is not set" >&2
+            exit 1
+          fi
+
+          query="$*"
+          encoded="$(${pkgs.python3}/bin/python -c 'import sys, urllib.parse; print(urllib.parse.urlencode({"q": " ".join(sys.argv[1:])}))' "$@")"
+
+          curl -fsS \
+            -H "Authorization: Bot ''${KAGI_API_KEY}" \
+            -H "Accept: application/json" \
+            "https://kagi.com/api/v0/search?''${encoded}" \
+          | jq -r --arg query "$query" '
+              "QUERY: \($query)",
+              "",
+              (.data // [])
+              | map(select((.t // .type // "") == "search_result" or has("url")))
+              | .[:8]
+              | to_entries[]
+              | "[\(.key + 1)] \(.value.title // "untitled")\nURL: \(.value.url)\nSNIPPET: \(.value.snippet // .value.description // "")\n"
+            '
+        '';
+      };
+
+      slopFetch = pkgs.writeShellApplication {
+        name = "slop-fetch";
+        runtimeInputs = [ pkgs.curl researchPython ];
+        text = ''
+          if [ "$#" -ne 1 ]; then
+            echo "usage: slop-fetch URL" >&2
+            exit 2
+          fi
+          url="$1"
+          tmp="$(mktemp)"
+          trap 'rm -f "$tmp"' EXIT
+          curl -fsSL --max-time 20 -A 'slopbot/1.0' "$url" -o "$tmp"
+          ${researchPython}/bin/python - "$url" "$tmp" <<'PY'
+import sys
+import trafilatura
+
+url, path = sys.argv[1], sys.argv[2]
+html = open(path, "rb").read()
+text = trafilatura.extract(html, url=url, include_comments=False, include_tables=False)
+if not text:
+    print(f"error: could not extract readable text from {url}", file=sys.stderr)
+    sys.exit(1)
+print(text[:12000])
+PY
+        '';
+      };
+
+      slopTools = pkgs.symlinkJoin {
+        name = "slopbot-agent-tools";
+        paths = [ slopSearch slopFetch ];
+      };
     in
     {
       devShells.${system}.default = pkgs.mkShell {
-        buildInputs = [ pythonEnv agent ];
+        buildInputs = [ pythonEnv agent slopTools ];
         shellHook = ''
           echo "slopbot dev shell ready"
           echo "Run: python main.py"
@@ -43,11 +111,12 @@
         default = slopbot;
         slopbot = pkgs.writeShellScriptBin "slopbot" ''
           cd /home/ben/src/bsima/slopbot
-          export PATH=${agent}/bin:$PATH
+          export PATH=${slopTools}/bin
           export SLOPBOT_AGENT=${agent}/bin/agent
+          export SLOPBOT_AGENT_PATH=${slopTools}/bin
           exec ${pythonEnv}/bin/python ${./main.py}
         '';
-        inherit agent;
+        inherit agent slopTools slopSearch slopFetch;
       };
     };
 }
diff --git a/main.py b/main.py
index 5e1e61a..af4f657 100644
--- a/main.py
+++ b/main.py
@@ -35,21 +35,20 @@ TICKER_RE = re.compile(r"\$([A-Z]{1,5})")
 # Regex to detect @mention of slop / slopbot
 MENTION_RE = re.compile(r"@\*\*slop[^*]*\*\*", re.IGNORECASE)
 
+BASE_DIR = Path(__file__).parent
+
 AGENT_TIMEOUT = int(os.environ.get("SLOPBOT_AGENT_TIMEOUT", "120"))
 AGENT_MODEL = os.environ.get("SLOPBOT_AGENT_MODEL", "parasail/kimi-k26")
 AGENT_BIN = os.environ.get("SLOPBOT_AGENT", "agent")
+AGENT_PATH = os.environ.get("SLOPBOT_AGENT_PATH", os.environ.get("PATH", ""))
 WORKERS = int(os.environ.get("SLOPBOT_WORKERS", "4"))
 AGENT_WORKERS = int(os.environ.get("SLOPBOT_AGENT_WORKERS", "1"))
-HISTORY_LIMIT = 20
+HISTORY_LIMIT = int(os.environ.get("SLOPBOT_HISTORY_LIMIT", "20"))
+SYSTEM_PROMPT_PATH = Path(os.environ.get("SLOPBOT_SYSTEM_PROMPT", "prompts/system.md"))
+if not SYSTEM_PROMPT_PATH.is_absolute():
+    SYSTEM_PROMPT_PATH = BASE_DIR / SYSTEM_PROMPT_PATH
 
-SYSTEM_PROMPT = """You are slop-bot in the meshheads.org Zulip chat: technically sharp,
-opinionated, libertarian-leaning, dryly funny, and peer-level with the audience.
-Assume the audience knows CS basics; do not over-explain and do not be obsequious.
-Use the provided topic history for conversational context, but answer the latest user.
-Search the web for current facts, news, sources, or prices using your search tool;
-when you use it, cite source URLs in your final answer.
-Keep replies short (2-4 sentences max unless asked for more).
-You are participating in an ongoing Zulip thread. Stay in character."""
+SYSTEM_PROMPT = SYSTEM_PROMPT_PATH.read_text().strip()
 
 log.info("Connecting to Zulip...")
 client = zulip.Client(config_file=".zuliprc")
@@ -112,14 +111,16 @@ def fetch_topic_history(msg: dict, limit: int = HISTORY_LIMIT) -> str:
 
 def _agent(prompt: str) -> tuple[int, str, str]:
     """Run agent once; return (returncode, stdout, stderr)."""
+    env = os.environ.copy()
+    env["PATH"] = AGENT_PATH
     result = subprocess.run(
         [
             AGENT_BIN,
             "--model", AGENT_MODEL,
-            "--eval-cwd", str(Path(__file__).parent),
+            "--eval-cwd", str(BASE_DIR),
             prompt,
         ],
-        capture_output=True, text=True, timeout=AGENT_TIMEOUT,
+        capture_output=True, text=True, timeout=AGENT_TIMEOUT, env=env,
     )
     return result.returncode, result.stdout.strip(), result.stderr.strip()
 
diff --git a/prompts/system.md b/prompts/system.md
new file mode 100644
index 0000000..4fa8a46
--- /dev/null
+++ b/prompts/system.md
@@ -0,0 +1,30 @@
+You are slop-bot in the meshheads.org Zulip chat: technically sharp,
+opinionated, libertarian-leaning, dryly funny, and peer-level with the audience.
+Assume the audience knows CS basics; do not over-explain and do not be obsequious.
+Use the provided topic history for conversational context, but answer the latest user.
+
+A major use case is quick internet research: news, source lookup, and concise summaries.
+You have a shell tool, but its PATH is intentionally locked down. Use only these commands for web research:
+
+- `slop-search QUERY` searches Kagi and returns titles, snippets, and source URLs.
+- `slop-fetch URL` fetches a page and extracts readable article text with trafilatura.
+
+Examples:
+
+```sh
+slop-search 'latest NVDA earnings guidance Reuters'
+slop-search 'Anthropic Claude Sonnet 4.6 release notes'
+slop-fetch 'https://example.com/article'
+```
+
+Research workflow:
+
+1. Run `slop-search` for current facts, news, source lookup, prices, or anything likely to have changed.
+2. Use the returned URLs. Do not cite URLs that did not appear in search results or fetch output.
+3. Use `slop-fetch URL` when snippets are not enough or the user asks for a summary of a specific article.
+4. Always include the source URLs you actually used in your final answer when you did internet research.
+5. Do not invent links, citations, article titles, publishers, dates, quotes, or references.
+6. If you cannot verify something from search results or fetched pages, say that instead of guessing.
+
+Keep replies short (2-4 sentences max unless asked for more).
+You are participating in an ongoing Zulip thread. Stay in character.
diff --git a/slopbot.service b/slopbot.service
index 396e9d0..7b582e0 100644
--- a/slopbot.service
+++ b/slopbot.service
@@ -6,7 +6,10 @@ Wants=network-online.target
 [Service]
 Type=simple
 WorkingDirectory=/home/ben/src/bsima/slopbot
+Environment=PATH=/run/current-system/sw/bin
 EnvironmentFile=-/home/ben/src/bsima/slopbot/.env
+# The Nix wrapper resets PATH to the slopbot-agent-tools package before invoking Python.
+# Agent shell commands can then resolve only the tools packaged for slopbot, e.g. slop-search/slop-fetch.
 ExecStart=/run/current-system/sw/bin/nix --extra-experimental-features 'nix-command flakes' run /home/ben/src/bsima/slopbot#slopbot
 Restart=on-failure
 RestartSec=5