1 files changed, 72 insertions, 3 deletions
diff --git a/flake.nix b/flake.nix
index 843f9c5..a70e64e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -29,10 +29,78 @@
         cargoBuildFlags = [ "-p" "agent" ];
         doCheck = false;
       };
+
+      researchPython = pkgs.python3.withPackages (ps: [
+        ps.trafilatura
+      ]);
+
+      slopSearch = pkgs.writeShellApplication {
+        name = "slop-search";
+        runtimeInputs = [ pkgs.curl pkgs.jq pkgs.python3 ];
+        text = ''
+          if [ "$#" -eq 0 ]; then
+            echo "usage: slop-search QUERY" >&2
+            exit 2
+          fi
+          if [ -z "''${KAGI_API_KEY:-}" ]; then
+            echo "error: KAGI_API_KEY is not set" >&2
+            exit 1
+          fi
+
+          query="$*"
+          encoded="$(${pkgs.python3}/bin/python -c 'import sys, urllib.parse; print(urllib.parse.urlencode({"q": " ".join(sys.argv[1:])}))' "$@")"
+
+          curl -fsS \
+            -H "Authorization: Bot ''${KAGI_API_KEY}" \
+            -H "Accept: application/json" \
+            "https://kagi.com/api/v0/search?''${encoded}" \
+          | jq -r --arg query "$query" '
+              "QUERY: \($query)",
+              "",
+              (.data // [])
+              | map(select((.t // .type // "") == "search_result" or has("url")))
+              | .[:8]
+              | to_entries[]
+              | "[\(.key + 1)] \(.value.title // "untitled")\nURL: \(.value.url)\nSNIPPET: \(.value.snippet // .value.description // "")\n"
+            '
+        '';
+      };
+
+      slopFetch = pkgs.writeShellApplication {
+        name = "slop-fetch";
+        runtimeInputs = [ pkgs.curl researchPython ];
+        text = ''
+          if [ "$#" -ne 1 ]; then
+            echo "usage: slop-fetch URL" >&2
+            exit 2
+          fi
+          url="$1"
+          tmp="$(mktemp)"
+          trap 'rm -f "$tmp"' EXIT
+          curl -fsSL --max-time 20 -A 'slopbot/1.0' "$url" -o "$tmp"
+          ${researchPython}/bin/python - "$url" "$tmp" <<'PY'
+import sys
+import trafilatura
+
+url, path = sys.argv[1], sys.argv[2]
+html = open(path, "rb").read()
+text = trafilatura.extract(html, url=url, include_comments=False, include_tables=False)
+if not text:
+    print(f"error: could not extract readable text from {url}", file=sys.stderr)
+    sys.exit(1)
+print(text[:12000])
+PY
+        '';
+      };
+
+      slopTools = pkgs.symlinkJoin {
+        name = "slopbot-agent-tools";
+        paths = [ slopSearch slopFetch ];
+      };
     in
     {
       devShells.${system}.default = pkgs.mkShell {
-        buildInputs = [ pythonEnv agent ];
+        buildInputs = [ pythonEnv agent slopTools ];
         shellHook = ''
           echo "slopbot dev shell ready"
           echo "Run: python main.py"
@@ -43,11 +111,12 @@
         default = slopbot;
         slopbot = pkgs.writeShellScriptBin "slopbot" ''
           cd /home/ben/src/bsima/slopbot
-          export PATH=${agent}/bin:$PATH
+          export PATH=${slopTools}/bin
           export SLOPBOT_AGENT=${agent}/bin/agent
+          export SLOPBOT_AGENT_PATH=${slopTools}/bin
           exec ${pythonEnv}/bin/python ${./main.py}
         '';
-        inherit agent;
+        inherit agent slopTools slopSearch slopFetch;
       };
     };
 }