ava: add Python/CLI tools and local whisper fallback

- Wrap ava binary with Python (requests, httpx, pandas, etc.)
- Add CLI tools: curl, pandoc, ffmpeg, imagemagick, csvkit
- Add local whisper-cli fallback when OPENAI_API_KEY unavailable

Amp-Thread-ID: https://ampcode.com/threads/T-019b2dc2-36e0-75e1-90c1-622901fc9847
Co-authored-by: Amp <amp@ampcode.com>

2 files changed, 44 insertions, 2 deletions

diff --git a/Omni/Agent/Telegram/Media.hs b/Omni/Agent/Telegram/Media.hs
index 47fbf91..0d62edd 100644
--- a/Omni/Agent/Telegram/Media.hs
+++ b/Omni/Agent/Telegram/Media.hs
@@ -54,9 +54,12 @@ import qualified Network.HTTP.Simple as HTTP
 import qualified Omni.Agent.Telegram.Types as Types
 import qualified Omni.Agent.Tools.Pdf as Pdf
 import qualified Omni.Test as Test
+import qualified System.Directory as Directory
 import System.Environment (lookupEnv)
+import qualified System.Exit as Exit
 import System.IO (hClose)
 import System.IO.Temp (withSystemTempFile)
+import qualified System.Process as Process
 
 main :: IO ()
 main = Test.run test
@@ -274,8 +277,34 @@ transcribeVoice :: Text -> BL.ByteString -> IO (Either Text Text)
 transcribeVoice _unusedApiKey audioBytes = do
   maybeKey <- lookupEnv "OPENAI_API_KEY"
   case maybeKey of
-    Nothing -> pure (Left "OPENAI_API_KEY not set - required for voice transcription")
-    Just key -> transcribeWithWhisper (Text.pack key) audioBytes
+    Nothing -> transcribeWithWhisperLocal audioBytes
+    Just key -> do
+      result <- transcribeWithWhisper (Text.pack key) audioBytes
+      case result of
+        Left _ -> transcribeWithWhisperLocal audioBytes
+        Right text -> pure (Right text)
+
+transcribeWithWhisperLocal :: BL.ByteString -> IO (Either Text Text)
+transcribeWithWhisperLocal audioBytes = do
+  tmpDir <- Directory.getTemporaryDirectory
+  let audioFile = tmpDir <> "/ava_voice_" <> show (BL.length audioBytes) <> ".ogg"
+  result <-
+    try <| do
+      BL.writeFile audioFile audioBytes
+      (exitCode, stdoutStr, stderrStr) <-
+        Process.readProcessWithExitCode
+          "whisper-cli"
+          ["--model", "/home/ava/models/ggml-base.en.bin", "--file", audioFile, "--no-timestamps"]
+          ""
+      Directory.removeFile audioFile
+      case exitCode of
+        Exit.ExitSuccess -> pure (Right (Text.strip (Text.pack stdoutStr)))
+        Exit.ExitFailure _ -> pure (Left (Text.pack stderrStr))
+  case result of
+    Left (e :: SomeException) -> do
+      _ <- try @SomeException (Directory.removeFile audioFile)
+      pure (Left ("Local whisper failed: " <> tshow e))
+    Right r -> pure r
 
 transcribeWithWhisper :: Text -> BL.ByteString -> IO (Either Text Text)
 transcribeWithWhisper apiKey audioBytes = do
diff --git a/Omni/Dev/Beryllium/Ava.nix b/Omni/Dev/Beryllium/Ava.nix
index f0765cd..2dc5e54 100644
--- a/Omni/Dev/Beryllium/Ava.nix
+++ b/Omni/Dev/Beryllium/Ava.nix
@@ -5,14 +5,20 @@
   # Python environment for Ava's python_exec tool
   avaPython = bild.python.pythonWith (p: [
     p.requests # HTTP requests
+    p.httpx # Async HTTP
     p.beautifulsoup4 # HTML/XML parsing
     p.lxml # Fast XML/HTML parser
     p.pandas # Data analysis
     p.numpy # Numerical computing
+    p.pillow # Image processing
     p.pyyaml # YAML parsing
     p.python-dateutil # Date utilities
   ]);
 
+  # Nixpkgs for CLI tools not in bild.pkgs
+  nixpkgs = import ../../Bild/Nixpkgs.nix;
+  pkgs = nixpkgs.nixos-24_11;
+
   # Wrap ava binary with tools in PATH
   avaWithTools = bild.stdenv.mkDerivation {
     name = "ava-wrapped";
@@ -28,6 +34,12 @@
         bild.pkgs.coreutils
         bild.pkgs.git
         bild.pkgs.sqlite
+        bild.pkgs.ffmpeg
+        pkgs.curl
+        pkgs.pandoc
+        pkgs.imagemagick
+        pkgs.csvkit
+        pkgs.openai-whisper-cpp
       ]}
     '';
   };
@@ -77,5 +89,6 @@ in {
     "d /home/ava/outreach/sent 0755 ava users -"
     "d /home/ava/users 0755 ava users -"
     "d /home/ava/.local/share/omni 0755 ava users -"
+    "d /home/ava/models 0755 ava users -"
   ];
 }