Batch web_reader tool, much faster

Added retry with backoff, parallel proccessing, editing pages down to main
content, summarization with haiku. It's so much faster and more reliable
now. Plus improved the logging system and distangled the status UI bar from the
logging module.

1 files changed, 11 insertions, 11 deletions

diff --git a/Omni/Agent/Worker.hs b/Omni/Agent/Worker.hs
index 3b0c563..d6afb73 100644
--- a/Omni/Agent/Worker.hs
+++ b/Omni/Agent/Worker.hs
@@ -20,8 +20,8 @@ import qualified Data.Text.Encoding as TE
 import qualified Data.Time
 import qualified Omni.Agent.Core as Core
 import qualified Omni.Agent.Engine as Engine
-import qualified Omni.Agent.Log as AgentLog
 import qualified Omni.Agent.Provider as Provider
+import qualified Omni.Agent.Status as AgentStatus
 import qualified Omni.Agent.Tools as Tools
 import qualified Omni.Fact as Fact
 import qualified Omni.Task.Core as TaskCore
@@ -36,8 +36,8 @@ start worker maybeTaskId = do
   if Core.workerQuiet worker
     then putText ("[worker] Starting for " <> Core.workerName worker)
     else do
-      AgentLog.init (Core.workerName worker)
-      AgentLog.log ("[worker] Starting for " <> Core.workerName worker)
+      AgentStatus.init (Core.workerName worker)
+      AgentStatus.log ("[worker] Starting for " <> Core.workerName worker)
   case maybeTaskId of
     Just tid -> logMsg worker ("[worker] Target task: " <> tid)
     Nothing -> logMsg worker "[worker] No specific task, will pick from ready queue"
@@ -48,7 +48,7 @@ logMsg :: Core.Worker -> Text -> IO ()
 logMsg worker msg =
   if Core.workerQuiet worker
     then putText msg
-    else AgentLog.log msg
+    else AgentStatus.log msg
 
 -- | Convert key-value pairs to JSON metadata string
 toMetadata :: [(Text, Text)] -> Text
@@ -86,10 +86,10 @@ runOnce worker maybeTaskId = do
     Nothing -> do
       case maybeTaskId of
         Just tid -> do
-          unless (Core.workerQuiet worker) <| AgentLog.updateActivity ("Task " <> tid <> " not found.")
+          unless (Core.workerQuiet worker) <| AgentStatus.updateActivity ("Task " <> tid <> " not found.")
           logMsg worker ("[worker] Task " <> tid <> " not found.")
         Nothing -> do
-          unless (Core.workerQuiet worker) <| AgentLog.updateActivity "No work found."
+          unless (Core.workerQuiet worker) <| AgentStatus.updateActivity "No work found."
           logMsg worker "[worker] No ready tasks found."
     Just task -> do
       processTask worker task
@@ -101,7 +101,7 @@ processTask worker task = do
   let quiet = Core.workerQuiet worker
   let say = logMsg worker
 
-  unless quiet <| AgentLog.update (\s -> s {AgentLog.statusTask = Just tid})
+  unless quiet <| AgentStatus.update (\s -> s {AgentStatus.statusTask = Just tid})
   say ("[worker] Claiming task " <> tid)
 
   -- Claim task
@@ -174,13 +174,13 @@ processTask worker task = do
           TaskCore.logActivity tid TaskCore.Completed (Just (toMetadata [("result", "no_changes")]))
           TaskCore.updateTaskStatusWithActor tid TaskCore.Done [] TaskCore.Junior
           say ("[worker] ✓ Task " <> tid <> " -> Done (no changes)")
-          unless quiet <| AgentLog.update (\s -> s {AgentLog.statusTask = Nothing})
+          unless quiet <| AgentStatus.update (\s -> s {AgentStatus.statusTask = Nothing})
         CommitSuccess -> do
           -- Commit succeeded, set to Review
           TaskCore.logActivity tid TaskCore.Completed (Just (toMetadata [("result", "committed")]))
           TaskCore.updateTaskStatusWithActor tid TaskCore.Review [] TaskCore.Junior
           say ("[worker] ✓ Task " <> tid <> " -> Review")
-          unless quiet <| AgentLog.update (\s -> s {AgentLog.statusTask = Nothing})
+          unless quiet <| AgentStatus.update (\s -> s {AgentStatus.statusTask = Nothing})
     EngineGuardrailViolation errMsg _ -> do
       say ("[worker] Guardrail violation: " <> errMsg)
       TaskCore.logActivity tid TaskCore.Failed (Just (toMetadata [("reason", "guardrail_violation")]))
@@ -189,7 +189,7 @@ processTask worker task = do
       -- Set to NeedsHelp so human can review
       TaskCore.updateTaskStatusWithActor tid TaskCore.NeedsHelp [] TaskCore.Junior
       say ("[worker] Task " <> tid <> " -> NeedsHelp (guardrail violation)")
-      unless quiet <| AgentLog.update (\s -> s {AgentLog.statusTask = Nothing})
+      unless quiet <| AgentStatus.update (\s -> s {AgentStatus.statusTask = Nothing})
     EngineError errMsg _ -> do
       say ("[worker] Engine error: " <> errMsg)
       TaskCore.logActivity tid TaskCore.Failed (Just (toMetadata [("reason", "engine_error")]))
@@ -303,7 +303,7 @@ runWithEngine worker repo task = do
       -- Build Engine config with callbacks
       totalCostRef <- newIORef (0 :: Double)
       let quiet = Core.workerQuiet worker
-          sayLog msg = if quiet then putText msg else AgentLog.log msg
+          sayLog msg = if quiet then putText msg else AgentStatus.log msg
           engineCfg =
             Engine.EngineConfig
               { Engine.engineLLM =