feat(server): io/agent worker groups, invoke limits for external IO capabilities, loop lag probe

- WORKER_CONCURRENCY / WORKER_AGENT_CONCURRENCY; --role worker (io), worker:agent, all (both)
- HTTP invoke of release.search/inspect, identity.resolve, catalog.lookup(remote):
  15s timeout + per-upstream semaphore (site 2, TMDB 4)
- worker event-loop lag probe: warn + count when drift > 5s

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/src/worker/index.ts

@@ -5,6 +5,18 @@ import { collectOutboxRoutes, tasksLogSink } from "../context.ts";
 import type { EventBusPort } from "../http/sse.ts";
 import type { ServerContext, ServerModule } from "../modules/types.ts";
 
+/** agent 组任务:整段会话占一槽数分钟,与文件复制等 io 任务分池 */
+export const AGENT_TASKS = ["runAgent", "resolveDecision"] as const;
+
+/**
+ * worker 分组:
+ *   all   — 单个引擎 worker 执行全部任务(测试 / 省略分组时的兼容行为)
+ *   io    — 除 AGENT_TASKS 外的全部任务 + OutboxRelay + 模块后台循环
+ *   agent — 只执行 AGENT_TASKS;不跑 relay 与模块循环
+ *   split — 同一进程里 io + agent 两个引擎 worker(--role all)
+ */
+export type WorkerGroup = "all" | "io" | "agent" | "split";
+
 export interface WorkerDeps {
   logger: Logger;
   heartbeatMs: number;
@@ -17,32 +29,117 @@ export interface WorkerDeps {
     /** 设置 outbox 中继实例到 ctx.outbox.relay */
     setRelay?: (relay: OutboxRelay) => void;
     outboxIntervalMs?: number;
+    /** io 组(或 all 组)并发 */
     concurrency?: number;
+    /** agent 组并发 */
+    agentConcurrency?: number;
+    /** 缺省 all */
+    group?: WorkerGroup;
+    /** 注册表里的全部任务名(io 组 = 全部 − AGENT_TASKS);group 为 io/split 时必填 */
+    taskNames?: readonly string[];
   };
+  /** 事件循环延迟探针:采样间隔与告警阈值(测试可调小) */
+  lagProbe?: { intervalMs?: number; warnMs?: number };
+}
+
+export interface WorkerStats {
+  /** 事件循环偏差超过阈值的次数 */
+  loopLagWarnings: number;
+  /** 观测到的最大偏差(ms) */
+  maxLoopLagMs: number;
 }
 
 export interface RunningWorker {
   stop(): Promise<void>;
+  stats(): WorkerStats;
+}
+
+/** 按分组拆出引擎 worker 的任务选择(versions 为任务名,匹配该任务全部版本) */
+export function workerPlan(
+  group: WorkerGroup,
+  taskNames: readonly string[],
+  concurrency: { io?: number; agent?: number },
+): { label: "all" | "io" | "agent"; versions?: string[]; concurrency?: number }[] {
+  const agentNames = taskNames.filter((n) => (AGENT_TASKS as readonly string[]).includes(n));
+  const ioNames = taskNames.filter((n) => !(AGENT_TASKS as readonly string[]).includes(n));
+  const io = { label: "io" as const, versions: ioNames, ...(concurrency.io ? { concurrency: concurrency.io } : {}) };
+  const agent = {
+    label: "agent" as const,
+    versions: agentNames,
+    ...(concurrency.agent ? { concurrency: concurrency.agent } : {}),
+  };
+  switch (group) {
+    case "all":
+      return [{ label: "all", ...(concurrency.io ? { concurrency: concurrency.io } : {}) }];
+    case "io":
+      return ioNames.length ? [io] : [];
+    case "agent":
+      return agentNames.length ? [agent] : [];
+    case "split":
+      return [...(ioNames.length ? [io] : []), ...(agentNames.length ? [agent] : [])];
+  }
+}
+
+/**
+ * 事件循环延迟探针:setInterval 实际触发时刻与预期的偏差超过 warnMs 记一次 warn + 计数。
+ * 偏差大说明有同步重活占住主线程(租约续期 15s、租约 30s,偏差逼近这个量级就有丢租约风险)。
+ */
+export function startLoopLagProbe(log: Logger, opts: { intervalMs?: number; warnMs?: number } = {}) {
+  const intervalMs = opts.intervalMs ?? 1000;
+  const warnMs = opts.warnMs ?? 5000;
+  const stats: WorkerStats = { loopLagWarnings: 0, maxLoopLagMs: 0 };
+  let expected = performance.now() + intervalMs;
+  const timer = setInterval(() => {
+    const now = performance.now();
+    const lag = Math.max(0, now - expected);
+    expected = now + intervalMs;
+    if (lag > stats.maxLoopLagMs) stats.maxLoopLagMs = Math.round(lag);
+    if (lag > warnMs) {
+      stats.loopLagWarnings += 1;
+      log.warn({ lagMs: Math.round(lag), warnings: stats.loopLagWarnings }, "worker 事件循环延迟超过阈值");
+    }
+  }, intervalMs);
+  return {
+    stats: () => ({ ...stats }),
+    stop: () => clearInterval(timer),
+  };
 }
 
 /**
  * worker 角色:心跳 + 任务引擎 worker(执行全部已注册任务)+ OutboxRelay(按模块的 outboxRoutes 路由)
  * + 各模块的后台循环(start)。stop 逆序停止;引擎连接本身由进程退出时关闭。
  */
-export async function startWorker({ logger, heartbeatMs, bus, runtime }: WorkerDeps): Promise<RunningWorker> {
-  const log = logger.child({ component: "worker" });
+export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe }: WorkerDeps): Promise<RunningWorker> {
+  const group = runtime?.group ?? "all";
+  const log = logger.child({ component: "worker", group });
+  const probe = startLoopLagProbe(log, lagProbe);
   let n = 0;
   const beat = () => {
     n += 1;
-    log.debug({ n }, "worker heartbeat");
-    bus?.publish(EventType.heartbeat, { role: "worker", n });
+    const s = probe.stats();
+    log.debug({ n, ...s }, "worker heartbeat");
+    bus?.publish(EventType.heartbeat, { role: group === "agent" ? "worker:agent" : "worker", n, ...s });
   };
 
   const started: ServerModule[] = [];
   let relay: OutboxRelay | null = null;
+  const plan: ReturnType<typeof workerPlan> = [];
   if (runtime) {
+    const { ctx } = runtime;
+    if (group !== "all" && !runtime.taskNames) throw new Error(`worker 分组 ${group} 需要 taskNames`);
+    plan.push(
+      ...workerPlan(group, runtime.taskNames ?? [], { io: runtime.concurrency, agent: runtime.agentConcurrency }),
+    );
+    for (const p of plan) {
+      await ctx.engine.startWorker({
+        ...(p.versions ? { versions: p.versions } : {}),
+        ...(p.concurrency ? { concurrency: p.concurrency } : {}),
+      });
+    }
+  }
+  // outbox 中继与模块后台循环只在 io 组(agent 组进程只执行会话任务)
+  if (runtime && group !== "agent") {
     const { ctx, modules } = runtime;
-    await ctx.engine.startWorker({ ...(runtime.concurrency ? { concurrency: runtime.concurrency } : {}) });
     relay = new OutboxRelay({
       db: ctx.db,
       engine: ctx.engine,
@@ -58,12 +155,21 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime }: WorkerD
     }
   }
 
-  log.info({ heartbeatMs, modules: started.map((m) => m.name) }, "worker started");
+  log.info(
+    {
+      heartbeatMs,
+      modules: started.map((m) => m.name),
+      pools: plan.map((p) => ({ group: p.label, concurrency: p.concurrency ?? 4, tasks: p.versions?.length ?? "all" })),
+    },
+    "worker started",
+  );
   beat();
   const timer = setInterval(beat, heartbeatMs);
   return {
+    stats: () => probe.stats(),
     async stop() {
       clearInterval(timer);
+      probe.stop();
       for (const m of started.reverse()) {
         try {
           await m.stop?.();