feat(worker): IO Worker 分道(WK-1)

io worker 进程内按任务声明的 lane 分文件 / 网络 / 维护三条道,各自并发(缺省 2/4/1)、各自排队;维护道在文件道或网络道有排队/运行时让路(连续 10 分钟放行一个),调度器内联的清理/回填/保留期裁剪也走维护道。并发在设置 → 自动化 → 高级热生效,env 覆盖;旧 WORKER_CONCURRENCY 映射到网络道。运行状态页按道显示运行中/并发/排队。归属表写进 docs/runbook/dev.md,ci 与注册表逐项核对。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-09-30 00:24:21 +08:00
1 parent 68f53390f6
commit da33cad81d
@@ -1,10 +1,19 @@
 import { EventType } from "@umt/contracts";
-import { OutboxRelay } from "@umt/tasks";
+import { OutboxRelay, type WorkerHandle } from "@umt/tasks";
 import type { Logger } from "pino";
 import { collectOutboxRoutes, tasksLogSink } from "../context.ts";
 import { type EventBusPort, sseConnectionCount } from "../http/sse.ts";
 import { type HeartbeatWriter, startHeartbeatWriter } from "../modules/system/heartbeat.ts";
 import type { ServerContext, ServerModule } from "../modules/types.ts";
+import {
+  groupTasksByLane,
+  IO_LANES,
+  type IoLane,
+  LaneController,
+  type LaneOverrides,
+  type LaneTask,
+  setLaneController,
+} from "./lanes.ts";

 /**
 * outbox → SSE 桥:这些前缀的 outbox 事件在中继投递成功后镜像到实时总线(payload 原样,带业务 id),
@@ -16,18 +25,20 @@ export const SSE_MIRRORED_OUTBOX_TYPES: readonly string[] = Object.values(EventT
  (t) => SSE_MIRROR_PREFIXES.some((p) => t.startsWith(p)) && !SSE_MIRROR_EXCLUDED.has(t),
 );

-/** agent 组任务:整段会话占一槽数分钟,与文件复制等 io 任务分池 */
-export const AGENT_TASKS = ["runAgent", "resolveDecision"] as const;
+/** agent 组任务:整段会话占一槽数分钟,与文件复制等 io 任务分池(以任务声明的 lane = "agent" 为准,这是兜底名单) */
+export { AGENT_TASKS } from "./lanes.ts";

 /**
 * worker 分组:
 *   all   — 单个引擎 worker 执行全部任务(测试 / 省略分组时的兼容行为)
- *   io    — 除 AGENT_TASKS 外的全部任务 + OutboxRelay + 模块后台循环
- *   agent — 只执行 AGENT_TASKS;不跑 relay 与模块循环
- *   split — 同一进程里 io + agent 两个引擎 worker(--role all)
+ *   io    — 除 agent 道外的全部任务,按 lane 分文件 / 网络 / 维护三个引擎 worker + OutboxRelay + 模块后台循环
+ *   agent — 只执行 agent 道任务;不跑 relay 与模块循环
+ *   split — 同一进程里 io 三条道 + agent 共四个引擎 worker(--role all)
 */
 export type WorkerGroup = "all" | "io" | "agent" | "split";

+export type PoolLabel = "all" | IoLane | "agent";
+
 export interface WorkerDeps {
  logger: Logger;
  heartbeatMs: number;
@@ -40,14 +51,21 @@ export interface WorkerDeps {
    /** 设置 outbox 中继实例到 ctx.outbox.relay */
    setRelay?: (relay: OutboxRelay) => void;
    outboxIntervalMs?: number;
-    /** io 组(或 all 组)并发 */
+    /** all 组(单池)并发 */
    concurrency?: number;
    /** agent 组并发 */
    agentConcurrency?: number;
    /** 缺省 all */
    group?: WorkerGroup;
-    /** 注册表里的全部任务名(io 组 = 全部 − AGENT_TASKS);group 为 io/split 时必填 */
-    taskNames?: readonly string[];
+    /** 注册表里的全部任务(名称 + lane);group 为 io/agent/split 时必填 */
+    tasks?: readonly LaneTask[];
+    /** 分道:环境变量覆盖与测试用的节奏参数 */
+    lanes?: {
+      overrides?: LaneOverrides;
+      refreshMs?: number;
+      busyCacheMs?: number;
+      maxYieldMs?: number;
+    };
  };
  /** 事件循环延迟探针:采样间隔与告警阈值(测试可调小) */
  lagProbe?: { intervalMs?: number; warnMs?: number };
@@ -63,31 +81,38 @@ export interface WorkerStats {
 export interface RunningWorker {
  stop(): Promise<void>;
  stats(): WorkerStats;
+  /** io / split 组的分道控制器 */
+  lanes: LaneController | null;
 }

-/** 按分组拆出引擎 worker 的任务选择(versions 为任务名,匹配该任务全部版本) */
+export interface PoolPlan {
+  label: PoolLabel;
+  /** 任务名(匹配该任务全部版本);all 池省略 */
+  versions?: string[];
+  concurrency?: number;
+}
+
+/** 按分组拆出引擎 worker 的任务选择;io 组按 lane 拆三条道(空道不起) */
 export function workerPlan(
  group: WorkerGroup,
-  taskNames: readonly string[],
-  concurrency: { io?: number; agent?: number },
-): { label: "all" | "io" | "agent"; versions?: string[]; concurrency?: number }[] {
-  const agentNames = taskNames.filter((n) => (AGENT_TASKS as readonly string[]).includes(n));
-  const ioNames = taskNames.filter((n) => !(AGENT_TASKS as readonly string[]).includes(n));
-  const io = { label: "io" as const, versions: ioNames, ...(concurrency.io ? { concurrency: concurrency.io } : {}) };
-  const agent = {
-    label: "agent" as const,
-    versions: agentNames,
-    ...(concurrency.agent ? { concurrency: concurrency.agent } : {}),
-  };
+  tasks: readonly LaneTask[],
+  concurrency: { all?: number; agent?: number } & LaneOverrides = {},
+): PoolPlan[] {
+  const { byLane } = groupTasksByLane(tasks);
+  const pool = (label: IoLane | "agent"): PoolPlan[] =>
+    byLane[label].length
+      ? [{ label, versions: byLane[label], ...(concurrency[label] ? { concurrency: concurrency[label] } : {}) }]
+      : [];
+  const io = IO_LANES.flatMap((l) => pool(l));
  switch (group) {
    case "all":
-      return [{ label: "all", ...(concurrency.io ? { concurrency: concurrency.io } : {}) }];
+      return [{ label: "all", ...(concurrency.all ? { concurrency: concurrency.all } : {}) }];
    case "io":
-      return ioNames.length ? [io] : [];
+      return io;
    case "agent":
-      return agentNames.length ? [agent] : [];
+      return pool("agent");
    case "split":
-      return [...(ioNames.length ? [io] : []), ...(agentNames.length ? [agent] : [])];
+      return [...io, ...pool("agent")];
  }
 }

@@ -134,18 +159,50 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe

  const started: ServerModule[] = [];
  let relay: OutboxRelay | null = null;
-  const plan: ReturnType<typeof workerPlan> = [];
+  let lanes: LaneController | null = null;
+  const plan: PoolPlan[] = [];
+  const handles = new Map<PoolLabel, WorkerHandle>();
  if (runtime) {
    const { ctx } = runtime;
-    if (group !== "all" && !runtime.taskNames) throw new Error(`worker 分组 ${group} 需要 taskNames`);
-    plan.push(
-      ...workerPlan(group, runtime.taskNames ?? [], { io: runtime.concurrency, agent: runtime.agentConcurrency }),
+    if (group !== "all" && !runtime.tasks) throw new Error(`worker 分组 ${group} 需要 tasks`);
+    const tasks = runtime.tasks ?? [];
+    plan.push(...workerPlan(group, tasks, { all: runtime.concurrency, agent: runtime.agentConcurrency }));
+    const laned = plan.filter((p): p is PoolPlan & { label: IoLane } =>
+      (IO_LANES as readonly string[]).includes(p.label),
    );
-    for (const p of plan) {
-      await ctx.engine.startWorker({
-        ...(p.versions ? { versions: p.versions } : {}),
-        ...(p.concurrency ? { concurrency: p.concurrency } : {}),
+    if (laned.length) {
+      const { byLane, missing } = groupTasksByLane(tasks);
+      if (missing.length)
+        log.warn({ tasks: missing }, "这些任务没有声明 lane，按缺省归入网络道（请在 defineTask 里补 lane）");
+      lanes = new LaneController({
+        engine: ctx.engine,
+        logger,
+        lanes: { file: byLane.file, network: byLane.network, maintenance: byLane.maintenance },
+        readSettings: async () => (await ctx.settings.get("automation")).workerLanes,
+        ...(runtime.lanes?.overrides ? { overrides: runtime.lanes.overrides } : {}),
+        ...(runtime.lanes?.refreshMs ? { refreshMs: runtime.lanes.refreshMs } : {}),
+        ...(runtime.lanes?.busyCacheMs !== undefined ? { busyCacheMs: runtime.lanes.busyCacheMs } : {}),
+        ...(runtime.lanes?.maxYieldMs ? { maxYieldMs: runtime.lanes.maxYieldMs } : {}),
      });
+      await lanes.start();
+      setLaneController(ctx, lanes);
+    }
+    for (const p of plan) {
+      const lane = (IO_LANES as readonly string[]).includes(p.label) ? (p.label as IoLane) : null;
+      const handle =
+        lane && lanes
+          ? await ctx.engine.startWorker({
+              versions: p.versions ?? [],
+              concurrency: lanes.slots(lane),
+              maxActive: () => lanes?.maxActive(lane) ?? 0,
+              ...(lane === "maintenance" ? { claimGate: () => lanes?.maintenanceMayStart() ?? true } : {}),
+            })
+          : await ctx.engine.startWorker({
+              ...(p.versions ? { versions: p.versions } : {}),
+              ...(p.concurrency ? { concurrency: p.concurrency } : {}),
+            });
+      if (handle) handles.set(p.label, handle);
+      if (lane) lanes?.attach(lane, handle || undefined);
    }
  }
  // outbox 中继与模块后台循环只在 io 组(agent 组进程只执行会话任务)
@@ -170,12 +227,14 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
    }
  }

-  // 进程与健康页:每 15s 把心跳/延迟/并发/运行中 run 数写进 process_heartbeat(写失败不影响 worker)
+  // 进程与健康页:每 15s 把心跳/延迟/并发/运行中 run 数写进 process_heartbeat(写失败不影响 worker);
+  // io 组另写各道的并发、本进程运行中、全库排队(extra.lanes),事件循环延迟照常记录(将来拆进程的判据)
  let hbWriter: HeartbeatWriter | null = null;
  if (runtime) {
    const { ctx } = runtime;
    const groupTasks = new Set(plan.flatMap((p) => p.versions ?? []));
    const countAll = plan.some((p) => !p.versions);
+    const agentPool = plan.find((p) => p.label === "agent");
    hbWriter = startHeartbeatWriter({
      db: ctx.db,
      logger,
@@ -183,16 +242,27 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
      sample: async () => {
        const running = await ctx.engine.list({ status: "running", limit: 500 }).catch(() => null);
        const s = probe.stats();
+        const laneStats = lanes ? await lanes.snapshot() : null;
+        const concurrency: Record<string, number> = {};
+        if (laneStats) {
+          for (const l of laneStats) concurrency[l.lane] = l.concurrency;
+          concurrency.io = laneStats.reduce((a, l) => a + l.concurrency, 0);
+        } else if (plan.some((p) => p.label === "all")) {
+          concurrency.io = runtime.concurrency ?? 4;
+        }
+        if (agentPool) concurrency.agent = handles.get("agent")?.concurrency ?? runtime.agentConcurrency ?? 4;
        return {
-          concurrency: {
-            ...(plan.some((p) => p.label !== "agent") ? { io: runtime.concurrency ?? 4 } : {}),
-            ...(plan.some((p) => p.label === "agent") ? { agent: runtime.agentConcurrency ?? 4 } : {}),
-          },
+          concurrency,
          runningRuns: running
            ? running.filter((r) => r.status === "running" && (countAll || groupTasks.has(r.taskName))).length
            : null,
          sseConnections: group === "split" ? sseConnectionCount() : null,
-          extra: { group, lagWarnings: s.loopLagWarnings, probeMaxLagMs: s.maxLoopLagMs },
+          extra: {
+            group,
+            lagWarnings: s.loopLagWarnings,
+            probeMaxLagMs: s.maxLoopLagMs,
+            ...(laneStats ? { lanes: laneStats } : {}),
+          },
        };
      },
    });
@@ -202,7 +272,14 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
    {
      heartbeatMs,
      modules: started.map((m) => m.name),
-      pools: plan.map((p) => ({ group: p.label, concurrency: p.concurrency ?? 4, tasks: p.versions?.length ?? "all" })),
+      pools: plan.map((p) => ({
+        group: p.label,
+        concurrency:
+          lanes && (IO_LANES as readonly string[]).includes(p.label)
+            ? lanes.limit(p.label as IoLane)
+            : (p.concurrency ?? 4),
+        tasks: p.versions?.length ?? "all",
+      })),
    },
    "worker started",
  );
@@ -210,9 +287,12 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
  const timer = setInterval(beat, heartbeatMs);
  return {
    stats: () => probe.stats(),
+    lanes,
    async stop() {
      clearInterval(timer);
      probe.stop();
+      lanes?.stop();
+      if (runtime) setLaneController(runtime.ctx, null);
      await hbWriter?.stop();
      for (const m of started.reverse()) {
        try {