feat(worker): IO Worker 分道(WK-1)

io worker 进程内按任务声明的 lane 分文件 / 网络 / 维护三条道,各自并发(缺省 2/4/1)、
各自排队;维护道在文件道或网络道有排队/运行时让路(连续 10 分钟放行一个),调度器内联的
清理/回填/保留期裁剪也走维护道。并发在 设置 → 自动化 → 高级 热生效,env 覆盖;旧
WORKER_CONCURRENCY 映射到网络道。运行状态页按道显示运行中/并发/排队。归属表写进
docs/runbook/dev.md,ci 与注册表逐项核对。

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-30 00:24:21 +08:00
1 parent 68f53390f6
commit da33cad81d
38 files changed
+1240 -105

No files matched your search

+12 -2
View File
@@ -4,6 +4,9 @@ import { z } from "zod";
/** compose 里 `${X:-}` 展开成空串:按未设置处理 */
const emptyToUndef = (v: unknown) => (v === "" ? undefined : v);
/** 可选正整数环境变量(空串 = 未设置) */
const optionalInt = (max: number) => z.preprocess(emptyToUndef, z.coerce.number().int().min(1).max(max).optional());
const LogLevel = z.enum(["fatal", "error", "warn", "info", "debug", "trace", "silent"]);
export const ConfigSchema = z.object({
@@ -29,8 +32,15 @@ export const ConfigSchema = z.object({
DEV_ADMIN_KEY: z.string().min(8).optional(),
/** worker 心跳间隔 */
WORKER_HEARTBEAT_MS: z.coerce.number().int().min(1000).default(30_000),
/** io 组 worker(除 runAgent/resolveDecision 外的全部任务)并发槽数 */
WORKER_CONCURRENCY: z.coerce.number().int().min(1).max(64).default(4),
/**
* 旧变量(WK-1 前的 io 组总并发):现在映射到网络道(WORKER_NETWORK_CONCURRENCY 未设时);
* 不设 = 用设置页的值。空串视为未设置(compose 里 `${WORKER_CONCURRENCY:-}`)。
*/
WORKER_CONCURRENCY: optionalInt(64),
/** io worker 三条道的并发(WK-1);设置了就覆盖 设置 → 自动化 → 高级 里的值 */
WORKER_FILE_CONCURRENCY: optionalInt(64),
WORKER_NETWORK_CONCURRENCY: optionalInt(64),
WORKER_MAINTENANCE_CONCURRENCY: optionalInt(64),
/** agent 组 worker(runAgent/resolveDecision:整段会话占一槽数分钟)并发槽数 */
WORKER_AGENT_CONCURRENCY: z.coerce.number().int().min(1).max(16).default(2),
/** 手动整理 dryRun / 按路径整理时递归遍历的文件数上限(超出截断并标记 truncated) */
+4
View File
@@ -41,6 +41,10 @@ export class DeferredTaskEngine implements TaskEngine {
get: TaskEngine["get"] = (...a) => this.e.get(...a);
list: TaskEngine["list"] = (...a) => this.e.list(...a);
startWorker: TaskEngine["startWorker"] = (...a) => this.e.startWorker(...a);
queueCounts: NonNullable<TaskEngine["queueCounts"]> = async (...a) => {
const q = this.e.queueCounts;
return q ? q.apply(this.e, a) : a[0].map(() => ({ queued: 0, running: 0 }));
};
stop: TaskEngine["stop"] = async () => {
await this.inner?.stop();
};
+5 -3
View File
@@ -3,7 +3,7 @@ import denoJson from "../deno.json" with { type: "json" };
import type { Config } from "./config.ts";
import type { MappingExport } from "./ops.ts";
// api / worker / worker:agent / all 是常驻角色(worker = io 组:除 runAgent/resolveDecision 外的任务 + outbox 中继 +
// api / worker / worker:agent / all 是常驻角色(worker = io 组:文件 / 网络 / 维护三条道 + outbox 中继 +
// 模块后台循环;worker:agent = 只跑 Agent 会话任务;all = api + 两组 worker 同进程);其余是容器入口 / 备份恢复脚本用的一次性子命令:
// migrate | preflight | probe
// export-mappings --out <file.json> [--compare <基准清单.json>] 映射清单导出(可与基准比行数)
@@ -77,6 +77,7 @@ const { SERVER_MODULES } = await import("./modules/index.ts");
const { diffCounts, enableReadOnlyReconcile, exportMappings, writeMappingExport } = await import("./ops.ts");
const { auditStorageWarnings, checkStorage, loadMountedPaths, runPreflight } = await import("./preflight.ts");
const { startWorker } = await import("./worker/index.ts");
const { laneOverridesFromEnv } = await import("./worker/lanes.ts");
let config: Config;
try {
@@ -253,9 +254,10 @@ if (role === "worker" || role === "worker:agent" || role === "all") {
modules: SERVER_MODULES,
setRelay: outbox.setRelay,
group: role === "all" ? "split" : role === "worker" ? "io" : "agent",
taskNames: booted.tasks.names(),
concurrency: config.WORKER_CONCURRENCY,
tasks: booted.tasks.all(),
agentConcurrency: config.WORKER_AGENT_CONCURRENCY,
// 三条道的并发读 设置 → 自动化 → 高级(热生效);环境变量设了就覆盖(旧 WORKER_CONCURRENCY → 网络道)
lanes: { overrides: laneOverridesFromEnv(config) },
},
});
stops.push(worker.stop);
@@ -76,6 +76,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
defineTask({
name: "evaluateAcquisition",
version: 1,
lane: "network",
inputSchema: z.object({
subscriptionId: id.optional(),
acquisitionSlotId: id.optional(),
@@ -91,6 +92,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
defineTask({
name: "submitDownload",
version: 1,
lane: "network",
inputSchema: z.object({ acquisitionPlanId: id, taskId: id.optional() }),
retryPolicy: NETWORK_RETRY,
handler: async (t) => {
@@ -117,6 +119,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
defineTask({
name: "verifyAndImport",
version: 1,
lane: "file",
inputSchema: z.object({ downloadTaskId: id.optional(), recordId: id.optional(), taskId: id.optional() }),
retryPolicy: FILE_RETRY,
handler: async (t) => {
@@ -174,6 +177,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
defineTask({
name: "cleanupDownloadCache",
version: 1,
lane: "file",
inputSchema: z.object({ downloaderInstanceId: id.optional(), taskId: id.optional() }),
retryPolicy: FILE_RETRY,
handler: async (t) => {
@@ -186,6 +190,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
defineTask({
name: "reconcileLibrary",
version: 1,
lane: "maintenance",
inputSchema: z.object({ libraryRootId: id.optional(), reconcileRunId: id.optional(), taskId: id.optional() }),
retryPolicy: FILE_RETRY,
handler: async (t) => {
+2
View File
@@ -37,6 +37,7 @@ export function registerAgentTasks(registry: TaskRegistry, env: () => AgentEnv)
defineTask({
name: "runAgent",
version: 1,
lane: "agent",
/** upstreamId:覆盖默认上游(W7-A);缺省用 agent_run.upstream_id,再缺省 settings.pi.defaultUpstreamId */
inputSchema: z.object({ agentRunId: id, upstreamId: id.optional() }),
retryPolicy: DB_RETRY,
@@ -175,6 +176,7 @@ export function registerAgentTasks(registry: TaskRegistry, env: () => AgentEnv)
defineTask({
name: "resolveDecision",
version: 1,
lane: "agent",
inputSchema: z.object({ decisionId: id, trigger: z.enum(["created", "updated"]).default("updated") }),
retryPolicy: DB_RETRY,
handler: async (t) => {
@@ -50,6 +50,7 @@ export function createAuthSettingsModule(opts: AuthSettingsModuleOptions = {}):
if (everyMs <= 0) return;
registerScheduledJob(ctx, {
key: "auth.cleanup",
lane: "maintenance",
defaults: {
name: "登录会话清理",
description: "删除 7 天前的登录尝试和 30 天前失效的会话",
+3
View File
@@ -181,6 +181,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
start(ctx: ServerContext) {
registerScheduledJob(ctx, {
key: "cache.sweep",
lane: "maintenance",
defaults: {
name: "缓存清理",
description: "删除过期的 TMDB 与识别缓存",
@@ -192,6 +193,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
});
registerScheduledJob(ctx, {
key: "cache.images_clean",
lane: "maintenance",
defaults: {
name: "图片缓存清理",
description: "删除过期的海报缓存",
@@ -206,6 +208,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
});
registerScheduledJob(ctx, {
key: "cache.images_lru",
lane: "maintenance",
defaults: {
name: "图片缓存容量控制",
description: "海报缓存超过容量上限时,删掉最久没用的",
@@ -58,6 +58,7 @@ export function createFileManagerModule(overrides: FmOverrides = {}): ServerModu
defineTask({
name: "purgeTrash",
version: 1,
lane: "maintenance",
inputSchema: z.object({}).passthrough(),
retryPolicy: { maximumAttempts: 3, initialInterval: "5s", backoffCoefficient: 2, maximumInterval: "60s" },
handler: async (t) =>
@@ -70,6 +71,7 @@ export function createFileManagerModule(overrides: FmOverrides = {}): ServerModu
defineTask({
name: IDENTIFY_REMOTE_TASK,
version: 1,
lane: "network",
inputSchema: z.object({ body: FilesIdentifyRequest, actorId: z.string() }),
retryPolicy: { maximumAttempts: 1 },
handler: async (t) =>
+1
View File
@@ -60,6 +60,7 @@ export function createPiModule(
defineTask({
name: "refreshPiTokens",
version: 1,
lane: "network",
inputSchema: z.object({}).passthrough(),
retryPolicy: { maximumAttempts: 2, initialInterval: "10s", backoffCoefficient: 2, maximumInterval: "60s" },
handler: async (t) =>
@@ -48,6 +48,12 @@ export interface ScheduledJobDef {
/** 稳定 key(表主键),如 feeds.poll */
key: string;
kind?: ScheduleKind;
/**
* 执行道(WK-1):"maintenance" = 在调度器里内联干重活的维护任务(清理、回填、保留期裁剪),
* 与 io worker 的维护道共用并发、文件道 / 网络道有活时让路(立即执行不让路)。
* 省略 = 轻量调度(只入队任务或很快结束),任务本身按它的 lane 执行。
*/
lane?: "maintenance";
defaults: ScheduledJobDefaults;
run: JobRunFn;
}
+12 -1
View File
@@ -2,6 +2,7 @@ import { SCHEDULE_CHANGED_EVENT } from "@umt/contracts";
import { type DB, type Kysely, type ScheduledJobRow, sql } from "@umt/db";
import type { Logger } from "pino";
import type { EventBusPort } from "../../http/sse.ts";
import { laneControllerOf } from "../../worker/lanes.ts";
import type { ServerContext } from "../types.ts";
import { runAgentCheck } from "./agent-checks.ts";
import type { JobRunFn, JobRunResult, ScheduleRegistry } from "./registry.ts";
@@ -158,10 +159,20 @@ export class Scheduler {
.limit(50)
.execute();
const started: string[] = [];
const lanes = laneControllerOf(this.ctx);
for (const job of due) {
const fn = this.runFor(job);
if (!fn) continue;
const p = this.execute(job, fn).finally(() => this.inflight.delete(job.id));
// 内联维护任务进维护道:占不到并发或正在让路就留到下一轮(不改 next_run_at);立即执行不让路
let release: (() => void) | null = null;
if (lanes && this.registry.get(job.id)?.lane === "maintenance") {
release = await lanes.enterMaintenance(job.run_requested_at !== null);
if (!release) continue;
}
const p = this.execute(job, fn).finally(() => {
release?.();
this.inflight.delete(job.id);
});
this.inflight.set(job.id, p);
started.push(job.id);
}
@@ -93,6 +93,7 @@ export function createSourcesIdentityModule(overrides: RuntimeOverrides = {}): S
// FIX-5 存量元数据回刷:部署后首次启动即运行,跑完自动停用;可在计划页手动再执行
registerScheduledJob(ctx, {
key: BACKFILL_LANGUAGES_JOB,
lane: "maintenance",
defaults: {
name: "元数据语言回刷",
description: "按语言设置重新拉一遍作品的标题、简介和每集信息;跑完自动停用,可手动再执行",
@@ -29,6 +29,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: "ingestFeed",
version: 1,
lane: "network",
inputSchema: z.object({ feedSourceId: id, manual: z.boolean().optional() }),
retryPolicy: NETWORK_RETRY,
handler: (t) => t.step.run("ingest", () => ingestFeed(rt(), t.input.feedSourceId, { manual: !!t.input.manual })),
@@ -36,6 +37,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: "resolveObservation",
version: 1,
lane: "network",
inputSchema: z
.object({
observationId: id.nullish(),
@@ -60,6 +62,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: AUTO_ENRICH_TASK,
version: 1,
lane: "network",
inputSchema: z.object({ poolEntryId: id }),
retryPolicy: { maximumAttempts: 1 },
handler: (t) => t.step.run("enrich", () => runAutoEnrich(rt().ctx, t.input.poolEntryId, t.runId, t.abortSignal)),
@@ -67,6 +70,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: "refreshMetadata",
version: 1,
lane: "network",
inputSchema: z.object({ mediaEntityId: id, full: z.boolean().optional() }),
retryPolicy: NETWORK_RETRY,
handler: (t) =>
@@ -75,6 +79,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: "searchSubscription",
version: 1,
lane: "network",
inputSchema: z.object({ subscriptionId: id, manual: z.boolean().optional() }),
retryPolicy: NETWORK_RETRY,
handler: (t) =>
@@ -83,6 +88,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: "snapshotSite",
version: 1,
lane: "network",
inputSchema: z.object({
siteId: id,
kind: z.enum(["periodic", "backfill", "manual"]).default("periodic"),
@@ -97,6 +103,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: "refreshSiteAccount",
version: 1,
lane: "network",
inputSchema: z.object({ accountId: id, mode: z.enum(["login", "userData"]).default("login") }),
// 网络失败只记 last_error 不抛错;不自动重试(站点登录频繁重试有封号风险)
retryPolicy: { maximumAttempts: 1 },
+21 -12
View File
@@ -1,12 +1,13 @@
import { readdir, stat } from "node:fs/promises";
import { checkRoot, DEFAULT_ROOT_SENTINEL, type DiskUsage, diskUsage } from "@umt/adapters";
import type {
HealthLevel,
SystemDisk,
SystemHealth,
SystemHealthHistory,
SystemMount,
SystemProcess,
import {
type HealthLevel,
type SystemDisk,
type SystemHealth,
type SystemHealthHistory,
type SystemMount,
type SystemProcess,
SystemWorkerLane,
} from "@umt/contracts";
import { type DB, type Kysely, type Selectable, sql } from "@umt/db";
import { CURRENT_HOST_WINDOW_MS, HEARTBEAT_ROW_RETENTION_MS, HEARTBEAT_STALE_MS } from "./heartbeat.ts";
@@ -61,7 +62,13 @@ function toProcess(r: HeartbeatRow, now: Date): SystemProcess {
const beat = toDate(r.last_beat_at);
const age = Math.max(0, now.getTime() - beat.getTime());
const extra = (r.extra ?? {}) as Record<string, unknown>;
const conc = (r.concurrency ?? {}) as { io?: number; agent?: number };
const conc = (r.concurrency ?? {}) as Record<string, unknown>;
const lanes = Array.isArray(extra.lanes)
? extra.lanes.flatMap((l) => {
const p = SystemWorkerLane.safeParse(l);
return p.success ? [p.data] : [];
})
: [];
const sse = extra.sseConnections;
return {
instanceId: r.instance_id,
@@ -76,10 +83,12 @@ function toProcess(r: HeartbeatRow, now: Date): SystemProcess {
stoppedAt: r.stopped_at ? toDate(r.stopped_at).toISOString() : null,
eventLoopLagMs: r.event_loop_lag_ms,
eventLoopLagMaxMs: r.event_loop_lag_max_ms,
concurrency: {
...(typeof conc.io === "number" ? { io: conc.io } : {}),
...(typeof conc.agent === "number" ? { agent: conc.agent } : {}),
},
concurrency: Object.fromEntries(
(["io", "agent", "file", "network", "maintenance"] as const).flatMap((k) =>
typeof conc[k] === "number" ? [[k, conc[k]]] : [],
),
),
lanes,
runningRuns: r.running_runs,
sseConnections: typeof sse === "number" ? sse : null,
extra,
+2 -1
View File
@@ -51,7 +51,8 @@ export function createLagSampler(intervalMs = 1000) {
}
export interface HeartbeatSample {
concurrency?: { io?: number; agent?: number };
/** io(各道合计)/ agent,以及 io 组的 file / network / maintenance */
concurrency?: Partial<Record<"io" | "agent" | "file" | "network" | "maintenance", number>>;
runningRuns?: number | null;
sseConnections?: number | null;
extra?: Record<string, unknown>;
+2
View File
@@ -185,6 +185,7 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
for (const j of RETENTION_JOBS) {
registerScheduledJob(ctx, {
key: j.jobId,
lane: "maintenance",
defaults: {
name: j.name,
description: j.description,
@@ -208,6 +209,7 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
if (everyMs <= 0) return;
registerScheduledJob(ctx, {
key: "system.heartbeat_cleanup",
lane: "maintenance",
defaults: {
name: "进程心跳清理",
description: "删除过期的进程状态记录",
+120 -40
View File
@@ -1,10 +1,19 @@
import { EventType } from "@umt/contracts";
import { OutboxRelay } from "@umt/tasks";
import { OutboxRelay, type WorkerHandle } from "@umt/tasks";
import type { Logger } from "pino";
import { collectOutboxRoutes, tasksLogSink } from "../context.ts";
import { type EventBusPort, sseConnectionCount } from "../http/sse.ts";
import { type HeartbeatWriter, startHeartbeatWriter } from "../modules/system/heartbeat.ts";
import type { ServerContext, ServerModule } from "../modules/types.ts";
import {
groupTasksByLane,
IO_LANES,
type IoLane,
LaneController,
type LaneOverrides,
type LaneTask,
setLaneController,
} from "./lanes.ts";
/**
* outbox → SSE 桥:这些前缀的 outbox 事件在中继投递成功后镜像到实时总线(payload 原样,带业务 id),
@@ -16,18 +25,20 @@ export const SSE_MIRRORED_OUTBOX_TYPES: readonly string[] = Object.values(EventT
(t) => SSE_MIRROR_PREFIXES.some((p) => t.startsWith(p)) && !SSE_MIRROR_EXCLUDED.has(t),
);
/** agent 组任务:整段会话占一槽数分钟,与文件复制等 io 任务分池 */
export const AGENT_TASKS = ["runAgent", "resolveDecision"] as const;
/** agent 组任务:整段会话占一槽数分钟,与文件复制等 io 任务分池(以任务声明的 lane = "agent" 为准,这是兜底名单) */
export { AGENT_TASKS } from "./lanes.ts";
/**
* worker 分组:
* all — 单个引擎 worker 执行全部任务(测试 / 省略分组时的兼容行为)
* io — 除 AGENT_TASKS 外的全部任务 + OutboxRelay + 模块后台循环
* agent — 只执行 AGENT_TASKS;不跑 relay 与模块循环
* split — 同一进程里 io + agent 两个引擎 worker(--role all)
* io — 除 agent 道外的全部任务,按 lane 分文件 / 网络 / 维护三个引擎 worker + OutboxRelay + 模块后台循环
* agent — 只执行 agent 道任务;不跑 relay 与模块循环
* split — 同一进程里 io 三条道 + agent 共四个引擎 worker(--role all)
*/
export type WorkerGroup = "all" | "io" | "agent" | "split";
export type PoolLabel = "all" | IoLane | "agent";
export interface WorkerDeps {
logger: Logger;
heartbeatMs: number;
@@ -40,14 +51,21 @@ export interface WorkerDeps {
/** 设置 outbox 中继实例到 ctx.outbox.relay */
setRelay?: (relay: OutboxRelay) => void;
outboxIntervalMs?: number;
/** io 组(或 all 组)并发 */
/** all 组(单池)并发 */
concurrency?: number;
/** agent 组并发 */
agentConcurrency?: number;
/** 缺省 all */
group?: WorkerGroup;
/** 注册表里的全部任务名(io 组 = 全部 − AGENT_TASKS);group 为 io/split 时必填 */
taskNames?: readonly string[];
/** 注册表里的全部任务(名称 + lane);group 为 io/agent/split 时必填 */
tasks?: readonly LaneTask[];
/** 分道:环境变量覆盖与测试用的节奏参数 */
lanes?: {
overrides?: LaneOverrides;
refreshMs?: number;
busyCacheMs?: number;
maxYieldMs?: number;
};
};
/** 事件循环延迟探针:采样间隔与告警阈值(测试可调小) */
lagProbe?: { intervalMs?: number; warnMs?: number };
@@ -63,31 +81,38 @@ export interface WorkerStats {
export interface RunningWorker {
stop(): Promise<void>;
stats(): WorkerStats;
/** io / split 组的分道控制器 */
lanes: LaneController | null;
}
/** 按分组拆出引擎 worker 的任务选择(versions 为任务名,匹配该任务全部版本) */
export interface PoolPlan {
label: PoolLabel;
/** 任务名(匹配该任务全部版本);all 池省略 */
versions?: string[];
concurrency?: number;
}
/** 按分组拆出引擎 worker 的任务选择;io 组按 lane 拆三条道(空道不起) */
export function workerPlan(
group: WorkerGroup,
taskNames: readonly string[],
concurrency: { io?: number; agent?: number },
): { label: "all" | "io" | "agent"; versions?: string[]; concurrency?: number }[] {
const agentNames = taskNames.filter((n) => (AGENT_TASKS as readonly string[]).includes(n));
const ioNames = taskNames.filter((n) => !(AGENT_TASKS as readonly string[]).includes(n));
const io = { label: "io" as const, versions: ioNames, ...(concurrency.io ? { concurrency: concurrency.io } : {}) };
const agent = {
label: "agent" as const,
versions: agentNames,
...(concurrency.agent ? { concurrency: concurrency.agent } : {}),
};
tasks: readonly LaneTask[],
concurrency: { all?: number; agent?: number } & LaneOverrides = {},
): PoolPlan[] {
const { byLane } = groupTasksByLane(tasks);
const pool = (label: IoLane | "agent"): PoolPlan[] =>
byLane[label].length
? [{ label, versions: byLane[label], ...(concurrency[label] ? { concurrency: concurrency[label] } : {}) }]
: [];
const io = IO_LANES.flatMap((l) => pool(l));
switch (group) {
case "all":
return [{ label: "all", ...(concurrency.io ? { concurrency: concurrency.io } : {}) }];
return [{ label: "all", ...(concurrency.all ? { concurrency: concurrency.all } : {}) }];
case "io":
return ioNames.length ? [io] : [];
return io;
case "agent":
return agentNames.length ? [agent] : [];
return pool("agent");
case "split":
return [...(ioNames.length ? [io] : []), ...(agentNames.length ? [agent] : [])];
return [...io, ...pool("agent")];
}
}
@@ -134,18 +159,50 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
const started: ServerModule[] = [];
let relay: OutboxRelay | null = null;
const plan: ReturnType<typeof workerPlan> = [];
let lanes: LaneController | null = null;
const plan: PoolPlan[] = [];
const handles = new Map<PoolLabel, WorkerHandle>();
if (runtime) {
const { ctx } = runtime;
if (group !== "all" && !runtime.taskNames) throw new Error(`worker 分组 ${group} 需要 taskNames`);
plan.push(
...workerPlan(group, runtime.taskNames ?? [], { io: runtime.concurrency, agent: runtime.agentConcurrency }),
if (group !== "all" && !runtime.tasks) throw new Error(`worker 分组 ${group} 需要 tasks`);
const tasks = runtime.tasks ?? [];
plan.push(...workerPlan(group, tasks, { all: runtime.concurrency, agent: runtime.agentConcurrency }));
const laned = plan.filter((p): p is PoolPlan & { label: IoLane } =>
(IO_LANES as readonly string[]).includes(p.label),
);
for (const p of plan) {
await ctx.engine.startWorker({
...(p.versions ? { versions: p.versions } : {}),
...(p.concurrency ? { concurrency: p.concurrency } : {}),
if (laned.length) {
const { byLane, missing } = groupTasksByLane(tasks);
if (missing.length)
log.warn({ tasks: missing }, "这些任务没有声明 lane,按缺省归入网络道(请在 defineTask 里补 lane)");
lanes = new LaneController({
engine: ctx.engine,
logger,
lanes: { file: byLane.file, network: byLane.network, maintenance: byLane.maintenance },
readSettings: async () => (await ctx.settings.get("automation")).workerLanes,
...(runtime.lanes?.overrides ? { overrides: runtime.lanes.overrides } : {}),
...(runtime.lanes?.refreshMs ? { refreshMs: runtime.lanes.refreshMs } : {}),
...(runtime.lanes?.busyCacheMs !== undefined ? { busyCacheMs: runtime.lanes.busyCacheMs } : {}),
...(runtime.lanes?.maxYieldMs ? { maxYieldMs: runtime.lanes.maxYieldMs } : {}),
});
await lanes.start();
setLaneController(ctx, lanes);
}
for (const p of plan) {
const lane = (IO_LANES as readonly string[]).includes(p.label) ? (p.label as IoLane) : null;
const handle =
lane && lanes
? await ctx.engine.startWorker({
versions: p.versions ?? [],
concurrency: lanes.slots(lane),
maxActive: () => lanes?.maxActive(lane) ?? 0,
...(lane === "maintenance" ? { claimGate: () => lanes?.maintenanceMayStart() ?? true } : {}),
})
: await ctx.engine.startWorker({
...(p.versions ? { versions: p.versions } : {}),
...(p.concurrency ? { concurrency: p.concurrency } : {}),
});
if (handle) handles.set(p.label, handle);
if (lane) lanes?.attach(lane, handle || undefined);
}
}
// outbox 中继与模块后台循环只在 io 组(agent 组进程只执行会话任务)
@@ -170,12 +227,14 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
}
}
// 进程与健康页:每 15s 把心跳/延迟/并发/运行中 run 数写进 process_heartbeat(写失败不影响 worker)
// 进程与健康页:每 15s 把心跳/延迟/并发/运行中 run 数写进 process_heartbeat(写失败不影响 worker);
// io 组另写各道的并发、本进程运行中、全库排队(extra.lanes),事件循环延迟照常记录(将来拆进程的判据)
let hbWriter: HeartbeatWriter | null = null;
if (runtime) {
const { ctx } = runtime;
const groupTasks = new Set(plan.flatMap((p) => p.versions ?? []));
const countAll = plan.some((p) => !p.versions);
const agentPool = plan.find((p) => p.label === "agent");
hbWriter = startHeartbeatWriter({
db: ctx.db,
logger,
@@ -183,16 +242,27 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
sample: async () => {
const running = await ctx.engine.list({ status: "running", limit: 500 }).catch(() => null);
const s = probe.stats();
const laneStats = lanes ? await lanes.snapshot() : null;
const concurrency: Record<string, number> = {};
if (laneStats) {
for (const l of laneStats) concurrency[l.lane] = l.concurrency;
concurrency.io = laneStats.reduce((a, l) => a + l.concurrency, 0);
} else if (plan.some((p) => p.label === "all")) {
concurrency.io = runtime.concurrency ?? 4;
}
if (agentPool) concurrency.agent = handles.get("agent")?.concurrency ?? runtime.agentConcurrency ?? 4;
return {
concurrency: {
...(plan.some((p) => p.label !== "agent") ? { io: runtime.concurrency ?? 4 } : {}),
...(plan.some((p) => p.label === "agent") ? { agent: runtime.agentConcurrency ?? 4 } : {}),
},
concurrency,
runningRuns: running
? running.filter((r) => r.status === "running" && (countAll || groupTasks.has(r.taskName))).length
: null,
sseConnections: group === "split" ? sseConnectionCount() : null,
extra: { group, lagWarnings: s.loopLagWarnings, probeMaxLagMs: s.maxLoopLagMs },
extra: {
group,
lagWarnings: s.loopLagWarnings,
probeMaxLagMs: s.maxLoopLagMs,
...(laneStats ? { lanes: laneStats } : {}),
},
};
},
});
@@ -202,7 +272,14 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
{
heartbeatMs,
modules: started.map((m) => m.name),
pools: plan.map((p) => ({ group: p.label, concurrency: p.concurrency ?? 4, tasks: p.versions?.length ?? "all" })),
pools: plan.map((p) => ({
group: p.label,
concurrency:
lanes && (IO_LANES as readonly string[]).includes(p.label)
? lanes.limit(p.label as IoLane)
: (p.concurrency ?? 4),
tasks: p.versions?.length ?? "all",
})),
},
"worker started",
);
@@ -210,9 +287,12 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
const timer = setInterval(beat, heartbeatMs);
return {
stats: () => probe.stats(),
lanes,
async stop() {
clearInterval(timer);
probe.stop();
lanes?.stop();
if (runtime) setLaneController(runtime.ctx, null);
await hbWriter?.stop();
for (const m of started.reverse()) {
try {
+271
View File
@@ -0,0 +1,271 @@
import { DEFAULT_WORKER_LANES, WORKER_LANE_MAX, type WorkerLaneSettings } from "@umt/contracts";
import { laneOf, type QueueCounts, type TaskDefinition, type TaskLane, type WorkerHandle } from "@umt/tasks";
import type { Logger } from "pino";
import type { Config } from "../config.ts";
// IO Worker 分道(WK-1,docs/runbook/dev.md「IO Worker 分道」):
// io worker 进程内按任务声明的 lane 起三个引擎 worker(文件 / 网络 / 维护),各自独立并发;
// 维护道"空闲时跑":文件道或网络道(全库)有排队或运行中的 run 时,维护道暂不领取新活(已在跑的不打断),
// 连续让路超过 MAINTENANCE_MAX_YIELD_MS 放行一个,防止网络道长期有活时维护永远跑不上。
// 调度器里内联执行的维护类定时任务(ScheduledJobDef.lane = "maintenance")与维护道共用并发与让路规则。
export const IO_LANES = ["file", "network", "maintenance"] as const;
export type IoLane = (typeof IO_LANES)[number];
export const LANE_LABEL: Record<IoLane, string> = { file: "文件", network: "网络", maintenance: "维护" };
/** 维护道最长连续让路时长,超过即放行一个 */
export const MAINTENANCE_MAX_YIELD_MS = 10 * 60_000;
export type LaneOverrides = Partial<Record<IoLane, number>>;
/** 环境变量覆盖:旧 WORKER_CONCURRENCY 映射到网络道(WORKER_NETWORK_CONCURRENCY 优先) */
export function laneOverridesFromEnv(
config: Pick<
Config,
"WORKER_CONCURRENCY" | "WORKER_FILE_CONCURRENCY" | "WORKER_NETWORK_CONCURRENCY" | "WORKER_MAINTENANCE_CONCURRENCY"
>,
): LaneOverrides {
const network = config.WORKER_NETWORK_CONCURRENCY ?? config.WORKER_CONCURRENCY;
return {
...(config.WORKER_FILE_CONCURRENCY ? { file: config.WORKER_FILE_CONCURRENCY } : {}),
...(network ? { network } : {}),
...(config.WORKER_MAINTENANCE_CONCURRENCY ? { maintenance: config.WORKER_MAINTENANCE_CONCURRENCY } : {}),
};
}
export interface LaneTask {
name: string;
lane?: TaskLane;
}
/** agent 会话任务:没声明 lane 时也归 agent(兼容 WK-1 前的固定名单) */
export const AGENT_TASKS = ["runAgent", "resolveDecision"] as const;
/**
* 按 lane 分组(同名多版本取第一次出现的声明);missing = 没声明 lane 的任务
* (AGENT_TASKS 归 agent,其余按缺省归网络道)。
*/
export function groupTasksByLane(tasks: readonly LaneTask[]): {
byLane: Record<TaskLane, string[]>;
missing: string[];
} {
const byLane: Record<TaskLane, string[]> = { file: [], network: [], maintenance: [], agent: [] };
const missing: string[] = [];
const seen = new Set<string>();
for (const t of tasks) {
if (seen.has(t.name)) continue;
seen.add(t.name);
if (t.lane === undefined) missing.push(t.name);
const lane =
t.lane === undefined && (AGENT_TASKS as readonly string[]).includes(t.name)
? "agent"
: laneOf(t as Pick<TaskDefinition, "lane">);
byLane[lane].push(t.name);
}
for (const k of Object.keys(byLane) as TaskLane[]) byLane[k].sort();
return { byLane, missing: missing.sort() };
}
export interface LaneStat {
lane: IoLane;
/** 当前生效的并发上限 */
concurrency: number;
/** 上限来源 */
source: "env" | "settings";
/** 本进程该道正在执行的 run 数(维护道含调度器内联的维护任务) */
running: number;
/** 全库该道已到点等待领取的 run 数;统计失败为 null */
queued: number | null;
/** 维护道:当前是否在给文件道 / 网络道让路 */
yielding?: boolean;
}
export interface LaneControllerOptions {
engine: { queueCounts?(groups: readonly (readonly string[])[]): Promise<QueueCounts[]> };
logger: Logger;
/** 各道的任务名 */
lanes: Record<IoLane, readonly string[]>;
/** 读设置页的并发(设置读取器自带缓存,settings.changed 事件立即失效) */
readSettings: () => Promise<WorkerLaneSettings>;
overrides?: LaneOverrides;
/** 重读设置的间隔,缺省 5s */
refreshMs?: number;
/** 忙闲判断缓存,缺省 1s */
busyCacheMs?: number;
maxYieldMs?: number;
now?: () => number;
}
export class LaneController {
private readonly limits: Record<IoLane, number>;
private readonly handles: Partial<Record<IoLane, WorkerHandle>> = {};
private inlineMaintenance = 0;
private yieldingSince: number | null = null;
private busyCache: { at: number; value: Promise<boolean> } | null = null;
private timer: ReturnType<typeof setInterval> | null = null;
private readonly log: Logger;
private readonly now: () => number;
constructor(private readonly o: LaneControllerOptions) {
this.log = o.logger.child({ component: "worker-lanes" });
this.now = o.now ?? (() => Date.now());
this.limits = { ...DEFAULT_WORKER_LANES, ...o.overrides };
}
/** 引擎 worker 的槽位数:设置页可调到 WORKER_LANE_MAX;环境变量更大时按环境变量开 */
slots(lane: IoLane): number {
return Math.max(WORKER_LANE_MAX, this.o.overrides?.[lane] ?? 0);
}
source(lane: IoLane): "env" | "settings" {
return this.o.overrides?.[lane] ? "env" : "settings";
}
limit(lane: IoLane): number {
return this.limits[lane];
}
tasks(lane: IoLane): readonly string[] {
return this.o.lanes[lane];
}
attach(lane: IoLane, handle: WorkerHandle | undefined): void {
if (handle) this.handles[lane] = handle;
}
/** 本进程该道正在执行的数 */
active(lane: IoLane): number {
const engine = this.handles[lane]?.active() ?? 0;
return lane === "maintenance" ? engine + this.inlineMaintenance : engine;
}
/** 引擎 worker 的动态上限;维护道扣掉调度器内联占用的 */
maxActive(lane: IoLane): number {
return lane === "maintenance" ? this.limits.maintenance - this.inlineMaintenance : this.limits[lane];
}
/** 重读设置 + 环境变量覆盖;变化时记日志 */
async refresh(): Promise<void> {
let fromSettings: WorkerLaneSettings = DEFAULT_WORKER_LANES;
try {
fromSettings = { ...DEFAULT_WORKER_LANES, ...(await this.o.readSettings()) };
} catch (err) {
this.log.debug({ err }, "读取分道并发设置失败,沿用上次的值");
return;
}
const changed: Record<string, number> = {};
for (const lane of IO_LANES) {
const next = Math.min(this.slots(lane), this.o.overrides?.[lane] ?? fromSettings[lane]);
if (next !== this.limits[lane]) changed[lane] = next;
this.limits[lane] = next;
}
if (Object.keys(changed).length) this.log.info({ changed, limits: { ...this.limits } }, "分道并发已更新");
}
start(): Promise<void> {
if (!this.timer) {
this.timer = setInterval(() => void this.refresh(), this.o.refreshMs ?? 5000);
try {
Deno.unrefTimer(this.timer);
} catch {
// 非 Deno 运行时
}
}
return this.refresh();
}
stop(): void {
if (this.timer) clearInterval(this.timer);
this.timer = null;
}
/** 文件道或网络道(全库)有排队或运行中的 run;统计失败按空闲处理(不让维护道因统计故障永远饿着) */
othersBusy(): Promise<boolean> {
const now = this.now();
if (!this.busyCache || now - this.busyCache.at >= (this.o.busyCacheMs ?? 1000)) {
const names = [...this.o.lanes.file, ...this.o.lanes.network];
const value = (async () => {
if (!this.o.engine.queueCounts || names.length === 0) return false;
try {
const [c] = await this.o.engine.queueCounts([names]);
return !!c && c.queued + c.running > 0;
} catch (err) {
this.log.debug({ err }, "分道忙闲统计失败");
return false;
}
})();
this.busyCache = { at: now, value };
}
return this.busyCache.value;
}
/** 维护道是否可以领取新活:其它两道空闲,或连续让路已超过上限(放行一个并重新计时) */
async maintenanceMayStart(): Promise<boolean> {
if (!(await this.othersBusy())) {
this.yieldingSince = null;
return true;
}
const now = this.now();
this.yieldingSince ??= now;
if (now - this.yieldingSince >= (this.o.maxYieldMs ?? MAINTENANCE_MAX_YIELD_MS)) {
this.log.warn({ yieldedMs: now - this.yieldingSince }, "维护道让路超时,放行一个维护任务");
this.yieldingSince = now;
return true;
}
return false;
}
yielding(): boolean {
return this.yieldingSince !== null;
}
/**
* 调度器内联维护任务进入维护道:占一个维护并发;force(手动立即执行)跳过让路与上限。
* 返回释放函数;不能进入时返回 null(任务保持到期状态,下一轮再试)。
*/
async enterMaintenance(force = false): Promise<(() => void) | null> {
if (!force && this.active("maintenance") >= this.limits.maintenance) return null;
this.inlineMaintenance += 1;
if (!force && !(await this.maintenanceMayStart())) {
this.inlineMaintenance -= 1;
return null;
}
let released = false;
return () => {
if (released) return;
released = true;
this.inlineMaintenance -= 1;
};
}
/** 心跳 / 运行状态页用:各道并发、本进程运行中、全库排队 */
async snapshot(): Promise<LaneStat[]> {
let counts: (QueueCounts | undefined)[] = [];
try {
counts = (await this.o.engine.queueCounts?.(IO_LANES.map((l) => this.o.lanes[l]))) ?? [];
} catch (err) {
this.log.debug({ err }, "分道排队统计失败");
}
return IO_LANES.map((lane, i) => ({
lane,
concurrency: this.limits[lane],
source: this.source(lane),
running: this.active(lane),
queued: counts[i]?.queued ?? null,
...(lane === "maintenance" ? { yielding: this.yielding() } : {}),
}));
}
}
// 同一 ServerContext 共用一个分道控制器(调度器据此给内联维护任务让路);没起 io worker 时为空
const controllers = new WeakMap<object, LaneController>();
export function setLaneController(ctx: object, c: LaneController | null): void {
if (c) controllers.set(ctx, c);
else controllers.delete(ctx);
}
export function laneControllerOf(ctx: object): LaneController | undefined {
return controllers.get(ctx);
}