feat(worker): IO Worker 分道(WK-1)

io worker 进程内按任务声明的 lane 分文件 / 网络 / 维护三条道,各自并发(缺省 2/4/1)、
各自排队;维护道在文件道或网络道有排队/运行时让路(连续 10 分钟放行一个),调度器内联的
清理/回填/保留期裁剪也走维护道。并发在 设置 → 自动化 → 高级 热生效,env 覆盖;旧
WORKER_CONCURRENCY 映射到网络道。运行状态页按道显示运行中/并发/排队。归属表写进
docs/runbook/dev.md,ci 与注册表逐项核对。

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-30 00:24:21 +08:00
1 parent 68f53390f6
commit da33cad81d
38 files changed
+1240 -105

No files matched your search

+12 -2
View File
@@ -4,6 +4,9 @@ import { z } from "zod";
/** compose 里 `${X:-}` 展开成空串:按未设置处理 */
const emptyToUndef = (v: unknown) => (v === "" ? undefined : v);
/** 可选正整数环境变量(空串 = 未设置) */
const optionalInt = (max: number) => z.preprocess(emptyToUndef, z.coerce.number().int().min(1).max(max).optional());
const LogLevel = z.enum(["fatal", "error", "warn", "info", "debug", "trace", "silent"]);
export const ConfigSchema = z.object({
@@ -29,8 +32,15 @@ export const ConfigSchema = z.object({
DEV_ADMIN_KEY: z.string().min(8).optional(),
/** worker 心跳间隔 */
WORKER_HEARTBEAT_MS: z.coerce.number().int().min(1000).default(30_000),
/** io 组 worker(除 runAgent/resolveDecision 外的全部任务)并发槽数 */
WORKER_CONCURRENCY: z.coerce.number().int().min(1).max(64).default(4),
/**
* 旧变量(WK-1 前的 io 组总并发):现在映射到网络道(WORKER_NETWORK_CONCURRENCY 未设时);
* 不设 = 用设置页的值。空串视为未设置(compose 里 `${WORKER_CONCURRENCY:-}`)。
*/
WORKER_CONCURRENCY: optionalInt(64),
/** io worker 三条道的并发(WK-1);设置了就覆盖 设置 → 自动化 → 高级 里的值 */
WORKER_FILE_CONCURRENCY: optionalInt(64),
WORKER_NETWORK_CONCURRENCY: optionalInt(64),
WORKER_MAINTENANCE_CONCURRENCY: optionalInt(64),
/** agent 组 worker(runAgent/resolveDecision:整段会话占一槽数分钟)并发槽数 */
WORKER_AGENT_CONCURRENCY: z.coerce.number().int().min(1).max(16).default(2),
/** 手动整理 dryRun / 按路径整理时递归遍历的文件数上限(超出截断并标记 truncated) */
+4
View File
@@ -41,6 +41,10 @@ export class DeferredTaskEngine implements TaskEngine {
get: TaskEngine["get"] = (...a) => this.e.get(...a);
list: TaskEngine["list"] = (...a) => this.e.list(...a);
startWorker: TaskEngine["startWorker"] = (...a) => this.e.startWorker(...a);
queueCounts: NonNullable<TaskEngine["queueCounts"]> = async (...a) => {
const q = this.e.queueCounts;
return q ? q.apply(this.e, a) : a[0].map(() => ({ queued: 0, running: 0 }));
};
stop: TaskEngine["stop"] = async () => {
await this.inner?.stop();
};
+5 -3
View File
@@ -3,7 +3,7 @@ import denoJson from "../deno.json" with { type: "json" };
import type { Config } from "./config.ts";
import type { MappingExport } from "./ops.ts";
// api / worker / worker:agent / all 是常驻角色(worker = io 组:除 runAgent/resolveDecision 外的任务 + outbox 中继 +
// api / worker / worker:agent / all 是常驻角色(worker = io 组:文件 / 网络 / 维护三条道 + outbox 中继 +
// 模块后台循环;worker:agent = 只跑 Agent 会话任务;all = api + 两组 worker 同进程);其余是容器入口 / 备份恢复脚本用的一次性子命令:
// migrate | preflight | probe
// export-mappings --out <file.json> [--compare <基准清单.json>] 映射清单导出(可与基准比行数)
@@ -77,6 +77,7 @@ const { SERVER_MODULES } = await import("./modules/index.ts");
const { diffCounts, enableReadOnlyReconcile, exportMappings, writeMappingExport } = await import("./ops.ts");
const { auditStorageWarnings, checkStorage, loadMountedPaths, runPreflight } = await import("./preflight.ts");
const { startWorker } = await import("./worker/index.ts");
const { laneOverridesFromEnv } = await import("./worker/lanes.ts");
let config: Config;
try {
@@ -253,9 +254,10 @@ if (role === "worker" || role === "worker:agent" || role === "all") {
modules: SERVER_MODULES,
setRelay: outbox.setRelay,
group: role === "all" ? "split" : role === "worker" ? "io" : "agent",
taskNames: booted.tasks.names(),
concurrency: config.WORKER_CONCURRENCY,
tasks: booted.tasks.all(),
agentConcurrency: config.WORKER_AGENT_CONCURRENCY,
// 三条道的并发读 设置 → 自动化 → 高级(热生效);环境变量设了就覆盖(旧 WORKER_CONCURRENCY → 网络道)
lanes: { overrides: laneOverridesFromEnv(config) },
},
});
stops.push(worker.stop);
@@ -76,6 +76,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
defineTask({
name: "evaluateAcquisition",
version: 1,
lane: "network",
inputSchema: z.object({
subscriptionId: id.optional(),
acquisitionSlotId: id.optional(),
@@ -91,6 +92,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
defineTask({
name: "submitDownload",
version: 1,
lane: "network",
inputSchema: z.object({ acquisitionPlanId: id, taskId: id.optional() }),
retryPolicy: NETWORK_RETRY,
handler: async (t) => {
@@ -117,6 +119,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
defineTask({
name: "verifyAndImport",
version: 1,
lane: "file",
inputSchema: z.object({ downloadTaskId: id.optional(), recordId: id.optional(), taskId: id.optional() }),
retryPolicy: FILE_RETRY,
handler: async (t) => {
@@ -174,6 +177,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
defineTask({
name: "cleanupDownloadCache",
version: 1,
lane: "file",
inputSchema: z.object({ downloaderInstanceId: id.optional(), taskId: id.optional() }),
retryPolicy: FILE_RETRY,
handler: async (t) => {
@@ -186,6 +190,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
defineTask({
name: "reconcileLibrary",
version: 1,
lane: "maintenance",
inputSchema: z.object({ libraryRootId: id.optional(), reconcileRunId: id.optional(), taskId: id.optional() }),
retryPolicy: FILE_RETRY,
handler: async (t) => {
+2
View File
@@ -37,6 +37,7 @@ export function registerAgentTasks(registry: TaskRegistry, env: () => AgentEnv)
defineTask({
name: "runAgent",
version: 1,
lane: "agent",
/** upstreamId:覆盖默认上游(W7-A);缺省用 agent_run.upstream_id,再缺省 settings.pi.defaultUpstreamId */
inputSchema: z.object({ agentRunId: id, upstreamId: id.optional() }),
retryPolicy: DB_RETRY,
@@ -175,6 +176,7 @@ export function registerAgentTasks(registry: TaskRegistry, env: () => AgentEnv)
defineTask({
name: "resolveDecision",
version: 1,
lane: "agent",
inputSchema: z.object({ decisionId: id, trigger: z.enum(["created", "updated"]).default("updated") }),
retryPolicy: DB_RETRY,
handler: async (t) => {
@@ -50,6 +50,7 @@ export function createAuthSettingsModule(opts: AuthSettingsModuleOptions = {}):
if (everyMs <= 0) return;
registerScheduledJob(ctx, {
key: "auth.cleanup",
lane: "maintenance",
defaults: {
name: "登录会话清理",
description: "删除 7 天前的登录尝试和 30 天前失效的会话",
+3
View File
@@ -181,6 +181,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
start(ctx: ServerContext) {
registerScheduledJob(ctx, {
key: "cache.sweep",
lane: "maintenance",
defaults: {
name: "缓存清理",
description: "删除过期的 TMDB 与识别缓存",
@@ -192,6 +193,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
});
registerScheduledJob(ctx, {
key: "cache.images_clean",
lane: "maintenance",
defaults: {
name: "图片缓存清理",
description: "删除过期的海报缓存",
@@ -206,6 +208,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
});
registerScheduledJob(ctx, {
key: "cache.images_lru",
lane: "maintenance",
defaults: {
name: "图片缓存容量控制",
description: "海报缓存超过容量上限时,删掉最久没用的",
@@ -58,6 +58,7 @@ export function createFileManagerModule(overrides: FmOverrides = {}): ServerModu
defineTask({
name: "purgeTrash",
version: 1,
lane: "maintenance",
inputSchema: z.object({}).passthrough(),
retryPolicy: { maximumAttempts: 3, initialInterval: "5s", backoffCoefficient: 2, maximumInterval: "60s" },
handler: async (t) =>
@@ -70,6 +71,7 @@ export function createFileManagerModule(overrides: FmOverrides = {}): ServerModu
defineTask({
name: IDENTIFY_REMOTE_TASK,
version: 1,
lane: "network",
inputSchema: z.object({ body: FilesIdentifyRequest, actorId: z.string() }),
retryPolicy: { maximumAttempts: 1 },
handler: async (t) =>
+1
View File
@@ -60,6 +60,7 @@ export function createPiModule(
defineTask({
name: "refreshPiTokens",
version: 1,
lane: "network",
inputSchema: z.object({}).passthrough(),
retryPolicy: { maximumAttempts: 2, initialInterval: "10s", backoffCoefficient: 2, maximumInterval: "60s" },
handler: async (t) =>
@@ -48,6 +48,12 @@ export interface ScheduledJobDef {
/** 稳定 key(表主键),如 feeds.poll */
key: string;
kind?: ScheduleKind;
/**
* 执行道(WK-1):"maintenance" = 在调度器里内联干重活的维护任务(清理、回填、保留期裁剪),
* 与 io worker 的维护道共用并发、文件道 / 网络道有活时让路(立即执行不让路)。
* 省略 = 轻量调度(只入队任务或很快结束),任务本身按它的 lane 执行。
*/
lane?: "maintenance";
defaults: ScheduledJobDefaults;
run: JobRunFn;
}
+12 -1
View File
@@ -2,6 +2,7 @@ import { SCHEDULE_CHANGED_EVENT } from "@umt/contracts";
import { type DB, type Kysely, type ScheduledJobRow, sql } from "@umt/db";
import type { Logger } from "pino";
import type { EventBusPort } from "../../http/sse.ts";
import { laneControllerOf } from "../../worker/lanes.ts";
import type { ServerContext } from "../types.ts";
import { runAgentCheck } from "./agent-checks.ts";
import type { JobRunFn, JobRunResult, ScheduleRegistry } from "./registry.ts";
@@ -158,10 +159,20 @@ export class Scheduler {
.limit(50)
.execute();
const started: string[] = [];
const lanes = laneControllerOf(this.ctx);
for (const job of due) {
const fn = this.runFor(job);
if (!fn) continue;
const p = this.execute(job, fn).finally(() => this.inflight.delete(job.id));
// 内联维护任务进维护道:占不到并发或正在让路就留到下一轮(不改 next_run_at);立即执行不让路
let release: (() => void) | null = null;
if (lanes && this.registry.get(job.id)?.lane === "maintenance") {
release = await lanes.enterMaintenance(job.run_requested_at !== null);
if (!release) continue;
}
const p = this.execute(job, fn).finally(() => {
release?.();
this.inflight.delete(job.id);
});
this.inflight.set(job.id, p);
started.push(job.id);
}
@@ -93,6 +93,7 @@ export function createSourcesIdentityModule(overrides: RuntimeOverrides = {}): S
// FIX-5 存量元数据回刷:部署后首次启动即运行,跑完自动停用;可在计划页手动再执行
registerScheduledJob(ctx, {
key: BACKFILL_LANGUAGES_JOB,
lane: "maintenance",
defaults: {
name: "元数据语言回刷",
description: "按语言设置重新拉一遍作品的标题、简介和每集信息;跑完自动停用,可手动再执行",
@@ -29,6 +29,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: "ingestFeed",
version: 1,
lane: "network",
inputSchema: z.object({ feedSourceId: id, manual: z.boolean().optional() }),
retryPolicy: NETWORK_RETRY,
handler: (t) => t.step.run("ingest", () => ingestFeed(rt(), t.input.feedSourceId, { manual: !!t.input.manual })),
@@ -36,6 +37,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: "resolveObservation",
version: 1,
lane: "network",
inputSchema: z
.object({
observationId: id.nullish(),
@@ -60,6 +62,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: AUTO_ENRICH_TASK,
version: 1,
lane: "network",
inputSchema: z.object({ poolEntryId: id }),
retryPolicy: { maximumAttempts: 1 },
handler: (t) => t.step.run("enrich", () => runAutoEnrich(rt().ctx, t.input.poolEntryId, t.runId, t.abortSignal)),
@@ -67,6 +70,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: "refreshMetadata",
version: 1,
lane: "network",
inputSchema: z.object({ mediaEntityId: id, full: z.boolean().optional() }),
retryPolicy: NETWORK_RETRY,
handler: (t) =>
@@ -75,6 +79,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: "searchSubscription",
version: 1,
lane: "network",
inputSchema: z.object({ subscriptionId: id, manual: z.boolean().optional() }),
retryPolicy: NETWORK_RETRY,
handler: (t) =>
@@ -83,6 +88,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: "snapshotSite",
version: 1,
lane: "network",
inputSchema: z.object({
siteId: id,
kind: z.enum(["periodic", "backfill", "manual"]).default("periodic"),
@@ -97,6 +103,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
defineTask({
name: "refreshSiteAccount",
version: 1,
lane: "network",
inputSchema: z.object({ accountId: id, mode: z.enum(["login", "userData"]).default("login") }),
// 网络失败只记 last_error 不抛错;不自动重试(站点登录频繁重试有封号风险)
retryPolicy: { maximumAttempts: 1 },
+21 -12
View File
@@ -1,12 +1,13 @@
import { readdir, stat } from "node:fs/promises";
import { checkRoot, DEFAULT_ROOT_SENTINEL, type DiskUsage, diskUsage } from "@umt/adapters";
import type {
HealthLevel,
SystemDisk,
SystemHealth,
SystemHealthHistory,
SystemMount,
SystemProcess,
import {
type HealthLevel,
type SystemDisk,
type SystemHealth,
type SystemHealthHistory,
type SystemMount,
type SystemProcess,
SystemWorkerLane,
} from "@umt/contracts";
import { type DB, type Kysely, type Selectable, sql } from "@umt/db";
import { CURRENT_HOST_WINDOW_MS, HEARTBEAT_ROW_RETENTION_MS, HEARTBEAT_STALE_MS } from "./heartbeat.ts";
@@ -61,7 +62,13 @@ function toProcess(r: HeartbeatRow, now: Date): SystemProcess {
const beat = toDate(r.last_beat_at);
const age = Math.max(0, now.getTime() - beat.getTime());
const extra = (r.extra ?? {}) as Record<string, unknown>;
const conc = (r.concurrency ?? {}) as { io?: number; agent?: number };
const conc = (r.concurrency ?? {}) as Record<string, unknown>;
const lanes = Array.isArray(extra.lanes)
? extra.lanes.flatMap((l) => {
const p = SystemWorkerLane.safeParse(l);
return p.success ? [p.data] : [];
})
: [];
const sse = extra.sseConnections;
return {
instanceId: r.instance_id,
@@ -76,10 +83,12 @@ function toProcess(r: HeartbeatRow, now: Date): SystemProcess {
stoppedAt: r.stopped_at ? toDate(r.stopped_at).toISOString() : null,
eventLoopLagMs: r.event_loop_lag_ms,
eventLoopLagMaxMs: r.event_loop_lag_max_ms,
concurrency: {
...(typeof conc.io === "number" ? { io: conc.io } : {}),
...(typeof conc.agent === "number" ? { agent: conc.agent } : {}),
},
concurrency: Object.fromEntries(
(["io", "agent", "file", "network", "maintenance"] as const).flatMap((k) =>
typeof conc[k] === "number" ? [[k, conc[k]]] : [],
),
),
lanes,
runningRuns: r.running_runs,
sseConnections: typeof sse === "number" ? sse : null,
extra,
+2 -1
View File
@@ -51,7 +51,8 @@ export function createLagSampler(intervalMs = 1000) {
}
export interface HeartbeatSample {
concurrency?: { io?: number; agent?: number };
/** io(各道合计)/ agent,以及 io 组的 file / network / maintenance */
concurrency?: Partial<Record<"io" | "agent" | "file" | "network" | "maintenance", number>>;
runningRuns?: number | null;
sseConnections?: number | null;
extra?: Record<string, unknown>;
+2
View File
@@ -185,6 +185,7 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
for (const j of RETENTION_JOBS) {
registerScheduledJob(ctx, {
key: j.jobId,
lane: "maintenance",
defaults: {
name: j.name,
description: j.description,
@@ -208,6 +209,7 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
if (everyMs <= 0) return;
registerScheduledJob(ctx, {
key: "system.heartbeat_cleanup",
lane: "maintenance",
defaults: {
name: "进程心跳清理",
description: "删除过期的进程状态记录",
+120 -40
View File
@@ -1,10 +1,19 @@
import { EventType } from "@umt/contracts";
import { OutboxRelay } from "@umt/tasks";
import { OutboxRelay, type WorkerHandle } from "@umt/tasks";
import type { Logger } from "pino";
import { collectOutboxRoutes, tasksLogSink } from "../context.ts";
import { type EventBusPort, sseConnectionCount } from "../http/sse.ts";
import { type HeartbeatWriter, startHeartbeatWriter } from "../modules/system/heartbeat.ts";
import type { ServerContext, ServerModule } from "../modules/types.ts";
import {
groupTasksByLane,
IO_LANES,
type IoLane,
LaneController,
type LaneOverrides,
type LaneTask,
setLaneController,
} from "./lanes.ts";
/**
* outbox → SSE 桥:这些前缀的 outbox 事件在中继投递成功后镜像到实时总线(payload 原样,带业务 id),
@@ -16,18 +25,20 @@ export const SSE_MIRRORED_OUTBOX_TYPES: readonly string[] = Object.values(EventT
(t) => SSE_MIRROR_PREFIXES.some((p) => t.startsWith(p)) && !SSE_MIRROR_EXCLUDED.has(t),
);
/** agent 组任务:整段会话占一槽数分钟,与文件复制等 io 任务分池 */
export const AGENT_TASKS = ["runAgent", "resolveDecision"] as const;
/** agent 组任务:整段会话占一槽数分钟,与文件复制等 io 任务分池(以任务声明的 lane = "agent" 为准,这是兜底名单) */
export { AGENT_TASKS } from "./lanes.ts";
/**
* worker 分组:
* all — 单个引擎 worker 执行全部任务(测试 / 省略分组时的兼容行为)
* io — 除 AGENT_TASKS 外的全部任务 + OutboxRelay + 模块后台循环
* agent — 只执行 AGENT_TASKS;不跑 relay 与模块循环
* split — 同一进程里 io + agent 两个引擎 worker(--role all)
* io — 除 agent 道外的全部任务,按 lane 分文件 / 网络 / 维护三个引擎 worker + OutboxRelay + 模块后台循环
* agent — 只执行 agent 道任务;不跑 relay 与模块循环
* split — 同一进程里 io 三条道 + agent 共四个引擎 worker(--role all)
*/
export type WorkerGroup = "all" | "io" | "agent" | "split";
export type PoolLabel = "all" | IoLane | "agent";
export interface WorkerDeps {
logger: Logger;
heartbeatMs: number;
@@ -40,14 +51,21 @@ export interface WorkerDeps {
/** 设置 outbox 中继实例到 ctx.outbox.relay */
setRelay?: (relay: OutboxRelay) => void;
outboxIntervalMs?: number;
/** io 组(或 all 组)并发 */
/** all 组(单池)并发 */
concurrency?: number;
/** agent 组并发 */
agentConcurrency?: number;
/** 缺省 all */
group?: WorkerGroup;
/** 注册表里的全部任务名(io 组 = 全部 − AGENT_TASKS);group 为 io/split 时必填 */
taskNames?: readonly string[];
/** 注册表里的全部任务(名称 + lane);group 为 io/agent/split 时必填 */
tasks?: readonly LaneTask[];
/** 分道:环境变量覆盖与测试用的节奏参数 */
lanes?: {
overrides?: LaneOverrides;
refreshMs?: number;
busyCacheMs?: number;
maxYieldMs?: number;
};
};
/** 事件循环延迟探针:采样间隔与告警阈值(测试可调小) */
lagProbe?: { intervalMs?: number; warnMs?: number };
@@ -63,31 +81,38 @@ export interface WorkerStats {
export interface RunningWorker {
stop(): Promise<void>;
stats(): WorkerStats;
/** io / split 组的分道控制器 */
lanes: LaneController | null;
}
/** 按分组拆出引擎 worker 的任务选择(versions 为任务名,匹配该任务全部版本) */
export interface PoolPlan {
label: PoolLabel;
/** 任务名(匹配该任务全部版本);all 池省略 */
versions?: string[];
concurrency?: number;
}
/** 按分组拆出引擎 worker 的任务选择;io 组按 lane 拆三条道(空道不起) */
export function workerPlan(
group: WorkerGroup,
taskNames: readonly string[],
concurrency: { io?: number; agent?: number },
): { label: "all" | "io" | "agent"; versions?: string[]; concurrency?: number }[] {
const agentNames = taskNames.filter((n) => (AGENT_TASKS as readonly string[]).includes(n));
const ioNames = taskNames.filter((n) => !(AGENT_TASKS as readonly string[]).includes(n));
const io = { label: "io" as const, versions: ioNames, ...(concurrency.io ? { concurrency: concurrency.io } : {}) };
const agent = {
label: "agent" as const,
versions: agentNames,
...(concurrency.agent ? { concurrency: concurrency.agent } : {}),
};
tasks: readonly LaneTask[],
concurrency: { all?: number; agent?: number } & LaneOverrides = {},
): PoolPlan[] {
const { byLane } = groupTasksByLane(tasks);
const pool = (label: IoLane | "agent"): PoolPlan[] =>
byLane[label].length
? [{ label, versions: byLane[label], ...(concurrency[label] ? { concurrency: concurrency[label] } : {}) }]
: [];
const io = IO_LANES.flatMap((l) => pool(l));
switch (group) {
case "all":
return [{ label: "all", ...(concurrency.io ? { concurrency: concurrency.io } : {}) }];
return [{ label: "all", ...(concurrency.all ? { concurrency: concurrency.all } : {}) }];
case "io":
return ioNames.length ? [io] : [];
return io;
case "agent":
return agentNames.length ? [agent] : [];
return pool("agent");
case "split":
return [...(ioNames.length ? [io] : []), ...(agentNames.length ? [agent] : [])];
return [...io, ...pool("agent")];
}
}
@@ -134,18 +159,50 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
const started: ServerModule[] = [];
let relay: OutboxRelay | null = null;
const plan: ReturnType<typeof workerPlan> = [];
let lanes: LaneController | null = null;
const plan: PoolPlan[] = [];
const handles = new Map<PoolLabel, WorkerHandle>();
if (runtime) {
const { ctx } = runtime;
if (group !== "all" && !runtime.taskNames) throw new Error(`worker 分组 ${group} 需要 taskNames`);
plan.push(
...workerPlan(group, runtime.taskNames ?? [], { io: runtime.concurrency, agent: runtime.agentConcurrency }),
if (group !== "all" && !runtime.tasks) throw new Error(`worker 分组 ${group} 需要 tasks`);
const tasks = runtime.tasks ?? [];
plan.push(...workerPlan(group, tasks, { all: runtime.concurrency, agent: runtime.agentConcurrency }));
const laned = plan.filter((p): p is PoolPlan & { label: IoLane } =>
(IO_LANES as readonly string[]).includes(p.label),
);
for (const p of plan) {
await ctx.engine.startWorker({
...(p.versions ? { versions: p.versions } : {}),
...(p.concurrency ? { concurrency: p.concurrency } : {}),
if (laned.length) {
const { byLane, missing } = groupTasksByLane(tasks);
if (missing.length)
log.warn({ tasks: missing }, "这些任务没有声明 lane,按缺省归入网络道(请在 defineTask 里补 lane)");
lanes = new LaneController({
engine: ctx.engine,
logger,
lanes: { file: byLane.file, network: byLane.network, maintenance: byLane.maintenance },
readSettings: async () => (await ctx.settings.get("automation")).workerLanes,
...(runtime.lanes?.overrides ? { overrides: runtime.lanes.overrides } : {}),
...(runtime.lanes?.refreshMs ? { refreshMs: runtime.lanes.refreshMs } : {}),
...(runtime.lanes?.busyCacheMs !== undefined ? { busyCacheMs: runtime.lanes.busyCacheMs } : {}),
...(runtime.lanes?.maxYieldMs ? { maxYieldMs: runtime.lanes.maxYieldMs } : {}),
});
await lanes.start();
setLaneController(ctx, lanes);
}
for (const p of plan) {
const lane = (IO_LANES as readonly string[]).includes(p.label) ? (p.label as IoLane) : null;
const handle =
lane && lanes
? await ctx.engine.startWorker({
versions: p.versions ?? [],
concurrency: lanes.slots(lane),
maxActive: () => lanes?.maxActive(lane) ?? 0,
...(lane === "maintenance" ? { claimGate: () => lanes?.maintenanceMayStart() ?? true } : {}),
})
: await ctx.engine.startWorker({
...(p.versions ? { versions: p.versions } : {}),
...(p.concurrency ? { concurrency: p.concurrency } : {}),
});
if (handle) handles.set(p.label, handle);
if (lane) lanes?.attach(lane, handle || undefined);
}
}
// outbox 中继与模块后台循环只在 io 组(agent 组进程只执行会话任务)
@@ -170,12 +227,14 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
}
}
// 进程与健康页:每 15s 把心跳/延迟/并发/运行中 run 数写进 process_heartbeat(写失败不影响 worker)
// 进程与健康页:每 15s 把心跳/延迟/并发/运行中 run 数写进 process_heartbeat(写失败不影响 worker);
// io 组另写各道的并发、本进程运行中、全库排队(extra.lanes),事件循环延迟照常记录(将来拆进程的判据)
let hbWriter: HeartbeatWriter | null = null;
if (runtime) {
const { ctx } = runtime;
const groupTasks = new Set(plan.flatMap((p) => p.versions ?? []));
const countAll = plan.some((p) => !p.versions);
const agentPool = plan.find((p) => p.label === "agent");
hbWriter = startHeartbeatWriter({
db: ctx.db,
logger,
@@ -183,16 +242,27 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
sample: async () => {
const running = await ctx.engine.list({ status: "running", limit: 500 }).catch(() => null);
const s = probe.stats();
const laneStats = lanes ? await lanes.snapshot() : null;
const concurrency: Record<string, number> = {};
if (laneStats) {
for (const l of laneStats) concurrency[l.lane] = l.concurrency;
concurrency.io = laneStats.reduce((a, l) => a + l.concurrency, 0);
} else if (plan.some((p) => p.label === "all")) {
concurrency.io = runtime.concurrency ?? 4;
}
if (agentPool) concurrency.agent = handles.get("agent")?.concurrency ?? runtime.agentConcurrency ?? 4;
return {
concurrency: {
...(plan.some((p) => p.label !== "agent") ? { io: runtime.concurrency ?? 4 } : {}),
...(plan.some((p) => p.label === "agent") ? { agent: runtime.agentConcurrency ?? 4 } : {}),
},
concurrency,
runningRuns: running
? running.filter((r) => r.status === "running" && (countAll || groupTasks.has(r.taskName))).length
: null,
sseConnections: group === "split" ? sseConnectionCount() : null,
extra: { group, lagWarnings: s.loopLagWarnings, probeMaxLagMs: s.maxLoopLagMs },
extra: {
group,
lagWarnings: s.loopLagWarnings,
probeMaxLagMs: s.maxLoopLagMs,
...(laneStats ? { lanes: laneStats } : {}),
},
};
},
});
@@ -202,7 +272,14 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
{
heartbeatMs,
modules: started.map((m) => m.name),
pools: plan.map((p) => ({ group: p.label, concurrency: p.concurrency ?? 4, tasks: p.versions?.length ?? "all" })),
pools: plan.map((p) => ({
group: p.label,
concurrency:
lanes && (IO_LANES as readonly string[]).includes(p.label)
? lanes.limit(p.label as IoLane)
: (p.concurrency ?? 4),
tasks: p.versions?.length ?? "all",
})),
},
"worker started",
);
@@ -210,9 +287,12 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
const timer = setInterval(beat, heartbeatMs);
return {
stats: () => probe.stats(),
lanes,
async stop() {
clearInterval(timer);
probe.stop();
lanes?.stop();
if (runtime) setLaneController(runtime.ctx, null);
await hbWriter?.stop();
for (const m of started.reverse()) {
try {
+271
View File
@@ -0,0 +1,271 @@
import { DEFAULT_WORKER_LANES, WORKER_LANE_MAX, type WorkerLaneSettings } from "@umt/contracts";
import { laneOf, type QueueCounts, type TaskDefinition, type TaskLane, type WorkerHandle } from "@umt/tasks";
import type { Logger } from "pino";
import type { Config } from "../config.ts";
// IO Worker 分道(WK-1,docs/runbook/dev.md「IO Worker 分道」):
// io worker 进程内按任务声明的 lane 起三个引擎 worker(文件 / 网络 / 维护),各自独立并发;
// 维护道"空闲时跑":文件道或网络道(全库)有排队或运行中的 run 时,维护道暂不领取新活(已在跑的不打断),
// 连续让路超过 MAINTENANCE_MAX_YIELD_MS 放行一个,防止网络道长期有活时维护永远跑不上。
// 调度器里内联执行的维护类定时任务(ScheduledJobDef.lane = "maintenance")与维护道共用并发与让路规则。
export const IO_LANES = ["file", "network", "maintenance"] as const;
export type IoLane = (typeof IO_LANES)[number];
export const LANE_LABEL: Record<IoLane, string> = { file: "文件", network: "网络", maintenance: "维护" };
/** 维护道最长连续让路时长,超过即放行一个 */
export const MAINTENANCE_MAX_YIELD_MS = 10 * 60_000;
export type LaneOverrides = Partial<Record<IoLane, number>>;
/** 环境变量覆盖:旧 WORKER_CONCURRENCY 映射到网络道(WORKER_NETWORK_CONCURRENCY 优先) */
export function laneOverridesFromEnv(
config: Pick<
Config,
"WORKER_CONCURRENCY" | "WORKER_FILE_CONCURRENCY" | "WORKER_NETWORK_CONCURRENCY" | "WORKER_MAINTENANCE_CONCURRENCY"
>,
): LaneOverrides {
const network = config.WORKER_NETWORK_CONCURRENCY ?? config.WORKER_CONCURRENCY;
return {
...(config.WORKER_FILE_CONCURRENCY ? { file: config.WORKER_FILE_CONCURRENCY } : {}),
...(network ? { network } : {}),
...(config.WORKER_MAINTENANCE_CONCURRENCY ? { maintenance: config.WORKER_MAINTENANCE_CONCURRENCY } : {}),
};
}
export interface LaneTask {
name: string;
lane?: TaskLane;
}
/** agent 会话任务:没声明 lane 时也归 agent(兼容 WK-1 前的固定名单) */
export const AGENT_TASKS = ["runAgent", "resolveDecision"] as const;
/**
* 按 lane 分组(同名多版本取第一次出现的声明);missing = 没声明 lane 的任务
* (AGENT_TASKS 归 agent,其余按缺省归网络道)。
*/
export function groupTasksByLane(tasks: readonly LaneTask[]): {
byLane: Record<TaskLane, string[]>;
missing: string[];
} {
const byLane: Record<TaskLane, string[]> = { file: [], network: [], maintenance: [], agent: [] };
const missing: string[] = [];
const seen = new Set<string>();
for (const t of tasks) {
if (seen.has(t.name)) continue;
seen.add(t.name);
if (t.lane === undefined) missing.push(t.name);
const lane =
t.lane === undefined && (AGENT_TASKS as readonly string[]).includes(t.name)
? "agent"
: laneOf(t as Pick<TaskDefinition, "lane">);
byLane[lane].push(t.name);
}
for (const k of Object.keys(byLane) as TaskLane[]) byLane[k].sort();
return { byLane, missing: missing.sort() };
}
export interface LaneStat {
lane: IoLane;
/** 当前生效的并发上限 */
concurrency: number;
/** 上限来源 */
source: "env" | "settings";
/** 本进程该道正在执行的 run 数(维护道含调度器内联的维护任务) */
running: number;
/** 全库该道已到点等待领取的 run 数;统计失败为 null */
queued: number | null;
/** 维护道:当前是否在给文件道 / 网络道让路 */
yielding?: boolean;
}
export interface LaneControllerOptions {
engine: { queueCounts?(groups: readonly (readonly string[])[]): Promise<QueueCounts[]> };
logger: Logger;
/** 各道的任务名 */
lanes: Record<IoLane, readonly string[]>;
/** 读设置页的并发(设置读取器自带缓存,settings.changed 事件立即失效) */
readSettings: () => Promise<WorkerLaneSettings>;
overrides?: LaneOverrides;
/** 重读设置的间隔,缺省 5s */
refreshMs?: number;
/** 忙闲判断缓存,缺省 1s */
busyCacheMs?: number;
maxYieldMs?: number;
now?: () => number;
}
export class LaneController {
private readonly limits: Record<IoLane, number>;
private readonly handles: Partial<Record<IoLane, WorkerHandle>> = {};
private inlineMaintenance = 0;
private yieldingSince: number | null = null;
private busyCache: { at: number; value: Promise<boolean> } | null = null;
private timer: ReturnType<typeof setInterval> | null = null;
private readonly log: Logger;
private readonly now: () => number;
constructor(private readonly o: LaneControllerOptions) {
this.log = o.logger.child({ component: "worker-lanes" });
this.now = o.now ?? (() => Date.now());
this.limits = { ...DEFAULT_WORKER_LANES, ...o.overrides };
}
/** 引擎 worker 的槽位数:设置页可调到 WORKER_LANE_MAX;环境变量更大时按环境变量开 */
slots(lane: IoLane): number {
return Math.max(WORKER_LANE_MAX, this.o.overrides?.[lane] ?? 0);
}
source(lane: IoLane): "env" | "settings" {
return this.o.overrides?.[lane] ? "env" : "settings";
}
limit(lane: IoLane): number {
return this.limits[lane];
}
tasks(lane: IoLane): readonly string[] {
return this.o.lanes[lane];
}
attach(lane: IoLane, handle: WorkerHandle | undefined): void {
if (handle) this.handles[lane] = handle;
}
/** 本进程该道正在执行的数 */
active(lane: IoLane): number {
const engine = this.handles[lane]?.active() ?? 0;
return lane === "maintenance" ? engine + this.inlineMaintenance : engine;
}
/** 引擎 worker 的动态上限;维护道扣掉调度器内联占用的 */
maxActive(lane: IoLane): number {
return lane === "maintenance" ? this.limits.maintenance - this.inlineMaintenance : this.limits[lane];
}
/** 重读设置 + 环境变量覆盖;变化时记日志 */
async refresh(): Promise<void> {
let fromSettings: WorkerLaneSettings = DEFAULT_WORKER_LANES;
try {
fromSettings = { ...DEFAULT_WORKER_LANES, ...(await this.o.readSettings()) };
} catch (err) {
this.log.debug({ err }, "读取分道并发设置失败,沿用上次的值");
return;
}
const changed: Record<string, number> = {};
for (const lane of IO_LANES) {
const next = Math.min(this.slots(lane), this.o.overrides?.[lane] ?? fromSettings[lane]);
if (next !== this.limits[lane]) changed[lane] = next;
this.limits[lane] = next;
}
if (Object.keys(changed).length) this.log.info({ changed, limits: { ...this.limits } }, "分道并发已更新");
}
start(): Promise<void> {
if (!this.timer) {
this.timer = setInterval(() => void this.refresh(), this.o.refreshMs ?? 5000);
try {
Deno.unrefTimer(this.timer);
} catch {
// 非 Deno 运行时
}
}
return this.refresh();
}
stop(): void {
if (this.timer) clearInterval(this.timer);
this.timer = null;
}
/** 文件道或网络道(全库)有排队或运行中的 run;统计失败按空闲处理(不让维护道因统计故障永远饿着) */
othersBusy(): Promise<boolean> {
const now = this.now();
if (!this.busyCache || now - this.busyCache.at >= (this.o.busyCacheMs ?? 1000)) {
const names = [...this.o.lanes.file, ...this.o.lanes.network];
const value = (async () => {
if (!this.o.engine.queueCounts || names.length === 0) return false;
try {
const [c] = await this.o.engine.queueCounts([names]);
return !!c && c.queued + c.running > 0;
} catch (err) {
this.log.debug({ err }, "分道忙闲统计失败");
return false;
}
})();
this.busyCache = { at: now, value };
}
return this.busyCache.value;
}
/** 维护道是否可以领取新活:其它两道空闲,或连续让路已超过上限(放行一个并重新计时) */
async maintenanceMayStart(): Promise<boolean> {
if (!(await this.othersBusy())) {
this.yieldingSince = null;
return true;
}
const now = this.now();
this.yieldingSince ??= now;
if (now - this.yieldingSince >= (this.o.maxYieldMs ?? MAINTENANCE_MAX_YIELD_MS)) {
this.log.warn({ yieldedMs: now - this.yieldingSince }, "维护道让路超时,放行一个维护任务");
this.yieldingSince = now;
return true;
}
return false;
}
yielding(): boolean {
return this.yieldingSince !== null;
}
/**
* 调度器内联维护任务进入维护道:占一个维护并发;force(手动立即执行)跳过让路与上限。
* 返回释放函数;不能进入时返回 null(任务保持到期状态,下一轮再试)。
*/
async enterMaintenance(force = false): Promise<(() => void) | null> {
if (!force && this.active("maintenance") >= this.limits.maintenance) return null;
this.inlineMaintenance += 1;
if (!force && !(await this.maintenanceMayStart())) {
this.inlineMaintenance -= 1;
return null;
}
let released = false;
return () => {
if (released) return;
released = true;
this.inlineMaintenance -= 1;
};
}
/** 心跳 / 运行状态页用:各道并发、本进程运行中、全库排队 */
async snapshot(): Promise<LaneStat[]> {
let counts: (QueueCounts | undefined)[] = [];
try {
counts = (await this.o.engine.queueCounts?.(IO_LANES.map((l) => this.o.lanes[l]))) ?? [];
} catch (err) {
this.log.debug({ err }, "分道排队统计失败");
}
return IO_LANES.map((lane, i) => ({
lane,
concurrency: this.limits[lane],
source: this.source(lane),
running: this.active(lane),
queued: counts[i]?.queued ?? null,
...(lane === "maintenance" ? { yielding: this.yielding() } : {}),
}));
}
}
// 同一 ServerContext 共用一个分道控制器(调度器据此给内联维护任务让路);没起 io worker 时为空
const controllers = new WeakMap<object, LaneController>();
export function setLaneController(ctx: object, c: LaneController | null): void {
if (c) controllers.set(ctx, c);
else controllers.delete(ctx);
}
export function laneControllerOf(ctx: object): LaneController | undefined {
return controllers.get(ctx);
}
@@ -75,7 +75,11 @@ describe.skipIf(!hasPg)("W2-A 管理 API(真实 PG)", () => {
describe("设置", () => {
it("GET 缺省值;PATCH 自动化总开关与只读对账;ctx.settings 缓存失效并广播 settings.changed", async () => {
const g = await json(await admin("GET", "/settings"));
expect(g.data.automation).toEqual({ enabled: true, readOnlyReconcile: false });
expect(g.data.automation).toEqual({
enabled: true,
readOnlyReconcile: false,
workerLanes: { file: 2, network: 4, maintenance: 1 },
});
expect((await h.ctx.settings.get("automation")).enabled).toBe(true);
const events: string[] = [];
const off = h.bus.subscribe((e) => events.push(e.type));
@@ -83,7 +87,7 @@ describe.skipIf(!hasPg)("W2-A 管理 API(真实 PG)", () => {
expect(r.status).toBe(200);
off();
expect(events).toContain("settings.changed");
expect(await h.ctx.settings.get("automation")).toEqual({ enabled: false, readOnlyReconcile: true });
expect(await h.ctx.settings.get("automation")).toMatchObject({ enabled: false, readOnlyReconcile: true });
// 其它分组不变
expect((await json(await admin("GET", "/settings"))).data.cache.hardThresholdPercent).toBe(90);
});
+19 -9
View File
@@ -71,19 +71,29 @@ describe("能力调用分级(HTTP 路径)", () => {
});
describe("worker 分组与事件循环探针", () => {
const names = ["evaluateAcquisition", "resolveDecision", "runAgent", "verifyAndImport"];
it("io / agent / split 的任务选择", () => {
expect(workerPlan("io", names, { io: 4 })).toEqual([
{ label: "io", versions: ["evaluateAcquisition", "verifyAndImport"], concurrency: 4 },
const tasks = [
{ name: "evaluateAcquisition", lane: "network" as const },
{ name: "resolveDecision", lane: "agent" as const },
{ name: "runAgent" },
{ name: "verifyAndImport", lane: "file" as const },
{ name: "purgeTrash", lane: "maintenance" as const },
];
it("io / agent / split 的任务选择(io 按 lane 分三条道;没声明 lane 的 runAgent 兜底归 agent)", () => {
expect(workerPlan("io", tasks).map((p) => [p.label, p.versions])).toEqual([
["file", ["verifyAndImport"]],
["network", ["evaluateAcquisition"]],
["maintenance", ["purgeTrash"]],
]);
expect(workerPlan("agent", names, { agent: 2 })).toEqual([
expect(workerPlan("agent", tasks, { agent: 2 })).toEqual([
{ label: "agent", versions: ["resolveDecision", "runAgent"], concurrency: 2 },
]);
expect(workerPlan("split", names, { io: 3, agent: 1 }).map((p) => [p.label, p.concurrency])).toEqual([
["io", 3],
["agent", 1],
expect(workerPlan("split", tasks, { agent: 1 }).map((p) => p.label)).toEqual([
"file",
"network",
"maintenance",
"agent",
]);
expect(workerPlan("all", names, {})).toEqual([{ label: "all" }]);
expect(workerPlan("all", tasks, {})).toEqual([{ label: "all" }]);
});
it("主线程被同步占住超过阈值 → 计数", async () => {
@@ -12,6 +12,7 @@ import {
} from "../../src/modules/schedules/registry.ts";
import { Scheduler } from "../../src/modules/schedules/scheduler.ts";
import type { ServerContext } from "../../src/modules/types.ts";
import { LaneController, setLaneController } from "../../src/worker/lanes.ts";
import { Browser, bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
import { DEV_KEY, json } from "../helpers.ts";
@@ -94,6 +95,58 @@ describe.skipIf(!hasPg)("定时任务注册表(真实 PG)", () => {
await h?.close();
});
it("WK-1:lane=maintenance 的内联任务在文件道 / 网络道有活时让路(保持到期),空闲后执行;立即执行不让路", async () => {
let busy = true;
const logger = ctx.logger;
const lanes = new LaneController({
engine: { queueCounts: async (g) => g.map(() => ({ queued: busy ? 1 : 0, running: 0 })) },
logger,
lanes: { file: ["f"], network: ["n"], maintenance: [] },
readSettings: async () => ({ file: 2, network: 4, maintenance: 1 }),
busyCacheMs: 0,
maxYieldMs: 3_600_000,
});
await lanes.refresh();
setLaneController(ctx, lanes);
try {
registerScheduledJob(ctx, counting("test.lane_maint", { lane: "maintenance" }));
registerScheduledJob(ctx, counting("test.lane_light"));
const started = await sched.tick();
await sched.idle();
expect(started).toContain("test.lane_light");
expect(started).not.toContain("test.lane_maint");
expect(calls.get("test.lane_maint") ?? 0).toBe(0);
expect(lanes.yielding()).toBe(true);
// 立即执行:不让路
await h.db
.updateTable("scheduled_job")
.set({ run_requested_at: new Date() })
.where("id", "=", "test.lane_maint")
.execute();
expect(await sched.tick()).toContain("test.lane_maint");
await sched.idle();
expect(calls.get("test.lane_maint")).toBe(1);
expect(lanes.active("maintenance")).toBe(0);
// 到期未跑的:空闲后执行
await h.db
.updateTable("scheduled_job")
.set({ next_run_at: new Date(Date.now() - 1000) })
.where("id", "=", "test.lane_maint")
.execute();
expect(await sched.tick()).not.toContain("test.lane_maint");
busy = false;
expect(await sched.tick()).toContain("test.lane_maint");
await sched.idle();
expect(calls.get("test.lane_maint")).toBe(2);
} finally {
setLaneController(ctx, null);
reg.unregister("test.lane_maint");
reg.unregister("test.lane_light");
}
});
it("登记默认值;用户改过的开关与调度不被再次登记覆盖;reset 恢复默认", async () => {
registerScheduledJob(ctx, counting("test.keep"));
await sched.tick();
+2
View File
@@ -51,6 +51,7 @@ describe("纯函数", () => {
eventLoopLagMs: 0,
eventLoopLagMaxMs: o.lag ?? 0,
concurrency: {},
lanes: [] as never[],
runningRuns: null,
sseConnections: null,
extra: {},
@@ -79,6 +80,7 @@ describe("纯函数", () => {
eventLoopLagMs: 0,
eventLoopLagMaxMs: 0,
concurrency: {},
lanes: [] as never[],
runningRuns: null,
sseConnections: null,
extra: {},
+319
View File
@@ -0,0 +1,319 @@
import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { type DB, migrateToLatest } from "@umt/db";
import { defineTask, type TaskLane } from "@umt/tasks";
import { Kysely, PostgresDialect } from "kysely";
import pg from "pg";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { z } from "zod";
import { ConfigSchema } from "../src/config.ts";
import { type Bootstrapped, bootstrap, buildTaskRegistry } from "../src/context.ts";
import { EventBus } from "../src/http/sse.ts";
import { SETTINGS_CHANGED_EVENT } from "../src/modules/auth-settings/provided.ts";
import { SERVER_MODULES } from "../src/modules/index.ts";
import type { ServerContext, ServerModule } from "../src/modules/types.ts";
import { type RunningWorker, startWorker } from "../src/worker/index.ts";
import { groupTasksByLane, LaneController, laneOverridesFromEnv } from "../src/worker/lanes.ts";
import { withSchemaLock } from "./auth-settings/harness.ts";
import { memoryLogger } from "./helpers.ts";
// WK-1 IO Worker 分道:归属表完整性(注册表 ↔ runbook)、控制器规则(并发热更新、让路、env 映射)、
// 真实 PG 上三条道互不阻塞与维护道让路。
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
const fakeCtx = { role: "all" } as unknown as ServerContext;
/** docs/runbook/dev.md「任务归属表」:| `name` | lane | 说明 | */
function runbookLaneTable(): Map<string, string> {
const md = readFileSync(fileURLToPath(new URL("../../../docs/runbook/dev.md", import.meta.url)), "utf8");
const section = md.slice(md.indexOf("#### 任务归属表"));
const end = section.indexOf("\n\n", section.indexOf("| 任务 |"));
const rows = section.slice(0, end).split("\n");
const out = new Map<string, string>();
for (const r of rows) {
const m = /^\| `([A-Za-z0-9_.-]+)` \| (\w+) \|/.exec(r);
if (m) out.set(m[1]!, m[2]!);
}
return out;
}
describe("任务归属表", () => {
const reg = buildTaskRegistry(SERVER_MODULES, fakeCtx);
it("注册表里每个任务都显式声明了 lane", () => {
const { missing } = groupTasksByLane(reg.all());
expect(missing, "这些任务没在 defineTask 里声明 lane").toEqual([]);
});
it("runbook 的归属表与注册表逐项一致(新增任务要同时写进 docs/runbook/dev.md)", () => {
const table = runbookLaneTable();
const actual = new Map(reg.all().map((d) => [d.name, d.lane as string]));
expect(Object.fromEntries(table)).toEqual(Object.fromEntries(actual));
});
it("没声明 lane:runAgent/resolveDecision 兜底归 agent,其余归网络道并列入 missing", () => {
const g = groupTasksByLane([{ name: "x" }, { name: "runAgent" }, { name: "f", lane: "file" }]);
expect(g.byLane.network).toEqual(["x"]);
expect(g.byLane.agent).toEqual(["runAgent"]);
expect(g.missing).toEqual(["runAgent", "x"]);
});
});
describe("LaneController 规则", () => {
const { logger } = memoryLogger();
const lanes = { file: ["f"], network: ["n"], maintenance: ["m"] };
it("旧 WORKER_CONCURRENCY 映射到网络道;新变量优先", () => {
expect(laneOverridesFromEnv({ WORKER_CONCURRENCY: 6 })).toEqual({ network: 6 });
expect(
laneOverridesFromEnv({ WORKER_CONCURRENCY: 6, WORKER_NETWORK_CONCURRENCY: 3, WORKER_FILE_CONCURRENCY: 1 }),
).toEqual({ network: 3, file: 1 });
expect(laneOverridesFromEnv({})).toEqual({});
});
it("并发读设置、热更新;环境变量覆盖设置;槽位按上限开", async () => {
let settings = { file: 2, network: 4, maintenance: 1 };
const c = new LaneController({
engine: {},
logger,
lanes,
readSettings: async () => settings,
overrides: { maintenance: 20 },
});
await c.refresh();
expect([c.limit("file"), c.limit("network"), c.limit("maintenance")]).toEqual([2, 4, 20]);
expect([c.slots("file"), c.slots("maintenance")]).toEqual([16, 20]);
expect(c.source("maintenance")).toBe("env");
settings = { file: 5, network: 1, maintenance: 3 };
await c.refresh();
expect([c.limit("file"), c.limit("network"), c.limit("maintenance")]).toEqual([5, 1, 20]);
});
it("维护道:其它道有活让路,空闲放行;连续让路超时放行一个;立即执行不让路;内联占并发", async () => {
let busy = { queued: 1, running: 0 };
let now = 0;
const c = new LaneController({
engine: { queueCounts: async (groups) => groups.map(() => busy) },
logger,
lanes,
readSettings: async () => ({ file: 2, network: 4, maintenance: 1 }),
busyCacheMs: 0,
maxYieldMs: 1000,
now: () => now,
});
await c.refresh();
expect(await c.enterMaintenance()).toBeNull();
expect(c.yielding()).toBe(true);
now = 500;
expect(await c.maintenanceMayStart()).toBe(false);
now = 1000;
expect(await c.maintenanceMayStart()).toBe(true); // 让路满 1s 放行一个
now = 1200;
expect(await c.maintenanceMayStart()).toBe(false); // 重新计时
const forced = await c.enterMaintenance(true);
expect(forced).not.toBeNull();
expect(c.active("maintenance")).toBe(1);
expect(c.maxActive("maintenance")).toBe(0); // 引擎维护 worker 让出被内联占用的并发
forced?.();
busy = { queued: 0, running: 0 };
const r1 = await c.enterMaintenance();
expect(r1).not.toBeNull();
expect(c.yielding()).toBe(false);
expect(await c.enterMaintenance()).toBeNull(); // 并发 1 已被占
r1?.();
const snap = await c.snapshot();
expect(snap.map((s) => [s.lane, s.concurrency, s.running, s.queued])).toEqual([
["file", 2, 0, 0],
["network", 4, 0, 0],
["maintenance", 1, 0, 0],
]);
});
});
// ── 真实 PG ──
const url = process.env.DATABASE_URL ?? "postgres://umt:umt@localhost:55433/umt";
const uniq = `${process.pid.toString(36)}${Math.random().toString(36).slice(2, 7)}`;
const biz = `server_lanes_${uniq}`;
const ow = `${biz}_ow`;
async function pgAvailable() {
const c = new pg.Client({ connectionString: url, connectionTimeoutMillis: 2000 });
try {
await c.connect();
await c.end();
return true;
} catch {
return false;
}
}
const hasPg = process.env.UMT_SKIP_DB_TESTS !== "1" && (await pgAvailable());
async function waitUntil(fn: () => boolean | Promise<boolean>, ms = 15_000) {
const until = Date.now() + ms;
while (Date.now() < until) {
if (await fn()) return;
await sleep(50);
}
throw new Error("等待超时");
}
describe.skipIf(!hasPg)("io worker 分道(真实 PG)", () => {
let db: Kysely<DB>;
let booted: Bootstrapped;
let worker: RunningWorker;
const bus = new EventBus(50);
const started: string[] = [];
const done: string[] = [];
const gates = new Map<string, () => void>();
const task = (name: string, lane: TaskLane, block: boolean) =>
defineTask({
name,
version: 1,
lane,
inputSchema: z.object({ id: z.string() }),
retryPolicy: { maximumAttempts: 1 },
handler: async (t) => {
started.push(t.input.id);
if (block) await new Promise<void>((resolve) => gates.set(t.input.id, resolve));
done.push(t.input.id);
return null;
},
});
const probe: ServerModule = {
name: "probe",
registerRoutes() {},
registerTasks(registry) {
registry.register(
task("laneFile", "file", true),
task("laneNet", "network", false),
task("laneNetBlock", "network", true),
task("laneMaint", "maintenance", false),
);
},
};
const enqueue = (name: string, id: string) =>
booted.ctx.engine.enqueue(name, { id }, { idempotencyKey: `${name}:${id}`, traceId: id });
beforeAll(async () => {
const admin = new pg.Client({ connectionString: url });
await admin.connect();
await admin.query(`create schema ${biz}`);
await admin.end();
db = new Kysely<DB>({
dialect: new PostgresDialect({
pool: new pg.Pool({ connectionString: url, max: 8, options: `-c search_path=${biz},public` }),
}),
});
await withSchemaLock(() => migrateToLatest(db, { migrationTableSchema: biz }));
await db
.insertInto("settings")
.values({
key: "automation",
value: JSON.stringify({
enabled: true,
readOnlyReconcile: false,
workerLanes: { file: 1, network: 2, maintenance: 1 },
}),
})
.execute();
const config = ConfigSchema.parse({ DATABASE_URL: url, SECRET_MASTER_KEY: "test-master-key-0123456789" });
const { logger } = memoryLogger();
booted = await bootstrap({
role: "worker",
config,
logger,
db,
events: bus,
modules: [probe],
engineSchema: ow,
engineMigrations: true,
});
worker = await startWorker({
logger,
heartbeatMs: 60_000,
runtime: {
ctx: booted.ctx,
modules: [probe],
setRelay: booted.outbox.setRelay,
outboxIntervalMs: 100,
group: "io",
tasks: booted.tasks.all(),
lanes: { refreshMs: 100, busyCacheMs: 0 },
},
});
}, 60_000);
afterAll(async () => {
for (const g of gates.values()) g();
await worker?.stop();
await booted?.engine.stop();
await db?.destroy();
await withSchemaLock(async () => {
const admin = new pg.Client({ connectionString: url });
await admin.connect();
await admin.query(`drop schema if exists ${biz} cascade`);
await admin.query(`drop schema if exists ${ow} cascade`);
await admin.end();
});
}, 60_000);
it("文件道占满(并发 1)时网络道照常领取;维护道在文件道有活时让路,空了再跑;改设置热生效", async () => {
const lanes = worker.lanes!;
expect(lanes.limit("file")).toBe(1);
await enqueue("laneFile", "f1");
await enqueue("laneFile", "f2");
await waitUntil(() => started.includes("f1") || started.includes("f2"));
const first = started.includes("f1") ? "f1" : "f2";
const second = first === "f1" ? "f2" : "f1";
// 文件道占满:网络任务仍被领取并完成;第二个文件任务排队
await enqueue("laneNet", "n1");
await waitUntil(() => done.includes("n1"));
expect(started).not.toContain(second);
// 维护道让路:文件道有运行中 + 排队
await enqueue("laneMaint", "m1");
await sleep(1500);
expect(started).not.toContain("m1");
expect(lanes.yielding()).toBe(true);
const snap = await lanes.snapshot();
expect(snap.find((s) => s.lane === "file")).toMatchObject({ concurrency: 1, running: 1, queued: 1 });
expect(snap.find((s) => s.lane === "maintenance")).toMatchObject({ queued: 1, yielding: true });
// 热生效:文件道并发改 2,排队的第二个文件任务被领取
await db
.updateTable("settings")
.set({
value: JSON.stringify({
enabled: true,
readOnlyReconcile: false,
workerLanes: { file: 2, network: 2, maintenance: 1 },
}),
})
.where("key", "=", "automation")
.execute();
await bus.publish(SETTINGS_CHANGED_EVENT, { keys: ["automation"] });
await waitUntil(() => started.includes(second));
expect(lanes.limit("file")).toBe(2);
expect(started).not.toContain("m1");
// 文件道清空 → 维护道领取
gates.get(first)?.();
gates.get(second)?.();
await waitUntil(() => done.includes("m1"));
expect(lanes.yielding()).toBe(false);
}, 60_000);
it("网络道占满时文件道照常领取", async () => {
await enqueue("laneNetBlock", "nb1");
await enqueue("laneNetBlock", "nb2");
await waitUntil(() => started.includes("nb1") && started.includes("nb2"));
await enqueue("laneFile", "f3");
await waitUntil(() => started.includes("f3"));
for (const id of ["nb1", "nb2", "f3"]) gates.get(id)?.();
await waitUntil(() => ["nb1", "nb2", "f3"].every((id) => done.includes(id)));
}, 60_000);
});