feat(worker): IO Worker 分道(WK-1)
io worker 进程内按任务声明的 lane 分文件 / 网络 / 维护三条道,各自并发(缺省 2/4/1)、 各自排队;维护道在文件道或网络道有排队/运行时让路(连续 10 分钟放行一个),调度器内联的 清理/回填/保留期裁剪也走维护道。并发在 设置 → 自动化 → 高级 热生效,env 覆盖;旧 WORKER_CONCURRENCY 映射到网络道。运行状态页按道显示运行中/并发/排队。归属表写进 docs/runbook/dev.md,ci 与注册表逐项核对。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
68f53390f6
commit
da33cad81d
38 files changed
+1240
-105
No files matched your search
@@ -4,6 +4,9 @@ import { z } from "zod";
|
||||
/** compose 里 `${X:-}` 展开成空串:按未设置处理 */
|
||||
const emptyToUndef = (v: unknown) => (v === "" ? undefined : v);
|
||||
|
||||
/** 可选正整数环境变量(空串 = 未设置) */
|
||||
const optionalInt = (max: number) => z.preprocess(emptyToUndef, z.coerce.number().int().min(1).max(max).optional());
|
||||
|
||||
const LogLevel = z.enum(["fatal", "error", "warn", "info", "debug", "trace", "silent"]);
|
||||
|
||||
export const ConfigSchema = z.object({
|
||||
@@ -29,8 +32,15 @@ export const ConfigSchema = z.object({
|
||||
DEV_ADMIN_KEY: z.string().min(8).optional(),
|
||||
/** worker 心跳间隔 */
|
||||
WORKER_HEARTBEAT_MS: z.coerce.number().int().min(1000).default(30_000),
|
||||
/** io 组 worker(除 runAgent/resolveDecision 外的全部任务)并发槽数 */
|
||||
WORKER_CONCURRENCY: z.coerce.number().int().min(1).max(64).default(4),
|
||||
/**
|
||||
* 旧变量(WK-1 前的 io 组总并发):现在映射到网络道(WORKER_NETWORK_CONCURRENCY 未设时);
|
||||
* 不设 = 用设置页的值。空串视为未设置(compose 里 `${WORKER_CONCURRENCY:-}`)。
|
||||
*/
|
||||
WORKER_CONCURRENCY: optionalInt(64),
|
||||
/** io worker 三条道的并发(WK-1);设置了就覆盖 设置 → 自动化 → 高级 里的值 */
|
||||
WORKER_FILE_CONCURRENCY: optionalInt(64),
|
||||
WORKER_NETWORK_CONCURRENCY: optionalInt(64),
|
||||
WORKER_MAINTENANCE_CONCURRENCY: optionalInt(64),
|
||||
/** agent 组 worker(runAgent/resolveDecision:整段会话占一槽数分钟)并发槽数 */
|
||||
WORKER_AGENT_CONCURRENCY: z.coerce.number().int().min(1).max(16).default(2),
|
||||
/** 手动整理 dryRun / 按路径整理时递归遍历的文件数上限(超出截断并标记 truncated) */
|
||||
|
||||
@@ -41,6 +41,10 @@ export class DeferredTaskEngine implements TaskEngine {
|
||||
get: TaskEngine["get"] = (...a) => this.e.get(...a);
|
||||
list: TaskEngine["list"] = (...a) => this.e.list(...a);
|
||||
startWorker: TaskEngine["startWorker"] = (...a) => this.e.startWorker(...a);
|
||||
queueCounts: NonNullable<TaskEngine["queueCounts"]> = async (...a) => {
|
||||
const q = this.e.queueCounts;
|
||||
return q ? q.apply(this.e, a) : a[0].map(() => ({ queued: 0, running: 0 }));
|
||||
};
|
||||
stop: TaskEngine["stop"] = async () => {
|
||||
await this.inner?.stop();
|
||||
};
|
||||
|
||||
@@ -3,7 +3,7 @@ import denoJson from "../deno.json" with { type: "json" };
|
||||
import type { Config } from "./config.ts";
|
||||
import type { MappingExport } from "./ops.ts";
|
||||
|
||||
// api / worker / worker:agent / all 是常驻角色(worker = io 组:除 runAgent/resolveDecision 外的任务 + outbox 中继 +
|
||||
// api / worker / worker:agent / all 是常驻角色(worker = io 组:文件 / 网络 / 维护三条道 + outbox 中继 +
|
||||
// 模块后台循环;worker:agent = 只跑 Agent 会话任务;all = api + 两组 worker 同进程);其余是容器入口 / 备份恢复脚本用的一次性子命令:
|
||||
// migrate | preflight | probe
|
||||
// export-mappings --out <file.json> [--compare <基准清单.json>] 映射清单导出(可与基准比行数)
|
||||
@@ -77,6 +77,7 @@ const { SERVER_MODULES } = await import("./modules/index.ts");
|
||||
const { diffCounts, enableReadOnlyReconcile, exportMappings, writeMappingExport } = await import("./ops.ts");
|
||||
const { auditStorageWarnings, checkStorage, loadMountedPaths, runPreflight } = await import("./preflight.ts");
|
||||
const { startWorker } = await import("./worker/index.ts");
|
||||
const { laneOverridesFromEnv } = await import("./worker/lanes.ts");
|
||||
|
||||
let config: Config;
|
||||
try {
|
||||
@@ -253,9 +254,10 @@ if (role === "worker" || role === "worker:agent" || role === "all") {
|
||||
modules: SERVER_MODULES,
|
||||
setRelay: outbox.setRelay,
|
||||
group: role === "all" ? "split" : role === "worker" ? "io" : "agent",
|
||||
taskNames: booted.tasks.names(),
|
||||
concurrency: config.WORKER_CONCURRENCY,
|
||||
tasks: booted.tasks.all(),
|
||||
agentConcurrency: config.WORKER_AGENT_CONCURRENCY,
|
||||
// 三条道的并发读 设置 → 自动化 → 高级(热生效);环境变量设了就覆盖(旧 WORKER_CONCURRENCY → 网络道)
|
||||
lanes: { overrides: laneOverridesFromEnv(config) },
|
||||
},
|
||||
});
|
||||
stops.push(worker.stop);
|
||||
|
||||
@@ -76,6 +76,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
|
||||
defineTask({
|
||||
name: "evaluateAcquisition",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({
|
||||
subscriptionId: id.optional(),
|
||||
acquisitionSlotId: id.optional(),
|
||||
@@ -91,6 +92,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
|
||||
defineTask({
|
||||
name: "submitDownload",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ acquisitionPlanId: id, taskId: id.optional() }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: async (t) => {
|
||||
@@ -117,6 +119,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
|
||||
defineTask({
|
||||
name: "verifyAndImport",
|
||||
version: 1,
|
||||
lane: "file",
|
||||
inputSchema: z.object({ downloadTaskId: id.optional(), recordId: id.optional(), taskId: id.optional() }),
|
||||
retryPolicy: FILE_RETRY,
|
||||
handler: async (t) => {
|
||||
@@ -174,6 +177,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
|
||||
defineTask({
|
||||
name: "cleanupDownloadCache",
|
||||
version: 1,
|
||||
lane: "file",
|
||||
inputSchema: z.object({ downloaderInstanceId: id.optional(), taskId: id.optional() }),
|
||||
retryPolicy: FILE_RETRY,
|
||||
handler: async (t) => {
|
||||
@@ -186,6 +190,7 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
|
||||
defineTask({
|
||||
name: "reconcileLibrary",
|
||||
version: 1,
|
||||
lane: "maintenance",
|
||||
inputSchema: z.object({ libraryRootId: id.optional(), reconcileRunId: id.optional(), taskId: id.optional() }),
|
||||
retryPolicy: FILE_RETRY,
|
||||
handler: async (t) => {
|
||||
|
||||
@@ -37,6 +37,7 @@ export function registerAgentTasks(registry: TaskRegistry, env: () => AgentEnv)
|
||||
defineTask({
|
||||
name: "runAgent",
|
||||
version: 1,
|
||||
lane: "agent",
|
||||
/** upstreamId:覆盖默认上游(W7-A);缺省用 agent_run.upstream_id,再缺省 settings.pi.defaultUpstreamId */
|
||||
inputSchema: z.object({ agentRunId: id, upstreamId: id.optional() }),
|
||||
retryPolicy: DB_RETRY,
|
||||
@@ -175,6 +176,7 @@ export function registerAgentTasks(registry: TaskRegistry, env: () => AgentEnv)
|
||||
defineTask({
|
||||
name: "resolveDecision",
|
||||
version: 1,
|
||||
lane: "agent",
|
||||
inputSchema: z.object({ decisionId: id, trigger: z.enum(["created", "updated"]).default("updated") }),
|
||||
retryPolicy: DB_RETRY,
|
||||
handler: async (t) => {
|
||||
|
||||
@@ -50,6 +50,7 @@ export function createAuthSettingsModule(opts: AuthSettingsModuleOptions = {}):
|
||||
if (everyMs <= 0) return;
|
||||
registerScheduledJob(ctx, {
|
||||
key: "auth.cleanup",
|
||||
lane: "maintenance",
|
||||
defaults: {
|
||||
name: "登录会话清理",
|
||||
description: "删除 7 天前的登录尝试和 30 天前失效的会话",
|
||||
|
||||
+3
@@ -181,6 +181,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
start(ctx: ServerContext) {
|
||||
registerScheduledJob(ctx, {
|
||||
key: "cache.sweep",
|
||||
lane: "maintenance",
|
||||
defaults: {
|
||||
name: "缓存清理",
|
||||
description: "删除过期的 TMDB 与识别缓存",
|
||||
@@ -192,6 +193,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
});
|
||||
registerScheduledJob(ctx, {
|
||||
key: "cache.images_clean",
|
||||
lane: "maintenance",
|
||||
defaults: {
|
||||
name: "图片缓存清理",
|
||||
description: "删除过期的海报缓存",
|
||||
@@ -206,6 +208,7 @@ export function createCacheModule(opts: CacheModuleOptions = {}): ServerModule &
|
||||
});
|
||||
registerScheduledJob(ctx, {
|
||||
key: "cache.images_lru",
|
||||
lane: "maintenance",
|
||||
defaults: {
|
||||
name: "图片缓存容量控制",
|
||||
description: "海报缓存超过容量上限时,删掉最久没用的",
|
||||
|
||||
@@ -58,6 +58,7 @@ export function createFileManagerModule(overrides: FmOverrides = {}): ServerModu
|
||||
defineTask({
|
||||
name: "purgeTrash",
|
||||
version: 1,
|
||||
lane: "maintenance",
|
||||
inputSchema: z.object({}).passthrough(),
|
||||
retryPolicy: { maximumAttempts: 3, initialInterval: "5s", backoffCoefficient: 2, maximumInterval: "60s" },
|
||||
handler: async (t) =>
|
||||
@@ -70,6 +71,7 @@ export function createFileManagerModule(overrides: FmOverrides = {}): ServerModu
|
||||
defineTask({
|
||||
name: IDENTIFY_REMOTE_TASK,
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ body: FilesIdentifyRequest, actorId: z.string() }),
|
||||
retryPolicy: { maximumAttempts: 1 },
|
||||
handler: async (t) =>
|
||||
|
||||
@@ -60,6 +60,7 @@ export function createPiModule(
|
||||
defineTask({
|
||||
name: "refreshPiTokens",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({}).passthrough(),
|
||||
retryPolicy: { maximumAttempts: 2, initialInterval: "10s", backoffCoefficient: 2, maximumInterval: "60s" },
|
||||
handler: async (t) =>
|
||||
|
||||
@@ -48,6 +48,12 @@ export interface ScheduledJobDef {
|
||||
/** 稳定 key(表主键),如 feeds.poll */
|
||||
key: string;
|
||||
kind?: ScheduleKind;
|
||||
/**
|
||||
* 执行道(WK-1):"maintenance" = 在调度器里内联干重活的维护任务(清理、回填、保留期裁剪),
|
||||
* 与 io worker 的维护道共用并发、文件道 / 网络道有活时让路(立即执行不让路)。
|
||||
* 省略 = 轻量调度(只入队任务或很快结束),任务本身按它的 lane 执行。
|
||||
*/
|
||||
lane?: "maintenance";
|
||||
defaults: ScheduledJobDefaults;
|
||||
run: JobRunFn;
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { SCHEDULE_CHANGED_EVENT } from "@umt/contracts";
|
||||
import { type DB, type Kysely, type ScheduledJobRow, sql } from "@umt/db";
|
||||
import type { Logger } from "pino";
|
||||
import type { EventBusPort } from "../../http/sse.ts";
|
||||
import { laneControllerOf } from "../../worker/lanes.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
import { runAgentCheck } from "./agent-checks.ts";
|
||||
import type { JobRunFn, JobRunResult, ScheduleRegistry } from "./registry.ts";
|
||||
@@ -158,10 +159,20 @@ export class Scheduler {
|
||||
.limit(50)
|
||||
.execute();
|
||||
const started: string[] = [];
|
||||
const lanes = laneControllerOf(this.ctx);
|
||||
for (const job of due) {
|
||||
const fn = this.runFor(job);
|
||||
if (!fn) continue;
|
||||
const p = this.execute(job, fn).finally(() => this.inflight.delete(job.id));
|
||||
// 内联维护任务进维护道:占不到并发或正在让路就留到下一轮(不改 next_run_at);立即执行不让路
|
||||
let release: (() => void) | null = null;
|
||||
if (lanes && this.registry.get(job.id)?.lane === "maintenance") {
|
||||
release = await lanes.enterMaintenance(job.run_requested_at !== null);
|
||||
if (!release) continue;
|
||||
}
|
||||
const p = this.execute(job, fn).finally(() => {
|
||||
release?.();
|
||||
this.inflight.delete(job.id);
|
||||
});
|
||||
this.inflight.set(job.id, p);
|
||||
started.push(job.id);
|
||||
}
|
||||
|
||||
@@ -93,6 +93,7 @@ export function createSourcesIdentityModule(overrides: RuntimeOverrides = {}): S
|
||||
// FIX-5 存量元数据回刷:部署后首次启动即运行,跑完自动停用;可在计划页手动再执行
|
||||
registerScheduledJob(ctx, {
|
||||
key: BACKFILL_LANGUAGES_JOB,
|
||||
lane: "maintenance",
|
||||
defaults: {
|
||||
name: "元数据语言回刷",
|
||||
description: "按语言设置重新拉一遍作品的标题、简介和每集信息;跑完自动停用,可手动再执行",
|
||||
|
||||
@@ -29,6 +29,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
|
||||
defineTask({
|
||||
name: "ingestFeed",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ feedSourceId: id, manual: z.boolean().optional() }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: (t) => t.step.run("ingest", () => ingestFeed(rt(), t.input.feedSourceId, { manual: !!t.input.manual })),
|
||||
@@ -36,6 +37,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
|
||||
defineTask({
|
||||
name: "resolveObservation",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z
|
||||
.object({
|
||||
observationId: id.nullish(),
|
||||
@@ -60,6 +62,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
|
||||
defineTask({
|
||||
name: AUTO_ENRICH_TASK,
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ poolEntryId: id }),
|
||||
retryPolicy: { maximumAttempts: 1 },
|
||||
handler: (t) => t.step.run("enrich", () => runAutoEnrich(rt().ctx, t.input.poolEntryId, t.runId, t.abortSignal)),
|
||||
@@ -67,6 +70,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
|
||||
defineTask({
|
||||
name: "refreshMetadata",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ mediaEntityId: id, full: z.boolean().optional() }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: (t) =>
|
||||
@@ -75,6 +79,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
|
||||
defineTask({
|
||||
name: "searchSubscription",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ subscriptionId: id, manual: z.boolean().optional() }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: (t) =>
|
||||
@@ -83,6 +88,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
|
||||
defineTask({
|
||||
name: "snapshotSite",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({
|
||||
siteId: id,
|
||||
kind: z.enum(["periodic", "backfill", "manual"]).default("periodic"),
|
||||
@@ -97,6 +103,7 @@ export function registerSourcesTasks(registry: TaskRegistry, rt: () => SourcesRu
|
||||
defineTask({
|
||||
name: "refreshSiteAccount",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ accountId: id, mode: z.enum(["login", "userData"]).default("login") }),
|
||||
// 网络失败只记 last_error 不抛错;不自动重试(站点登录频繁重试有封号风险)
|
||||
retryPolicy: { maximumAttempts: 1 },
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import { readdir, stat } from "node:fs/promises";
|
||||
import { checkRoot, DEFAULT_ROOT_SENTINEL, type DiskUsage, diskUsage } from "@umt/adapters";
|
||||
import type {
|
||||
HealthLevel,
|
||||
SystemDisk,
|
||||
SystemHealth,
|
||||
SystemHealthHistory,
|
||||
SystemMount,
|
||||
SystemProcess,
|
||||
import {
|
||||
type HealthLevel,
|
||||
type SystemDisk,
|
||||
type SystemHealth,
|
||||
type SystemHealthHistory,
|
||||
type SystemMount,
|
||||
type SystemProcess,
|
||||
SystemWorkerLane,
|
||||
} from "@umt/contracts";
|
||||
import { type DB, type Kysely, type Selectable, sql } from "@umt/db";
|
||||
import { CURRENT_HOST_WINDOW_MS, HEARTBEAT_ROW_RETENTION_MS, HEARTBEAT_STALE_MS } from "./heartbeat.ts";
|
||||
@@ -61,7 +62,13 @@ function toProcess(r: HeartbeatRow, now: Date): SystemProcess {
|
||||
const beat = toDate(r.last_beat_at);
|
||||
const age = Math.max(0, now.getTime() - beat.getTime());
|
||||
const extra = (r.extra ?? {}) as Record<string, unknown>;
|
||||
const conc = (r.concurrency ?? {}) as { io?: number; agent?: number };
|
||||
const conc = (r.concurrency ?? {}) as Record<string, unknown>;
|
||||
const lanes = Array.isArray(extra.lanes)
|
||||
? extra.lanes.flatMap((l) => {
|
||||
const p = SystemWorkerLane.safeParse(l);
|
||||
return p.success ? [p.data] : [];
|
||||
})
|
||||
: [];
|
||||
const sse = extra.sseConnections;
|
||||
return {
|
||||
instanceId: r.instance_id,
|
||||
@@ -76,10 +83,12 @@ function toProcess(r: HeartbeatRow, now: Date): SystemProcess {
|
||||
stoppedAt: r.stopped_at ? toDate(r.stopped_at).toISOString() : null,
|
||||
eventLoopLagMs: r.event_loop_lag_ms,
|
||||
eventLoopLagMaxMs: r.event_loop_lag_max_ms,
|
||||
concurrency: {
|
||||
...(typeof conc.io === "number" ? { io: conc.io } : {}),
|
||||
...(typeof conc.agent === "number" ? { agent: conc.agent } : {}),
|
||||
},
|
||||
concurrency: Object.fromEntries(
|
||||
(["io", "agent", "file", "network", "maintenance"] as const).flatMap((k) =>
|
||||
typeof conc[k] === "number" ? [[k, conc[k]]] : [],
|
||||
),
|
||||
),
|
||||
lanes,
|
||||
runningRuns: r.running_runs,
|
||||
sseConnections: typeof sse === "number" ? sse : null,
|
||||
extra,
|
||||
|
||||
@@ -51,7 +51,8 @@ export function createLagSampler(intervalMs = 1000) {
|
||||
}
|
||||
|
||||
export interface HeartbeatSample {
|
||||
concurrency?: { io?: number; agent?: number };
|
||||
/** io(各道合计)/ agent,以及 io 组的 file / network / maintenance */
|
||||
concurrency?: Partial<Record<"io" | "agent" | "file" | "network" | "maintenance", number>>;
|
||||
runningRuns?: number | null;
|
||||
sseConnections?: number | null;
|
||||
extra?: Record<string, unknown>;
|
||||
|
||||
@@ -185,6 +185,7 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
|
||||
for (const j of RETENTION_JOBS) {
|
||||
registerScheduledJob(ctx, {
|
||||
key: j.jobId,
|
||||
lane: "maintenance",
|
||||
defaults: {
|
||||
name: j.name,
|
||||
description: j.description,
|
||||
@@ -208,6 +209,7 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule
|
||||
if (everyMs <= 0) return;
|
||||
registerScheduledJob(ctx, {
|
||||
key: "system.heartbeat_cleanup",
|
||||
lane: "maintenance",
|
||||
defaults: {
|
||||
name: "进程心跳清理",
|
||||
description: "删除过期的进程状态记录",
|
||||
|
||||
+120
-40
@@ -1,10 +1,19 @@
|
||||
import { EventType } from "@umt/contracts";
|
||||
import { OutboxRelay } from "@umt/tasks";
|
||||
import { OutboxRelay, type WorkerHandle } from "@umt/tasks";
|
||||
import type { Logger } from "pino";
|
||||
import { collectOutboxRoutes, tasksLogSink } from "../context.ts";
|
||||
import { type EventBusPort, sseConnectionCount } from "../http/sse.ts";
|
||||
import { type HeartbeatWriter, startHeartbeatWriter } from "../modules/system/heartbeat.ts";
|
||||
import type { ServerContext, ServerModule } from "../modules/types.ts";
|
||||
import {
|
||||
groupTasksByLane,
|
||||
IO_LANES,
|
||||
type IoLane,
|
||||
LaneController,
|
||||
type LaneOverrides,
|
||||
type LaneTask,
|
||||
setLaneController,
|
||||
} from "./lanes.ts";
|
||||
|
||||
/**
|
||||
* outbox → SSE 桥:这些前缀的 outbox 事件在中继投递成功后镜像到实时总线(payload 原样,带业务 id),
|
||||
@@ -16,18 +25,20 @@ export const SSE_MIRRORED_OUTBOX_TYPES: readonly string[] = Object.values(EventT
|
||||
(t) => SSE_MIRROR_PREFIXES.some((p) => t.startsWith(p)) && !SSE_MIRROR_EXCLUDED.has(t),
|
||||
);
|
||||
|
||||
/** agent 组任务:整段会话占一槽数分钟,与文件复制等 io 任务分池 */
|
||||
export const AGENT_TASKS = ["runAgent", "resolveDecision"] as const;
|
||||
/** agent 组任务:整段会话占一槽数分钟,与文件复制等 io 任务分池(以任务声明的 lane = "agent" 为准,这是兜底名单) */
|
||||
export { AGENT_TASKS } from "./lanes.ts";
|
||||
|
||||
/**
|
||||
* worker 分组:
|
||||
* all — 单个引擎 worker 执行全部任务(测试 / 省略分组时的兼容行为)
|
||||
* io — 除 AGENT_TASKS 外的全部任务 + OutboxRelay + 模块后台循环
|
||||
* agent — 只执行 AGENT_TASKS;不跑 relay 与模块循环
|
||||
* split — 同一进程里 io + agent 两个引擎 worker(--role all)
|
||||
* io — 除 agent 道外的全部任务,按 lane 分文件 / 网络 / 维护三个引擎 worker + OutboxRelay + 模块后台循环
|
||||
* agent — 只执行 agent 道任务;不跑 relay 与模块循环
|
||||
* split — 同一进程里 io 三条道 + agent 共四个引擎 worker(--role all)
|
||||
*/
|
||||
export type WorkerGroup = "all" | "io" | "agent" | "split";
|
||||
|
||||
export type PoolLabel = "all" | IoLane | "agent";
|
||||
|
||||
export interface WorkerDeps {
|
||||
logger: Logger;
|
||||
heartbeatMs: number;
|
||||
@@ -40,14 +51,21 @@ export interface WorkerDeps {
|
||||
/** 设置 outbox 中继实例到 ctx.outbox.relay */
|
||||
setRelay?: (relay: OutboxRelay) => void;
|
||||
outboxIntervalMs?: number;
|
||||
/** io 组(或 all 组)并发 */
|
||||
/** all 组(单池)并发 */
|
||||
concurrency?: number;
|
||||
/** agent 组并发 */
|
||||
agentConcurrency?: number;
|
||||
/** 缺省 all */
|
||||
group?: WorkerGroup;
|
||||
/** 注册表里的全部任务名(io 组 = 全部 − AGENT_TASKS);group 为 io/split 时必填 */
|
||||
taskNames?: readonly string[];
|
||||
/** 注册表里的全部任务(名称 + lane);group 为 io/agent/split 时必填 */
|
||||
tasks?: readonly LaneTask[];
|
||||
/** 分道:环境变量覆盖与测试用的节奏参数 */
|
||||
lanes?: {
|
||||
overrides?: LaneOverrides;
|
||||
refreshMs?: number;
|
||||
busyCacheMs?: number;
|
||||
maxYieldMs?: number;
|
||||
};
|
||||
};
|
||||
/** 事件循环延迟探针:采样间隔与告警阈值(测试可调小) */
|
||||
lagProbe?: { intervalMs?: number; warnMs?: number };
|
||||
@@ -63,31 +81,38 @@ export interface WorkerStats {
|
||||
export interface RunningWorker {
|
||||
stop(): Promise<void>;
|
||||
stats(): WorkerStats;
|
||||
/** io / split 组的分道控制器 */
|
||||
lanes: LaneController | null;
|
||||
}
|
||||
|
||||
/** 按分组拆出引擎 worker 的任务选择(versions 为任务名,匹配该任务全部版本) */
|
||||
export interface PoolPlan {
|
||||
label: PoolLabel;
|
||||
/** 任务名(匹配该任务全部版本);all 池省略 */
|
||||
versions?: string[];
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
/** 按分组拆出引擎 worker 的任务选择;io 组按 lane 拆三条道(空道不起) */
|
||||
export function workerPlan(
|
||||
group: WorkerGroup,
|
||||
taskNames: readonly string[],
|
||||
concurrency: { io?: number; agent?: number },
|
||||
): { label: "all" | "io" | "agent"; versions?: string[]; concurrency?: number }[] {
|
||||
const agentNames = taskNames.filter((n) => (AGENT_TASKS as readonly string[]).includes(n));
|
||||
const ioNames = taskNames.filter((n) => !(AGENT_TASKS as readonly string[]).includes(n));
|
||||
const io = { label: "io" as const, versions: ioNames, ...(concurrency.io ? { concurrency: concurrency.io } : {}) };
|
||||
const agent = {
|
||||
label: "agent" as const,
|
||||
versions: agentNames,
|
||||
...(concurrency.agent ? { concurrency: concurrency.agent } : {}),
|
||||
};
|
||||
tasks: readonly LaneTask[],
|
||||
concurrency: { all?: number; agent?: number } & LaneOverrides = {},
|
||||
): PoolPlan[] {
|
||||
const { byLane } = groupTasksByLane(tasks);
|
||||
const pool = (label: IoLane | "agent"): PoolPlan[] =>
|
||||
byLane[label].length
|
||||
? [{ label, versions: byLane[label], ...(concurrency[label] ? { concurrency: concurrency[label] } : {}) }]
|
||||
: [];
|
||||
const io = IO_LANES.flatMap((l) => pool(l));
|
||||
switch (group) {
|
||||
case "all":
|
||||
return [{ label: "all", ...(concurrency.io ? { concurrency: concurrency.io } : {}) }];
|
||||
return [{ label: "all", ...(concurrency.all ? { concurrency: concurrency.all } : {}) }];
|
||||
case "io":
|
||||
return ioNames.length ? [io] : [];
|
||||
return io;
|
||||
case "agent":
|
||||
return agentNames.length ? [agent] : [];
|
||||
return pool("agent");
|
||||
case "split":
|
||||
return [...(ioNames.length ? [io] : []), ...(agentNames.length ? [agent] : [])];
|
||||
return [...io, ...pool("agent")];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,18 +159,50 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
|
||||
|
||||
const started: ServerModule[] = [];
|
||||
let relay: OutboxRelay | null = null;
|
||||
const plan: ReturnType<typeof workerPlan> = [];
|
||||
let lanes: LaneController | null = null;
|
||||
const plan: PoolPlan[] = [];
|
||||
const handles = new Map<PoolLabel, WorkerHandle>();
|
||||
if (runtime) {
|
||||
const { ctx } = runtime;
|
||||
if (group !== "all" && !runtime.taskNames) throw new Error(`worker 分组 ${group} 需要 taskNames`);
|
||||
plan.push(
|
||||
...workerPlan(group, runtime.taskNames ?? [], { io: runtime.concurrency, agent: runtime.agentConcurrency }),
|
||||
if (group !== "all" && !runtime.tasks) throw new Error(`worker 分组 ${group} 需要 tasks`);
|
||||
const tasks = runtime.tasks ?? [];
|
||||
plan.push(...workerPlan(group, tasks, { all: runtime.concurrency, agent: runtime.agentConcurrency }));
|
||||
const laned = plan.filter((p): p is PoolPlan & { label: IoLane } =>
|
||||
(IO_LANES as readonly string[]).includes(p.label),
|
||||
);
|
||||
for (const p of plan) {
|
||||
await ctx.engine.startWorker({
|
||||
...(p.versions ? { versions: p.versions } : {}),
|
||||
...(p.concurrency ? { concurrency: p.concurrency } : {}),
|
||||
if (laned.length) {
|
||||
const { byLane, missing } = groupTasksByLane(tasks);
|
||||
if (missing.length)
|
||||
log.warn({ tasks: missing }, "这些任务没有声明 lane,按缺省归入网络道(请在 defineTask 里补 lane)");
|
||||
lanes = new LaneController({
|
||||
engine: ctx.engine,
|
||||
logger,
|
||||
lanes: { file: byLane.file, network: byLane.network, maintenance: byLane.maintenance },
|
||||
readSettings: async () => (await ctx.settings.get("automation")).workerLanes,
|
||||
...(runtime.lanes?.overrides ? { overrides: runtime.lanes.overrides } : {}),
|
||||
...(runtime.lanes?.refreshMs ? { refreshMs: runtime.lanes.refreshMs } : {}),
|
||||
...(runtime.lanes?.busyCacheMs !== undefined ? { busyCacheMs: runtime.lanes.busyCacheMs } : {}),
|
||||
...(runtime.lanes?.maxYieldMs ? { maxYieldMs: runtime.lanes.maxYieldMs } : {}),
|
||||
});
|
||||
await lanes.start();
|
||||
setLaneController(ctx, lanes);
|
||||
}
|
||||
for (const p of plan) {
|
||||
const lane = (IO_LANES as readonly string[]).includes(p.label) ? (p.label as IoLane) : null;
|
||||
const handle =
|
||||
lane && lanes
|
||||
? await ctx.engine.startWorker({
|
||||
versions: p.versions ?? [],
|
||||
concurrency: lanes.slots(lane),
|
||||
maxActive: () => lanes?.maxActive(lane) ?? 0,
|
||||
...(lane === "maintenance" ? { claimGate: () => lanes?.maintenanceMayStart() ?? true } : {}),
|
||||
})
|
||||
: await ctx.engine.startWorker({
|
||||
...(p.versions ? { versions: p.versions } : {}),
|
||||
...(p.concurrency ? { concurrency: p.concurrency } : {}),
|
||||
});
|
||||
if (handle) handles.set(p.label, handle);
|
||||
if (lane) lanes?.attach(lane, handle || undefined);
|
||||
}
|
||||
}
|
||||
// outbox 中继与模块后台循环只在 io 组(agent 组进程只执行会话任务)
|
||||
@@ -170,12 +227,14 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
|
||||
}
|
||||
}
|
||||
|
||||
// 进程与健康页:每 15s 把心跳/延迟/并发/运行中 run 数写进 process_heartbeat(写失败不影响 worker)
|
||||
// 进程与健康页:每 15s 把心跳/延迟/并发/运行中 run 数写进 process_heartbeat(写失败不影响 worker);
|
||||
// io 组另写各道的并发、本进程运行中、全库排队(extra.lanes),事件循环延迟照常记录(将来拆进程的判据)
|
||||
let hbWriter: HeartbeatWriter | null = null;
|
||||
if (runtime) {
|
||||
const { ctx } = runtime;
|
||||
const groupTasks = new Set(plan.flatMap((p) => p.versions ?? []));
|
||||
const countAll = plan.some((p) => !p.versions);
|
||||
const agentPool = plan.find((p) => p.label === "agent");
|
||||
hbWriter = startHeartbeatWriter({
|
||||
db: ctx.db,
|
||||
logger,
|
||||
@@ -183,16 +242,27 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
|
||||
sample: async () => {
|
||||
const running = await ctx.engine.list({ status: "running", limit: 500 }).catch(() => null);
|
||||
const s = probe.stats();
|
||||
const laneStats = lanes ? await lanes.snapshot() : null;
|
||||
const concurrency: Record<string, number> = {};
|
||||
if (laneStats) {
|
||||
for (const l of laneStats) concurrency[l.lane] = l.concurrency;
|
||||
concurrency.io = laneStats.reduce((a, l) => a + l.concurrency, 0);
|
||||
} else if (plan.some((p) => p.label === "all")) {
|
||||
concurrency.io = runtime.concurrency ?? 4;
|
||||
}
|
||||
if (agentPool) concurrency.agent = handles.get("agent")?.concurrency ?? runtime.agentConcurrency ?? 4;
|
||||
return {
|
||||
concurrency: {
|
||||
...(plan.some((p) => p.label !== "agent") ? { io: runtime.concurrency ?? 4 } : {}),
|
||||
...(plan.some((p) => p.label === "agent") ? { agent: runtime.agentConcurrency ?? 4 } : {}),
|
||||
},
|
||||
concurrency,
|
||||
runningRuns: running
|
||||
? running.filter((r) => r.status === "running" && (countAll || groupTasks.has(r.taskName))).length
|
||||
: null,
|
||||
sseConnections: group === "split" ? sseConnectionCount() : null,
|
||||
extra: { group, lagWarnings: s.loopLagWarnings, probeMaxLagMs: s.maxLoopLagMs },
|
||||
extra: {
|
||||
group,
|
||||
lagWarnings: s.loopLagWarnings,
|
||||
probeMaxLagMs: s.maxLoopLagMs,
|
||||
...(laneStats ? { lanes: laneStats } : {}),
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
@@ -202,7 +272,14 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
|
||||
{
|
||||
heartbeatMs,
|
||||
modules: started.map((m) => m.name),
|
||||
pools: plan.map((p) => ({ group: p.label, concurrency: p.concurrency ?? 4, tasks: p.versions?.length ?? "all" })),
|
||||
pools: plan.map((p) => ({
|
||||
group: p.label,
|
||||
concurrency:
|
||||
lanes && (IO_LANES as readonly string[]).includes(p.label)
|
||||
? lanes.limit(p.label as IoLane)
|
||||
: (p.concurrency ?? 4),
|
||||
tasks: p.versions?.length ?? "all",
|
||||
})),
|
||||
},
|
||||
"worker started",
|
||||
);
|
||||
@@ -210,9 +287,12 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe
|
||||
const timer = setInterval(beat, heartbeatMs);
|
||||
return {
|
||||
stats: () => probe.stats(),
|
||||
lanes,
|
||||
async stop() {
|
||||
clearInterval(timer);
|
||||
probe.stop();
|
||||
lanes?.stop();
|
||||
if (runtime) setLaneController(runtime.ctx, null);
|
||||
await hbWriter?.stop();
|
||||
for (const m of started.reverse()) {
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
import { DEFAULT_WORKER_LANES, WORKER_LANE_MAX, type WorkerLaneSettings } from "@umt/contracts";
|
||||
import { laneOf, type QueueCounts, type TaskDefinition, type TaskLane, type WorkerHandle } from "@umt/tasks";
|
||||
import type { Logger } from "pino";
|
||||
import type { Config } from "../config.ts";
|
||||
|
||||
// IO Worker 分道(WK-1,docs/runbook/dev.md「IO Worker 分道」):
|
||||
// io worker 进程内按任务声明的 lane 起三个引擎 worker(文件 / 网络 / 维护),各自独立并发;
|
||||
// 维护道"空闲时跑":文件道或网络道(全库)有排队或运行中的 run 时,维护道暂不领取新活(已在跑的不打断),
|
||||
// 连续让路超过 MAINTENANCE_MAX_YIELD_MS 放行一个,防止网络道长期有活时维护永远跑不上。
|
||||
// 调度器里内联执行的维护类定时任务(ScheduledJobDef.lane = "maintenance")与维护道共用并发与让路规则。
|
||||
|
||||
export const IO_LANES = ["file", "network", "maintenance"] as const;
|
||||
export type IoLane = (typeof IO_LANES)[number];
|
||||
|
||||
export const LANE_LABEL: Record<IoLane, string> = { file: "文件", network: "网络", maintenance: "维护" };
|
||||
|
||||
/** 维护道最长连续让路时长,超过即放行一个 */
|
||||
export const MAINTENANCE_MAX_YIELD_MS = 10 * 60_000;
|
||||
|
||||
export type LaneOverrides = Partial<Record<IoLane, number>>;
|
||||
|
||||
/** 环境变量覆盖:旧 WORKER_CONCURRENCY 映射到网络道(WORKER_NETWORK_CONCURRENCY 优先) */
|
||||
export function laneOverridesFromEnv(
|
||||
config: Pick<
|
||||
Config,
|
||||
"WORKER_CONCURRENCY" | "WORKER_FILE_CONCURRENCY" | "WORKER_NETWORK_CONCURRENCY" | "WORKER_MAINTENANCE_CONCURRENCY"
|
||||
>,
|
||||
): LaneOverrides {
|
||||
const network = config.WORKER_NETWORK_CONCURRENCY ?? config.WORKER_CONCURRENCY;
|
||||
return {
|
||||
...(config.WORKER_FILE_CONCURRENCY ? { file: config.WORKER_FILE_CONCURRENCY } : {}),
|
||||
...(network ? { network } : {}),
|
||||
...(config.WORKER_MAINTENANCE_CONCURRENCY ? { maintenance: config.WORKER_MAINTENANCE_CONCURRENCY } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export interface LaneTask {
|
||||
name: string;
|
||||
lane?: TaskLane;
|
||||
}
|
||||
|
||||
/** agent 会话任务:没声明 lane 时也归 agent(兼容 WK-1 前的固定名单) */
|
||||
export const AGENT_TASKS = ["runAgent", "resolveDecision"] as const;
|
||||
|
||||
/**
|
||||
* 按 lane 分组(同名多版本取第一次出现的声明);missing = 没声明 lane 的任务
|
||||
* (AGENT_TASKS 归 agent,其余按缺省归网络道)。
|
||||
*/
|
||||
export function groupTasksByLane(tasks: readonly LaneTask[]): {
|
||||
byLane: Record<TaskLane, string[]>;
|
||||
missing: string[];
|
||||
} {
|
||||
const byLane: Record<TaskLane, string[]> = { file: [], network: [], maintenance: [], agent: [] };
|
||||
const missing: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const t of tasks) {
|
||||
if (seen.has(t.name)) continue;
|
||||
seen.add(t.name);
|
||||
if (t.lane === undefined) missing.push(t.name);
|
||||
const lane =
|
||||
t.lane === undefined && (AGENT_TASKS as readonly string[]).includes(t.name)
|
||||
? "agent"
|
||||
: laneOf(t as Pick<TaskDefinition, "lane">);
|
||||
byLane[lane].push(t.name);
|
||||
}
|
||||
for (const k of Object.keys(byLane) as TaskLane[]) byLane[k].sort();
|
||||
return { byLane, missing: missing.sort() };
|
||||
}
|
||||
|
||||
export interface LaneStat {
|
||||
lane: IoLane;
|
||||
/** 当前生效的并发上限 */
|
||||
concurrency: number;
|
||||
/** 上限来源 */
|
||||
source: "env" | "settings";
|
||||
/** 本进程该道正在执行的 run 数(维护道含调度器内联的维护任务) */
|
||||
running: number;
|
||||
/** 全库该道已到点等待领取的 run 数;统计失败为 null */
|
||||
queued: number | null;
|
||||
/** 维护道:当前是否在给文件道 / 网络道让路 */
|
||||
yielding?: boolean;
|
||||
}
|
||||
|
||||
export interface LaneControllerOptions {
|
||||
engine: { queueCounts?(groups: readonly (readonly string[])[]): Promise<QueueCounts[]> };
|
||||
logger: Logger;
|
||||
/** 各道的任务名 */
|
||||
lanes: Record<IoLane, readonly string[]>;
|
||||
/** 读设置页的并发(设置读取器自带缓存,settings.changed 事件立即失效) */
|
||||
readSettings: () => Promise<WorkerLaneSettings>;
|
||||
overrides?: LaneOverrides;
|
||||
/** 重读设置的间隔,缺省 5s */
|
||||
refreshMs?: number;
|
||||
/** 忙闲判断缓存,缺省 1s */
|
||||
busyCacheMs?: number;
|
||||
maxYieldMs?: number;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
export class LaneController {
|
||||
private readonly limits: Record<IoLane, number>;
|
||||
private readonly handles: Partial<Record<IoLane, WorkerHandle>> = {};
|
||||
private inlineMaintenance = 0;
|
||||
private yieldingSince: number | null = null;
|
||||
private busyCache: { at: number; value: Promise<boolean> } | null = null;
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
private readonly log: Logger;
|
||||
private readonly now: () => number;
|
||||
|
||||
constructor(private readonly o: LaneControllerOptions) {
|
||||
this.log = o.logger.child({ component: "worker-lanes" });
|
||||
this.now = o.now ?? (() => Date.now());
|
||||
this.limits = { ...DEFAULT_WORKER_LANES, ...o.overrides };
|
||||
}
|
||||
|
||||
/** 引擎 worker 的槽位数:设置页可调到 WORKER_LANE_MAX;环境变量更大时按环境变量开 */
|
||||
slots(lane: IoLane): number {
|
||||
return Math.max(WORKER_LANE_MAX, this.o.overrides?.[lane] ?? 0);
|
||||
}
|
||||
|
||||
source(lane: IoLane): "env" | "settings" {
|
||||
return this.o.overrides?.[lane] ? "env" : "settings";
|
||||
}
|
||||
|
||||
limit(lane: IoLane): number {
|
||||
return this.limits[lane];
|
||||
}
|
||||
|
||||
tasks(lane: IoLane): readonly string[] {
|
||||
return this.o.lanes[lane];
|
||||
}
|
||||
|
||||
attach(lane: IoLane, handle: WorkerHandle | undefined): void {
|
||||
if (handle) this.handles[lane] = handle;
|
||||
}
|
||||
|
||||
/** 本进程该道正在执行的数 */
|
||||
active(lane: IoLane): number {
|
||||
const engine = this.handles[lane]?.active() ?? 0;
|
||||
return lane === "maintenance" ? engine + this.inlineMaintenance : engine;
|
||||
}
|
||||
|
||||
/** 引擎 worker 的动态上限;维护道扣掉调度器内联占用的 */
|
||||
maxActive(lane: IoLane): number {
|
||||
return lane === "maintenance" ? this.limits.maintenance - this.inlineMaintenance : this.limits[lane];
|
||||
}
|
||||
|
||||
/** 重读设置 + 环境变量覆盖;变化时记日志 */
|
||||
async refresh(): Promise<void> {
|
||||
let fromSettings: WorkerLaneSettings = DEFAULT_WORKER_LANES;
|
||||
try {
|
||||
fromSettings = { ...DEFAULT_WORKER_LANES, ...(await this.o.readSettings()) };
|
||||
} catch (err) {
|
||||
this.log.debug({ err }, "读取分道并发设置失败,沿用上次的值");
|
||||
return;
|
||||
}
|
||||
const changed: Record<string, number> = {};
|
||||
for (const lane of IO_LANES) {
|
||||
const next = Math.min(this.slots(lane), this.o.overrides?.[lane] ?? fromSettings[lane]);
|
||||
if (next !== this.limits[lane]) changed[lane] = next;
|
||||
this.limits[lane] = next;
|
||||
}
|
||||
if (Object.keys(changed).length) this.log.info({ changed, limits: { ...this.limits } }, "分道并发已更新");
|
||||
}
|
||||
|
||||
start(): Promise<void> {
|
||||
if (!this.timer) {
|
||||
this.timer = setInterval(() => void this.refresh(), this.o.refreshMs ?? 5000);
|
||||
try {
|
||||
Deno.unrefTimer(this.timer);
|
||||
} catch {
|
||||
// 非 Deno 运行时
|
||||
}
|
||||
}
|
||||
return this.refresh();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (this.timer) clearInterval(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
|
||||
/** 文件道或网络道(全库)有排队或运行中的 run;统计失败按空闲处理(不让维护道因统计故障永远饿着) */
|
||||
othersBusy(): Promise<boolean> {
|
||||
const now = this.now();
|
||||
if (!this.busyCache || now - this.busyCache.at >= (this.o.busyCacheMs ?? 1000)) {
|
||||
const names = [...this.o.lanes.file, ...this.o.lanes.network];
|
||||
const value = (async () => {
|
||||
if (!this.o.engine.queueCounts || names.length === 0) return false;
|
||||
try {
|
||||
const [c] = await this.o.engine.queueCounts([names]);
|
||||
return !!c && c.queued + c.running > 0;
|
||||
} catch (err) {
|
||||
this.log.debug({ err }, "分道忙闲统计失败");
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
this.busyCache = { at: now, value };
|
||||
}
|
||||
return this.busyCache.value;
|
||||
}
|
||||
|
||||
/** 维护道是否可以领取新活:其它两道空闲,或连续让路已超过上限(放行一个并重新计时) */
|
||||
async maintenanceMayStart(): Promise<boolean> {
|
||||
if (!(await this.othersBusy())) {
|
||||
this.yieldingSince = null;
|
||||
return true;
|
||||
}
|
||||
const now = this.now();
|
||||
this.yieldingSince ??= now;
|
||||
if (now - this.yieldingSince >= (this.o.maxYieldMs ?? MAINTENANCE_MAX_YIELD_MS)) {
|
||||
this.log.warn({ yieldedMs: now - this.yieldingSince }, "维护道让路超时,放行一个维护任务");
|
||||
this.yieldingSince = now;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
yielding(): boolean {
|
||||
return this.yieldingSince !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 调度器内联维护任务进入维护道:占一个维护并发;force(手动立即执行)跳过让路与上限。
|
||||
* 返回释放函数;不能进入时返回 null(任务保持到期状态,下一轮再试)。
|
||||
*/
|
||||
async enterMaintenance(force = false): Promise<(() => void) | null> {
|
||||
if (!force && this.active("maintenance") >= this.limits.maintenance) return null;
|
||||
this.inlineMaintenance += 1;
|
||||
if (!force && !(await this.maintenanceMayStart())) {
|
||||
this.inlineMaintenance -= 1;
|
||||
return null;
|
||||
}
|
||||
let released = false;
|
||||
return () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
this.inlineMaintenance -= 1;
|
||||
};
|
||||
}
|
||||
|
||||
/** 心跳 / 运行状态页用:各道并发、本进程运行中、全库排队 */
|
||||
async snapshot(): Promise<LaneStat[]> {
|
||||
let counts: (QueueCounts | undefined)[] = [];
|
||||
try {
|
||||
counts = (await this.o.engine.queueCounts?.(IO_LANES.map((l) => this.o.lanes[l]))) ?? [];
|
||||
} catch (err) {
|
||||
this.log.debug({ err }, "分道排队统计失败");
|
||||
}
|
||||
return IO_LANES.map((lane, i) => ({
|
||||
lane,
|
||||
concurrency: this.limits[lane],
|
||||
source: this.source(lane),
|
||||
running: this.active(lane),
|
||||
queued: counts[i]?.queued ?? null,
|
||||
...(lane === "maintenance" ? { yielding: this.yielding() } : {}),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// 同一 ServerContext 共用一个分道控制器(调度器据此给内联维护任务让路);没起 io worker 时为空
|
||||
const controllers = new WeakMap<object, LaneController>();
|
||||
|
||||
export function setLaneController(ctx: object, c: LaneController | null): void {
|
||||
if (c) controllers.set(ctx, c);
|
||||
else controllers.delete(ctx);
|
||||
}
|
||||
|
||||
export function laneControllerOf(ctx: object): LaneController | undefined {
|
||||
return controllers.get(ctx);
|
||||
}
|
||||
@@ -75,7 +75,11 @@ describe.skipIf(!hasPg)("W2-A 管理 API(真实 PG)", () => {
|
||||
describe("设置", () => {
|
||||
it("GET 缺省值;PATCH 自动化总开关与只读对账;ctx.settings 缓存失效并广播 settings.changed", async () => {
|
||||
const g = await json(await admin("GET", "/settings"));
|
||||
expect(g.data.automation).toEqual({ enabled: true, readOnlyReconcile: false });
|
||||
expect(g.data.automation).toEqual({
|
||||
enabled: true,
|
||||
readOnlyReconcile: false,
|
||||
workerLanes: { file: 2, network: 4, maintenance: 1 },
|
||||
});
|
||||
expect((await h.ctx.settings.get("automation")).enabled).toBe(true);
|
||||
const events: string[] = [];
|
||||
const off = h.bus.subscribe((e) => events.push(e.type));
|
||||
@@ -83,7 +87,7 @@ describe.skipIf(!hasPg)("W2-A 管理 API(真实 PG)", () => {
|
||||
expect(r.status).toBe(200);
|
||||
off();
|
||||
expect(events).toContain("settings.changed");
|
||||
expect(await h.ctx.settings.get("automation")).toEqual({ enabled: false, readOnlyReconcile: true });
|
||||
expect(await h.ctx.settings.get("automation")).toMatchObject({ enabled: false, readOnlyReconcile: true });
|
||||
// 其它分组不变
|
||||
expect((await json(await admin("GET", "/settings"))).data.cache.hardThresholdPercent).toBe(90);
|
||||
});
|
||||
|
||||
@@ -71,19 +71,29 @@ describe("能力调用分级(HTTP 路径)", () => {
|
||||
});
|
||||
|
||||
describe("worker 分组与事件循环探针", () => {
|
||||
const names = ["evaluateAcquisition", "resolveDecision", "runAgent", "verifyAndImport"];
|
||||
it("io / agent / split 的任务选择", () => {
|
||||
expect(workerPlan("io", names, { io: 4 })).toEqual([
|
||||
{ label: "io", versions: ["evaluateAcquisition", "verifyAndImport"], concurrency: 4 },
|
||||
const tasks = [
|
||||
{ name: "evaluateAcquisition", lane: "network" as const },
|
||||
{ name: "resolveDecision", lane: "agent" as const },
|
||||
{ name: "runAgent" },
|
||||
{ name: "verifyAndImport", lane: "file" as const },
|
||||
{ name: "purgeTrash", lane: "maintenance" as const },
|
||||
];
|
||||
it("io / agent / split 的任务选择(io 按 lane 分三条道;没声明 lane 的 runAgent 兜底归 agent)", () => {
|
||||
expect(workerPlan("io", tasks).map((p) => [p.label, p.versions])).toEqual([
|
||||
["file", ["verifyAndImport"]],
|
||||
["network", ["evaluateAcquisition"]],
|
||||
["maintenance", ["purgeTrash"]],
|
||||
]);
|
||||
expect(workerPlan("agent", names, { agent: 2 })).toEqual([
|
||||
expect(workerPlan("agent", tasks, { agent: 2 })).toEqual([
|
||||
{ label: "agent", versions: ["resolveDecision", "runAgent"], concurrency: 2 },
|
||||
]);
|
||||
expect(workerPlan("split", names, { io: 3, agent: 1 }).map((p) => [p.label, p.concurrency])).toEqual([
|
||||
["io", 3],
|
||||
["agent", 1],
|
||||
expect(workerPlan("split", tasks, { agent: 1 }).map((p) => p.label)).toEqual([
|
||||
"file",
|
||||
"network",
|
||||
"maintenance",
|
||||
"agent",
|
||||
]);
|
||||
expect(workerPlan("all", names, {})).toEqual([{ label: "all" }]);
|
||||
expect(workerPlan("all", tasks, {})).toEqual([{ label: "all" }]);
|
||||
});
|
||||
|
||||
it("主线程被同步占住超过阈值 → 计数", async () => {
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
} from "../../src/modules/schedules/registry.ts";
|
||||
import { Scheduler } from "../../src/modules/schedules/scheduler.ts";
|
||||
import type { ServerContext } from "../../src/modules/types.ts";
|
||||
import { LaneController, setLaneController } from "../../src/worker/lanes.ts";
|
||||
import { Browser, bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
|
||||
import { DEV_KEY, json } from "../helpers.ts";
|
||||
|
||||
@@ -94,6 +95,58 @@ describe.skipIf(!hasPg)("定时任务注册表(真实 PG)", () => {
|
||||
await h?.close();
|
||||
});
|
||||
|
||||
it("WK-1:lane=maintenance 的内联任务在文件道 / 网络道有活时让路(保持到期),空闲后执行;立即执行不让路", async () => {
|
||||
let busy = true;
|
||||
const logger = ctx.logger;
|
||||
const lanes = new LaneController({
|
||||
engine: { queueCounts: async (g) => g.map(() => ({ queued: busy ? 1 : 0, running: 0 })) },
|
||||
logger,
|
||||
lanes: { file: ["f"], network: ["n"], maintenance: [] },
|
||||
readSettings: async () => ({ file: 2, network: 4, maintenance: 1 }),
|
||||
busyCacheMs: 0,
|
||||
maxYieldMs: 3_600_000,
|
||||
});
|
||||
await lanes.refresh();
|
||||
setLaneController(ctx, lanes);
|
||||
try {
|
||||
registerScheduledJob(ctx, counting("test.lane_maint", { lane: "maintenance" }));
|
||||
registerScheduledJob(ctx, counting("test.lane_light"));
|
||||
const started = await sched.tick();
|
||||
await sched.idle();
|
||||
expect(started).toContain("test.lane_light");
|
||||
expect(started).not.toContain("test.lane_maint");
|
||||
expect(calls.get("test.lane_maint") ?? 0).toBe(0);
|
||||
expect(lanes.yielding()).toBe(true);
|
||||
|
||||
// 立即执行:不让路
|
||||
await h.db
|
||||
.updateTable("scheduled_job")
|
||||
.set({ run_requested_at: new Date() })
|
||||
.where("id", "=", "test.lane_maint")
|
||||
.execute();
|
||||
expect(await sched.tick()).toContain("test.lane_maint");
|
||||
await sched.idle();
|
||||
expect(calls.get("test.lane_maint")).toBe(1);
|
||||
expect(lanes.active("maintenance")).toBe(0);
|
||||
|
||||
// 到期未跑的:空闲后执行
|
||||
await h.db
|
||||
.updateTable("scheduled_job")
|
||||
.set({ next_run_at: new Date(Date.now() - 1000) })
|
||||
.where("id", "=", "test.lane_maint")
|
||||
.execute();
|
||||
expect(await sched.tick()).not.toContain("test.lane_maint");
|
||||
busy = false;
|
||||
expect(await sched.tick()).toContain("test.lane_maint");
|
||||
await sched.idle();
|
||||
expect(calls.get("test.lane_maint")).toBe(2);
|
||||
} finally {
|
||||
setLaneController(ctx, null);
|
||||
reg.unregister("test.lane_maint");
|
||||
reg.unregister("test.lane_light");
|
||||
}
|
||||
});
|
||||
|
||||
it("登记默认值;用户改过的开关与调度不被再次登记覆盖;reset 恢复默认", async () => {
|
||||
registerScheduledJob(ctx, counting("test.keep"));
|
||||
await sched.tick();
|
||||
|
||||
@@ -51,6 +51,7 @@ describe("纯函数", () => {
|
||||
eventLoopLagMs: 0,
|
||||
eventLoopLagMaxMs: o.lag ?? 0,
|
||||
concurrency: {},
|
||||
lanes: [] as never[],
|
||||
runningRuns: null,
|
||||
sseConnections: null,
|
||||
extra: {},
|
||||
@@ -79,6 +80,7 @@ describe("纯函数", () => {
|
||||
eventLoopLagMs: 0,
|
||||
eventLoopLagMaxMs: 0,
|
||||
concurrency: {},
|
||||
lanes: [] as never[],
|
||||
runningRuns: null,
|
||||
sseConnections: null,
|
||||
extra: {},
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { type DB, migrateToLatest } from "@umt/db";
|
||||
import { defineTask, type TaskLane } from "@umt/tasks";
|
||||
import { Kysely, PostgresDialect } from "kysely";
|
||||
import pg from "pg";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { ConfigSchema } from "../src/config.ts";
|
||||
import { type Bootstrapped, bootstrap, buildTaskRegistry } from "../src/context.ts";
|
||||
import { EventBus } from "../src/http/sse.ts";
|
||||
import { SETTINGS_CHANGED_EVENT } from "../src/modules/auth-settings/provided.ts";
|
||||
import { SERVER_MODULES } from "../src/modules/index.ts";
|
||||
import type { ServerContext, ServerModule } from "../src/modules/types.ts";
|
||||
import { type RunningWorker, startWorker } from "../src/worker/index.ts";
|
||||
import { groupTasksByLane, LaneController, laneOverridesFromEnv } from "../src/worker/lanes.ts";
|
||||
import { withSchemaLock } from "./auth-settings/harness.ts";
|
||||
import { memoryLogger } from "./helpers.ts";
|
||||
|
||||
// WK-1 IO Worker 分道:归属表完整性(注册表 ↔ runbook)、控制器规则(并发热更新、让路、env 映射)、
|
||||
// 真实 PG 上三条道互不阻塞与维护道让路。
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
const fakeCtx = { role: "all" } as unknown as ServerContext;
|
||||
|
||||
/** docs/runbook/dev.md「任务归属表」:| `name` | lane | 说明 | */
|
||||
function runbookLaneTable(): Map<string, string> {
|
||||
const md = readFileSync(fileURLToPath(new URL("../../../docs/runbook/dev.md", import.meta.url)), "utf8");
|
||||
const section = md.slice(md.indexOf("#### 任务归属表"));
|
||||
const end = section.indexOf("\n\n", section.indexOf("| 任务 |"));
|
||||
const rows = section.slice(0, end).split("\n");
|
||||
const out = new Map<string, string>();
|
||||
for (const r of rows) {
|
||||
const m = /^\| `([A-Za-z0-9_.-]+)` \| (\w+) \|/.exec(r);
|
||||
if (m) out.set(m[1]!, m[2]!);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
describe("任务归属表", () => {
|
||||
const reg = buildTaskRegistry(SERVER_MODULES, fakeCtx);
|
||||
|
||||
it("注册表里每个任务都显式声明了 lane", () => {
|
||||
const { missing } = groupTasksByLane(reg.all());
|
||||
expect(missing, "这些任务没在 defineTask 里声明 lane").toEqual([]);
|
||||
});
|
||||
|
||||
it("runbook 的归属表与注册表逐项一致(新增任务要同时写进 docs/runbook/dev.md)", () => {
|
||||
const table = runbookLaneTable();
|
||||
const actual = new Map(reg.all().map((d) => [d.name, d.lane as string]));
|
||||
expect(Object.fromEntries(table)).toEqual(Object.fromEntries(actual));
|
||||
});
|
||||
|
||||
it("没声明 lane:runAgent/resolveDecision 兜底归 agent,其余归网络道并列入 missing", () => {
|
||||
const g = groupTasksByLane([{ name: "x" }, { name: "runAgent" }, { name: "f", lane: "file" }]);
|
||||
expect(g.byLane.network).toEqual(["x"]);
|
||||
expect(g.byLane.agent).toEqual(["runAgent"]);
|
||||
expect(g.missing).toEqual(["runAgent", "x"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("LaneController 规则", () => {
|
||||
const { logger } = memoryLogger();
|
||||
const lanes = { file: ["f"], network: ["n"], maintenance: ["m"] };
|
||||
|
||||
it("旧 WORKER_CONCURRENCY 映射到网络道;新变量优先", () => {
|
||||
expect(laneOverridesFromEnv({ WORKER_CONCURRENCY: 6 })).toEqual({ network: 6 });
|
||||
expect(
|
||||
laneOverridesFromEnv({ WORKER_CONCURRENCY: 6, WORKER_NETWORK_CONCURRENCY: 3, WORKER_FILE_CONCURRENCY: 1 }),
|
||||
).toEqual({ network: 3, file: 1 });
|
||||
expect(laneOverridesFromEnv({})).toEqual({});
|
||||
});
|
||||
|
||||
it("并发读设置、热更新;环境变量覆盖设置;槽位按上限开", async () => {
|
||||
let settings = { file: 2, network: 4, maintenance: 1 };
|
||||
const c = new LaneController({
|
||||
engine: {},
|
||||
logger,
|
||||
lanes,
|
||||
readSettings: async () => settings,
|
||||
overrides: { maintenance: 20 },
|
||||
});
|
||||
await c.refresh();
|
||||
expect([c.limit("file"), c.limit("network"), c.limit("maintenance")]).toEqual([2, 4, 20]);
|
||||
expect([c.slots("file"), c.slots("maintenance")]).toEqual([16, 20]);
|
||||
expect(c.source("maintenance")).toBe("env");
|
||||
settings = { file: 5, network: 1, maintenance: 3 };
|
||||
await c.refresh();
|
||||
expect([c.limit("file"), c.limit("network"), c.limit("maintenance")]).toEqual([5, 1, 20]);
|
||||
});
|
||||
|
||||
it("维护道:其它道有活让路,空闲放行;连续让路超时放行一个;立即执行不让路;内联占并发", async () => {
|
||||
let busy = { queued: 1, running: 0 };
|
||||
let now = 0;
|
||||
const c = new LaneController({
|
||||
engine: { queueCounts: async (groups) => groups.map(() => busy) },
|
||||
logger,
|
||||
lanes,
|
||||
readSettings: async () => ({ file: 2, network: 4, maintenance: 1 }),
|
||||
busyCacheMs: 0,
|
||||
maxYieldMs: 1000,
|
||||
now: () => now,
|
||||
});
|
||||
await c.refresh();
|
||||
expect(await c.enterMaintenance()).toBeNull();
|
||||
expect(c.yielding()).toBe(true);
|
||||
now = 500;
|
||||
expect(await c.maintenanceMayStart()).toBe(false);
|
||||
now = 1000;
|
||||
expect(await c.maintenanceMayStart()).toBe(true); // 让路满 1s 放行一个
|
||||
now = 1200;
|
||||
expect(await c.maintenanceMayStart()).toBe(false); // 重新计时
|
||||
const forced = await c.enterMaintenance(true);
|
||||
expect(forced).not.toBeNull();
|
||||
expect(c.active("maintenance")).toBe(1);
|
||||
expect(c.maxActive("maintenance")).toBe(0); // 引擎维护 worker 让出被内联占用的并发
|
||||
forced?.();
|
||||
busy = { queued: 0, running: 0 };
|
||||
const r1 = await c.enterMaintenance();
|
||||
expect(r1).not.toBeNull();
|
||||
expect(c.yielding()).toBe(false);
|
||||
expect(await c.enterMaintenance()).toBeNull(); // 并发 1 已被占
|
||||
r1?.();
|
||||
const snap = await c.snapshot();
|
||||
expect(snap.map((s) => [s.lane, s.concurrency, s.running, s.queued])).toEqual([
|
||||
["file", 2, 0, 0],
|
||||
["network", 4, 0, 0],
|
||||
["maintenance", 1, 0, 0],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 真实 PG ──
|
||||
|
||||
const url = process.env.DATABASE_URL ?? "postgres://umt:umt@localhost:55433/umt";
|
||||
const uniq = `${process.pid.toString(36)}${Math.random().toString(36).slice(2, 7)}`;
|
||||
const biz = `server_lanes_${uniq}`;
|
||||
const ow = `${biz}_ow`;
|
||||
|
||||
async function pgAvailable() {
|
||||
const c = new pg.Client({ connectionString: url, connectionTimeoutMillis: 2000 });
|
||||
try {
|
||||
await c.connect();
|
||||
await c.end();
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const hasPg = process.env.UMT_SKIP_DB_TESTS !== "1" && (await pgAvailable());
|
||||
|
||||
async function waitUntil(fn: () => boolean | Promise<boolean>, ms = 15_000) {
|
||||
const until = Date.now() + ms;
|
||||
while (Date.now() < until) {
|
||||
if (await fn()) return;
|
||||
await sleep(50);
|
||||
}
|
||||
throw new Error("等待超时");
|
||||
}
|
||||
|
||||
describe.skipIf(!hasPg)("io worker 分道(真实 PG)", () => {
|
||||
let db: Kysely<DB>;
|
||||
let booted: Bootstrapped;
|
||||
let worker: RunningWorker;
|
||||
const bus = new EventBus(50);
|
||||
const started: string[] = [];
|
||||
const done: string[] = [];
|
||||
const gates = new Map<string, () => void>();
|
||||
|
||||
const task = (name: string, lane: TaskLane, block: boolean) =>
|
||||
defineTask({
|
||||
name,
|
||||
version: 1,
|
||||
lane,
|
||||
inputSchema: z.object({ id: z.string() }),
|
||||
retryPolicy: { maximumAttempts: 1 },
|
||||
handler: async (t) => {
|
||||
started.push(t.input.id);
|
||||
if (block) await new Promise<void>((resolve) => gates.set(t.input.id, resolve));
|
||||
done.push(t.input.id);
|
||||
return null;
|
||||
},
|
||||
});
|
||||
const probe: ServerModule = {
|
||||
name: "probe",
|
||||
registerRoutes() {},
|
||||
registerTasks(registry) {
|
||||
registry.register(
|
||||
task("laneFile", "file", true),
|
||||
task("laneNet", "network", false),
|
||||
task("laneNetBlock", "network", true),
|
||||
task("laneMaint", "maintenance", false),
|
||||
);
|
||||
},
|
||||
};
|
||||
const enqueue = (name: string, id: string) =>
|
||||
booted.ctx.engine.enqueue(name, { id }, { idempotencyKey: `${name}:${id}`, traceId: id });
|
||||
|
||||
beforeAll(async () => {
|
||||
const admin = new pg.Client({ connectionString: url });
|
||||
await admin.connect();
|
||||
await admin.query(`create schema ${biz}`);
|
||||
await admin.end();
|
||||
db = new Kysely<DB>({
|
||||
dialect: new PostgresDialect({
|
||||
pool: new pg.Pool({ connectionString: url, max: 8, options: `-c search_path=${biz},public` }),
|
||||
}),
|
||||
});
|
||||
await withSchemaLock(() => migrateToLatest(db, { migrationTableSchema: biz }));
|
||||
await db
|
||||
.insertInto("settings")
|
||||
.values({
|
||||
key: "automation",
|
||||
value: JSON.stringify({
|
||||
enabled: true,
|
||||
readOnlyReconcile: false,
|
||||
workerLanes: { file: 1, network: 2, maintenance: 1 },
|
||||
}),
|
||||
})
|
||||
.execute();
|
||||
const config = ConfigSchema.parse({ DATABASE_URL: url, SECRET_MASTER_KEY: "test-master-key-0123456789" });
|
||||
const { logger } = memoryLogger();
|
||||
booted = await bootstrap({
|
||||
role: "worker",
|
||||
config,
|
||||
logger,
|
||||
db,
|
||||
events: bus,
|
||||
modules: [probe],
|
||||
engineSchema: ow,
|
||||
engineMigrations: true,
|
||||
});
|
||||
worker = await startWorker({
|
||||
logger,
|
||||
heartbeatMs: 60_000,
|
||||
runtime: {
|
||||
ctx: booted.ctx,
|
||||
modules: [probe],
|
||||
setRelay: booted.outbox.setRelay,
|
||||
outboxIntervalMs: 100,
|
||||
group: "io",
|
||||
tasks: booted.tasks.all(),
|
||||
lanes: { refreshMs: 100, busyCacheMs: 0 },
|
||||
},
|
||||
});
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
for (const g of gates.values()) g();
|
||||
await worker?.stop();
|
||||
await booted?.engine.stop();
|
||||
await db?.destroy();
|
||||
await withSchemaLock(async () => {
|
||||
const admin = new pg.Client({ connectionString: url });
|
||||
await admin.connect();
|
||||
await admin.query(`drop schema if exists ${biz} cascade`);
|
||||
await admin.query(`drop schema if exists ${ow} cascade`);
|
||||
await admin.end();
|
||||
});
|
||||
}, 60_000);
|
||||
|
||||
it("文件道占满(并发 1)时网络道照常领取;维护道在文件道有活时让路,空了再跑;改设置热生效", async () => {
|
||||
const lanes = worker.lanes!;
|
||||
expect(lanes.limit("file")).toBe(1);
|
||||
|
||||
await enqueue("laneFile", "f1");
|
||||
await enqueue("laneFile", "f2");
|
||||
await waitUntil(() => started.includes("f1") || started.includes("f2"));
|
||||
const first = started.includes("f1") ? "f1" : "f2";
|
||||
const second = first === "f1" ? "f2" : "f1";
|
||||
|
||||
// 文件道占满:网络任务仍被领取并完成;第二个文件任务排队
|
||||
await enqueue("laneNet", "n1");
|
||||
await waitUntil(() => done.includes("n1"));
|
||||
expect(started).not.toContain(second);
|
||||
|
||||
// 维护道让路:文件道有运行中 + 排队
|
||||
await enqueue("laneMaint", "m1");
|
||||
await sleep(1500);
|
||||
expect(started).not.toContain("m1");
|
||||
expect(lanes.yielding()).toBe(true);
|
||||
const snap = await lanes.snapshot();
|
||||
expect(snap.find((s) => s.lane === "file")).toMatchObject({ concurrency: 1, running: 1, queued: 1 });
|
||||
expect(snap.find((s) => s.lane === "maintenance")).toMatchObject({ queued: 1, yielding: true });
|
||||
|
||||
// 热生效:文件道并发改 2,排队的第二个文件任务被领取
|
||||
await db
|
||||
.updateTable("settings")
|
||||
.set({
|
||||
value: JSON.stringify({
|
||||
enabled: true,
|
||||
readOnlyReconcile: false,
|
||||
workerLanes: { file: 2, network: 2, maintenance: 1 },
|
||||
}),
|
||||
})
|
||||
.where("key", "=", "automation")
|
||||
.execute();
|
||||
await bus.publish(SETTINGS_CHANGED_EVENT, { keys: ["automation"] });
|
||||
await waitUntil(() => started.includes(second));
|
||||
expect(lanes.limit("file")).toBe(2);
|
||||
expect(started).not.toContain("m1");
|
||||
|
||||
// 文件道清空 → 维护道领取
|
||||
gates.get(first)?.();
|
||||
gates.get(second)?.();
|
||||
await waitUntil(() => done.includes("m1"));
|
||||
expect(lanes.yielding()).toBe(false);
|
||||
}, 60_000);
|
||||
|
||||
it("网络道占满时文件道照常领取", async () => {
|
||||
await enqueue("laneNetBlock", "nb1");
|
||||
await enqueue("laneNetBlock", "nb2");
|
||||
await waitUntil(() => started.includes("nb1") && started.includes("nb2"));
|
||||
await enqueue("laneFile", "f3");
|
||||
await waitUntil(() => started.includes("f3"));
|
||||
for (const id of ["nb1", "nb2", "f3"]) gates.get(id)?.();
|
||||
await waitUntil(() => ["nb1", "nb2", "f3"].every((id) => done.includes(id)));
|
||||
}, 60_000);
|
||||
});
|
||||
Reference in new issue
Block a user