feat(server): io/agent worker groups, invoke limits for external IO capabilities, loop lag probe
- WORKER_CONCURRENCY / WORKER_AGENT_CONCURRENCY; --role worker (io), worker:agent, all (both) - HTTP invoke of release.search/inspect, identity.resolve, catalog.lookup(remote): 15s timeout + per-upstream semaphore (site 2, TMDB 4) - worker event-loop lag probe: warn + count when drift > 5s Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
fe9693d4ca
commit
473c5e94a5
7 files changed
+399
-19
No files matched your search
@@ -15,6 +15,12 @@ export const ConfigSchema = z.object({
|
|||||||
DEV_ADMIN_KEY: z.string().min(8).optional(),
|
DEV_ADMIN_KEY: z.string().min(8).optional(),
|
||||||
/** worker 心跳间隔 */
|
/** worker 心跳间隔 */
|
||||||
WORKER_HEARTBEAT_MS: z.coerce.number().int().min(1000).default(30_000),
|
WORKER_HEARTBEAT_MS: z.coerce.number().int().min(1000).default(30_000),
|
||||||
|
/** io 组 worker(除 runAgent/resolveDecision 外的全部任务)并发槽数 */
|
||||||
|
WORKER_CONCURRENCY: z.coerce.number().int().min(1).max(64).default(4),
|
||||||
|
/** agent 组 worker(runAgent/resolveDecision:整段会话占一槽数分钟)并发槽数 */
|
||||||
|
WORKER_AGENT_CONCURRENCY: z.coerce.number().int().min(1).max(16).default(2),
|
||||||
|
/** 手动整理 dryRun / 按路径整理时递归遍历的文件数上限(超出截断并标记 truncated) */
|
||||||
|
ORGANIZE_WALK_MAX_FILES: z.coerce.number().int().min(1).max(1_000_000).default(5000),
|
||||||
/** web 静态产物目录(单容器模式):设置后同端口托管 SPA,非 /api 路径回退 index.html */
|
/** web 静态产物目录(单容器模式):设置后同端口托管 SPA,非 /api 路径回退 index.html */
|
||||||
WEB_ROOT: z.string().min(1).optional(),
|
WEB_ROOT: z.string().min(1).optional(),
|
||||||
/**
|
/**
|
||||||
|
|||||||
+23
-4
@@ -12,11 +12,22 @@ import { diffCounts, enableReadOnlyReconcile, exportMappings, type MappingExport
|
|||||||
import { auditStorageWarnings, checkStorage, loadMountedPaths, runPreflight } from "./preflight.ts";
|
import { auditStorageWarnings, checkStorage, loadMountedPaths, runPreflight } from "./preflight.ts";
|
||||||
import { startWorker } from "./worker/index.ts";
|
import { startWorker } from "./worker/index.ts";
|
||||||
|
|
||||||
// api/worker/all 是常驻角色;其余是容器入口 / 备份恢复脚本用的一次性子命令:
|
// api / worker / worker:agent / all 是常驻角色(worker = io 组:除 runAgent/resolveDecision 外的任务 + outbox 中继 +
|
||||||
|
// 模块后台循环;worker:agent = 只跑 Agent 会话任务;all = api + 两组 worker 同进程);其余是容器入口 / 备份恢复脚本用的一次性子命令:
|
||||||
// migrate | preflight | probe
|
// migrate | preflight | probe
|
||||||
// export-mappings --out <file.json> [--compare <基准清单.json>] 映射清单导出(可与基准比行数)
|
// export-mappings --out <file.json> [--compare <基准清单.json>] 映射清单导出(可与基准比行数)
|
||||||
// read-only-on [--reason <文本>] 打开 automation.readOnlyReconcile
|
// read-only-on [--reason <文本>] 打开 automation.readOnlyReconcile
|
||||||
const Roles = ["api", "worker", "all", "migrate", "preflight", "probe", "export-mappings", "read-only-on"] as const;
|
const Roles = [
|
||||||
|
"api",
|
||||||
|
"worker",
|
||||||
|
"worker:agent",
|
||||||
|
"all",
|
||||||
|
"migrate",
|
||||||
|
"preflight",
|
||||||
|
"probe",
|
||||||
|
"export-mappings",
|
||||||
|
"read-only-on",
|
||||||
|
] as const;
|
||||||
type Role = (typeof Roles)[number];
|
type Role = (typeof Roles)[number];
|
||||||
const OneShot = ["migrate", "preflight", "export-mappings", "read-only-on"] as const;
|
const OneShot = ["migrate", "preflight", "export-mappings", "read-only-on"] as const;
|
||||||
type OneShotRole = (typeof OneShot)[number];
|
type OneShotRole = (typeof OneShot)[number];
|
||||||
@@ -195,12 +206,20 @@ if (role === "api" || role === "all") {
|
|||||||
await server.finished;
|
await server.finished;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (role === "worker" || role === "all") {
|
if (role === "worker" || role === "worker:agent" || role === "all") {
|
||||||
const worker = await startWorker({
|
const worker = await startWorker({
|
||||||
logger,
|
logger,
|
||||||
heartbeatMs: config.WORKER_HEARTBEAT_MS,
|
heartbeatMs: config.WORKER_HEARTBEAT_MS,
|
||||||
bus,
|
bus,
|
||||||
runtime: { ctx, modules: SERVER_MODULES, setRelay: outbox.setRelay },
|
runtime: {
|
||||||
|
ctx,
|
||||||
|
modules: SERVER_MODULES,
|
||||||
|
setRelay: outbox.setRelay,
|
||||||
|
group: role === "all" ? "split" : role === "worker" ? "io" : "agent",
|
||||||
|
taskNames: booted.tasks.names(),
|
||||||
|
concurrency: config.WORKER_CONCURRENCY,
|
||||||
|
agentConcurrency: config.WORKER_AGENT_CONCURRENCY,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
stops.push(worker.stop);
|
stops.push(worker.stop);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
// HTTP 路径上的能力调用分级(W4-D P1):外部 IO 型能力(远程站点 / TMDB)在请求内执行时
|
||||||
|
// 强制 15s 超时 + 按上游限并发,避免慢站点把 API 进程的连接与事件循环拖住。
|
||||||
|
// Agent 在 worker:agent 进程里调用同一能力不经过这里(由运行预算约束)。
|
||||||
|
|
||||||
|
/** 外部 IO 能力的 HTTP 超时 */
|
||||||
|
export const EXTERNAL_INVOKE_TIMEOUT_MS = 15_000;
|
||||||
|
|
||||||
|
/** 每个上游的并发上限:站点实时搜索/详情刷新、TMDB 查询 */
|
||||||
|
export const UPSTREAM_LIMITS = { site: 2, tmdb: 4 } as const;
|
||||||
|
export type Upstream = keyof typeof UPSTREAM_LIMITS;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 判断一次调用是否会打外部上游;返回上游名,null 表示纯本地读写(不限流、不加超时)。
|
||||||
|
* 只看输入里确实会触发远程的开关,纯快照查询不受影响。
|
||||||
|
*/
|
||||||
|
export function externalUpstream(capabilityId: string, input: unknown): Upstream | null {
|
||||||
|
const i = (input ?? {}) as Record<string, unknown>;
|
||||||
|
switch (capabilityId) {
|
||||||
|
case "release.search":
|
||||||
|
return i.allowLiveSearch === true && typeof i.query === "string" && i.query.length > 0 ? "site" : null;
|
||||||
|
case "release.inspect":
|
||||||
|
return i.refreshDetail === true ? "site" : null;
|
||||||
|
case "identity.resolve":
|
||||||
|
return "tmdb";
|
||||||
|
case "catalog.lookup":
|
||||||
|
return i.includeRemote === true ? "tmdb" : null;
|
||||||
|
default:
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class UpstreamBusyError extends Error {
|
||||||
|
constructor(readonly upstream: Upstream) {
|
||||||
|
super(`${upstream === "tmdb" ? "TMDB" : "站点"}并发已满,请稍后重试`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class UpstreamTimeoutError extends Error {
|
||||||
|
constructor(readonly upstream: Upstream) {
|
||||||
|
super(`调用${upstream === "tmdb" ? " TMDB" : "站点"}超时(${EXTERNAL_INVOKE_TIMEOUT_MS / 1000}s),已放弃等待`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 计数信号量:acquire 在 signal 触发前拿不到许可则抛 signal.reason */
|
||||||
|
export class Semaphore {
|
||||||
|
private active = 0;
|
||||||
|
private readonly waiters: Array<() => void> = [];
|
||||||
|
|
||||||
|
constructor(readonly limit: number) {}
|
||||||
|
|
||||||
|
get inUse(): number {
|
||||||
|
return this.active;
|
||||||
|
}
|
||||||
|
|
||||||
|
async acquire(signal: AbortSignal): Promise<() => void> {
|
||||||
|
if (this.active >= this.limit) {
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
const wake = () => {
|
||||||
|
signal.removeEventListener("abort", onAbort);
|
||||||
|
resolve();
|
||||||
|
};
|
||||||
|
const onAbort = () => {
|
||||||
|
const i = this.waiters.indexOf(wake);
|
||||||
|
if (i >= 0) this.waiters.splice(i, 1);
|
||||||
|
reject(signal.reason);
|
||||||
|
};
|
||||||
|
if (signal.aborted) return reject(signal.reason);
|
||||||
|
this.waiters.push(wake);
|
||||||
|
signal.addEventListener("abort", onAbort, { once: true });
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
this.active += 1;
|
||||||
|
}
|
||||||
|
let released = false;
|
||||||
|
return () => {
|
||||||
|
if (released) return;
|
||||||
|
released = true;
|
||||||
|
const next = this.waiters.shift();
|
||||||
|
// 许可直接转交给下一个等待者(active 不变)
|
||||||
|
if (next) next();
|
||||||
|
else this.active -= 1;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface InvokeLimiter {
|
||||||
|
/**
|
||||||
|
* 包一层:外部 IO 型调用排队拿许可 + 超时。超时后 HTTP 立即返回,但许可要等底层调用真正结束才归还,
|
||||||
|
* 这样并发上限约束的是"实际在打上游的调用数"。
|
||||||
|
*/
|
||||||
|
run<T>(
|
||||||
|
capabilityId: string,
|
||||||
|
input: unknown,
|
||||||
|
parent: AbortSignal | undefined,
|
||||||
|
fn: (signal: AbortSignal | undefined) => Promise<T>,
|
||||||
|
): Promise<T>;
|
||||||
|
inUse(upstream: Upstream): number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createInvokeLimiter(
|
||||||
|
opts: { timeoutMs?: number; limits?: Partial<Record<Upstream, number>> } = {},
|
||||||
|
): InvokeLimiter {
|
||||||
|
const timeoutMs = opts.timeoutMs ?? EXTERNAL_INVOKE_TIMEOUT_MS;
|
||||||
|
const sems = {
|
||||||
|
site: new Semaphore(opts.limits?.site ?? UPSTREAM_LIMITS.site),
|
||||||
|
tmdb: new Semaphore(opts.limits?.tmdb ?? UPSTREAM_LIMITS.tmdb),
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
inUse: (u) => sems[u].inUse,
|
||||||
|
async run(capabilityId, input, parent, fn) {
|
||||||
|
const upstream = externalUpstream(capabilityId, input);
|
||||||
|
if (!upstream) return fn(parent);
|
||||||
|
const timeout = AbortSignal.timeout(timeoutMs);
|
||||||
|
const signal = parent ? AbortSignal.any([parent, timeout]) : timeout;
|
||||||
|
let release: () => void;
|
||||||
|
try {
|
||||||
|
release = await sems[upstream].acquire(signal);
|
||||||
|
} catch (err) {
|
||||||
|
if (timeout.aborted) throw new UpstreamBusyError(upstream);
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
const work = fn(signal).finally(release);
|
||||||
|
let onAbort: (() => void) | undefined;
|
||||||
|
const aborted = new Promise<never>((_, reject) => {
|
||||||
|
onAbort = () => reject(timeout.aborted ? new UpstreamTimeoutError(upstream) : signal.reason);
|
||||||
|
if (signal.aborted) onAbort();
|
||||||
|
else signal.addEventListener("abort", onAbort, { once: true });
|
||||||
|
});
|
||||||
|
// 超时后底层调用继续跑完(结果丢弃);两边都挂空 catch,避免未处理的 rejection
|
||||||
|
work.catch(() => {});
|
||||||
|
aborted.catch(() => {});
|
||||||
|
try {
|
||||||
|
return await Promise.race([work, aborted]);
|
||||||
|
} finally {
|
||||||
|
if (onAbort) signal.removeEventListener("abort", onAbort);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -14,6 +14,7 @@ import type { ModuleApp, ServerContext } from "../types.ts";
|
|||||||
import { planBinding } from "./authorize.ts";
|
import { planBinding } from "./authorize.ts";
|
||||||
import { cancelTask, retryTask, taskToApi } from "./capabilities.ts";
|
import { cancelTask, retryTask, taskToApi } from "./capabilities.ts";
|
||||||
import { route } from "./http.ts";
|
import { route } from "./http.ts";
|
||||||
|
import { createInvokeLimiter, type InvokeLimiter, UpstreamBusyError, UpstreamTimeoutError } from "./invoke-limits.ts";
|
||||||
import { AgentRunError, abortRun, appendMessage, startRun } from "./service.ts";
|
import { AgentRunError, abortRun, appendMessage, startRun } from "./service.ts";
|
||||||
|
|
||||||
// Agent 会话 API、持续授权 / 单次审批、能力目录与调用、业务任务。
|
// Agent 会话 API、持续授权 / 单次审批、能力目录与调用、业务任务。
|
||||||
@@ -85,7 +86,11 @@ const decode = (c?: string) => {
|
|||||||
return at && id ? { at, id } : null;
|
return at && id ? { at, id } : null;
|
||||||
};
|
};
|
||||||
|
|
||||||
export function registerAgentRoutes(app: ModuleApp, ctx: ServerContext) {
|
export function registerAgentRoutes(
|
||||||
|
app: ModuleApp,
|
||||||
|
ctx: ServerContext,
|
||||||
|
limiter: InvokeLimiter = createInvokeLimiter(),
|
||||||
|
) {
|
||||||
const db = () => ctx.db;
|
const db = () => ctx.db;
|
||||||
|
|
||||||
// ── Agent 会话 ──
|
// ── Agent 会话 ──
|
||||||
@@ -328,13 +333,16 @@ export function registerAgentRoutes(app: ModuleApp, ctx: ServerContext) {
|
|||||||
if (body.preview && !entry.descriptor.supportsPreview)
|
if (body.preview && !entry.descriptor.supportsPreview)
|
||||||
throw new ApiException("bad_request", `${entry.descriptor.id} 不支持预览`);
|
throw new ApiException("bad_request", `${entry.descriptor.id} 不支持预览`);
|
||||||
try {
|
try {
|
||||||
const output = await ctx.capabilities.invoke(params.capabilityId, body.input, {
|
// 外部 IO 型能力(站点实时搜索/详情刷新、TMDB):15s 超时 + 按上游限并发
|
||||||
actorId,
|
const output = await limiter.run(params.capabilityId, body.input, c.req.raw.signal, (signal) =>
|
||||||
taskId: `http:${c.get("requestId") ?? "req"}`,
|
ctx.capabilities.invoke(params.capabilityId, body.input, {
|
||||||
scopeIds: [],
|
actorId,
|
||||||
signal: c.req.raw.signal,
|
taskId: `http:${c.get("requestId") ?? "req"}`,
|
||||||
source: "web",
|
scopeIds: [],
|
||||||
});
|
signal: signal ?? c.req.raw.signal,
|
||||||
|
source: "web",
|
||||||
|
}),
|
||||||
|
);
|
||||||
const o = (output ?? {}) as { taskId?: unknown; planId?: unknown; plan?: { id?: unknown } | null };
|
const o = (output ?? {}) as { taskId?: unknown; planId?: unknown; plan?: { id?: unknown } | null };
|
||||||
return {
|
return {
|
||||||
output,
|
output,
|
||||||
@@ -499,6 +507,8 @@ export function registerAgentRoutes(app: ModuleApp, ctx: ServerContext) {
|
|||||||
|
|
||||||
/** 能力错误 → HTTP 错误码 */
|
/** 能力错误 → HTTP 错误码 */
|
||||||
function capabilityHttpError(err: unknown): never {
|
function capabilityHttpError(err: unknown): never {
|
||||||
|
if (err instanceof UpstreamBusyError) throw new ApiException("rate_limited", err.message);
|
||||||
|
if (err instanceof UpstreamTimeoutError) throw new ApiException("upstream_unavailable", err.message);
|
||||||
if (!isCapabilityError(err)) throw err;
|
if (!isCapabilityError(err)) throw err;
|
||||||
const code =
|
const code =
|
||||||
err.code === "unauthorized"
|
err.code === "unauthorized"
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import type { EventBusPort } from "../http/sse.ts";
|
|||||||
// 每个 W2 子任务是一个 ServerModule,只改 modules/<自己的目录>/ 与 contracts/db 中本域的文件。
|
// 每个 W2 子任务是一个 ServerModule,只改 modules/<自己的目录>/ 与 contracts/db 中本域的文件。
|
||||||
// 跨模块只通过 DB 表 + outbox_event(事件类型在 contracts/events.ts)+ contracts 类型耦合,不互相 import。
|
// 跨模块只通过 DB 表 + outbox_event(事件类型在 contracts/events.ts)+ contracts 类型耦合,不互相 import。
|
||||||
|
|
||||||
export type ServerRole = "api" | "worker" | "all";
|
export type ServerRole = "api" | "worker" | "worker:agent" | "all";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 模块路由挂载点:已挂在 /api/v1 下、已经过 API Key 鉴权(c.var.principal 可用)。
|
* 模块路由挂载点:已挂在 /api/v1 下、已经过 API Key 鉴权(c.var.principal 可用)。
|
||||||
|
|||||||
@@ -5,6 +5,18 @@ import { collectOutboxRoutes, tasksLogSink } from "../context.ts";
|
|||||||
import type { EventBusPort } from "../http/sse.ts";
|
import type { EventBusPort } from "../http/sse.ts";
|
||||||
import type { ServerContext, ServerModule } from "../modules/types.ts";
|
import type { ServerContext, ServerModule } from "../modules/types.ts";
|
||||||
|
|
||||||
|
/** agent 组任务:整段会话占一槽数分钟,与文件复制等 io 任务分池 */
|
||||||
|
export const AGENT_TASKS = ["runAgent", "resolveDecision"] as const;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* worker 分组:
|
||||||
|
* all — 单个引擎 worker 执行全部任务(测试 / 省略分组时的兼容行为)
|
||||||
|
* io — 除 AGENT_TASKS 外的全部任务 + OutboxRelay + 模块后台循环
|
||||||
|
* agent — 只执行 AGENT_TASKS;不跑 relay 与模块循环
|
||||||
|
* split — 同一进程里 io + agent 两个引擎 worker(--role all)
|
||||||
|
*/
|
||||||
|
export type WorkerGroup = "all" | "io" | "agent" | "split";
|
||||||
|
|
||||||
export interface WorkerDeps {
|
export interface WorkerDeps {
|
||||||
logger: Logger;
|
logger: Logger;
|
||||||
heartbeatMs: number;
|
heartbeatMs: number;
|
||||||
@@ -17,32 +29,117 @@ export interface WorkerDeps {
|
|||||||
/** 设置 outbox 中继实例到 ctx.outbox.relay */
|
/** 设置 outbox 中继实例到 ctx.outbox.relay */
|
||||||
setRelay?: (relay: OutboxRelay) => void;
|
setRelay?: (relay: OutboxRelay) => void;
|
||||||
outboxIntervalMs?: number;
|
outboxIntervalMs?: number;
|
||||||
|
/** io 组(或 all 组)并发 */
|
||||||
concurrency?: number;
|
concurrency?: number;
|
||||||
|
/** agent 组并发 */
|
||||||
|
agentConcurrency?: number;
|
||||||
|
/** 缺省 all */
|
||||||
|
group?: WorkerGroup;
|
||||||
|
/** 注册表里的全部任务名(io 组 = 全部 − AGENT_TASKS);group 为 io/split 时必填 */
|
||||||
|
taskNames?: readonly string[];
|
||||||
};
|
};
|
||||||
|
/** 事件循环延迟探针:采样间隔与告警阈值(测试可调小) */
|
||||||
|
lagProbe?: { intervalMs?: number; warnMs?: number };
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WorkerStats {
|
||||||
|
/** 事件循环偏差超过阈值的次数 */
|
||||||
|
loopLagWarnings: number;
|
||||||
|
/** 观测到的最大偏差(ms) */
|
||||||
|
maxLoopLagMs: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RunningWorker {
|
export interface RunningWorker {
|
||||||
stop(): Promise<void>;
|
stop(): Promise<void>;
|
||||||
|
stats(): WorkerStats;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 按分组拆出引擎 worker 的任务选择(versions 为任务名,匹配该任务全部版本) */
|
||||||
|
export function workerPlan(
|
||||||
|
group: WorkerGroup,
|
||||||
|
taskNames: readonly string[],
|
||||||
|
concurrency: { io?: number; agent?: number },
|
||||||
|
): { label: "all" | "io" | "agent"; versions?: string[]; concurrency?: number }[] {
|
||||||
|
const agentNames = taskNames.filter((n) => (AGENT_TASKS as readonly string[]).includes(n));
|
||||||
|
const ioNames = taskNames.filter((n) => !(AGENT_TASKS as readonly string[]).includes(n));
|
||||||
|
const io = { label: "io" as const, versions: ioNames, ...(concurrency.io ? { concurrency: concurrency.io } : {}) };
|
||||||
|
const agent = {
|
||||||
|
label: "agent" as const,
|
||||||
|
versions: agentNames,
|
||||||
|
...(concurrency.agent ? { concurrency: concurrency.agent } : {}),
|
||||||
|
};
|
||||||
|
switch (group) {
|
||||||
|
case "all":
|
||||||
|
return [{ label: "all", ...(concurrency.io ? { concurrency: concurrency.io } : {}) }];
|
||||||
|
case "io":
|
||||||
|
return ioNames.length ? [io] : [];
|
||||||
|
case "agent":
|
||||||
|
return agentNames.length ? [agent] : [];
|
||||||
|
case "split":
|
||||||
|
return [...(ioNames.length ? [io] : []), ...(agentNames.length ? [agent] : [])];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 事件循环延迟探针:setInterval 实际触发时刻与预期的偏差超过 warnMs 记一次 warn + 计数。
|
||||||
|
* 偏差大说明有同步重活占住主线程(租约续期 15s、租约 30s,偏差逼近这个量级就有丢租约风险)。
|
||||||
|
*/
|
||||||
|
export function startLoopLagProbe(log: Logger, opts: { intervalMs?: number; warnMs?: number } = {}) {
|
||||||
|
const intervalMs = opts.intervalMs ?? 1000;
|
||||||
|
const warnMs = opts.warnMs ?? 5000;
|
||||||
|
const stats: WorkerStats = { loopLagWarnings: 0, maxLoopLagMs: 0 };
|
||||||
|
let expected = performance.now() + intervalMs;
|
||||||
|
const timer = setInterval(() => {
|
||||||
|
const now = performance.now();
|
||||||
|
const lag = Math.max(0, now - expected);
|
||||||
|
expected = now + intervalMs;
|
||||||
|
if (lag > stats.maxLoopLagMs) stats.maxLoopLagMs = Math.round(lag);
|
||||||
|
if (lag > warnMs) {
|
||||||
|
stats.loopLagWarnings += 1;
|
||||||
|
log.warn({ lagMs: Math.round(lag), warnings: stats.loopLagWarnings }, "worker 事件循环延迟超过阈值");
|
||||||
|
}
|
||||||
|
}, intervalMs);
|
||||||
|
return {
|
||||||
|
stats: () => ({ ...stats }),
|
||||||
|
stop: () => clearInterval(timer),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* worker 角色:心跳 + 任务引擎 worker(执行全部已注册任务)+ OutboxRelay(按模块的 outboxRoutes 路由)
|
* worker 角色:心跳 + 任务引擎 worker(执行全部已注册任务)+ OutboxRelay(按模块的 outboxRoutes 路由)
|
||||||
* + 各模块的后台循环(start)。stop 逆序停止;引擎连接本身由进程退出时关闭。
|
* + 各模块的后台循环(start)。stop 逆序停止;引擎连接本身由进程退出时关闭。
|
||||||
*/
|
*/
|
||||||
export async function startWorker({ logger, heartbeatMs, bus, runtime }: WorkerDeps): Promise<RunningWorker> {
|
export async function startWorker({ logger, heartbeatMs, bus, runtime, lagProbe }: WorkerDeps): Promise<RunningWorker> {
|
||||||
const log = logger.child({ component: "worker" });
|
const group = runtime?.group ?? "all";
|
||||||
|
const log = logger.child({ component: "worker", group });
|
||||||
|
const probe = startLoopLagProbe(log, lagProbe);
|
||||||
let n = 0;
|
let n = 0;
|
||||||
const beat = () => {
|
const beat = () => {
|
||||||
n += 1;
|
n += 1;
|
||||||
log.debug({ n }, "worker heartbeat");
|
const s = probe.stats();
|
||||||
bus?.publish(EventType.heartbeat, { role: "worker", n });
|
log.debug({ n, ...s }, "worker heartbeat");
|
||||||
|
bus?.publish(EventType.heartbeat, { role: group === "agent" ? "worker:agent" : "worker", n, ...s });
|
||||||
};
|
};
|
||||||
|
|
||||||
const started: ServerModule[] = [];
|
const started: ServerModule[] = [];
|
||||||
let relay: OutboxRelay | null = null;
|
let relay: OutboxRelay | null = null;
|
||||||
|
const plan: ReturnType<typeof workerPlan> = [];
|
||||||
if (runtime) {
|
if (runtime) {
|
||||||
|
const { ctx } = runtime;
|
||||||
|
if (group !== "all" && !runtime.taskNames) throw new Error(`worker 分组 ${group} 需要 taskNames`);
|
||||||
|
plan.push(
|
||||||
|
...workerPlan(group, runtime.taskNames ?? [], { io: runtime.concurrency, agent: runtime.agentConcurrency }),
|
||||||
|
);
|
||||||
|
for (const p of plan) {
|
||||||
|
await ctx.engine.startWorker({
|
||||||
|
...(p.versions ? { versions: p.versions } : {}),
|
||||||
|
...(p.concurrency ? { concurrency: p.concurrency } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// outbox 中继与模块后台循环只在 io 组(agent 组进程只执行会话任务)
|
||||||
|
if (runtime && group !== "agent") {
|
||||||
const { ctx, modules } = runtime;
|
const { ctx, modules } = runtime;
|
||||||
await ctx.engine.startWorker({ ...(runtime.concurrency ? { concurrency: runtime.concurrency } : {}) });
|
|
||||||
relay = new OutboxRelay({
|
relay = new OutboxRelay({
|
||||||
db: ctx.db,
|
db: ctx.db,
|
||||||
engine: ctx.engine,
|
engine: ctx.engine,
|
||||||
@@ -58,12 +155,21 @@ export async function startWorker({ logger, heartbeatMs, bus, runtime }: WorkerD
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.info({ heartbeatMs, modules: started.map((m) => m.name) }, "worker started");
|
log.info(
|
||||||
|
{
|
||||||
|
heartbeatMs,
|
||||||
|
modules: started.map((m) => m.name),
|
||||||
|
pools: plan.map((p) => ({ group: p.label, concurrency: p.concurrency ?? 4, tasks: p.versions?.length ?? "all" })),
|
||||||
|
},
|
||||||
|
"worker started",
|
||||||
|
);
|
||||||
beat();
|
beat();
|
||||||
const timer = setInterval(beat, heartbeatMs);
|
const timer = setInterval(beat, heartbeatMs);
|
||||||
return {
|
return {
|
||||||
|
stats: () => probe.stats(),
|
||||||
async stop() {
|
async stop() {
|
||||||
clearInterval(timer);
|
clearInterval(timer);
|
||||||
|
probe.stop();
|
||||||
for (const m of started.reverse()) {
|
for (const m of started.reverse()) {
|
||||||
try {
|
try {
|
||||||
await m.stop?.();
|
await m.stop?.();
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { createLogger } from "../src/logger.ts";
|
||||||
|
import {
|
||||||
|
createInvokeLimiter,
|
||||||
|
externalUpstream,
|
||||||
|
Semaphore,
|
||||||
|
UpstreamBusyError,
|
||||||
|
UpstreamTimeoutError,
|
||||||
|
} from "../src/modules/agent/invoke-limits.ts";
|
||||||
|
import { startLoopLagProbe, workerPlan } from "../src/worker/index.ts";
|
||||||
|
|
||||||
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||||
|
|
||||||
|
describe("能力调用分级(HTTP 路径)", () => {
|
||||||
|
it("只把会打外部上游的调用归类", () => {
|
||||||
|
expect(externalUpstream("release.search", { query: "x", allowLiveSearch: true })).toBe("site");
|
||||||
|
expect(externalUpstream("release.search", { query: "x", allowLiveSearch: false })).toBeNull();
|
||||||
|
expect(externalUpstream("release.inspect", { refreshDetail: true })).toBe("site");
|
||||||
|
expect(externalUpstream("release.inspect", {})).toBeNull();
|
||||||
|
expect(externalUpstream("identity.resolve", {})).toBe("tmdb");
|
||||||
|
expect(externalUpstream("catalog.lookup", { includeRemote: true })).toBe("tmdb");
|
||||||
|
expect(externalUpstream("catalog.lookup", { includeRemote: false })).toBeNull();
|
||||||
|
expect(externalUpstream("inventory.query", {})).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("超时:HTTP 立即放弃,许可等底层结束才归还", async () => {
|
||||||
|
const lim = createInvokeLimiter({ timeoutMs: 50, limits: { tmdb: 1 } });
|
||||||
|
let finished = false;
|
||||||
|
const slow = lim.run("identity.resolve", {}, undefined, async () => {
|
||||||
|
await sleep(200);
|
||||||
|
finished = true;
|
||||||
|
return 1;
|
||||||
|
});
|
||||||
|
await expect(slow).rejects.toBeInstanceOf(UpstreamTimeoutError);
|
||||||
|
expect(finished).toBe(false);
|
||||||
|
expect(lim.inUse("tmdb")).toBe(1);
|
||||||
|
// 许可仍被占:排队到超时 → busy
|
||||||
|
await expect(lim.run("identity.resolve", {}, undefined, async () => 2)).rejects.toBeInstanceOf(UpstreamBusyError);
|
||||||
|
await sleep(250);
|
||||||
|
expect(finished).toBe(true);
|
||||||
|
expect(lim.inUse("tmdb")).toBe(0);
|
||||||
|
await expect(lim.run("identity.resolve", {}, undefined, async () => 3)).resolves.toBe(3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("并发上限:同时在跑的外部调用不超过 limit;本地调用不受限", async () => {
|
||||||
|
const lim = createInvokeLimiter({ timeoutMs: 5000, limits: { site: 2 } });
|
||||||
|
let running = 0;
|
||||||
|
let peak = 0;
|
||||||
|
const job = () =>
|
||||||
|
lim.run("release.search", { query: "q", allowLiveSearch: true }, undefined, async () => {
|
||||||
|
running++;
|
||||||
|
peak = Math.max(peak, running);
|
||||||
|
await sleep(30);
|
||||||
|
running--;
|
||||||
|
});
|
||||||
|
await Promise.all(Array.from({ length: 7 }, job));
|
||||||
|
expect(peak).toBe(2);
|
||||||
|
await expect(lim.run("inventory.query", {}, undefined, async () => "local")).resolves.toBe("local");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("Semaphore:等待者被中止时移出队列", async () => {
|
||||||
|
const s = new Semaphore(1);
|
||||||
|
const r1 = await s.acquire(new AbortController().signal);
|
||||||
|
const ac = new AbortController();
|
||||||
|
const p = s.acquire(ac.signal);
|
||||||
|
ac.abort(new Error("gone"));
|
||||||
|
await expect(p).rejects.toThrow("gone");
|
||||||
|
r1();
|
||||||
|
expect(s.inUse).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("worker 分组与事件循环探针", () => {
|
||||||
|
const names = ["evaluateAcquisition", "resolveDecision", "runAgent", "verifyAndImport"];
|
||||||
|
it("io / agent / split 的任务选择", () => {
|
||||||
|
expect(workerPlan("io", names, { io: 4 })).toEqual([
|
||||||
|
{ label: "io", versions: ["evaluateAcquisition", "verifyAndImport"], concurrency: 4 },
|
||||||
|
]);
|
||||||
|
expect(workerPlan("agent", names, { agent: 2 })).toEqual([
|
||||||
|
{ label: "agent", versions: ["resolveDecision", "runAgent"], concurrency: 2 },
|
||||||
|
]);
|
||||||
|
expect(workerPlan("split", names, { io: 3, agent: 1 }).map((p) => [p.label, p.concurrency])).toEqual([
|
||||||
|
["io", 3],
|
||||||
|
["agent", 1],
|
||||||
|
]);
|
||||||
|
expect(workerPlan("all", names, {})).toEqual([{ label: "all" }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("主线程被同步占住超过阈值 → 计数", async () => {
|
||||||
|
const probe = startLoopLagProbe(createLogger({ level: "silent" }), { intervalMs: 20, warnMs: 100 });
|
||||||
|
const until = performance.now() + 250;
|
||||||
|
while (performance.now() < until) {
|
||||||
|
// 同步忙等,模拟阻塞事件循环
|
||||||
|
}
|
||||||
|
await sleep(60);
|
||||||
|
probe.stop();
|
||||||
|
expect(probe.stats().loopLagWarnings).toBeGreaterThanOrEqual(1);
|
||||||
|
expect(probe.stats().maxLoopLagMs).toBeGreaterThan(100);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in new issue
Block a user