rootandClaude Opus 5.5
da33cad81d
io worker 进程内按任务声明的 lane 分文件 / 网络 / 维护三条道,各自并发(缺省 2/4/1)、 各自排队;维护道在文件道或网络道有排队/运行时让路(连续 10 分钟放行一个),调度器内联的 清理/回填/保留期裁剪也走维护道。并发在 设置 → 自动化 → 高级 热生效,env 覆盖;旧 WORKER_CONCURRENCY 映射到网络道。运行状态页按道显示运行中/并发/排队。归属表写进 docs/runbook/dev.md,ci 与注册表逐项核对。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
282 lines
11 KiB
TypeScript
282 lines
11 KiB
TypeScript
import { parseArgs } from "node:util";
|
||
import denoJson from "../deno.json" with { type: "json" };
|
||
import type { Config } from "./config.ts";
|
||
import type { MappingExport } from "./ops.ts";
|
||
|
||
// api / worker / worker:agent / all 是常驻角色(worker = io 组:文件 / 网络 / 维护三条道 + outbox 中继 +
|
||
// 模块后台循环;worker:agent = 只跑 Agent 会话任务;all = api + 两组 worker 同进程);其余是容器入口 / 备份恢复脚本用的一次性子命令:
|
||
// migrate | preflight | probe
|
||
// export-mappings --out <file.json> [--compare <基准清单.json>] 映射清单导出(可与基准比行数)
|
||
// read-only-on [--reason <文本>] 打开 automation.readOnlyReconcile
|
||
// rotate-master-key 用 env SECRET_MASTER_KEY_NEW 重新加密 secret 表
|
||
// (逐行事务、幂等可重跑;步骤与回滚见 docs/runbook/auth.md)
|
||
// fsworker fsproxy 执行子进程(编译二进制里由 fsproxy 以
|
||
// `<自身> --role fsworker` 拉起,stdin 一条请求)
|
||
const Roles = [
|
||
"api",
|
||
"worker",
|
||
"worker:agent",
|
||
"all",
|
||
"migrate",
|
||
"preflight",
|
||
"probe",
|
||
"export-mappings",
|
||
"read-only-on",
|
||
"rotate-master-key",
|
||
"fsworker",
|
||
] as const;
|
||
type Role = (typeof Roles)[number];
|
||
const OneShot = ["migrate", "preflight", "export-mappings", "read-only-on", "rotate-master-key"] as const;
|
||
type OneShotRole = (typeof OneShot)[number];
|
||
const isOneShot = (r: Role): r is OneShotRole => (OneShot as readonly Role[]).includes(r);
|
||
|
||
const { values } = parseArgs({
|
||
args: Deno.args,
|
||
options: {
|
||
role: { type: "string", default: "all" },
|
||
out: { type: "string" },
|
||
compare: { type: "string" },
|
||
reason: { type: "string" },
|
||
},
|
||
strict: true,
|
||
});
|
||
const role = values.role as Role;
|
||
if (!Roles.includes(role)) {
|
||
console.error(`--role 必须是 ${Roles.join("|")}`);
|
||
Deno.exit(2);
|
||
}
|
||
|
||
// fsproxy 执行子进程:必须在加载 db/任务引擎/模块之前分派——每个文件操作起一次,只加载 fs 操作表,启动保持轻量
|
||
if (role === "fsworker") {
|
||
const { fsWorkerMain } = await import("@umt/adapters/fsworker");
|
||
await fsWorkerMain();
|
||
Deno.exit(0);
|
||
}
|
||
|
||
// 健康探测:只打本机回环端口,不需要完整配置(HEALTHCHECK 用,镜像里不装 curl)
|
||
if (role === "probe") {
|
||
const port = Deno.env.get("PORT") ?? "4380";
|
||
try {
|
||
const res = await fetch(`http://127.0.0.1:${port}/api/v1/health`, { signal: AbortSignal.timeout(3000) });
|
||
const body = (await res.json()) as { data?: { status?: string } };
|
||
Deno.exit(res.ok && body.data?.status === "ok" ? 0 : 1);
|
||
} catch {
|
||
Deno.exit(1);
|
||
}
|
||
}
|
||
|
||
// 其余依赖动态加载(顺序同原静态 import),让上面的 fsworker / probe 分支不必求值整张模块图
|
||
const { createDb, migrateToLatest } = await import("@umt/db");
|
||
const { migrateEngineSchema } = await import("@umt/tasks");
|
||
const { DATA_SUBDIRS, dataPaths, loadConfig } = await import("./config.ts");
|
||
const { bootstrap } = await import("./context.ts");
|
||
const { createApp } = await import("./http/app.ts");
|
||
const { PgEventBus } = await import("./http/sse-pg.ts");
|
||
const { createLogger } = await import("./logger.ts");
|
||
const { SERVER_MODULES } = await import("./modules/index.ts");
|
||
const { diffCounts, enableReadOnlyReconcile, exportMappings, writeMappingExport } = await import("./ops.ts");
|
||
const { auditStorageWarnings, checkStorage, loadMountedPaths, runPreflight } = await import("./preflight.ts");
|
||
const { startWorker } = await import("./worker/index.ts");
|
||
const { laneOverridesFromEnv } = await import("./worker/lanes.ts");
|
||
|
||
let config: Config;
|
||
try {
|
||
config = loadConfig();
|
||
} catch (err) {
|
||
console.error((err as Error).message);
|
||
Deno.exit(2);
|
||
}
|
||
|
||
// 运行期写入统一落在 DATA_DIR;Agent 会话目录经环境变量传给 agent 模块(它只读 AGENT_SESSION_DIR)
|
||
const paths = dataPaths(config);
|
||
Deno.env.set("AGENT_SESSION_DIR", paths.sessionDir);
|
||
|
||
const logger = createLogger({
|
||
level: config.LOG_LEVEL,
|
||
pretty: config.NODE_ENV === "development" && Deno.stdout.isTerminal(),
|
||
});
|
||
const db = createDb({ connectionString: config.DATABASE_URL });
|
||
|
||
if (isOneShot(role)) {
|
||
let code = 0;
|
||
try {
|
||
code = await runOneShot(role);
|
||
} catch (err) {
|
||
logger.error({ err }, `${role} 失败`);
|
||
code = 1;
|
||
} finally {
|
||
await db.destroy();
|
||
}
|
||
Deno.exit(code);
|
||
}
|
||
|
||
async function runOneShot(r: OneShotRole): Promise<number> {
|
||
switch (r) {
|
||
case "rotate-master-key": {
|
||
if (!config.SECRET_MASTER_KEY_NEW) {
|
||
logger.error("缺少环境变量 SECRET_MASTER_KEY_NEW（新主密钥）；SECRET_MASTER_KEY 保持旧密钥");
|
||
return 2;
|
||
}
|
||
const { rotateMasterKey } = await import("./modules/auth-settings/provided.ts");
|
||
const res = await rotateMasterKey({ db, logger }, config.SECRET_MASTER_KEY, config.SECRET_MASTER_KEY_NEW);
|
||
return res.failed.length ? 1 : 0;
|
||
}
|
||
case "migrate": {
|
||
const { results } = await migrateToLatest(db);
|
||
for (const m of results ?? []) logger.info({ migration: m.migrationName, status: m.status }, "migration");
|
||
logger.info({ applied: results?.length ?? 0 }, "迁移完成");
|
||
// 任务引擎自己的表(openworkflow schema);api/worker 进程启动时不再迁移
|
||
await migrateEngineSchema(config.DATABASE_URL);
|
||
logger.info("任务引擎迁移完成");
|
||
return 0;
|
||
}
|
||
case "preflight": {
|
||
const problems = await runPreflight({ config, db });
|
||
// 库/下载目录由用户挂载:不可读写只告警(并写审计),不阻止启动;数据目录不可写才阻止
|
||
const mounts = problems.some((p) => p.check === "db") ? [] : await loadMountedPaths(db);
|
||
const storage = await checkStorage({ ...paths, subdirs: DATA_SUBDIRS, mounts });
|
||
problems.push(...storage.problems);
|
||
for (const w of storage.warnings) {
|
||
logger.warn({ kind: w.kind, id: w.id, path: w.path }, `挂载路径不可用：${w.name} ${w.reason}`);
|
||
}
|
||
await auditStorageWarnings(db, storage.warnings).catch((err) => logger.warn({ err }, "存储告警写审计失败"));
|
||
// 验收开关误设提示(验收 07 S-10):只有 UMT_FS_TEST_HOOKS=1 时才真正生效
|
||
if (Deno.env.get("UMT_FS_TEST_HANG_AFTER_BYTES"))
|
||
logger.warn(
|
||
{ active: Deno.env.get("UMT_FS_TEST_HOOKS") === "1" },
|
||
"设置了 UMT_FS_TEST_HANG_AFTER_BYTES（fsproxy 挂死验收开关）；生产环境不要设置，生效时所有数据类文件操作都会卡死到被 SIGKILL",
|
||
);
|
||
for (const p of problems) logger.error({ check: p.check }, p.message);
|
||
if (problems.length) return 1;
|
||
logger.info(
|
||
{
|
||
dataDir: paths.dataDir,
|
||
sessionDir: paths.sessionDir,
|
||
mounts: mounts.length,
|
||
warnings: storage.warnings.length,
|
||
},
|
||
"启动自检通过",
|
||
);
|
||
return 0;
|
||
}
|
||
case "export-mappings": {
|
||
const out = values.out ?? `${paths.exportsDir}/mappings-${new Date().toISOString().replace(/[:.]/g, "-")}.json`;
|
||
const data = await exportMappings(db, denoJson.version);
|
||
await writeMappingExport(out, data);
|
||
logger.info({ out, counts: data.counts }, "映射清单已导出");
|
||
if (!values.compare) return 0;
|
||
const base = JSON.parse(await Deno.readTextFile(values.compare)) as MappingExport;
|
||
const diff = diffCounts(base.counts, data.counts);
|
||
if (diff.length) {
|
||
logger.error({ diff }, "映射清单行数与基准不一致");
|
||
return 3;
|
||
}
|
||
logger.info({ compare: values.compare }, "映射清单行数与基准一致");
|
||
return 0;
|
||
}
|
||
case "read-only-on":
|
||
await enableReadOnlyReconcile(db, values.reason ?? "restore");
|
||
logger.info("已打开只读对账模式 automation.readOnlyReconcile=true");
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
// SSE 总线:sse_event 表 + LISTEN/NOTIFY,api 与 worker 分进程时事件互相可见
|
||
let bus: Awaited<ReturnType<typeof PgEventBus.start>>;
|
||
try {
|
||
bus = await PgEventBus.start({
|
||
db,
|
||
connectionString: config.DATABASE_URL,
|
||
logger: logger.child({ component: "sse-bus" }),
|
||
});
|
||
} catch (err) {
|
||
logger.error({ err }, "SSE 总线启动失败");
|
||
await db.destroy();
|
||
Deno.exit(1);
|
||
}
|
||
const stops: Array<() => Promise<void>> = [];
|
||
|
||
// 装配 ServerContext:模块注册任务/能力,连接任务引擎(api 角色也要:入队、信号、取消)
|
||
let booted: Awaited<ReturnType<typeof bootstrap>>;
|
||
try {
|
||
booted = await bootstrap({ role, config, logger, db, events: bus, modules: SERVER_MODULES });
|
||
} catch (err) {
|
||
logger.error({ err }, "启动装配失败");
|
||
await db.destroy();
|
||
Deno.exit(1);
|
||
}
|
||
const { ctx, outbox, engine } = booted;
|
||
|
||
if (role === "api" || role === "all") {
|
||
const app = createApp({
|
||
config,
|
||
logger,
|
||
bus,
|
||
db,
|
||
role,
|
||
version: denoJson.version,
|
||
...(config.WEB_ROOT ? { webRoot: config.WEB_ROOT } : {}),
|
||
modules: SERVER_MODULES,
|
||
ctx,
|
||
});
|
||
const ac = new AbortController();
|
||
const server = Deno.serve(
|
||
{
|
||
port: config.PORT,
|
||
hostname: config.HOST,
|
||
signal: ac.signal,
|
||
onListen: ({ port }) =>
|
||
logger.info(
|
||
{ port, role, webRoot: config.WEB_ROOT ?? null },
|
||
`API 已启动 http://localhost:${port}/api/v1/health`,
|
||
),
|
||
},
|
||
app.fetch,
|
||
);
|
||
stops.push(async () => {
|
||
// abort 后 Deno.serve 会断开 SSE 等长连接,finished 在所有连接关闭后 resolve
|
||
ac.abort();
|
||
await server.finished;
|
||
});
|
||
if (role === "api") {
|
||
// 纯 api 进程不走 worker 的模块 stop:这里停 system 模块的心跳写入器,把本进程的心跳行标 stopped_at
|
||
const sys = SERVER_MODULES.find((m) => m.name === "system");
|
||
if (sys?.stop) stops.push(async () => await sys.stop?.());
|
||
}
|
||
}
|
||
if (role === "worker" || role === "worker:agent" || role === "all") {
|
||
const worker = await startWorker({
|
||
logger,
|
||
heartbeatMs: config.WORKER_HEARTBEAT_MS,
|
||
bus,
|
||
runtime: {
|
||
ctx,
|
||
modules: SERVER_MODULES,
|
||
setRelay: outbox.setRelay,
|
||
group: role === "all" ? "split" : role === "worker" ? "io" : "agent",
|
||
tasks: booted.tasks.all(),
|
||
agentConcurrency: config.WORKER_AGENT_CONCURRENCY,
|
||
// 三条道的并发读 设置 → 自动化 → 高级(热生效);环境变量设了就覆盖(旧 WORKER_CONCURRENCY → 网络道)
|
||
lanes: { overrides: laneOverridesFromEnv(config) },
|
||
},
|
||
});
|
||
stops.push(worker.stop);
|
||
}
|
||
|
||
let stopping = false;
|
||
async function shutdown(signal: Deno.Signal) {
|
||
if (stopping) return;
|
||
stopping = true;
|
||
logger.info({ signal }, "正在退出");
|
||
// 兜底:10 秒内没退干净就强退,避免容器 stop 超时被 SIGKILL
|
||
const force = setTimeout(() => Deno.exit(1), 10_000);
|
||
await Promise.allSettled(stops.map((s) => s()));
|
||
await engine.stop().catch((err) => logger.error({ err }, "任务引擎关闭失败"));
|
||
await bus.stop().catch(() => {});
|
||
await db.destroy();
|
||
clearTimeout(force);
|
||
Deno.exit(0);
|
||
}
|
||
Deno.addSignalListener("SIGINT", () => void shutdown("SIGINT"));
|
||
Deno.addSignalListener("SIGTERM", () => void shutdown("SIGTERM"));
|