io worker 进程内按任务声明的 lane 分文件 / 网络 / 维护三条道,各自并发(缺省 2/4/1)、 各自排队;维护道在文件道或网络道有排队/运行时让路(连续 10 分钟放行一个),调度器内联的 清理/回填/保留期裁剪也走维护道。并发在 设置 → 自动化 → 高级 热生效,env 覆盖;旧 WORKER_CONCURRENCY 映射到网络道。运行状态页按道显示运行中/并发/排队。归属表写进 docs/runbook/dev.md,ci 与注册表逐项核对。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
417 lines
17 KiB
TypeScript
417 lines
17 KiB
TypeScript
import { randomUUID } from "node:crypto";
|
|
import { sql } from "@umt/db";
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import { ensureSetupToken } from "../../src/modules/auth-settings/auth-service.ts";
|
|
import { nextCronAfter } from "../../src/modules/schedules/cron.ts";
|
|
import { createSchedulesModule } from "../../src/modules/schedules/index.ts";
|
|
import {
|
|
registerScheduledJob,
|
|
type ScheduledJobDef,
|
|
type ScheduleRegistry,
|
|
scheduleRegistry,
|
|
} from "../../src/modules/schedules/registry.ts";
|
|
import { Scheduler } from "../../src/modules/schedules/scheduler.ts";
|
|
import type { ServerContext } from "../../src/modules/types.ts";
|
|
import { LaneController, setLaneController } from "../../src/worker/lanes.ts";
|
|
import { Browser, bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
|
|
import { DEV_KEY, json } from "../helpers.ts";
|
|
|
|
// W7-C 定时任务注册表:登记不覆盖用户改动、调度单实例锁、立即执行幂等、失败记录、跳过本次、
|
|
// agent_check 入队 runAgent 与结果回填、端点鉴权(只接受控制台会话)。
|
|
|
|
const hasPg = await pgAvailable();
|
|
const PASSWORD = "correct horse battery";
|
|
|
|
describe("cron", () => {
|
|
it("按 5 段表达式取下一次时间", () => {
|
|
const base = new Date(2026, 8, 28, 10, 7, 30); // 周一
|
|
expect(nextCronAfter("*/15 * * * *", base)).toEqual(new Date(2026, 8, 28, 10, 15, 0));
|
|
expect(nextCronAfter("30 3 * * *", base)).toEqual(new Date(2026, 8, 29, 3, 30, 0));
|
|
expect(nextCronAfter("0 9 * * 0", base)).toEqual(new Date(2026, 9, 4, 9, 0, 0));
|
|
expect(nextCronAfter("0 0 1 1 *", base)).toEqual(new Date(2027, 0, 1, 0, 0, 0));
|
|
expect(() => nextCronAfter("61 * * * *", base)).toThrow();
|
|
});
|
|
});
|
|
|
|
interface Enq {
|
|
task: string;
|
|
input: unknown;
|
|
key: string;
|
|
}
|
|
|
|
describe.skipIf(!hasPg)("定时任务注册表(真实 PG)", () => {
|
|
let h: Harness;
|
|
let admin: Browser;
|
|
const enqueued: Enq[] = [];
|
|
const mod = createSchedulesModule();
|
|
let ctx: ServerContext;
|
|
let reg: ScheduleRegistry;
|
|
let sched: Scheduler;
|
|
const calls = new Map<string, number>();
|
|
|
|
const api = async (method: string, path: string, body?: unknown) => {
|
|
const res = await admin.req(method, path, body);
|
|
return { status: res.status, body: await json(res) };
|
|
};
|
|
const job = async (id: string) =>
|
|
await h.db.selectFrom("scheduled_job").selectAll().where("id", "=", id).executeTakeFirstOrThrow();
|
|
const counting = (key: string, extra: Partial<ScheduledJobDef> = {}): ScheduledJobDef => ({
|
|
key,
|
|
defaults: { name: key, group: "maintain", schedule: { type: "interval", everyMs: 60_000 } },
|
|
run: async () => {
|
|
calls.set(key, (calls.get(key) ?? 0) + 1);
|
|
return { summary: "done" };
|
|
},
|
|
...extra,
|
|
});
|
|
|
|
beforeAll(async () => {
|
|
h = await createHarness({ extraModules: [mod] });
|
|
ctx = h.ctx;
|
|
// runAgent 入队用的假引擎(同键去重)
|
|
const keys = new Set<string>();
|
|
(ctx as { engine: ServerContext["engine"] }).engine = {
|
|
...ctx.engine,
|
|
async enqueue(task: string, input: unknown, opts: { idempotencyKey: string }) {
|
|
const dup = keys.has(opts.idempotencyKey);
|
|
if (!dup) {
|
|
keys.add(opts.idempotencyKey);
|
|
enqueued.push({ task, input, key: opts.idempotencyKey });
|
|
}
|
|
return { runId: randomUUID(), deduplicated: dup };
|
|
},
|
|
} as ServerContext["engine"];
|
|
reg = scheduleRegistry(ctx);
|
|
sched = new Scheduler(ctx, reg, { tickMs: 3_600_000, lockRetryMs: 50, harvestMs: 0 });
|
|
const token = await ensureSetupToken(ctx);
|
|
admin = new Browser(h.app);
|
|
expect(
|
|
(await admin.req("POST", "/auth/setup", { setupToken: token, username: "admin", password: PASSWORD })).status,
|
|
).toBe(200);
|
|
}, 60_000);
|
|
|
|
afterAll(async () => {
|
|
await sched?.stop();
|
|
await h?.close();
|
|
});
|
|
|
|
it("WK-1:lane=maintenance 的内联任务在文件道 / 网络道有活时让路(保持到期),空闲后执行;立即执行不让路", async () => {
|
|
let busy = true;
|
|
const logger = ctx.logger;
|
|
const lanes = new LaneController({
|
|
engine: { queueCounts: async (g) => g.map(() => ({ queued: busy ? 1 : 0, running: 0 })) },
|
|
logger,
|
|
lanes: { file: ["f"], network: ["n"], maintenance: [] },
|
|
readSettings: async () => ({ file: 2, network: 4, maintenance: 1 }),
|
|
busyCacheMs: 0,
|
|
maxYieldMs: 3_600_000,
|
|
});
|
|
await lanes.refresh();
|
|
setLaneController(ctx, lanes);
|
|
try {
|
|
registerScheduledJob(ctx, counting("test.lane_maint", { lane: "maintenance" }));
|
|
registerScheduledJob(ctx, counting("test.lane_light"));
|
|
const started = await sched.tick();
|
|
await sched.idle();
|
|
expect(started).toContain("test.lane_light");
|
|
expect(started).not.toContain("test.lane_maint");
|
|
expect(calls.get("test.lane_maint") ?? 0).toBe(0);
|
|
expect(lanes.yielding()).toBe(true);
|
|
|
|
// 立即执行:不让路
|
|
await h.db
|
|
.updateTable("scheduled_job")
|
|
.set({ run_requested_at: new Date() })
|
|
.where("id", "=", "test.lane_maint")
|
|
.execute();
|
|
expect(await sched.tick()).toContain("test.lane_maint");
|
|
await sched.idle();
|
|
expect(calls.get("test.lane_maint")).toBe(1);
|
|
expect(lanes.active("maintenance")).toBe(0);
|
|
|
|
// 到期未跑的:空闲后执行
|
|
await h.db
|
|
.updateTable("scheduled_job")
|
|
.set({ next_run_at: new Date(Date.now() - 1000) })
|
|
.where("id", "=", "test.lane_maint")
|
|
.execute();
|
|
expect(await sched.tick()).not.toContain("test.lane_maint");
|
|
busy = false;
|
|
expect(await sched.tick()).toContain("test.lane_maint");
|
|
await sched.idle();
|
|
expect(calls.get("test.lane_maint")).toBe(2);
|
|
} finally {
|
|
setLaneController(ctx, null);
|
|
reg.unregister("test.lane_maint");
|
|
reg.unregister("test.lane_light");
|
|
}
|
|
});
|
|
|
|
it("登记默认值;用户改过的开关与调度不被再次登记覆盖;reset 恢复默认", async () => {
|
|
registerScheduledJob(ctx, counting("test.keep"));
|
|
await sched.tick();
|
|
await sched.idle();
|
|
let row = await job("test.keep");
|
|
expect(row).toMatchObject({ enabled: true, customized: false, schedule: { type: "interval", everyMs: 60_000 } });
|
|
|
|
const r = await api("PATCH", "/schedules/test.keep", {
|
|
enabled: false,
|
|
schedule: { type: "interval", everyMs: 120_000 },
|
|
});
|
|
expect(r.status).toBe(200);
|
|
expect(r.body.data).toMatchObject({ enabled: false, customized: true });
|
|
|
|
// 模块再次启动,默认值变了
|
|
registerScheduledJob(
|
|
ctx,
|
|
counting("test.keep", {
|
|
defaults: { name: "改名了", group: "download", schedule: { type: "interval", everyMs: 30_000 } },
|
|
}),
|
|
);
|
|
await sched.tick();
|
|
row = await job("test.keep");
|
|
expect(row).toMatchObject({
|
|
name: "改名了",
|
|
board_group: "download",
|
|
enabled: false,
|
|
schedule: { type: "interval", everyMs: 120_000 },
|
|
default_schedule: { type: "interval", everyMs: 30_000 },
|
|
});
|
|
|
|
const reset = await api("PATCH", "/schedules/test.keep", { reset: true });
|
|
expect(reset.body.data).toMatchObject({
|
|
enabled: true,
|
|
customized: false,
|
|
schedule: { type: "interval", everyMs: 30_000 },
|
|
});
|
|
|
|
// 间隔下限与 cron 校验
|
|
expect((await api("PATCH", "/schedules/test.keep", { schedule: { type: "interval", everyMs: 500 } })).status).toBe(
|
|
422,
|
|
);
|
|
expect(
|
|
(await api("PATCH", "/schedules/test.keep", { schedule: { type: "cron", cron: "99 * * * *" } })).status,
|
|
).toBe(422);
|
|
const cron = await api("PATCH", "/schedules/test.keep", { schedule: { type: "cron", cron: "0 4 * * *" } });
|
|
expect(cron.status).toBe(200);
|
|
expect(new Date(cron.body.data.nextRunAt).getHours()).toBe(4);
|
|
});
|
|
|
|
it("config:没改过的行随代码默认值整体更新;PATCH 过的保留表内值", async () => {
|
|
const def = (config: Record<string, unknown>) =>
|
|
counting("test.cfg", {
|
|
defaults: { name: "c", group: "download", schedule: { type: "interval", everyMs: 60_000 }, config },
|
|
});
|
|
registerScheduledJob(ctx, def({ list: ["a"] }));
|
|
await sched.tick();
|
|
registerScheduledJob(ctx, def({ list: ["a", "b"] }));
|
|
await sched.tick();
|
|
expect((await job("test.cfg")).config).toEqual({ list: ["a", "b"] });
|
|
expect((await api("PATCH", "/schedules/test.cfg", { config: { list: ["x"] } })).status).toBe(200);
|
|
registerScheduledJob(ctx, def({ list: ["a", "b", "c"], extra: 1 }));
|
|
await sched.tick();
|
|
expect((await job("test.cfg")).config).toEqual({ list: ["x"], extra: 1 });
|
|
});
|
|
|
|
it("立即执行幂等(合并重复请求)且不改下次运行时间", async () => {
|
|
registerScheduledJob(
|
|
ctx,
|
|
counting("test.now", { defaults: { ...counting("x").defaults, firstRun: "after_interval" } }),
|
|
);
|
|
await sched.tick();
|
|
const before = await job("test.now");
|
|
expect(before.next_run_at!.getTime()).toBeGreaterThan(Date.now());
|
|
const a = await api("POST", "/schedules/test.now/run-now");
|
|
const b = await api("POST", "/schedules/test.now/run-now");
|
|
expect(a.body.data.queued).toBe(true);
|
|
expect(b.body.data.queued).toBe(false);
|
|
const started = await sched.tick();
|
|
expect(started).toContain("test.now");
|
|
await sched.idle();
|
|
expect(await sched.tick()).not.toContain("test.now");
|
|
await sched.idle();
|
|
expect(calls.get("test.now")).toBe(1);
|
|
const after = await job("test.now");
|
|
expect(after.next_run_at).toEqual(before.next_run_at);
|
|
expect(after).toMatchObject({ last_status: "ok", run_count: 1, run_requested_at: null, running_since: null });
|
|
const runs = await api("GET", "/schedules/test.now/runs");
|
|
expect(runs.body.data.items).toHaveLength(1);
|
|
expect(runs.body.data.items[0]).toMatchObject({ trigger: "manual", status: "ok", summary: "done" });
|
|
});
|
|
|
|
it("失败写入 last_error / fail_count 与历史;跳过本次顺延一个周期", async () => {
|
|
registerScheduledJob(ctx, {
|
|
key: "test.fail",
|
|
defaults: { name: "会失败", group: "maintain", schedule: { type: "interval", everyMs: 60_000 } },
|
|
run: async () => {
|
|
throw new Error("qB 连不上");
|
|
},
|
|
});
|
|
await sched.tick();
|
|
await sched.idle();
|
|
const row = await job("test.fail");
|
|
expect(row).toMatchObject({ last_status: "failed", last_error: "qB 连不上", fail_count: 1, run_count: 1 });
|
|
// interval:下次 = 结束 + 周期
|
|
expect(row.next_run_at!.getTime() - Date.now()).toBeGreaterThan(50_000);
|
|
|
|
const skip = await api("POST", "/schedules/test.fail/skip");
|
|
expect(skip.status).toBe(200);
|
|
expect(new Date(skip.body.data.nextRunAt).getTime() - row.next_run_at!.getTime()).toBe(60_000);
|
|
const runs = (await api("GET", "/schedules/test.fail/runs")).body.data.items;
|
|
expect(runs.map((r: { trigger: string; status: string }) => `${r.trigger}:${r.status}`)).toEqual([
|
|
"skip:skipped",
|
|
"schedule:failed",
|
|
]);
|
|
expect(runs[1].error).toBe("qB 连不上");
|
|
});
|
|
|
|
it("单实例锁:两个调度器只有一个执行;执行者停止后另一个接手", async () => {
|
|
const regA = scheduleRegistry({ db: {} } as ServerContext);
|
|
const regB = scheduleRegistry({ db: {} } as ServerContext);
|
|
const ran: string[] = [];
|
|
for (const [name, r] of [
|
|
["A", regA],
|
|
["B", regB],
|
|
] as const) {
|
|
r.register({
|
|
key: "test.lock",
|
|
defaults: { name: "锁", group: "maintain", schedule: { type: "interval", everyMs: 1 } },
|
|
run: async () => {
|
|
ran.push(name);
|
|
return {};
|
|
},
|
|
});
|
|
}
|
|
const a = new Scheduler(ctx, regA, { tickMs: 20, lockRetryMs: 30 });
|
|
const b = new Scheduler(ctx, regB, { tickMs: 20, lockRetryMs: 30 });
|
|
// 表里其它测试的任务不要被 a/b 执行:只登记了 test.lock,其余 task 行没有执行体会被跳过
|
|
a.start();
|
|
await new Promise((r) => setTimeout(r, 150));
|
|
b.start();
|
|
await new Promise((r) => setTimeout(r, 400));
|
|
expect(a.leader).toBe(true);
|
|
expect(b.leader).toBe(false);
|
|
expect(ran.length).toBeGreaterThan(2);
|
|
expect(new Set(ran)).toEqual(new Set(["A"]));
|
|
const board = await api("GET", "/schedules");
|
|
expect(board.body.data.schedulerOnline).toBe(true);
|
|
|
|
await a.stop();
|
|
await new Promise((r) => setTimeout(r, 400));
|
|
expect(b.leader).toBe(true);
|
|
expect(ran).toContain("B");
|
|
await b.stop();
|
|
await sql`update scheduled_job set enabled = false where id = 'test.lock'`.execute(h.db);
|
|
});
|
|
|
|
it("agent_check:新建巡检 → 立即执行入队 runAgent;结束后回填结果并写入待处理", async () => {
|
|
const created = await api("POST", "/schedules", {
|
|
name: "自定义巡检",
|
|
schedule: { type: "interval", everyMs: 3_600_000 },
|
|
config: {
|
|
goalTemplate: "巡检 {{date}}:待处理 {{openDecisions}} 条",
|
|
capabilityAllowlist: ["task.inspect", "inventory.query", "download.submit"],
|
|
budget: { maxToolCalls: 5 },
|
|
reportAsDecision: true,
|
|
},
|
|
});
|
|
expect(created.status).toBe(200);
|
|
const id = created.body.data.id as string;
|
|
expect(created.body.data).toMatchObject({ kind: "agent_check", builtin: false, group: "agent", enabled: false });
|
|
|
|
expect((await api("POST", `/schedules/${id}/run-now`)).body.data.queued).toBe(true);
|
|
await sched.tick();
|
|
await sched.idle();
|
|
const [run] = (await api("GET", `/schedules/${id}/runs`)).body.data.items;
|
|
expect(run).toMatchObject({ trigger: "manual", status: "ok" });
|
|
expect(run.agentRunId).toBeTruthy();
|
|
const ar = await h.db
|
|
.selectFrom("agent_run")
|
|
.selectAll()
|
|
.where("id", "=", run.agentRunId)
|
|
.executeTakeFirstOrThrow();
|
|
expect(ar).toMatchObject({ trigger: "schedule", mode: "plan_only", created_by: `schedule:${id}` });
|
|
expect(ar.goal).toMatch(/^巡检 \d{4}-\d{2}-\d{2}:待处理 0 条$/);
|
|
// 只读巡检:execute 类能力(download.submit)被剔除
|
|
expect((ar.scope as { capabilities: string[] }).capabilities).toEqual(["task.inspect", "inventory.query"]);
|
|
expect(ar.budget).toMatchObject({ maxToolCalls: 5 });
|
|
expect(enqueued.filter((e) => e.task === "runAgent")).toHaveLength(1);
|
|
|
|
// runAgent 结束 → 回填
|
|
await h.db
|
|
.updateTable("agent_run")
|
|
.set({ status: "succeeded", summary: "一切正常" })
|
|
.where("id", "=", ar.id)
|
|
.execute();
|
|
expect(await sched.harvest()).toBe(1);
|
|
const [after] = (await api("GET", `/schedules/${id}/runs`)).body.data.items;
|
|
expect(after).toMatchObject({ agentStatus: "succeeded", summary: "一切正常" });
|
|
const d = await h.db.selectFrom("decision").selectAll().where("agent_run_id", "=", ar.id).executeTakeFirstOrThrow();
|
|
expect(d).toMatchObject({ kind: "other", severity: "info", summary: "一切正常" });
|
|
expect(await sched.harvest()).toBe(0);
|
|
|
|
// INT-3:看板给巡检带最近会话的费用与模型汇总
|
|
await h.db
|
|
.updateTable("agent_run")
|
|
.set({ provider: "anthropic", model: "claude-x", usage: JSON.stringify({ costUsd: 0.25 }) })
|
|
.where("id", "=", ar.id)
|
|
.execute();
|
|
const listed = (await api("GET", "/schedules")).body.data.items.find((j: { id: string }) => j.id === id);
|
|
expect(listed.agentRuns).toMatchObject({
|
|
count: 1,
|
|
lastCostUsd: 0.25,
|
|
avgCostUsd: 0.25,
|
|
weekCount: 1,
|
|
weekCostUsd: 0.25,
|
|
lastProvider: "anthropic",
|
|
lastModel: "claude-x",
|
|
models: [{ provider: "anthropic", model: "claude-x", count: 1 }],
|
|
recent: [{ id: ar.id, status: "succeeded", summary: "一切正常", costUsd: 0.25 }],
|
|
});
|
|
const plain = (await api("GET", "/schedules")).body.data.items.find((j: { id: string }) => j.id === "test.keep");
|
|
expect(plain.agentRuns).toBeUndefined();
|
|
|
|
// 自定义可改名、可删;内置不可删
|
|
expect((await api("PATCH", `/schedules/${id}`, { name: "夜间巡检" })).body.data.name).toBe("夜间巡检");
|
|
expect((await api("DELETE", "/schedules/test.keep")).status).toBe(409);
|
|
expect((await api("DELETE", `/schedules/${id}`)).status).toBe(200);
|
|
expect((await api("GET", `/schedules/${id}/runs`)).status).toBe(404);
|
|
});
|
|
|
|
it("内置巡检默认关闭;触发条件不满足时计划触发记 skipped,不唤醒 Agent", async () => {
|
|
await mod.start?.(ctx);
|
|
await mod.stop?.();
|
|
await sched.tick();
|
|
const board = (await api("GET", "/schedules")).body.data;
|
|
const checks = board.items.filter((j: { kind: string }) => j.kind === "agent_check");
|
|
expect(checks.map((j: { id: string }) => j.id).sort()).toEqual([
|
|
"check.downloads",
|
|
"check.pending",
|
|
"check.weekly",
|
|
]);
|
|
expect(checks.every((j: { enabled: boolean }) => !j.enabled)).toBe(true);
|
|
const backup = board.items.find((j: { id: string }) => j.id === "maintain.backup");
|
|
expect(backup).toMatchObject({ available: false, enabled: false });
|
|
expect((await api("PATCH", "/schedules/maintain.backup", { enabled: true })).status).toBe(409);
|
|
|
|
// 上一个用例写入的巡检报告算未关闭事项,先关掉
|
|
await sql`update decision set status = 'dismissed'`.execute(h.db);
|
|
await api("PATCH", "/schedules/check.pending", { enabled: true });
|
|
await sql`update scheduled_job set next_run_at = now() - interval '1 minute' where id = 'check.pending'`.execute(
|
|
h.db,
|
|
);
|
|
const before = enqueued.length;
|
|
await sched.tick();
|
|
await sched.idle();
|
|
const row = await job("check.pending");
|
|
expect(row.last_status).toBe("skipped");
|
|
expect(enqueued.length).toBe(before);
|
|
await api("PATCH", "/schedules/check.pending", { enabled: false });
|
|
});
|
|
|
|
it("端点只接受控制台会话", async () => {
|
|
const res = await bearer(h.app, DEV_KEY)("GET", "/schedules");
|
|
expect(res.status).toBe(403);
|
|
});
|
|
});
|