rootandClaude Fable 5.1
1262340856
- TaskEngine port + OpenWorkflowEngine ([email protected].1, schema openworkflow): enqueue/signal/cancel/get/list/startWorker/stop - defineTask registry with zod input schema; 9 stub task definitions (§3.1-8) - TaskContext: step.run/sleep/waitForFact, redacting logger, traceId, attemptToken, abortSignal driven by PG cancel flag - signal persists fact to tasks_fact first; waitForFact checks DB before waiting - OutboxRelay: FOR UPDATE SKIP LOCKED, idempotencyKey outbox:<id> - withAttemptToken fencing on tasks_op_ledger (own migration, merged in W2) - tests: kill -9 crash recovery (child worker), cancel, idempotency, concurrent relays, early fact, lease-loss fencing Co-Authored-By: Claude Fable 5.1 <[email protected]>
97 lines
3.4 KiB
TypeScript
97 lines
3.4 KiB
TypeScript
import { type ChildProcess, spawn } from "node:child_process";
|
|
import { fileURLToPath } from "node:url";
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import { createTestRegistry } from "./fixtures/tasks";
|
|
import {
|
|
connectDb,
|
|
createSideEffectTable,
|
|
dropSchemas,
|
|
pgAvailable,
|
|
resetSchemas,
|
|
setupEngine,
|
|
sideEffects,
|
|
type TestDb,
|
|
testSchemas,
|
|
waitFor,
|
|
} from "./helpers";
|
|
|
|
const schemas = testSchemas("crash");
|
|
const hasPg = await pgAvailable();
|
|
const WORKER = fileURLToPath(new URL("./fixtures/worker.ts", import.meta.url));
|
|
const PKG_DIR = fileURLToPath(new URL("..", import.meta.url));
|
|
|
|
function startWorker(env: Record<string, string>): Promise<ChildProcess> {
|
|
const child = spawn(process.execPath, ["--import", "tsx", WORKER], {
|
|
cwd: PKG_DIR,
|
|
env: { ...process.env, SCHEMA: schemas.biz, OW_SCHEMA: schemas.ow, ...env },
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
let err = "";
|
|
child.stderr!.on("data", (d) => {
|
|
err += d;
|
|
});
|
|
return new Promise((resolve, reject) => {
|
|
child.stdout!.on("data", (d) => {
|
|
if (String(d).includes("worker ready")) resolve(child);
|
|
});
|
|
child.on("exit", (code) => reject(new Error(`worker exited early code=${code}: ${err}`)));
|
|
});
|
|
}
|
|
|
|
describe.skipIf(!hasPg)("崩溃恢复(worker 子进程 kill -9)", () => {
|
|
let db: TestDb;
|
|
let engine: Awaited<ReturnType<typeof setupEngine>>;
|
|
const children: ChildProcess[] = [];
|
|
|
|
beforeAll(async () => {
|
|
await resetSchemas(schemas);
|
|
db = connectDb(schemas.biz);
|
|
await createSideEffectTable(db);
|
|
engine = await setupEngine(db, schemas, createTestRegistry(db)); // 只做客户端,不起 worker
|
|
}, 30_000);
|
|
|
|
afterAll(async () => {
|
|
for (const c of children) if (c.exitCode === null && c.signalCode === null) c.kill("SIGKILL");
|
|
await engine?.stop();
|
|
await db?.destroy();
|
|
await dropSchemas(schemas);
|
|
}, 30_000);
|
|
|
|
it("第 2 步完成后 kill -9,新 worker 续跑:第 1、2 步不重跑,第 3 步完成", async () => {
|
|
const w1 = await startWorker({ HANG_STEP3: "1" });
|
|
children.push(w1);
|
|
const { runId } = await engine.enqueue("threeSteps", { label: "x" }, { idempotencyKey: "crash-1", traceId: "t" });
|
|
|
|
// 第 3 步开始 = 第 1、2 步已持久化
|
|
await waitFor(async () => (await sideEffects(db, runId)).some((s) => s.step === "step3"), 20_000, "step3 start");
|
|
w1.kill("SIGKILL");
|
|
await new Promise((r) => w1.once("exit", r));
|
|
|
|
const w2 = await startWorker({});
|
|
children.push(w2);
|
|
// 租约 30s 到期后被新 worker 认领
|
|
const done = await waitFor(
|
|
async () => {
|
|
const v = await engine.get(runId);
|
|
return v?.status === "completed" ? v : null;
|
|
},
|
|
75_000,
|
|
"run completed",
|
|
);
|
|
expect(done.output).toEqual({ label: "x", sum: 6 });
|
|
|
|
const effects = await sideEffects(db, runId);
|
|
const count = (step: string, note: string | null = null) =>
|
|
effects.filter((e) => e.step === step && e.note === note).length;
|
|
expect(count("step1")).toBe(1);
|
|
expect(count("step2")).toBe(1);
|
|
expect(count("step3", "start")).toBe(2); // 被中断的步骤在新 worker 上重跑(步骤体须幂等)
|
|
expect(count("step3", "end")).toBe(1);
|
|
const step3Pids = effects.filter((e) => e.step === "step3").map((e) => e.pid);
|
|
expect(step3Pids).toEqual([w1.pid, w2.pid, w2.pid]);
|
|
|
|
w2.kill("SIGTERM");
|
|
await new Promise((r) => w2.once("exit", r));
|
|
}, 120_000);
|
|
});
|