import { type ChildProcess, spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { createTestRegistry } from "./fixtures/tasks"; import { connectDb, createSideEffectTable, dropSchemas, pgAvailable, resetSchemas, setupEngine, sideEffects, type TestDb, testSchemas, waitFor, } from "./helpers"; const schemas = testSchemas("crash"); const hasPg = await pgAvailable(); const WORKER = fileURLToPath(new URL("./fixtures/worker.ts", import.meta.url)); const PKG_DIR = fileURLToPath(new URL("..", import.meta.url)); function startWorker(env: Record): Promise { const child = spawn(process.execPath, ["--import", "tsx", WORKER], { cwd: PKG_DIR, env: { ...process.env, SCHEMA: schemas.biz, OW_SCHEMA: schemas.ow, ...env }, stdio: ["ignore", "pipe", "pipe"], }); let err = ""; child.stderr!.on("data", (d) => { err += d; }); return new Promise((resolve, reject) => { child.stdout!.on("data", (d) => { if (String(d).includes("worker ready")) resolve(child); }); child.on("exit", (code) => reject(new Error(`worker exited early code=${code}: ${err}`))); }); } describe.skipIf(!hasPg)("崩溃恢复(worker 子进程 kill -9)", () => { let db: TestDb; let engine: Awaited>; const children: ChildProcess[] = []; beforeAll(async () => { await resetSchemas(schemas); db = connectDb(schemas.biz); await createSideEffectTable(db); engine = await setupEngine(db, schemas, createTestRegistry(db)); // 只做客户端,不起 worker }, 30_000); afterAll(async () => { for (const c of children) if (c.exitCode === null && c.signalCode === null) c.kill("SIGKILL"); await engine?.stop(); await db?.destroy(); await dropSchemas(schemas); }, 30_000); it("第 2 步完成后 kill -9,新 worker 续跑:第 1、2 步不重跑,第 3 步完成", async () => { const w1 = await startWorker({ HANG_STEP3: "1" }); children.push(w1); const { runId } = await engine.enqueue("threeSteps", { label: "x" }, { idempotencyKey: "crash-1", traceId: "t" }); // 第 3 步开始 = 第 1、2 步已持久化 await waitFor(async () => (await sideEffects(db, runId)).some((s) => s.step === "step3"), 20_000, "step3 start"); w1.kill("SIGKILL"); await new Promise((r) => w1.once("exit", r)); const w2 = await startWorker({}); children.push(w2); // 租约 30s 到期后被新 worker 认领 const done = await waitFor( async () => { const v = await engine.get(runId); return v?.status === "completed" ? v : null; }, 75_000, "run completed", ); expect(done.output).toEqual({ label: "x", sum: 6 }); const effects = await sideEffects(db, runId); const count = (step: string, note: string | null = null) => effects.filter((e) => e.step === step && e.note === note).length; expect(count("step1")).toBe(1); expect(count("step2")).toBe(1); expect(count("step3", "start")).toBe(2); // 被中断的步骤在新 worker 上重跑(步骤体须幂等) expect(count("step3", "end")).toBe(1); const step3Pids = effects.filter((e) => e.step === "step3").map((e) => e.pid); expect(step3Pids).toEqual([w1.pid, w2.pid, w2.pid]); w2.kill("SIGTERM"); await new Promise((r) => w2.once("exit", r)); }, 120_000); });