feat(tasks): OpenWorkflow-backed TaskEngine, task registry, outbox relay and op-ledger fencing (W1-4)

- TaskEngine port + OpenWorkflowEngine ([email protected].1, schema openworkflow):
  enqueue/signal/cancel/get/list/startWorker/stop
- defineTask registry with zod input schema; 9 stub task definitions (§3.1-8)
- TaskContext: step.run/sleep/waitForFact, redacting logger, traceId,
  attemptToken, abortSignal driven by PG cancel flag
- signal persists fact to tasks_fact first; waitForFact checks DB before waiting
- OutboxRelay: FOR UPDATE SKIP LOCKED, idempotencyKey outbox:<id>
- withAttemptToken fencing on tasks_op_ledger (own migration, merged in W2)
- tests: kill -9 crash recovery (child worker), cancel, idempotency,
  concurrent relays, early fact, lease-loss fencing

Co-Authored-By: Claude Fable 5.1 <[email protected]>

packages/tasks/test/crash-recovery.test.ts

@@ -0,0 +1,96 @@
+import { type ChildProcess, spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { createTestRegistry } from "./fixtures/tasks";
+import {
+  connectDb,
+  createSideEffectTable,
+  dropSchemas,
+  pgAvailable,
+  resetSchemas,
+  setupEngine,
+  sideEffects,
+  type TestDb,
+  testSchemas,
+  waitFor,
+} from "./helpers";
+
+const schemas = testSchemas("crash");
+const hasPg = await pgAvailable();
+const WORKER = fileURLToPath(new URL("./fixtures/worker.ts", import.meta.url));
+const PKG_DIR = fileURLToPath(new URL("..", import.meta.url));
+
+function startWorker(env: Record<string, string>): Promise<ChildProcess> {
+  const child = spawn(process.execPath, ["--import", "tsx", WORKER], {
+    cwd: PKG_DIR,
+    env: { ...process.env, SCHEMA: schemas.biz, OW_SCHEMA: schemas.ow, ...env },
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+  let err = "";
+  child.stderr!.on("data", (d) => {
+    err += d;
+  });
+  return new Promise((resolve, reject) => {
+    child.stdout!.on("data", (d) => {
+      if (String(d).includes("worker ready")) resolve(child);
+    });
+    child.on("exit", (code) => reject(new Error(`worker exited early code=${code}: ${err}`)));
+  });
+}
+
+describe.skipIf(!hasPg)("崩溃恢复(worker 子进程 kill -9)", () => {
+  let db: TestDb;
+  let engine: Awaited<ReturnType<typeof setupEngine>>;
+  const children: ChildProcess[] = [];
+
+  beforeAll(async () => {
+    await resetSchemas(schemas);
+    db = connectDb(schemas.biz);
+    await createSideEffectTable(db);
+    engine = await setupEngine(db, schemas, createTestRegistry(db)); // 只做客户端,不起 worker
+  }, 30_000);
+
+  afterAll(async () => {
+    for (const c of children) if (c.exitCode === null && c.signalCode === null) c.kill("SIGKILL");
+    await engine?.stop();
+    await db?.destroy();
+    await dropSchemas(schemas);
+  }, 30_000);
+
+  it("第 2 步完成后 kill -9,新 worker 续跑:第 1、2 步不重跑,第 3 步完成", async () => {
+    const w1 = await startWorker({ HANG_STEP3: "1" });
+    children.push(w1);
+    const { runId } = await engine.enqueue("threeSteps", { label: "x" }, { idempotencyKey: "crash-1", traceId: "t" });
+
+    // 第 3 步开始 = 第 1、2 步已持久化
+    await waitFor(async () => (await sideEffects(db, runId)).some((s) => s.step === "step3"), 20_000, "step3 start");
+    w1.kill("SIGKILL");
+    await new Promise((r) => w1.once("exit", r));
+
+    const w2 = await startWorker({});
+    children.push(w2);
+    // 租约 30s 到期后被新 worker 认领
+    const done = await waitFor(
+      async () => {
+        const v = await engine.get(runId);
+        return v?.status === "completed" ? v : null;
+      },
+      75_000,
+      "run completed",
+    );
+    expect(done.output).toEqual({ label: "x", sum: 6 });
+
+    const effects = await sideEffects(db, runId);
+    const count = (step: string, note: string | null = null) =>
+      effects.filter((e) => e.step === step && e.note === note).length;
+    expect(count("step1")).toBe(1);
+    expect(count("step2")).toBe(1);
+    expect(count("step3", "start")).toBe(2); // 被中断的步骤在新 worker 上重跑(步骤体须幂等)
+    expect(count("step3", "end")).toBe(1);
+    const step3Pids = effects.filter((e) => e.step === "step3").map((e) => e.pid);
+    expect(step3Pids).toEqual([w1.pid, w2.pid, w2.pid]);
+
+    w2.kill("SIGTERM");
+    await new Promise((r) => w2.once("exit", r));
+  }, 120_000);
+});