feat(tasks): OpenWorkflow-backed TaskEngine, task registry, outbox relay and op-ledger fencing (W1-4)
- TaskEngine port + OpenWorkflowEngine ([email protected].1, schema openworkflow): enqueue/signal/cancel/get/list/startWorker/stop - defineTask registry with zod input schema; 9 stub task definitions (§3.1-8) - TaskContext: step.run/sleep/waitForFact, redacting logger, traceId, attemptToken, abortSignal driven by PG cancel flag - signal persists fact to tasks_fact first; waitForFact checks DB before waiting - OutboxRelay: FOR UPDATE SKIP LOCKED, idempotencyKey outbox:<id> - withAttemptToken fencing on tasks_op_ledger (own migration, merged in W2) - tests: kill -9 crash recovery (child worker), cancel, idempotency, concurrent relays, early fact, lease-loss fencing Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
rootandClaude Fable 5.1 committed
1 parent
ec023a8405
commit
1262340856
22 files changed
+1818
-8
No files matched your search
@@ -0,0 +1,96 @@
|
||||
import { type ChildProcess, spawn } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { createTestRegistry } from "./fixtures/tasks";
|
||||
import {
|
||||
connectDb,
|
||||
createSideEffectTable,
|
||||
dropSchemas,
|
||||
pgAvailable,
|
||||
resetSchemas,
|
||||
setupEngine,
|
||||
sideEffects,
|
||||
type TestDb,
|
||||
testSchemas,
|
||||
waitFor,
|
||||
} from "./helpers";
|
||||
|
||||
const schemas = testSchemas("crash");
|
||||
const hasPg = await pgAvailable();
|
||||
const WORKER = fileURLToPath(new URL("./fixtures/worker.ts", import.meta.url));
|
||||
const PKG_DIR = fileURLToPath(new URL("..", import.meta.url));
|
||||
|
||||
function startWorker(env: Record<string, string>): Promise<ChildProcess> {
|
||||
const child = spawn(process.execPath, ["--import", "tsx", WORKER], {
|
||||
cwd: PKG_DIR,
|
||||
env: { ...process.env, SCHEMA: schemas.biz, OW_SCHEMA: schemas.ow, ...env },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let err = "";
|
||||
child.stderr!.on("data", (d) => {
|
||||
err += d;
|
||||
});
|
||||
return new Promise((resolve, reject) => {
|
||||
child.stdout!.on("data", (d) => {
|
||||
if (String(d).includes("worker ready")) resolve(child);
|
||||
});
|
||||
child.on("exit", (code) => reject(new Error(`worker exited early code=${code}: ${err}`)));
|
||||
});
|
||||
}
|
||||
|
||||
describe.skipIf(!hasPg)("崩溃恢复(worker 子进程 kill -9)", () => {
|
||||
let db: TestDb;
|
||||
let engine: Awaited<ReturnType<typeof setupEngine>>;
|
||||
const children: ChildProcess[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
await resetSchemas(schemas);
|
||||
db = connectDb(schemas.biz);
|
||||
await createSideEffectTable(db);
|
||||
engine = await setupEngine(db, schemas, createTestRegistry(db)); // 只做客户端,不起 worker
|
||||
}, 30_000);
|
||||
|
||||
afterAll(async () => {
|
||||
for (const c of children) if (c.exitCode === null && c.signalCode === null) c.kill("SIGKILL");
|
||||
await engine?.stop();
|
||||
await db?.destroy();
|
||||
await dropSchemas(schemas);
|
||||
}, 30_000);
|
||||
|
||||
it("第 2 步完成后 kill -9,新 worker 续跑:第 1、2 步不重跑,第 3 步完成", async () => {
|
||||
const w1 = await startWorker({ HANG_STEP3: "1" });
|
||||
children.push(w1);
|
||||
const { runId } = await engine.enqueue("threeSteps", { label: "x" }, { idempotencyKey: "crash-1", traceId: "t" });
|
||||
|
||||
// 第 3 步开始 = 第 1、2 步已持久化
|
||||
await waitFor(async () => (await sideEffects(db, runId)).some((s) => s.step === "step3"), 20_000, "step3 start");
|
||||
w1.kill("SIGKILL");
|
||||
await new Promise((r) => w1.once("exit", r));
|
||||
|
||||
const w2 = await startWorker({});
|
||||
children.push(w2);
|
||||
// 租约 30s 到期后被新 worker 认领
|
||||
const done = await waitFor(
|
||||
async () => {
|
||||
const v = await engine.get(runId);
|
||||
return v?.status === "completed" ? v : null;
|
||||
},
|
||||
75_000,
|
||||
"run completed",
|
||||
);
|
||||
expect(done.output).toEqual({ label: "x", sum: 6 });
|
||||
|
||||
const effects = await sideEffects(db, runId);
|
||||
const count = (step: string, note: string | null = null) =>
|
||||
effects.filter((e) => e.step === step && e.note === note).length;
|
||||
expect(count("step1")).toBe(1);
|
||||
expect(count("step2")).toBe(1);
|
||||
expect(count("step3", "start")).toBe(2); // 被中断的步骤在新 worker 上重跑(步骤体须幂等)
|
||||
expect(count("step3", "end")).toBe(1);
|
||||
const step3Pids = effects.filter((e) => e.step === "step3").map((e) => e.pid);
|
||||
expect(step3Pids).toEqual([w1.pid, w2.pid, w2.pid]);
|
||||
|
||||
w2.kill("SIGTERM");
|
||||
await new Promise((r) => w2.once("exit", r));
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -0,0 +1,109 @@
|
||||
import { sql } from "kysely";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import type { OpenWorkflowEngine } from "../src";
|
||||
import { createTestRegistry } from "./fixtures/tasks";
|
||||
import {
|
||||
connectDb,
|
||||
createSideEffectTable,
|
||||
dropSchemas,
|
||||
pgAvailable,
|
||||
resetSchemas,
|
||||
setupEngine,
|
||||
sideEffects,
|
||||
type TestDb,
|
||||
testSchemas,
|
||||
waitFor,
|
||||
} from "./helpers";
|
||||
|
||||
const schemas = testSchemas("engine");
|
||||
const hasPg = await pgAvailable();
|
||||
|
||||
describe.skipIf(!hasPg)("OpenWorkflowEngine(进程内 worker)", () => {
|
||||
let db: TestDb;
|
||||
let engine: OpenWorkflowEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
await resetSchemas(schemas);
|
||||
db = connectDb(schemas.biz);
|
||||
await createSideEffectTable(db);
|
||||
engine = await setupEngine(db, schemas, createTestRegistry(db));
|
||||
await engine.startWorker({ concurrency: 4 });
|
||||
}, 30_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await engine?.stop();
|
||||
await db?.destroy();
|
||||
await dropSchemas(schemas);
|
||||
}, 60_000);
|
||||
|
||||
it("enqueue 幂等:同 key 返回同一 run,不同 key 新建;traceId 进入 handler", async () => {
|
||||
const a = await engine.enqueue("echo", { n: 1 }, { idempotencyKey: "k-1", traceId: "t-1", priority: 5 });
|
||||
const b = await engine.enqueue("echo", { n: 2 }, { idempotencyKey: "k-1", traceId: "t-2" });
|
||||
const c = await engine.enqueue("echo", { n: 3 }, { idempotencyKey: "k-2", traceId: "t-3" });
|
||||
expect(a.deduplicated).toBe(false);
|
||||
expect(b).toEqual({ runId: a.runId, deduplicated: true });
|
||||
expect(c.runId).not.toBe(a.runId);
|
||||
|
||||
const done = await waitFor(async () => {
|
||||
const v = await engine.get(a.runId);
|
||||
return v?.status === "completed" ? v : null;
|
||||
}, 15_000);
|
||||
expect(done.output).toEqual({ n: 1, trace: "t-1" });
|
||||
expect(done).toMatchObject({ taskName: "echo", version: 1, traceId: "t-1", priority: 5, idempotencyKey: "k-1" });
|
||||
|
||||
const listed = await engine.list({ taskName: "echo" });
|
||||
expect(listed.map((r) => r.runId).sort()).toEqual([a.runId, c.runId].sort());
|
||||
}, 30_000);
|
||||
|
||||
it("enqueue 拒绝不合 schema 的输入与未注册任务", async () => {
|
||||
await expect(engine.enqueue("echo", { n: "x" }, { idempotencyKey: "bad", traceId: "t" })).rejects.toThrow();
|
||||
await expect(engine.enqueue("nope", {}, { idempotencyKey: "bad2", traceId: "t" })).rejects.toThrow(
|
||||
/not registered/,
|
||||
);
|
||||
});
|
||||
|
||||
it("取消:进行中的长步骤经 abortSignal 中止,后续步骤不执行", async () => {
|
||||
const { runId } = await engine.enqueue("cancellable", {}, { idempotencyKey: "c-1", traceId: "t" });
|
||||
await waitFor(async () => (await sideEffects(db, runId)).some((s) => s.step === "long"), 15_000, "long start");
|
||||
expect(await engine.cancel(runId, "user requested")).toBe(true);
|
||||
|
||||
await waitFor(
|
||||
async () => (await sideEffects(db, runId)).some((s) => s.note === "aborted"),
|
||||
5_000,
|
||||
"abort propagated",
|
||||
);
|
||||
const v = await engine.get(runId);
|
||||
expect(v?.status).toBe("canceled");
|
||||
expect(v?.cancelReason).toBe("user requested");
|
||||
await new Promise((r) => setTimeout(r, 1500));
|
||||
expect((await sideEffects(db, runId)).map((s) => s.step)).not.toContain("after");
|
||||
// 已取消再取消:幂等
|
||||
expect(await engine.cancel(runId)).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
it("取消已完成的 run 返回 false", async () => {
|
||||
const { runId } = await engine.enqueue("echo", { n: 9 }, { idempotencyKey: "c-done", traceId: "t" });
|
||||
await waitFor(async () => (await engine.get(runId))?.status === "completed", 15_000);
|
||||
expect(await engine.cancel(runId)).toBe(false);
|
||||
}, 30_000);
|
||||
|
||||
it("waitForFact:事实先于等待到达不丢失;等待中到达的事实经信号唤醒", async () => {
|
||||
const { runId } = await engine.enqueue("waitFacts", {}, { idempotencyKey: "f-1", traceId: "t" });
|
||||
// 此刻 handler 还在 before 步骤(1.5s),没有等待者,引擎信号会丢,但事实已落库
|
||||
const early = await engine.signal(runId, "a", { approved: true });
|
||||
expect(early.delivered).toBe(0);
|
||||
|
||||
await waitFor(async () => (await sideEffects(db, runId)).some((s) => s.step === "got-a"), 15_000, "got-a");
|
||||
await waitFor(async () => (await engine.get(runId))?.status === "waiting", 10_000, "waiting on b");
|
||||
const late = await engine.signal(runId, "b", 42);
|
||||
expect(late.delivered).toBe(1);
|
||||
|
||||
const v = await waitFor(async () => {
|
||||
const r = await engine.get(runId);
|
||||
return r?.status === "completed" ? r : null;
|
||||
}, 15_000);
|
||||
expect(v.output).toEqual({ a: { approved: true }, b: 42 });
|
||||
const facts = await sql<{ n: string }>`SELECT count(*) n FROM tasks_fact WHERE run_id = ${runId}`.execute(db);
|
||||
expect(Number(facts.rows[0]!.n)).toBe(2);
|
||||
}, 45_000);
|
||||
});
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
import { sql } from "kysely";
|
||||
import { z } from "zod";
|
||||
import { defineTask, TaskRegistry } from "../../src";
|
||||
import type { TestDb } from "../helpers";
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
/** 测试任务;副作用写 side_effect 表,用来判断步骤是否重跑 */
|
||||
export function createTestRegistry(db: TestDb) {
|
||||
const mark = (runId: string, step: string, note: string | null = null) =>
|
||||
sql`INSERT INTO side_effect (run_id, step, note, pid) VALUES (${runId}, ${step}, ${note}, ${process.pid})`.execute(
|
||||
db,
|
||||
);
|
||||
|
||||
const threeSteps = defineTask({
|
||||
name: "threeSteps",
|
||||
version: 1,
|
||||
inputSchema: z.object({ label: z.string() }),
|
||||
handler: async (ctx) => {
|
||||
const a = await ctx.step.run("step1", async () => {
|
||||
await mark(ctx.runId, "step1");
|
||||
return 1;
|
||||
});
|
||||
const b = await ctx.step.run("step2", async () => {
|
||||
await mark(ctx.runId, "step2");
|
||||
return 2;
|
||||
});
|
||||
const c = await ctx.step.run("step3", async () => {
|
||||
await mark(ctx.runId, "step3", "start");
|
||||
// 崩溃测试:第一个 worker 卡在这里被 kill -9
|
||||
if (process.env.HANG_STEP3 === "1") await sleep(10 * 60_000);
|
||||
await mark(ctx.runId, "step3", "end");
|
||||
return 3;
|
||||
});
|
||||
return { label: ctx.input.label, sum: a + b + c };
|
||||
},
|
||||
});
|
||||
|
||||
const cancellable = defineTask({
|
||||
name: "cancellable",
|
||||
version: 1,
|
||||
inputSchema: z.object({}),
|
||||
handler: async (ctx) => {
|
||||
await ctx.step.run("first", () => mark(ctx.runId, "first").then(() => 1));
|
||||
await ctx.step.run("long", async () => {
|
||||
await mark(ctx.runId, "long", "start");
|
||||
for (let i = 0; i < 300; i++) {
|
||||
if (ctx.abortSignal.aborted) {
|
||||
await mark(ctx.runId, "long", "aborted");
|
||||
throw ctx.abortSignal.reason;
|
||||
}
|
||||
await sleep(100);
|
||||
}
|
||||
return 1;
|
||||
});
|
||||
await ctx.step.run("after", () => mark(ctx.runId, "after").then(() => 1));
|
||||
return "done";
|
||||
},
|
||||
});
|
||||
|
||||
const waitFacts = defineTask({
|
||||
name: "waitFacts",
|
||||
version: 1,
|
||||
inputSchema: z.object({}),
|
||||
handler: async (ctx) => {
|
||||
await ctx.step.run("before", async () => {
|
||||
await mark(ctx.runId, "before");
|
||||
await sleep(1500);
|
||||
return 1;
|
||||
});
|
||||
const a = await ctx.step.waitForFact("a", { timeout: "60s" });
|
||||
await ctx.step.run("got-a", () => mark(ctx.runId, "got-a").then(() => 1));
|
||||
const b = await ctx.step.waitForFact("b", { timeout: "60s" });
|
||||
return { a, b };
|
||||
},
|
||||
});
|
||||
|
||||
const echo = defineTask({
|
||||
name: "echo",
|
||||
version: 1,
|
||||
inputSchema: z.object({ n: z.number() }),
|
||||
handler: async (ctx) => ctx.step.run("echo", async () => ({ n: ctx.input.n, trace: ctx.traceId })),
|
||||
});
|
||||
|
||||
return new TaskRegistry().register(threeSteps, cancellable, waitFacts, echo);
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
// 崩溃恢复测试用的 worker 子进程:SCHEMA / OW_SCHEMA 指定测试 schema,HANG_STEP3=1 让第 3 步卡住
|
||||
import { connectDb, setupEngine } from "../helpers";
|
||||
import { createTestRegistry } from "./tasks";
|
||||
|
||||
const db = connectDb(process.env.SCHEMA!);
|
||||
const engine = await setupEngine(db, { ow: process.env.OW_SCHEMA! }, createTestRegistry(db));
|
||||
await engine.startWorker({ versions: ["threeSteps@1"], concurrency: 2 });
|
||||
console.log(`worker ready pid=${process.pid}`);
|
||||
process.on("SIGTERM", async () => {
|
||||
await engine.stop();
|
||||
await db.destroy();
|
||||
process.exit(0);
|
||||
});
|
||||
@@ -0,0 +1,102 @@
|
||||
import { Kysely, PostgresDialect, sql } from "kysely";
|
||||
import pg from "pg";
|
||||
import { migrateTasksTables, OpenWorkflowEngine, silentSink, type TaskRegistry } from "../src";
|
||||
|
||||
export const PG_URL =
|
||||
process.env.TASKS_TEST_DATABASE_URL ?? process.env.DATABASE_URL ?? "postgres://umt:umt@localhost:55433/umt";
|
||||
|
||||
/** 探测开发库;不可达时集成测试跳过(CI 机器应提供 55433) */
|
||||
export async function pgAvailable(): Promise<boolean> {
|
||||
const c = new pg.Client({ connectionString: PG_URL, connectionTimeoutMillis: 2000 });
|
||||
try {
|
||||
await c.connect();
|
||||
await c.end();
|
||||
return true;
|
||||
} catch {
|
||||
console.warn(`[tasks] PostgreSQL ${PG_URL} 不可达,跳过集成测试`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 测试直接用原生 SQL
|
||||
export type TestDb = Kysely<any>;
|
||||
|
||||
/** 每个测试文件用独立 schema:业务表 <name>,引擎表 <name>_ow */
|
||||
export function testSchemas(suffix: string) {
|
||||
const biz = `tasks_test_${suffix}`;
|
||||
return { biz, ow: `${biz}_ow` };
|
||||
}
|
||||
|
||||
export function connectDb(schema: string): TestDb {
|
||||
const pool = new pg.Pool({ connectionString: PG_URL, max: 6, options: `-c search_path=${schema}` });
|
||||
return new Kysely({ dialect: new PostgresDialect({ pool }) });
|
||||
}
|
||||
|
||||
export async function resetSchemas(schemas: { biz: string; ow: string }) {
|
||||
const admin = connectDb("public");
|
||||
try {
|
||||
await sql.raw(`DROP SCHEMA IF EXISTS ${schemas.biz} CASCADE`).execute(admin);
|
||||
await sql.raw(`DROP SCHEMA IF EXISTS ${schemas.ow} CASCADE`).execute(admin);
|
||||
await sql.raw(`CREATE SCHEMA ${schemas.biz}`).execute(admin);
|
||||
} finally {
|
||||
await admin.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
export async function dropSchemas(schemas: { biz: string; ow: string }) {
|
||||
const admin = connectDb("public");
|
||||
try {
|
||||
await sql.raw(`DROP SCHEMA IF EXISTS ${schemas.biz} CASCADE`).execute(admin);
|
||||
await sql.raw(`DROP SCHEMA IF EXISTS ${schemas.ow} CASCADE`).execute(admin);
|
||||
} finally {
|
||||
await admin.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
/** 与 packages/db 0001_init 的 outbox_event 同构 */
|
||||
export async function createOutboxTable(db: TestDb) {
|
||||
await sql`CREATE TABLE outbox_event (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
at timestamptz NOT NULL DEFAULT now(),
|
||||
type text NOT NULL,
|
||||
payload jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
published_at timestamptz)`.execute(db);
|
||||
}
|
||||
|
||||
export async function createSideEffectTable(db: TestDb) {
|
||||
await sql`CREATE TABLE IF NOT EXISTS side_effect (
|
||||
id bigserial PRIMARY KEY, run_id text NOT NULL, step text NOT NULL, note text, pid int,
|
||||
at timestamptz NOT NULL DEFAULT now())`.execute(db);
|
||||
}
|
||||
|
||||
export async function sideEffects(db: TestDb, runId: string) {
|
||||
const r = await sql<{ step: string; note: string | null; pid: number }>`
|
||||
SELECT step, note, pid FROM side_effect WHERE run_id = ${runId} ORDER BY id`.execute(db);
|
||||
return r.rows;
|
||||
}
|
||||
|
||||
export async function setupEngine(db: TestDb, schemas: { ow: string }, registry: TaskRegistry) {
|
||||
await migrateTasksTables(db);
|
||||
return OpenWorkflowEngine.connect({
|
||||
databaseUrl: PG_URL,
|
||||
db,
|
||||
registry,
|
||||
schema: schemas.ow,
|
||||
logSink: silentSink,
|
||||
cancelPollMs: 200,
|
||||
});
|
||||
}
|
||||
|
||||
export async function waitFor<T>(
|
||||
fn: () => Promise<T | null | undefined | false>,
|
||||
timeoutMs: number,
|
||||
label = "condition",
|
||||
) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
for (;;) {
|
||||
const v = await fn();
|
||||
if (v) return v;
|
||||
if (Date.now() > deadline) throw new Error(`timeout waiting for ${label}`);
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { PACKAGE_NAME } from "../src";
|
||||
|
||||
it("包入口可加载", () => {
|
||||
expect(PACKAGE_NAME).toBe("@umt/tasks");
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import { sql } from "kysely";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { FencingError, getOperation, migrateTasksTables, withAttemptToken } from "../src";
|
||||
import { connectDb, dropSchemas, pgAvailable, resetSchemas, type TestDb, testSchemas } from "./helpers";
|
||||
|
||||
const schemas = testSchemas("ledger");
|
||||
const hasPg = await pgAvailable();
|
||||
|
||||
describe.skipIf(!hasPg)("withAttemptToken(操作账本 fencing)", () => {
|
||||
let db: TestDb;
|
||||
|
||||
beforeAll(async () => {
|
||||
await resetSchemas(schemas);
|
||||
db = connectDb(schemas.biz);
|
||||
await migrateTasksTables(db);
|
||||
await sql`CREATE TABLE download_task (id text PRIMARY KEY, hash text)`.execute(db);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.destroy();
|
||||
await dropSchemas(schemas);
|
||||
});
|
||||
|
||||
it("成功后记录结果;再次调用直接返回结果不重做", async () => {
|
||||
let calls = 0;
|
||||
const run = () =>
|
||||
withAttemptToken(
|
||||
db,
|
||||
"op-1",
|
||||
async (info) => {
|
||||
calls++;
|
||||
return { hash: "h1", attempt: info.attempt };
|
||||
},
|
||||
{
|
||||
runId: "r1",
|
||||
commit: async (trx, result) => {
|
||||
await sql`INSERT INTO download_task (id, hash) VALUES ('d1', ${result.hash})`.execute(trx);
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(await run()).toEqual({ hash: "h1", attempt: 1 });
|
||||
expect(await run()).toEqual({ hash: "h1", attempt: 1 });
|
||||
expect(calls).toBe(1);
|
||||
const row = await getOperation(db, "op-1");
|
||||
expect(row).toMatchObject({ status: "succeeded", attempts: 1, runId: "r1" });
|
||||
});
|
||||
|
||||
it("租约丢失:旧 attempt 执行完后提交被拒绝,新 attempt 的结果生效", async () => {
|
||||
let releaseOld!: () => void;
|
||||
const oldGate = new Promise<void>((r) => {
|
||||
releaseOld = r;
|
||||
});
|
||||
let oldStarted!: () => void;
|
||||
const oldStartedP = new Promise<void>((r) => {
|
||||
oldStarted = r;
|
||||
});
|
||||
|
||||
// 旧 worker(被 SIGSTOP 之类卡住):已开始执行,尚未提交
|
||||
const oldAttempt = withAttemptToken(
|
||||
db,
|
||||
"op-2",
|
||||
async () => {
|
||||
oldStarted();
|
||||
await oldGate;
|
||||
return "old";
|
||||
},
|
||||
{
|
||||
commit: async (trx) => void (await sql`INSERT INTO download_task (id, hash) VALUES ('d2', 'old')`.execute(trx)),
|
||||
},
|
||||
);
|
||||
await oldStartedP;
|
||||
|
||||
// 新 worker 接管同一操作:resumed=true,需要先对账外部状态
|
||||
let resumed = false;
|
||||
const fresh = await withAttemptToken(
|
||||
db,
|
||||
"op-2",
|
||||
async (info) => {
|
||||
resumed = info.resumed;
|
||||
return "new";
|
||||
},
|
||||
{
|
||||
commit: async (trx) => void (await sql`INSERT INTO download_task (id, hash) VALUES ('d2', 'new')`.execute(trx)),
|
||||
},
|
||||
);
|
||||
expect(fresh).toBe("new");
|
||||
expect(resumed).toBe(true);
|
||||
|
||||
releaseOld();
|
||||
await expect(oldAttempt).rejects.toBeInstanceOf(FencingError);
|
||||
const r = await sql<{ hash: string }>`SELECT hash FROM download_task WHERE id = 'd2'`.execute(db);
|
||||
expect(r.rows).toEqual([{ hash: "new" }]);
|
||||
expect(await getOperation(db, "op-2")).toMatchObject({ status: "succeeded", attempts: 2, result: "new" });
|
||||
});
|
||||
|
||||
it("fn 抛错:记录 last_error,不写结果,可重试", async () => {
|
||||
await expect(
|
||||
withAttemptToken(db, "op-3", async () => {
|
||||
throw new Error("qb timeout");
|
||||
}),
|
||||
).rejects.toThrow("qb timeout");
|
||||
expect(await getOperation(db, "op-3")).toMatchObject({ status: "started", lastError: "qb timeout" });
|
||||
expect(await withAttemptToken(db, "op-3", async (i) => i.attempt)).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import { sql } from "kysely";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { type EnqueueOptions, OutboxRelay, type TaskEngine } from "../src";
|
||||
import { createTestRegistry } from "./fixtures/tasks";
|
||||
import {
|
||||
connectDb,
|
||||
createOutboxTable,
|
||||
createSideEffectTable,
|
||||
dropSchemas,
|
||||
pgAvailable,
|
||||
resetSchemas,
|
||||
setupEngine,
|
||||
type TestDb,
|
||||
testSchemas,
|
||||
} from "./helpers";
|
||||
|
||||
const schemas = testSchemas("outbox");
|
||||
const hasPg = await pgAvailable();
|
||||
|
||||
describe.skipIf(!hasPg)("OutboxRelay", () => {
|
||||
let db: TestDb;
|
||||
let engine: Awaited<ReturnType<typeof setupEngine>>;
|
||||
|
||||
beforeAll(async () => {
|
||||
await resetSchemas(schemas);
|
||||
db = connectDb(schemas.biz);
|
||||
await createOutboxTable(db);
|
||||
await createSideEffectTable(db);
|
||||
engine = await setupEngine(db, schemas, createTestRegistry(db));
|
||||
}, 30_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await engine?.stop();
|
||||
await db?.destroy();
|
||||
await dropSchemas(schemas);
|
||||
}, 30_000);
|
||||
|
||||
it("两个 relay 并发处理 60 条事件:每条只 enqueue 一次,全部标记已投递", async () => {
|
||||
// 业务事务写 outbox
|
||||
await db.transaction().execute(async (trx) => {
|
||||
for (let i = 0; i < 60; i++) {
|
||||
await sql`INSERT INTO outbox_event (type, payload) VALUES ('echo.requested', ${JSON.stringify({ n: i, traceId: `tr-${i}` })}::jsonb)`.execute(
|
||||
trx,
|
||||
);
|
||||
}
|
||||
await sql`INSERT INTO outbox_event (type, payload) VALUES ('other.type', '{}'::jsonb)`.execute(trx);
|
||||
});
|
||||
|
||||
const calls: string[] = [];
|
||||
// 包一层计数,确认没有重复 enqueue(引擎幂等只是兜底)
|
||||
const counting: Pick<TaskEngine, "enqueue"> = {
|
||||
enqueue: async (task: string, input: unknown, opts: EnqueueOptions) => {
|
||||
calls.push(opts.idempotencyKey);
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
return engine.enqueue(task, input, opts);
|
||||
},
|
||||
};
|
||||
const routes = { "echo.requested": (ev: { payload: unknown }) => ({ task: "echo", input: ev.payload }) };
|
||||
const relays = [0, 1].map((i) => new OutboxRelay({ db, engine: counting, routes, batchSize: 7 + i }));
|
||||
|
||||
const drain = async (r: OutboxRelay) => {
|
||||
const perRelay = { published: 0 };
|
||||
for (;;) {
|
||||
const res = await r.pollOnce();
|
||||
perRelay.published += res.published;
|
||||
if (res.published === 0 && res.failed === 0) return perRelay.published;
|
||||
}
|
||||
};
|
||||
const [p0, p1] = await Promise.all(relays.map(drain));
|
||||
|
||||
expect(p0! + p1!).toBe(60);
|
||||
expect(p0).toBeGreaterThan(0);
|
||||
expect(p1).toBeGreaterThan(0);
|
||||
expect(calls).toHaveLength(60);
|
||||
expect(new Set(calls).size).toBe(60);
|
||||
|
||||
const pending = await sql<{ type: string }>`SELECT type FROM outbox_event WHERE published_at IS NULL`.execute(db);
|
||||
expect(pending.rows).toEqual([{ type: "other.type" }]); // 未配置路由的 type 不认领
|
||||
|
||||
const runs = await sql<{ n: string; keys: string }>`
|
||||
SELECT count(*) n, count(DISTINCT idempotency_key) keys FROM tasks_run_meta WHERE task_name = 'echo'`.execute(db);
|
||||
expect(runs.rows[0]).toEqual({ n: "60", keys: "60" });
|
||||
const traced = await sql<{ trace_id: string }>`
|
||||
SELECT m.trace_id FROM tasks_run_meta m
|
||||
JOIN outbox_event o ON m.idempotency_key = 'outbox:' || o.id::text
|
||||
WHERE o.payload->>'n' = '3'`.execute(db);
|
||||
expect(traced.rows).toEqual([{ trace_id: "tr-3" }]);
|
||||
}, 60_000);
|
||||
|
||||
it("enqueue 失败的事件保持未投递,下一轮重试", async () => {
|
||||
await sql`INSERT INTO outbox_event (type, payload) VALUES ('echo.requested', '{"n":"not-a-number"}'::jsonb)`.execute(
|
||||
db,
|
||||
);
|
||||
const relay = new OutboxRelay({
|
||||
db,
|
||||
engine,
|
||||
routes: { "echo.requested": (ev) => ({ task: "echo", input: ev.payload }) },
|
||||
logSink: () => {},
|
||||
});
|
||||
expect(await relay.pollOnce()).toEqual({ published: 0, failed: 1 });
|
||||
const left = await sql<{ n: string }>`
|
||||
SELECT count(*) n FROM outbox_event WHERE published_at IS NULL AND type = 'echo.requested'`.execute(db);
|
||||
expect(left.rows[0]!.n).toBe("1");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { createDefaultRegistry, defineTask, redact, TASK_NAMES, TaskRegistry, UnknownTaskError } from "../src";
|
||||
|
||||
describe("TaskRegistry", () => {
|
||||
it("注册任务书 §3.1-8 的九个空壳任务", () => {
|
||||
const reg = createDefaultRegistry();
|
||||
expect(reg.names()).toEqual([...TASK_NAMES].sort());
|
||||
for (const n of TASK_NAMES) expect(reg.latest(n).version).toBe(1);
|
||||
});
|
||||
|
||||
it("latest 取最高版本;select 按 name@version / @version 过滤", () => {
|
||||
const mk = (name: string, version: number) =>
|
||||
defineTask({ name, version, inputSchema: z.object({}), handler: async () => null });
|
||||
const reg = new TaskRegistry().register(mk("a", 1), mk("a", 2), mk("b", 1));
|
||||
expect(reg.latest("a").version).toBe(2);
|
||||
expect(reg.select(["a@1"]).map((d) => `${d.name}@${d.version}`)).toEqual(["a@1"]);
|
||||
expect(reg.select(["@1"]).map((d) => `${d.name}@${d.version}`)).toEqual(["a@1", "b@1"]);
|
||||
expect(reg.select(["a"])).toHaveLength(2);
|
||||
expect(reg.select()).toHaveLength(3);
|
||||
expect(() => reg.register(mk("a", 1))).toThrow(/already registered/);
|
||||
expect(() => reg.latest("zzz")).toThrow(UnknownTaskError);
|
||||
expect(() => mk("a", 0)).toThrow(/invalid task version/);
|
||||
});
|
||||
|
||||
it("空壳 handler 回显输入", async () => {
|
||||
const def = createDefaultRegistry().latest("submitDownload");
|
||||
const input = def.inputSchema.parse({ acquisitionPlanId: "p1" });
|
||||
const out = await def.handler({ input, logger: { info() {} } } as never);
|
||||
expect(out).toEqual({ todo: true, task: "submitDownload", input: { acquisitionPlanId: "p1" } });
|
||||
expect(() => def.inputSchema.parse({})).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("redact", () => {
|
||||
it("敏感键与 URL 凭证参数脱敏", () => {
|
||||
expect(
|
||||
redact({
|
||||
url: "https://pt.example/download.php?id=1&passkey=abcdef123",
|
||||
nested: { apiKey: "k", password: "p", ok: 1 },
|
||||
list: ["http://x/rss?sign=zz&x=1"],
|
||||
}),
|
||||
).toEqual({
|
||||
url: "https://pt.example/download.php?id=1&passkey=***",
|
||||
nested: { apiKey: "***", password: "***", ok: 1 },
|
||||
list: ["http://x/rss?sign=***&x=1"],
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user