refactor(core): migrate to Bun and converge acquisition policies and Agent boundaries
This commit is contained in:
root committed
1 parent
3f1c6d02f6
commit
27c0f27434
268 files changed
+7468
-7852
No files matched your search
@@ -13,7 +13,7 @@
|
||||
| 导出 | 作用 |
|
||||
| --- | --- |
|
||||
| `OpenWorkflowEngine.connect({ databaseUrl, db, registry })` | `TaskEngine` 实现:`enqueue / signal / cancel / get / list / startWorker / stop` |
|
||||
| `defineTask` / `TaskRegistry` / `createDefaultRegistry()` | 任务定义与注册表;默认注册表里有 §3.1-8 的 9 个任务(目前是空壳) |
|
||||
| `defineTask` / `TaskRegistry` | 任务定义与注册表;实际业务 handler 由 server 所属模块注册,未注册任务抛 `UnknownTaskError` |
|
||||
| `OutboxRelay` | 用 `FOR UPDATE SKIP LOCKED` 认领 `outbox_event` 中 `status=pending` 且 `next_attempt_at` 已到的行,按 type 路由后 enqueue(幂等键取行的 `idempotency_key`,缺省 `outbox:<id>`),成功标记 `dispatched` + `published_at`;失败 attempts+1 指数退避(上限 5 分钟),达 `maxAttempts` 置 `dead`;多进程并发时不会重复 |
|
||||
| `migrateEngineSchema(databaseUrl, schema?)` | 只跑引擎自己的迁移后断开(部署时 `--role migrate`) |
|
||||
| `withAttemptToken(db, opKey, fn, { commit })` | 操作账本 fencing,表是 `tasks_op_ledger` |
|
||||
@@ -24,9 +24,9 @@
|
||||
- **取消**:`cancel(runId, reason)` 先写 `tasks_run_meta.cancel_requested_at`,再取消引擎里的 run。handler 执行期间会轮询这个标志(默认 1s)并触发 `abortSignal`;每个 `step.run/sleep` 开始前也会检查一次。进行中的步骤只能靠自己检查 `abortSignal` 来中止。已完成的外部动作不会撤销,补偿要写成显式步骤。
|
||||
- **等待外部条件**:`signal(runId, name, payload)` 先把事实写进 `tasks_fact`,再发送引擎信号。`step.waitForFact(name, { check?, timeout })` 每一轮都先查库,查不到才带超时等待信号。所以事实比等待方先到也不会丢。审批、下载完成这类业务事实,先写业务表,再用 `check` 去读业务表。
|
||||
|
||||
## 新增/填充任务(W2 规则)
|
||||
## 新增业务任务
|
||||
|
||||
1. 在 `src/definitions.ts` 里用 `defineTask({ name, version, inputSchema, retryPolicy, handler })` 定义任务。输入只放业务 id,handler 自己去读业务表。重试策略要显式写:外部网络类 5 次(1s→60s),文件类 3 次。
|
||||
1. 在 server 所属业务模块的 `registerTasks` 中用 `defineTask({ name, version, inputSchema, retryPolicy, handler })` 定义任务。输入只放业务 id,handler 自己去读业务表。显式声明 `lane` 与符合该业务副作用的重试策略；server 的任务分道表与回归同步更新。
|
||||
2. **步骤体必须幂等**。worker 被 kill 后,正在执行的步骤会在新 worker 上**从头重跑**(最长要等约 30s 租约到期)。已完成的步骤不会重跑。
|
||||
3. **有外部副作用的步骤都走 `withAttemptToken`**,opKey 按语义生成,例如 `submit:<planId>`:
|
||||
- 已经有结果时直接返回;
|
||||
@@ -57,4 +57,4 @@
|
||||
|
||||
## 测试
|
||||
|
||||
`deno task test`(或本包 `deno task test`)会连接开发库,每个测试 schema 先用 `@umt/db` 的 `migrateToLatest` 建全部业务表(`DATABASE_URL`,默认 `postgres://umt:umt@localhost:55433/umt`)。每个测试文件使用独立的 `tasks_test_<名>_<pid+随机>` schema(并发跑多份也不冲突),测试前后都会清理;库不可达时跳过集成测试。覆盖范围:kill -9 后续跑(子进程 worker)、取消、幂等 enqueue、两个 relay 并发、事实先到达、fencing、分组 worker 只认领自己的任务。
|
||||
`bun run test`(或本包 `bun run test`)会连接开发库,每个测试 schema 先用 `@umt/db` 的 `migrateToLatest` 建全部业务表(`DATABASE_URL`,默认 `postgres://umt:umt@localhost:55433/umt`)。每个测试文件使用独立的 `tasks_test_<名>_<pid+随机>` schema(并发跑多份也不冲突),测试前后都会清理;单包运行时库不可达会跳过集成测试；根目录 `bun run test:all` 先检查连接,不可达直接失败。覆盖范围:kill -9 后续跑(子进程 worker)、取消、幂等 enqueue、两个 relay 并发、事实先到达、fencing、分组 worker 只认领自己的任务。
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "@umt/tasks",
|
||||
"version": "0.3.44",
|
||||
"exports": { ".": "./src/index.ts" },
|
||||
"tasks": {
|
||||
"test": "deno run -A npm:vitest run",
|
||||
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
|
||||
}
|
||||
}
|
||||
@@ -4,5 +4,13 @@
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"version": "0.3.44",
|
||||
"scripts": {
|
||||
"test": "bun x --bun vitest run",
|
||||
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
|
||||
},
|
||||
"dependencies": {
|
||||
"@umt/db": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
import { z } from "zod";
|
||||
import { defineTask, TaskRegistry } from "./registry.ts";
|
||||
|
||||
// 任务书 §3.1-8 的九类任务。W1 只落空壳:输入 schema + 回显 handler,W2 填充处理器。
|
||||
// 填充规则见 README「新增/填充任务」:副作用走 withAttemptToken,外部条件用 step.waitForFact,
|
||||
// 业务状态写业务表,不要放进 handler 返回值。
|
||||
|
||||
const id = z.string().min(1);
|
||||
|
||||
/** 空壳 handler:记录一条日志并回显输入 */
|
||||
function stub<I>(name: string) {
|
||||
return async (ctx: { input: I; logger: { info(msg: string, f?: Record<string, unknown>): void } }) => {
|
||||
// TODO(W2): 实现真实处理器
|
||||
ctx.logger.info(`${name}: stub handler (TODO W2)`);
|
||||
return { todo: true as const, task: name, input: ctx.input };
|
||||
};
|
||||
}
|
||||
|
||||
const NETWORK_RETRY = {
|
||||
maximumAttempts: 5,
|
||||
initialInterval: "1s",
|
||||
backoffCoefficient: 2,
|
||||
maximumInterval: "60s",
|
||||
} as const;
|
||||
const FILE_RETRY = {
|
||||
maximumAttempts: 3,
|
||||
initialInterval: "2s",
|
||||
backoffCoefficient: 2,
|
||||
maximumInterval: "60s",
|
||||
} as const;
|
||||
const DB_RETRY = { maximumAttempts: 3, initialInterval: "1s", backoffCoefficient: 2, maximumInterval: "30s" } as const;
|
||||
|
||||
export const ingestFeed = defineTask({
|
||||
name: "ingestFeed",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ feedSourceId: id }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: stub("ingestFeed"),
|
||||
});
|
||||
|
||||
export const resolveObservation = defineTask({
|
||||
name: "resolveObservation",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ observationId: id }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: stub("resolveObservation"),
|
||||
});
|
||||
|
||||
export const evaluateAcquisition = defineTask({
|
||||
name: "evaluateAcquisition",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({
|
||||
subscriptionId: id.optional(),
|
||||
acquisitionSlotId: id.optional(),
|
||||
sourceReleaseId: id.optional(),
|
||||
}),
|
||||
retryPolicy: DB_RETRY,
|
||||
handler: stub("evaluateAcquisition"),
|
||||
});
|
||||
|
||||
export const submitDownload = defineTask({
|
||||
name: "submitDownload",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ acquisitionPlanId: id }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: stub("submitDownload"),
|
||||
});
|
||||
|
||||
export const verifyAndImport = defineTask({
|
||||
name: "verifyAndImport",
|
||||
version: 1,
|
||||
lane: "file",
|
||||
inputSchema: z.object({ downloadTaskId: id }),
|
||||
retryPolicy: FILE_RETRY,
|
||||
handler: stub("verifyAndImport"),
|
||||
});
|
||||
|
||||
export const refreshMetadata = defineTask({
|
||||
name: "refreshMetadata",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ mediaEntityId: id }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: stub("refreshMetadata"),
|
||||
});
|
||||
|
||||
export const reconcileLibrary = defineTask({
|
||||
name: "reconcileLibrary",
|
||||
version: 1,
|
||||
lane: "maintenance",
|
||||
inputSchema: z.object({ libraryRootId: id.optional(), reconcileRunId: id.optional() }),
|
||||
retryPolicy: FILE_RETRY,
|
||||
handler: stub("reconcileLibrary"),
|
||||
});
|
||||
|
||||
export const cleanupDownloadCache = defineTask({
|
||||
name: "cleanupDownloadCache",
|
||||
version: 1,
|
||||
lane: "file",
|
||||
inputSchema: z.object({ downloaderInstanceId: id.optional() }),
|
||||
retryPolicy: FILE_RETRY,
|
||||
handler: stub("cleanupDownloadCache"),
|
||||
});
|
||||
|
||||
export const resolveDecision = defineTask({
|
||||
name: "resolveDecision",
|
||||
version: 1,
|
||||
lane: "agent",
|
||||
inputSchema: z.object({ decisionId: id }),
|
||||
retryPolicy: DB_RETRY,
|
||||
handler: stub("resolveDecision"),
|
||||
});
|
||||
|
||||
export const TASK_NAMES = [
|
||||
"ingestFeed",
|
||||
"resolveObservation",
|
||||
"evaluateAcquisition",
|
||||
"submitDownload",
|
||||
"verifyAndImport",
|
||||
"refreshMetadata",
|
||||
"reconcileLibrary",
|
||||
"cleanupDownloadCache",
|
||||
"resolveDecision",
|
||||
] as const;
|
||||
export type TaskName = (typeof TASK_NAMES)[number];
|
||||
|
||||
/** v0.1 的默认注册表;W2 把真实 handler 换进来(同名同版本替换,或升版本并保留旧版本) */
|
||||
export function createDefaultRegistry(): TaskRegistry {
|
||||
return new TaskRegistry().register(
|
||||
ingestFeed,
|
||||
resolveObservation,
|
||||
evaluateAcquisition,
|
||||
submitDownload,
|
||||
verifyAndImport,
|
||||
refreshMetadata,
|
||||
reconcileLibrary,
|
||||
cleanupDownloadCache,
|
||||
resolveDecision,
|
||||
);
|
||||
}
|
||||
@@ -1,20 +1,6 @@
|
||||
// @umt/tasks 公开入口:TaskEngine 端口 + OpenWorkflow 实现、任务注册表、outbox 中继、操作账本 fencing
|
||||
export const PACKAGE_NAME = "@umt/tasks";
|
||||
|
||||
export {
|
||||
cleanupDownloadCache,
|
||||
createDefaultRegistry,
|
||||
evaluateAcquisition,
|
||||
ingestFeed,
|
||||
reconcileLibrary,
|
||||
refreshMetadata,
|
||||
resolveDecision,
|
||||
resolveObservation,
|
||||
submitDownload,
|
||||
TASK_NAMES,
|
||||
type TaskName,
|
||||
verifyAndImport,
|
||||
} from "./definitions.ts";
|
||||
export { FencingError, TaskCanceledError, UnknownTaskError } from "./errors.ts";
|
||||
export {
|
||||
consoleSink,
|
||||
|
||||
@@ -22,7 +22,7 @@ const WORKER = fileURLToPath(new URL("./fixtures/worker.ts", import.meta.url));
|
||||
const PKG_DIR = fileURLToPath(new URL("..", import.meta.url));
|
||||
|
||||
function startWorker(env: Record<string, string>): Promise<ChildProcess> {
|
||||
const child = spawn(process.execPath, ["run", "-A", WORKER], {
|
||||
const child = spawn(process.execPath, [WORKER], {
|
||||
cwd: PKG_DIR,
|
||||
env: { ...process.env, SCHEMA: schemas.biz, OW_SCHEMA: schemas.ow, ...env },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { createDefaultRegistry, defineTask, redact, TASK_NAMES, TaskRegistry, UnknownTaskError } from "../src/index.ts";
|
||||
import { defineTask, redact, TaskRegistry, UnknownTaskError } from "../src/index.ts";
|
||||
|
||||
describe("TaskRegistry", () => {
|
||||
it("注册任务书 §3.1-8 的九个空壳任务", () => {
|
||||
const reg = createDefaultRegistry();
|
||||
expect(reg.names()).toEqual([...TASK_NAMES].sort());
|
||||
for (const n of TASK_NAMES) expect(reg.latest(n).version).toBe(1);
|
||||
expect(reg.has("ingestFeed")).toBe(true);
|
||||
expect(reg.has("nope")).toBe(false);
|
||||
it("注册表默认为空,未注册任务明确报错", () => {
|
||||
const reg = new TaskRegistry();
|
||||
expect(reg.names()).toEqual([]);
|
||||
expect(reg.has("submitDownload")).toBe(false);
|
||||
expect(() => reg.latest("submitDownload")).toThrow(UnknownTaskError);
|
||||
});
|
||||
|
||||
it("latest 取最高版本;select 按 name@version / @version 过滤", () => {
|
||||
@@ -24,14 +23,6 @@ describe("TaskRegistry", () => {
|
||||
expect(() => reg.latest("zzz")).toThrow(UnknownTaskError);
|
||||
expect(() => mk("a", 0)).toThrow(/invalid task version/);
|
||||
});
|
||||
|
||||
it("空壳 handler 回显输入", async () => {
|
||||
const def = createDefaultRegistry().latest("submitDownload");
|
||||
const input = def.inputSchema.parse({ acquisitionPlanId: "p1" });
|
||||
const out = await def.handler({ input, logger: { info() {} } } as never);
|
||||
expect(out).toEqual({ todo: true, task: "submitDownload", input: { acquisitionPlanId: "p1" } });
|
||||
expect(() => def.inputSchema.parse({})).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("redact", () => {
|
||||
|
||||
Reference in new issue
Block a user