refactor(core): migrate to Bun and converge acquisition policies and Agent boundaries
This commit is contained in:
1 parent
3f1c6d02f6
commit
27c0f27434
268 files changed
+7468
-7852
No files matched your search
@@ -14,4 +14,4 @@
|
||||
| [`fs`](src/fs/README.md) | `copyWithVerify`、`moveSafe`、`crossDeviceMove`、`checkRoot`、`scanIncremental`、`diskUsage`、`ffprobe`、`FsProxy`;另有子路径导出 `@umt/adapters/fsworker`(`fsWorkerMain`,供 `--role fsworker`) | 校验复制、跨卷移动、库根哨兵、增量扫描;FsProxy 把文件操作放进可杀子进程(挂死 → SIGKILL → TIMEOUT) |
|
||||
| [`browser`](src/browser/README.md) | `BrowserProvider` 合同、`createNoneBrowserProvider`、camofox 适配骨架、`redactBrowserUrl` | 浏览器服务接口(M1 只有接口与骨架) |
|
||||
|
||||
测试:`deno task test`(vitest,外部服务全部用 `@umt/testkit` 的 mock)。
|
||||
测试:`bun run test`(vitest,外部服务全部用 `@umt/testkit` 的 mock)。
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "@umt/adapters",
|
||||
"version": "0.3.44",
|
||||
"exports": { ".": "./src/index.ts", "./fsworker": "./src/fs/fsworker.ts", "./alist-mock": "./src/alist/mock.ts" },
|
||||
"tasks": {
|
||||
"test": "deno run -A npm:vitest run",
|
||||
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
|
||||
}
|
||||
}
|
||||
@@ -6,5 +6,15 @@
|
||||
".": "./src/index.ts",
|
||||
"./fsworker": "./src/fs/fsworker.ts",
|
||||
"./alist-mock": "./src/alist/mock.ts"
|
||||
},
|
||||
"version": "0.3.44",
|
||||
"scripts": {
|
||||
"test": "bun x --bun vitest run",
|
||||
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
|
||||
},
|
||||
"dependencies": {
|
||||
"@umt/contracts": "workspace:*",
|
||||
"@umt/testkit": "workspace:*",
|
||||
"@umt/runtime": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -12,16 +12,16 @@ import { type AlistAuth, AlistClient } from "./client.ts";
|
||||
// UMT_ALIST_REAL_ROOT2 可选:另一个挂载(驱动)下的可写目录,测跨驱动复制 / 移动(异步任务)
|
||||
// 跑法:docs/runbook/storage-alist.md「真机验收」。
|
||||
|
||||
const URL_ = Deno.env.get("UMT_ALIST_REAL_URL");
|
||||
const ROOT = Deno.env.get("UMT_ALIST_REAL_ROOT");
|
||||
const ROOT2 = Deno.env.get("UMT_ALIST_REAL_ROOT2");
|
||||
const auth: AlistAuth | null = Deno.env.get("UMT_ALIST_REAL_TOKEN")
|
||||
? { kind: "token", token: Deno.env.get("UMT_ALIST_REAL_TOKEN")! }
|
||||
: Deno.env.get("UMT_ALIST_REAL_USER")
|
||||
const URL_ = process.env.UMT_ALIST_REAL_URL;
|
||||
const ROOT = process.env.UMT_ALIST_REAL_ROOT;
|
||||
const ROOT2 = process.env.UMT_ALIST_REAL_ROOT2;
|
||||
const auth: AlistAuth | null = process.env.UMT_ALIST_REAL_TOKEN
|
||||
? { kind: "token", token: process.env.UMT_ALIST_REAL_TOKEN! }
|
||||
: process.env.UMT_ALIST_REAL_USER
|
||||
? {
|
||||
kind: "password",
|
||||
username: Deno.env.get("UMT_ALIST_REAL_USER")!,
|
||||
password: Deno.env.get("UMT_ALIST_REAL_PASSWORD") ?? "",
|
||||
username: process.env.UMT_ALIST_REAL_USER!,
|
||||
password: process.env.UMT_ALIST_REAL_PASSWORD ?? "",
|
||||
}
|
||||
: null;
|
||||
const enabled = !!URL_ && !!ROOT && !!auth;
|
||||
|
||||
@@ -388,14 +388,17 @@ describe("上传", () => {
|
||||
|
||||
it("空闲超时:服务端不读数据时中止", async () => {
|
||||
const mock = createMockAlist({ token: "t" });
|
||||
const stuck: typeof fetch = (async (input: Request | URL | string, init?: RequestInit) => {
|
||||
const stuck: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = (async (
|
||||
input: Request | URL | string,
|
||||
init?: RequestInit,
|
||||
) => {
|
||||
const req = input instanceof Request ? input : new Request(String(input), init);
|
||||
if (new URL(req.url).pathname === "/api/fs/put")
|
||||
return new Promise<Response>((_, reject) =>
|
||||
init?.signal?.addEventListener("abort", () => reject(init.signal!.reason)),
|
||||
);
|
||||
return mock.fetch(input, init);
|
||||
}) as typeof fetch;
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
const client = new AlistClient({
|
||||
baseUrl: mock.base,
|
||||
auth: { kind: "token", token: "t" },
|
||||
@@ -453,14 +456,17 @@ describe("下载", () => {
|
||||
mock.put("/s1/a.mkv", bytes(1000));
|
||||
const realFetch = mock.fetch;
|
||||
let expired = false;
|
||||
const f: typeof fetch = (async (input: Request | URL | string, init?: RequestInit) => {
|
||||
const f: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = (async (
|
||||
input: Request | URL | string,
|
||||
init?: RequestInit,
|
||||
) => {
|
||||
const u = String(input instanceof Request ? input.url : input);
|
||||
if (u.includes("/d/") && !expired) {
|
||||
expired = true;
|
||||
mock.expireSigns();
|
||||
}
|
||||
return realFetch(input, init);
|
||||
}) as typeof fetch;
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
const c2 = new AlistClient({ baseUrl: mock.base, auth: { kind: "token", token: "perm-token" }, fetch: f, qps: 0 });
|
||||
await c2.download("/s1/a.mkv", join(dir, "a.mkv"));
|
||||
expect((await stat(join(dir, "a.mkv"))).size).toBe(1000);
|
||||
@@ -478,7 +484,10 @@ describe("下载", () => {
|
||||
const mock = createMockAlist({ token: "t" });
|
||||
mock.put("/s1/a.mkv", "x");
|
||||
const calls: { url: string; auth: string | null }[] = [];
|
||||
const f: typeof fetch = (async (input: Request | URL | string, init?: RequestInit) => {
|
||||
const f: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = (async (
|
||||
input: Request | URL | string,
|
||||
init?: RequestInit,
|
||||
) => {
|
||||
const req = input instanceof Request ? input : new Request(String(input), init);
|
||||
calls.push({ url: req.url, auth: req.headers.get("authorization") });
|
||||
if (req.url.startsWith("https://cdn.example")) return new Response("x");
|
||||
@@ -489,7 +498,7 @@ describe("下载", () => {
|
||||
return Response.json(j);
|
||||
}
|
||||
return res;
|
||||
}) as typeof fetch;
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
const c = new AlistClient({ baseUrl: mock.base, auth: { kind: "token", token: "t" }, fetch: f, qps: 0 });
|
||||
await c.download("/s1/a.mkv", join(dir, "a.mkv"));
|
||||
expect(calls.find((x) => x.url.startsWith("https://cdn.example"))!.auth).toBeNull();
|
||||
|
||||
@@ -24,7 +24,7 @@ export interface AlistClientOptions {
|
||||
auth: AlistAuth;
|
||||
/** 路径密码(Alist「元信息」里给目录设的密码),list / get 时带上 */
|
||||
pathPassword?: string | null;
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
/** 普通请求超时(ms),缺省 30 s */
|
||||
timeoutMs?: number;
|
||||
/** 数据流无进度空闲超时(ms),缺省 120 s */
|
||||
@@ -203,7 +203,7 @@ function idleWatch(ms: number, outer?: AbortSignal) {
|
||||
|
||||
export class AlistClient {
|
||||
readonly baseUrl: string;
|
||||
private readonly fetchFn: typeof fetch;
|
||||
private readonly fetchFn: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
private readonly limiter: AlistLimiter;
|
||||
private readonly o: Required<
|
||||
Pick<
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
// 令牌无效 code 401;copy 一律异步任务;同挂载 move 立即完成、跨挂载 move 是任务;put 覆盖同名;/d 支持 Range 与 sign。
|
||||
// 顶层目录视为「挂载」(驱动),用于区分同驱动 / 跨驱动。
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { type NetAddr, serve } from "@umt/runtime";
|
||||
|
||||
export interface MockFile {
|
||||
data: Uint8Array;
|
||||
@@ -43,7 +44,7 @@ export interface MockAlist {
|
||||
readonly base: string;
|
||||
setOrigin(origin: string): void;
|
||||
handler: (req: Request) => Promise<Response>;
|
||||
fetch: typeof fetch;
|
||||
fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
files: Map<string, MockFile>;
|
||||
dirs: Map<string, number>;
|
||||
tasks: Map<string, Task>;
|
||||
@@ -432,7 +433,7 @@ export function createMockAlist(opts: MockAlistOptions = {}): MockAlist {
|
||||
// 老版本(noTaskIds):没有任务 id 的复制 / 移动,在之后的任意 get 上推进
|
||||
if (opts.noTaskIds) for (const t of tasks.values()) advanceTask(t);
|
||||
return handler(req);
|
||||
}) as typeof fetch,
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
|
||||
files,
|
||||
dirs,
|
||||
tasks,
|
||||
@@ -471,13 +472,13 @@ export function createMockAlist(opts: MockAlistOptions = {}): MockAlist {
|
||||
|
||||
/** 起一个真 HTTP 监听(ffprobe / 下载代理等需要真实 URL 的测试用);返回 base 与关闭函数 */
|
||||
export function serveMockAlist(mock: MockAlist): { base: string; close: () => Promise<void> } {
|
||||
const server = Deno.serve({ port: 0, hostname: "127.0.0.1", onListen: () => {} }, (req) => {
|
||||
const server = serve({ port: 0, hostname: "127.0.0.1", onListen: () => {} }, (req) => {
|
||||
const u = new URL(req.url);
|
||||
return mock
|
||||
.handler(new Request(`${mock.base}${u.pathname}${u.search}`, req))
|
||||
.catch(() => new Response("connection reset", { status: 502 }));
|
||||
});
|
||||
const addr = server.addr as Deno.NetAddr;
|
||||
const addr = server.addr as NetAddr;
|
||||
const prev = mock.base;
|
||||
const base = `http://127.0.0.1:${addr.port}`;
|
||||
mock.setOrigin(base);
|
||||
|
||||
@@ -41,7 +41,7 @@ function realUpstream(handler: (c: Call) => Reply | undefined) {
|
||||
status: r.status ?? 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}) as typeof fetch;
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
return { calls, fetchImpl };
|
||||
}
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ export interface CamofoxConfig {
|
||||
downloadSink?: DownloadSink;
|
||||
/** waitForDownload 轮询间隔 */
|
||||
downloadPollMs?: number;
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
logger?: BrowserLogger;
|
||||
now?: () => Date;
|
||||
newId?: () => string;
|
||||
@@ -157,7 +157,7 @@ const LOGIN_PATH = /(?:^|\/)(?:login|takelogin|signin|sign_in|auth)(?:\.php)?(?:
|
||||
export class CamofoxBrowserProvider implements BrowserProvider {
|
||||
readonly kind = "camofox" as const;
|
||||
private readonly base: string;
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
private readonly lock: ProfileLock;
|
||||
private readonly sessions = new Map<string, SessionState>();
|
||||
private readonly leases = new Map<string, BrowserControlLease>();
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* 截图的解码、缩放与编码(VIS-1):交给视觉模型的截图缩放到不超过 maxWidth(缺省 1280)像素宽,
|
||||
* 证据缩略图更小。PNG 自己解码 / 编码(node:zlib),JPEG 用 jpeg-js(纯 JS,无原生依赖,deno compile 可嵌入)。
|
||||
* 证据缩略图更小。PNG 自己解码 / 编码(node:zlib),JPEG 用 jpeg-js(纯 JS,无原生依赖,bun build --compile 可嵌入)。
|
||||
* 只支持浏览器截图实际会出现的格式:8 位 PNG(灰度 / RGB / 调色板 / 带 alpha,非隔行)与基线 JPEG。
|
||||
*/
|
||||
import { deflateSync, inflateSync } from "node:zlib";
|
||||
|
||||
@@ -26,7 +26,7 @@ export interface EmbyConfig {
|
||||
retryBaseMs?: number;
|
||||
/** 搜索/分页单页大小,默认 50 */
|
||||
pageSize?: number;
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
}
|
||||
|
||||
export type Observed<T> =
|
||||
@@ -240,7 +240,7 @@ export class EmbyClient {
|
||||
readonly instanceId: string;
|
||||
private readonly base: string;
|
||||
private readonly flavor: "emby" | "jellyfin";
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
private userId: string | undefined;
|
||||
|
||||
constructor(private readonly cfg: EmbyConfig) {
|
||||
|
||||
@@ -4,13 +4,13 @@
|
||||
|
||||
## 挂死隔离:`FsProxy`(对标 MP fsproxy)
|
||||
|
||||
`new FsProxy({ hangTimeoutMs, quickTimeoutMs, progressIntervalMs, killGraceMs, mode })` 把单个文件操作(copy / crossMove / move / match / sha256 / stat / checkRoot / sweep / unlink)放进一次性执行子进程 `fsworker.ts`(`deno run --no-config -A`,约 50 ms 启动),stdin 一条 JSON 请求,stdout 逐行心跳 + 结果。
|
||||
`new FsProxy({ hangTimeoutMs, quickTimeoutMs, progressIntervalMs, killGraceMs, mode })` 把单个文件操作(copy / crossMove / move / match / sha256 / stat / checkRoot / sweep / unlink)放进一次性执行子进程 `fsworker.ts`(`bun fsworker.ts`),通过 `@umt/runtime/worker` 发送带 id 的逐行 JSON 请求、进度与结果,单条消息上限 16 MiB。
|
||||
|
||||
- 数据类操作(复制、跨卷搬运、哈希)按**进度停滞**判挂死:`hangTimeoutMs`(缺省 120 s)内字节数与阶段都没前进 → SIGTERM(worker 中止并清掉自己的 partial)→ `killGraceMs` 后 SIGKILL → `FsError('TIMEOUT')`。大文件复制几小时不会误杀。
|
||||
- 快操作按总时长 `quickTimeoutMs`(缺省 30 s)。`statSafe` 超时返回 `{kind:"error",code:"TIMEOUT"}`,`checkRoot` 超时返回 `offline(timeout)`——**挂死不等于文件缺失**。
|
||||
- 卡在 D 状态的子进程被 `unref`,不拖住本进程;被 SIGKILL 的 partial 由下次写前清理收走。
|
||||
- 子进程里 sha256 走流式(带心跳),不走 `sha256sum`(看不到进度);吞吐见 `docs/porting/moviepilot-rust-transfer.md` 基准记录。
|
||||
- `mode: "auto"`:`Deno.execPath()` 是 deno CLI → `deno run -A fsworker.ts`;否则视为 `deno compile` 出的 umt-server 二进制 → `<execPath> --role fsworker`(`apps/server/src/main.ts` 在加载其余模块前分派,启动约 150 ms)。编译产物的 `--allow-run` 必须含自身绝对路径(镜像内 `/app/umt-server`,见 `apps/server/deno.json`);起不来(NotCapable/ENOENT)自动回退 in-process(同样的看门狗,超时立即报 TIMEOUT 并中止,但卡死系统调用占住的线程无法回收),`stats().fallbackReason` 记原因。`stats().workerKind` = script / self / custom。
|
||||
- 服务端缺省 `mode: "subprocess"`,启动失败直接报错;显式选择 `mode: "auto"`:`process.execPath` 是 bun CLI → `bun fsworker.ts`;否则视为 `bun build --compile` 出的 umt-server 二进制 → `<execPath> --role fsworker`(`apps/server/src/main.ts` 在加载其余模块前分派)。子进程起不来(ENOENT/EACCES)自动回退 in-process(同样的看门狗,超时立即报 TIMEOUT 并中止,但卡死系统调用占住的线程无法回收),`stats().fallbackReason` 记原因。`stats().workerKind` = script / self / custom。
|
||||
- 容器验收开关:worker 进程设 `UMT_FS_TEST_HANG_AFTER_BYTES=<n>` 后,每个数据类操作的 worker 读到 n 字节即卡死 JS 线程并屏蔽 SIGTERM,用来在生产二进制里验证 停滞超时 → SIGKILL → TIMEOUT;生产勿设。
|
||||
- 测试开关 `_test: { hangAfterBytes, ignoreTerm }`:worker 在累计进度达到阈值后阻塞 JS 线程,模拟卡死的系统调用。
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { link, lstat, mkdir, open, readdir, rename, stat, unlink, utimes } from
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { Transform } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { Command } from "@umt/runtime";
|
||||
import { errnoOf, FsError, toFsError } from "./errors.ts";
|
||||
import { assertLocalPaths } from "./remote-guard.ts";
|
||||
|
||||
@@ -133,23 +134,10 @@ export function sha256Stats(): { counts: Record<Sha256Via, number>; subprocessUn
|
||||
return { counts: { ...hashCounters }, subprocessUnavailable: sha256sumUnavailable };
|
||||
}
|
||||
|
||||
type DenoLike = {
|
||||
Command: new (
|
||||
cmd: string,
|
||||
opts: { args: string[]; stdout: "piped"; stderr: "piped"; signal?: AbortSignal },
|
||||
) => { output(): Promise<{ success: boolean; code: number; stdout: Uint8Array; stderr: Uint8Array }> };
|
||||
errors: { NotFound: new () => Error; NotCapable?: new () => Error; PermissionDenied: new () => Error };
|
||||
};
|
||||
|
||||
async function sha256Subprocess(p: string, signal?: AbortSignal): Promise<string | null> {
|
||||
const D = (globalThis as { Deno?: DenoLike }).Deno;
|
||||
if (!D?.Command) {
|
||||
sha256sumUnavailable = "no_deno_command";
|
||||
return null;
|
||||
}
|
||||
let out: Awaited<ReturnType<InstanceType<DenoLike["Command"]>["output"]>>;
|
||||
let out: Awaited<ReturnType<Command["output"]>>;
|
||||
try {
|
||||
out = await new D.Command("sha256sum", {
|
||||
out = await new Command("sha256sum", {
|
||||
args: ["-b", "--", p],
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
@@ -157,9 +145,9 @@ async function sha256Subprocess(p: string, signal?: AbortSignal): Promise<string
|
||||
}).output();
|
||||
} catch (e) {
|
||||
if (signal?.aborted) throw new FsError("ABORTED", "aborted", { path: p, cause: e });
|
||||
const notCapable = D.errors.NotCapable && e instanceof D.errors.NotCapable;
|
||||
if (e instanceof D.errors.NotFound || e instanceof D.errors.PermissionDenied || notCapable) {
|
||||
sha256sumUnavailable = e instanceof D.errors.NotFound ? "not_installed" : "not_permitted";
|
||||
const code = (e as NodeJS.ErrnoException).code;
|
||||
if (code === "ENOENT" || code === "EACCES" || code === "EPERM") {
|
||||
sha256sumUnavailable = code === "ENOENT" ? "not_installed" : "not_permitted";
|
||||
return null;
|
||||
}
|
||||
throw toFsError(e, p);
|
||||
@@ -425,7 +413,7 @@ export async function copyWithVerify(src: string, dst: string, opts: CopyOptions
|
||||
receipt.sampleDigest = b;
|
||||
}
|
||||
opts.signal?.throwIfAborted();
|
||||
await utimes(tmp, srcStat.atime, srcStat.mtime);
|
||||
await utimes(tmp, srcStat.atimeMs / 1000, srcStat.mtimeMs / 1000);
|
||||
if (existing) receipt.replaced = await replaceCommit(tmp, dst, existing, opts.replaceOld);
|
||||
else await commitNoClobber(tmp, dst);
|
||||
await fsyncPath(dirname(dst));
|
||||
|
||||
@@ -108,3 +108,6 @@ export async function runFsOp(req: FsOpRequest, ctx: FsOpContext): Promise<unkno
|
||||
throw new FsError("IO", `unknown fs op: ${(req as { op: string }).op}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** 文件请求、扫描结果的单条协议消息上限。文件内容不通过管道传输。 */
|
||||
export const FS_MESSAGE_BYTES = 16 * 1024 * 1024;
|
||||
@@ -132,7 +132,7 @@ describe("FsProxy(子进程)", () => {
|
||||
});
|
||||
|
||||
describe("FsProxy 启动方式", () => {
|
||||
it("deno CLI 下缺省 run fsworker.ts;显式 workerCommand 记为 custom", () => {
|
||||
it("bun CLI 下缺省 run fsworker.ts;显式 workerCommand 记为 custom", () => {
|
||||
const auto = defaultWorkerCommand();
|
||||
expect(auto?.kind).toBe("script");
|
||||
expect(auto?.args.at(-1)).toMatch(/fsworker\.ts$/);
|
||||
@@ -148,6 +148,13 @@ describe("FsProxy 启动方式", () => {
|
||||
expect(px.stats()).toMatchObject({ mode: "inprocess", fallbackReason: expect.stringMatching(/^spawn_failed/) });
|
||||
});
|
||||
|
||||
it("强制子进程时启动失败报错,不在主进程执行文件操作", async () => {
|
||||
const t = await tree({ "a.mkv": "x" });
|
||||
const px = new FsProxy({ mode: "subprocess", workerCommand: { cmd: "/nonexistent/umt-worker", args: [] } });
|
||||
expect(await px.statSafe(t.path("a.mkv"))).toMatchObject({ kind: "error", code: "IO" });
|
||||
expect(px.stats()).toMatchObject({ mode: "subprocess", fallbackReason: null });
|
||||
});
|
||||
|
||||
it("验收开关只在 UMT_FS_TEST_HOOKS=1 时生效(单独误设 HANG_AFTER_BYTES 无效)", () => {
|
||||
const env = (m: Record<string, string>) => (k: string) => m[k];
|
||||
expect(fsTestHookFromEnv(env({ UMT_FS_TEST_HANG_AFTER_BYTES: "1024" }))).toBeUndefined();
|
||||
@@ -162,16 +169,16 @@ describe("FsProxy 启动方式", () => {
|
||||
it("UMT_FS_TEST_HANG_AFTER_BYTES(容器验收开关)→ 数据类操作卡死 → SIGKILL → TIMEOUT;快操作不受影响", async () => {
|
||||
const t = await tree({ "src/a.mkv": 3 * MiB, "dst/": null });
|
||||
const px = new FsProxy({ mode: "subprocess", ...fast });
|
||||
Deno.env.set("UMT_FS_TEST_HANG_AFTER_BYTES", String(MiB));
|
||||
Deno.env.set("UMT_FS_TEST_HOOKS", "1");
|
||||
process.env.UMT_FS_TEST_HANG_AFTER_BYTES = String(MiB);
|
||||
process.env.UMT_FS_TEST_HOOKS = "1";
|
||||
try {
|
||||
expect(await px.statSafe(t.path("src/a.mkv"))).toMatchObject({ kind: "ok" });
|
||||
const e = await px.copy(t.path("src/a.mkv"), t.path("dst/a.mkv")).catch((x) => x);
|
||||
expect(e.code).toBe("TIMEOUT");
|
||||
expect(px.stats()).toMatchObject({ timeouts: 1, killed: 1 });
|
||||
} finally {
|
||||
Deno.env.delete("UMT_FS_TEST_HANG_AFTER_BYTES");
|
||||
Deno.env.delete("UMT_FS_TEST_HOOKS");
|
||||
delete process.env.UMT_FS_TEST_HANG_AFTER_BYTES;
|
||||
delete process.env.UMT_FS_TEST_HOOKS;
|
||||
}
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
@@ -5,18 +5,20 @@
|
||||
// 统一转成 FsError('TIMEOUT')——普通可重试 IO 错误,交给任务层退避重试。卡在 D 状态的子进程被 unref,不拖住本进程。
|
||||
// stat/checkRoot 超时返回 error/offline(timeout),**不**当成文件缺失(REQ §2.1 原则 4)。
|
||||
//
|
||||
// 与 MP 的差异:MP 是常驻 worker + 一把锁串行;我们每个操作一个子进程(~50 ms 启动,入库按文件计可忽略),
|
||||
// 与 MP 的差异:MP 是常驻 worker + 一把锁串行;我们每个操作一个子进程,
|
||||
// 天然并发、互不拖累,也没有"上一个请求挂死把 worker 占住"的问题。
|
||||
// 启动方式:execPath 是 deno CLI(开发/测试)→ `deno run -A fsworker.ts`;否则视为 deno compile 出的 umt-server
|
||||
// 二进制 → `<execPath> --role fsworker`(入口 main.ts 在加载其余模块前分派)。编译产物的 --allow-run 须含自身绝对路径
|
||||
// (镜像内 /app/umt-server,见 apps/server/deno.json);起不来(NotCapable 等)回退 in-process:
|
||||
// 启动方式:execPath 是 bun CLI(开发/测试)→ `bun fsworker.ts`;否则视为 bun build --compile 出的 umt-server
|
||||
// 二进制 → `<execPath> --role fsworker`(入口 main.ts 在加载其余模块前分派)。
|
||||
// 子进程无法启动时回退 in-process:
|
||||
// 同样的心跳看门狗,超时立即以 TIMEOUT 失败并中止操作,但卡死的系统调用占住的线程无法回收(见 README)。
|
||||
import { basename } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { WorkerConnection, WorkerTransportError } from "@umt/runtime/worker";
|
||||
import type { CopyReceipt, Sha256Result } from "./copy.ts";
|
||||
import { FsError, type FsErrorCode } from "./errors.ts";
|
||||
import {
|
||||
DATA_OPS,
|
||||
FS_MESSAGE_BYTES,
|
||||
type FsOpRequest,
|
||||
type FsProgress,
|
||||
type FsTestHook,
|
||||
@@ -32,7 +34,7 @@ import type { RootHealth, StatResult } from "./stat.ts";
|
||||
export type { FsProgress } from "./fsops.ts";
|
||||
|
||||
export interface FsProxyConfig {
|
||||
/** auto:能起 deno 子进程就用子进程,否则 in-process;subprocess 强制子进程(起不来报错) */
|
||||
/** auto:能起 bun 子进程就用子进程,否则 in-process;subprocess 强制子进程(起不来报错) */
|
||||
mode?: "auto" | "subprocess" | "inprocess";
|
||||
/** 数据类操作:进度停滞多久判挂死(缺省 120 s,同 MP FS_PROXY_STALL_TIMEOUT) */
|
||||
hangTimeoutMs?: number;
|
||||
@@ -62,15 +64,6 @@ export interface FsProxyStats {
|
||||
fallbackReason: string | null;
|
||||
}
|
||||
|
||||
type DenoChild = {
|
||||
stdin: WritableStream<Uint8Array>;
|
||||
stdout: ReadableStream<Uint8Array>;
|
||||
stderr: ReadableStream<Uint8Array>;
|
||||
status: Promise<{ success: boolean; code: number; signal: string | null }>;
|
||||
kill(sig?: string): void;
|
||||
unref(): void;
|
||||
};
|
||||
|
||||
const WORKER_PATH = (() => {
|
||||
try {
|
||||
const u = new URL("./fsworker.ts", import.meta.url);
|
||||
@@ -80,31 +73,12 @@ const WORKER_PATH = (() => {
|
||||
}
|
||||
})();
|
||||
|
||||
/** 当前运行时能用的缺省 worker 启动命令:deno CLI → run 脚本;编译二进制 → 自身 --role fsworker;都不行 → null */
|
||||
/** 当前运行时能用的缺省 worker 启动命令:bun CLI → run 脚本;编译二进制 → 自身 --role fsworker;都不行 → null */
|
||||
export function defaultWorkerCommand(): { cmd: string; args: string[]; kind: "script" | "self" } | null {
|
||||
const D = (globalThis as { Deno?: { execPath?: () => string; Command?: unknown } }).Deno;
|
||||
if (!D?.Command || !D.execPath) return null;
|
||||
let exe: string;
|
||||
try {
|
||||
exe = D.execPath();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (/^deno(\.exe)?$/.test(basename(exe))) {
|
||||
const exe = process.execPath;
|
||||
if (/^bun(\.exe)?$/.test(basename(exe))) {
|
||||
if (!WORKER_PATH) return null;
|
||||
return {
|
||||
cmd: exe,
|
||||
args: [
|
||||
"run",
|
||||
"--no-config",
|
||||
"--quiet",
|
||||
"--no-prompt",
|
||||
// Deno 对 /dev、/proc 下的路径(如 /dev/shm 上的库根)要求 --allow-all;worker 只执行本包固定的操作表
|
||||
"--allow-all",
|
||||
WORKER_PATH,
|
||||
],
|
||||
kind: "script",
|
||||
};
|
||||
return { cmd: exe, args: [WORKER_PATH], kind: "script" };
|
||||
}
|
||||
return { cmd: exe, args: ["--role", "fsworker"], kind: "self" };
|
||||
}
|
||||
@@ -122,12 +96,7 @@ export function fsTestHookFromEnv(get: (k: string) => string | undefined): FsTes
|
||||
}
|
||||
|
||||
function envTestHook(): FsTestHook | undefined {
|
||||
try {
|
||||
const env = (globalThis as { Deno?: { env?: { get(k: string): string | undefined } } }).Deno?.env;
|
||||
return env ? fsTestHookFromEnv((k) => env.get(k)) : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
return fsTestHookFromEnv((k) => process.env[k]);
|
||||
}
|
||||
|
||||
const timeoutError = (op: string, why: string, path?: string) =>
|
||||
@@ -154,7 +123,7 @@ function pathOf(req: FsOpRequest): string | undefined {
|
||||
|
||||
export class FsProxy {
|
||||
readonly cfg: Required<Omit<FsProxyConfig, "workerCommand" | "mode">> & Pick<FsProxyConfig, "workerCommand">;
|
||||
/** 子进程怎么起:script = deno run fsworker.ts;self = 编译二进制 --role fsworker;custom = 显式 workerCommand */
|
||||
/** 子进程怎么起:script = bun fsworker.ts;self = 编译二进制 --role fsworker;custom = 显式 workerCommand */
|
||||
readonly workerKind: "script" | "self" | "custom" | null;
|
||||
#mode: "subprocess" | "inprocess";
|
||||
#forced: boolean;
|
||||
@@ -177,9 +146,9 @@ export class FsProxy {
|
||||
if (mode === "inprocess") this.#mode = "inprocess";
|
||||
else if (wc) this.#mode = "subprocess";
|
||||
else {
|
||||
if (mode === "subprocess") throw new Error("fsproxy: 当前运行时无法启动 deno 子进程,且未提供 workerCommand");
|
||||
if (mode === "subprocess") throw new Error("fsproxy: 当前运行时无法启动 bun 子进程,且未提供 workerCommand");
|
||||
this.#mode = "inprocess";
|
||||
this.#fallbackReason = "no_deno_cli";
|
||||
this.#fallbackReason = "no_bun_cli";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,7 +220,8 @@ export class FsProxy {
|
||||
try {
|
||||
return await this.#runSubprocess(req, call);
|
||||
} catch (e) {
|
||||
if (!(e instanceof SpawnError) || this.#forced) throw e instanceof SpawnError ? e.cause : e;
|
||||
if (!(e instanceof SpawnError)) throw e;
|
||||
if (this.#forced) throw new FsError("IO", `文件 worker 无法启动:${e.message}`, { cause: e.cause });
|
||||
this.#mode = "inprocess";
|
||||
this.#fallbackReason = e.message;
|
||||
}
|
||||
@@ -310,144 +280,72 @@ export class FsProxy {
|
||||
}
|
||||
}
|
||||
|
||||
async #runSubprocess(req: FsOpRequest, call: FsCallOptions): Promise<unknown> {
|
||||
const D = (globalThis as unknown as { Deno: DenoLike }).Deno;
|
||||
const cmd = this.cfg.workerCommand!;
|
||||
let child: DenoChild;
|
||||
async #runSubprocess(req: FsOpRequest, call: FsCallOptions = {}): Promise<unknown> {
|
||||
let connection: WorkerConnection;
|
||||
try {
|
||||
child = new D.Command(cmd.cmd, {
|
||||
args: cmd.args,
|
||||
stdin: "piped",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
env: { UMT_FS_PROGRESS_INTERVAL_MS: String(this.cfg.progressIntervalMs) },
|
||||
}).spawn();
|
||||
} catch (e) {
|
||||
throw new SpawnError(`spawn_failed:${(e as Error)?.name ?? "Error"}`, e);
|
||||
connection = new WorkerConnection(this.cfg.workerCommand!, {
|
||||
values: { UMT_FS_PROGRESS_INTERVAL_MS: String(this.cfg.progressIntervalMs) },
|
||||
});
|
||||
} catch (error) {
|
||||
throw new SpawnError(`spawn_failed:${(error as Error).name}`, error);
|
||||
}
|
||||
this.#stats.spawned += 1;
|
||||
const limit = this.#limitFor(req);
|
||||
const w = child.stdin.getWriter();
|
||||
let last = Date.now();
|
||||
let lastKey = "";
|
||||
const terminate = (error: FsError) => connection.terminate(error, this.cfg.killGraceMs, () => this.#stats.killed++);
|
||||
const onAbort = () => terminate(new FsError("ABORTED", "aborted", { path: pathOf(req) }));
|
||||
const watch = setInterval(
|
||||
() => {
|
||||
if (Date.now() - last > limit.ms) {
|
||||
this.#stats.timeouts += 1;
|
||||
clearInterval(watch);
|
||||
terminate(timeoutError(req.op, limit.why, pathOf(req)));
|
||||
}
|
||||
},
|
||||
Math.max(10, Math.min(1000, limit.ms / 4)),
|
||||
);
|
||||
call.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
const hook = call._test ?? (DATA_OPS.has(req.op) ? envTestHook() : undefined);
|
||||
await w.write(new TextEncoder().encode(JSON.stringify(hook ? { ...req, _test: hook } : req)));
|
||||
await w.close().catch(() => undefined);
|
||||
|
||||
let stderr = "";
|
||||
const errPump = (async () => {
|
||||
const dec = new TextDecoder();
|
||||
try {
|
||||
for await (const c of child.stderr) stderr = (stderr + dec.decode(c, { stream: true })).slice(-2000);
|
||||
} catch {
|
||||
// 被 kill 时流会出错
|
||||
}
|
||||
})();
|
||||
|
||||
return await new Promise<unknown>((resolve, reject) => {
|
||||
let settled = false;
|
||||
let last = Date.now();
|
||||
let lastKey = "";
|
||||
let killTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
/** 已判超时/已取消:之后 worker 自己报的结果(如 SIGTERM 后的 ABORTED)一律以此为准 */
|
||||
let terminating: FsError | null = null;
|
||||
const finish = (fn: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearInterval(watch);
|
||||
call.signal?.removeEventListener("abort", onAbort);
|
||||
fn();
|
||||
};
|
||||
const terminate = (err: FsError) => {
|
||||
if (settled || terminating) return;
|
||||
terminating = err;
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
// 已退出
|
||||
}
|
||||
clearInterval(watch);
|
||||
killTimer = setTimeout(() => {
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
this.#stats.killed += 1;
|
||||
} catch {
|
||||
// SIGTERM 后已退出
|
||||
}
|
||||
// D 状态的子进程可能连 SIGKILL 都要等系统调用返回:不等它,unref 后直接报错
|
||||
child.unref();
|
||||
finish(() => reject(err));
|
||||
}, this.cfg.killGraceMs);
|
||||
};
|
||||
const onAbort = () =>
|
||||
terminate(new FsError("ABORTED", "aborted", { ...(pathOf(req) ? { path: pathOf(req) } : {}) }));
|
||||
call.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
const watch = setInterval(
|
||||
() => {
|
||||
if (Date.now() - last > limit.ms) {
|
||||
this.#stats.timeouts += 1;
|
||||
terminate(timeoutError(req.op, limit.why, pathOf(req)));
|
||||
}
|
||||
try {
|
||||
const pending = connection.request(
|
||||
{
|
||||
id: "1",
|
||||
task: req.op,
|
||||
input: hook ? { ...req, _test: hook } : req,
|
||||
},
|
||||
Math.max(10, Math.min(1000, limit.ms / 4)),
|
||||
);
|
||||
|
||||
(async () => {
|
||||
const dec = new TextDecoder();
|
||||
let buf = "";
|
||||
let final: WorkerFinal | null = null;
|
||||
try {
|
||||
for await (const c of child.stdout) {
|
||||
buf += dec.decode(c, { stream: true });
|
||||
let i = buf.indexOf("\n");
|
||||
while (i >= 0) {
|
||||
const line = buf.slice(0, i);
|
||||
buf = buf.slice(i + 1);
|
||||
i = buf.indexOf("\n");
|
||||
if (!line.trim()) continue;
|
||||
const msg = JSON.parse(line) as WorkerFinal & FsProgress;
|
||||
if (msg.t === "hb") {
|
||||
const key = `${msg.phase}:${msg.bytes}`;
|
||||
if (!limit.stall || key !== lastKey) {
|
||||
lastKey = key;
|
||||
if (limit.stall) last = Date.now();
|
||||
}
|
||||
call.onProgress?.({ phase: msg.phase, bytes: msg.bytes, total: msg.total });
|
||||
} else final = msg;
|
||||
{
|
||||
maxRequestBytes: FS_MESSAGE_BYTES,
|
||||
maxResponseBytes: FS_MESSAGE_BYTES,
|
||||
onProgress: (value) => {
|
||||
const p = value as FsProgress;
|
||||
if (!p || typeof p.phase !== "string" || !Number.isFinite(p.bytes) || !Number.isFinite(p.total))
|
||||
throw new Error("文件 worker 进度格式无效");
|
||||
const key = `${p.phase}:${p.bytes}`;
|
||||
if (limit.stall && key !== lastKey) {
|
||||
lastKey = key;
|
||||
last = Date.now();
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// kill 时 stdout 断开
|
||||
}
|
||||
const status = await child.status;
|
||||
await errPump;
|
||||
if (terminating) {
|
||||
// SIGTERM 生效(worker 清理后退出)就不必等到宽限期满
|
||||
clearTimeout(killTimer);
|
||||
const err = terminating;
|
||||
return finish(() => reject(err));
|
||||
}
|
||||
if (final?.t === "ok") return finish(() => resolve(final!.result));
|
||||
if (final?.t === "err")
|
||||
return finish(() =>
|
||||
reject(
|
||||
new FsError(final!.code ?? "IO", final!.message ?? "fs worker error", {
|
||||
...(final!.path ? { path: final!.path } : {}),
|
||||
}),
|
||||
),
|
||||
);
|
||||
finish(() =>
|
||||
reject(
|
||||
new FsError("IO", `fs worker 异常退出(code=${status.code} signal=${status.signal}):${stderr.trim()}`, {
|
||||
...(pathOf(req) ? { path: pathOf(req) } : {}),
|
||||
}),
|
||||
),
|
||||
);
|
||||
})();
|
||||
});
|
||||
call.onProgress?.(p);
|
||||
},
|
||||
},
|
||||
);
|
||||
if (call.signal?.aborted) onAbort();
|
||||
const reply = await pending;
|
||||
if (reply.type === "result") return reply.result;
|
||||
throw new FsError(reply.error.code as FsErrorCode, reply.error.message, reply.error.detail);
|
||||
} catch (error) {
|
||||
if (error instanceof FsError) throw error;
|
||||
if (error instanceof WorkerTransportError && error.kind === "spawn")
|
||||
throw new SpawnError(`spawn_failed:${(error.cause as Error)?.name ?? "Error"}`, error.cause);
|
||||
throw new FsError("IO", (error as Error).message, { path: pathOf(req), cause: error });
|
||||
} finally {
|
||||
clearInterval(watch);
|
||||
call.signal?.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type WorkerFinal = { t: string; result?: unknown; code?: FsErrorCode; message?: string; path?: string };
|
||||
|
||||
class SpawnError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
@@ -457,20 +355,6 @@ class SpawnError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
type DenoLike = {
|
||||
execPath(): string;
|
||||
Command: new (
|
||||
cmd: string,
|
||||
opts: {
|
||||
args: string[];
|
||||
stdin: "piped";
|
||||
stdout: "piped";
|
||||
stderr: "piped";
|
||||
env?: Record<string, string>;
|
||||
},
|
||||
) => { spawn(): DenoChild };
|
||||
};
|
||||
|
||||
let shared: FsProxy | null = null;
|
||||
/** 进程级缺省代理(配置由 server 启动时 configureFsProxy 注入;未配置时用缺省阈值) */
|
||||
export function fsProxy(): FsProxy {
|
||||
|
||||
@@ -1,34 +1,25 @@
|
||||
// fsproxy 执行子进程:从 stdin 读一条 JSON 请求,执行单个文件操作,stdout 逐行 JSON 上报
|
||||
// {t:"hb",phase,bytes,total}(按 progressIntervalMs 节流,阶段切换与阶段完成立即报)/ {t:"ok",result} / {t:"err",code,message}。
|
||||
// SIGTERM → 中止当前操作(copy 会清掉自己的 partial)后退出;挂死在系统调用里时由父进程 SIGKILL 回收。
|
||||
// 只依赖 node 内置模块,`deno run --no-config` 毫秒级启动。编译二进制可在入口按 --role fsworker 调 fsWorkerMain()。
|
||||
import { decodeWorkerRequest, readWorkerRequestText, writeWorkerMessage } from "@umt/runtime/worker";
|
||||
import { toFsError } from "./errors.ts";
|
||||
import { type FsOpRequest, type FsProgress, runFsOp } from "./fsops.ts";
|
||||
|
||||
const enc = new TextEncoder();
|
||||
|
||||
function send(obj: unknown): void {
|
||||
try {
|
||||
Deno.stdout.writeSync(enc.encode(`${JSON.stringify(obj)}\n`));
|
||||
} catch {
|
||||
// 父进程已退出:无处可报
|
||||
}
|
||||
}
|
||||
import { FS_MESSAGE_BYTES, type FsOpRequest, type FsProgress, runFsOp } from "./fsops.ts";
|
||||
|
||||
// 一次性文件 worker;固定操作分派、进度停滞策略与文件清理语义保留在文件模块。
|
||||
/** 阻塞 JS 线程(测试用,模拟卡死的系统调用:信号监听器也不会运行) */
|
||||
function blockForever(): never {
|
||||
const a = new Int32Array(new SharedArrayBuffer(4));
|
||||
for (;;) Atomics.wait(a, 0, 0);
|
||||
}
|
||||
|
||||
export async function fsWorkerMain(progressIntervalMs = Number(Deno.env.get("UMT_FS_PROGRESS_INTERVAL_MS") ?? 1000)) {
|
||||
const req = JSON.parse(await new Response(Deno.stdin.readable).text()) as FsOpRequest;
|
||||
export async function fsWorkerMain(progressIntervalMs = Number(process.env.UMT_FS_PROGRESS_INTERVAL_MS ?? 1000)) {
|
||||
const request = decodeWorkerRequest(await readWorkerRequestText(FS_MESSAGE_BYTES));
|
||||
const req = request.input as FsOpRequest;
|
||||
if (!req || req.op !== request.task) throw new Error("文件 worker 任务与操作不一致");
|
||||
const send = (value: FsProgress) => writeWorkerMessage({ id: request.id, type: "progress", value }, FS_MESSAGE_BYTES);
|
||||
const ac = new AbortController();
|
||||
Deno.addSignalListener("SIGTERM", () => {
|
||||
process.on("SIGTERM", () => {
|
||||
if (req._test?.ignoreTerm) return;
|
||||
ac.abort();
|
||||
// 操作卡在系统调用里时 abort 解不开:给清理一点时间后直接退出
|
||||
setTimeout(() => Deno.exit(143), 500);
|
||||
setTimeout(() => process.exit(143), 500);
|
||||
});
|
||||
let lastAt = 0;
|
||||
let lastPhase = "";
|
||||
@@ -45,28 +36,29 @@ export async function fsWorkerMain(progressIntervalMs = Number(Deno.env.get("UMT
|
||||
) {
|
||||
lastAt = now;
|
||||
lastPhase = p.phase;
|
||||
send({ t: "hb", ...p });
|
||||
send(p);
|
||||
}
|
||||
if (hang !== undefined && p.bytes >= hang) blockForever();
|
||||
};
|
||||
if (req._test?.hangAfterBytes === 0) {
|
||||
send({ t: "hb", phase: "start", bytes: 0, total: 0 });
|
||||
send({ phase: "start", bytes: 0, total: 0 });
|
||||
blockForever();
|
||||
}
|
||||
try {
|
||||
const result = await runFsOp(req, { signal: ac.signal, progress });
|
||||
send({ t: "ok", result });
|
||||
writeWorkerMessage({ id: request.id, type: "result", result }, FS_MESSAGE_BYTES);
|
||||
} catch (e) {
|
||||
const fe = toFsError(e);
|
||||
send({
|
||||
t: "err",
|
||||
code: ac.signal.aborted ? "ABORTED" : fe.code,
|
||||
message: fe.message,
|
||||
...(fe.detail.path ? { path: fe.detail.path } : {}),
|
||||
...(fe.detail.syscall ? { syscall: fe.detail.syscall } : {}),
|
||||
});
|
||||
writeWorkerMessage(
|
||||
{
|
||||
id: request.id,
|
||||
type: "error",
|
||||
error: { code: ac.signal.aborted ? "ABORTED" : fe.code, message: fe.message, detail: fe.detail },
|
||||
},
|
||||
FS_MESSAGE_BYTES,
|
||||
);
|
||||
}
|
||||
Deno.exit(0);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (import.meta.main) await fsWorkerMain();
|
||||
@@ -44,7 +44,7 @@ export interface MetaRequestCache {
|
||||
}
|
||||
|
||||
export interface MetaClientOptions {
|
||||
fetch?: typeof globalThis.fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
/** 缺省 15s */
|
||||
timeoutMs?: number;
|
||||
/** API 地址(bgm api.bgm.tv / 豆瓣 rexxar) */
|
||||
|
||||
@@ -39,7 +39,7 @@ export interface QbInstanceConfig {
|
||||
sequentialDownload?: boolean;
|
||||
firstLastPiecePrio?: boolean;
|
||||
/** 注入 fetch(测试/代理) */
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
}
|
||||
|
||||
export type QbAuthMode = "password" | "api_key" | "none";
|
||||
@@ -314,7 +314,7 @@ export class QbClient implements DownloaderClient {
|
||||
private readonly retryBaseMs: number;
|
||||
private readonly lookup: { attempts: number; intervalMs: number };
|
||||
private readonly probePrefix: string;
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
private sid: string | null = null;
|
||||
private loginInFlight: Promise<void> | null = null;
|
||||
/** qB 5.x 把 pause/resume 改名为 stop/start;探测到后缓存 */
|
||||
|
||||
@@ -7,7 +7,7 @@ import { redactText, redactUrl } from "./redact.ts";
|
||||
import { FeedError, type FeedFetchResult, type FeedFetchState } from "./types.ts";
|
||||
|
||||
export interface RssFetcherOptions {
|
||||
fetch?: typeof globalThis.fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
now?: () => Date;
|
||||
/** 单次请求超时,默认 30s(MP `timeout or 30`) */
|
||||
timeoutMs?: number;
|
||||
@@ -31,7 +31,7 @@ export interface FetchFeedOptions {
|
||||
/** 本次请求的 UA(覆盖 RssFetcherOptions.userAgent;BR-4 按站点账号取) */
|
||||
userAgent?: string;
|
||||
/** 本次请求用的 fetch(覆盖 RssFetcherOptions.fetch;BR-4 站点代理 = 与站点其它请求同一出口) */
|
||||
fetch?: typeof globalThis.fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
}
|
||||
|
||||
export function initialFeedState(): FeedFetchState {
|
||||
|
||||
@@ -22,7 +22,7 @@ const CF_JS_PAGE = `<html><head><title>Just a moment...</title></head><body>
|
||||
<script>window._cf_chl_opt={cType: 'non-interactive'}</script></body></html>`;
|
||||
const NORMAL = `<html><head><title>种子 - 观众</title><script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script></head><body>ok</body></html>`;
|
||||
|
||||
function ctx(fetchImpl: typeof fetch) {
|
||||
function ctx(fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>) {
|
||||
return {
|
||||
siteId: "s1",
|
||||
fetch: fetchImpl,
|
||||
@@ -32,7 +32,10 @@ function ctx(fetchImpl: typeof fetch) {
|
||||
};
|
||||
}
|
||||
const reply = (status: number, body: string, headers: Record<string, string> = {}) =>
|
||||
(async () => new Response(body, { status, headers: { "content-type": "text/html", ...headers } })) as typeof fetch;
|
||||
(async () => new Response(body, { status, headers: { "content-type": "text/html", ...headers } })) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>;
|
||||
|
||||
describe("challengeRuleOfHtml", () => {
|
||||
it("挑战页标题 / 脚本特征命中;正常页面里注入的 challenge-platform 脚本不算", () => {
|
||||
@@ -62,7 +65,7 @@ describe("detectChallenge", () => {
|
||||
|
||||
describe("siteFetch 挑战分类", () => {
|
||||
const url = "https://audiences.example/torrents.php";
|
||||
const kind = async (f: typeof fetch) => {
|
||||
const kind = async (f: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>) => {
|
||||
try {
|
||||
await siteFetch(ctx(f), url);
|
||||
return "ok";
|
||||
|
||||
@@ -130,7 +130,7 @@ async function peekBody(
|
||||
res: Response,
|
||||
max: number,
|
||||
): Promise<{ head: Uint8Array<ArrayBuffer>; complete: boolean; res: Response }> {
|
||||
const reader = (res.body as ReadableStream<Uint8Array>).getReader();
|
||||
const reader = (res.body as unknown as ReadableStream<Uint8Array>).getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
let complete = false;
|
||||
|
||||
@@ -5,7 +5,7 @@ import type { SecretResolver } from "./types.ts";
|
||||
|
||||
export interface SiteAdapterDeps {
|
||||
secrets: SecretResolver;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
/** 同站共享的限流器;不传则按站点定义新建(列表/搜索/详情/下载共享) */
|
||||
limiter?: SiteRateLimiter;
|
||||
now?: () => Date;
|
||||
|
||||
@@ -6,7 +6,7 @@ import { SiteError } from "./types.ts";
|
||||
|
||||
export interface SiteHttpContext {
|
||||
siteId: string;
|
||||
fetch: typeof globalThis.fetch;
|
||||
fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
limiter: SiteRateLimiter;
|
||||
timeoutMs: number;
|
||||
userAgent?: string;
|
||||
|
||||
@@ -53,7 +53,7 @@ export interface NexusLoginInput {
|
||||
/** base32 TOTP 密钥 */
|
||||
totpSecret?: string | null;
|
||||
userAgent: string;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
limiter?: { acquire(signal?: AbortSignal): Promise<void> };
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
|
||||
@@ -38,7 +38,7 @@ function adapter(body: string, status = 200, headers: Record<string, string> = {
|
||||
new Response(body, {
|
||||
status,
|
||||
headers: { "content-type": "text/html; charset=utf-8", ...headers },
|
||||
})) as typeof fetch;
|
||||
})) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
const a = createSpiderSiteAdapter(audiencesDefinition, {
|
||||
baseUrl: "https://aud.test/",
|
||||
secrets: staticSecretResolver({ [audiencesDefinition.id]: { cookie: "c_secure_uid=1; c_secure_pass=abc" } }),
|
||||
|
||||
@@ -176,7 +176,7 @@ describe("详情页扩展内容(憨憨 2026 真实结构:分节 div,无 #kdescr)
|
||||
paths.push(`${u.pathname}${u.search}`);
|
||||
const body = u.pathname === "/viewfilelist.php" ? HHANCLUB_FILELIST_FRAGMENT : HHANCLUB_DETAIL_PAGE_2026;
|
||||
return new Response(body, { status: 200, headers: { "content-type": "text/html; charset=utf-8" } });
|
||||
}) as typeof globalThis.fetch;
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
const limiter = createSiteRateLimiter({ minIntervalMs: 0 });
|
||||
const a = createSpiderSiteAdapter(hhanclubDefinition, {
|
||||
fetch,
|
||||
|
||||
@@ -30,7 +30,7 @@ export function testDef(
|
||||
export function adapterWith(
|
||||
def: SpecializedSiteDefinition,
|
||||
spider: SpecializedSpider,
|
||||
fetchImpl: typeof globalThis.fetch,
|
||||
fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
|
||||
secrets: Partial<Record<SiteCredentialKind, string>> = { cookie: "c=COOKIESECRET", apiKey: "APIKEYSECRET" },
|
||||
): SiteAdapter {
|
||||
return createSpecializedSiteAdapter(def, spider, {
|
||||
@@ -42,10 +42,14 @@ export function adapterWith(
|
||||
|
||||
/** 包一层 fetch:命中 predicate 的请求返回合法 .torrent 字节 */
|
||||
export function withTorrent(
|
||||
inner: typeof globalThis.fetch,
|
||||
inner: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
|
||||
predicate: (url: URL) => boolean,
|
||||
name = "demo",
|
||||
): { fetch: typeof globalThis.fetch; infoHash: string; hits: Array<{ url: URL; headers: Headers }> } {
|
||||
): {
|
||||
fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
infoHash: string;
|
||||
hits: Array<{ url: URL; headers: Headers }>;
|
||||
} {
|
||||
const t = buildFakeTorrent({ name });
|
||||
const hits: Array<{ url: URL; headers: Headers }> = [];
|
||||
const f = (async (input: string | URL | Request, init?: RequestInit) => {
|
||||
@@ -58,7 +62,7 @@ export function withTorrent(
|
||||
});
|
||||
}
|
||||
return inner(input, init);
|
||||
}) as typeof globalThis.fetch;
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
return { fetch: f, infoHash: t.infoHash, hits };
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import type { SiteUserInfo, SiteUserParserOptions } from "./types.ts";
|
||||
|
||||
export async function runParser(
|
||||
cls: SiteParserClass,
|
||||
opts: Partial<SiteUserParserOptions> & { fetch: typeof globalThis.fetch },
|
||||
opts: Partial<SiteUserParserOptions> & { fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> },
|
||||
): Promise<SiteUserInfo> {
|
||||
const p = new cls({ siteName: "demo", url: "https://pt.example.com/", ...opts });
|
||||
await p.parse();
|
||||
|
||||
@@ -92,7 +92,7 @@ export interface SiteUserParserOptions {
|
||||
apiUrl?: string | null;
|
||||
/** 是否读取未读消息正文(MP 全局设置 `SITE_MESSAGE`);默认 false。注意 MTorrent 读取后会标记已读 */
|
||||
fetchMessages?: boolean;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
/** 单请求超时(MP 固定 60s) */
|
||||
timeoutMs?: number;
|
||||
/** 共享站点限流器(与 list/search 共用) */
|
||||
|
||||
@@ -119,7 +119,7 @@ export interface TmdbConfig {
|
||||
* 语义对照 MP @cached(skip_none、业务失败不缓存、results 为空按 emptyTtlMs)。
|
||||
*/
|
||||
sharedCache?: TmdbSharedCache;
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
@@ -185,7 +185,7 @@ export class TmdbClient {
|
||||
private readonly langs: string[];
|
||||
private readonly cache: TtlLruCache<unknown>;
|
||||
private readonly limiter: RequestLimiter;
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
|
||||
constructor(private readonly cfg: TmdbConfig) {
|
||||
if (!cfg.apiKey && !cfg.accessToken) throw new TypeError("TMDB apiKey or accessToken required");
|
||||
|
||||
@@ -219,7 +219,7 @@ describe("TmdbClient 缓存/限流/错误分类", () => {
|
||||
const e1 = await dead.details("movie", 1).catch((x) => x);
|
||||
expect(e1.kind).toBe("network");
|
||||
// 模拟把 URL(含 api_key 参数与编码后的明文)拼进信息的 fetch 实现
|
||||
const leaky: typeof fetch = async (input) => {
|
||||
const leaky: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = async (input) => {
|
||||
throw new TypeError(`error sending request for url (${String(input)}): key ${encodeURIComponent(KEY)}`);
|
||||
};
|
||||
const e2 = await createTmdbClient({ baseUrl: "http://tmdb.local/3", apiKey: KEY, retries: 0, fetch: leaky })
|
||||
|
||||
@@ -16,7 +16,7 @@ export interface BraveClientOptions {
|
||||
baseUrl?: string;
|
||||
/** 缺省 10s */
|
||||
timeoutMs?: number;
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
}
|
||||
|
||||
export interface BraveSearchParams {
|
||||
|
||||
@@ -18,7 +18,7 @@ export interface FetchPageOptions {
|
||||
/** 最多跟随几次重定向,缺省 3 */
|
||||
maxRedirects?: number;
|
||||
userAgent?: string;
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
resolve?: HostResolver;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ describe("Brave 客户端", () => {
|
||||
fetch: (async (url: string, init: RequestInit) => {
|
||||
seen = { url, headers: new Headers(init.headers) };
|
||||
return jsonRes(BRAVE_BODY);
|
||||
}) as typeof fetch,
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
|
||||
});
|
||||
const r = await c.search({
|
||||
q: "MoviePilot v3",
|
||||
@@ -85,8 +85,10 @@ describe("Brave 客户端", () => {
|
||||
])("HTTP %i → %s,错误信息不含 Key", async (status, kind) => {
|
||||
const c = createBraveClient({
|
||||
apiKey: KEY,
|
||||
fetch: (async () =>
|
||||
jsonRes({ error: `bad token ${KEY}` }, status, { "x-ratelimit-reset": "1, 1000" })) as typeof fetch,
|
||||
fetch: (async () => jsonRes({ error: `bad token ${KEY}` }, status, { "x-ratelimit-reset": "1, 1000" })) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
});
|
||||
const err = (await c.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError;
|
||||
expect(err).toBeInstanceOf(WebSearchError);
|
||||
@@ -100,7 +102,7 @@ describe("Brave 客户端", () => {
|
||||
apiKey: KEY,
|
||||
fetch: (async () => {
|
||||
throw new TypeError(`connect fail ${KEY}`);
|
||||
}) as typeof fetch,
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
|
||||
});
|
||||
const e1 = (await net.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError;
|
||||
expect(e1.kind).toBe("network");
|
||||
@@ -109,9 +111,10 @@ describe("Brave 客户端", () => {
|
||||
apiKey: KEY,
|
||||
timeoutMs: 50,
|
||||
fetch: ((_u: string, init: RequestInit) =>
|
||||
new Promise((_r, rej) =>
|
||||
init.signal?.addEventListener("abort", () => rej(new Error("aborted"))),
|
||||
)) as typeof fetch,
|
||||
new Promise((_r, rej) => init.signal?.addEventListener("abort", () => rej(new Error("aborted"))))) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
});
|
||||
expect(((await slow.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError).kind).toBe("timeout");
|
||||
});
|
||||
@@ -164,8 +167,10 @@ describe("读取网页", () => {
|
||||
it("HTML 转纯文本:去脚本样式表单,保留标题与段落,URL 凭证脱敏", async () => {
|
||||
const p = await fetchPage("https://example.org/notice", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () =>
|
||||
new Response(html, { headers: { "content-type": "text/html; charset=utf-8" } })) as typeof fetch,
|
||||
fetch: (async () => new Response(html, { headers: { "content-type": "text/html; charset=utf-8" } })) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
});
|
||||
expect(p.title).toBe("站点公告");
|
||||
expect(p.text).toContain("9 月 20 日维护。");
|
||||
@@ -180,7 +185,10 @@ describe("读取网页", () => {
|
||||
const big = "a".repeat(600 * 1024);
|
||||
const p = await fetchPage("https://example.org/big.txt", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () => new Response(big, { headers: { "content-type": "text/plain" } })) as typeof fetch,
|
||||
fetch: (async () => new Response(big, { headers: { "content-type": "text/plain" } })) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
});
|
||||
expect(p.bytes).toBe(512 * 1024);
|
||||
expect(p.truncated).toBe(true);
|
||||
@@ -190,22 +198,28 @@ describe("读取网页", () => {
|
||||
it("非文本类型拒绝;重定向到内网拒绝;超时", async () => {
|
||||
const pdf = await fetchPage("https://example.org/a.pdf", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () => new Response("%PDF", { headers: { "content-type": "application/pdf" } })) as typeof fetch,
|
||||
fetch: (async () => new Response("%PDF", { headers: { "content-type": "application/pdf" } })) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
}).catch((e) => e);
|
||||
expect((pdf as WebSearchError).kind).toBe("unsupported");
|
||||
const redir = await fetchPage("https://example.org/r", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () =>
|
||||
new Response(null, { status: 302, headers: { location: "http://192.168.1.1/admin" } })) as typeof fetch,
|
||||
fetch: (async () => new Response(null, { status: 302, headers: { location: "http://192.168.1.1/admin" } })) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
}).catch((e) => e);
|
||||
expect((redir as WebSearchError).kind).toBe("blocked");
|
||||
const slow = await fetchPage("https://example.org/slow", {
|
||||
resolve: publicDns,
|
||||
timeoutMs: 50,
|
||||
fetch: ((_u: string, init: RequestInit) =>
|
||||
new Promise((_r, rej) =>
|
||||
init.signal?.addEventListener("abort", () => rej(new Error("aborted"))),
|
||||
)) as typeof fetch,
|
||||
new Promise((_r, rej) => init.signal?.addEventListener("abort", () => rej(new Error("aborted"))))) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
}).catch((e) => e);
|
||||
expect((slow as WebSearchError).kind).toBe("timeout");
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* 把 @umt/testkit 的 mock RSS / NexusPHP fixture 导出成文件,供 MP 差分生成脚本(scripts/mp-sync/gen-differential.py rss|indexer)读取。
|
||||
* 运行:deno run -A packages/adapters/test/differential/export-inputs.ts
|
||||
* 运行:bun packages/adapters/test/differential/export-inputs.ts
|
||||
* 只写 `inputs/rss/tk-*.xml` 与 `inputs/indexer/tk-*.html.txt`;手写样本(hw-*)不受影响。
|
||||
*/
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "@umt/agent",
|
||||
"version": "0.3.44",
|
||||
"exports": { ".": "./src/index.ts" },
|
||||
"tasks": {
|
||||
"test": "deno run -A npm:vitest run",
|
||||
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
|
||||
}
|
||||
}
|
||||
@@ -4,5 +4,13 @@
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"version": "0.3.44",
|
||||
"scripts": {
|
||||
"test": "bun x --bun vitest run",
|
||||
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
|
||||
},
|
||||
"dependencies": {
|
||||
"@umt/contracts": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
---
|
||||
name: release-facts
|
||||
title: 资源属性补全
|
||||
description: 收录规则因分辨率、编码、来源或发布组等属性未知无法判断时,查看资源详情并查证缺失事实,补齐后由原规则重新判断。
|
||||
triggers:
|
||||
- 资源属性(未知|缺失|补全)
|
||||
- release_facts
|
||||
capabilities:
|
||||
- release.inspect
|
||||
- release.apply_facts
|
||||
- decision.list
|
||||
- web.search
|
||||
- web.fetch
|
||||
sessions: [chat, investigate, child]
|
||||
---
|
||||
|
||||
## 适用意图
|
||||
|
||||
收录规则所依赖的分辨率、编码、来源或发布组未知,资源属性事项等待查证后重评。
|
||||
|
||||
## 步骤
|
||||
|
||||
1. 查看事项与缺失字段,用 release.inspect(refreshDetail: true)读取详情、MediaInfo 和文件信息。刷新失败时说明错误,不能把旧页作为刷新成功的证据。
|
||||
2. 只补齐该事项缺少且有明确证据的属性。分辨率写实际规格;编码、来源和发布组按详情记载,不从作品类型猜测。
|
||||
3. 有持续授权时调用 release.apply_facts,带事项、资源、事实和具体出处。原生详情刷新已经补齐字段时可以提交空 facts,服务端核实后触发重评。
|
||||
4. 无授权或证据不足时保留明确建议,交给用户协同处理。不要修改收录范围、保留目标、组合优先级,也不要直接选另一个资源替代规则判断。
|
||||
5. 补齐事实不代表资源合格或已下载;最终以原规则重新评估的结果为准。
|
||||
|
||||
## 需要批准
|
||||
|
||||
`release.apply_facts` 需要覆盖本作品的专用持续授权;已有下载或身份授权不替代它。只读详情与网页查询沿用读取权限。
|
||||
|
||||
## 汇报模板
|
||||
|
||||
已查证资源编码为 H.265,依据详情页 MediaInfo;补齐后已交原收录规则重新判断。若证据不足,说明具体缺哪项资料与下一步。
|
||||
|
||||
## 不要做
|
||||
|
||||
不改允许分辨率、保留目标、组合排序、作品身份和季集映射;不覆盖已明确的解析值,不把未知当作不满足,也不把补齐等同于已经下载。
|
||||
@@ -51,6 +51,7 @@ export const INTENT_ROUTES: readonly IntentRoute[] = [
|
||||
{ cap: "inventory.query", note: "库里已有的文件与收录版本" },
|
||||
{ cap: "release.search", note: "找可下载的资源" },
|
||||
{ cap: "release.inspect", note: "看某条资源的文件清单与促销" },
|
||||
{ cap: "release.apply_facts", note: "查证后补齐缺失资源属性,再由规则判断" },
|
||||
{ cap: "download.plan", note: "准备下载,返回是否需要批准" },
|
||||
{ cap: "download.submit", note: "提交到下载器" },
|
||||
],
|
||||
|
||||
@@ -17,8 +17,10 @@ import type { PromptSection, SessionKind, SkillDoc } from "./types.ts";
|
||||
// pi 1.x 把 harness 移到 pi-durable 后不再提供 skills,这里只保留本项目用到的部分:
|
||||
// 一层目录下的 <name>/SKILL.md,不读 .gitignore、不收根目录的散 .md、不跟随符号链接。
|
||||
|
||||
/** 仓库内的 Skill 目录;deno compile 用 --include 嵌入(apps/server/deno.json 的 compile 任务) */
|
||||
export const SKILLS_DIR = fileURLToPath(new URL("../../skills", import.meta.url));
|
||||
/** 仓库内的 Skill 目录;bun build --compile 用 --include 嵌入(apps/server/package.json 的 compile 任务) */
|
||||
export const SKILLS_DIR = process.execPath.endsWith("/umt-server")
|
||||
? join(process.execPath.replace(/[^/]+$/, ""), "skills")
|
||||
: fileURLToPath(new URL("../../skills", import.meta.url));
|
||||
|
||||
const DEFAULT_SESSIONS: SessionKind[] = ["chat", "patrol", "investigate", "child"];
|
||||
const SESSION_KINDS = new Set<SessionKind>(["chat", "patrol", "investigate", "child", "vision"]);
|
||||
|
||||
@@ -354,14 +354,10 @@ describe("PiAgentRuntime + FakeModelProvider", () => {
|
||||
|
||||
it("JSONL 恢复:子进程在 download.submit 执行中被 SIGKILL,新进程 resume 得到“结果未知”且不重复提交", async () => {
|
||||
const sessionDir = await tempDir();
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
["run", "-A", join(import.meta.dirname!, "fixtures/crash-run.ts"), sessionDir],
|
||||
{
|
||||
cwd: join(import.meta.dirname!, ".."),
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
const child = spawn(process.execPath, [join(import.meta.dirname!, "fixtures/crash-run.ts"), sessionDir], {
|
||||
cwd: join(import.meta.dirname!, ".."),
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let stderr = "";
|
||||
child.stderr.on("data", (d) => {
|
||||
stderr += String(d);
|
||||
|
||||
@@ -150,14 +150,14 @@ describe("按上游装配", () => {
|
||||
it("测试补全带 sessionId:OpenCode Go 请求头有 x-opencode-session", async () => {
|
||||
const seen: { url: string; headers: Headers }[] = [];
|
||||
const orig = globalThis.fetch;
|
||||
globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const req = new Request(input, init);
|
||||
globalThis.fetch = (async (input: string | Request | URL, init?: RequestInit) => {
|
||||
const req = new Request(input instanceof Request ? input : String(input), init);
|
||||
seen.push({ url: req.url, headers: req.headers });
|
||||
return new Response(JSON.stringify({ error: { message: "stubbed" } }), {
|
||||
status: 500,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}) as typeof fetch;
|
||||
}) as typeof globalThis.fetch;
|
||||
try {
|
||||
const c = new PiUpstreamClient(
|
||||
spec({ provider: "opencode-go", authKind: "api_key" }),
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "@umt/contracts",
|
||||
"version": "0.3.44",
|
||||
"exports": { ".": "./src/index.ts" },
|
||||
"tasks": {
|
||||
"test": "deno run -A npm:vitest run",
|
||||
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
|
||||
}
|
||||
}
|
||||
@@ -4,5 +4,11 @@
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
}
|
||||
},
|
||||
"version": "0.3.44",
|
||||
"scripts": {
|
||||
"test": "bun x --bun vitest run",
|
||||
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
|
||||
},
|
||||
"dependencies": {}
|
||||
}
|
||||
@@ -76,6 +76,37 @@ export function tmdbImageUrl(path: string | null | undefined, size: TmdbImageSiz
|
||||
return `/api/v1/img/tmdb/${size}/${p}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 外部图片代理允许的域名:豆瓣图床(img1–9.doubanio.com 等)与 Bangumi 图床。豆瓣图床对不带 Referer 的请求回 418,
|
||||
* 浏览器 no-referrer 直连拿不到图,所以经服务端带上来源站的 Referer 取(MP SECURITY_IMAGE_DOMAINS 同此)。
|
||||
*/
|
||||
export const EXTERNAL_IMAGE_HOSTS: readonly { suffix: string; referer: string }[] = [
|
||||
{ suffix: ".doubanio.com", referer: "https://movie.douban.com/" },
|
||||
{ suffix: "lain.bgm.tv", referer: "https://bgm.tv/" },
|
||||
];
|
||||
|
||||
/** 外部图片地址是否在代理白名单里(只认 https、图片扩展名) */
|
||||
export function externalImageReferer(url: string): string | null {
|
||||
let u: URL;
|
||||
try {
|
||||
u = new URL(url);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (u.protocol !== "https:" || !/\.(jpe?g|png|webp)$/i.test(u.pathname)) return null;
|
||||
const host = u.hostname.toLowerCase();
|
||||
const hit = EXTERNAL_IMAGE_HOSTS.find((h) =>
|
||||
h.suffix.startsWith(".") ? host.endsWith(h.suffix) : host === h.suffix,
|
||||
);
|
||||
return hit?.referer ?? null;
|
||||
}
|
||||
|
||||
/** 豆瓣 / Bangumi 封面 → 本站图片代理 /api/v1/img/ext?u=…;不在白名单的原样返回 */
|
||||
export function externalImageUrl(url: string | null | undefined): string | null {
|
||||
if (!url) return null;
|
||||
return externalImageReferer(url) ? `/api/v1/img/ext?u=${encodeURIComponent(url)}` : url;
|
||||
}
|
||||
|
||||
// ── 通用缓存(kv_cache)按 region 管理(设置 → 缓存) ──
|
||||
|
||||
/** region 名:小写字母、数字、`.`、`_`、`-` */
|
||||
|
||||
@@ -40,6 +40,7 @@ import {
|
||||
SiteKind,
|
||||
SiteLastHealth,
|
||||
SiteLoginResult,
|
||||
SiteReleasePage,
|
||||
SiteUserDataPoint,
|
||||
SourceRelease,
|
||||
} from "./sources.ts";
|
||||
@@ -116,7 +117,31 @@ export const ReleaseSearchOutput = z.object({
|
||||
});
|
||||
|
||||
export const ReleaseInspectInput = z.object({ sourceReleaseId: Id, refreshDetail: z.boolean().default(false) });
|
||||
export const ReleaseInspectOutput = z.object({ release: SourceRelease, inventory: z.array(MediaAsset) });
|
||||
export const ReleaseInspectOutput = z.object({
|
||||
release: SourceRelease,
|
||||
inventory: z.array(MediaAsset),
|
||||
page: SiteReleasePage.nullable().optional(),
|
||||
refreshError: z.string().nullable().optional(),
|
||||
});
|
||||
export const ReleaseStaticFacts = z
|
||||
.object({
|
||||
resolution: z.enum(["4320p", "2160p", "1440p", "1080p", "1080i", "720p", "576p", "480p"]).optional(),
|
||||
videoCodec: z.enum(["H.265", "H.264", "AV1", "VC-1", "MPEG-2", "XviD", "DivX"]).optional(),
|
||||
source: z.enum(["REMUX", "BluRay-Disc", "BluRay", "WEB-DL", "WEBRip", "HDTV", "HDRip", "DVDRip", "DVD"]).optional(),
|
||||
releaseGroup: z.string().trim().min(1).max(200).optional(),
|
||||
})
|
||||
.strict();
|
||||
export type ReleaseStaticFacts = z.infer<typeof ReleaseStaticFacts>;
|
||||
export const ReleaseApplyFactsInput = z
|
||||
.object({
|
||||
decisionId: Id,
|
||||
sourceReleaseId: Id,
|
||||
facts: ReleaseStaticFacts,
|
||||
evidence: z.string().trim().min(10).max(2000),
|
||||
evidenceUrl: z.url().optional(),
|
||||
})
|
||||
.strict();
|
||||
export const ReleaseApplyFactsOutput = z.object({ resolved: z.boolean(), remaining: z.array(z.string()) });
|
||||
|
||||
export const IdentityResolveInput = z
|
||||
.object({
|
||||
@@ -1136,6 +1161,7 @@ export const CAPABILITY_IO = {
|
||||
"catalog.lookup": { input: CatalogLookupInput, output: CatalogLookupOutput },
|
||||
"release.search": { input: ReleaseSearchInput, output: ReleaseSearchOutput },
|
||||
"release.inspect": { input: ReleaseInspectInput, output: ReleaseInspectOutput },
|
||||
"release.apply_facts": { input: ReleaseApplyFactsInput, output: ReleaseApplyFactsOutput },
|
||||
"identity.resolve": { input: IdentityResolveInput, output: IdentityResolveOutput },
|
||||
"download.plan": { input: DownloadPlanInput, output: DownloadPlanOutput },
|
||||
"download.submit": { input: DownloadSubmitInput, output: DownloadSubmitOutput },
|
||||
|
||||
@@ -57,6 +57,7 @@ export const CAPABILITY_IDS = [
|
||||
"catalog.lookup",
|
||||
"release.search",
|
||||
"release.inspect",
|
||||
"release.apply_facts",
|
||||
"identity.resolve",
|
||||
"download.plan",
|
||||
"download.submit",
|
||||
@@ -185,6 +186,18 @@ export const CAPABILITY_CATALOG: Record<KnownCapabilityId, CapabilityDescriptor>
|
||||
true,
|
||||
true,
|
||||
),
|
||||
"release.apply_facts": d(
|
||||
"release.apply_facts",
|
||||
"补全资源属性",
|
||||
"查证资源详情后,仅补全事项里缺失的分辨率、编码、来源或发布组;规则重新判断收录。不改变已有事实和策略。",
|
||||
"execute",
|
||||
"ReleaseApplyFacts",
|
||||
"organize.execute",
|
||||
"medium",
|
||||
true,
|
||||
true,
|
||||
"safe",
|
||||
),
|
||||
"download.plan": d(
|
||||
"download.plan",
|
||||
"准备下载",
|
||||
@@ -704,6 +717,7 @@ export const CAPABILITY_ACTIVITY: Record<KnownCapabilityId, CapabilityActivity>
|
||||
"catalog.lookup": act("查", "作品信息"),
|
||||
"release.search": act("搜", "资源", "次"),
|
||||
"release.inspect": act("看", "资源", "个"),
|
||||
"release.apply_facts": act("补全", "资源属性", "个"),
|
||||
"identity.resolve": act("识别", "作品", "个"),
|
||||
"download.plan": { ...act("准备下载", "资源", "个"), bare: true },
|
||||
"download.submit": act("提交", "下载", "个"),
|
||||
|
||||
@@ -184,6 +184,7 @@ export const ReleaseResolvedPayload = z.object({
|
||||
});
|
||||
|
||||
export const DecisionCreatedPayload = z.object({
|
||||
dedupeKey: z.string().optional(),
|
||||
decisionId: Id,
|
||||
kind: DecisionKind,
|
||||
severity: Severity,
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { KnownCapabilityId } from "./capability.ts";
|
||||
import type { EventType } from "./events.ts";
|
||||
import type { HookPoint } from "./hooks.ts";
|
||||
|
||||
// 功能模块约定(ADR-006):外围功能各自声明一份 manifest,汇总成功能清单。
|
||||
// 功能清单只给智能体看(系统提示的「功能清单」一节),不做开关;前端插槽默认直接显示。
|
||||
@@ -31,8 +30,6 @@ export interface FeatureManifest {
|
||||
pages: readonly string[];
|
||||
/** 挂在哪些前端插槽 */
|
||||
slots: readonly SlotPosition[];
|
||||
/** 注册的链式钩子 */
|
||||
hooks: readonly HookPoint[];
|
||||
}
|
||||
|
||||
/** 功能清单。新增外围功能:在这里加一份,服务端模块 manifest 指向它(featureManifest(id)) */
|
||||
@@ -47,7 +44,6 @@ export const FEATURES = [
|
||||
capabilities: [],
|
||||
pages: ["作品页「补字幕」", "整理记录行菜单「补字幕」", "站点页「字幕站」"],
|
||||
slots: ["entity.actions", "organize.record.actions"],
|
||||
hooks: [],
|
||||
},
|
||||
{
|
||||
id: "browser",
|
||||
@@ -64,7 +60,6 @@ export const FEATURES = [
|
||||
],
|
||||
pages: ["站点页「原站视图」"],
|
||||
slots: [],
|
||||
hooks: [],
|
||||
},
|
||||
{
|
||||
id: "desktop",
|
||||
@@ -75,7 +70,6 @@ export const FEATURES = [
|
||||
capabilities: [],
|
||||
pages: ["原站视图「桌面模式」"],
|
||||
slots: [],
|
||||
hooks: [],
|
||||
},
|
||||
{
|
||||
id: "websearch",
|
||||
@@ -86,7 +80,6 @@ export const FEATURES = [
|
||||
capabilities: ["web.search", "web.fetch"],
|
||||
pages: ["设置 → 智能体 → 网页搜索"],
|
||||
slots: [],
|
||||
hooks: [],
|
||||
},
|
||||
{
|
||||
id: "mcp",
|
||||
@@ -97,7 +90,6 @@ export const FEATURES = [
|
||||
capabilities: [],
|
||||
pages: ["设置 → 智能体 → 工具与 MCP"],
|
||||
slots: [],
|
||||
hooks: [],
|
||||
},
|
||||
{
|
||||
id: "schedules",
|
||||
@@ -108,7 +100,6 @@ export const FEATURES = [
|
||||
capabilities: ["schedule.draft_check"],
|
||||
pages: ["日志 → 定时任务"],
|
||||
slots: [],
|
||||
hooks: [],
|
||||
},
|
||||
] as const satisfies readonly FeatureManifest[];
|
||||
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
// 链式钩子挂点(ADR-006,参考 MoviePilot ChainEventType):同步、按注册顺序执行、可改写或拦截。
|
||||
// 这里只登记挂点名称与语义(给功能清单、智能体提示与文档用);入参 / 返回值类型在服务端
|
||||
// apps/server/src/hooks/points.ts,因为它们用到 domain 的解析结果等类型。
|
||||
|
||||
/**
|
||||
* - rewrite:每个钩子拿到上一个钩子的结果,返回新值或 undefined(不改);
|
||||
* - intercept:任一钩子返回拒绝原因即停止,后面的钩子不再执行。
|
||||
*/
|
||||
export type HookKind = "rewrite" | "intercept";
|
||||
|
||||
export const HOOK_POINTS = {
|
||||
"name.recognize": {
|
||||
kind: "rewrite",
|
||||
label: "名称识别",
|
||||
when: "资源标题解析之后、按名称召回作品之前",
|
||||
effect: "改写解析结果(标题、年份、季集、外部 ID)",
|
||||
},
|
||||
"organize.rename": {
|
||||
kind: "rewrite",
|
||||
label: "整理改名",
|
||||
when: "按命名模板生成目标路径之后",
|
||||
effect: "改写主视频的目标文件名(同主干的字幕等附属文件跟着改)",
|
||||
},
|
||||
"organize.intercept": {
|
||||
kind: "intercept",
|
||||
label: "整理拦截",
|
||||
when: "整理计划生成后、写整理记录之前",
|
||||
effect: "返回原因即拒绝本次整理(记录为失败,原因 intercepted:<原因>)",
|
||||
},
|
||||
"resource.selection": {
|
||||
kind: "rewrite",
|
||||
label: "资源选择",
|
||||
when: "订阅 / 收录版本重评时候选排好序之后、逐条评估之前",
|
||||
effect: "调整候选顺序或过滤掉一部分(不能加入新候选)",
|
||||
},
|
||||
"subscription.completion.check": {
|
||||
kind: "intercept",
|
||||
label: "订阅完成判定",
|
||||
when: "订阅满足完成条件、标记完成之前",
|
||||
effect: "追加完成条件:返回原因即本次不完成,订阅继续追踪",
|
||||
},
|
||||
} as const satisfies Record<string, { kind: HookKind; label: string; when: string; effect: string }>;
|
||||
|
||||
export type HookPoint = keyof typeof HOOK_POINTS;
|
||||
@@ -15,7 +15,6 @@ export * from "./episode-mapping.ts";
|
||||
export * from "./events.ts";
|
||||
export * from "./execution.ts";
|
||||
export * from "./features.ts";
|
||||
export * from "./hooks.ts";
|
||||
export * from "./identify-words.ts";
|
||||
export * from "./identity.ts";
|
||||
export * from "./instances.ts";
|
||||
|
||||
@@ -85,7 +85,7 @@ export const Subscription = EntityBase.extend({
|
||||
mediaEntityTitle: z.string().nullable().optional(),
|
||||
});
|
||||
export type Subscription = z.infer<typeof Subscription>;
|
||||
/** 自动补全其他版本:inherit = 取引用规则组(后者优先)的 autoFill,都没设为开 */
|
||||
/** 自动补全其他版本:inherit = 取主版本收录规则链(后者优先)的 autoFill,都没设为关 */
|
||||
export const AutoFillMode = z.enum(["inherit", "on", "off"]);
|
||||
export type AutoFillMode = z.infer<typeof AutoFillMode>;
|
||||
/** 补全预算:只约束非主版本的自动补全计划(按订阅、按自然日);同时仍计入全局每日预算 */
|
||||
@@ -121,7 +121,8 @@ export type SubscriptionEffect = z.infer<typeof SubscriptionEffect>;
|
||||
* siteIds 站点范围;episodeRange 集范围(对照 MP start/total_episode);searchIntervalHours 主动搜索周期(0 = 关);
|
||||
* stopAfter 截止时间;downloaderInstanceId 指定下载器;ruleGroupIds 规则组;keywords 追加检索词;
|
||||
* include/exclude 订阅级关键词正则(作用于全部收录版本);libraryRootId 库根;downloadSavePath 下载保存路径;
|
||||
* autoFill/fillBudget 多版本自动补全;resolution/source/effect 质量预设(MP resolution/quality/effect)。
|
||||
* autoFill/fillBudget 多版本自动补全;resolution/source/effect 仅保留 MP 导入/API 质量预设兼容。
|
||||
* 正常界面逐版本维护质量和规则引用,普通订阅保存保留旧兼容字段。
|
||||
* include/exclude 保存时同步为订阅私有规则组(scope.subscriptionId,随订阅删除),评估只读规则组。
|
||||
* 写入时兼容 MP 字段与旧键(sites/downloader/save_path/start_episode/filter_groups/keyword/autoFillVersions…),
|
||||
* 服务端规范化后只存本表列出的键,见 docs/runbook/subscriptions.md。
|
||||
@@ -714,6 +715,17 @@ export type PreferenceSuppressionUpdate = z.infer<typeof PreferenceSuppressionUp
|
||||
|
||||
// ── 收录策略规则与候选评估(W2-C) ──
|
||||
|
||||
/** 分类名称沿用整理分类;显式选择策略时不再按分类过滤。 */
|
||||
export const AcquisitionPolicyScope = z
|
||||
.object({
|
||||
mediaKinds: z.array(SubscriptionMediaKind).optional(),
|
||||
categories: z
|
||||
.object({ movie: z.array(z.string().min(1)).optional(), tv: z.array(z.string().min(1)).optional() })
|
||||
.optional(),
|
||||
})
|
||||
.passthrough();
|
||||
export type AcquisitionPolicyScope = z.infer<typeof AcquisitionPolicyScope>;
|
||||
|
||||
/** Slot 模板:策略规则里用于自动建 Slot(新作品收录 / 订阅缺省 Slot);字段同 AcquisitionSlot 规则列 */
|
||||
export const SlotTemplate = z.object({
|
||||
name: z.string().min(1).max(64),
|
||||
|
||||
@@ -229,16 +229,20 @@ describe("关键合同行为", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("capabilitySpec 组装目录元数据与真实 schema;有副作用的类别不可重放", () => {
|
||||
it("capabilitySpec 组装目录元数据与真实 schema;写入仅明确的幂等例外可重放", () => {
|
||||
for (const id of C.CAPABILITY_IDS) {
|
||||
const spec = C.capabilitySpec(id);
|
||||
const { inputSchema: _i, outputSchema: _o, ...meta } = C.CAPABILITY_CATALOG[id];
|
||||
expect(spec).toMatchObject(meta);
|
||||
expect(spec.inputSchema).toBe(C.CAPABILITY_IO[id].input);
|
||||
expect(spec.outputSchema).toBe(C.CAPABILITY_IO[id].output);
|
||||
// 例外:site.health_check 只读首页并写回健康列,幂等可重跑;metadata.enrich 只存补全建议(7 天标题缓存),重跑无副作用
|
||||
if (spec.kind !== "read" && spec.kind !== "preview" && id !== "site.health_check" && id !== "metadata.enrich")
|
||||
expect(spec.replay, id).toBe("never");
|
||||
// 健康列、补全建议、带事项与事件去重的资源事实可幂等重跑;其它写入仍不可重放。
|
||||
if (spec.kind !== "read" && spec.kind !== "preview") {
|
||||
if (["site.health_check", "metadata.enrich", "release.apply_facts"].includes(id)) {
|
||||
expect(spec.replay, id).toBe("safe");
|
||||
expect(spec.idempotent, id).toBe(true);
|
||||
} else expect(spec.replay, id).toBe("never");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ PostgreSQL 访问层:Kysely 客户端与连接池、表类型(`src/types.ts`)、
|
||||
## 规则
|
||||
|
||||
- **迁移是唯一事实来源**。所有 schema 变更只能新增迁移,已发布迁移不得修改。迁移用原生 DDL 书写(`src/migration-helpers.ts`:`entityTable()` 统一加 uuid 主键 `gen_random_uuid()`、`created_at/updated_at` 与 updated_at 触发器;`oneOf()` 生成状态 CHECK)。`migrations/` 内相对导入必须带 `.ts` 扩展名(Kysely 用原生 `import()` 加载)。
|
||||
- **`src/types.ts` 是生成物**:`deno task db:migrate && deno task gen:types`。CHECK 枚举自动变成字面量联合(如 `AcquisitionSlotStatus`)。`test/schema.test.ts` 会在隔离 schema 里重新反射并比对,漂移即失败;`packages/contracts/test/db-sync.test.ts` 再校验 contracts 的 Row schema 与枚举和这里一致。
|
||||
- **`src/types.ts` 是生成物**:`bun run db:migrate && bun run gen:types`。CHECK 枚举自动变成字面量联合(如 `AcquisitionSlotStatus`)。`test/schema.test.ts` 会在隔离 schema 里重新反射并比对,漂移即失败;`packages/contracts/test/db-sync.test.ts` 再校验 contracts 的 Row schema 与枚举和这里一致。
|
||||
- 状态列一律 `text + CHECK`,不用 PG enum(便于迁移加值)。JSONB 只放证据、规则参数、原文快照与演进字段;稳定查询字段用关系列。
|
||||
- 凭证只存 `secret_ref`(→ `secret.id`),明文不入库、不进日志。
|
||||
- `createDb()` 把 `int8` 解析为 `number`(超出 2^53 抛错)、`date` 保持 `YYYY-MM-DD` 字符串。写 jsonb 数组时须先 `JSON.stringify`(pg 会把 JS 数组当 PG 数组)。
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"name": "@umt/db",
|
||||
"version": "0.3.44",
|
||||
"exports": { ".": "./src/index.ts" },
|
||||
"tasks": {
|
||||
"test": "deno run -A npm:vitest run",
|
||||
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json",
|
||||
"migrate": "deno run -A --env-file=../../.env scripts/migrate.ts latest",
|
||||
"reset": "deno run -A --env-file=../../.env scripts/migrate.ts reset",
|
||||
"gen:types": "deno run -A --env-file=../../.env scripts/gen-types.ts"
|
||||
}
|
||||
}
|
||||
@@ -4,5 +4,16 @@
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"version": "0.3.44",
|
||||
"scripts": {
|
||||
"test": "bun x --bun vitest run",
|
||||
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json",
|
||||
"migrate": "bun --env-file=../../.env scripts/migrate.ts latest",
|
||||
"reset": "bun --env-file=../../.env scripts/migrate.ts reset",
|
||||
"gen:types": "bun --env-file=../../.env scripts/gen-types.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@umt/runtime": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import { fileURLToPath } from "node:url";
|
||||
import { type Kysely, sql } from "kysely";
|
||||
|
||||
// 从已迁移的数据库反射出 src/types.ts:列类型 + CHECK (col in (...)) 枚举字面量联合。
|
||||
// 迁移是唯一事实来源;改表后运行 `deno task gen:types`,测试会校验两者无漂移。
|
||||
// 迁移是唯一事实来源;改表后运行 `bun run gen:types`,测试会校验两者无漂移。
|
||||
|
||||
interface ColumnInfo {
|
||||
table_name: string;
|
||||
@@ -77,7 +77,7 @@ export async function generateTypesSource(db: Kysely<any>): Promise<string> {
|
||||
|
||||
const out: string[] = [
|
||||
"// 本文件由 scripts/gen-types.ts 从迁移后的数据库反射生成,请勿手改。",
|
||||
"// 改表:新增迁移 → deno task db:migrate → deno task gen:types。",
|
||||
"// 改表:新增迁移 → bun run db:migrate → bun run gen:types。",
|
||||
"",
|
||||
'import type { ColumnType, Generated, Insertable, Selectable, Updateable } from "kysely";',
|
||||
"",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// 用法:deno task gen:types —— 对 DATABASE_URL 的当前 schema 反射并重写 src/types.ts
|
||||
// 用法:bun run gen:types —— 对 DATABASE_URL 的当前 schema 反射并重写 src/types.ts
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { createDb } from "../src/index.ts";
|
||||
import { formatWithBiome, generateTypesSource, TYPES_FILE } from "./gen-types-lib.ts";
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// 用法:deno task db:migrate / db:reset(= deno run -A --env-file=.env packages/db/scripts/migrate.ts latest|reset)
|
||||
// 用法:bun run db:migrate / db:reset(= bun --env-file=.env packages/db/scripts/migrate.ts latest|reset)
|
||||
import { createDb, migrateToLatest, resetSchema } from "../src/index.ts";
|
||||
|
||||
const cmd = process.argv[2] ?? "latest";
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// 本文件由 scripts/gen-types.ts 从迁移后的数据库反射生成,请勿手改。
|
||||
// 改表:新增迁移 → deno task db:migrate → deno task gen:types。
|
||||
// 改表:新增迁移 → bun run db:migrate → bun run gen:types。
|
||||
|
||||
import type { ColumnType, Generated, Insertable, Selectable, Updateable } from "kysely";
|
||||
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ import { Kysely, PostgresDialect } from "kysely";
|
||||
import pg from "pg";
|
||||
import { PgTokenBucket } from "../../src/rate-limit.ts";
|
||||
|
||||
const [url, schema, key, n, capacity, rate] = Deno.args as [string, string, string, string, string, string];
|
||||
const [url, schema, key, n, capacity, rate] = process.argv.slice(2) as [string, string, string, string, string, string];
|
||||
const db = new Kysely<unknown>({
|
||||
dialect: new PostgresDialect({
|
||||
pool: new pg.Pool({ connectionString: url, max: 1, options: `-c search_path=${schema},public` }),
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Command } from "@umt/runtime";
|
||||
import { Kysely, PostgresDialect, sql } from "kysely";
|
||||
import pg from "pg";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
@@ -70,8 +71,8 @@ describe.skipIf(skip)("PgTokenBucket(真实 PG)", () => {
|
||||
const rate = 20;
|
||||
const perProc = 12;
|
||||
const procs = [0, 1, 2].map(() =>
|
||||
new Deno.Command(Deno.execPath(), {
|
||||
args: ["run", "-A", script, url, schema, "t:procs", String(perProc), String(capacity), String(rate)],
|
||||
new Command(process.execPath, {
|
||||
args: [script, url, schema, "t:procs", String(perProc), String(capacity), String(rate)],
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
}).output(),
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
纯业务逻辑(无 IO):发布标题解析、身份匹配、Slot 满足度、收录模式/预算/磁盘阈值、路径绑定与命名、整理计划、下载状态映射。
|
||||
|
||||
- 禁止 import 任何网络/文件/数据库库,也不读全局配置与时钟;时间(`now`)、设备号、文件清单、已存在目标等事实一律由调用方传入。
|
||||
- 不依赖 `@umt/contracts`:这里的类型是领域类型,W2 在 server/tasks 层映射到 zod 合同。
|
||||
- 唯一入口 `src/index.ts`(`export *` 各模块)。测试:`cd packages/domain && deno task test`。
|
||||
- 同一语义的类型和纯规则可以依赖 `@umt/contracts`,例如 `ContentType`;含义不同的领域类型通过 `@umt/mapping` 转换。保持无 IO,不依赖 server / db / adapters。
|
||||
- 唯一入口 `src/index.ts`(`export *` 各模块)。测试:`cd packages/domain && bun run test`。
|
||||
|
||||
## 模块地图
|
||||
|
||||
@@ -49,7 +49,7 @@
|
||||
| move 跨盘 | EXDEV 隐式降级复制,删源失败仍成功 | 只标记 `crossDevice`,执行层显式三步 |
|
||||
| Move 删种 | 不看 H&R | H&R 未满足/未知 → 阻塞冲突 |
|
||||
|
||||
## 给 W2 的接入提示
|
||||
## 接入提示
|
||||
|
||||
- 解析:下载完成后用 `parseReleasePath({ file, parent, grandParent })` 或 `planOrganize({ sourceRoot })`(只在种子根内取目录名);RSS/站点标题用 `parseRelease(title, { subtitle })`。`ParsedRelease.sizeBytes` 可用站点字段覆盖(`{ ...parsed, sizeBytes }`)后再交给 Slot 评估。
|
||||
- 身份:本地别名召回(pg_trgm)与 TMDB 结果都转成 `IdentityCandidate`(本地用 `provider: "local"` + `localEntityId` + `linkedExternalIds`);`resolveIdentity` 为 `auto` 才自动绑定,其余进待识别池。
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "@umt/domain",
|
||||
"version": "0.3.44",
|
||||
"exports": { ".": "./src/index.ts" },
|
||||
"tasks": {
|
||||
"test": "deno run -A npm:vitest run",
|
||||
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
|
||||
}
|
||||
}
|
||||
@@ -4,5 +4,13 @@
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"version": "0.3.44",
|
||||
"scripts": {
|
||||
"test": "bun x --bun vitest run",
|
||||
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
|
||||
},
|
||||
"dependencies": {
|
||||
"@umt/contracts": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ export {
|
||||
resolveAutoFill,
|
||||
} from "./fill.ts";
|
||||
export { type AcquisitionMode, determineMode, type ModeDecision, type ModeInput } from "./mode.ts";
|
||||
export { type ScopedAcquisitionPolicy, selectAcquisitionPolicy } from "./policy.ts";
|
||||
export {
|
||||
DEFAULT_PRIORITY_ORDER,
|
||||
type PriorityClass,
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import type { AcquisitionPolicyScope } from "@umt/contracts";
|
||||
|
||||
export interface ScopedAcquisitionPolicy {
|
||||
id: string;
|
||||
scope: AcquisitionPolicyScope;
|
||||
}
|
||||
|
||||
/** 输入已按发布时间排序;分类明确命中优先于类型默认,显式选择不回退到别的策略。 */
|
||||
export function selectAcquisitionPolicy<T extends ScopedAcquisitionPolicy>(
|
||||
policies: readonly T[],
|
||||
input: { explicitId?: string | null; mediaKind: "movie" | "series" | "variety"; category: string | null },
|
||||
): T | undefined {
|
||||
if (input.explicitId) return policies.find((p) => p.id === input.explicitId);
|
||||
const type = input.mediaKind === "movie" ? "movie" : "tv";
|
||||
let selected: T | undefined;
|
||||
let specificity = -1;
|
||||
for (const p of policies) {
|
||||
if (p.scope.mediaKinds?.length && !p.scope.mediaKinds.includes(input.mediaKind)) continue;
|
||||
const categories = p.scope.categories;
|
||||
if (categories && Object.values(categories).some((v) => v?.length)) {
|
||||
if (!input.category || !categories[type]?.includes(input.category)) continue;
|
||||
}
|
||||
const score = categories?.[type]?.length ? 2 : p.scope.mediaKinds?.length ? 1 : 0;
|
||||
if (score > specificity) {
|
||||
selected = p;
|
||||
specificity = score;
|
||||
}
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { ContentType } from "@umt/contracts";
|
||||
import type { CompiledGlobalWord } from "../identify-words/apply.ts";
|
||||
|
||||
/** 解析器版本:规则变化时递增,缓存/识别结果按此失效 */
|
||||
@@ -35,15 +36,8 @@ export type Cut =
|
||||
| "Ultimate"
|
||||
| "Special Edition";
|
||||
|
||||
/** 综艺内容类型(REQ §5.1);v0.1 只识别标记,不做分段建模 */
|
||||
export type ContentType =
|
||||
| "unknown"
|
||||
| "main"
|
||||
| "extra"
|
||||
| "behind_the_scenes"
|
||||
| "performance_only"
|
||||
| "preview"
|
||||
| "reaction";
|
||||
// 内容类型在 contracts 定义一次;领域与接口使用相同语义。
|
||||
export type { ContentType };
|
||||
|
||||
export type Segment = "full" | "upper" | "middle" | "lower";
|
||||
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { ParsedRelease } from "../release-parser/types.ts";
|
||||
import type { HardField } from "../slot/types.ts";
|
||||
import { type AtomCatalog, testAtom } from "./evaluate.ts";
|
||||
import type { RuleContext, RuleExpr, RuleGroupSpec } from "./types.ts";
|
||||
|
||||
// 只对有明确事实字段的条件判断未知;正则文本缺词仍是已知不命中。
|
||||
const fields: Record<string, HardField> = {
|
||||
"4K": "resolution",
|
||||
"1080P": "resolution",
|
||||
"720P": "resolution",
|
||||
BLU: "source",
|
||||
BLURAY: "source",
|
||||
REMUX: "source",
|
||||
WEBDL: "source",
|
||||
H265: "codec",
|
||||
H264: "codec",
|
||||
FREE: "promotion",
|
||||
};
|
||||
type PartialResult = { hit: boolean | null; missing: HardField[] };
|
||||
|
||||
function partial(e: RuleExpr, r: ParsedRelease, ctx: RuleContext, catalog: AtomCatalog): PartialResult {
|
||||
if (e.op === "atom") {
|
||||
const id = e.id.toUpperCase();
|
||||
const f = fields[id];
|
||||
const value =
|
||||
f === "resolution"
|
||||
? r.resolution
|
||||
: f === "source"
|
||||
? r.source
|
||||
: f === "codec"
|
||||
? r.videoCodec
|
||||
: ctx.downloadVolumeFactor;
|
||||
if (f && value === undefined) return { hit: null, missing: [f] };
|
||||
const custom = catalog.get(id);
|
||||
const missing: HardField[] = [];
|
||||
if (custom?.minSeeders !== undefined && ctx.seeders === undefined) missing.push("seeders");
|
||||
if (custom?.free && ctx.downloadVolumeFactor === undefined) missing.push("promotion");
|
||||
if (custom?.unitSizeMB && r.sizeBytes === undefined) missing.push("size");
|
||||
const input = { r, ctx, text: [r.raw ?? "", ctx.subtitle ?? "", ...(ctx.labels ?? [])].join(" ") };
|
||||
// 已知的正则条件失败时,其他缺失字段不会改变结论。
|
||||
if (custom && missing.length) {
|
||||
const known = { ...custom, minSeeders: undefined, free: undefined, unitSizeMB: undefined };
|
||||
const knownCatalog = new Map(catalog).set(id, known);
|
||||
if (!testAtom(id, input, knownCatalog).hit) return { hit: false, missing: [] };
|
||||
return { hit: null, missing };
|
||||
}
|
||||
return { hit: testAtom(id, input, catalog).hit, missing: [] };
|
||||
}
|
||||
if (e.op === "not") {
|
||||
const result = partial(e.arg, r, ctx, catalog);
|
||||
return { ...result, hit: result.hit === null ? null : !result.hit };
|
||||
}
|
||||
const results = e.args.map((a) => partial(a, r, ctx, catalog));
|
||||
const decisive = e.op !== "and";
|
||||
if (results.some((x) => x.hit === decisive)) return { hit: decisive, missing: [] };
|
||||
const unknown = results.filter((x) => x.hit === null);
|
||||
return unknown.length ? { hit: null, missing: unknown.flatMap((x) => x.missing) } : { hit: !decisive, missing: [] };
|
||||
}
|
||||
|
||||
/** 未知的高档不能被当成不命中而落入低档,NOT 也不能把未知当成事实。 */
|
||||
export function unknownRuleFields(
|
||||
groups: readonly RuleGroupSpec[],
|
||||
r: ParsedRelease,
|
||||
ctx: RuleContext,
|
||||
catalog: AtomCatalog,
|
||||
): HardField[] {
|
||||
const missing: HardField[] = [];
|
||||
for (const group of groups) {
|
||||
for (const expr of group.hardExcludes ?? []) missing.push(...partial(expr, r, ctx, catalog).missing);
|
||||
for (const layer of group.layers) {
|
||||
const result = partial(layer.expr, r, ctx, catalog);
|
||||
missing.push(...result.missing);
|
||||
if (result.hit === true) break;
|
||||
}
|
||||
}
|
||||
return [...new Set(missing)];
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
import type { ParsedRelease, ReleaseSource, Resolution } from "../release-parser/types.ts";
|
||||
import { catalogOf, evaluateRuleChain } from "../rules/evaluate.ts";
|
||||
import type { RuleChainResult } from "../rules/types.ts";
|
||||
import { unknownRuleFields } from "../rules/unknown.ts";
|
||||
import { seasonReleaseKey, upgradeSpec } from "./season-release.ts";
|
||||
import type {
|
||||
EvaluationContext,
|
||||
HardField,
|
||||
@@ -310,6 +312,8 @@ export function describeViolation(v: Violation): string {
|
||||
return `优先级没有提高(现有 ${JSON.stringify(e)},这个资源 ${JSON.stringify(a)})`;
|
||||
case "upgrade.not_full_pack":
|
||||
return "全集洗版只接受整季包";
|
||||
case "upgrade.different_season_release":
|
||||
return "与当季选定的成套版本不同,等待同套资源";
|
||||
case "upgrade.no_improvable_units":
|
||||
return "这个资源包含的季集都没有可升级的";
|
||||
default:
|
||||
@@ -348,6 +352,37 @@ function tierFailures(rule: SlotTierRule, r: ParsedRelease, ctx: EvaluationConte
|
||||
return out;
|
||||
}
|
||||
|
||||
function unknownTierFields(spec: SlotSpec, r: ParsedRelease, ctx: EvaluationContext): HardField[] {
|
||||
for (const tier of spec.soft?.tiers ?? []) {
|
||||
if (!tierFailures(tier, r, ctx).length) break;
|
||||
const missing: HardField[] = [];
|
||||
const filled = { ...r };
|
||||
if (!r.resolution && tier.resolutions?.length) {
|
||||
missing.push("resolution");
|
||||
filled.resolution = tier.resolutions[0];
|
||||
}
|
||||
if (!r.source && (tier.sources?.length || tier.excludeSources?.length || tier.requireEncode)) {
|
||||
missing.push("source");
|
||||
filled.source =
|
||||
tier.sources?.[0] ??
|
||||
(tier.requireEncode
|
||||
? "BluRay"
|
||||
: (Object.keys(DEFAULT_SOURCE_TIERS) as ReleaseSource[]).find((s) => !tier.excludeSources?.includes(s)));
|
||||
filled.isEncode = tier.requireEncode ?? filled.source === "BluRay";
|
||||
}
|
||||
if (!r.videoCodec && tier.codecs?.length) {
|
||||
missing.push("codec");
|
||||
filled.videoCodec = tier.codecs[0];
|
||||
}
|
||||
if (!r.releaseGroup && tier.groups?.length) {
|
||||
missing.push("group");
|
||||
filled.releaseGroup = { name: tier.groups.join("@"), source: "known" };
|
||||
}
|
||||
if (missing.length && !tierFailures(tier, filled, ctx).length) return missing;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* 优先级阶梯(对应 MP 规则组 `A > B > C`):逐层匹配,第 k 层(0 起)命中 → pri = 100 - k;
|
||||
* 未配置阶梯 → pri = 100、level = null(不参与排序差异);配置了但都不命中 → pri = 0。
|
||||
@@ -465,9 +500,21 @@ export function evaluateCandidate(
|
||||
current?: QualityFacts | null,
|
||||
ctx: EvaluationContext = {},
|
||||
): SlotEvaluation {
|
||||
if (current) spec = upgradeSpec(spec);
|
||||
const hard = checkHardConditions(spec, r, ctx);
|
||||
const { missing, accepted } = hard;
|
||||
const chain = ruleChainOf(spec, r, ctx);
|
||||
if (hard.violations.length === 0) {
|
||||
const unknown = spec.rules ? unknownRuleFields(spec.rules.groups, r, ctx, catalogOf(spec.rules.atoms)) : [];
|
||||
if (chain?.decidingGroup === undefined || chain.decidingGroup === null)
|
||||
unknown.push(...unknownTierFields(spec, r, ctx));
|
||||
if (unknown.length)
|
||||
return {
|
||||
decision: "unknown_field",
|
||||
missing: [...new Set([...hard.missing, ...unknown])],
|
||||
trace: unknown.map((f) => `组合规则需要补充:${f}`),
|
||||
};
|
||||
}
|
||||
const { missing, accepted } = hard;
|
||||
const withRules = chain ? { rules: chain } : {};
|
||||
const violations = [...hard.violations, ...chainViolations(chain, spec)];
|
||||
const ladder = effectiveLadder(spec, r, ctx, chain);
|
||||
@@ -524,6 +571,53 @@ export function evaluateCandidate(
|
||||
const up = spec.upgrade;
|
||||
if (!up?.enabled) return reject({ field: "upgrade", code: "upgrade_disabled" });
|
||||
if (up.mode === "full_pack" && (r.episodes || r.airDate)) return reject({ field: "upgrade", code: "not_full_pack" });
|
||||
if (r.isEncode && up.encodeGroupAllowlist?.length) {
|
||||
if (!r.releaseGroup) return { decision: "unknown_field", missing: ["group"], trace: ["合格压制组需要查证发布组"] };
|
||||
const groups = r.releaseGroup ? r.releaseGroup.name.split("@").map(lc) : [];
|
||||
if (!up.encodeGroupAllowlist.some((g) => groups.includes(lc(g))))
|
||||
return reject({
|
||||
field: "upgrade",
|
||||
code: "encode_not_qualified",
|
||||
expected: up.encodeGroupAllowlist,
|
||||
actual: r.releaseGroup?.name,
|
||||
});
|
||||
}
|
||||
if (up.consistency === "season_release") {
|
||||
const key = seasonReleaseKey(r);
|
||||
const targetKey = ctx.seasonReleaseTarget ? seasonReleaseKey(ctx.seasonReleaseTarget) : null;
|
||||
if (key && !targetKey)
|
||||
return { decision: "unknown_field", missing: ["current_asset"], trace: ["当季成套目标的规格未知"] };
|
||||
if (!key)
|
||||
return {
|
||||
decision: "unknown_field",
|
||||
missing: [
|
||||
...(!r.resolution ? ["resolution" as const] : []),
|
||||
...(!r.source ? ["source" as const] : []),
|
||||
...(!r.videoCodec ? ["codec" as const] : []),
|
||||
...(!r.releaseGroup?.name ? ["group" as const] : []),
|
||||
],
|
||||
trace: ["成套洗版需要完整版本规格"],
|
||||
};
|
||||
if (key !== targetKey) return reject({ field: "upgrade", code: "different_season_release" });
|
||||
const fromPri = currentPri(spec, current);
|
||||
if (key !== seasonReleaseKey(current) && ladder.pri === fromPri) {
|
||||
const from = qualityTier(current, spec);
|
||||
if (tier.source < from.source || tier.resolution < from.resolution)
|
||||
return reject({ field: "upgrade", code: "lower_tier", expected: from, actual: tier });
|
||||
return {
|
||||
decision: "upgrade",
|
||||
tier,
|
||||
fromTier: from,
|
||||
softRank: rank,
|
||||
reasons: [...reasons, "season_release_aligned", ...priReasons],
|
||||
pri: ladder.pri,
|
||||
fromPri,
|
||||
ladder,
|
||||
trace: ["保持优先级,向当季同套版本对齐", ...ladder.trace],
|
||||
...withRules,
|
||||
};
|
||||
}
|
||||
}
|
||||
const basis = up.basis ?? (tiered ? "priority" : "tier");
|
||||
if (basis === "priority") {
|
||||
const fromPri = currentPri(spec, current);
|
||||
@@ -559,16 +653,6 @@ export function evaluateCandidate(
|
||||
dSource === 0 && dRes === 0 ? "same_tier" : dSource < 0 || dRes < 0 ? "lower_tier" : "below_min_improvement";
|
||||
return reject({ field: "upgrade", code, expected: { minTierSteps: minSteps, from }, actual: tier });
|
||||
}
|
||||
if (r.isEncode && up.encodeGroupAllowlist?.length) {
|
||||
const groups = r.releaseGroup ? r.releaseGroup.name.split("@").map(lc) : [];
|
||||
if (!up.encodeGroupAllowlist.some((g) => groups.includes(lc(g))))
|
||||
return reject({
|
||||
field: "upgrade",
|
||||
code: "encode_not_qualified",
|
||||
expected: up.encodeGroupAllowlist,
|
||||
actual: r.releaseGroup?.name,
|
||||
});
|
||||
}
|
||||
return {
|
||||
decision: "upgrade",
|
||||
tier,
|
||||
|
||||
@@ -59,6 +59,7 @@ export {
|
||||
type SubscriptionGroupResult,
|
||||
subscriptionGroup,
|
||||
} from "./overview.ts";
|
||||
export { seasonReleaseKey, selectSeasonRelease, upgradeSpec } from "./season-release.ts";
|
||||
export {
|
||||
canTransition,
|
||||
type SlotEvent,
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { ParsedRelease } from "../release-parser/types.ts";
|
||||
import { compareRank, currentPri, qualityTier, softRank } from "./evaluate.ts";
|
||||
import type { QualityFacts, SlotSpec } from "./types.ts";
|
||||
|
||||
/** 缺少关键规格时不能声称同套;集号、文件名、种子 ID 不属于版本身份。 */
|
||||
export function seasonReleaseKey(q: QualityFacts): string | null {
|
||||
if (!q.resolution || !q.source || !q.videoCodec || !q.releaseGroup?.name) return null;
|
||||
return JSON.stringify([
|
||||
q.resolution,
|
||||
q.source,
|
||||
q.videoCodec,
|
||||
q.releaseGroup.name
|
||||
.toLowerCase()
|
||||
.split("@")
|
||||
.map((g) => g.trim())
|
||||
.sort(),
|
||||
[...(q.hdr ?? [])].sort(),
|
||||
q.cut ?? "default",
|
||||
]);
|
||||
}
|
||||
|
||||
export function upgradeSpec(spec: SlotSpec): SlotSpec {
|
||||
return {
|
||||
...spec,
|
||||
...(spec.upgradeRules ? { rules: spec.upgradeRules } : {}),
|
||||
...(spec.upgradeHard ? { hard: spec.upgradeHard } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** 优先级、软偏好、规范化规格形成稳定顺序;同档对齐不允许质量降档。 */
|
||||
export function selectSeasonRelease(spec: SlotSpec, versions: readonly QualityFacts[]): QualityFacts | undefined {
|
||||
let target: QualityFacts | undefined;
|
||||
const rank = (q: QualityFacts) => {
|
||||
const tier = qualityTier(q, spec);
|
||||
return [
|
||||
currentPri(spec, q),
|
||||
tier.source,
|
||||
tier.resolution,
|
||||
...softRank(spec, { raw: "", hdr: [], audio: [], ...q } as unknown as ParsedRelease),
|
||||
];
|
||||
};
|
||||
for (const q of versions) {
|
||||
const key = seasonReleaseKey(q);
|
||||
if (!key) continue;
|
||||
const compared = target ? compareRank(rank(q), rank(target)) : 1;
|
||||
if (compared > 0 || (compared === 0 && key < seasonReleaseKey(target!)!)) target = q;
|
||||
}
|
||||
return target;
|
||||
}
|
||||
@@ -91,6 +91,10 @@ export interface SlotSoftPreferences {
|
||||
|
||||
export interface SlotUpgradePolicy {
|
||||
enabled: boolean;
|
||||
/** 洗版阶段显式规则组;未设时沿用既有洗版默认链。 */
|
||||
ruleGroupIds?: string[];
|
||||
/** 追更可混合,洗版按季向同组、同来源和编码的版本收敛。 */
|
||||
consistency?: "season_release";
|
||||
/** 最小改进:来源等级至少提升几级,默认 1(同等级不反复更换) */
|
||||
minTierSteps?: number;
|
||||
/** 来源等级表;默认 REMUX/原盘/蓝光二压=3,WEB-DL=2,WEBRip/HDTV/HDRip=1,DVD=0 */
|
||||
@@ -128,6 +132,9 @@ export interface SlotSpec {
|
||||
hard: SlotHardConditions;
|
||||
/** 规则组:硬排除 → 拒绝;各组须命中某层(ladderRequired);pri 取最后一个带层的组(覆盖 soft.tiers 的 pri) */
|
||||
rules?: SlotRuleChain;
|
||||
/** 服务入口解析的洗版规则,不参与首次补齐。 */
|
||||
upgradeRules?: SlotRuleChain;
|
||||
upgradeHard?: SlotHardConditions;
|
||||
unknownPolicy?: UnknownPolicy | Partial<Record<HardField, UnknownPolicy>>;
|
||||
soft?: SlotSoftPreferences;
|
||||
upgrade?: SlotUpgradePolicy;
|
||||
@@ -151,6 +158,8 @@ export type QualityFacts = Pick<
|
||||
|
||||
/** 评估上下文:站点字段与折算口径(不在标题解析里的事实由调用方给) */
|
||||
export interface EvaluationContext {
|
||||
/** 季级成套目标,由同一收录版本的有效资产及在途计划确定。 */
|
||||
seasonReleaseTarget?: QualityFacts;
|
||||
siteId?: string;
|
||||
/** 副标题/标签等参与关键词匹配的文本 */
|
||||
subtitle?: string;
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
evaluateCandidate,
|
||||
type ParsedRelease,
|
||||
parseRelease,
|
||||
parseRuleString,
|
||||
type QualityFacts,
|
||||
type RuleGroupSpec,
|
||||
rankForSlot,
|
||||
type SlotSpec,
|
||||
seasonReleaseKey,
|
||||
selectAcquisitionPolicy,
|
||||
selectSeasonRelease,
|
||||
} from "../src/index.ts";
|
||||
|
||||
const release = (title: string, facts: Partial<ParsedRelease> = {}): ParsedRelease => ({
|
||||
...parseRelease(title),
|
||||
...facts,
|
||||
});
|
||||
const group = (name: string, rule: string): RuleGroupSpec => ({ name, layers: parseRuleString(rule).layers });
|
||||
const current: QualityFacts = release("Show.S01E01.1080p.WEB-DL.H265-OTHER");
|
||||
const target = release("Show.S01E02.1080p.WEB-DL.H265-CHD");
|
||||
const consistent: SlotSpec = {
|
||||
name: "正片",
|
||||
hard: { resolutions: ["1080p", "2160p"] },
|
||||
rules: { groups: [group("允许收录", "1080P | 4K")] },
|
||||
upgrade: { enabled: true, consistency: "season_release" },
|
||||
};
|
||||
|
||||
describe("按分类选择默认收录策略", () => {
|
||||
const policies = [
|
||||
{ id: "global", scope: {} },
|
||||
{ id: "series", scope: { mediaKinds: ["series" as const] } },
|
||||
{ id: "animation", scope: { categories: { tv: ["动漫剧集"] } } },
|
||||
];
|
||||
|
||||
it("订阅显式指定的策略覆盖类型和分类范围", () => {
|
||||
expect(
|
||||
selectAcquisitionPolicy(policies, { explicitId: "series", mediaKind: "movie", category: "动画电影" })?.id,
|
||||
).toBe("series");
|
||||
});
|
||||
|
||||
it("显式指定的策略不存在时不悄悄回退", () => {
|
||||
expect(
|
||||
selectAcquisitionPolicy(policies, { explicitId: "missing", mediaKind: "series", category: "动漫剧集" }),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("分类匹配优先于类型默认,类型默认优先于全局默认", () => {
|
||||
expect(selectAcquisitionPolicy(policies, { mediaKind: "series", category: "动漫剧集" })?.id).toBe("animation");
|
||||
expect(selectAcquisitionPolicy(policies, { mediaKind: "series", category: "华语剧集" })?.id).toBe("series");
|
||||
expect(selectAcquisitionPolicy(policies, { mediaKind: "movie", category: "动画电影" })?.id).toBe("global");
|
||||
});
|
||||
|
||||
it.each(["movie", "series", "variety"] as const)("%s 不串用另一类型的同名分类策略", (mediaKind) => {
|
||||
const scoped = [
|
||||
{ id: "variety", scope: { mediaKinds: ["variety" as const], categories: { tv: ["同名分类"] } } },
|
||||
{ id: "movie", scope: { categories: { movie: ["同名分类"] } } },
|
||||
{ id: "series", scope: { mediaKinds: ["series" as const], categories: { tv: ["同名分类"] } } },
|
||||
];
|
||||
expect(selectAcquisitionPolicy(scoped, { mediaKind, category: "同名分类" })?.id).toBe(mediaKind);
|
||||
});
|
||||
|
||||
it("同一范围按传入的最新发布顺序取第一个", () => {
|
||||
const latest = { id: "latest", scope: { categories: { tv: ["动漫剧集"] } } };
|
||||
const old = { id: "old", scope: { categories: { tv: ["动漫剧集"] } } };
|
||||
expect(selectAcquisitionPolicy([latest, old], { mediaKind: "series", category: "动漫剧集" })).toBe(latest);
|
||||
expect(selectAcquisitionPolicy([old, latest], { mediaKind: "series", category: "动漫剧集" })).toBe(old);
|
||||
});
|
||||
|
||||
it("分类未知时跳过分类限定策略,仍可使用类型默认", () => {
|
||||
expect(selectAcquisitionPolicy(policies, { mediaKind: "series", category: null })?.id).toBe("series");
|
||||
});
|
||||
|
||||
it("类型或分类没有匹配且无默认时返回 undefined", () => {
|
||||
const scoped = policies.slice(1);
|
||||
expect(selectAcquisitionPolicy(scoped, { mediaKind: "movie", category: "动漫剧集" })).toBeUndefined();
|
||||
expect(selectAcquisitionPolicy([policies[2]!], { mediaKind: "series", category: null })).toBeUndefined();
|
||||
expect(selectAcquisitionPolicy([], { mediaKind: "variety", category: "综艺" })).toBeUndefined();
|
||||
});
|
||||
|
||||
it("空的分类列表仍按类型默认处理", () => {
|
||||
const typeDefault = { id: "type", scope: { mediaKinds: ["series" as const], categories: { movie: [], tv: [] } } };
|
||||
expect(selectAcquisitionPolicy([policies[0]!, typeDefault], { mediaKind: "series", category: null })).toBe(
|
||||
typeDefault,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("整季同套的规格与目标选择", () => {
|
||||
it("发布组大小写、联合组顺序、HDR 顺序不产生新的版本身份", () => {
|
||||
const first = {
|
||||
...target,
|
||||
releaseGroup: { name: "CHD@FLUX", source: "suffix" as const },
|
||||
hdr: ["DV", "HDR10"] as const,
|
||||
};
|
||||
const second = {
|
||||
...target,
|
||||
releaseGroup: { name: "flux@chd", source: "known" as const },
|
||||
hdr: ["HDR10", "DV"] as const,
|
||||
};
|
||||
expect(seasonReleaseKey({ ...first, hdr: [...first.hdr] })).toBe(
|
||||
seasonReleaseKey({ ...second, hdr: [...second.hdr] }),
|
||||
);
|
||||
});
|
||||
|
||||
it.each(["resolution", "source", "videoCodec", "releaseGroup"] as const)(
|
||||
"缺少 %s 时不声称属于完整成套版本",
|
||||
(field) => {
|
||||
expect(seasonReleaseKey({ ...target, [field]: undefined })).toBeNull();
|
||||
},
|
||||
);
|
||||
|
||||
it("来源、分辨率、编码和剪辑不同分别是不同的成套版本", () => {
|
||||
for (const change of [
|
||||
{ source: "BluRay" as const },
|
||||
{ resolution: "2160p" as const },
|
||||
{ videoCodec: "H.264" as const },
|
||||
{ cut: "Extended" as const },
|
||||
{ hdr: ["DV" as const] },
|
||||
]) {
|
||||
expect(seasonReleaseKey({ ...target, ...change })).not.toBe(seasonReleaseKey(target));
|
||||
}
|
||||
});
|
||||
|
||||
const unrestricted: SlotSpec = { name: "主版本", hard: {} };
|
||||
it.each([
|
||||
{
|
||||
name: "来源等级优先于更高分辨率和软偏好",
|
||||
spec: { ...unrestricted, soft: { groups: ["CHD"] } },
|
||||
weaker: release("Show.S01E01.2160p.WEB-DL.H265-CHD"),
|
||||
better: release("Show.S01E02.1080p.BluRay.H264-OTHER"),
|
||||
},
|
||||
{
|
||||
name: "同来源时分辨率优先于软偏好",
|
||||
spec: { ...unrestricted, soft: { groups: ["CHD"] } },
|
||||
weaker: target,
|
||||
better: release("Show.S01E02.2160p.WEB-DL.H265-OTHER"),
|
||||
},
|
||||
{
|
||||
name: "同档规格按发布组软偏好排序",
|
||||
spec: { ...unrestricted, soft: { groups: ["CHD", "OTHER"] } },
|
||||
weaker: release("Show.S01E01.1080p.WEB-DL.H265-OTHER"),
|
||||
better: target,
|
||||
},
|
||||
{
|
||||
name: "同档规格按 HDR 软偏好排序",
|
||||
spec: { ...unrestricted, soft: { hdr: ["DV" as const, "HDR10" as const] } },
|
||||
weaker: release("Show.S01E01.1080p.WEB-DL.HDR.H265-CHD"),
|
||||
better: release("Show.S01E02.1080p.WEB-DL.DV.H265-CHD"),
|
||||
},
|
||||
{
|
||||
name: "同档规格按编码软偏好排序",
|
||||
spec: { ...unrestricted, soft: { codecs: ["H.265" as const, "H.264" as const] } },
|
||||
weaker: release("Show.S01E01.1080p.WEB-DL.H264-CHD"),
|
||||
better: target,
|
||||
},
|
||||
])("$name,成套选择与候选排序一致", ({ spec, weaker, better }) => {
|
||||
expect(rankForSlot(spec, [weaker, better])[0]?.candidate).toBe(better);
|
||||
expect(selectSeasonRelease(spec, [weaker, better])).toBe(better);
|
||||
expect(selectSeasonRelease(spec, [better, weaker])).toBe(better);
|
||||
});
|
||||
|
||||
it("组合优先级先于来源和分辨率等级", () => {
|
||||
const spec: SlotSpec = { ...unrestricted, rules: { groups: [group("来源偏好", "WEBDL > BLURAY")] } };
|
||||
const blue = release("Show.S01E01.2160p.BluRay.H265-OTHER");
|
||||
expect(rankForSlot(spec, [blue, target])[0]?.candidate).toBe(target);
|
||||
expect(selectSeasonRelease(spec, [blue, target])).toBe(target);
|
||||
});
|
||||
|
||||
it("规格不完整的版本不参与成套选择", () => {
|
||||
const incomplete = { ...target, videoCodec: undefined, pri: 1000 };
|
||||
expect(selectSeasonRelease(unrestricted, [incomplete, target])).toBe(target);
|
||||
expect(selectSeasonRelease(unrestricted, [incomplete])).toBeUndefined();
|
||||
});
|
||||
|
||||
it("同档没有偏好时目标不随数据库返回顺序反复变化", () => {
|
||||
const other = release("Show.S01E02.1080p.WEB-DL.H265-OTHER");
|
||||
expect(selectSeasonRelease(unrestricted, [target, other])).toBe(selectSeasonRelease(unrestricted, [other, target]));
|
||||
});
|
||||
});
|
||||
|
||||
describe("追更和洗版分别使用规则", () => {
|
||||
const phased: SlotSpec = {
|
||||
name: "主版本",
|
||||
hard: { resolutions: ["1080p", "2160p"] },
|
||||
rules: { groups: [group("追更", "WEBDL")] },
|
||||
upgradeRules: { groups: [group("洗版", "BLURAY > WEBDL")] },
|
||||
upgrade: { enabled: true },
|
||||
};
|
||||
const blue = release("Show.S01E02.1080p.BluRay.H265-CHD");
|
||||
|
||||
it("补齐缺集按追更规则,洗版规则不提前过滤", () => {
|
||||
expect(evaluateCandidate(phased, target)).toMatchObject({ decision: "accept", pri: 100 });
|
||||
expect(evaluateCandidate(phased, blue)).toMatchObject({
|
||||
decision: "reject",
|
||||
violations: [expect.objectContaining({ field: "rule", expected: "追更" })],
|
||||
});
|
||||
});
|
||||
|
||||
it("已有当前资产时改用洗版规则,并按洗版规则重算当前档位", () => {
|
||||
expect(evaluateCandidate(phased, blue, current)).toMatchObject({ decision: "upgrade", pri: 100, fromPri: 99 });
|
||||
expect(evaluateCandidate(phased, target, current)).toMatchObject({
|
||||
decision: "reject",
|
||||
violations: [expect.objectContaining({ code: "not_higher_priority" })],
|
||||
});
|
||||
});
|
||||
|
||||
it("洗版专用硬条件不阻止追更,但在已有资产时生效", () => {
|
||||
const spec: SlotSpec = { ...phased, upgradeHard: { resolutions: ["2160p"] } };
|
||||
expect(evaluateCandidate(spec, target).decision).toBe("accept");
|
||||
expect(evaluateCandidate(spec, blue, current)).toMatchObject({
|
||||
decision: "reject",
|
||||
violations: [expect.objectContaining({ field: "resolution", code: "not_allowed" })],
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["1080p", "2160p"] as const)("允许分辨率 %s 在追更和洗版阶段均可使用", (resolution) => {
|
||||
const spec: SlotSpec = { name: "主版本", hard: phased.hard, upgrade: { enabled: true } };
|
||||
const candidate = release(`Show.S01E02.${resolution}.BluRay.H265-CHD`);
|
||||
expect(evaluateCandidate(spec, candidate).decision).toBe("accept");
|
||||
expect(evaluateCandidate(spec, candidate, current).decision).toBe("upgrade");
|
||||
});
|
||||
|
||||
it.each(["720p", "4320p"] as const)("范围外的分辨率 %s 不因更优来源或整季对齐而放行", (resolution) => {
|
||||
const candidate = release(`Show.S01E02.${resolution}.BluRay.H265-CHD`);
|
||||
expect(evaluateCandidate(consistent, candidate).decision).toBe("reject");
|
||||
expect(evaluateCandidate(consistent, candidate, current, { seasonReleaseTarget: candidate })).toMatchObject({
|
||||
decision: "reject",
|
||||
violations: expect.arrayContaining([expect.objectContaining({ field: "resolution", code: "not_allowed" })]),
|
||||
});
|
||||
});
|
||||
|
||||
it("分辨率未知时保持待补充,不把未知当作允许范围", () => {
|
||||
expect(evaluateCandidate(phased, { ...target, resolution: undefined })).toMatchObject({
|
||||
decision: "unknown_field",
|
||||
missing: ["resolution"],
|
||||
});
|
||||
});
|
||||
|
||||
it("组合条件的编码未知时,不能因 NOT 自动命中低档", () => {
|
||||
const spec: SlotSpec = { name: "主版本", hard: {}, rules: { groups: [group("编码", "H265 > !H265")] } };
|
||||
expect(evaluateCandidate(spec, release("Show.S01E02.1080p.WEB-DL-CHD"))).toMatchObject({
|
||||
decision: "unknown_field",
|
||||
missing: ["codec"],
|
||||
});
|
||||
});
|
||||
|
||||
it("高档分辨率已明确不满足时,无关的未知编码不阻止已匹配的低档", () => {
|
||||
const spec: SlotSpec = {
|
||||
name: "主版本",
|
||||
hard: {},
|
||||
rules: { groups: [group("组合档位", "4K & H265 > 1080P")] },
|
||||
};
|
||||
expect(evaluateCandidate(spec, release("Show.S01E02.1080p.WEB-DL-CHD"))).toMatchObject({
|
||||
decision: "accept",
|
||||
pri: 99,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("整季同套的同档对齐不绕过升级限制", () => {
|
||||
it("同档的不同发布组可以逐集对齐到选定的整季版本", () => {
|
||||
expect(evaluateCandidate(consistent, target, current, { seasonReleaseTarget: target })).toMatchObject({
|
||||
decision: "upgrade",
|
||||
pri: 100,
|
||||
fromPri: 100,
|
||||
reasons: expect.arrayContaining(["season_release_aligned"]),
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "来源", source: "BluRay" as const, resolution: "1080p" as const },
|
||||
{ name: "分辨率", source: "WEB-DL" as const, resolution: "2160p" as const },
|
||||
])("同档对齐不能降低当前文件的$name", ({ source, resolution }) => {
|
||||
expect(
|
||||
evaluateCandidate(consistent, target, { ...current, source, resolution }, { seasonReleaseTarget: target }),
|
||||
).toMatchObject({ decision: "reject", violations: [expect.objectContaining({ code: "lower_tier" })] });
|
||||
});
|
||||
|
||||
it("当前文件已经同套时不重复整理或下载", () => {
|
||||
expect(evaluateCandidate(consistent, target, target, { seasonReleaseTarget: target }).decision).toBe("reject");
|
||||
const normalizedCurrent = { ...target, releaseGroup: { name: "chd", source: "known" as const } };
|
||||
expect(evaluateCandidate(consistent, target, normalizedCurrent, { seasonReleaseTarget: target }).decision).toBe(
|
||||
"reject",
|
||||
);
|
||||
});
|
||||
|
||||
it("另一套同档资源不能替换已选定的当季目标", () => {
|
||||
const other = release("Show.S01E02.1080p.WEB-DL.H265-FLUX");
|
||||
expect(evaluateCandidate(consistent, other, current, { seasonReleaseTarget: target })).toMatchObject({
|
||||
decision: "reject",
|
||||
violations: [expect.objectContaining({ code: "different_season_release" })],
|
||||
});
|
||||
});
|
||||
|
||||
it("关闭升级时,同档对齐也不能发起替换", () => {
|
||||
const spec = { ...consistent, upgrade: { ...consistent.upgrade!, enabled: false } };
|
||||
expect(evaluateCandidate(spec, target, current, { seasonReleaseTarget: target })).toMatchObject({
|
||||
decision: "reject",
|
||||
violations: [expect.objectContaining({ code: "upgrade_disabled" })],
|
||||
});
|
||||
});
|
||||
|
||||
it("全集洗版不能因同档对齐而接受单集或播出日期资源", () => {
|
||||
const spec: SlotSpec = { ...consistent, upgrade: { ...consistent.upgrade!, mode: "full_pack" } };
|
||||
for (const candidate of [target, { ...target, episodes: undefined, airDate: "20261013" }]) {
|
||||
expect(evaluateCandidate(spec, candidate, current, { seasonReleaseTarget: candidate })).toMatchObject({
|
||||
decision: "reject",
|
||||
violations: [expect.objectContaining({ code: "not_full_pack" })],
|
||||
});
|
||||
}
|
||||
const pack = release("Show.S01.1080p.WEB-DL.H265-CHD");
|
||||
expect(pack.episodes).toBeUndefined();
|
||||
expect(evaluateCandidate(spec, pack, current, { seasonReleaseTarget: pack }).decision).toBe("upgrade");
|
||||
});
|
||||
|
||||
it("不合格蓝光压制组不能因同档对齐绕过名单", () => {
|
||||
const spec: SlotSpec = {
|
||||
...consistent,
|
||||
upgrade: { ...consistent.upgrade!, encodeGroupAllowlist: ["CHD"] },
|
||||
};
|
||||
const old = release("Show.S01E01.1080p.BluRay.x265-OTHER");
|
||||
const unqualified = release("Show.S01E02.1080p.BluRay.x265-BADGRP");
|
||||
expect(unqualified.isEncode).toBe(true);
|
||||
expect(evaluateCandidate(spec, unqualified, old, { seasonReleaseTarget: unqualified })).toMatchObject({
|
||||
decision: "reject",
|
||||
violations: [expect.objectContaining({ code: "encode_not_qualified" })],
|
||||
});
|
||||
const qualified = release("Show.S01E02.1080p.BluRay.x265-chd");
|
||||
expect(evaluateCandidate(spec, qualified, old, { seasonReleaseTarget: qualified }).decision).toBe("upgrade");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ field: "resolution", missing: "resolution" },
|
||||
{ field: "source", missing: "source" },
|
||||
{ field: "videoCodec", missing: "codec" },
|
||||
{ field: "releaseGroup", missing: "group" },
|
||||
] as const)("候选缺少 $field 时不执行成套替换", ({ field, missing }) => {
|
||||
const candidate = { ...target, [field]: undefined };
|
||||
expect(evaluateCandidate(consistent, candidate, current, { seasonReleaseTarget: target })).toMatchObject({
|
||||
decision: "unknown_field",
|
||||
missing: expect.arrayContaining([missing]),
|
||||
});
|
||||
});
|
||||
|
||||
it("季目标缺失或规格不完整时等待,不自行认定当前候选是目标", () => {
|
||||
expect(evaluateCandidate(consistent, target, current)).toMatchObject({
|
||||
decision: "unknown_field",
|
||||
missing: ["current_asset"],
|
||||
});
|
||||
expect(
|
||||
evaluateCandidate(consistent, target, current, { seasonReleaseTarget: { ...target, videoCodec: undefined } }),
|
||||
).toMatchObject({ decision: "unknown_field", missing: ["current_asset"] });
|
||||
});
|
||||
});
|
||||
@@ -194,7 +194,7 @@ describe("MP 过滤规则差分(filter-cases.json)", () => {
|
||||
});
|
||||
});
|
||||
|
||||
/** 生成/更新登记时用:UMT_FILTER_DUMP=1 deno task test 打印当前不一致 */
|
||||
/** 生成/更新登记时用:UMT_FILTER_DUMP=1 bun run test 打印当前不一致 */
|
||||
if (process.env.UMT_FILTER_DUMP) {
|
||||
const rows: Record<string, unknown> = {};
|
||||
for (const c of fx.cases) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# @umt/mapping
|
||||
|
||||
domain ↔ contracts 的集中映射(W1 审查 B5)。只依赖 `@umt/domain` 与 `@umt/contracts`,不依赖 db / adapters。server 各模块需要在两套取值之间转换时一律走这里,不要在模块里各写一份。
|
||||
domain ↔ contracts 的语义映射。只依赖 `@umt/domain` 与 `@umt/contracts`,不依赖 db / adapters。server 各模块需要在两套取值之间转换时一律走这里,不要在模块里各写一份。
|
||||
|
||||
db 行(snake_case)↔ contracts(camelCase)的转换不在本包;adapters 的 `SiteRelease` 以结构类型 `SiteReleaseLike` 接入(兼容性由 `apps/server/test/mapping-compat.test.ts` 在类型层保证)。
|
||||
|
||||
@@ -10,12 +10,14 @@ db 行(snake_case)↔ contracts(camelCase)的转换不在本包;adapters 的 `Si
|
||||
| `evidence.ts` | `toContractEvidenceLevel` / `toDomainConfidence` / `toDomainMediaType` / `toContractMediaKind` / `toDomainEntityType` / `toContractEntityType` | `needs_confirm ↔ needs_review`,`manual → rule_confirmed`,`rejected → null`;`tv ↔ series/variety` |
|
||||
| `identity.ts` | `toContractResolveStatus` / `toReleaseResolutionStatus` / `rankedToIdentityCreate` / `localIdentityCandidate` | `auto/ambiguous/no_match → resolved|proposed/ambiguous/unidentifiable → source_release.resolution_status`;排名候选 → `ExternalIdentityCreate`;本地作品 → 本地优先候选 |
|
||||
| `slot.ts` | `toContractSlotStatus` / `toDomainSlotStatus` / `slotSpecFromRow` / `slotSpecToColumns` / `toContractSlotEvaluation` | Slot 状态;规则 JSON 列 ↔ `SlotSpec`;评估结果 → `{ decision, acceptable, rejected[] }` |
|
||||
| `release.ts` | `toParsedColumns` / `fromParsedColumns` / `toContractContentType` / `toDomainContentType` / `toContractSegment` / `toDomainSegment` | `ParsedRelease ↔ source_release.parsed + parser_version`(版本不符返回 null,重解析) |
|
||||
| `release.ts` | `toParsedColumns` / `fromParsedColumns` / `toContractSegment` / `toDomainSegment` | `ParsedRelease ↔ source_release.parsed + parser_version`(版本不符返回 null,重解析) |
|
||||
| `site-release.ts` | `siteReleaseToSourceRelease` / `sourceReleaseToSiteRelease` / `SiteReleaseLike` | 站点资源 ↔ `SourceReleaseCreate` / `SourceRelease`;downloadUrl 不入库 |
|
||||
| `path-binding.ts` | `toDomainPathBinding` / `toPathBindingCreate` / `parseTemplateVersion` | `entityId/scope/season/templateVersion:number/active ↔ mediaEntityId/bindingKind/seasonNumber/templateVersion:string/status` |
|
||||
| `organize.ts` | `organizePlanToRecord` / `organizePlanToFileResults` / `organizePlanToCapabilityOutput` / `organizationStatusFromFiles` / `toOrganizationAction` / `toOrganizeOperation` / `toFileRole` | `OrganizePlan →` organization_record / organization_file_result 草稿 / organize.plan 输出 |
|
||||
|
||||
## 规则
|
||||
|
||||
- 同一语义的类型直接共用,不维护同值枚举的双向转换表。只在取值或含义存在实际差异时添加映射。
|
||||
|
||||
- 映射表一律写成 `Record<一侧的联合类型, …>`:domain 增删取值时 typecheck 失败;contracts 侧在 `test/enums.test.ts` 按 `z.enum(...).options` 逐个断言,新增取值时测试失败。
|
||||
- 新增映射先写测试(往返、schema 合法)。输出给 contracts 的对象都用对应 zod schema 在测试里 `safeParse` 一遍。
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "@umt/mapping",
|
||||
"version": "0.3.44",
|
||||
"exports": { ".": "./src/index.ts" },
|
||||
"tasks": {
|
||||
"test": "deno run -A npm:vitest run",
|
||||
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
|
||||
}
|
||||
}
|
||||
@@ -4,5 +4,14 @@
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"version": "0.3.44",
|
||||
"scripts": {
|
||||
"test": "bun x --bun vitest run",
|
||||
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
|
||||
},
|
||||
"dependencies": {
|
||||
"@umt/contracts": "workspace:*",
|
||||
"@umt/domain": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,5 @@
|
||||
import type { ContentType as ContractContentType, Segment as ContractSegment, JsonObject } from "@umt/contracts";
|
||||
import {
|
||||
type ContentType as DomainContentType,
|
||||
type Segment as DomainSegment,
|
||||
PARSER_VERSION,
|
||||
type ParsedRelease,
|
||||
} from "@umt/domain";
|
||||
import type { Segment as ContractSegment, JsonObject } from "@umt/contracts";
|
||||
import { type Segment as DomainSegment, PARSER_VERSION, type ParsedRelease } from "@umt/domain";
|
||||
|
||||
// 标题解析结果 ↔ source_release.parsed / parser_version 列;综艺内容类型与分段 ↔ content_unit 列。
|
||||
|
||||
@@ -28,29 +23,6 @@ export function fromParsedColumns(cols: { parsed: JsonObject; parserVersion: str
|
||||
return p as unknown as ParsedRelease;
|
||||
}
|
||||
|
||||
const CONTENT_TYPE: Record<DomainContentType, ContractContentType> = {
|
||||
unknown: "unknown",
|
||||
main: "main",
|
||||
extra: "extra",
|
||||
behind_the_scenes: "behind_the_scenes",
|
||||
performance_only: "performance_only",
|
||||
preview: "preview",
|
||||
reaction: "reaction",
|
||||
};
|
||||
const CONTENT_TYPE_BACK: Record<ContractContentType, DomainContentType> = {
|
||||
main: "main",
|
||||
extra: "extra",
|
||||
behind_the_scenes: "behind_the_scenes",
|
||||
performance_only: "performance_only",
|
||||
preview: "preview",
|
||||
reaction: "reaction",
|
||||
unknown: "unknown",
|
||||
};
|
||||
export const DOMAIN_CONTENT_TYPES = Object.keys(CONTENT_TYPE) as DomainContentType[];
|
||||
|
||||
export const toContractContentType = (t: DomainContentType): ContractContentType => CONTENT_TYPE[t];
|
||||
export const toDomainContentType = (t: ContractContentType): DomainContentType => CONTENT_TYPE_BACK[t];
|
||||
|
||||
const SEGMENT: Record<DomainSegment, ContractSegment> = {
|
||||
full: "full",
|
||||
upper: "upper",
|
||||
|
||||
@@ -117,15 +117,13 @@ describe("身份解析状态", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("Slot / 内容类型 / 分段 / 整理动作", () => {
|
||||
describe("Slot / 分段 / 整理动作", () => {
|
||||
it("Slot 状态两侧一一对应", () => {
|
||||
expect(sorted(M.DOMAIN_SLOT_STATUSES)).toEqual(sorted(C.SlotStatus.options));
|
||||
for (const s of C.SlotStatus.options) expect(M.toContractSlotStatus(M.toDomainSlotStatus(s))).toBe(s);
|
||||
});
|
||||
|
||||
it("综艺内容类型两侧一一对应;分段 unknown ↔ 缺失", () => {
|
||||
expect(sorted(M.DOMAIN_CONTENT_TYPES)).toEqual(sorted(C.ContentType.options));
|
||||
for (const t of C.ContentType.options) expect(M.toContractContentType(M.toDomainContentType(t))).toBe(t);
|
||||
it("分段 unknown ↔ 缺失", () => {
|
||||
expect(sorted([...M.DOMAIN_SEGMENTS, "unknown"])).toEqual(sorted(C.Segment.options));
|
||||
for (const s of C.Segment.options) expect(M.toContractSegment(M.toDomainSegment(s))).toBe(s);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "@umt/runtime",
|
||||
"version": "0.3.44",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts",
|
||||
"./worker": "./src/worker.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json",
|
||||
"test": "bun x --bun vitest run"
|
||||
},
|
||||
"dependencies": {}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { readFileSync, writeSync } from "node:fs";
|
||||
import { mkdir, open, opendir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { hostname } from "node:os";
|
||||
import { Readable, Writable } from "node:stream";
|
||||
|
||||
export const runtimeHostname = hostname;
|
||||
export const runtimeReadTextFile = (path: string) => readFile(path, "utf8");
|
||||
export const runtimeReadTextFileSync = (path: string) => readFileSync(path, "utf8");
|
||||
export const runtimeWriteTextFile = (path: string, text: string) => writeFile(path, text);
|
||||
export const runtimeMkdir = mkdir;
|
||||
export const runtimeRename = rename;
|
||||
export const runtimeRemove = rm;
|
||||
export const runtimeReadDir = opendir;
|
||||
export async function runtimeStat(path: string) {
|
||||
const s = await stat(path);
|
||||
return { size: s.size, isFile: s.isFile(), isDirectory: s.isDirectory() };
|
||||
}
|
||||
export async function runtimeOpen(
|
||||
path: string,
|
||||
opts: { read?: boolean; write?: boolean; create?: boolean; truncate?: boolean },
|
||||
) {
|
||||
const f = await open(path, opts.write ? (opts.truncate ? "w" : "a") : "r");
|
||||
return {
|
||||
stat: async () => {
|
||||
const s = await f.stat();
|
||||
return { size: s.size, isFile: s.isFile(), isDirectory: s.isDirectory() };
|
||||
},
|
||||
close: () => f.close(),
|
||||
write: async (bytes: Uint8Array) => (await f.write(bytes)).bytesWritten,
|
||||
get readable() {
|
||||
return Readable.toWeb(f.createReadStream({ autoClose: true })) as unknown as ReadableStream<Uint8Array>;
|
||||
},
|
||||
};
|
||||
}
|
||||
export const runtimeWriteSync = (bytes: Uint8Array) => writeSync(1, bytes);
|
||||
export const runtimeWrite = async (bytes: Uint8Array) => runtimeWriteSync(bytes);
|
||||
export const runtimeUnrefTimer = (timer: unknown) => (timer as { unref?: () => void }).unref?.();
|
||||
|
||||
type CommandOptions = {
|
||||
args?: string[];
|
||||
stdin?: "piped" | "null" | "inherit";
|
||||
stdout?: "piped" | "null" | "inherit";
|
||||
stderr?: "piped" | "null" | "inherit";
|
||||
env?: Record<string, string>;
|
||||
clearEnv?: boolean;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
export type ChildProcess = ReturnType<Command["spawn"]>;
|
||||
export class Command {
|
||||
constructor(
|
||||
private cmd: string,
|
||||
private opts: CommandOptions = {},
|
||||
) {}
|
||||
spawn() {
|
||||
const o = this.opts;
|
||||
const io = (s: string | undefined) => (s === "piped" ? "pipe" : s === "null" ? "ignore" : "inherit");
|
||||
const p = spawn(this.cmd, o.args ?? [], {
|
||||
stdio: [io(o.stdin), io(o.stdout ?? "piped"), io(o.stderr ?? "piped")],
|
||||
env: o.clearEnv ? o.env : { ...process.env, ...o.env },
|
||||
signal: o.signal,
|
||||
});
|
||||
const status = new Promise<{ success: boolean; code: number; signal: string | null }>((resolve, reject) => {
|
||||
p.once("error", reject);
|
||||
p.once("close", (code, signal) => resolve({ success: code === 0, code: code ?? 1, signal }));
|
||||
});
|
||||
// Attach a handler immediately: callers may still be writing stdin when spawn fails.
|
||||
status.catch(() => {});
|
||||
const child = {
|
||||
pid: p.pid!,
|
||||
stdin: p.stdin ? Writable.toWeb(p.stdin) : new WritableStream<Uint8Array>(),
|
||||
stdout: p.stdout
|
||||
? (Readable.toWeb(p.stdout) as unknown as ReadableStream<Uint8Array>)
|
||||
: new ReadableStream<Uint8Array>({
|
||||
start(c) {
|
||||
c.close();
|
||||
},
|
||||
}),
|
||||
stderr: p.stderr
|
||||
? (Readable.toWeb(p.stderr) as unknown as ReadableStream<Uint8Array>)
|
||||
: new ReadableStream<Uint8Array>({
|
||||
start(c) {
|
||||
c.close();
|
||||
},
|
||||
}),
|
||||
status,
|
||||
kill: (signal: string = "SIGTERM") => {
|
||||
p.kill(signal as NodeJS.Signals);
|
||||
},
|
||||
unref: () => {
|
||||
p.unref();
|
||||
},
|
||||
output: async () => {
|
||||
const [s, stdout, stderr] = await Promise.all([
|
||||
status,
|
||||
new Response(child.stdout).bytes(),
|
||||
new Response(child.stderr).bytes(),
|
||||
]);
|
||||
return { ...s, stdout, stderr };
|
||||
},
|
||||
};
|
||||
return child;
|
||||
}
|
||||
output() {
|
||||
return this.spawn().output();
|
||||
}
|
||||
}
|
||||
|
||||
export interface NetAddr {
|
||||
hostname: string;
|
||||
port: number;
|
||||
}
|
||||
export interface HttpServer {
|
||||
addr: NetAddr;
|
||||
finished: Promise<void>;
|
||||
shutdown(): Promise<void>;
|
||||
}
|
||||
type SocketData = { client: WebSocket };
|
||||
const upgrades = new WeakMap<Request, Bun.Server<SocketData>>();
|
||||
export function serve(
|
||||
opts: { port: number; hostname?: string; signal?: AbortSignal; onListen?: (addr: NetAddr) => void },
|
||||
fetcher: (req: Request) => Response | Promise<Response>,
|
||||
): HttpServer {
|
||||
let finish!: () => void;
|
||||
const finished = new Promise<void>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
const server = Bun.serve<SocketData>({
|
||||
port: opts.port,
|
||||
hostname: opts.hostname,
|
||||
async fetch(req, server) {
|
||||
upgrades.set(req, server);
|
||||
const response = await fetcher(req);
|
||||
upgrades.delete(req);
|
||||
return upgradedRequests.has(req) ? undefined : response;
|
||||
},
|
||||
websocket: {
|
||||
open(ws) {
|
||||
const c = ws.data.client;
|
||||
socketBindings.get(c)?.(ws);
|
||||
socketBindings.delete(c);
|
||||
Object.defineProperty(c, "readyState", { value: WebSocket.OPEN, configurable: true });
|
||||
c.dispatchEvent(new Event("open"));
|
||||
},
|
||||
message(ws, data) {
|
||||
ws.data.client.dispatchEvent(
|
||||
new MessageEvent("message", {
|
||||
data:
|
||||
typeof data === "string" ? data : data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength),
|
||||
}),
|
||||
);
|
||||
},
|
||||
close(ws, code, reason) {
|
||||
const c = ws.data.client;
|
||||
Object.defineProperty(c, "readyState", { value: WebSocket.CLOSED, configurable: true });
|
||||
c.dispatchEvent(new CloseEvent("close", { code, reason }));
|
||||
},
|
||||
},
|
||||
});
|
||||
const addr = { hostname: opts.hostname ?? "0.0.0.0", port: server.port! };
|
||||
opts.onListen?.(addr);
|
||||
const shutdown = async () => {
|
||||
await server.stop(true);
|
||||
finish();
|
||||
};
|
||||
opts.signal?.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
void shutdown();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
if (opts.signal?.aborted) void shutdown();
|
||||
return { addr, finished, shutdown };
|
||||
}
|
||||
const upgradedRequests = new WeakSet<Request>();
|
||||
export function upgradeWebSocket(req: Request, opts: { protocol?: string } = {}) {
|
||||
const server = upgrades.get(req);
|
||||
if (!server) throw new Error("WebSocket upgrade requires a Bun HTTP server");
|
||||
// EventTarget facade preserves the bridge's browser WebSocket callbacks.
|
||||
const client = new EventTarget() as WebSocket;
|
||||
let ws: Bun.ServerWebSocket<SocketData> | undefined;
|
||||
Object.defineProperty(client, "readyState", { value: WebSocket.CONNECTING, configurable: true });
|
||||
for (const event of ["open", "message", "close", "error"]) {
|
||||
let callback: ((e: Event) => void) | null = null;
|
||||
Object.defineProperty(client, `on${event}`, {
|
||||
get: () => callback,
|
||||
set: (fn) => {
|
||||
if (callback) client.removeEventListener(event, callback);
|
||||
callback = fn;
|
||||
if (fn) client.addEventListener(event, fn);
|
||||
},
|
||||
});
|
||||
}
|
||||
client.send = (data) => {
|
||||
ws?.send(data as string | ArrayBuffer);
|
||||
};
|
||||
client.close = (code, reason) => {
|
||||
ws?.close(code, reason);
|
||||
};
|
||||
const data = { client };
|
||||
// Capture the native socket in open before dispatching the bridge callbacks.
|
||||
socketBindings.set(client, (socket) => {
|
||||
ws = socket;
|
||||
});
|
||||
if (!server.upgrade(req, { data, headers: opts.protocol ? { "Sec-WebSocket-Protocol": opts.protocol } : undefined }))
|
||||
throw new Error("WebSocket upgrade failed");
|
||||
const response = new Response(null);
|
||||
upgradedRequests.add(req);
|
||||
return { socket: client, response };
|
||||
}
|
||||
const socketBindings = new WeakMap<WebSocket, (ws: Bun.ServerWebSocket<SocketData>) => void>();
|
||||
@@ -0,0 +1,262 @@
|
||||
import { Readable } from "node:stream";
|
||||
import { type ChildProcess, Command, runtimeWriteSync } from "./index.ts";
|
||||
|
||||
/** 本地执行协议。业务类型、重试、权限与超时策略由调用方负责。 */
|
||||
export interface WorkerRequest {
|
||||
id: string;
|
||||
task: string;
|
||||
input: unknown;
|
||||
}
|
||||
export interface WorkerError {
|
||||
code: string;
|
||||
message: string;
|
||||
detail?: Record<string, unknown>;
|
||||
}
|
||||
export type WorkerReply =
|
||||
| { id: string; type: "result"; result: unknown }
|
||||
| { id: string; type: "error"; error: WorkerError };
|
||||
export type WorkerMessage = WorkerReply | { id: string; type: "progress"; value: unknown };
|
||||
|
||||
export class WorkerTransportError extends Error {
|
||||
constructor(
|
||||
readonly kind: "spawn" | "protocol" | "exit" | "size",
|
||||
message: string,
|
||||
override readonly cause?: unknown,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "WorkerTransportError";
|
||||
}
|
||||
}
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
export function encodeWorkerMessage(value: WorkerRequest | WorkerMessage, maxBytes: number): Uint8Array {
|
||||
const bytes = encoder.encode(`${JSON.stringify(value)}\n`);
|
||||
if (bytes.byteLength > maxBytes) throw new WorkerTransportError("size", `worker 消息超过 ${maxBytes} 字节`);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/** 按字节限制一行的大小;UTF-8 与换行可以跨管道分块。EOF 不能代替消息结束符。 */
|
||||
export async function* workerLines(
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||
maxBytes: number,
|
||||
): AsyncGenerator<string> {
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true });
|
||||
let parts: string[] = [];
|
||||
let length = 0;
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
if (length) throw new WorkerTransportError("protocol", "worker 消息未以换行结束");
|
||||
return;
|
||||
}
|
||||
let start = 0;
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
if (value[i] !== 10) continue;
|
||||
length += i - start + 1;
|
||||
if (length > maxBytes) throw new WorkerTransportError("size", `worker 消息超过 ${maxBytes} 字节`);
|
||||
parts.push(decoder.decode(value.subarray(start, i), { stream: true }), decoder.decode());
|
||||
const line = parts.join("");
|
||||
parts = [];
|
||||
length = 0;
|
||||
start = i + 1;
|
||||
if (!line) throw new WorkerTransportError("protocol", "worker 返回空消息");
|
||||
yield line;
|
||||
}
|
||||
length += value.length - start;
|
||||
if (length > maxBytes) throw new WorkerTransportError("size", `worker 消息超过 ${maxBytes} 字节`);
|
||||
parts.push(decoder.decode(value.subarray(start), { stream: true }));
|
||||
}
|
||||
}
|
||||
|
||||
export async function readWorkerRequestText(maxBytes: number): Promise<string> {
|
||||
const stream = Readable.toWeb(process.stdin) as unknown as ReadableStream<Uint8Array>;
|
||||
const reader = stream.getReader();
|
||||
try {
|
||||
let text: string | undefined;
|
||||
for await (const line of workerLines(reader, maxBytes)) {
|
||||
if (text !== undefined) throw new WorkerTransportError("protocol", "一次性 worker 只能接收一个请求");
|
||||
text = line;
|
||||
}
|
||||
if (text === undefined) throw new WorkerTransportError("protocol", "worker 没有收到请求");
|
||||
return text;
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
export function decodeWorkerRequest(text: string): WorkerRequest {
|
||||
const req = JSON.parse(text) as WorkerRequest;
|
||||
if (!req || typeof req.id !== "string" || !req.id || typeof req.task !== "string" || !("input" in req))
|
||||
throw new WorkerTransportError("protocol", "worker 请求格式无效");
|
||||
return req;
|
||||
}
|
||||
|
||||
export function writeWorkerMessage(message: WorkerMessage, maxBytes: number): void {
|
||||
const bytes = encodeWorkerMessage(message, maxBytes);
|
||||
let offset = 0;
|
||||
while (offset < bytes.length) offset += runtimeWriteSync(bytes.subarray(offset));
|
||||
}
|
||||
|
||||
function decodeMessage(line: string, id: string): WorkerMessage {
|
||||
const value = JSON.parse(line) as WorkerMessage;
|
||||
if (!value || value.id !== id) throw new WorkerTransportError("protocol", "worker 响应不属于当前请求");
|
||||
if (value.type === "result" && "result" in value) return value;
|
||||
if (value.type === "progress" && "value" in value) return value;
|
||||
if (value.type === "error" && typeof value.error?.code === "string" && typeof value.error?.message === "string")
|
||||
return value;
|
||||
throw new WorkerTransportError("protocol", "worker 响应格式无效");
|
||||
}
|
||||
|
||||
/** 一个一次性进程连接。只管通信与回收,不重试、不决定超时、不安装权限策略。 */
|
||||
export class WorkerConnection {
|
||||
readonly pid: number;
|
||||
private readonly child: ChildProcess;
|
||||
private readonly stdout: ReadableStreamDefaultReader<Uint8Array>;
|
||||
private readonly stderrReader: ReadableStreamDefaultReader<Uint8Array>;
|
||||
private reject: ((error: unknown) => void) | null = null;
|
||||
private terminating: unknown;
|
||||
private killTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
private settled = false;
|
||||
private started = false;
|
||||
private stderr = "";
|
||||
|
||||
constructor(command: { cmd: string; args: string[] }, env?: { values: Record<string, string>; clear?: boolean }) {
|
||||
try {
|
||||
this.child = new Command(command.cmd, {
|
||||
args: command.args,
|
||||
stdin: "piped",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
env: env?.values,
|
||||
clearEnv: env?.clear,
|
||||
}).spawn();
|
||||
} catch (error) {
|
||||
throw new WorkerTransportError("spawn", "worker 子进程无法启动", error);
|
||||
}
|
||||
this.pid = this.child.pid;
|
||||
this.stdout = this.child.stdout.getReader();
|
||||
this.stderrReader = this.child.stderr.getReader();
|
||||
}
|
||||
|
||||
/** graceMs > 0 时先 SIGTERM 供文件操作清理;解析可直接 SIGKILL。等待始终有界。 */
|
||||
terminate(error: unknown, graceMs = 0, onForceKill?: () => void): void {
|
||||
if (this.settled || this.terminating !== undefined) return;
|
||||
this.terminating = error;
|
||||
const force = () => {
|
||||
try {
|
||||
this.child.kill("SIGKILL");
|
||||
onForceKill?.();
|
||||
} catch {
|
||||
// 已退出
|
||||
}
|
||||
this.child.unref();
|
||||
this.fail(error);
|
||||
};
|
||||
if (graceMs === 0) {
|
||||
force();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.child.kill("SIGTERM");
|
||||
} catch {
|
||||
// 已退出
|
||||
}
|
||||
this.killTimer = setTimeout(force, graceMs);
|
||||
}
|
||||
|
||||
request(
|
||||
req: WorkerRequest,
|
||||
options: { maxRequestBytes: number; maxResponseBytes: number; onProgress?: (value: unknown) => void },
|
||||
): Promise<WorkerReply> {
|
||||
if (this.started) throw new Error("一次性 worker 连接不能重复请求");
|
||||
this.started = true;
|
||||
if (this.terminating !== undefined) return Promise.reject(this.terminating);
|
||||
if (!this.pid)
|
||||
return this.child.status.then(
|
||||
() => {
|
||||
this.cleanup();
|
||||
throw new WorkerTransportError("exit", "worker 未启动");
|
||||
},
|
||||
(error) => {
|
||||
this.cleanup();
|
||||
throw new WorkerTransportError("spawn", "worker 子进程无法启动", error);
|
||||
},
|
||||
);
|
||||
return new Promise((resolve, reject) => {
|
||||
this.reject = reject;
|
||||
const errPump = this.readStderr();
|
||||
let final: WorkerReply | undefined;
|
||||
const pump = (async () => {
|
||||
for await (const line of workerLines(this.stdout, options.maxResponseBytes)) {
|
||||
const msg = decodeMessage(line, req.id);
|
||||
if (final) throw new WorkerTransportError("protocol", "worker 结果之后仍发送消息");
|
||||
if (msg.type === "progress") options.onProgress?.(msg.value);
|
||||
else final = msg;
|
||||
}
|
||||
})();
|
||||
pump.catch((error) => this.terminate(this.transportError(error)));
|
||||
this.child.status.then(
|
||||
async (status) => {
|
||||
try {
|
||||
await Promise.all([pump, errPump]);
|
||||
if (this.settled) return;
|
||||
if (this.terminating !== undefined) return this.fail(this.terminating);
|
||||
if (!status.success || !final)
|
||||
throw new WorkerTransportError(
|
||||
"exit",
|
||||
`worker 异常退出(code=${status.code} signal=${status.signal}):${this.stderr}`,
|
||||
);
|
||||
this.cleanup();
|
||||
resolve(final);
|
||||
} catch (error) {
|
||||
this.fail(this.terminating ?? this.transportError(error));
|
||||
}
|
||||
},
|
||||
(error) => this.fail(new WorkerTransportError("spawn", "worker 子进程无法启动", error)),
|
||||
);
|
||||
(async () => {
|
||||
const bytes = encodeWorkerMessage(req, options.maxRequestBytes);
|
||||
const writer = this.child.stdin.getWriter();
|
||||
try {
|
||||
await writer.write(bytes);
|
||||
await writer.close();
|
||||
} finally {
|
||||
writer.releaseLock();
|
||||
}
|
||||
})().catch((error) => this.terminate(this.transportError(error)));
|
||||
if (this.terminating !== undefined) this.fail(this.terminating);
|
||||
});
|
||||
}
|
||||
|
||||
private transportError(error: unknown): WorkerTransportError {
|
||||
return error instanceof WorkerTransportError
|
||||
? error
|
||||
: new WorkerTransportError("protocol", `worker 通信失败:${(error as Error).message}`, error);
|
||||
}
|
||||
|
||||
private async readStderr(): Promise<void> {
|
||||
const decoder = new TextDecoder();
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await this.stderrReader.read();
|
||||
if (done) return;
|
||||
this.stderr = (this.stderr + decoder.decode(value, { stream: true })).slice(-2000);
|
||||
}
|
||||
} catch {
|
||||
// 终止时管道可能断开
|
||||
}
|
||||
}
|
||||
|
||||
private fail(error: unknown): void {
|
||||
if (this.settled) return;
|
||||
this.cleanup();
|
||||
this.reject?.(error);
|
||||
}
|
||||
|
||||
private cleanup(): void {
|
||||
this.settled = true;
|
||||
clearTimeout(this.killTimer);
|
||||
void this.stdout.cancel().catch(() => {});
|
||||
void this.stderrReader.cancel().catch(() => {});
|
||||
}
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
import { decodeWorkerRequest, readWorkerRequestText, writeWorkerMessage } from "../../src/worker.ts";
|
||||
|
||||
const req = decodeWorkerRequest(await readWorkerRequestText(4096));
|
||||
const send = (type: "result", result: unknown) => writeWorkerMessage({ id: req.id, type, result }, 4096);
|
||||
switch (req.task) {
|
||||
case "normal":
|
||||
writeWorkerMessage({ id: req.id, type: "progress", value: { bytes: 2 } }, 4096);
|
||||
send("result", req.input);
|
||||
break;
|
||||
case "large":
|
||||
writeWorkerMessage({ id: req.id, type: "result", result: "中文".repeat(200_000) }, 2 * 1024 * 1024);
|
||||
break;
|
||||
case "wrong-id":
|
||||
writeWorkerMessage({ id: "other", type: "result", result: true }, 4096);
|
||||
break;
|
||||
case "truncated":
|
||||
process.stdout.write('{"id":"1"');
|
||||
break;
|
||||
case "oversized":
|
||||
process.stdout.write("x".repeat(8192));
|
||||
break;
|
||||
case "after-result":
|
||||
send("result", true);
|
||||
send("result", false);
|
||||
break;
|
||||
case "failed-exit":
|
||||
send("result", true);
|
||||
process.exitCode = 1;
|
||||
break;
|
||||
case "hang":
|
||||
setInterval(() => {}, 1000);
|
||||
break;
|
||||
case "stderr":
|
||||
process.stderr.write("x".repeat(65536));
|
||||
send("result", true);
|
||||
break;
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { WorkerConnection, workerLines } from "../src/worker.ts";
|
||||
|
||||
const fixture = fileURLToPath(new URL("./fixtures/worker.ts", import.meta.url));
|
||||
const limits = { maxRequestBytes: 4096, maxResponseBytes: 4096 };
|
||||
function connection() {
|
||||
return new WorkerConnection({ cmd: process.execPath, args: [fixture] });
|
||||
}
|
||||
const request = (task: string) => ({ id: "1", task, input: "中文" });
|
||||
|
||||
describe("worker 管道与回收", () => {
|
||||
it("拆开的 UTF-8 和多行消息仍能正确分帧", async () => {
|
||||
const bytes = new TextEncoder().encode("中文\n第二行\n");
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(c) {
|
||||
for (const b of bytes) c.enqueue(Uint8Array.of(b));
|
||||
c.close();
|
||||
},
|
||||
});
|
||||
const lines = [];
|
||||
for await (const line of workerLines(stream.getReader(), 64)) lines.push(line);
|
||||
expect(lines).toEqual(["中文", "第二行"]);
|
||||
});
|
||||
|
||||
it("转发进度、返回结果,并持续排空大量 stderr", async () => {
|
||||
const seen: unknown[] = [];
|
||||
expect(await connection().request(request("normal"), { ...limits, onProgress: (p) => seen.push(p) })).toEqual({
|
||||
id: "1",
|
||||
type: "result",
|
||||
result: "中文",
|
||||
});
|
||||
expect(seen).toEqual([{ bytes: 2 }]);
|
||||
expect(await connection().request(request("stderr"), limits)).toMatchObject({ result: true });
|
||||
});
|
||||
|
||||
it("大结果跨多个管道分块也完整返回", async () => {
|
||||
const reply = await connection().request(request("large"), { ...limits, maxResponseBytes: 2 * 1024 * 1024 });
|
||||
expect(reply).toEqual({ id: "1", type: "result", result: "中文".repeat(200_000) });
|
||||
});
|
||||
|
||||
it.each([
|
||||
["wrong-id", "protocol"],
|
||||
["truncated", "protocol"],
|
||||
["oversized", "size"],
|
||||
["after-result", "protocol"],
|
||||
["failed-exit", "exit"],
|
||||
])("拒绝 %s,而不接受部分或错误响应", async (task, kind) => {
|
||||
await expect(connection().request(request(task), limits)).rejects.toMatchObject({ kind });
|
||||
});
|
||||
|
||||
it("请求大小超限也回收已启动的进程", async () => {
|
||||
const c = connection();
|
||||
await expect(c.request({ ...request("hang"), input: "x".repeat(8192) }, limits)).rejects.toMatchObject({
|
||||
kind: "size",
|
||||
});
|
||||
});
|
||||
|
||||
it("启动失败保留 spawn 分类", async () => {
|
||||
const c = new WorkerConnection({ cmd: "/nonexistent/umt-worker", args: [] });
|
||||
await expect(c.request(request("normal"), limits)).rejects.toMatchObject({ kind: "spawn" });
|
||||
});
|
||||
|
||||
it("执行中的请求可终止,返回调用方的原因", async () => {
|
||||
const c = connection();
|
||||
const pending = c.request(request("hang"), limits);
|
||||
const reason = new Error("调用方取消");
|
||||
c.terminate(reason, 50);
|
||||
await expect(pending).rejects.toBe(reason);
|
||||
});
|
||||
|
||||
it("进度消费者抛错时回收连接,不让任务继续等待", async () => {
|
||||
await expect(
|
||||
connection().request(request("normal"), {
|
||||
...limits,
|
||||
onProgress: () => {
|
||||
throw new Error("进度无效");
|
||||
},
|
||||
}),
|
||||
).rejects.toMatchObject({ kind: "protocol", message: expect.stringContaining("进度无效") });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1 @@
|
||||
{ "extends": "../../config/tsconfig.base.json", "include": ["src", "test"] }
|
||||
@@ -13,7 +13,7 @@
|
||||
| 导出 | 作用 |
|
||||
| --- | --- |
|
||||
| `OpenWorkflowEngine.connect({ databaseUrl, db, registry })` | `TaskEngine` 实现:`enqueue / signal / cancel / get / list / startWorker / stop` |
|
||||
| `defineTask` / `TaskRegistry` / `createDefaultRegistry()` | 任务定义与注册表;默认注册表里有 §3.1-8 的 9 个任务(目前是空壳) |
|
||||
| `defineTask` / `TaskRegistry` | 任务定义与注册表;实际业务 handler 由 server 所属模块注册,未注册任务抛 `UnknownTaskError` |
|
||||
| `OutboxRelay` | 用 `FOR UPDATE SKIP LOCKED` 认领 `outbox_event` 中 `status=pending` 且 `next_attempt_at` 已到的行,按 type 路由后 enqueue(幂等键取行的 `idempotency_key`,缺省 `outbox:<id>`),成功标记 `dispatched` + `published_at`;失败 attempts+1 指数退避(上限 5 分钟),达 `maxAttempts` 置 `dead`;多进程并发时不会重复 |
|
||||
| `migrateEngineSchema(databaseUrl, schema?)` | 只跑引擎自己的迁移后断开(部署时 `--role migrate`) |
|
||||
| `withAttemptToken(db, opKey, fn, { commit })` | 操作账本 fencing,表是 `tasks_op_ledger` |
|
||||
@@ -24,9 +24,9 @@
|
||||
- **取消**:`cancel(runId, reason)` 先写 `tasks_run_meta.cancel_requested_at`,再取消引擎里的 run。handler 执行期间会轮询这个标志(默认 1s)并触发 `abortSignal`;每个 `step.run/sleep` 开始前也会检查一次。进行中的步骤只能靠自己检查 `abortSignal` 来中止。已完成的外部动作不会撤销,补偿要写成显式步骤。
|
||||
- **等待外部条件**:`signal(runId, name, payload)` 先把事实写进 `tasks_fact`,再发送引擎信号。`step.waitForFact(name, { check?, timeout })` 每一轮都先查库,查不到才带超时等待信号。所以事实比等待方先到也不会丢。审批、下载完成这类业务事实,先写业务表,再用 `check` 去读业务表。
|
||||
|
||||
## 新增/填充任务(W2 规则)
|
||||
## 新增业务任务
|
||||
|
||||
1. 在 `src/definitions.ts` 里用 `defineTask({ name, version, inputSchema, retryPolicy, handler })` 定义任务。输入只放业务 id,handler 自己去读业务表。重试策略要显式写:外部网络类 5 次(1s→60s),文件类 3 次。
|
||||
1. 在 server 所属业务模块的 `registerTasks` 中用 `defineTask({ name, version, inputSchema, retryPolicy, handler })` 定义任务。输入只放业务 id,handler 自己去读业务表。显式声明 `lane` 与符合该业务副作用的重试策略;server 的任务分道表与回归同步更新。
|
||||
2. **步骤体必须幂等**。worker 被 kill 后,正在执行的步骤会在新 worker 上**从头重跑**(最长要等约 30s 租约到期)。已完成的步骤不会重跑。
|
||||
3. **有外部副作用的步骤都走 `withAttemptToken`**,opKey 按语义生成,例如 `submit:<planId>`:
|
||||
- 已经有结果时直接返回;
|
||||
@@ -57,4 +57,4 @@
|
||||
|
||||
## 测试
|
||||
|
||||
`deno task test`(或本包 `deno task test`)会连接开发库,每个测试 schema 先用 `@umt/db` 的 `migrateToLatest` 建全部业务表(`DATABASE_URL`,默认 `postgres://umt:umt@localhost:55433/umt`)。每个测试文件使用独立的 `tasks_test_<名>_<pid+随机>` schema(并发跑多份也不冲突),测试前后都会清理;库不可达时跳过集成测试。覆盖范围:kill -9 后续跑(子进程 worker)、取消、幂等 enqueue、两个 relay 并发、事实先到达、fencing、分组 worker 只认领自己的任务。
|
||||
`bun run test`(或本包 `bun run test`)会连接开发库,每个测试 schema 先用 `@umt/db` 的 `migrateToLatest` 建全部业务表(`DATABASE_URL`,默认 `postgres://umt:umt@localhost:55433/umt`)。每个测试文件使用独立的 `tasks_test_<名>_<pid+随机>` schema(并发跑多份也不冲突),测试前后都会清理;单包运行时库不可达会跳过集成测试;根目录 `bun run test:all` 先检查连接,不可达直接失败。覆盖范围:kill -9 后续跑(子进程 worker)、取消、幂等 enqueue、两个 relay 并发、事实先到达、fencing、分组 worker 只认领自己的任务。
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "@umt/tasks",
|
||||
"version": "0.3.44",
|
||||
"exports": { ".": "./src/index.ts" },
|
||||
"tasks": {
|
||||
"test": "deno run -A npm:vitest run",
|
||||
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
|
||||
}
|
||||
}
|
||||
@@ -4,5 +4,13 @@
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"version": "0.3.44",
|
||||
"scripts": {
|
||||
"test": "bun x --bun vitest run",
|
||||
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
|
||||
},
|
||||
"dependencies": {
|
||||
"@umt/db": "workspace:*"
|
||||
}
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
import { z } from "zod";
|
||||
import { defineTask, TaskRegistry } from "./registry.ts";
|
||||
|
||||
// 任务书 §3.1-8 的九类任务。W1 只落空壳:输入 schema + 回显 handler,W2 填充处理器。
|
||||
// 填充规则见 README「新增/填充任务」:副作用走 withAttemptToken,外部条件用 step.waitForFact,
|
||||
// 业务状态写业务表,不要放进 handler 返回值。
|
||||
|
||||
const id = z.string().min(1);
|
||||
|
||||
/** 空壳 handler:记录一条日志并回显输入 */
|
||||
function stub<I>(name: string) {
|
||||
return async (ctx: { input: I; logger: { info(msg: string, f?: Record<string, unknown>): void } }) => {
|
||||
// TODO(W2): 实现真实处理器
|
||||
ctx.logger.info(`${name}: stub handler (TODO W2)`);
|
||||
return { todo: true as const, task: name, input: ctx.input };
|
||||
};
|
||||
}
|
||||
|
||||
const NETWORK_RETRY = {
|
||||
maximumAttempts: 5,
|
||||
initialInterval: "1s",
|
||||
backoffCoefficient: 2,
|
||||
maximumInterval: "60s",
|
||||
} as const;
|
||||
const FILE_RETRY = {
|
||||
maximumAttempts: 3,
|
||||
initialInterval: "2s",
|
||||
backoffCoefficient: 2,
|
||||
maximumInterval: "60s",
|
||||
} as const;
|
||||
const DB_RETRY = { maximumAttempts: 3, initialInterval: "1s", backoffCoefficient: 2, maximumInterval: "30s" } as const;
|
||||
|
||||
export const ingestFeed = defineTask({
|
||||
name: "ingestFeed",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ feedSourceId: id }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: stub("ingestFeed"),
|
||||
});
|
||||
|
||||
export const resolveObservation = defineTask({
|
||||
name: "resolveObservation",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ observationId: id }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: stub("resolveObservation"),
|
||||
});
|
||||
|
||||
export const evaluateAcquisition = defineTask({
|
||||
name: "evaluateAcquisition",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({
|
||||
subscriptionId: id.optional(),
|
||||
acquisitionSlotId: id.optional(),
|
||||
sourceReleaseId: id.optional(),
|
||||
}),
|
||||
retryPolicy: DB_RETRY,
|
||||
handler: stub("evaluateAcquisition"),
|
||||
});
|
||||
|
||||
export const submitDownload = defineTask({
|
||||
name: "submitDownload",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ acquisitionPlanId: id }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: stub("submitDownload"),
|
||||
});
|
||||
|
||||
export const verifyAndImport = defineTask({
|
||||
name: "verifyAndImport",
|
||||
version: 1,
|
||||
lane: "file",
|
||||
inputSchema: z.object({ downloadTaskId: id }),
|
||||
retryPolicy: FILE_RETRY,
|
||||
handler: stub("verifyAndImport"),
|
||||
});
|
||||
|
||||
export const refreshMetadata = defineTask({
|
||||
name: "refreshMetadata",
|
||||
version: 1,
|
||||
lane: "network",
|
||||
inputSchema: z.object({ mediaEntityId: id }),
|
||||
retryPolicy: NETWORK_RETRY,
|
||||
handler: stub("refreshMetadata"),
|
||||
});
|
||||
|
||||
export const reconcileLibrary = defineTask({
|
||||
name: "reconcileLibrary",
|
||||
version: 1,
|
||||
lane: "maintenance",
|
||||
inputSchema: z.object({ libraryRootId: id.optional(), reconcileRunId: id.optional() }),
|
||||
retryPolicy: FILE_RETRY,
|
||||
handler: stub("reconcileLibrary"),
|
||||
});
|
||||
|
||||
export const cleanupDownloadCache = defineTask({
|
||||
name: "cleanupDownloadCache",
|
||||
version: 1,
|
||||
lane: "file",
|
||||
inputSchema: z.object({ downloaderInstanceId: id.optional() }),
|
||||
retryPolicy: FILE_RETRY,
|
||||
handler: stub("cleanupDownloadCache"),
|
||||
});
|
||||
|
||||
export const resolveDecision = defineTask({
|
||||
name: "resolveDecision",
|
||||
version: 1,
|
||||
lane: "agent",
|
||||
inputSchema: z.object({ decisionId: id }),
|
||||
retryPolicy: DB_RETRY,
|
||||
handler: stub("resolveDecision"),
|
||||
});
|
||||
|
||||
export const TASK_NAMES = [
|
||||
"ingestFeed",
|
||||
"resolveObservation",
|
||||
"evaluateAcquisition",
|
||||
"submitDownload",
|
||||
"verifyAndImport",
|
||||
"refreshMetadata",
|
||||
"reconcileLibrary",
|
||||
"cleanupDownloadCache",
|
||||
"resolveDecision",
|
||||
] as const;
|
||||
export type TaskName = (typeof TASK_NAMES)[number];
|
||||
|
||||
/** v0.1 的默认注册表;W2 把真实 handler 换进来(同名同版本替换,或升版本并保留旧版本) */
|
||||
export function createDefaultRegistry(): TaskRegistry {
|
||||
return new TaskRegistry().register(
|
||||
ingestFeed,
|
||||
resolveObservation,
|
||||
evaluateAcquisition,
|
||||
submitDownload,
|
||||
verifyAndImport,
|
||||
refreshMetadata,
|
||||
reconcileLibrary,
|
||||
cleanupDownloadCache,
|
||||
resolveDecision,
|
||||
);
|
||||
}
|
||||
@@ -1,20 +1,6 @@
|
||||
// @umt/tasks 公开入口:TaskEngine 端口 + OpenWorkflow 实现、任务注册表、outbox 中继、操作账本 fencing
|
||||
export const PACKAGE_NAME = "@umt/tasks";
|
||||
|
||||
export {
|
||||
cleanupDownloadCache,
|
||||
createDefaultRegistry,
|
||||
evaluateAcquisition,
|
||||
ingestFeed,
|
||||
reconcileLibrary,
|
||||
refreshMetadata,
|
||||
resolveDecision,
|
||||
resolveObservation,
|
||||
submitDownload,
|
||||
TASK_NAMES,
|
||||
type TaskName,
|
||||
verifyAndImport,
|
||||
} from "./definitions.ts";
|
||||
export { FencingError, TaskCanceledError, UnknownTaskError } from "./errors.ts";
|
||||
export {
|
||||
consoleSink,
|
||||
|
||||
@@ -22,7 +22,7 @@ const WORKER = fileURLToPath(new URL("./fixtures/worker.ts", import.meta.url));
|
||||
const PKG_DIR = fileURLToPath(new URL("..", import.meta.url));
|
||||
|
||||
function startWorker(env: Record<string, string>): Promise<ChildProcess> {
|
||||
const child = spawn(process.execPath, ["run", "-A", WORKER], {
|
||||
const child = spawn(process.execPath, [WORKER], {
|
||||
cwd: PKG_DIR,
|
||||
env: { ...process.env, SCHEMA: schemas.biz, OW_SCHEMA: schemas.ow, ...env },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { createDefaultRegistry, defineTask, redact, TASK_NAMES, TaskRegistry, UnknownTaskError } from "../src/index.ts";
|
||||
import { defineTask, redact, TaskRegistry, UnknownTaskError } from "../src/index.ts";
|
||||
|
||||
describe("TaskRegistry", () => {
|
||||
it("注册任务书 §3.1-8 的九个空壳任务", () => {
|
||||
const reg = createDefaultRegistry();
|
||||
expect(reg.names()).toEqual([...TASK_NAMES].sort());
|
||||
for (const n of TASK_NAMES) expect(reg.latest(n).version).toBe(1);
|
||||
expect(reg.has("ingestFeed")).toBe(true);
|
||||
expect(reg.has("nope")).toBe(false);
|
||||
it("注册表默认为空,未注册任务明确报错", () => {
|
||||
const reg = new TaskRegistry();
|
||||
expect(reg.names()).toEqual([]);
|
||||
expect(reg.has("submitDownload")).toBe(false);
|
||||
expect(() => reg.latest("submitDownload")).toThrow(UnknownTaskError);
|
||||
});
|
||||
|
||||
it("latest 取最高版本;select 按 name@version / @version 过滤", () => {
|
||||
@@ -24,14 +23,6 @@ describe("TaskRegistry", () => {
|
||||
expect(() => reg.latest("zzz")).toThrow(UnknownTaskError);
|
||||
expect(() => mk("a", 0)).toThrow(/invalid task version/);
|
||||
});
|
||||
|
||||
it("空壳 handler 回显输入", async () => {
|
||||
const def = createDefaultRegistry().latest("submitDownload");
|
||||
const input = def.inputSchema.parse({ acquisitionPlanId: "p1" });
|
||||
const out = await def.handler({ input, logger: { info() {} } } as never);
|
||||
expect(out).toEqual({ todo: true, task: "submitDownload", input: { acquisitionPlanId: "p1" } });
|
||||
expect(() => def.inputSchema.parse({})).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("redact", () => {
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "@umt/testkit",
|
||||
"version": "0.3.44",
|
||||
"exports": { ".": "./src/index.ts" },
|
||||
"tasks": {
|
||||
"test": "deno run -A npm:vitest run",
|
||||
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
|
||||
}
|
||||
}
|
||||
@@ -4,5 +4,11 @@
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
}
|
||||
},
|
||||
"version": "0.3.44",
|
||||
"scripts": {
|
||||
"test": "bun x --bun vitest run",
|
||||
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
|
||||
},
|
||||
"dependencies": {}
|
||||
}
|
||||
@@ -21,7 +21,7 @@ export interface MockFetchRequest {
|
||||
}
|
||||
|
||||
export interface MockFetch {
|
||||
fetch: typeof globalThis.fetch;
|
||||
fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
requests: MockFetchRequest[];
|
||||
/** 追加/覆盖路由 */
|
||||
route(key: string, handler: MockRouteHandler): void;
|
||||
@@ -87,7 +87,7 @@ export function createMockFetch(routes: Record<string, MockRouteHandler> = {}):
|
||||
return new Response("not found", { status: 404 });
|
||||
};
|
||||
return {
|
||||
fetch: fetchImpl as typeof globalThis.fetch,
|
||||
fetch: fetchImpl as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
|
||||
requests,
|
||||
route(key, handler) {
|
||||
table.delete(key);
|
||||
|
||||
Reference in new issue
Block a user