refactor(core): migrate to Bun and converge acquisition policies and Agent boundaries

This commit is contained in:
root committed 2026-10-13 14:37:03 +08:00
1 parent 3f1c6d02f6
commit 27c0f27434
268 files changed
+7468 -7852

No files matched your search

+1 -1
View File
@@ -14,4 +14,4 @@
| [`fs`](src/fs/README.md) | `copyWithVerify`、`moveSafe`、`crossDeviceMove`、`checkRoot`、`scanIncremental`、`diskUsage`、`ffprobe`、`FsProxy`;另有子路径导出 `@umt/adapters/fsworker`(`fsWorkerMain`,供 `--role fsworker`) | 校验复制、跨卷移动、库根哨兵、增量扫描;FsProxy 把文件操作放进可杀子进程(挂死 → SIGKILL → TIMEOUT) |
| [`browser`](src/browser/README.md) | `BrowserProvider` 合同、`createNoneBrowserProvider`、camofox 适配骨架、`redactBrowserUrl` | 浏览器服务接口(M1 只有接口与骨架) |
测试:`deno task test`(vitest,外部服务全部用 `@umt/testkit` 的 mock)。
测试:`bun run test`(vitest,外部服务全部用 `@umt/testkit` 的 mock)。
-9
View File
@@ -1,9 +0,0 @@
{
"name": "@umt/adapters",
"version": "0.3.44",
"exports": { ".": "./src/index.ts", "./fsworker": "./src/fs/fsworker.ts", "./alist-mock": "./src/alist/mock.ts" },
"tasks": {
"test": "deno run -A npm:vitest run",
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
}
}
+10
View File
@@ -6,5 +6,15 @@
".": "./src/index.ts",
"./fsworker": "./src/fs/fsworker.ts",
"./alist-mock": "./src/alist/mock.ts"
},
"version": "0.3.44",
"scripts": {
"test": "bun x --bun vitest run",
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
},
"dependencies": {
"@umt/contracts": "workspace:*",
"@umt/testkit": "workspace:*",
"@umt/runtime": "workspace:*"
}
}
@@ -12,16 +12,16 @@ import { type AlistAuth, AlistClient } from "./client.ts";
// UMT_ALIST_REAL_ROOT2 可选:另一个挂载(驱动)下的可写目录,测跨驱动复制 / 移动(异步任务)
// 跑法:docs/runbook/storage-alist.md「真机验收」。
const URL_ = Deno.env.get("UMT_ALIST_REAL_URL");
const ROOT = Deno.env.get("UMT_ALIST_REAL_ROOT");
const ROOT2 = Deno.env.get("UMT_ALIST_REAL_ROOT2");
const auth: AlistAuth | null = Deno.env.get("UMT_ALIST_REAL_TOKEN")
? { kind: "token", token: Deno.env.get("UMT_ALIST_REAL_TOKEN")! }
: Deno.env.get("UMT_ALIST_REAL_USER")
const URL_ = process.env.UMT_ALIST_REAL_URL;
const ROOT = process.env.UMT_ALIST_REAL_ROOT;
const ROOT2 = process.env.UMT_ALIST_REAL_ROOT2;
const auth: AlistAuth | null = process.env.UMT_ALIST_REAL_TOKEN
? { kind: "token", token: process.env.UMT_ALIST_REAL_TOKEN! }
: process.env.UMT_ALIST_REAL_USER
? {
kind: "password",
username: Deno.env.get("UMT_ALIST_REAL_USER")!,
password: Deno.env.get("UMT_ALIST_REAL_PASSWORD") ?? "",
username: process.env.UMT_ALIST_REAL_USER!,
password: process.env.UMT_ALIST_REAL_PASSWORD ?? "",
}
: null;
const enabled = !!URL_ && !!ROOT && !!auth;
+15 -6
View File
@@ -388,14 +388,17 @@ describe("上传", () => {
it("空闲超时:服务端不读数据时中止", async () => {
const mock = createMockAlist({ token: "t" });
const stuck: typeof fetch = (async (input: Request | URL | string, init?: RequestInit) => {
const stuck: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = (async (
input: Request | URL | string,
init?: RequestInit,
) => {
const req = input instanceof Request ? input : new Request(String(input), init);
if (new URL(req.url).pathname === "/api/fs/put")
return new Promise<Response>((_, reject) =>
init?.signal?.addEventListener("abort", () => reject(init.signal!.reason)),
);
return mock.fetch(input, init);
}) as typeof fetch;
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
const client = new AlistClient({
baseUrl: mock.base,
auth: { kind: "token", token: "t" },
@@ -453,14 +456,17 @@ describe("下载", () => {
mock.put("/s1/a.mkv", bytes(1000));
const realFetch = mock.fetch;
let expired = false;
const f: typeof fetch = (async (input: Request | URL | string, init?: RequestInit) => {
const f: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = (async (
input: Request | URL | string,
init?: RequestInit,
) => {
const u = String(input instanceof Request ? input.url : input);
if (u.includes("/d/") && !expired) {
expired = true;
mock.expireSigns();
}
return realFetch(input, init);
}) as typeof fetch;
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
const c2 = new AlistClient({ baseUrl: mock.base, auth: { kind: "token", token: "perm-token" }, fetch: f, qps: 0 });
await c2.download("/s1/a.mkv", join(dir, "a.mkv"));
expect((await stat(join(dir, "a.mkv"))).size).toBe(1000);
@@ -478,7 +484,10 @@ describe("下载", () => {
const mock = createMockAlist({ token: "t" });
mock.put("/s1/a.mkv", "x");
const calls: { url: string; auth: string | null }[] = [];
const f: typeof fetch = (async (input: Request | URL | string, init?: RequestInit) => {
const f: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = (async (
input: Request | URL | string,
init?: RequestInit,
) => {
const req = input instanceof Request ? input : new Request(String(input), init);
calls.push({ url: req.url, auth: req.headers.get("authorization") });
if (req.url.startsWith("https://cdn.example")) return new Response("x");
@@ -489,7 +498,7 @@ describe("下载", () => {
return Response.json(j);
}
return res;
}) as typeof fetch;
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
const c = new AlistClient({ baseUrl: mock.base, auth: { kind: "token", token: "t" }, fetch: f, qps: 0 });
await c.download("/s1/a.mkv", join(dir, "a.mkv"));
expect(calls.find((x) => x.url.startsWith("https://cdn.example"))!.auth).toBeNull();
+2 -2
View File
@@ -24,7 +24,7 @@ export interface AlistClientOptions {
auth: AlistAuth;
/** 路径密码(Alist「元信息」里给目录设的密码),list / get 时带上 */
pathPassword?: string | null;
fetch?: typeof fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
/** 普通请求超时(ms),缺省 30 s */
timeoutMs?: number;
/** 数据流无进度空闲超时(ms),缺省 120 s */
@@ -203,7 +203,7 @@ function idleWatch(ms: number, outer?: AbortSignal) {
export class AlistClient {
readonly baseUrl: string;
private readonly fetchFn: typeof fetch;
private readonly fetchFn: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
private readonly limiter: AlistLimiter;
private readonly o: Required<
Pick<
+5 -4
View File
@@ -3,6 +3,7 @@
// 令牌无效 code 401;copy 一律异步任务;同挂载 move 立即完成、跨挂载 move 是任务;put 覆盖同名;/d 支持 Range 与 sign。
// 顶层目录视为「挂载」(驱动),用于区分同驱动 / 跨驱动。
import { createHash, randomUUID } from "node:crypto";
import { type NetAddr, serve } from "@umt/runtime";
export interface MockFile {
data: Uint8Array;
@@ -43,7 +44,7 @@ export interface MockAlist {
readonly base: string;
setOrigin(origin: string): void;
handler: (req: Request) => Promise<Response>;
fetch: typeof fetch;
fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
files: Map<string, MockFile>;
dirs: Map<string, number>;
tasks: Map<string, Task>;
@@ -432,7 +433,7 @@ export function createMockAlist(opts: MockAlistOptions = {}): MockAlist {
// 老版本(noTaskIds):没有任务 id 的复制 / 移动,在之后的任意 get 上推进
if (opts.noTaskIds) for (const t of tasks.values()) advanceTask(t);
return handler(req);
}) as typeof fetch,
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
files,
dirs,
tasks,
@@ -471,13 +472,13 @@ export function createMockAlist(opts: MockAlistOptions = {}): MockAlist {
/** 起一个真 HTTP 监听(ffprobe / 下载代理等需要真实 URL 的测试用);返回 base 与关闭函数 */
export function serveMockAlist(mock: MockAlist): { base: string; close: () => Promise<void> } {
const server = Deno.serve({ port: 0, hostname: "127.0.0.1", onListen: () => {} }, (req) => {
const server = serve({ port: 0, hostname: "127.0.0.1", onListen: () => {} }, (req) => {
const u = new URL(req.url);
return mock
.handler(new Request(`${mock.base}${u.pathname}${u.search}`, req))
.catch(() => new Response("connection reset", { status: 502 }));
});
const addr = server.addr as Deno.NetAddr;
const addr = server.addr as NetAddr;
const prev = mock.base;
const base = `http://127.0.0.1:${addr.port}`;
mock.setOrigin(base);
@@ -41,7 +41,7 @@ function realUpstream(handler: (c: Call) => Reply | undefined) {
status: r.status ?? 200,
headers: { "content-type": "application/json" },
});
}) as typeof fetch;
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
return { calls, fetchImpl };
}
@@ -75,7 +75,7 @@ export interface CamofoxConfig {
downloadSink?: DownloadSink;
/** waitForDownload 轮询间隔 */
downloadPollMs?: number;
fetch?: typeof fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
logger?: BrowserLogger;
now?: () => Date;
newId?: () => string;
@@ -157,7 +157,7 @@ const LOGIN_PATH = /(?:^|\/)(?:login|takelogin|signin|sign_in|auth)(?:\.php)?(?:
export class CamofoxBrowserProvider implements BrowserProvider {
readonly kind = "camofox" as const;
private readonly base: string;
private readonly fetchImpl: typeof fetch;
private readonly fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
private readonly lock: ProfileLock;
private readonly sessions = new Map<string, SessionState>();
private readonly leases = new Map<string, BrowserControlLease>();
+1 -1
View File
@@ -1,6 +1,6 @@
/**
* 截图的解码、缩放与编码(VIS-1):交给视觉模型的截图缩放到不超过 maxWidth(缺省 1280)像素宽,
* 证据缩略图更小。PNG 自己解码 / 编码(node:zlib),JPEG 用 jpeg-js(纯 JS,无原生依赖,deno compile 可嵌入)。
* 证据缩略图更小。PNG 自己解码 / 编码(node:zlib),JPEG 用 jpeg-js(纯 JS,无原生依赖,bun build --compile 可嵌入)。
* 只支持浏览器截图实际会出现的格式:8 位 PNG(灰度 / RGB / 调色板 / 带 alpha,非隔行)与基线 JPEG。
*/
import { deflateSync, inflateSync } from "node:zlib";
+2 -2
View File
@@ -26,7 +26,7 @@ export interface EmbyConfig {
retryBaseMs?: number;
/** 搜索/分页单页大小,默认 50 */
pageSize?: number;
fetch?: typeof fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
}
export type Observed<T> =
@@ -240,7 +240,7 @@ export class EmbyClient {
readonly instanceId: string;
private readonly base: string;
private readonly flavor: "emby" | "jellyfin";
private readonly fetchImpl: typeof fetch;
private readonly fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
private userId: string | undefined;
constructor(private readonly cfg: EmbyConfig) {
+2 -2
View File
@@ -4,13 +4,13 @@
## 挂死隔离:`FsProxy`(对标 MP fsproxy)
`new FsProxy({ hangTimeoutMs, quickTimeoutMs, progressIntervalMs, killGraceMs, mode })` 把单个文件操作(copy / crossMove / move / match / sha256 / stat / checkRoot / sweep / unlink)放进一次性执行子进程 `fsworker.ts`(`deno run --no-config -A`,约 50 ms 启动),stdin 一条 JSON 请求,stdout 逐行心跳 + 结果。
`new FsProxy({ hangTimeoutMs, quickTimeoutMs, progressIntervalMs, killGraceMs, mode })` 把单个文件操作(copy / crossMove / move / match / sha256 / stat / checkRoot / sweep / unlink)放进一次性执行子进程 `fsworker.ts`(`bun fsworker.ts`),通过 `@umt/runtime/worker` 发送带 id 的逐行 JSON 请求、进度与结果,单条消息上限 16 MiB。
- 数据类操作(复制、跨卷搬运、哈希)按**进度停滞**判挂死:`hangTimeoutMs`(缺省 120 s)内字节数与阶段都没前进 → SIGTERM(worker 中止并清掉自己的 partial)→ `killGraceMs` 后 SIGKILL → `FsError('TIMEOUT')`。大文件复制几小时不会误杀。
- 快操作按总时长 `quickTimeoutMs`(缺省 30 s)。`statSafe` 超时返回 `{kind:"error",code:"TIMEOUT"}`,`checkRoot` 超时返回 `offline(timeout)`——**挂死不等于文件缺失**。
- 卡在 D 状态的子进程被 `unref`,不拖住本进程;被 SIGKILL 的 partial 由下次写前清理收走。
- 子进程里 sha256 走流式(带心跳),不走 `sha256sum`(看不到进度);吞吐见 `docs/porting/moviepilot-rust-transfer.md` 基准记录。
- `mode: "auto"`:`Deno.execPath()` 是 deno CLI → `deno run -A fsworker.ts`;否则视为 `deno compile` 出的 umt-server 二进制 → `<execPath> --role fsworker`(`apps/server/src/main.ts` 在加载其余模块前分派,启动约 150 ms)。编译产物的 `--allow-run` 必须含自身绝对路径(镜像内 `/app/umt-server`,见 `apps/server/deno.json`);起不来(NotCapable/ENOENT)自动回退 in-process(同样的看门狗,超时立即报 TIMEOUT 并中止,但卡死系统调用占住的线程无法回收),`stats().fallbackReason` 记原因。`stats().workerKind` = script / self / custom。
- 服务端缺省 `mode: "subprocess"`,启动失败直接报错;显式选择 `mode: "auto"`:`process.execPath` 是 bun CLI → `bun fsworker.ts`;否则视为 `bun build --compile` 出的 umt-server 二进制 → `<execPath> --role fsworker`(`apps/server/src/main.ts` 在加载其余模块前分派)。子进程起不来(ENOENT/EACCES)自动回退 in-process(同样的看门狗,超时立即报 TIMEOUT 并中止,但卡死系统调用占住的线程无法回收),`stats().fallbackReason` 记原因。`stats().workerKind` = script / self / custom。
- 容器验收开关:worker 进程设 `UMT_FS_TEST_HANG_AFTER_BYTES=<n>` 后,每个数据类操作的 worker 读到 n 字节即卡死 JS 线程并屏蔽 SIGTERM,用来在生产二进制里验证 停滞超时 → SIGKILL → TIMEOUT;生产勿设。
- 测试开关 `_test: { hangAfterBytes, ignoreTerm }`:worker 在累计进度达到阈值后阻塞 JS 线程,模拟卡死的系统调用。
+7 -19
View File
@@ -6,6 +6,7 @@ import { link, lstat, mkdir, open, readdir, rename, stat, unlink, utimes } from
import { basename, dirname, join } from "node:path";
import { Transform } from "node:stream";
import { pipeline } from "node:stream/promises";
import { Command } from "@umt/runtime";
import { errnoOf, FsError, toFsError } from "./errors.ts";
import { assertLocalPaths } from "./remote-guard.ts";
@@ -133,23 +134,10 @@ export function sha256Stats(): { counts: Record<Sha256Via, number>; subprocessUn
return { counts: { ...hashCounters }, subprocessUnavailable: sha256sumUnavailable };
}
type DenoLike = {
Command: new (
cmd: string,
opts: { args: string[]; stdout: "piped"; stderr: "piped"; signal?: AbortSignal },
) => { output(): Promise<{ success: boolean; code: number; stdout: Uint8Array; stderr: Uint8Array }> };
errors: { NotFound: new () => Error; NotCapable?: new () => Error; PermissionDenied: new () => Error };
};
async function sha256Subprocess(p: string, signal?: AbortSignal): Promise<string | null> {
const D = (globalThis as { Deno?: DenoLike }).Deno;
if (!D?.Command) {
sha256sumUnavailable = "no_deno_command";
return null;
}
let out: Awaited<ReturnType<InstanceType<DenoLike["Command"]>["output"]>>;
let out: Awaited<ReturnType<Command["output"]>>;
try {
out = await new D.Command("sha256sum", {
out = await new Command("sha256sum", {
args: ["-b", "--", p],
stdout: "piped",
stderr: "piped",
@@ -157,9 +145,9 @@ async function sha256Subprocess(p: string, signal?: AbortSignal): Promise<string
}).output();
} catch (e) {
if (signal?.aborted) throw new FsError("ABORTED", "aborted", { path: p, cause: e });
const notCapable = D.errors.NotCapable && e instanceof D.errors.NotCapable;
if (e instanceof D.errors.NotFound || e instanceof D.errors.PermissionDenied || notCapable) {
sha256sumUnavailable = e instanceof D.errors.NotFound ? "not_installed" : "not_permitted";
const code = (e as NodeJS.ErrnoException).code;
if (code === "ENOENT" || code === "EACCES" || code === "EPERM") {
sha256sumUnavailable = code === "ENOENT" ? "not_installed" : "not_permitted";
return null;
}
throw toFsError(e, p);
@@ -425,7 +413,7 @@ export async function copyWithVerify(src: string, dst: string, opts: CopyOptions
receipt.sampleDigest = b;
}
opts.signal?.throwIfAborted();
await utimes(tmp, srcStat.atime, srcStat.mtime);
await utimes(tmp, srcStat.atimeMs / 1000, srcStat.mtimeMs / 1000);
if (existing) receipt.replaced = await replaceCommit(tmp, dst, existing, opts.replaceOld);
else await commitNoClobber(tmp, dst);
await fsyncPath(dirname(dst));
+3
View File
@@ -108,3 +108,6 @@ export async function runFsOp(req: FsOpRequest, ctx: FsOpContext): Promise<unkno
throw new FsError("IO", `unknown fs op: ${(req as { op: string }).op}`);
}
}
/** 文件请求、扫描结果的单条协议消息上限。文件内容不通过管道传输。 */
export const FS_MESSAGE_BYTES = 16 * 1024 * 1024;
+12 -5
View File
@@ -132,7 +132,7 @@ describe("FsProxy(子进程)", () => {
});
describe("FsProxy 启动方式", () => {
it("deno CLI 下缺省 run fsworker.ts;显式 workerCommand 记为 custom", () => {
it("bun CLI 下缺省 run fsworker.ts;显式 workerCommand 记为 custom", () => {
const auto = defaultWorkerCommand();
expect(auto?.kind).toBe("script");
expect(auto?.args.at(-1)).toMatch(/fsworker\.ts$/);
@@ -148,6 +148,13 @@ describe("FsProxy 启动方式", () => {
expect(px.stats()).toMatchObject({ mode: "inprocess", fallbackReason: expect.stringMatching(/^spawn_failed/) });
});
it("强制子进程时启动失败报错,不在主进程执行文件操作", async () => {
const t = await tree({ "a.mkv": "x" });
const px = new FsProxy({ mode: "subprocess", workerCommand: { cmd: "/nonexistent/umt-worker", args: [] } });
expect(await px.statSafe(t.path("a.mkv"))).toMatchObject({ kind: "error", code: "IO" });
expect(px.stats()).toMatchObject({ mode: "subprocess", fallbackReason: null });
});
it("验收开关只在 UMT_FS_TEST_HOOKS=1 时生效(单独误设 HANG_AFTER_BYTES 无效)", () => {
const env = (m: Record<string, string>) => (k: string) => m[k];
expect(fsTestHookFromEnv(env({ UMT_FS_TEST_HANG_AFTER_BYTES: "1024" }))).toBeUndefined();
@@ -162,16 +169,16 @@ describe("FsProxy 启动方式", () => {
it("UMT_FS_TEST_HANG_AFTER_BYTES(容器验收开关)→ 数据类操作卡死 → SIGKILL → TIMEOUT;快操作不受影响", async () => {
const t = await tree({ "src/a.mkv": 3 * MiB, "dst/": null });
const px = new FsProxy({ mode: "subprocess", ...fast });
Deno.env.set("UMT_FS_TEST_HANG_AFTER_BYTES", String(MiB));
Deno.env.set("UMT_FS_TEST_HOOKS", "1");
process.env.UMT_FS_TEST_HANG_AFTER_BYTES = String(MiB);
process.env.UMT_FS_TEST_HOOKS = "1";
try {
expect(await px.statSafe(t.path("src/a.mkv"))).toMatchObject({ kind: "ok" });
const e = await px.copy(t.path("src/a.mkv"), t.path("dst/a.mkv")).catch((x) => x);
expect(e.code).toBe("TIMEOUT");
expect(px.stats()).toMatchObject({ timeouts: 1, killed: 1 });
} finally {
Deno.env.delete("UMT_FS_TEST_HANG_AFTER_BYTES");
Deno.env.delete("UMT_FS_TEST_HOOKS");
delete process.env.UMT_FS_TEST_HANG_AFTER_BYTES;
delete process.env.UMT_FS_TEST_HOOKS;
}
}, 30_000);
});
+73 -189
View File
@@ -5,18 +5,20 @@
// 统一转成 FsError('TIMEOUT')——普通可重试 IO 错误,交给任务层退避重试。卡在 D 状态的子进程被 unref,不拖住本进程。
// stat/checkRoot 超时返回 error/offline(timeout),**不**当成文件缺失(REQ §2.1 原则 4)。
//
// 与 MP 的差异:MP 是常驻 worker + 一把锁串行;我们每个操作一个子进程(~50 ms 启动,入库按文件计可忽略),
// 与 MP 的差异:MP 是常驻 worker + 一把锁串行;我们每个操作一个子进程,
// 天然并发、互不拖累,也没有"上一个请求挂死把 worker 占住"的问题。
// 启动方式:execPath 是 deno CLI(开发/测试)→ `deno run -A fsworker.ts`;否则视为 deno compile 出的 umt-server
// 二进制 → `<execPath> --role fsworker`(入口 main.ts 在加载其余模块前分派)。编译产物的 --allow-run 须含自身绝对路径
// (镜像内 /app/umt-server,见 apps/server/deno.json);起不来(NotCapable 等)回退 in-process:
// 启动方式:execPath 是 bun CLI(开发/测试)→ `bun fsworker.ts`;否则视为 bun build --compile 出的 umt-server
// 二进制 → `<execPath> --role fsworker`(入口 main.ts 在加载其余模块前分派)。
// 子进程无法启动时回退 in-process:
// 同样的心跳看门狗,超时立即以 TIMEOUT 失败并中止操作,但卡死的系统调用占住的线程无法回收(见 README)。
import { basename } from "node:path";
import { fileURLToPath } from "node:url";
import { WorkerConnection, WorkerTransportError } from "@umt/runtime/worker";
import type { CopyReceipt, Sha256Result } from "./copy.ts";
import { FsError, type FsErrorCode } from "./errors.ts";
import {
DATA_OPS,
FS_MESSAGE_BYTES,
type FsOpRequest,
type FsProgress,
type FsTestHook,
@@ -32,7 +34,7 @@ import type { RootHealth, StatResult } from "./stat.ts";
export type { FsProgress } from "./fsops.ts";
export interface FsProxyConfig {
/** auto:能起 deno 子进程就用子进程,否则 in-process;subprocess 强制子进程(起不来报错) */
/** auto:能起 bun 子进程就用子进程,否则 in-process;subprocess 强制子进程(起不来报错) */
mode?: "auto" | "subprocess" | "inprocess";
/** 数据类操作:进度停滞多久判挂死(缺省 120 s,同 MP FS_PROXY_STALL_TIMEOUT) */
hangTimeoutMs?: number;
@@ -62,15 +64,6 @@ export interface FsProxyStats {
fallbackReason: string | null;
}
type DenoChild = {
stdin: WritableStream<Uint8Array>;
stdout: ReadableStream<Uint8Array>;
stderr: ReadableStream<Uint8Array>;
status: Promise<{ success: boolean; code: number; signal: string | null }>;
kill(sig?: string): void;
unref(): void;
};
const WORKER_PATH = (() => {
try {
const u = new URL("./fsworker.ts", import.meta.url);
@@ -80,31 +73,12 @@ const WORKER_PATH = (() => {
}
})();
/** 当前运行时能用的缺省 worker 启动命令:deno CLI → run 脚本;编译二进制 → 自身 --role fsworker;都不行 → null */
/** 当前运行时能用的缺省 worker 启动命令:bun CLI → run 脚本;编译二进制 → 自身 --role fsworker;都不行 → null */
export function defaultWorkerCommand(): { cmd: string; args: string[]; kind: "script" | "self" } | null {
const D = (globalThis as { Deno?: { execPath?: () => string; Command?: unknown } }).Deno;
if (!D?.Command || !D.execPath) return null;
let exe: string;
try {
exe = D.execPath();
} catch {
return null;
}
if (/^deno(\.exe)?$/.test(basename(exe))) {
const exe = process.execPath;
if (/^bun(\.exe)?$/.test(basename(exe))) {
if (!WORKER_PATH) return null;
return {
cmd: exe,
args: [
"run",
"--no-config",
"--quiet",
"--no-prompt",
// Deno 对 /dev、/proc 下的路径(如 /dev/shm 上的库根)要求 --allow-all;worker 只执行本包固定的操作表
"--allow-all",
WORKER_PATH,
],
kind: "script",
};
return { cmd: exe, args: [WORKER_PATH], kind: "script" };
}
return { cmd: exe, args: ["--role", "fsworker"], kind: "self" };
}
@@ -122,12 +96,7 @@ export function fsTestHookFromEnv(get: (k: string) => string | undefined): FsTes
}
function envTestHook(): FsTestHook | undefined {
try {
const env = (globalThis as { Deno?: { env?: { get(k: string): string | undefined } } }).Deno?.env;
return env ? fsTestHookFromEnv((k) => env.get(k)) : undefined;
} catch {
return undefined;
}
return fsTestHookFromEnv((k) => process.env[k]);
}
const timeoutError = (op: string, why: string, path?: string) =>
@@ -154,7 +123,7 @@ function pathOf(req: FsOpRequest): string | undefined {
export class FsProxy {
readonly cfg: Required<Omit<FsProxyConfig, "workerCommand" | "mode">> & Pick<FsProxyConfig, "workerCommand">;
/** 子进程怎么起:script = deno run fsworker.ts;self = 编译二进制 --role fsworker;custom = 显式 workerCommand */
/** 子进程怎么起:script = bun fsworker.ts;self = 编译二进制 --role fsworker;custom = 显式 workerCommand */
readonly workerKind: "script" | "self" | "custom" | null;
#mode: "subprocess" | "inprocess";
#forced: boolean;
@@ -177,9 +146,9 @@ export class FsProxy {
if (mode === "inprocess") this.#mode = "inprocess";
else if (wc) this.#mode = "subprocess";
else {
if (mode === "subprocess") throw new Error("fsproxy: 当前运行时无法启动 deno 子进程,且未提供 workerCommand");
if (mode === "subprocess") throw new Error("fsproxy: 当前运行时无法启动 bun 子进程,且未提供 workerCommand");
this.#mode = "inprocess";
this.#fallbackReason = "no_deno_cli";
this.#fallbackReason = "no_bun_cli";
}
}
@@ -251,7 +220,8 @@ export class FsProxy {
try {
return await this.#runSubprocess(req, call);
} catch (e) {
if (!(e instanceof SpawnError) || this.#forced) throw e instanceof SpawnError ? e.cause : e;
if (!(e instanceof SpawnError)) throw e;
if (this.#forced) throw new FsError("IO", `文件 worker 无法启动:${e.message}`, { cause: e.cause });
this.#mode = "inprocess";
this.#fallbackReason = e.message;
}
@@ -310,144 +280,72 @@ export class FsProxy {
}
}
async #runSubprocess(req: FsOpRequest, call: FsCallOptions): Promise<unknown> {
const D = (globalThis as unknown as { Deno: DenoLike }).Deno;
const cmd = this.cfg.workerCommand!;
let child: DenoChild;
async #runSubprocess(req: FsOpRequest, call: FsCallOptions = {}): Promise<unknown> {
let connection: WorkerConnection;
try {
child = new D.Command(cmd.cmd, {
args: cmd.args,
stdin: "piped",
stdout: "piped",
stderr: "piped",
env: { UMT_FS_PROGRESS_INTERVAL_MS: String(this.cfg.progressIntervalMs) },
}).spawn();
} catch (e) {
throw new SpawnError(`spawn_failed:${(e as Error)?.name ?? "Error"}`, e);
connection = new WorkerConnection(this.cfg.workerCommand!, {
values: { UMT_FS_PROGRESS_INTERVAL_MS: String(this.cfg.progressIntervalMs) },
});
} catch (error) {
throw new SpawnError(`spawn_failed:${(error as Error).name}`, error);
}
this.#stats.spawned += 1;
const limit = this.#limitFor(req);
const w = child.stdin.getWriter();
let last = Date.now();
let lastKey = "";
const terminate = (error: FsError) => connection.terminate(error, this.cfg.killGraceMs, () => this.#stats.killed++);
const onAbort = () => terminate(new FsError("ABORTED", "aborted", { path: pathOf(req) }));
const watch = setInterval(
() => {
if (Date.now() - last > limit.ms) {
this.#stats.timeouts += 1;
clearInterval(watch);
terminate(timeoutError(req.op, limit.why, pathOf(req)));
}
},
Math.max(10, Math.min(1000, limit.ms / 4)),
);
call.signal?.addEventListener("abort", onAbort, { once: true });
const hook = call._test ?? (DATA_OPS.has(req.op) ? envTestHook() : undefined);
await w.write(new TextEncoder().encode(JSON.stringify(hook ? { ...req, _test: hook } : req)));
await w.close().catch(() => undefined);
let stderr = "";
const errPump = (async () => {
const dec = new TextDecoder();
try {
for await (const c of child.stderr) stderr = (stderr + dec.decode(c, { stream: true })).slice(-2000);
} catch {
// 被 kill 时流会出错
}
})();
return await new Promise<unknown>((resolve, reject) => {
let settled = false;
let last = Date.now();
let lastKey = "";
let killTimer: ReturnType<typeof setTimeout> | undefined;
/** 已判超时/已取消:之后 worker 自己报的结果(如 SIGTERM 后的 ABORTED)一律以此为准 */
let terminating: FsError | null = null;
const finish = (fn: () => void) => {
if (settled) return;
settled = true;
clearInterval(watch);
call.signal?.removeEventListener("abort", onAbort);
fn();
};
const terminate = (err: FsError) => {
if (settled || terminating) return;
terminating = err;
try {
child.kill("SIGTERM");
} catch {
// 已退出
}
clearInterval(watch);
killTimer = setTimeout(() => {
try {
child.kill("SIGKILL");
this.#stats.killed += 1;
} catch {
// SIGTERM 后已退出
}
// D 状态的子进程可能连 SIGKILL 都要等系统调用返回:不等它,unref 后直接报错
child.unref();
finish(() => reject(err));
}, this.cfg.killGraceMs);
};
const onAbort = () =>
terminate(new FsError("ABORTED", "aborted", { ...(pathOf(req) ? { path: pathOf(req) } : {}) }));
call.signal?.addEventListener("abort", onAbort, { once: true });
const watch = setInterval(
() => {
if (Date.now() - last > limit.ms) {
this.#stats.timeouts += 1;
terminate(timeoutError(req.op, limit.why, pathOf(req)));
}
try {
const pending = connection.request(
{
id: "1",
task: req.op,
input: hook ? { ...req, _test: hook } : req,
},
Math.max(10, Math.min(1000, limit.ms / 4)),
);
(async () => {
const dec = new TextDecoder();
let buf = "";
let final: WorkerFinal | null = null;
try {
for await (const c of child.stdout) {
buf += dec.decode(c, { stream: true });
let i = buf.indexOf("\n");
while (i >= 0) {
const line = buf.slice(0, i);
buf = buf.slice(i + 1);
i = buf.indexOf("\n");
if (!line.trim()) continue;
const msg = JSON.parse(line) as WorkerFinal & FsProgress;
if (msg.t === "hb") {
const key = `${msg.phase}:${msg.bytes}`;
if (!limit.stall || key !== lastKey) {
lastKey = key;
if (limit.stall) last = Date.now();
}
call.onProgress?.({ phase: msg.phase, bytes: msg.bytes, total: msg.total });
} else final = msg;
{
maxRequestBytes: FS_MESSAGE_BYTES,
maxResponseBytes: FS_MESSAGE_BYTES,
onProgress: (value) => {
const p = value as FsProgress;
if (!p || typeof p.phase !== "string" || !Number.isFinite(p.bytes) || !Number.isFinite(p.total))
throw new Error("文件 worker 进度格式无效");
const key = `${p.phase}:${p.bytes}`;
if (limit.stall && key !== lastKey) {
lastKey = key;
last = Date.now();
}
}
} catch {
// kill 时 stdout 断开
}
const status = await child.status;
await errPump;
if (terminating) {
// SIGTERM 生效(worker 清理后退出)就不必等到宽限期满
clearTimeout(killTimer);
const err = terminating;
return finish(() => reject(err));
}
if (final?.t === "ok") return finish(() => resolve(final!.result));
if (final?.t === "err")
return finish(() =>
reject(
new FsError(final!.code ?? "IO", final!.message ?? "fs worker error", {
...(final!.path ? { path: final!.path } : {}),
}),
),
);
finish(() =>
reject(
new FsError("IO", `fs worker 异常退出(code=${status.code} signal=${status.signal}):${stderr.trim()}`, {
...(pathOf(req) ? { path: pathOf(req) } : {}),
}),
),
);
})();
});
call.onProgress?.(p);
},
},
);
if (call.signal?.aborted) onAbort();
const reply = await pending;
if (reply.type === "result") return reply.result;
throw new FsError(reply.error.code as FsErrorCode, reply.error.message, reply.error.detail);
} catch (error) {
if (error instanceof FsError) throw error;
if (error instanceof WorkerTransportError && error.kind === "spawn")
throw new SpawnError(`spawn_failed:${(error.cause as Error)?.name ?? "Error"}`, error.cause);
throw new FsError("IO", (error as Error).message, { path: pathOf(req), cause: error });
} finally {
clearInterval(watch);
call.signal?.removeEventListener("abort", onAbort);
}
}
}
type WorkerFinal = { t: string; result?: unknown; code?: FsErrorCode; message?: string; path?: string };
class SpawnError extends Error {
constructor(
message: string,
@@ -457,20 +355,6 @@ class SpawnError extends Error {
}
}
type DenoLike = {
execPath(): string;
Command: new (
cmd: string,
opts: {
args: string[];
stdin: "piped";
stdout: "piped";
stderr: "piped";
env?: Record<string, string>;
},
) => { spawn(): DenoChild };
};
let shared: FsProxy | null = null;
/** 进程级缺省代理(配置由 server 启动时 configureFsProxy 注入;未配置时用缺省阈值) */
export function fsProxy(): FsProxy {
+22 -30
View File
@@ -1,34 +1,25 @@
// fsproxy 执行子进程:从 stdin 读一条 JSON 请求,执行单个文件操作,stdout 逐行 JSON 上报
// {t:"hb",phase,bytes,total}(按 progressIntervalMs 节流,阶段切换与阶段完成立即报)/ {t:"ok",result} / {t:"err",code,message}。
// SIGTERM → 中止当前操作(copy 会清掉自己的 partial)后退出;挂死在系统调用里时由父进程 SIGKILL 回收。
// 只依赖 node 内置模块,`deno run --no-config` 毫秒级启动。编译二进制可在入口按 --role fsworker 调 fsWorkerMain()。
import { decodeWorkerRequest, readWorkerRequestText, writeWorkerMessage } from "@umt/runtime/worker";
import { toFsError } from "./errors.ts";
import { type FsOpRequest, type FsProgress, runFsOp } from "./fsops.ts";
const enc = new TextEncoder();
function send(obj: unknown): void {
try {
Deno.stdout.writeSync(enc.encode(`${JSON.stringify(obj)}\n`));
} catch {
// 父进程已退出:无处可报
}
}
import { FS_MESSAGE_BYTES, type FsOpRequest, type FsProgress, runFsOp } from "./fsops.ts";
// 一次性文件 worker;固定操作分派、进度停滞策略与文件清理语义保留在文件模块。
/** 阻塞 JS 线程(测试用,模拟卡死的系统调用:信号监听器也不会运行) */
function blockForever(): never {
const a = new Int32Array(new SharedArrayBuffer(4));
for (;;) Atomics.wait(a, 0, 0);
}
export async function fsWorkerMain(progressIntervalMs = Number(Deno.env.get("UMT_FS_PROGRESS_INTERVAL_MS") ?? 1000)) {
const req = JSON.parse(await new Response(Deno.stdin.readable).text()) as FsOpRequest;
export async function fsWorkerMain(progressIntervalMs = Number(process.env.UMT_FS_PROGRESS_INTERVAL_MS ?? 1000)) {
const request = decodeWorkerRequest(await readWorkerRequestText(FS_MESSAGE_BYTES));
const req = request.input as FsOpRequest;
if (!req || req.op !== request.task) throw new Error("文件 worker 任务与操作不一致");
const send = (value: FsProgress) => writeWorkerMessage({ id: request.id, type: "progress", value }, FS_MESSAGE_BYTES);
const ac = new AbortController();
Deno.addSignalListener("SIGTERM", () => {
process.on("SIGTERM", () => {
if (req._test?.ignoreTerm) return;
ac.abort();
// 操作卡在系统调用里时 abort 解不开:给清理一点时间后直接退出
setTimeout(() => Deno.exit(143), 500);
setTimeout(() => process.exit(143), 500);
});
let lastAt = 0;
let lastPhase = "";
@@ -45,28 +36,29 @@ export async function fsWorkerMain(progressIntervalMs = Number(Deno.env.get("UMT
) {
lastAt = now;
lastPhase = p.phase;
send({ t: "hb", ...p });
send(p);
}
if (hang !== undefined && p.bytes >= hang) blockForever();
};
if (req._test?.hangAfterBytes === 0) {
send({ t: "hb", phase: "start", bytes: 0, total: 0 });
send({ phase: "start", bytes: 0, total: 0 });
blockForever();
}
try {
const result = await runFsOp(req, { signal: ac.signal, progress });
send({ t: "ok", result });
writeWorkerMessage({ id: request.id, type: "result", result }, FS_MESSAGE_BYTES);
} catch (e) {
const fe = toFsError(e);
send({
t: "err",
code: ac.signal.aborted ? "ABORTED" : fe.code,
message: fe.message,
...(fe.detail.path ? { path: fe.detail.path } : {}),
...(fe.detail.syscall ? { syscall: fe.detail.syscall } : {}),
});
writeWorkerMessage(
{
id: request.id,
type: "error",
error: { code: ac.signal.aborted ? "ABORTED" : fe.code, message: fe.message, detail: fe.detail },
},
FS_MESSAGE_BYTES,
);
}
Deno.exit(0);
process.exit(0);
}
if (import.meta.main) await fsWorkerMain();
+1 -1
View File
@@ -44,7 +44,7 @@ export interface MetaRequestCache {
}
export interface MetaClientOptions {
fetch?: typeof globalThis.fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
/** 缺省 15s */
timeoutMs?: number;
/** API 地址(bgm api.bgm.tv / 豆瓣 rexxar) */
+2 -2
View File
@@ -39,7 +39,7 @@ export interface QbInstanceConfig {
sequentialDownload?: boolean;
firstLastPiecePrio?: boolean;
/** 注入 fetch(测试/代理) */
fetch?: typeof fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
}
export type QbAuthMode = "password" | "api_key" | "none";
@@ -314,7 +314,7 @@ export class QbClient implements DownloaderClient {
private readonly retryBaseMs: number;
private readonly lookup: { attempts: number; intervalMs: number };
private readonly probePrefix: string;
private readonly fetchImpl: typeof fetch;
private readonly fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
private sid: string | null = null;
private loginInFlight: Promise<void> | null = null;
/** qB 5.x 把 pause/resume 改名为 stop/start;探测到后缓存 */
+2 -2
View File
@@ -7,7 +7,7 @@ import { redactText, redactUrl } from "./redact.ts";
import { FeedError, type FeedFetchResult, type FeedFetchState } from "./types.ts";
export interface RssFetcherOptions {
fetch?: typeof globalThis.fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
now?: () => Date;
/** 单次请求超时,默认 30s(MP `timeout or 30`) */
timeoutMs?: number;
@@ -31,7 +31,7 @@ export interface FetchFeedOptions {
/** 本次请求的 UA(覆盖 RssFetcherOptions.userAgent;BR-4 按站点账号取) */
userAgent?: string;
/** 本次请求用的 fetch(覆盖 RssFetcherOptions.fetch;BR-4 站点代理 = 与站点其它请求同一出口) */
fetch?: typeof globalThis.fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
}
export function initialFeedState(): FeedFetchState {
@@ -22,7 +22,7 @@ const CF_JS_PAGE = `<html><head><title>Just a moment...</title></head><body>
<script>window._cf_chl_opt={cType: 'non-interactive'}</script></body></html>`;
const NORMAL = `<html><head><title>种子 - 观众</title><script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script></head><body>ok</body></html>`;
function ctx(fetchImpl: typeof fetch) {
function ctx(fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>) {
return {
siteId: "s1",
fetch: fetchImpl,
@@ -32,7 +32,10 @@ function ctx(fetchImpl: typeof fetch) {
};
}
const reply = (status: number, body: string, headers: Record<string, string> = {}) =>
(async () => new Response(body, { status, headers: { "content-type": "text/html", ...headers } })) as typeof fetch;
(async () => new Response(body, { status, headers: { "content-type": "text/html", ...headers } })) as (
input: RequestInfo | URL,
init?: RequestInit,
) => Promise<Response>;
describe("challengeRuleOfHtml", () => {
it("挑战页标题 / 脚本特征命中;正常页面里注入的 challenge-platform 脚本不算", () => {
@@ -62,7 +65,7 @@ describe("detectChallenge", () => {
describe("siteFetch 挑战分类", () => {
const url = "https://audiences.example/torrents.php";
const kind = async (f: typeof fetch) => {
const kind = async (f: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>) => {
try {
await siteFetch(ctx(f), url);
return "ok";
+1 -1
View File
@@ -130,7 +130,7 @@ async function peekBody(
res: Response,
max: number,
): Promise<{ head: Uint8Array<ArrayBuffer>; complete: boolean; res: Response }> {
const reader = (res.body as ReadableStream<Uint8Array>).getReader();
const reader = (res.body as unknown as ReadableStream<Uint8Array>).getReader();
const chunks: Uint8Array[] = [];
let total = 0;
let complete = false;
+1 -1
View File
@@ -5,7 +5,7 @@ import type { SecretResolver } from "./types.ts";
export interface SiteAdapterDeps {
secrets: SecretResolver;
fetch?: typeof globalThis.fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
/** 同站共享的限流器;不传则按站点定义新建(列表/搜索/详情/下载共享) */
limiter?: SiteRateLimiter;
now?: () => Date;
+1 -1
View File
@@ -6,7 +6,7 @@ import { SiteError } from "./types.ts";
export interface SiteHttpContext {
siteId: string;
fetch: typeof globalThis.fetch;
fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
limiter: SiteRateLimiter;
timeoutMs: number;
userAgent?: string;
@@ -53,7 +53,7 @@ export interface NexusLoginInput {
/** base32 TOTP 密钥 */
totpSecret?: string | null;
userAgent: string;
fetch?: typeof globalThis.fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
limiter?: { acquire(signal?: AbortSignal): Promise<void> };
signal?: AbortSignal;
timeoutMs?: number;
@@ -38,7 +38,7 @@ function adapter(body: string, status = 200, headers: Record<string, string> = {
new Response(body, {
status,
headers: { "content-type": "text/html; charset=utf-8", ...headers },
})) as typeof fetch;
})) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
const a = createSpiderSiteAdapter(audiencesDefinition, {
baseUrl: "https://aud.test/",
secrets: staticSecretResolver({ [audiencesDefinition.id]: { cookie: "c_secure_uid=1; c_secure_pass=abc" } }),
@@ -176,7 +176,7 @@ describe("详情页扩展内容(憨憨 2026 真实结构:分节 div,无 #kdescr)
paths.push(`${u.pathname}${u.search}`);
const body = u.pathname === "/viewfilelist.php" ? HHANCLUB_FILELIST_FRAGMENT : HHANCLUB_DETAIL_PAGE_2026;
return new Response(body, { status: 200, headers: { "content-type": "text/html; charset=utf-8" } });
}) as typeof globalThis.fetch;
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
const limiter = createSiteRateLimiter({ minIntervalMs: 0 });
const a = createSpiderSiteAdapter(hhanclubDefinition, {
fetch,
@@ -30,7 +30,7 @@ export function testDef(
export function adapterWith(
def: SpecializedSiteDefinition,
spider: SpecializedSpider,
fetchImpl: typeof globalThis.fetch,
fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
secrets: Partial<Record<SiteCredentialKind, string>> = { cookie: "c=COOKIESECRET", apiKey: "APIKEYSECRET" },
): SiteAdapter {
return createSpecializedSiteAdapter(def, spider, {
@@ -42,10 +42,14 @@ export function adapterWith(
/** 包一层 fetch:命中 predicate 的请求返回合法 .torrent 字节 */
export function withTorrent(
inner: typeof globalThis.fetch,
inner: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
predicate: (url: URL) => boolean,
name = "demo",
): { fetch: typeof globalThis.fetch; infoHash: string; hits: Array<{ url: URL; headers: Headers }> } {
): {
fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
infoHash: string;
hits: Array<{ url: URL; headers: Headers }>;
} {
const t = buildFakeTorrent({ name });
const hits: Array<{ url: URL; headers: Headers }> = [];
const f = (async (input: string | URL | Request, init?: RequestInit) => {
@@ -58,7 +62,7 @@ export function withTorrent(
});
}
return inner(input, init);
}) as typeof globalThis.fetch;
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
return { fetch: f, infoHash: t.infoHash, hits };
}
@@ -4,7 +4,7 @@ import type { SiteUserInfo, SiteUserParserOptions } from "./types.ts";
export async function runParser(
cls: SiteParserClass,
opts: Partial<SiteUserParserOptions> & { fetch: typeof globalThis.fetch },
opts: Partial<SiteUserParserOptions> & { fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> },
): Promise<SiteUserInfo> {
const p = new cls({ siteName: "demo", url: "https://pt.example.com/", ...opts });
await p.parse();
@@ -92,7 +92,7 @@ export interface SiteUserParserOptions {
apiUrl?: string | null;
/** 是否读取未读消息正文(MP 全局设置 `SITE_MESSAGE`);默认 false。注意 MTorrent 读取后会标记已读 */
fetchMessages?: boolean;
fetch?: typeof globalThis.fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
/** 单请求超时(MP 固定 60s) */
timeoutMs?: number;
/** 共享站点限流器(与 list/search 共用) */
+2 -2
View File
@@ -119,7 +119,7 @@ export interface TmdbConfig {
* 语义对照 MP @cached(skip_none、业务失败不缓存、results 为空按 emptyTtlMs)。
*/
sharedCache?: TmdbSharedCache;
fetch?: typeof fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
now?: () => number;
}
@@ -185,7 +185,7 @@ export class TmdbClient {
private readonly langs: string[];
private readonly cache: TtlLruCache<unknown>;
private readonly limiter: RequestLimiter;
private readonly fetchImpl: typeof fetch;
private readonly fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
constructor(private readonly cfg: TmdbConfig) {
if (!cfg.apiKey && !cfg.accessToken) throw new TypeError("TMDB apiKey or accessToken required");
+1 -1
View File
@@ -219,7 +219,7 @@ describe("TmdbClient 缓存/限流/错误分类", () => {
const e1 = await dead.details("movie", 1).catch((x) => x);
expect(e1.kind).toBe("network");
// 模拟把 URL(含 api_key 参数与编码后的明文)拼进信息的 fetch 实现
const leaky: typeof fetch = async (input) => {
const leaky: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = async (input) => {
throw new TypeError(`error sending request for url (${String(input)}): key ${encodeURIComponent(KEY)}`);
};
const e2 = await createTmdbClient({ baseUrl: "http://tmdb.local/3", apiKey: KEY, retries: 0, fetch: leaky })
+1 -1
View File
@@ -16,7 +16,7 @@ export interface BraveClientOptions {
baseUrl?: string;
/** 缺省 10s */
timeoutMs?: number;
fetch?: typeof fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
}
export interface BraveSearchParams {
+1 -1
View File
@@ -18,7 +18,7 @@ export interface FetchPageOptions {
/** 最多跟随几次重定向,缺省 3 */
maxRedirects?: number;
userAgent?: string;
fetch?: typeof fetch;
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
resolve?: HostResolver;
signal?: AbortSignal;
}
@@ -41,7 +41,7 @@ describe("Brave 客户端", () => {
fetch: (async (url: string, init: RequestInit) => {
seen = { url, headers: new Headers(init.headers) };
return jsonRes(BRAVE_BODY);
}) as typeof fetch,
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
});
const r = await c.search({
q: "MoviePilot v3",
@@ -85,8 +85,10 @@ describe("Brave 客户端", () => {
])("HTTP %i → %s,错误信息不含 Key", async (status, kind) => {
const c = createBraveClient({
apiKey: KEY,
fetch: (async () =>
jsonRes({ error: `bad token ${KEY}` }, status, { "x-ratelimit-reset": "1, 1000" })) as typeof fetch,
fetch: (async () => jsonRes({ error: `bad token ${KEY}` }, status, { "x-ratelimit-reset": "1, 1000" })) as (
input: RequestInfo | URL,
init?: RequestInit,
) => Promise<Response>,
});
const err = (await c.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError;
expect(err).toBeInstanceOf(WebSearchError);
@@ -100,7 +102,7 @@ describe("Brave 客户端", () => {
apiKey: KEY,
fetch: (async () => {
throw new TypeError(`connect fail ${KEY}`);
}) as typeof fetch,
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
});
const e1 = (await net.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError;
expect(e1.kind).toBe("network");
@@ -109,9 +111,10 @@ describe("Brave 客户端", () => {
apiKey: KEY,
timeoutMs: 50,
fetch: ((_u: string, init: RequestInit) =>
new Promise((_r, rej) =>
init.signal?.addEventListener("abort", () => rej(new Error("aborted"))),
)) as typeof fetch,
new Promise((_r, rej) => init.signal?.addEventListener("abort", () => rej(new Error("aborted"))))) as (
input: RequestInfo | URL,
init?: RequestInit,
) => Promise<Response>,
});
expect(((await slow.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError).kind).toBe("timeout");
});
@@ -164,8 +167,10 @@ describe("读取网页", () => {
it("HTML 转纯文本:去脚本样式表单,保留标题与段落,URL 凭证脱敏", async () => {
const p = await fetchPage("https://example.org/notice", {
resolve: publicDns,
fetch: (async () =>
new Response(html, { headers: { "content-type": "text/html; charset=utf-8" } })) as typeof fetch,
fetch: (async () => new Response(html, { headers: { "content-type": "text/html; charset=utf-8" } })) as (
input: RequestInfo | URL,
init?: RequestInit,
) => Promise<Response>,
});
expect(p.title).toBe("站点公告");
expect(p.text).toContain("9 月 20 日维护。");
@@ -180,7 +185,10 @@ describe("读取网页", () => {
const big = "a".repeat(600 * 1024);
const p = await fetchPage("https://example.org/big.txt", {
resolve: publicDns,
fetch: (async () => new Response(big, { headers: { "content-type": "text/plain" } })) as typeof fetch,
fetch: (async () => new Response(big, { headers: { "content-type": "text/plain" } })) as (
input: RequestInfo | URL,
init?: RequestInit,
) => Promise<Response>,
});
expect(p.bytes).toBe(512 * 1024);
expect(p.truncated).toBe(true);
@@ -190,22 +198,28 @@ describe("读取网页", () => {
it("非文本类型拒绝;重定向到内网拒绝;超时", async () => {
const pdf = await fetchPage("https://example.org/a.pdf", {
resolve: publicDns,
fetch: (async () => new Response("%PDF", { headers: { "content-type": "application/pdf" } })) as typeof fetch,
fetch: (async () => new Response("%PDF", { headers: { "content-type": "application/pdf" } })) as (
input: RequestInfo | URL,
init?: RequestInit,
) => Promise<Response>,
}).catch((e) => e);
expect((pdf as WebSearchError).kind).toBe("unsupported");
const redir = await fetchPage("https://example.org/r", {
resolve: publicDns,
fetch: (async () =>
new Response(null, { status: 302, headers: { location: "http://192.168.1.1/admin" } })) as typeof fetch,
fetch: (async () => new Response(null, { status: 302, headers: { location: "http://192.168.1.1/admin" } })) as (
input: RequestInfo | URL,
init?: RequestInit,
) => Promise<Response>,
}).catch((e) => e);
expect((redir as WebSearchError).kind).toBe("blocked");
const slow = await fetchPage("https://example.org/slow", {
resolve: publicDns,
timeoutMs: 50,
fetch: ((_u: string, init: RequestInit) =>
new Promise((_r, rej) =>
init.signal?.addEventListener("abort", () => rej(new Error("aborted"))),
)) as typeof fetch,
new Promise((_r, rej) => init.signal?.addEventListener("abort", () => rej(new Error("aborted"))))) as (
input: RequestInfo | URL,
init?: RequestInit,
) => Promise<Response>,
}).catch((e) => e);
expect((slow as WebSearchError).kind).toBe("timeout");
});
@@ -1,6 +1,6 @@
/**
* 把 @umt/testkit 的 mock RSS / NexusPHP fixture 导出成文件,供 MP 差分生成脚本(scripts/mp-sync/gen-differential.py rss|indexer)读取。
* 运行:deno run -A packages/adapters/test/differential/export-inputs.ts
* 运行:bun packages/adapters/test/differential/export-inputs.ts
* 只写 `inputs/rss/tk-*.xml` 与 `inputs/indexer/tk-*.html.txt`;手写样本(hw-*)不受影响。
*/
import { mkdirSync, writeFileSync } from "node:fs";
-9
View File
@@ -1,9 +0,0 @@
{
"name": "@umt/agent",
"version": "0.3.44",
"exports": { ".": "./src/index.ts" },
"tasks": {
"test": "deno run -A npm:vitest run",
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
}
}
+8
View File
@@ -4,5 +4,13 @@
"type": "module",
"exports": {
".": "./src/index.ts"
},
"version": "0.3.44",
"scripts": {
"test": "bun x --bun vitest run",
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
},
"dependencies": {
"@umt/contracts": "workspace:*"
}
}
@@ -0,0 +1,39 @@
---
name: release-facts
title: 资源属性补全
description: 收录规则因分辨率、编码、来源或发布组等属性未知无法判断时,查看资源详情并查证缺失事实,补齐后由原规则重新判断。
triggers:
- 资源属性(未知|缺失|补全)
- release_facts
capabilities:
- release.inspect
- release.apply_facts
- decision.list
- web.search
- web.fetch
sessions: [chat, investigate, child]
---
## 适用意图
收录规则所依赖的分辨率、编码、来源或发布组未知,资源属性事项等待查证后重评。
## 步骤
1. 查看事项与缺失字段,用 release.inspect(refreshDetail: true)读取详情、MediaInfo 和文件信息。刷新失败时说明错误,不能把旧页作为刷新成功的证据。
2. 只补齐该事项缺少且有明确证据的属性。分辨率写实际规格;编码、来源和发布组按详情记载,不从作品类型猜测。
3. 有持续授权时调用 release.apply_facts,带事项、资源、事实和具体出处。原生详情刷新已经补齐字段时可以提交空 facts,服务端核实后触发重评。
4. 无授权或证据不足时保留明确建议,交给用户协同处理。不要修改收录范围、保留目标、组合优先级,也不要直接选另一个资源替代规则判断。
5. 补齐事实不代表资源合格或已下载;最终以原规则重新评估的结果为准。
## 需要批准
`release.apply_facts` 需要覆盖本作品的专用持续授权;已有下载或身份授权不替代它。只读详情与网页查询沿用读取权限。
## 汇报模板
已查证资源编码为 H.265,依据详情页 MediaInfo;补齐后已交原收录规则重新判断。若证据不足,说明具体缺哪项资料与下一步。
## 不要做
不改允许分辨率、保留目标、组合排序、作品身份和季集映射;不覆盖已明确的解析值,不把未知当作不满足,也不把补齐等同于已经下载。
@@ -51,6 +51,7 @@ export const INTENT_ROUTES: readonly IntentRoute[] = [
{ cap: "inventory.query", note: "库里已有的文件与收录版本" },
{ cap: "release.search", note: "找可下载的资源" },
{ cap: "release.inspect", note: "看某条资源的文件清单与促销" },
{ cap: "release.apply_facts", note: "查证后补齐缺失资源属性,再由规则判断" },
{ cap: "download.plan", note: "准备下载,返回是否需要批准" },
{ cap: "download.submit", note: "提交到下载器" },
],
+4 -2
View File
@@ -17,8 +17,10 @@ import type { PromptSection, SessionKind, SkillDoc } from "./types.ts";
// pi 1.x 把 harness 移到 pi-durable 后不再提供 skills,这里只保留本项目用到的部分:
// 一层目录下的 <name>/SKILL.md,不读 .gitignore、不收根目录的散 .md、不跟随符号链接。
/** 仓库内的 Skill 目录;deno compile 用 --include 嵌入(apps/server/deno.json 的 compile 任务) */
export const SKILLS_DIR = fileURLToPath(new URL("../../skills", import.meta.url));
/** 仓库内的 Skill 目录;bun build --compile 用 --include 嵌入(apps/server/package.json 的 compile 任务) */
export const SKILLS_DIR = process.execPath.endsWith("/umt-server")
? join(process.execPath.replace(/[^/]+$/, ""), "skills")
: fileURLToPath(new URL("../../skills", import.meta.url));
const DEFAULT_SESSIONS: SessionKind[] = ["chat", "patrol", "investigate", "child"];
const SESSION_KINDS = new Set<SessionKind>(["chat", "patrol", "investigate", "child", "vision"]);
+4 -8
View File
@@ -354,14 +354,10 @@ describe("PiAgentRuntime + FakeModelProvider", () => {
it("JSONL 恢复:子进程在 download.submit 执行中被 SIGKILL,新进程 resume 得到“结果未知”且不重复提交", async () => {
const sessionDir = await tempDir();
const child = spawn(
process.execPath,
["run", "-A", join(import.meta.dirname!, "fixtures/crash-run.ts"), sessionDir],
{
cwd: join(import.meta.dirname!, ".."),
stdio: ["ignore", "pipe", "pipe"],
},
);
const child = spawn(process.execPath, [join(import.meta.dirname!, "fixtures/crash-run.ts"), sessionDir], {
cwd: join(import.meta.dirname!, ".."),
stdio: ["ignore", "pipe", "pipe"],
});
let stderr = "";
child.stderr.on("data", (d) => {
stderr += String(d);
+3 -3
View File
@@ -150,14 +150,14 @@ describe("按上游装配", () => {
it("测试补全带 sessionId:OpenCode Go 请求头有 x-opencode-session", async () => {
const seen: { url: string; headers: Headers }[] = [];
const orig = globalThis.fetch;
globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
const req = new Request(input, init);
globalThis.fetch = (async (input: string | Request | URL, init?: RequestInit) => {
const req = new Request(input instanceof Request ? input : String(input), init);
seen.push({ url: req.url, headers: req.headers });
return new Response(JSON.stringify({ error: { message: "stubbed" } }), {
status: 500,
headers: { "content-type": "application/json" },
});
}) as typeof fetch;
}) as typeof globalThis.fetch;
try {
const c = new PiUpstreamClient(
spec({ provider: "opencode-go", authKind: "api_key" }),
-9
View File
@@ -1,9 +0,0 @@
{
"name": "@umt/contracts",
"version": "0.3.44",
"exports": { ".": "./src/index.ts" },
"tasks": {
"test": "deno run -A npm:vitest run",
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
}
}
+7 -1
View File
@@ -4,5 +4,11 @@
"type": "module",
"exports": {
".": "./src/index.ts"
}
},
"version": "0.3.44",
"scripts": {
"test": "bun x --bun vitest run",
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
},
"dependencies": {}
}
+31
View File
@@ -76,6 +76,37 @@ export function tmdbImageUrl(path: string | null | undefined, size: TmdbImageSiz
return `/api/v1/img/tmdb/${size}/${p}`;
}
/**
* 外部图片代理允许的域名:豆瓣图床(img1–9.doubanio.com 等)与 Bangumi 图床。豆瓣图床对不带 Referer 的请求回 418,
* 浏览器 no-referrer 直连拿不到图,所以经服务端带上来源站的 Referer 取(MP SECURITY_IMAGE_DOMAINS 同此)。
*/
export const EXTERNAL_IMAGE_HOSTS: readonly { suffix: string; referer: string }[] = [
{ suffix: ".doubanio.com", referer: "https://movie.douban.com/" },
{ suffix: "lain.bgm.tv", referer: "https://bgm.tv/" },
];
/** 外部图片地址是否在代理白名单里(只认 https、图片扩展名) */
export function externalImageReferer(url: string): string | null {
let u: URL;
try {
u = new URL(url);
} catch {
return null;
}
if (u.protocol !== "https:" || !/\.(jpe?g|png|webp)$/i.test(u.pathname)) return null;
const host = u.hostname.toLowerCase();
const hit = EXTERNAL_IMAGE_HOSTS.find((h) =>
h.suffix.startsWith(".") ? host.endsWith(h.suffix) : host === h.suffix,
);
return hit?.referer ?? null;
}
/** 豆瓣 / Bangumi 封面 → 本站图片代理 /api/v1/img/ext?u=…;不在白名单的原样返回 */
export function externalImageUrl(url: string | null | undefined): string | null {
if (!url) return null;
return externalImageReferer(url) ? `/api/v1/img/ext?u=${encodeURIComponent(url)}` : url;
}
// ── 通用缓存(kv_cache)按 region 管理(设置 → 缓存) ──
/** region 名:小写字母、数字、`.`、`_`、`-` */
+27 -1
View File
@@ -40,6 +40,7 @@ import {
SiteKind,
SiteLastHealth,
SiteLoginResult,
SiteReleasePage,
SiteUserDataPoint,
SourceRelease,
} from "./sources.ts";
@@ -116,7 +117,31 @@ export const ReleaseSearchOutput = z.object({
});
export const ReleaseInspectInput = z.object({ sourceReleaseId: Id, refreshDetail: z.boolean().default(false) });
export const ReleaseInspectOutput = z.object({ release: SourceRelease, inventory: z.array(MediaAsset) });
export const ReleaseInspectOutput = z.object({
release: SourceRelease,
inventory: z.array(MediaAsset),
page: SiteReleasePage.nullable().optional(),
refreshError: z.string().nullable().optional(),
});
export const ReleaseStaticFacts = z
.object({
resolution: z.enum(["4320p", "2160p", "1440p", "1080p", "1080i", "720p", "576p", "480p"]).optional(),
videoCodec: z.enum(["H.265", "H.264", "AV1", "VC-1", "MPEG-2", "XviD", "DivX"]).optional(),
source: z.enum(["REMUX", "BluRay-Disc", "BluRay", "WEB-DL", "WEBRip", "HDTV", "HDRip", "DVDRip", "DVD"]).optional(),
releaseGroup: z.string().trim().min(1).max(200).optional(),
})
.strict();
export type ReleaseStaticFacts = z.infer<typeof ReleaseStaticFacts>;
export const ReleaseApplyFactsInput = z
.object({
decisionId: Id,
sourceReleaseId: Id,
facts: ReleaseStaticFacts,
evidence: z.string().trim().min(10).max(2000),
evidenceUrl: z.url().optional(),
})
.strict();
export const ReleaseApplyFactsOutput = z.object({ resolved: z.boolean(), remaining: z.array(z.string()) });
export const IdentityResolveInput = z
.object({
@@ -1136,6 +1161,7 @@ export const CAPABILITY_IO = {
"catalog.lookup": { input: CatalogLookupInput, output: CatalogLookupOutput },
"release.search": { input: ReleaseSearchInput, output: ReleaseSearchOutput },
"release.inspect": { input: ReleaseInspectInput, output: ReleaseInspectOutput },
"release.apply_facts": { input: ReleaseApplyFactsInput, output: ReleaseApplyFactsOutput },
"identity.resolve": { input: IdentityResolveInput, output: IdentityResolveOutput },
"download.plan": { input: DownloadPlanInput, output: DownloadPlanOutput },
"download.submit": { input: DownloadSubmitInput, output: DownloadSubmitOutput },
+14
View File
@@ -57,6 +57,7 @@ export const CAPABILITY_IDS = [
"catalog.lookup",
"release.search",
"release.inspect",
"release.apply_facts",
"identity.resolve",
"download.plan",
"download.submit",
@@ -185,6 +186,18 @@ export const CAPABILITY_CATALOG: Record<KnownCapabilityId, CapabilityDescriptor>
true,
true,
),
"release.apply_facts": d(
"release.apply_facts",
"补全资源属性",
"查证资源详情后,仅补全事项里缺失的分辨率、编码、来源或发布组;规则重新判断收录。不改变已有事实和策略。",
"execute",
"ReleaseApplyFacts",
"organize.execute",
"medium",
true,
true,
"safe",
),
"download.plan": d(
"download.plan",
"准备下载",
@@ -704,6 +717,7 @@ export const CAPABILITY_ACTIVITY: Record<KnownCapabilityId, CapabilityActivity>
"catalog.lookup": act("查", "作品信息"),
"release.search": act("搜", "资源", "次"),
"release.inspect": act("看", "资源", "个"),
"release.apply_facts": act("补全", "资源属性", "个"),
"identity.resolve": act("识别", "作品", "个"),
"download.plan": { ...act("准备下载", "资源", "个"), bare: true },
"download.submit": act("提交", "下载", "个"),
+1
View File
@@ -184,6 +184,7 @@ export const ReleaseResolvedPayload = z.object({
});
export const DecisionCreatedPayload = z.object({
dedupeKey: z.string().optional(),
decisionId: Id,
kind: DecisionKind,
severity: Severity,
-9
View File
@@ -1,6 +1,5 @@
import type { KnownCapabilityId } from "./capability.ts";
import type { EventType } from "./events.ts";
import type { HookPoint } from "./hooks.ts";
// 功能模块约定(ADR-006):外围功能各自声明一份 manifest,汇总成功能清单。
// 功能清单只给智能体看(系统提示的「功能清单」一节),不做开关;前端插槽默认直接显示。
@@ -31,8 +30,6 @@ export interface FeatureManifest {
pages: readonly string[];
/** 挂在哪些前端插槽 */
slots: readonly SlotPosition[];
/** 注册的链式钩子 */
hooks: readonly HookPoint[];
}
/** 功能清单。新增外围功能:在这里加一份,服务端模块 manifest 指向它(featureManifest(id)) */
@@ -47,7 +44,6 @@ export const FEATURES = [
capabilities: [],
pages: ["作品页「补字幕」", "整理记录行菜单「补字幕」", "站点页「字幕站」"],
slots: ["entity.actions", "organize.record.actions"],
hooks: [],
},
{
id: "browser",
@@ -64,7 +60,6 @@ export const FEATURES = [
],
pages: ["站点页「原站视图」"],
slots: [],
hooks: [],
},
{
id: "desktop",
@@ -75,7 +70,6 @@ export const FEATURES = [
capabilities: [],
pages: ["原站视图「桌面模式」"],
slots: [],
hooks: [],
},
{
id: "websearch",
@@ -86,7 +80,6 @@ export const FEATURES = [
capabilities: ["web.search", "web.fetch"],
pages: ["设置 → 智能体 → 网页搜索"],
slots: [],
hooks: [],
},
{
id: "mcp",
@@ -97,7 +90,6 @@ export const FEATURES = [
capabilities: [],
pages: ["设置 → 智能体 → 工具与 MCP"],
slots: [],
hooks: [],
},
{
id: "schedules",
@@ -108,7 +100,6 @@ export const FEATURES = [
capabilities: ["schedule.draft_check"],
pages: ["日志 → 定时任务"],
slots: [],
hooks: [],
},
] as const satisfies readonly FeatureManifest[];
-44
View File
@@ -1,44 +0,0 @@
// 链式钩子挂点(ADR-006,参考 MoviePilot ChainEventType):同步、按注册顺序执行、可改写或拦截。
// 这里只登记挂点名称与语义(给功能清单、智能体提示与文档用);入参 / 返回值类型在服务端
// apps/server/src/hooks/points.ts,因为它们用到 domain 的解析结果等类型。
/**
* - rewrite:每个钩子拿到上一个钩子的结果,返回新值或 undefined(不改);
* - intercept:任一钩子返回拒绝原因即停止,后面的钩子不再执行。
*/
export type HookKind = "rewrite" | "intercept";
export const HOOK_POINTS = {
"name.recognize": {
kind: "rewrite",
label: "名称识别",
when: "资源标题解析之后、按名称召回作品之前",
effect: "改写解析结果(标题、年份、季集、外部 ID)",
},
"organize.rename": {
kind: "rewrite",
label: "整理改名",
when: "按命名模板生成目标路径之后",
effect: "改写主视频的目标文件名(同主干的字幕等附属文件跟着改)",
},
"organize.intercept": {
kind: "intercept",
label: "整理拦截",
when: "整理计划生成后、写整理记录之前",
effect: "返回原因即拒绝本次整理(记录为失败,原因 intercepted:<原因>)",
},
"resource.selection": {
kind: "rewrite",
label: "资源选择",
when: "订阅 / 收录版本重评时候选排好序之后、逐条评估之前",
effect: "调整候选顺序或过滤掉一部分(不能加入新候选)",
},
"subscription.completion.check": {
kind: "intercept",
label: "订阅完成判定",
when: "订阅满足完成条件、标记完成之前",
effect: "追加完成条件:返回原因即本次不完成,订阅继续追踪",
},
} as const satisfies Record<string, { kind: HookKind; label: string; when: string; effect: string }>;
export type HookPoint = keyof typeof HOOK_POINTS;
-1
View File
@@ -15,7 +15,6 @@ export * from "./episode-mapping.ts";
export * from "./events.ts";
export * from "./execution.ts";
export * from "./features.ts";
export * from "./hooks.ts";
export * from "./identify-words.ts";
export * from "./identity.ts";
export * from "./instances.ts";
+14 -2
View File
@@ -85,7 +85,7 @@ export const Subscription = EntityBase.extend({
mediaEntityTitle: z.string().nullable().optional(),
});
export type Subscription = z.infer<typeof Subscription>;
/** 自动补全其他版本:inherit = 取引用规则组(后者优先)的 autoFill,都没设为开 */
/** 自动补全其他版本:inherit = 取主版本收录规则链(后者优先)的 autoFill,都没设为关 */
export const AutoFillMode = z.enum(["inherit", "on", "off"]);
export type AutoFillMode = z.infer<typeof AutoFillMode>;
/** 补全预算:只约束非主版本的自动补全计划(按订阅、按自然日);同时仍计入全局每日预算 */
@@ -121,7 +121,8 @@ export type SubscriptionEffect = z.infer<typeof SubscriptionEffect>;
* siteIds 站点范围;episodeRange 集范围(对照 MP start/total_episode);searchIntervalHours 主动搜索周期(0 = 关);
* stopAfter 截止时间;downloaderInstanceId 指定下载器;ruleGroupIds 规则组;keywords 追加检索词;
* include/exclude 订阅级关键词正则(作用于全部收录版本);libraryRootId 库根;downloadSavePath 下载保存路径;
* autoFill/fillBudget 多版本自动补全;resolution/source/effect 质量预设(MP resolution/quality/effect)。
* autoFill/fillBudget 多版本自动补全;resolution/source/effect 仅保留 MP 导入/API 质量预设兼容。
* 正常界面逐版本维护质量和规则引用,普通订阅保存保留旧兼容字段。
* include/exclude 保存时同步为订阅私有规则组(scope.subscriptionId,随订阅删除),评估只读规则组。
* 写入时兼容 MP 字段与旧键(sites/downloader/save_path/start_episode/filter_groups/keyword/autoFillVersions…),
* 服务端规范化后只存本表列出的键,见 docs/runbook/subscriptions.md。
@@ -714,6 +715,17 @@ export type PreferenceSuppressionUpdate = z.infer<typeof PreferenceSuppressionUp
// ── 收录策略规则与候选评估(W2-C) ──
/** 分类名称沿用整理分类;显式选择策略时不再按分类过滤。 */
export const AcquisitionPolicyScope = z
.object({
mediaKinds: z.array(SubscriptionMediaKind).optional(),
categories: z
.object({ movie: z.array(z.string().min(1)).optional(), tv: z.array(z.string().min(1)).optional() })
.optional(),
})
.passthrough();
export type AcquisitionPolicyScope = z.infer<typeof AcquisitionPolicyScope>;
/** Slot 模板:策略规则里用于自动建 Slot(新作品收录 / 订阅缺省 Slot);字段同 AcquisitionSlot 规则列 */
export const SlotTemplate = z.object({
name: z.string().min(1).max(64),
+8 -4
View File
@@ -229,16 +229,20 @@ describe("关键合同行为", () => {
}
});
it("capabilitySpec 组装目录元数据与真实 schema;有副作用的类别不可重放", () => {
it("capabilitySpec 组装目录元数据与真实 schema;写入仅明确的幂等例外可重放", () => {
for (const id of C.CAPABILITY_IDS) {
const spec = C.capabilitySpec(id);
const { inputSchema: _i, outputSchema: _o, ...meta } = C.CAPABILITY_CATALOG[id];
expect(spec).toMatchObject(meta);
expect(spec.inputSchema).toBe(C.CAPABILITY_IO[id].input);
expect(spec.outputSchema).toBe(C.CAPABILITY_IO[id].output);
// 例外:site.health_check 只读首页并写回健康列,幂等可重跑;metadata.enrich 只存补全建议(7 天标题缓存),重跑无副作用
if (spec.kind !== "read" && spec.kind !== "preview" && id !== "site.health_check" && id !== "metadata.enrich")
expect(spec.replay, id).toBe("never");
// 健康列、补全建议、带事项与事件去重的资源事实可幂等重跑;其它写入仍不可重放。
if (spec.kind !== "read" && spec.kind !== "preview") {
if (["site.health_check", "metadata.enrich", "release.apply_facts"].includes(id)) {
expect(spec.replay, id).toBe("safe");
expect(spec.idempotent, id).toBe(true);
} else expect(spec.replay, id).toBe("never");
}
}
});
+1 -1
View File
@@ -5,7 +5,7 @@ PostgreSQL 访问层:Kysely 客户端与连接池、表类型(`src/types.ts`)、
## 规则
- **迁移是唯一事实来源**。所有 schema 变更只能新增迁移,已发布迁移不得修改。迁移用原生 DDL 书写(`src/migration-helpers.ts`:`entityTable()` 统一加 uuid 主键 `gen_random_uuid()`、`created_at/updated_at` 与 updated_at 触发器;`oneOf()` 生成状态 CHECK)。`migrations/` 内相对导入必须带 `.ts` 扩展名(Kysely 用原生 `import()` 加载)。
- **`src/types.ts` 是生成物**:`deno task db:migrate && deno task gen:types`。CHECK 枚举自动变成字面量联合(如 `AcquisitionSlotStatus`)。`test/schema.test.ts` 会在隔离 schema 里重新反射并比对,漂移即失败;`packages/contracts/test/db-sync.test.ts` 再校验 contracts 的 Row schema 与枚举和这里一致。
- **`src/types.ts` 是生成物**:`bun run db:migrate && bun run gen:types`。CHECK 枚举自动变成字面量联合(如 `AcquisitionSlotStatus`)。`test/schema.test.ts` 会在隔离 schema 里重新反射并比对,漂移即失败;`packages/contracts/test/db-sync.test.ts` 再校验 contracts 的 Row schema 与枚举和这里一致。
- 状态列一律 `text + CHECK`,不用 PG enum(便于迁移加值)。JSONB 只放证据、规则参数、原文快照与演进字段;稳定查询字段用关系列。
- 凭证只存 `secret_ref`(→ `secret.id`),明文不入库、不进日志。
- `createDb()` 把 `int8` 解析为 `number`(超出 2^53 抛错)、`date` 保持 `YYYY-MM-DD` 字符串。写 jsonb 数组时须先 `JSON.stringify`(pg 会把 JS 数组当 PG 数组)。
-12
View File
@@ -1,12 +0,0 @@
{
"name": "@umt/db",
"version": "0.3.44",
"exports": { ".": "./src/index.ts" },
"tasks": {
"test": "deno run -A npm:vitest run",
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json",
"migrate": "deno run -A --env-file=../../.env scripts/migrate.ts latest",
"reset": "deno run -A --env-file=../../.env scripts/migrate.ts reset",
"gen:types": "deno run -A --env-file=../../.env scripts/gen-types.ts"
}
}
+11
View File
@@ -4,5 +4,16 @@
"type": "module",
"exports": {
".": "./src/index.ts"
},
"version": "0.3.44",
"scripts": {
"test": "bun x --bun vitest run",
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json",
"migrate": "bun --env-file=../../.env scripts/migrate.ts latest",
"reset": "bun --env-file=../../.env scripts/migrate.ts reset",
"gen:types": "bun --env-file=../../.env scripts/gen-types.ts"
},
"dependencies": {
"@umt/runtime": "workspace:*"
}
}
+2 -2
View File
@@ -3,7 +3,7 @@ import { fileURLToPath } from "node:url";
import { type Kysely, sql } from "kysely";
// 从已迁移的数据库反射出 src/types.ts:列类型 + CHECK (col in (...)) 枚举字面量联合。
// 迁移是唯一事实来源;改表后运行 `deno task gen:types`,测试会校验两者无漂移。
// 迁移是唯一事实来源;改表后运行 `bun run gen:types`,测试会校验两者无漂移。
interface ColumnInfo {
table_name: string;
@@ -77,7 +77,7 @@ export async function generateTypesSource(db: Kysely<any>): Promise<string> {
const out: string[] = [
"// 本文件由 scripts/gen-types.ts 从迁移后的数据库反射生成,请勿手改。",
"// 改表:新增迁移 → deno task db:migrate → deno task gen:types。",
"// 改表:新增迁移 → bun run db:migrate → bun run gen:types。",
"",
'import type { ColumnType, Generated, Insertable, Selectable, Updateable } from "kysely";',
"",
+1 -1
View File
@@ -1,4 +1,4 @@
// 用法:deno task gen:types —— 对 DATABASE_URL 的当前 schema 反射并重写 src/types.ts
// 用法:bun run gen:types —— 对 DATABASE_URL 的当前 schema 反射并重写 src/types.ts
import { writeFileSync } from "node:fs";
import { createDb } from "../src/index.ts";
import { formatWithBiome, generateTypesSource, TYPES_FILE } from "./gen-types-lib.ts";
+1 -1
View File
@@ -1,4 +1,4 @@
// 用法:deno task db:migrate / db:reset(= deno run -A --env-file=.env packages/db/scripts/migrate.ts latest|reset)
// 用法:bun run db:migrate / db:reset(= bun --env-file=.env packages/db/scripts/migrate.ts latest|reset)
import { createDb, migrateToLatest, resetSchema } from "../src/index.ts";
const cmd = process.argv[2] ?? "latest";
+1 -1
View File
@@ -1,5 +1,5 @@
// 本文件由 scripts/gen-types.ts 从迁移后的数据库反射生成,请勿手改。
// 改表:新增迁移 → deno task db:migrate → deno task gen:types。
// 改表:新增迁移 → bun run db:migrate → bun run gen:types。
import type { ColumnType, Generated, Insertable, Selectable, Updateable } from "kysely";
+1 -1
View File
@@ -3,7 +3,7 @@ import { Kysely, PostgresDialect } from "kysely";
import pg from "pg";
import { PgTokenBucket } from "../../src/rate-limit.ts";
const [url, schema, key, n, capacity, rate] = Deno.args as [string, string, string, string, string, string];
const [url, schema, key, n, capacity, rate] = process.argv.slice(2) as [string, string, string, string, string, string];
const db = new Kysely<unknown>({
dialect: new PostgresDialect({
pool: new pg.Pool({ connectionString: url, max: 1, options: `-c search_path=${schema},public` }),
+3 -2
View File
@@ -1,4 +1,5 @@
import { fileURLToPath } from "node:url";
import { Command } from "@umt/runtime";
import { Kysely, PostgresDialect, sql } from "kysely";
import pg from "pg";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
@@ -70,8 +71,8 @@ describe.skipIf(skip)("PgTokenBucket(真实 PG)", () => {
const rate = 20;
const perProc = 12;
const procs = [0, 1, 2].map(() =>
new Deno.Command(Deno.execPath(), {
args: ["run", "-A", script, url, schema, "t:procs", String(perProc), String(capacity), String(rate)],
new Command(process.execPath, {
args: [script, url, schema, "t:procs", String(perProc), String(capacity), String(rate)],
stdout: "piped",
stderr: "piped",
}).output(),
+3 -3
View File
@@ -3,8 +3,8 @@
纯业务逻辑(无 IO):发布标题解析、身份匹配、Slot 满足度、收录模式/预算/磁盘阈值、路径绑定与命名、整理计划、下载状态映射。
- 禁止 import 任何网络/文件/数据库库,也不读全局配置与时钟;时间(`now`)、设备号、文件清单、已存在目标等事实一律由调用方传入。
- 不依赖 `@umt/contracts`:这里的类型是领域类型,W2 在 server/tasks 层映射到 zod 合同。
- 唯一入口 `src/index.ts`(`export *` 各模块)。测试:`cd packages/domain && deno task test`。
- 同一语义的类型和纯规则可以依赖 `@umt/contracts`,例如 `ContentType`;含义不同的领域类型通过 `@umt/mapping` 转换。保持无 IO,不依赖 server / db / adapters。
- 唯一入口 `src/index.ts`(`export *` 各模块)。测试:`cd packages/domain && bun run test`。
## 模块地图
@@ -49,7 +49,7 @@
| move 跨盘 | EXDEV 隐式降级复制,删源失败仍成功 | 只标记 `crossDevice`,执行层显式三步 |
| Move 删种 | 不看 H&R | H&R 未满足/未知 → 阻塞冲突 |
## 给 W2 的接入提示
## 接入提示
- 解析:下载完成后用 `parseReleasePath({ file, parent, grandParent })` 或 `planOrganize({ sourceRoot })`(只在种子根内取目录名);RSS/站点标题用 `parseRelease(title, { subtitle })`。`ParsedRelease.sizeBytes` 可用站点字段覆盖(`{ ...parsed, sizeBytes }`)后再交给 Slot 评估。
- 身份:本地别名召回(pg_trgm)与 TMDB 结果都转成 `IdentityCandidate`(本地用 `provider: "local"` + `localEntityId` + `linkedExternalIds`);`resolveIdentity` 为 `auto` 才自动绑定,其余进待识别池。
-9
View File
@@ -1,9 +0,0 @@
{
"name": "@umt/domain",
"version": "0.3.44",
"exports": { ".": "./src/index.ts" },
"tasks": {
"test": "deno run -A npm:vitest run",
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
}
}
+8
View File
@@ -4,5 +4,13 @@
"type": "module",
"exports": {
".": "./src/index.ts"
},
"version": "0.3.44",
"scripts": {
"test": "bun x --bun vitest run",
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
},
"dependencies": {
"@umt/contracts": "workspace:*"
}
}
+1
View File
@@ -25,6 +25,7 @@ export {
resolveAutoFill,
} from "./fill.ts";
export { type AcquisitionMode, determineMode, type ModeDecision, type ModeInput } from "./mode.ts";
export { type ScopedAcquisitionPolicy, selectAcquisitionPolicy } from "./policy.ts";
export {
DEFAULT_PRIORITY_ORDER,
type PriorityClass,
+30
View File
@@ -0,0 +1,30 @@
import type { AcquisitionPolicyScope } from "@umt/contracts";
export interface ScopedAcquisitionPolicy {
id: string;
scope: AcquisitionPolicyScope;
}
/** 输入已按发布时间排序;分类明确命中优先于类型默认,显式选择不回退到别的策略。 */
export function selectAcquisitionPolicy<T extends ScopedAcquisitionPolicy>(
policies: readonly T[],
input: { explicitId?: string | null; mediaKind: "movie" | "series" | "variety"; category: string | null },
): T | undefined {
if (input.explicitId) return policies.find((p) => p.id === input.explicitId);
const type = input.mediaKind === "movie" ? "movie" : "tv";
let selected: T | undefined;
let specificity = -1;
for (const p of policies) {
if (p.scope.mediaKinds?.length && !p.scope.mediaKinds.includes(input.mediaKind)) continue;
const categories = p.scope.categories;
if (categories && Object.values(categories).some((v) => v?.length)) {
if (!input.category || !categories[type]?.includes(input.category)) continue;
}
const score = categories?.[type]?.length ? 2 : p.scope.mediaKinds?.length ? 1 : 0;
if (score > specificity) {
selected = p;
specificity = score;
}
}
return selected;
}
+3 -9
View File
@@ -1,3 +1,4 @@
import type { ContentType } from "@umt/contracts";
import type { CompiledGlobalWord } from "../identify-words/apply.ts";
/** 解析器版本:规则变化时递增,缓存/识别结果按此失效 */
@@ -35,15 +36,8 @@ export type Cut =
| "Ultimate"
| "Special Edition";
/** 综艺内容类型(REQ §5.1);v0.1 只识别标记,不做分段建模 */
export type ContentType =
| "unknown"
| "main"
| "extra"
| "behind_the_scenes"
| "performance_only"
| "preview"
| "reaction";
// 内容类型在 contracts 定义一次;领域与接口使用相同语义。
export type { ContentType };
export type Segment = "full" | "upper" | "middle" | "lower";
+77
View File
@@ -0,0 +1,77 @@
import type { ParsedRelease } from "../release-parser/types.ts";
import type { HardField } from "../slot/types.ts";
import { type AtomCatalog, testAtom } from "./evaluate.ts";
import type { RuleContext, RuleExpr, RuleGroupSpec } from "./types.ts";
// 只对有明确事实字段的条件判断未知;正则文本缺词仍是已知不命中。
const fields: Record<string, HardField> = {
"4K": "resolution",
"1080P": "resolution",
"720P": "resolution",
BLU: "source",
BLURAY: "source",
REMUX: "source",
WEBDL: "source",
H265: "codec",
H264: "codec",
FREE: "promotion",
};
type PartialResult = { hit: boolean | null; missing: HardField[] };
function partial(e: RuleExpr, r: ParsedRelease, ctx: RuleContext, catalog: AtomCatalog): PartialResult {
if (e.op === "atom") {
const id = e.id.toUpperCase();
const f = fields[id];
const value =
f === "resolution"
? r.resolution
: f === "source"
? r.source
: f === "codec"
? r.videoCodec
: ctx.downloadVolumeFactor;
if (f && value === undefined) return { hit: null, missing: [f] };
const custom = catalog.get(id);
const missing: HardField[] = [];
if (custom?.minSeeders !== undefined && ctx.seeders === undefined) missing.push("seeders");
if (custom?.free && ctx.downloadVolumeFactor === undefined) missing.push("promotion");
if (custom?.unitSizeMB && r.sizeBytes === undefined) missing.push("size");
const input = { r, ctx, text: [r.raw ?? "", ctx.subtitle ?? "", ...(ctx.labels ?? [])].join(" ") };
// 已知的正则条件失败时,其他缺失字段不会改变结论。
if (custom && missing.length) {
const known = { ...custom, minSeeders: undefined, free: undefined, unitSizeMB: undefined };
const knownCatalog = new Map(catalog).set(id, known);
if (!testAtom(id, input, knownCatalog).hit) return { hit: false, missing: [] };
return { hit: null, missing };
}
return { hit: testAtom(id, input, catalog).hit, missing: [] };
}
if (e.op === "not") {
const result = partial(e.arg, r, ctx, catalog);
return { ...result, hit: result.hit === null ? null : !result.hit };
}
const results = e.args.map((a) => partial(a, r, ctx, catalog));
const decisive = e.op !== "and";
if (results.some((x) => x.hit === decisive)) return { hit: decisive, missing: [] };
const unknown = results.filter((x) => x.hit === null);
return unknown.length ? { hit: null, missing: unknown.flatMap((x) => x.missing) } : { hit: !decisive, missing: [] };
}
/** 未知的高档不能被当成不命中而落入低档,NOT 也不能把未知当成事实。 */
export function unknownRuleFields(
groups: readonly RuleGroupSpec[],
r: ParsedRelease,
ctx: RuleContext,
catalog: AtomCatalog,
): HardField[] {
const missing: HardField[] = [];
for (const group of groups) {
for (const expr of group.hardExcludes ?? []) missing.push(...partial(expr, r, ctx, catalog).missing);
for (const layer of group.layers) {
const result = partial(layer.expr, r, ctx, catalog);
missing.push(...result.missing);
if (result.hit === true) break;
}
}
return [...new Set(missing)];
}
+95 -11
View File
@@ -1,6 +1,8 @@
import type { ParsedRelease, ReleaseSource, Resolution } from "../release-parser/types.ts";
import { catalogOf, evaluateRuleChain } from "../rules/evaluate.ts";
import type { RuleChainResult } from "../rules/types.ts";
import { unknownRuleFields } from "../rules/unknown.ts";
import { seasonReleaseKey, upgradeSpec } from "./season-release.ts";
import type {
EvaluationContext,
HardField,
@@ -310,6 +312,8 @@ export function describeViolation(v: Violation): string {
return `优先级没有提高(现有 ${JSON.stringify(e)},这个资源 ${JSON.stringify(a)})`;
case "upgrade.not_full_pack":
return "全集洗版只接受整季包";
case "upgrade.different_season_release":
return "与当季选定的成套版本不同,等待同套资源";
case "upgrade.no_improvable_units":
return "这个资源包含的季集都没有可升级的";
default:
@@ -348,6 +352,37 @@ function tierFailures(rule: SlotTierRule, r: ParsedRelease, ctx: EvaluationConte
return out;
}
function unknownTierFields(spec: SlotSpec, r: ParsedRelease, ctx: EvaluationContext): HardField[] {
for (const tier of spec.soft?.tiers ?? []) {
if (!tierFailures(tier, r, ctx).length) break;
const missing: HardField[] = [];
const filled = { ...r };
if (!r.resolution && tier.resolutions?.length) {
missing.push("resolution");
filled.resolution = tier.resolutions[0];
}
if (!r.source && (tier.sources?.length || tier.excludeSources?.length || tier.requireEncode)) {
missing.push("source");
filled.source =
tier.sources?.[0] ??
(tier.requireEncode
? "BluRay"
: (Object.keys(DEFAULT_SOURCE_TIERS) as ReleaseSource[]).find((s) => !tier.excludeSources?.includes(s)));
filled.isEncode = tier.requireEncode ?? filled.source === "BluRay";
}
if (!r.videoCodec && tier.codecs?.length) {
missing.push("codec");
filled.videoCodec = tier.codecs[0];
}
if (!r.releaseGroup && tier.groups?.length) {
missing.push("group");
filled.releaseGroup = { name: tier.groups.join("@"), source: "known" };
}
if (missing.length && !tierFailures(tier, filled, ctx).length) return missing;
}
return [];
}
/**
* 优先级阶梯(对应 MP 规则组 `A > B > C`):逐层匹配,第 k 层(0 起)命中 → pri = 100 - k;
* 未配置阶梯 → pri = 100、level = null(不参与排序差异);配置了但都不命中 → pri = 0。
@@ -465,9 +500,21 @@ export function evaluateCandidate(
current?: QualityFacts | null,
ctx: EvaluationContext = {},
): SlotEvaluation {
if (current) spec = upgradeSpec(spec);
const hard = checkHardConditions(spec, r, ctx);
const { missing, accepted } = hard;
const chain = ruleChainOf(spec, r, ctx);
if (hard.violations.length === 0) {
const unknown = spec.rules ? unknownRuleFields(spec.rules.groups, r, ctx, catalogOf(spec.rules.atoms)) : [];
if (chain?.decidingGroup === undefined || chain.decidingGroup === null)
unknown.push(...unknownTierFields(spec, r, ctx));
if (unknown.length)
return {
decision: "unknown_field",
missing: [...new Set([...hard.missing, ...unknown])],
trace: unknown.map((f) => `组合规则需要补充:${f}`),
};
}
const { missing, accepted } = hard;
const withRules = chain ? { rules: chain } : {};
const violations = [...hard.violations, ...chainViolations(chain, spec)];
const ladder = effectiveLadder(spec, r, ctx, chain);
@@ -524,6 +571,53 @@ export function evaluateCandidate(
const up = spec.upgrade;
if (!up?.enabled) return reject({ field: "upgrade", code: "upgrade_disabled" });
if (up.mode === "full_pack" && (r.episodes || r.airDate)) return reject({ field: "upgrade", code: "not_full_pack" });
if (r.isEncode && up.encodeGroupAllowlist?.length) {
if (!r.releaseGroup) return { decision: "unknown_field", missing: ["group"], trace: ["合格压制组需要查证发布组"] };
const groups = r.releaseGroup ? r.releaseGroup.name.split("@").map(lc) : [];
if (!up.encodeGroupAllowlist.some((g) => groups.includes(lc(g))))
return reject({
field: "upgrade",
code: "encode_not_qualified",
expected: up.encodeGroupAllowlist,
actual: r.releaseGroup?.name,
});
}
if (up.consistency === "season_release") {
const key = seasonReleaseKey(r);
const targetKey = ctx.seasonReleaseTarget ? seasonReleaseKey(ctx.seasonReleaseTarget) : null;
if (key && !targetKey)
return { decision: "unknown_field", missing: ["current_asset"], trace: ["当季成套目标的规格未知"] };
if (!key)
return {
decision: "unknown_field",
missing: [
...(!r.resolution ? ["resolution" as const] : []),
...(!r.source ? ["source" as const] : []),
...(!r.videoCodec ? ["codec" as const] : []),
...(!r.releaseGroup?.name ? ["group" as const] : []),
],
trace: ["成套洗版需要完整版本规格"],
};
if (key !== targetKey) return reject({ field: "upgrade", code: "different_season_release" });
const fromPri = currentPri(spec, current);
if (key !== seasonReleaseKey(current) && ladder.pri === fromPri) {
const from = qualityTier(current, spec);
if (tier.source < from.source || tier.resolution < from.resolution)
return reject({ field: "upgrade", code: "lower_tier", expected: from, actual: tier });
return {
decision: "upgrade",
tier,
fromTier: from,
softRank: rank,
reasons: [...reasons, "season_release_aligned", ...priReasons],
pri: ladder.pri,
fromPri,
ladder,
trace: ["保持优先级,向当季同套版本对齐", ...ladder.trace],
...withRules,
};
}
}
const basis = up.basis ?? (tiered ? "priority" : "tier");
if (basis === "priority") {
const fromPri = currentPri(spec, current);
@@ -559,16 +653,6 @@ export function evaluateCandidate(
dSource === 0 && dRes === 0 ? "same_tier" : dSource < 0 || dRes < 0 ? "lower_tier" : "below_min_improvement";
return reject({ field: "upgrade", code, expected: { minTierSteps: minSteps, from }, actual: tier });
}
if (r.isEncode && up.encodeGroupAllowlist?.length) {
const groups = r.releaseGroup ? r.releaseGroup.name.split("@").map(lc) : [];
if (!up.encodeGroupAllowlist.some((g) => groups.includes(lc(g))))
return reject({
field: "upgrade",
code: "encode_not_qualified",
expected: up.encodeGroupAllowlist,
actual: r.releaseGroup?.name,
});
}
return {
decision: "upgrade",
tier,
+1
View File
@@ -59,6 +59,7 @@ export {
type SubscriptionGroupResult,
subscriptionGroup,
} from "./overview.ts";
export { seasonReleaseKey, selectSeasonRelease, upgradeSpec } from "./season-release.ts";
export {
canTransition,
type SlotEvent,
@@ -0,0 +1,49 @@
import type { ParsedRelease } from "../release-parser/types.ts";
import { compareRank, currentPri, qualityTier, softRank } from "./evaluate.ts";
import type { QualityFacts, SlotSpec } from "./types.ts";
/** 缺少关键规格时不能声称同套;集号、文件名、种子 ID 不属于版本身份。 */
export function seasonReleaseKey(q: QualityFacts): string | null {
if (!q.resolution || !q.source || !q.videoCodec || !q.releaseGroup?.name) return null;
return JSON.stringify([
q.resolution,
q.source,
q.videoCodec,
q.releaseGroup.name
.toLowerCase()
.split("@")
.map((g) => g.trim())
.sort(),
[...(q.hdr ?? [])].sort(),
q.cut ?? "default",
]);
}
export function upgradeSpec(spec: SlotSpec): SlotSpec {
return {
...spec,
...(spec.upgradeRules ? { rules: spec.upgradeRules } : {}),
...(spec.upgradeHard ? { hard: spec.upgradeHard } : {}),
};
}
/** 优先级、软偏好、规范化规格形成稳定顺序;同档对齐不允许质量降档。 */
export function selectSeasonRelease(spec: SlotSpec, versions: readonly QualityFacts[]): QualityFacts | undefined {
let target: QualityFacts | undefined;
const rank = (q: QualityFacts) => {
const tier = qualityTier(q, spec);
return [
currentPri(spec, q),
tier.source,
tier.resolution,
...softRank(spec, { raw: "", hdr: [], audio: [], ...q } as unknown as ParsedRelease),
];
};
for (const q of versions) {
const key = seasonReleaseKey(q);
if (!key) continue;
const compared = target ? compareRank(rank(q), rank(target)) : 1;
if (compared > 0 || (compared === 0 && key < seasonReleaseKey(target!)!)) target = q;
}
return target;
}
+9
View File
@@ -91,6 +91,10 @@ export interface SlotSoftPreferences {
export interface SlotUpgradePolicy {
enabled: boolean;
/** 洗版阶段显式规则组;未设时沿用既有洗版默认链。 */
ruleGroupIds?: string[];
/** 追更可混合,洗版按季向同组、同来源和编码的版本收敛。 */
consistency?: "season_release";
/** 最小改进:来源等级至少提升几级,默认 1(同等级不反复更换) */
minTierSteps?: number;
/** 来源等级表;默认 REMUX/原盘/蓝光二压=3,WEB-DL=2,WEBRip/HDTV/HDRip=1,DVD=0 */
@@ -128,6 +132,9 @@ export interface SlotSpec {
hard: SlotHardConditions;
/** 规则组:硬排除 → 拒绝;各组须命中某层(ladderRequired);pri 取最后一个带层的组(覆盖 soft.tiers 的 pri) */
rules?: SlotRuleChain;
/** 服务入口解析的洗版规则,不参与首次补齐。 */
upgradeRules?: SlotRuleChain;
upgradeHard?: SlotHardConditions;
unknownPolicy?: UnknownPolicy | Partial<Record<HardField, UnknownPolicy>>;
soft?: SlotSoftPreferences;
upgrade?: SlotUpgradePolicy;
@@ -151,6 +158,8 @@ export type QualityFacts = Pick<
/** 评估上下文:站点字段与折算口径(不在标题解析里的事实由调用方给) */
export interface EvaluationContext {
/** 季级成套目标,由同一收录版本的有效资产及在途计划确定。 */
seasonReleaseTarget?: QualityFacts;
siteId?: string;
/** 副标题/标签等参与关键词匹配的文本 */
subtitle?: string;
@@ -0,0 +1,356 @@
import { describe, expect, it } from "vitest";
import {
evaluateCandidate,
type ParsedRelease,
parseRelease,
parseRuleString,
type QualityFacts,
type RuleGroupSpec,
rankForSlot,
type SlotSpec,
seasonReleaseKey,
selectAcquisitionPolicy,
selectSeasonRelease,
} from "../src/index.ts";
const release = (title: string, facts: Partial<ParsedRelease> = {}): ParsedRelease => ({
...parseRelease(title),
...facts,
});
const group = (name: string, rule: string): RuleGroupSpec => ({ name, layers: parseRuleString(rule).layers });
const current: QualityFacts = release("Show.S01E01.1080p.WEB-DL.H265-OTHER");
const target = release("Show.S01E02.1080p.WEB-DL.H265-CHD");
const consistent: SlotSpec = {
name: "正片",
hard: { resolutions: ["1080p", "2160p"] },
rules: { groups: [group("允许收录", "1080P | 4K")] },
upgrade: { enabled: true, consistency: "season_release" },
};
describe("按分类选择默认收录策略", () => {
const policies = [
{ id: "global", scope: {} },
{ id: "series", scope: { mediaKinds: ["series" as const] } },
{ id: "animation", scope: { categories: { tv: ["动漫剧集"] } } },
];
it("订阅显式指定的策略覆盖类型和分类范围", () => {
expect(
selectAcquisitionPolicy(policies, { explicitId: "series", mediaKind: "movie", category: "动画电影" })?.id,
).toBe("series");
});
it("显式指定的策略不存在时不悄悄回退", () => {
expect(
selectAcquisitionPolicy(policies, { explicitId: "missing", mediaKind: "series", category: "动漫剧集" }),
).toBeUndefined();
});
it("分类匹配优先于类型默认,类型默认优先于全局默认", () => {
expect(selectAcquisitionPolicy(policies, { mediaKind: "series", category: "动漫剧集" })?.id).toBe("animation");
expect(selectAcquisitionPolicy(policies, { mediaKind: "series", category: "华语剧集" })?.id).toBe("series");
expect(selectAcquisitionPolicy(policies, { mediaKind: "movie", category: "动画电影" })?.id).toBe("global");
});
it.each(["movie", "series", "variety"] as const)("%s 不串用另一类型的同名分类策略", (mediaKind) => {
const scoped = [
{ id: "variety", scope: { mediaKinds: ["variety" as const], categories: { tv: ["同名分类"] } } },
{ id: "movie", scope: { categories: { movie: ["同名分类"] } } },
{ id: "series", scope: { mediaKinds: ["series" as const], categories: { tv: ["同名分类"] } } },
];
expect(selectAcquisitionPolicy(scoped, { mediaKind, category: "同名分类" })?.id).toBe(mediaKind);
});
it("同一范围按传入的最新发布顺序取第一个", () => {
const latest = { id: "latest", scope: { categories: { tv: ["动漫剧集"] } } };
const old = { id: "old", scope: { categories: { tv: ["动漫剧集"] } } };
expect(selectAcquisitionPolicy([latest, old], { mediaKind: "series", category: "动漫剧集" })).toBe(latest);
expect(selectAcquisitionPolicy([old, latest], { mediaKind: "series", category: "动漫剧集" })).toBe(old);
});
it("分类未知时跳过分类限定策略,仍可使用类型默认", () => {
expect(selectAcquisitionPolicy(policies, { mediaKind: "series", category: null })?.id).toBe("series");
});
it("类型或分类没有匹配且无默认时返回 undefined", () => {
const scoped = policies.slice(1);
expect(selectAcquisitionPolicy(scoped, { mediaKind: "movie", category: "动漫剧集" })).toBeUndefined();
expect(selectAcquisitionPolicy([policies[2]!], { mediaKind: "series", category: null })).toBeUndefined();
expect(selectAcquisitionPolicy([], { mediaKind: "variety", category: "综艺" })).toBeUndefined();
});
it("空的分类列表仍按类型默认处理", () => {
const typeDefault = { id: "type", scope: { mediaKinds: ["series" as const], categories: { movie: [], tv: [] } } };
expect(selectAcquisitionPolicy([policies[0]!, typeDefault], { mediaKind: "series", category: null })).toBe(
typeDefault,
);
});
});
describe("整季同套的规格与目标选择", () => {
it("发布组大小写、联合组顺序、HDR 顺序不产生新的版本身份", () => {
const first = {
...target,
releaseGroup: { name: "CHD@FLUX", source: "suffix" as const },
hdr: ["DV", "HDR10"] as const,
};
const second = {
...target,
releaseGroup: { name: "flux@chd", source: "known" as const },
hdr: ["HDR10", "DV"] as const,
};
expect(seasonReleaseKey({ ...first, hdr: [...first.hdr] })).toBe(
seasonReleaseKey({ ...second, hdr: [...second.hdr] }),
);
});
it.each(["resolution", "source", "videoCodec", "releaseGroup"] as const)(
"缺少 %s 时不声称属于完整成套版本",
(field) => {
expect(seasonReleaseKey({ ...target, [field]: undefined })).toBeNull();
},
);
it("来源、分辨率、编码和剪辑不同分别是不同的成套版本", () => {
for (const change of [
{ source: "BluRay" as const },
{ resolution: "2160p" as const },
{ videoCodec: "H.264" as const },
{ cut: "Extended" as const },
{ hdr: ["DV" as const] },
]) {
expect(seasonReleaseKey({ ...target, ...change })).not.toBe(seasonReleaseKey(target));
}
});
const unrestricted: SlotSpec = { name: "主版本", hard: {} };
it.each([
{
name: "来源等级优先于更高分辨率和软偏好",
spec: { ...unrestricted, soft: { groups: ["CHD"] } },
weaker: release("Show.S01E01.2160p.WEB-DL.H265-CHD"),
better: release("Show.S01E02.1080p.BluRay.H264-OTHER"),
},
{
name: "同来源时分辨率优先于软偏好",
spec: { ...unrestricted, soft: { groups: ["CHD"] } },
weaker: target,
better: release("Show.S01E02.2160p.WEB-DL.H265-OTHER"),
},
{
name: "同档规格按发布组软偏好排序",
spec: { ...unrestricted, soft: { groups: ["CHD", "OTHER"] } },
weaker: release("Show.S01E01.1080p.WEB-DL.H265-OTHER"),
better: target,
},
{
name: "同档规格按 HDR 软偏好排序",
spec: { ...unrestricted, soft: { hdr: ["DV" as const, "HDR10" as const] } },
weaker: release("Show.S01E01.1080p.WEB-DL.HDR.H265-CHD"),
better: release("Show.S01E02.1080p.WEB-DL.DV.H265-CHD"),
},
{
name: "同档规格按编码软偏好排序",
spec: { ...unrestricted, soft: { codecs: ["H.265" as const, "H.264" as const] } },
weaker: release("Show.S01E01.1080p.WEB-DL.H264-CHD"),
better: target,
},
])("$name,成套选择与候选排序一致", ({ spec, weaker, better }) => {
expect(rankForSlot(spec, [weaker, better])[0]?.candidate).toBe(better);
expect(selectSeasonRelease(spec, [weaker, better])).toBe(better);
expect(selectSeasonRelease(spec, [better, weaker])).toBe(better);
});
it("组合优先级先于来源和分辨率等级", () => {
const spec: SlotSpec = { ...unrestricted, rules: { groups: [group("来源偏好", "WEBDL > BLURAY")] } };
const blue = release("Show.S01E01.2160p.BluRay.H265-OTHER");
expect(rankForSlot(spec, [blue, target])[0]?.candidate).toBe(target);
expect(selectSeasonRelease(spec, [blue, target])).toBe(target);
});
it("规格不完整的版本不参与成套选择", () => {
const incomplete = { ...target, videoCodec: undefined, pri: 1000 };
expect(selectSeasonRelease(unrestricted, [incomplete, target])).toBe(target);
expect(selectSeasonRelease(unrestricted, [incomplete])).toBeUndefined();
});
it("同档没有偏好时目标不随数据库返回顺序反复变化", () => {
const other = release("Show.S01E02.1080p.WEB-DL.H265-OTHER");
expect(selectSeasonRelease(unrestricted, [target, other])).toBe(selectSeasonRelease(unrestricted, [other, target]));
});
});
describe("追更和洗版分别使用规则", () => {
const phased: SlotSpec = {
name: "主版本",
hard: { resolutions: ["1080p", "2160p"] },
rules: { groups: [group("追更", "WEBDL")] },
upgradeRules: { groups: [group("洗版", "BLURAY > WEBDL")] },
upgrade: { enabled: true },
};
const blue = release("Show.S01E02.1080p.BluRay.H265-CHD");
it("补齐缺集按追更规则,洗版规则不提前过滤", () => {
expect(evaluateCandidate(phased, target)).toMatchObject({ decision: "accept", pri: 100 });
expect(evaluateCandidate(phased, blue)).toMatchObject({
decision: "reject",
violations: [expect.objectContaining({ field: "rule", expected: "追更" })],
});
});
it("已有当前资产时改用洗版规则,并按洗版规则重算当前档位", () => {
expect(evaluateCandidate(phased, blue, current)).toMatchObject({ decision: "upgrade", pri: 100, fromPri: 99 });
expect(evaluateCandidate(phased, target, current)).toMatchObject({
decision: "reject",
violations: [expect.objectContaining({ code: "not_higher_priority" })],
});
});
it("洗版专用硬条件不阻止追更,但在已有资产时生效", () => {
const spec: SlotSpec = { ...phased, upgradeHard: { resolutions: ["2160p"] } };
expect(evaluateCandidate(spec, target).decision).toBe("accept");
expect(evaluateCandidate(spec, blue, current)).toMatchObject({
decision: "reject",
violations: [expect.objectContaining({ field: "resolution", code: "not_allowed" })],
});
});
it.each(["1080p", "2160p"] as const)("允许分辨率 %s 在追更和洗版阶段均可使用", (resolution) => {
const spec: SlotSpec = { name: "主版本", hard: phased.hard, upgrade: { enabled: true } };
const candidate = release(`Show.S01E02.${resolution}.BluRay.H265-CHD`);
expect(evaluateCandidate(spec, candidate).decision).toBe("accept");
expect(evaluateCandidate(spec, candidate, current).decision).toBe("upgrade");
});
it.each(["720p", "4320p"] as const)("范围外的分辨率 %s 不因更优来源或整季对齐而放行", (resolution) => {
const candidate = release(`Show.S01E02.${resolution}.BluRay.H265-CHD`);
expect(evaluateCandidate(consistent, candidate).decision).toBe("reject");
expect(evaluateCandidate(consistent, candidate, current, { seasonReleaseTarget: candidate })).toMatchObject({
decision: "reject",
violations: expect.arrayContaining([expect.objectContaining({ field: "resolution", code: "not_allowed" })]),
});
});
it("分辨率未知时保持待补充,不把未知当作允许范围", () => {
expect(evaluateCandidate(phased, { ...target, resolution: undefined })).toMatchObject({
decision: "unknown_field",
missing: ["resolution"],
});
});
it("组合条件的编码未知时,不能因 NOT 自动命中低档", () => {
const spec: SlotSpec = { name: "主版本", hard: {}, rules: { groups: [group("编码", "H265 > !H265")] } };
expect(evaluateCandidate(spec, release("Show.S01E02.1080p.WEB-DL-CHD"))).toMatchObject({
decision: "unknown_field",
missing: ["codec"],
});
});
it("高档分辨率已明确不满足时,无关的未知编码不阻止已匹配的低档", () => {
const spec: SlotSpec = {
name: "主版本",
hard: {},
rules: { groups: [group("组合档位", "4K & H265 > 1080P")] },
};
expect(evaluateCandidate(spec, release("Show.S01E02.1080p.WEB-DL-CHD"))).toMatchObject({
decision: "accept",
pri: 99,
});
});
});
describe("整季同套的同档对齐不绕过升级限制", () => {
it("同档的不同发布组可以逐集对齐到选定的整季版本", () => {
expect(evaluateCandidate(consistent, target, current, { seasonReleaseTarget: target })).toMatchObject({
decision: "upgrade",
pri: 100,
fromPri: 100,
reasons: expect.arrayContaining(["season_release_aligned"]),
});
});
it.each([
{ name: "来源", source: "BluRay" as const, resolution: "1080p" as const },
{ name: "分辨率", source: "WEB-DL" as const, resolution: "2160p" as const },
])("同档对齐不能降低当前文件的$name", ({ source, resolution }) => {
expect(
evaluateCandidate(consistent, target, { ...current, source, resolution }, { seasonReleaseTarget: target }),
).toMatchObject({ decision: "reject", violations: [expect.objectContaining({ code: "lower_tier" })] });
});
it("当前文件已经同套时不重复整理或下载", () => {
expect(evaluateCandidate(consistent, target, target, { seasonReleaseTarget: target }).decision).toBe("reject");
const normalizedCurrent = { ...target, releaseGroup: { name: "chd", source: "known" as const } };
expect(evaluateCandidate(consistent, target, normalizedCurrent, { seasonReleaseTarget: target }).decision).toBe(
"reject",
);
});
it("另一套同档资源不能替换已选定的当季目标", () => {
const other = release("Show.S01E02.1080p.WEB-DL.H265-FLUX");
expect(evaluateCandidate(consistent, other, current, { seasonReleaseTarget: target })).toMatchObject({
decision: "reject",
violations: [expect.objectContaining({ code: "different_season_release" })],
});
});
it("关闭升级时,同档对齐也不能发起替换", () => {
const spec = { ...consistent, upgrade: { ...consistent.upgrade!, enabled: false } };
expect(evaluateCandidate(spec, target, current, { seasonReleaseTarget: target })).toMatchObject({
decision: "reject",
violations: [expect.objectContaining({ code: "upgrade_disabled" })],
});
});
it("全集洗版不能因同档对齐而接受单集或播出日期资源", () => {
const spec: SlotSpec = { ...consistent, upgrade: { ...consistent.upgrade!, mode: "full_pack" } };
for (const candidate of [target, { ...target, episodes: undefined, airDate: "20261013" }]) {
expect(evaluateCandidate(spec, candidate, current, { seasonReleaseTarget: candidate })).toMatchObject({
decision: "reject",
violations: [expect.objectContaining({ code: "not_full_pack" })],
});
}
const pack = release("Show.S01.1080p.WEB-DL.H265-CHD");
expect(pack.episodes).toBeUndefined();
expect(evaluateCandidate(spec, pack, current, { seasonReleaseTarget: pack }).decision).toBe("upgrade");
});
it("不合格蓝光压制组不能因同档对齐绕过名单", () => {
const spec: SlotSpec = {
...consistent,
upgrade: { ...consistent.upgrade!, encodeGroupAllowlist: ["CHD"] },
};
const old = release("Show.S01E01.1080p.BluRay.x265-OTHER");
const unqualified = release("Show.S01E02.1080p.BluRay.x265-BADGRP");
expect(unqualified.isEncode).toBe(true);
expect(evaluateCandidate(spec, unqualified, old, { seasonReleaseTarget: unqualified })).toMatchObject({
decision: "reject",
violations: [expect.objectContaining({ code: "encode_not_qualified" })],
});
const qualified = release("Show.S01E02.1080p.BluRay.x265-chd");
expect(evaluateCandidate(spec, qualified, old, { seasonReleaseTarget: qualified }).decision).toBe("upgrade");
});
it.each([
{ field: "resolution", missing: "resolution" },
{ field: "source", missing: "source" },
{ field: "videoCodec", missing: "codec" },
{ field: "releaseGroup", missing: "group" },
] as const)("候选缺少 $field 时不执行成套替换", ({ field, missing }) => {
const candidate = { ...target, [field]: undefined };
expect(evaluateCandidate(consistent, candidate, current, { seasonReleaseTarget: target })).toMatchObject({
decision: "unknown_field",
missing: expect.arrayContaining([missing]),
});
});
it("季目标缺失或规格不完整时等待,不自行认定当前候选是目标", () => {
expect(evaluateCandidate(consistent, target, current)).toMatchObject({
decision: "unknown_field",
missing: ["current_asset"],
});
expect(
evaluateCandidate(consistent, target, current, { seasonReleaseTarget: { ...target, videoCodec: undefined } }),
).toMatchObject({ decision: "unknown_field", missing: ["current_asset"] });
});
});
@@ -194,7 +194,7 @@ describe("MP 过滤规则差分(filter-cases.json)", () => {
});
});
/** 生成/更新登记时用:UMT_FILTER_DUMP=1 deno task test 打印当前不一致 */
/** 生成/更新登记时用:UMT_FILTER_DUMP=1 bun run test 打印当前不一致 */
if (process.env.UMT_FILTER_DUMP) {
const rows: Record<string, unknown> = {};
for (const c of fx.cases) {
+4 -2
View File
@@ -1,6 +1,6 @@
# @umt/mapping
domain ↔ contracts 的集中映射(W1 审查 B5)。只依赖 `@umt/domain` 与 `@umt/contracts`,不依赖 db / adapters。server 各模块需要在两套取值之间转换时一律走这里,不要在模块里各写一份。
domain ↔ contracts 的语义映射。只依赖 `@umt/domain` 与 `@umt/contracts`,不依赖 db / adapters。server 各模块需要在两套取值之间转换时一律走这里,不要在模块里各写一份。
db 行(snake_case)↔ contracts(camelCase)的转换不在本包;adapters 的 `SiteRelease` 以结构类型 `SiteReleaseLike` 接入(兼容性由 `apps/server/test/mapping-compat.test.ts` 在类型层保证)。
@@ -10,12 +10,14 @@ db 行(snake_case)↔ contracts(camelCase)的转换不在本包;adapters 的 `Si
| `evidence.ts` | `toContractEvidenceLevel` / `toDomainConfidence` / `toDomainMediaType` / `toContractMediaKind` / `toDomainEntityType` / `toContractEntityType` | `needs_confirm ↔ needs_review`,`manual → rule_confirmed`,`rejected → null`;`tv ↔ series/variety` |
| `identity.ts` | `toContractResolveStatus` / `toReleaseResolutionStatus` / `rankedToIdentityCreate` / `localIdentityCandidate` | `auto/ambiguous/no_match → resolved|proposed/ambiguous/unidentifiable → source_release.resolution_status`;排名候选 → `ExternalIdentityCreate`;本地作品 → 本地优先候选 |
| `slot.ts` | `toContractSlotStatus` / `toDomainSlotStatus` / `slotSpecFromRow` / `slotSpecToColumns` / `toContractSlotEvaluation` | Slot 状态;规则 JSON 列 ↔ `SlotSpec`;评估结果 → `{ decision, acceptable, rejected[] }` |
| `release.ts` | `toParsedColumns` / `fromParsedColumns` / `toContractContentType` / `toDomainContentType` / `toContractSegment` / `toDomainSegment` | `ParsedRelease ↔ source_release.parsed + parser_version`(版本不符返回 null,重解析) |
| `release.ts` | `toParsedColumns` / `fromParsedColumns` / `toContractSegment` / `toDomainSegment` | `ParsedRelease ↔ source_release.parsed + parser_version`(版本不符返回 null,重解析) |
| `site-release.ts` | `siteReleaseToSourceRelease` / `sourceReleaseToSiteRelease` / `SiteReleaseLike` | 站点资源 ↔ `SourceReleaseCreate` / `SourceRelease`;downloadUrl 不入库 |
| `path-binding.ts` | `toDomainPathBinding` / `toPathBindingCreate` / `parseTemplateVersion` | `entityId/scope/season/templateVersion:number/active ↔ mediaEntityId/bindingKind/seasonNumber/templateVersion:string/status` |
| `organize.ts` | `organizePlanToRecord` / `organizePlanToFileResults` / `organizePlanToCapabilityOutput` / `organizationStatusFromFiles` / `toOrganizationAction` / `toOrganizeOperation` / `toFileRole` | `OrganizePlan →` organization_record / organization_file_result 草稿 / organize.plan 输出 |
## 规则
- 同一语义的类型直接共用,不维护同值枚举的双向转换表。只在取值或含义存在实际差异时添加映射。
- 映射表一律写成 `Record<一侧的联合类型, …>`:domain 增删取值时 typecheck 失败;contracts 侧在 `test/enums.test.ts` 按 `z.enum(...).options` 逐个断言,新增取值时测试失败。
- 新增映射先写测试(往返、schema 合法)。输出给 contracts 的对象都用对应 zod schema 在测试里 `safeParse` 一遍。
-9
View File
@@ -1,9 +0,0 @@
{
"name": "@umt/mapping",
"version": "0.3.44",
"exports": { ".": "./src/index.ts" },
"tasks": {
"test": "deno run -A npm:vitest run",
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
}
}
+9
View File
@@ -4,5 +4,14 @@
"type": "module",
"exports": {
".": "./src/index.ts"
},
"version": "0.3.44",
"scripts": {
"test": "bun x --bun vitest run",
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
},
"dependencies": {
"@umt/contracts": "workspace:*",
"@umt/domain": "workspace:*"
}
}
+2 -30
View File
@@ -1,10 +1,5 @@
import type { ContentType as ContractContentType, Segment as ContractSegment, JsonObject } from "@umt/contracts";
import {
type ContentType as DomainContentType,
type Segment as DomainSegment,
PARSER_VERSION,
type ParsedRelease,
} from "@umt/domain";
import type { Segment as ContractSegment, JsonObject } from "@umt/contracts";
import { type Segment as DomainSegment, PARSER_VERSION, type ParsedRelease } from "@umt/domain";
// 标题解析结果 ↔ source_release.parsed / parser_version 列;综艺内容类型与分段 ↔ content_unit 列。
@@ -28,29 +23,6 @@ export function fromParsedColumns(cols: { parsed: JsonObject; parserVersion: str
return p as unknown as ParsedRelease;
}
const CONTENT_TYPE: Record<DomainContentType, ContractContentType> = {
unknown: "unknown",
main: "main",
extra: "extra",
behind_the_scenes: "behind_the_scenes",
performance_only: "performance_only",
preview: "preview",
reaction: "reaction",
};
const CONTENT_TYPE_BACK: Record<ContractContentType, DomainContentType> = {
main: "main",
extra: "extra",
behind_the_scenes: "behind_the_scenes",
performance_only: "performance_only",
preview: "preview",
reaction: "reaction",
unknown: "unknown",
};
export const DOMAIN_CONTENT_TYPES = Object.keys(CONTENT_TYPE) as DomainContentType[];
export const toContractContentType = (t: DomainContentType): ContractContentType => CONTENT_TYPE[t];
export const toDomainContentType = (t: ContractContentType): DomainContentType => CONTENT_TYPE_BACK[t];
const SEGMENT: Record<DomainSegment, ContractSegment> = {
full: "full",
upper: "upper",
+2 -4
View File
@@ -117,15 +117,13 @@ describe("身份解析状态", () => {
});
});
describe("Slot / 内容类型 / 分段 / 整理动作", () => {
describe("Slot / 分段 / 整理动作", () => {
it("Slot 状态两侧一一对应", () => {
expect(sorted(M.DOMAIN_SLOT_STATUSES)).toEqual(sorted(C.SlotStatus.options));
for (const s of C.SlotStatus.options) expect(M.toContractSlotStatus(M.toDomainSlotStatus(s))).toBe(s);
});
it("综艺内容类型两侧一一对应;分段 unknown ↔ 缺失", () => {
expect(sorted(M.DOMAIN_CONTENT_TYPES)).toEqual(sorted(C.ContentType.options));
for (const t of C.ContentType.options) expect(M.toContractContentType(M.toDomainContentType(t))).toBe(t);
it("分段 unknown ↔ 缺失", () => {
expect(sorted([...M.DOMAIN_SEGMENTS, "unknown"])).toEqual(sorted(C.Segment.options));
for (const s of C.Segment.options) expect(M.toContractSegment(M.toDomainSegment(s))).toBe(s);
});
+15
View File
@@ -0,0 +1,15 @@
{
"name": "@umt/runtime",
"version": "0.3.44",
"private": true,
"type": "module",
"exports": {
".": "./src/index.ts",
"./worker": "./src/worker.ts"
},
"scripts": {
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json",
"test": "bun x --bun vitest run"
},
"dependencies": {}
}
+212
View File
@@ -0,0 +1,212 @@
import { spawn } from "node:child_process";
import { readFileSync, writeSync } from "node:fs";
import { mkdir, open, opendir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
import { hostname } from "node:os";
import { Readable, Writable } from "node:stream";
export const runtimeHostname = hostname;
export const runtimeReadTextFile = (path: string) => readFile(path, "utf8");
export const runtimeReadTextFileSync = (path: string) => readFileSync(path, "utf8");
export const runtimeWriteTextFile = (path: string, text: string) => writeFile(path, text);
export const runtimeMkdir = mkdir;
export const runtimeRename = rename;
export const runtimeRemove = rm;
export const runtimeReadDir = opendir;
export async function runtimeStat(path: string) {
const s = await stat(path);
return { size: s.size, isFile: s.isFile(), isDirectory: s.isDirectory() };
}
export async function runtimeOpen(
path: string,
opts: { read?: boolean; write?: boolean; create?: boolean; truncate?: boolean },
) {
const f = await open(path, opts.write ? (opts.truncate ? "w" : "a") : "r");
return {
stat: async () => {
const s = await f.stat();
return { size: s.size, isFile: s.isFile(), isDirectory: s.isDirectory() };
},
close: () => f.close(),
write: async (bytes: Uint8Array) => (await f.write(bytes)).bytesWritten,
get readable() {
return Readable.toWeb(f.createReadStream({ autoClose: true })) as unknown as ReadableStream<Uint8Array>;
},
};
}
export const runtimeWriteSync = (bytes: Uint8Array) => writeSync(1, bytes);
export const runtimeWrite = async (bytes: Uint8Array) => runtimeWriteSync(bytes);
export const runtimeUnrefTimer = (timer: unknown) => (timer as { unref?: () => void }).unref?.();
type CommandOptions = {
args?: string[];
stdin?: "piped" | "null" | "inherit";
stdout?: "piped" | "null" | "inherit";
stderr?: "piped" | "null" | "inherit";
env?: Record<string, string>;
clearEnv?: boolean;
signal?: AbortSignal;
};
export type ChildProcess = ReturnType<Command["spawn"]>;
export class Command {
constructor(
private cmd: string,
private opts: CommandOptions = {},
) {}
spawn() {
const o = this.opts;
const io = (s: string | undefined) => (s === "piped" ? "pipe" : s === "null" ? "ignore" : "inherit");
const p = spawn(this.cmd, o.args ?? [], {
stdio: [io(o.stdin), io(o.stdout ?? "piped"), io(o.stderr ?? "piped")],
env: o.clearEnv ? o.env : { ...process.env, ...o.env },
signal: o.signal,
});
const status = new Promise<{ success: boolean; code: number; signal: string | null }>((resolve, reject) => {
p.once("error", reject);
p.once("close", (code, signal) => resolve({ success: code === 0, code: code ?? 1, signal }));
});
// Attach a handler immediately: callers may still be writing stdin when spawn fails.
status.catch(() => {});
const child = {
pid: p.pid!,
stdin: p.stdin ? Writable.toWeb(p.stdin) : new WritableStream<Uint8Array>(),
stdout: p.stdout
? (Readable.toWeb(p.stdout) as unknown as ReadableStream<Uint8Array>)
: new ReadableStream<Uint8Array>({
start(c) {
c.close();
},
}),
stderr: p.stderr
? (Readable.toWeb(p.stderr) as unknown as ReadableStream<Uint8Array>)
: new ReadableStream<Uint8Array>({
start(c) {
c.close();
},
}),
status,
kill: (signal: string = "SIGTERM") => {
p.kill(signal as NodeJS.Signals);
},
unref: () => {
p.unref();
},
output: async () => {
const [s, stdout, stderr] = await Promise.all([
status,
new Response(child.stdout).bytes(),
new Response(child.stderr).bytes(),
]);
return { ...s, stdout, stderr };
},
};
return child;
}
output() {
return this.spawn().output();
}
}
export interface NetAddr {
hostname: string;
port: number;
}
export interface HttpServer {
addr: NetAddr;
finished: Promise<void>;
shutdown(): Promise<void>;
}
type SocketData = { client: WebSocket };
const upgrades = new WeakMap<Request, Bun.Server<SocketData>>();
export function serve(
opts: { port: number; hostname?: string; signal?: AbortSignal; onListen?: (addr: NetAddr) => void },
fetcher: (req: Request) => Response | Promise<Response>,
): HttpServer {
let finish!: () => void;
const finished = new Promise<void>((resolve) => {
finish = resolve;
});
const server = Bun.serve<SocketData>({
port: opts.port,
hostname: opts.hostname,
async fetch(req, server) {
upgrades.set(req, server);
const response = await fetcher(req);
upgrades.delete(req);
return upgradedRequests.has(req) ? undefined : response;
},
websocket: {
open(ws) {
const c = ws.data.client;
socketBindings.get(c)?.(ws);
socketBindings.delete(c);
Object.defineProperty(c, "readyState", { value: WebSocket.OPEN, configurable: true });
c.dispatchEvent(new Event("open"));
},
message(ws, data) {
ws.data.client.dispatchEvent(
new MessageEvent("message", {
data:
typeof data === "string" ? data : data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength),
}),
);
},
close(ws, code, reason) {
const c = ws.data.client;
Object.defineProperty(c, "readyState", { value: WebSocket.CLOSED, configurable: true });
c.dispatchEvent(new CloseEvent("close", { code, reason }));
},
},
});
const addr = { hostname: opts.hostname ?? "0.0.0.0", port: server.port! };
opts.onListen?.(addr);
const shutdown = async () => {
await server.stop(true);
finish();
};
opts.signal?.addEventListener(
"abort",
() => {
void shutdown();
},
{ once: true },
);
if (opts.signal?.aborted) void shutdown();
return { addr, finished, shutdown };
}
const upgradedRequests = new WeakSet<Request>();
export function upgradeWebSocket(req: Request, opts: { protocol?: string } = {}) {
const server = upgrades.get(req);
if (!server) throw new Error("WebSocket upgrade requires a Bun HTTP server");
// EventTarget facade preserves the bridge's browser WebSocket callbacks.
const client = new EventTarget() as WebSocket;
let ws: Bun.ServerWebSocket<SocketData> | undefined;
Object.defineProperty(client, "readyState", { value: WebSocket.CONNECTING, configurable: true });
for (const event of ["open", "message", "close", "error"]) {
let callback: ((e: Event) => void) | null = null;
Object.defineProperty(client, `on${event}`, {
get: () => callback,
set: (fn) => {
if (callback) client.removeEventListener(event, callback);
callback = fn;
if (fn) client.addEventListener(event, fn);
},
});
}
client.send = (data) => {
ws?.send(data as string | ArrayBuffer);
};
client.close = (code, reason) => {
ws?.close(code, reason);
};
const data = { client };
// Capture the native socket in open before dispatching the bridge callbacks.
socketBindings.set(client, (socket) => {
ws = socket;
});
if (!server.upgrade(req, { data, headers: opts.protocol ? { "Sec-WebSocket-Protocol": opts.protocol } : undefined }))
throw new Error("WebSocket upgrade failed");
const response = new Response(null);
upgradedRequests.add(req);
return { socket: client, response };
}
const socketBindings = new WeakMap<WebSocket, (ws: Bun.ServerWebSocket<SocketData>) => void>();
+262
View File
@@ -0,0 +1,262 @@
import { Readable } from "node:stream";
import { type ChildProcess, Command, runtimeWriteSync } from "./index.ts";
/** 本地执行协议。业务类型、重试、权限与超时策略由调用方负责。 */
export interface WorkerRequest {
id: string;
task: string;
input: unknown;
}
export interface WorkerError {
code: string;
message: string;
detail?: Record<string, unknown>;
}
export type WorkerReply =
| { id: string; type: "result"; result: unknown }
| { id: string; type: "error"; error: WorkerError };
export type WorkerMessage = WorkerReply | { id: string; type: "progress"; value: unknown };
export class WorkerTransportError extends Error {
constructor(
readonly kind: "spawn" | "protocol" | "exit" | "size",
message: string,
override readonly cause?: unknown,
) {
super(message);
this.name = "WorkerTransportError";
}
}
const encoder = new TextEncoder();
export function encodeWorkerMessage(value: WorkerRequest | WorkerMessage, maxBytes: number): Uint8Array {
const bytes = encoder.encode(`${JSON.stringify(value)}\n`);
if (bytes.byteLength > maxBytes) throw new WorkerTransportError("size", `worker 消息超过 ${maxBytes} 字节`);
return bytes;
}
/** 按字节限制一行的大小;UTF-8 与换行可以跨管道分块。EOF 不能代替消息结束符。 */
export async function* workerLines(
reader: ReadableStreamDefaultReader<Uint8Array>,
maxBytes: number,
): AsyncGenerator<string> {
const decoder = new TextDecoder("utf-8", { fatal: true });
let parts: string[] = [];
let length = 0;
for (;;) {
const { done, value } = await reader.read();
if (done) {
if (length) throw new WorkerTransportError("protocol", "worker 消息未以换行结束");
return;
}
let start = 0;
for (let i = 0; i < value.length; i++) {
if (value[i] !== 10) continue;
length += i - start + 1;
if (length > maxBytes) throw new WorkerTransportError("size", `worker 消息超过 ${maxBytes} 字节`);
parts.push(decoder.decode(value.subarray(start, i), { stream: true }), decoder.decode());
const line = parts.join("");
parts = [];
length = 0;
start = i + 1;
if (!line) throw new WorkerTransportError("protocol", "worker 返回空消息");
yield line;
}
length += value.length - start;
if (length > maxBytes) throw new WorkerTransportError("size", `worker 消息超过 ${maxBytes} 字节`);
parts.push(decoder.decode(value.subarray(start), { stream: true }));
}
}
export async function readWorkerRequestText(maxBytes: number): Promise<string> {
const stream = Readable.toWeb(process.stdin) as unknown as ReadableStream<Uint8Array>;
const reader = stream.getReader();
try {
let text: string | undefined;
for await (const line of workerLines(reader, maxBytes)) {
if (text !== undefined) throw new WorkerTransportError("protocol", "一次性 worker 只能接收一个请求");
text = line;
}
if (text === undefined) throw new WorkerTransportError("protocol", "worker 没有收到请求");
return text;
} finally {
reader.releaseLock();
}
}
export function decodeWorkerRequest(text: string): WorkerRequest {
const req = JSON.parse(text) as WorkerRequest;
if (!req || typeof req.id !== "string" || !req.id || typeof req.task !== "string" || !("input" in req))
throw new WorkerTransportError("protocol", "worker 请求格式无效");
return req;
}
export function writeWorkerMessage(message: WorkerMessage, maxBytes: number): void {
const bytes = encodeWorkerMessage(message, maxBytes);
let offset = 0;
while (offset < bytes.length) offset += runtimeWriteSync(bytes.subarray(offset));
}
function decodeMessage(line: string, id: string): WorkerMessage {
const value = JSON.parse(line) as WorkerMessage;
if (!value || value.id !== id) throw new WorkerTransportError("protocol", "worker 响应不属于当前请求");
if (value.type === "result" && "result" in value) return value;
if (value.type === "progress" && "value" in value) return value;
if (value.type === "error" && typeof value.error?.code === "string" && typeof value.error?.message === "string")
return value;
throw new WorkerTransportError("protocol", "worker 响应格式无效");
}
/** 一个一次性进程连接。只管通信与回收,不重试、不决定超时、不安装权限策略。 */
export class WorkerConnection {
readonly pid: number;
private readonly child: ChildProcess;
private readonly stdout: ReadableStreamDefaultReader<Uint8Array>;
private readonly stderrReader: ReadableStreamDefaultReader<Uint8Array>;
private reject: ((error: unknown) => void) | null = null;
private terminating: unknown;
private killTimer: ReturnType<typeof setTimeout> | undefined;
private settled = false;
private started = false;
private stderr = "";
constructor(command: { cmd: string; args: string[] }, env?: { values: Record<string, string>; clear?: boolean }) {
try {
this.child = new Command(command.cmd, {
args: command.args,
stdin: "piped",
stdout: "piped",
stderr: "piped",
env: env?.values,
clearEnv: env?.clear,
}).spawn();
} catch (error) {
throw new WorkerTransportError("spawn", "worker 子进程无法启动", error);
}
this.pid = this.child.pid;
this.stdout = this.child.stdout.getReader();
this.stderrReader = this.child.stderr.getReader();
}
/** graceMs > 0 时先 SIGTERM 供文件操作清理;解析可直接 SIGKILL。等待始终有界。 */
terminate(error: unknown, graceMs = 0, onForceKill?: () => void): void {
if (this.settled || this.terminating !== undefined) return;
this.terminating = error;
const force = () => {
try {
this.child.kill("SIGKILL");
onForceKill?.();
} catch {
// 已退出
}
this.child.unref();
this.fail(error);
};
if (graceMs === 0) {
force();
return;
}
try {
this.child.kill("SIGTERM");
} catch {
// 已退出
}
this.killTimer = setTimeout(force, graceMs);
}
request(
req: WorkerRequest,
options: { maxRequestBytes: number; maxResponseBytes: number; onProgress?: (value: unknown) => void },
): Promise<WorkerReply> {
if (this.started) throw new Error("一次性 worker 连接不能重复请求");
this.started = true;
if (this.terminating !== undefined) return Promise.reject(this.terminating);
if (!this.pid)
return this.child.status.then(
() => {
this.cleanup();
throw new WorkerTransportError("exit", "worker 未启动");
},
(error) => {
this.cleanup();
throw new WorkerTransportError("spawn", "worker 子进程无法启动", error);
},
);
return new Promise((resolve, reject) => {
this.reject = reject;
const errPump = this.readStderr();
let final: WorkerReply | undefined;
const pump = (async () => {
for await (const line of workerLines(this.stdout, options.maxResponseBytes)) {
const msg = decodeMessage(line, req.id);
if (final) throw new WorkerTransportError("protocol", "worker 结果之后仍发送消息");
if (msg.type === "progress") options.onProgress?.(msg.value);
else final = msg;
}
})();
pump.catch((error) => this.terminate(this.transportError(error)));
this.child.status.then(
async (status) => {
try {
await Promise.all([pump, errPump]);
if (this.settled) return;
if (this.terminating !== undefined) return this.fail(this.terminating);
if (!status.success || !final)
throw new WorkerTransportError(
"exit",
`worker 异常退出(code=${status.code} signal=${status.signal}):${this.stderr}`,
);
this.cleanup();
resolve(final);
} catch (error) {
this.fail(this.terminating ?? this.transportError(error));
}
},
(error) => this.fail(new WorkerTransportError("spawn", "worker 子进程无法启动", error)),
);
(async () => {
const bytes = encodeWorkerMessage(req, options.maxRequestBytes);
const writer = this.child.stdin.getWriter();
try {
await writer.write(bytes);
await writer.close();
} finally {
writer.releaseLock();
}
})().catch((error) => this.terminate(this.transportError(error)));
if (this.terminating !== undefined) this.fail(this.terminating);
});
}
private transportError(error: unknown): WorkerTransportError {
return error instanceof WorkerTransportError
? error
: new WorkerTransportError("protocol", `worker 通信失败:${(error as Error).message}`, error);
}
private async readStderr(): Promise<void> {
const decoder = new TextDecoder();
try {
for (;;) {
const { done, value } = await this.stderrReader.read();
if (done) return;
this.stderr = (this.stderr + decoder.decode(value, { stream: true })).slice(-2000);
}
} catch {
// 终止时管道可能断开
}
}
private fail(error: unknown): void {
if (this.settled) return;
this.cleanup();
this.reject?.(error);
}
private cleanup(): void {
this.settled = true;
clearTimeout(this.killTimer);
void this.stdout.cancel().catch(() => {});
void this.stderrReader.cancel().catch(() => {});
}
}
+37
View File
@@ -0,0 +1,37 @@
import { decodeWorkerRequest, readWorkerRequestText, writeWorkerMessage } from "../../src/worker.ts";
const req = decodeWorkerRequest(await readWorkerRequestText(4096));
const send = (type: "result", result: unknown) => writeWorkerMessage({ id: req.id, type, result }, 4096);
switch (req.task) {
case "normal":
writeWorkerMessage({ id: req.id, type: "progress", value: { bytes: 2 } }, 4096);
send("result", req.input);
break;
case "large":
writeWorkerMessage({ id: req.id, type: "result", result: "中文".repeat(200_000) }, 2 * 1024 * 1024);
break;
case "wrong-id":
writeWorkerMessage({ id: "other", type: "result", result: true }, 4096);
break;
case "truncated":
process.stdout.write('{"id":"1"');
break;
case "oversized":
process.stdout.write("x".repeat(8192));
break;
case "after-result":
send("result", true);
send("result", false);
break;
case "failed-exit":
send("result", true);
process.exitCode = 1;
break;
case "hang":
setInterval(() => {}, 1000);
break;
case "stderr":
process.stderr.write("x".repeat(65536));
send("result", true);
break;
}
+82
View File
@@ -0,0 +1,82 @@
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { WorkerConnection, workerLines } from "../src/worker.ts";
const fixture = fileURLToPath(new URL("./fixtures/worker.ts", import.meta.url));
const limits = { maxRequestBytes: 4096, maxResponseBytes: 4096 };
function connection() {
return new WorkerConnection({ cmd: process.execPath, args: [fixture] });
}
const request = (task: string) => ({ id: "1", task, input: "中文" });
describe("worker 管道与回收", () => {
it("拆开的 UTF-8 和多行消息仍能正确分帧", async () => {
const bytes = new TextEncoder().encode("中文\n第二行\n");
const stream = new ReadableStream<Uint8Array>({
start(c) {
for (const b of bytes) c.enqueue(Uint8Array.of(b));
c.close();
},
});
const lines = [];
for await (const line of workerLines(stream.getReader(), 64)) lines.push(line);
expect(lines).toEqual(["中文", "第二行"]);
});
it("转发进度、返回结果,并持续排空大量 stderr", async () => {
const seen: unknown[] = [];
expect(await connection().request(request("normal"), { ...limits, onProgress: (p) => seen.push(p) })).toEqual({
id: "1",
type: "result",
result: "中文",
});
expect(seen).toEqual([{ bytes: 2 }]);
expect(await connection().request(request("stderr"), limits)).toMatchObject({ result: true });
});
it("大结果跨多个管道分块也完整返回", async () => {
const reply = await connection().request(request("large"), { ...limits, maxResponseBytes: 2 * 1024 * 1024 });
expect(reply).toEqual({ id: "1", type: "result", result: "中文".repeat(200_000) });
});
it.each([
["wrong-id", "protocol"],
["truncated", "protocol"],
["oversized", "size"],
["after-result", "protocol"],
["failed-exit", "exit"],
])("拒绝 %s,而不接受部分或错误响应", async (task, kind) => {
await expect(connection().request(request(task), limits)).rejects.toMatchObject({ kind });
});
it("请求大小超限也回收已启动的进程", async () => {
const c = connection();
await expect(c.request({ ...request("hang"), input: "x".repeat(8192) }, limits)).rejects.toMatchObject({
kind: "size",
});
});
it("启动失败保留 spawn 分类", async () => {
const c = new WorkerConnection({ cmd: "/nonexistent/umt-worker", args: [] });
await expect(c.request(request("normal"), limits)).rejects.toMatchObject({ kind: "spawn" });
});
it("执行中的请求可终止,返回调用方的原因", async () => {
const c = connection();
const pending = c.request(request("hang"), limits);
const reason = new Error("调用方取消");
c.terminate(reason, 50);
await expect(pending).rejects.toBe(reason);
});
it("进度消费者抛错时回收连接,不让任务继续等待", async () => {
await expect(
connection().request(request("normal"), {
...limits,
onProgress: () => {
throw new Error("进度无效");
},
}),
).rejects.toMatchObject({ kind: "protocol", message: expect.stringContaining("进度无效") });
});
});
+1
View File
@@ -0,0 +1 @@
{ "extends": "../../config/tsconfig.base.json", "include": ["src", "test"] }
+4 -4
View File
@@ -13,7 +13,7 @@
| 导出 | 作用 |
| --- | --- |
| `OpenWorkflowEngine.connect({ databaseUrl, db, registry })` | `TaskEngine` 实现:`enqueue / signal / cancel / get / list / startWorker / stop` |
| `defineTask` / `TaskRegistry` / `createDefaultRegistry()` | 任务定义与注册表;默认注册表里有 §3.1-8 的 9 个任务(目前是空壳) |
| `defineTask` / `TaskRegistry` | 任务定义与注册表;实际业务 handler 由 server 所属模块注册,未注册任务抛 `UnknownTaskError` |
| `OutboxRelay` | 用 `FOR UPDATE SKIP LOCKED` 认领 `outbox_event` 中 `status=pending` 且 `next_attempt_at` 已到的行,按 type 路由后 enqueue(幂等键取行的 `idempotency_key`,缺省 `outbox:<id>`),成功标记 `dispatched` + `published_at`;失败 attempts+1 指数退避(上限 5 分钟),达 `maxAttempts` 置 `dead`;多进程并发时不会重复 |
| `migrateEngineSchema(databaseUrl, schema?)` | 只跑引擎自己的迁移后断开(部署时 `--role migrate`) |
| `withAttemptToken(db, opKey, fn, { commit })` | 操作账本 fencing,表是 `tasks_op_ledger` |
@@ -24,9 +24,9 @@
- **取消**:`cancel(runId, reason)` 先写 `tasks_run_meta.cancel_requested_at`,再取消引擎里的 run。handler 执行期间会轮询这个标志(默认 1s)并触发 `abortSignal`;每个 `step.run/sleep` 开始前也会检查一次。进行中的步骤只能靠自己检查 `abortSignal` 来中止。已完成的外部动作不会撤销,补偿要写成显式步骤。
- **等待外部条件**:`signal(runId, name, payload)` 先把事实写进 `tasks_fact`,再发送引擎信号。`step.waitForFact(name, { check?, timeout })` 每一轮都先查库,查不到才带超时等待信号。所以事实比等待方先到也不会丢。审批、下载完成这类业务事实,先写业务表,再用 `check` 去读业务表。
## 新增/填充任务(W2 规则)
## 新增业务任务
1. 在 `src/definitions.ts` 里用 `defineTask({ name, version, inputSchema, retryPolicy, handler })` 定义任务。输入只放业务 id,handler 自己去读业务表。重试策略要显式写:外部网络类 5 次(1s→60s),文件类 3 次。
1. 在 server 所属业务模块的 `registerTasks` 中用 `defineTask({ name, version, inputSchema, retryPolicy, handler })` 定义任务。输入只放业务 id,handler 自己去读业务表。显式声明 `lane` 与符合该业务副作用的重试策略;server 的任务分道表与回归同步更新。
2. **步骤体必须幂等**。worker 被 kill 后,正在执行的步骤会在新 worker 上**从头重跑**(最长要等约 30s 租约到期)。已完成的步骤不会重跑。
3. **有外部副作用的步骤都走 `withAttemptToken`**,opKey 按语义生成,例如 `submit:<planId>`:
- 已经有结果时直接返回;
@@ -57,4 +57,4 @@
## 测试
`deno task test`(或本包 `deno task test`)会连接开发库,每个测试 schema 先用 `@umt/db` 的 `migrateToLatest` 建全部业务表(`DATABASE_URL`,默认 `postgres://umt:umt@localhost:55433/umt`)。每个测试文件使用独立的 `tasks_test_<名>_<pid+随机>` schema(并发跑多份也不冲突),测试前后都会清理;库不可达时跳过集成测试。覆盖范围:kill -9 后续跑(子进程 worker)、取消、幂等 enqueue、两个 relay 并发、事实先到达、fencing、分组 worker 只认领自己的任务。
`bun run test`(或本包 `bun run test`)会连接开发库,每个测试 schema 先用 `@umt/db` 的 `migrateToLatest` 建全部业务表(`DATABASE_URL`,默认 `postgres://umt:umt@localhost:55433/umt`)。每个测试文件使用独立的 `tasks_test_<名>_<pid+随机>` schema(并发跑多份也不冲突),测试前后都会清理;单包运行时库不可达会跳过集成测试;根目录 `bun run test:all` 先检查连接,不可达直接失败。覆盖范围:kill -9 后续跑(子进程 worker)、取消、幂等 enqueue、两个 relay 并发、事实先到达、fencing、分组 worker 只认领自己的任务。
-9
View File
@@ -1,9 +0,0 @@
{
"name": "@umt/tasks",
"version": "0.3.44",
"exports": { ".": "./src/index.ts" },
"tasks": {
"test": "deno run -A npm:vitest run",
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
}
}
+8
View File
@@ -4,5 +4,13 @@
"type": "module",
"exports": {
".": "./src/index.ts"
},
"version": "0.3.44",
"scripts": {
"test": "bun x --bun vitest run",
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
},
"dependencies": {
"@umt/db": "workspace:*"
}
}
-144
View File
@@ -1,144 +0,0 @@
import { z } from "zod";
import { defineTask, TaskRegistry } from "./registry.ts";
// 任务书 §3.1-8 的九类任务。W1 只落空壳:输入 schema + 回显 handler,W2 填充处理器。
// 填充规则见 README「新增/填充任务」:副作用走 withAttemptToken,外部条件用 step.waitForFact,
// 业务状态写业务表,不要放进 handler 返回值。
const id = z.string().min(1);
/** 空壳 handler:记录一条日志并回显输入 */
function stub<I>(name: string) {
return async (ctx: { input: I; logger: { info(msg: string, f?: Record<string, unknown>): void } }) => {
// TODO(W2): 实现真实处理器
ctx.logger.info(`${name}: stub handler (TODO W2)`);
return { todo: true as const, task: name, input: ctx.input };
};
}
const NETWORK_RETRY = {
maximumAttempts: 5,
initialInterval: "1s",
backoffCoefficient: 2,
maximumInterval: "60s",
} as const;
const FILE_RETRY = {
maximumAttempts: 3,
initialInterval: "2s",
backoffCoefficient: 2,
maximumInterval: "60s",
} as const;
const DB_RETRY = { maximumAttempts: 3, initialInterval: "1s", backoffCoefficient: 2, maximumInterval: "30s" } as const;
export const ingestFeed = defineTask({
name: "ingestFeed",
version: 1,
lane: "network",
inputSchema: z.object({ feedSourceId: id }),
retryPolicy: NETWORK_RETRY,
handler: stub("ingestFeed"),
});
export const resolveObservation = defineTask({
name: "resolveObservation",
version: 1,
lane: "network",
inputSchema: z.object({ observationId: id }),
retryPolicy: NETWORK_RETRY,
handler: stub("resolveObservation"),
});
export const evaluateAcquisition = defineTask({
name: "evaluateAcquisition",
version: 1,
lane: "network",
inputSchema: z.object({
subscriptionId: id.optional(),
acquisitionSlotId: id.optional(),
sourceReleaseId: id.optional(),
}),
retryPolicy: DB_RETRY,
handler: stub("evaluateAcquisition"),
});
export const submitDownload = defineTask({
name: "submitDownload",
version: 1,
lane: "network",
inputSchema: z.object({ acquisitionPlanId: id }),
retryPolicy: NETWORK_RETRY,
handler: stub("submitDownload"),
});
export const verifyAndImport = defineTask({
name: "verifyAndImport",
version: 1,
lane: "file",
inputSchema: z.object({ downloadTaskId: id }),
retryPolicy: FILE_RETRY,
handler: stub("verifyAndImport"),
});
export const refreshMetadata = defineTask({
name: "refreshMetadata",
version: 1,
lane: "network",
inputSchema: z.object({ mediaEntityId: id }),
retryPolicy: NETWORK_RETRY,
handler: stub("refreshMetadata"),
});
export const reconcileLibrary = defineTask({
name: "reconcileLibrary",
version: 1,
lane: "maintenance",
inputSchema: z.object({ libraryRootId: id.optional(), reconcileRunId: id.optional() }),
retryPolicy: FILE_RETRY,
handler: stub("reconcileLibrary"),
});
export const cleanupDownloadCache = defineTask({
name: "cleanupDownloadCache",
version: 1,
lane: "file",
inputSchema: z.object({ downloaderInstanceId: id.optional() }),
retryPolicy: FILE_RETRY,
handler: stub("cleanupDownloadCache"),
});
export const resolveDecision = defineTask({
name: "resolveDecision",
version: 1,
lane: "agent",
inputSchema: z.object({ decisionId: id }),
retryPolicy: DB_RETRY,
handler: stub("resolveDecision"),
});
export const TASK_NAMES = [
"ingestFeed",
"resolveObservation",
"evaluateAcquisition",
"submitDownload",
"verifyAndImport",
"refreshMetadata",
"reconcileLibrary",
"cleanupDownloadCache",
"resolveDecision",
] as const;
export type TaskName = (typeof TASK_NAMES)[number];
/** v0.1 的默认注册表;W2 把真实 handler 换进来(同名同版本替换,或升版本并保留旧版本) */
export function createDefaultRegistry(): TaskRegistry {
return new TaskRegistry().register(
ingestFeed,
resolveObservation,
evaluateAcquisition,
submitDownload,
verifyAndImport,
refreshMetadata,
reconcileLibrary,
cleanupDownloadCache,
resolveDecision,
);
}
-14
View File
@@ -1,20 +1,6 @@
// @umt/tasks 公开入口:TaskEngine 端口 + OpenWorkflow 实现、任务注册表、outbox 中继、操作账本 fencing
export const PACKAGE_NAME = "@umt/tasks";
export {
cleanupDownloadCache,
createDefaultRegistry,
evaluateAcquisition,
ingestFeed,
reconcileLibrary,
refreshMetadata,
resolveDecision,
resolveObservation,
submitDownload,
TASK_NAMES,
type TaskName,
verifyAndImport,
} from "./definitions.ts";
export { FencingError, TaskCanceledError, UnknownTaskError } from "./errors.ts";
export {
consoleSink,
+1 -1
View File
@@ -22,7 +22,7 @@ const WORKER = fileURLToPath(new URL("./fixtures/worker.ts", import.meta.url));
const PKG_DIR = fileURLToPath(new URL("..", import.meta.url));
function startWorker(env: Record<string, string>): Promise<ChildProcess> {
const child = spawn(process.execPath, ["run", "-A", WORKER], {
const child = spawn(process.execPath, [WORKER], {
cwd: PKG_DIR,
env: { ...process.env, SCHEMA: schemas.biz, OW_SCHEMA: schemas.ow, ...env },
stdio: ["ignore", "pipe", "pipe"],
+6 -15
View File
@@ -1,14 +1,13 @@
import { describe, expect, it } from "vitest";
import { z } from "zod";
import { createDefaultRegistry, defineTask, redact, TASK_NAMES, TaskRegistry, UnknownTaskError } from "../src/index.ts";
import { defineTask, redact, TaskRegistry, UnknownTaskError } from "../src/index.ts";
describe("TaskRegistry", () => {
it("注册任务书 §3.1-8 的九个空壳任务", () => {
const reg = createDefaultRegistry();
expect(reg.names()).toEqual([...TASK_NAMES].sort());
for (const n of TASK_NAMES) expect(reg.latest(n).version).toBe(1);
expect(reg.has("ingestFeed")).toBe(true);
expect(reg.has("nope")).toBe(false);
it("注册表默认为空,未注册任务明确报错", () => {
const reg = new TaskRegistry();
expect(reg.names()).toEqual([]);
expect(reg.has("submitDownload")).toBe(false);
expect(() => reg.latest("submitDownload")).toThrow(UnknownTaskError);
});
it("latest 取最高版本;select 按 name@version / @version 过滤", () => {
@@ -24,14 +23,6 @@ describe("TaskRegistry", () => {
expect(() => reg.latest("zzz")).toThrow(UnknownTaskError);
expect(() => mk("a", 0)).toThrow(/invalid task version/);
});
it("空壳 handler 回显输入", async () => {
const def = createDefaultRegistry().latest("submitDownload");
const input = def.inputSchema.parse({ acquisitionPlanId: "p1" });
const out = await def.handler({ input, logger: { info() {} } } as never);
expect(out).toEqual({ todo: true, task: "submitDownload", input: { acquisitionPlanId: "p1" } });
expect(() => def.inputSchema.parse({})).toThrow();
});
});
describe("redact", () => {
-9
View File
@@ -1,9 +0,0 @@
{
"name": "@umt/testkit",
"version": "0.3.44",
"exports": { ".": "./src/index.ts" },
"tasks": {
"test": "deno run -A npm:vitest run",
"typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json"
}
}
+7 -1
View File
@@ -4,5 +4,11 @@
"type": "module",
"exports": {
".": "./src/index.ts"
}
},
"version": "0.3.44",
"scripts": {
"test": "bun x --bun vitest run",
"typecheck": "bun x --bun tsc --noEmit -p tsconfig.json"
},
"dependencies": {}
}
+2 -2
View File
@@ -21,7 +21,7 @@ export interface MockFetchRequest {
}
export interface MockFetch {
fetch: typeof globalThis.fetch;
fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
requests: MockFetchRequest[];
/** 追加/覆盖路由 */
route(key: string, handler: MockRouteHandler): void;
@@ -87,7 +87,7 @@ export function createMockFetch(routes: Record<string, MockRouteHandler> = {}):
return new Response("not found", { status: 404 });
};
return {
fetch: fetchImpl as typeof globalThis.fetch,
fetch: fetchImpl as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
requests,
route(key, handler) {
table.delete(key);