refactor(core): migrate to Bun and converge acquisition policies and Agent boundaries
This commit is contained in:
1 parent
3f1c6d02f6
commit
27c0f27434
268 files changed
+7468
-7852
No files matched your search
@@ -12,16 +12,16 @@ import { type AlistAuth, AlistClient } from "./client.ts";
|
||||
// UMT_ALIST_REAL_ROOT2 可选:另一个挂载(驱动)下的可写目录,测跨驱动复制 / 移动(异步任务)
|
||||
// 跑法:docs/runbook/storage-alist.md「真机验收」。
|
||||
|
||||
const URL_ = Deno.env.get("UMT_ALIST_REAL_URL");
|
||||
const ROOT = Deno.env.get("UMT_ALIST_REAL_ROOT");
|
||||
const ROOT2 = Deno.env.get("UMT_ALIST_REAL_ROOT2");
|
||||
const auth: AlistAuth | null = Deno.env.get("UMT_ALIST_REAL_TOKEN")
|
||||
? { kind: "token", token: Deno.env.get("UMT_ALIST_REAL_TOKEN")! }
|
||||
: Deno.env.get("UMT_ALIST_REAL_USER")
|
||||
const URL_ = process.env.UMT_ALIST_REAL_URL;
|
||||
const ROOT = process.env.UMT_ALIST_REAL_ROOT;
|
||||
const ROOT2 = process.env.UMT_ALIST_REAL_ROOT2;
|
||||
const auth: AlistAuth | null = process.env.UMT_ALIST_REAL_TOKEN
|
||||
? { kind: "token", token: process.env.UMT_ALIST_REAL_TOKEN! }
|
||||
: process.env.UMT_ALIST_REAL_USER
|
||||
? {
|
||||
kind: "password",
|
||||
username: Deno.env.get("UMT_ALIST_REAL_USER")!,
|
||||
password: Deno.env.get("UMT_ALIST_REAL_PASSWORD") ?? "",
|
||||
username: process.env.UMT_ALIST_REAL_USER!,
|
||||
password: process.env.UMT_ALIST_REAL_PASSWORD ?? "",
|
||||
}
|
||||
: null;
|
||||
const enabled = !!URL_ && !!ROOT && !!auth;
|
||||
|
||||
@@ -388,14 +388,17 @@ describe("上传", () => {
|
||||
|
||||
it("空闲超时:服务端不读数据时中止", async () => {
|
||||
const mock = createMockAlist({ token: "t" });
|
||||
const stuck: typeof fetch = (async (input: Request | URL | string, init?: RequestInit) => {
|
||||
const stuck: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = (async (
|
||||
input: Request | URL | string,
|
||||
init?: RequestInit,
|
||||
) => {
|
||||
const req = input instanceof Request ? input : new Request(String(input), init);
|
||||
if (new URL(req.url).pathname === "/api/fs/put")
|
||||
return new Promise<Response>((_, reject) =>
|
||||
init?.signal?.addEventListener("abort", () => reject(init.signal!.reason)),
|
||||
);
|
||||
return mock.fetch(input, init);
|
||||
}) as typeof fetch;
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
const client = new AlistClient({
|
||||
baseUrl: mock.base,
|
||||
auth: { kind: "token", token: "t" },
|
||||
@@ -453,14 +456,17 @@ describe("下载", () => {
|
||||
mock.put("/s1/a.mkv", bytes(1000));
|
||||
const realFetch = mock.fetch;
|
||||
let expired = false;
|
||||
const f: typeof fetch = (async (input: Request | URL | string, init?: RequestInit) => {
|
||||
const f: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = (async (
|
||||
input: Request | URL | string,
|
||||
init?: RequestInit,
|
||||
) => {
|
||||
const u = String(input instanceof Request ? input.url : input);
|
||||
if (u.includes("/d/") && !expired) {
|
||||
expired = true;
|
||||
mock.expireSigns();
|
||||
}
|
||||
return realFetch(input, init);
|
||||
}) as typeof fetch;
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
const c2 = new AlistClient({ baseUrl: mock.base, auth: { kind: "token", token: "perm-token" }, fetch: f, qps: 0 });
|
||||
await c2.download("/s1/a.mkv", join(dir, "a.mkv"));
|
||||
expect((await stat(join(dir, "a.mkv"))).size).toBe(1000);
|
||||
@@ -478,7 +484,10 @@ describe("下载", () => {
|
||||
const mock = createMockAlist({ token: "t" });
|
||||
mock.put("/s1/a.mkv", "x");
|
||||
const calls: { url: string; auth: string | null }[] = [];
|
||||
const f: typeof fetch = (async (input: Request | URL | string, init?: RequestInit) => {
|
||||
const f: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = (async (
|
||||
input: Request | URL | string,
|
||||
init?: RequestInit,
|
||||
) => {
|
||||
const req = input instanceof Request ? input : new Request(String(input), init);
|
||||
calls.push({ url: req.url, auth: req.headers.get("authorization") });
|
||||
if (req.url.startsWith("https://cdn.example")) return new Response("x");
|
||||
@@ -489,7 +498,7 @@ describe("下载", () => {
|
||||
return Response.json(j);
|
||||
}
|
||||
return res;
|
||||
}) as typeof fetch;
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
const c = new AlistClient({ baseUrl: mock.base, auth: { kind: "token", token: "t" }, fetch: f, qps: 0 });
|
||||
await c.download("/s1/a.mkv", join(dir, "a.mkv"));
|
||||
expect(calls.find((x) => x.url.startsWith("https://cdn.example"))!.auth).toBeNull();
|
||||
|
||||
@@ -24,7 +24,7 @@ export interface AlistClientOptions {
|
||||
auth: AlistAuth;
|
||||
/** 路径密码(Alist「元信息」里给目录设的密码),list / get 时带上 */
|
||||
pathPassword?: string | null;
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
/** 普通请求超时(ms),缺省 30 s */
|
||||
timeoutMs?: number;
|
||||
/** 数据流无进度空闲超时(ms),缺省 120 s */
|
||||
@@ -203,7 +203,7 @@ function idleWatch(ms: number, outer?: AbortSignal) {
|
||||
|
||||
export class AlistClient {
|
||||
readonly baseUrl: string;
|
||||
private readonly fetchFn: typeof fetch;
|
||||
private readonly fetchFn: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
private readonly limiter: AlistLimiter;
|
||||
private readonly o: Required<
|
||||
Pick<
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
// 令牌无效 code 401;copy 一律异步任务;同挂载 move 立即完成、跨挂载 move 是任务;put 覆盖同名;/d 支持 Range 与 sign。
|
||||
// 顶层目录视为「挂载」(驱动),用于区分同驱动 / 跨驱动。
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { type NetAddr, serve } from "@umt/runtime";
|
||||
|
||||
export interface MockFile {
|
||||
data: Uint8Array;
|
||||
@@ -43,7 +44,7 @@ export interface MockAlist {
|
||||
readonly base: string;
|
||||
setOrigin(origin: string): void;
|
||||
handler: (req: Request) => Promise<Response>;
|
||||
fetch: typeof fetch;
|
||||
fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
files: Map<string, MockFile>;
|
||||
dirs: Map<string, number>;
|
||||
tasks: Map<string, Task>;
|
||||
@@ -432,7 +433,7 @@ export function createMockAlist(opts: MockAlistOptions = {}): MockAlist {
|
||||
// 老版本(noTaskIds):没有任务 id 的复制 / 移动,在之后的任意 get 上推进
|
||||
if (opts.noTaskIds) for (const t of tasks.values()) advanceTask(t);
|
||||
return handler(req);
|
||||
}) as typeof fetch,
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
|
||||
files,
|
||||
dirs,
|
||||
tasks,
|
||||
@@ -471,13 +472,13 @@ export function createMockAlist(opts: MockAlistOptions = {}): MockAlist {
|
||||
|
||||
/** 起一个真 HTTP 监听(ffprobe / 下载代理等需要真实 URL 的测试用);返回 base 与关闭函数 */
|
||||
export function serveMockAlist(mock: MockAlist): { base: string; close: () => Promise<void> } {
|
||||
const server = Deno.serve({ port: 0, hostname: "127.0.0.1", onListen: () => {} }, (req) => {
|
||||
const server = serve({ port: 0, hostname: "127.0.0.1", onListen: () => {} }, (req) => {
|
||||
const u = new URL(req.url);
|
||||
return mock
|
||||
.handler(new Request(`${mock.base}${u.pathname}${u.search}`, req))
|
||||
.catch(() => new Response("connection reset", { status: 502 }));
|
||||
});
|
||||
const addr = server.addr as Deno.NetAddr;
|
||||
const addr = server.addr as NetAddr;
|
||||
const prev = mock.base;
|
||||
const base = `http://127.0.0.1:${addr.port}`;
|
||||
mock.setOrigin(base);
|
||||
|
||||
@@ -41,7 +41,7 @@ function realUpstream(handler: (c: Call) => Reply | undefined) {
|
||||
status: r.status ?? 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}) as typeof fetch;
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
return { calls, fetchImpl };
|
||||
}
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ export interface CamofoxConfig {
|
||||
downloadSink?: DownloadSink;
|
||||
/** waitForDownload 轮询间隔 */
|
||||
downloadPollMs?: number;
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
logger?: BrowserLogger;
|
||||
now?: () => Date;
|
||||
newId?: () => string;
|
||||
@@ -157,7 +157,7 @@ const LOGIN_PATH = /(?:^|\/)(?:login|takelogin|signin|sign_in|auth)(?:\.php)?(?:
|
||||
export class CamofoxBrowserProvider implements BrowserProvider {
|
||||
readonly kind = "camofox" as const;
|
||||
private readonly base: string;
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
private readonly lock: ProfileLock;
|
||||
private readonly sessions = new Map<string, SessionState>();
|
||||
private readonly leases = new Map<string, BrowserControlLease>();
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* 截图的解码、缩放与编码(VIS-1):交给视觉模型的截图缩放到不超过 maxWidth(缺省 1280)像素宽,
|
||||
* 证据缩略图更小。PNG 自己解码 / 编码(node:zlib),JPEG 用 jpeg-js(纯 JS,无原生依赖,deno compile 可嵌入)。
|
||||
* 证据缩略图更小。PNG 自己解码 / 编码(node:zlib),JPEG 用 jpeg-js(纯 JS,无原生依赖,bun build --compile 可嵌入)。
|
||||
* 只支持浏览器截图实际会出现的格式:8 位 PNG(灰度 / RGB / 调色板 / 带 alpha,非隔行)与基线 JPEG。
|
||||
*/
|
||||
import { deflateSync, inflateSync } from "node:zlib";
|
||||
|
||||
@@ -26,7 +26,7 @@ export interface EmbyConfig {
|
||||
retryBaseMs?: number;
|
||||
/** 搜索/分页单页大小,默认 50 */
|
||||
pageSize?: number;
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
}
|
||||
|
||||
export type Observed<T> =
|
||||
@@ -240,7 +240,7 @@ export class EmbyClient {
|
||||
readonly instanceId: string;
|
||||
private readonly base: string;
|
||||
private readonly flavor: "emby" | "jellyfin";
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
private userId: string | undefined;
|
||||
|
||||
constructor(private readonly cfg: EmbyConfig) {
|
||||
|
||||
@@ -4,13 +4,13 @@
|
||||
|
||||
## 挂死隔离:`FsProxy`(对标 MP fsproxy)
|
||||
|
||||
`new FsProxy({ hangTimeoutMs, quickTimeoutMs, progressIntervalMs, killGraceMs, mode })` 把单个文件操作(copy / crossMove / move / match / sha256 / stat / checkRoot / sweep / unlink)放进一次性执行子进程 `fsworker.ts`(`deno run --no-config -A`,约 50 ms 启动),stdin 一条 JSON 请求,stdout 逐行心跳 + 结果。
|
||||
`new FsProxy({ hangTimeoutMs, quickTimeoutMs, progressIntervalMs, killGraceMs, mode })` 把单个文件操作(copy / crossMove / move / match / sha256 / stat / checkRoot / sweep / unlink)放进一次性执行子进程 `fsworker.ts`(`bun fsworker.ts`),通过 `@umt/runtime/worker` 发送带 id 的逐行 JSON 请求、进度与结果,单条消息上限 16 MiB。
|
||||
|
||||
- 数据类操作(复制、跨卷搬运、哈希)按**进度停滞**判挂死:`hangTimeoutMs`(缺省 120 s)内字节数与阶段都没前进 → SIGTERM(worker 中止并清掉自己的 partial)→ `killGraceMs` 后 SIGKILL → `FsError('TIMEOUT')`。大文件复制几小时不会误杀。
|
||||
- 快操作按总时长 `quickTimeoutMs`(缺省 30 s)。`statSafe` 超时返回 `{kind:"error",code:"TIMEOUT"}`,`checkRoot` 超时返回 `offline(timeout)`——**挂死不等于文件缺失**。
|
||||
- 卡在 D 状态的子进程被 `unref`,不拖住本进程;被 SIGKILL 的 partial 由下次写前清理收走。
|
||||
- 子进程里 sha256 走流式(带心跳),不走 `sha256sum`(看不到进度);吞吐见 `docs/porting/moviepilot-rust-transfer.md` 基准记录。
|
||||
- `mode: "auto"`:`Deno.execPath()` 是 deno CLI → `deno run -A fsworker.ts`;否则视为 `deno compile` 出的 umt-server 二进制 → `<execPath> --role fsworker`(`apps/server/src/main.ts` 在加载其余模块前分派,启动约 150 ms)。编译产物的 `--allow-run` 必须含自身绝对路径(镜像内 `/app/umt-server`,见 `apps/server/deno.json`);起不来(NotCapable/ENOENT)自动回退 in-process(同样的看门狗,超时立即报 TIMEOUT 并中止,但卡死系统调用占住的线程无法回收),`stats().fallbackReason` 记原因。`stats().workerKind` = script / self / custom。
|
||||
- 服务端缺省 `mode: "subprocess"`,启动失败直接报错;显式选择 `mode: "auto"`:`process.execPath` 是 bun CLI → `bun fsworker.ts`;否则视为 `bun build --compile` 出的 umt-server 二进制 → `<execPath> --role fsworker`(`apps/server/src/main.ts` 在加载其余模块前分派)。子进程起不来(ENOENT/EACCES)自动回退 in-process(同样的看门狗,超时立即报 TIMEOUT 并中止,但卡死系统调用占住的线程无法回收),`stats().fallbackReason` 记原因。`stats().workerKind` = script / self / custom。
|
||||
- 容器验收开关:worker 进程设 `UMT_FS_TEST_HANG_AFTER_BYTES=<n>` 后,每个数据类操作的 worker 读到 n 字节即卡死 JS 线程并屏蔽 SIGTERM,用来在生产二进制里验证 停滞超时 → SIGKILL → TIMEOUT;生产勿设。
|
||||
- 测试开关 `_test: { hangAfterBytes, ignoreTerm }`:worker 在累计进度达到阈值后阻塞 JS 线程,模拟卡死的系统调用。
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { link, lstat, mkdir, open, readdir, rename, stat, unlink, utimes } from
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { Transform } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { Command } from "@umt/runtime";
|
||||
import { errnoOf, FsError, toFsError } from "./errors.ts";
|
||||
import { assertLocalPaths } from "./remote-guard.ts";
|
||||
|
||||
@@ -133,23 +134,10 @@ export function sha256Stats(): { counts: Record<Sha256Via, number>; subprocessUn
|
||||
return { counts: { ...hashCounters }, subprocessUnavailable: sha256sumUnavailable };
|
||||
}
|
||||
|
||||
type DenoLike = {
|
||||
Command: new (
|
||||
cmd: string,
|
||||
opts: { args: string[]; stdout: "piped"; stderr: "piped"; signal?: AbortSignal },
|
||||
) => { output(): Promise<{ success: boolean; code: number; stdout: Uint8Array; stderr: Uint8Array }> };
|
||||
errors: { NotFound: new () => Error; NotCapable?: new () => Error; PermissionDenied: new () => Error };
|
||||
};
|
||||
|
||||
async function sha256Subprocess(p: string, signal?: AbortSignal): Promise<string | null> {
|
||||
const D = (globalThis as { Deno?: DenoLike }).Deno;
|
||||
if (!D?.Command) {
|
||||
sha256sumUnavailable = "no_deno_command";
|
||||
return null;
|
||||
}
|
||||
let out: Awaited<ReturnType<InstanceType<DenoLike["Command"]>["output"]>>;
|
||||
let out: Awaited<ReturnType<Command["output"]>>;
|
||||
try {
|
||||
out = await new D.Command("sha256sum", {
|
||||
out = await new Command("sha256sum", {
|
||||
args: ["-b", "--", p],
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
@@ -157,9 +145,9 @@ async function sha256Subprocess(p: string, signal?: AbortSignal): Promise<string
|
||||
}).output();
|
||||
} catch (e) {
|
||||
if (signal?.aborted) throw new FsError("ABORTED", "aborted", { path: p, cause: e });
|
||||
const notCapable = D.errors.NotCapable && e instanceof D.errors.NotCapable;
|
||||
if (e instanceof D.errors.NotFound || e instanceof D.errors.PermissionDenied || notCapable) {
|
||||
sha256sumUnavailable = e instanceof D.errors.NotFound ? "not_installed" : "not_permitted";
|
||||
const code = (e as NodeJS.ErrnoException).code;
|
||||
if (code === "ENOENT" || code === "EACCES" || code === "EPERM") {
|
||||
sha256sumUnavailable = code === "ENOENT" ? "not_installed" : "not_permitted";
|
||||
return null;
|
||||
}
|
||||
throw toFsError(e, p);
|
||||
@@ -425,7 +413,7 @@ export async function copyWithVerify(src: string, dst: string, opts: CopyOptions
|
||||
receipt.sampleDigest = b;
|
||||
}
|
||||
opts.signal?.throwIfAborted();
|
||||
await utimes(tmp, srcStat.atime, srcStat.mtime);
|
||||
await utimes(tmp, srcStat.atimeMs / 1000, srcStat.mtimeMs / 1000);
|
||||
if (existing) receipt.replaced = await replaceCommit(tmp, dst, existing, opts.replaceOld);
|
||||
else await commitNoClobber(tmp, dst);
|
||||
await fsyncPath(dirname(dst));
|
||||
|
||||
@@ -108,3 +108,6 @@ export async function runFsOp(req: FsOpRequest, ctx: FsOpContext): Promise<unkno
|
||||
throw new FsError("IO", `unknown fs op: ${(req as { op: string }).op}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** 文件请求、扫描结果的单条协议消息上限。文件内容不通过管道传输。 */
|
||||
export const FS_MESSAGE_BYTES = 16 * 1024 * 1024;
|
||||
@@ -132,7 +132,7 @@ describe("FsProxy(子进程)", () => {
|
||||
});
|
||||
|
||||
describe("FsProxy 启动方式", () => {
|
||||
it("deno CLI 下缺省 run fsworker.ts;显式 workerCommand 记为 custom", () => {
|
||||
it("bun CLI 下缺省 run fsworker.ts;显式 workerCommand 记为 custom", () => {
|
||||
const auto = defaultWorkerCommand();
|
||||
expect(auto?.kind).toBe("script");
|
||||
expect(auto?.args.at(-1)).toMatch(/fsworker\.ts$/);
|
||||
@@ -148,6 +148,13 @@ describe("FsProxy 启动方式", () => {
|
||||
expect(px.stats()).toMatchObject({ mode: "inprocess", fallbackReason: expect.stringMatching(/^spawn_failed/) });
|
||||
});
|
||||
|
||||
it("强制子进程时启动失败报错,不在主进程执行文件操作", async () => {
|
||||
const t = await tree({ "a.mkv": "x" });
|
||||
const px = new FsProxy({ mode: "subprocess", workerCommand: { cmd: "/nonexistent/umt-worker", args: [] } });
|
||||
expect(await px.statSafe(t.path("a.mkv"))).toMatchObject({ kind: "error", code: "IO" });
|
||||
expect(px.stats()).toMatchObject({ mode: "subprocess", fallbackReason: null });
|
||||
});
|
||||
|
||||
it("验收开关只在 UMT_FS_TEST_HOOKS=1 时生效(单独误设 HANG_AFTER_BYTES 无效)", () => {
|
||||
const env = (m: Record<string, string>) => (k: string) => m[k];
|
||||
expect(fsTestHookFromEnv(env({ UMT_FS_TEST_HANG_AFTER_BYTES: "1024" }))).toBeUndefined();
|
||||
@@ -162,16 +169,16 @@ describe("FsProxy 启动方式", () => {
|
||||
it("UMT_FS_TEST_HANG_AFTER_BYTES(容器验收开关)→ 数据类操作卡死 → SIGKILL → TIMEOUT;快操作不受影响", async () => {
|
||||
const t = await tree({ "src/a.mkv": 3 * MiB, "dst/": null });
|
||||
const px = new FsProxy({ mode: "subprocess", ...fast });
|
||||
Deno.env.set("UMT_FS_TEST_HANG_AFTER_BYTES", String(MiB));
|
||||
Deno.env.set("UMT_FS_TEST_HOOKS", "1");
|
||||
process.env.UMT_FS_TEST_HANG_AFTER_BYTES = String(MiB);
|
||||
process.env.UMT_FS_TEST_HOOKS = "1";
|
||||
try {
|
||||
expect(await px.statSafe(t.path("src/a.mkv"))).toMatchObject({ kind: "ok" });
|
||||
const e = await px.copy(t.path("src/a.mkv"), t.path("dst/a.mkv")).catch((x) => x);
|
||||
expect(e.code).toBe("TIMEOUT");
|
||||
expect(px.stats()).toMatchObject({ timeouts: 1, killed: 1 });
|
||||
} finally {
|
||||
Deno.env.delete("UMT_FS_TEST_HANG_AFTER_BYTES");
|
||||
Deno.env.delete("UMT_FS_TEST_HOOKS");
|
||||
delete process.env.UMT_FS_TEST_HANG_AFTER_BYTES;
|
||||
delete process.env.UMT_FS_TEST_HOOKS;
|
||||
}
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
@@ -5,18 +5,20 @@
|
||||
// 统一转成 FsError('TIMEOUT')——普通可重试 IO 错误,交给任务层退避重试。卡在 D 状态的子进程被 unref,不拖住本进程。
|
||||
// stat/checkRoot 超时返回 error/offline(timeout),**不**当成文件缺失(REQ §2.1 原则 4)。
|
||||
//
|
||||
// 与 MP 的差异:MP 是常驻 worker + 一把锁串行;我们每个操作一个子进程(~50 ms 启动,入库按文件计可忽略),
|
||||
// 与 MP 的差异:MP 是常驻 worker + 一把锁串行;我们每个操作一个子进程,
|
||||
// 天然并发、互不拖累,也没有"上一个请求挂死把 worker 占住"的问题。
|
||||
// 启动方式:execPath 是 deno CLI(开发/测试)→ `deno run -A fsworker.ts`;否则视为 deno compile 出的 umt-server
|
||||
// 二进制 → `<execPath> --role fsworker`(入口 main.ts 在加载其余模块前分派)。编译产物的 --allow-run 须含自身绝对路径
|
||||
// (镜像内 /app/umt-server,见 apps/server/deno.json);起不来(NotCapable 等)回退 in-process:
|
||||
// 启动方式:execPath 是 bun CLI(开发/测试)→ `bun fsworker.ts`;否则视为 bun build --compile 出的 umt-server
|
||||
// 二进制 → `<execPath> --role fsworker`(入口 main.ts 在加载其余模块前分派)。
|
||||
// 子进程无法启动时回退 in-process:
|
||||
// 同样的心跳看门狗,超时立即以 TIMEOUT 失败并中止操作,但卡死的系统调用占住的线程无法回收(见 README)。
|
||||
import { basename } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { WorkerConnection, WorkerTransportError } from "@umt/runtime/worker";
|
||||
import type { CopyReceipt, Sha256Result } from "./copy.ts";
|
||||
import { FsError, type FsErrorCode } from "./errors.ts";
|
||||
import {
|
||||
DATA_OPS,
|
||||
FS_MESSAGE_BYTES,
|
||||
type FsOpRequest,
|
||||
type FsProgress,
|
||||
type FsTestHook,
|
||||
@@ -32,7 +34,7 @@ import type { RootHealth, StatResult } from "./stat.ts";
|
||||
export type { FsProgress } from "./fsops.ts";
|
||||
|
||||
export interface FsProxyConfig {
|
||||
/** auto:能起 deno 子进程就用子进程,否则 in-process;subprocess 强制子进程(起不来报错) */
|
||||
/** auto:能起 bun 子进程就用子进程,否则 in-process;subprocess 强制子进程(起不来报错) */
|
||||
mode?: "auto" | "subprocess" | "inprocess";
|
||||
/** 数据类操作:进度停滞多久判挂死(缺省 120 s,同 MP FS_PROXY_STALL_TIMEOUT) */
|
||||
hangTimeoutMs?: number;
|
||||
@@ -62,15 +64,6 @@ export interface FsProxyStats {
|
||||
fallbackReason: string | null;
|
||||
}
|
||||
|
||||
type DenoChild = {
|
||||
stdin: WritableStream<Uint8Array>;
|
||||
stdout: ReadableStream<Uint8Array>;
|
||||
stderr: ReadableStream<Uint8Array>;
|
||||
status: Promise<{ success: boolean; code: number; signal: string | null }>;
|
||||
kill(sig?: string): void;
|
||||
unref(): void;
|
||||
};
|
||||
|
||||
const WORKER_PATH = (() => {
|
||||
try {
|
||||
const u = new URL("./fsworker.ts", import.meta.url);
|
||||
@@ -80,31 +73,12 @@ const WORKER_PATH = (() => {
|
||||
}
|
||||
})();
|
||||
|
||||
/** 当前运行时能用的缺省 worker 启动命令:deno CLI → run 脚本;编译二进制 → 自身 --role fsworker;都不行 → null */
|
||||
/** 当前运行时能用的缺省 worker 启动命令:bun CLI → run 脚本;编译二进制 → 自身 --role fsworker;都不行 → null */
|
||||
export function defaultWorkerCommand(): { cmd: string; args: string[]; kind: "script" | "self" } | null {
|
||||
const D = (globalThis as { Deno?: { execPath?: () => string; Command?: unknown } }).Deno;
|
||||
if (!D?.Command || !D.execPath) return null;
|
||||
let exe: string;
|
||||
try {
|
||||
exe = D.execPath();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (/^deno(\.exe)?$/.test(basename(exe))) {
|
||||
const exe = process.execPath;
|
||||
if (/^bun(\.exe)?$/.test(basename(exe))) {
|
||||
if (!WORKER_PATH) return null;
|
||||
return {
|
||||
cmd: exe,
|
||||
args: [
|
||||
"run",
|
||||
"--no-config",
|
||||
"--quiet",
|
||||
"--no-prompt",
|
||||
// Deno 对 /dev、/proc 下的路径(如 /dev/shm 上的库根)要求 --allow-all;worker 只执行本包固定的操作表
|
||||
"--allow-all",
|
||||
WORKER_PATH,
|
||||
],
|
||||
kind: "script",
|
||||
};
|
||||
return { cmd: exe, args: [WORKER_PATH], kind: "script" };
|
||||
}
|
||||
return { cmd: exe, args: ["--role", "fsworker"], kind: "self" };
|
||||
}
|
||||
@@ -122,12 +96,7 @@ export function fsTestHookFromEnv(get: (k: string) => string | undefined): FsTes
|
||||
}
|
||||
|
||||
function envTestHook(): FsTestHook | undefined {
|
||||
try {
|
||||
const env = (globalThis as { Deno?: { env?: { get(k: string): string | undefined } } }).Deno?.env;
|
||||
return env ? fsTestHookFromEnv((k) => env.get(k)) : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
return fsTestHookFromEnv((k) => process.env[k]);
|
||||
}
|
||||
|
||||
const timeoutError = (op: string, why: string, path?: string) =>
|
||||
@@ -154,7 +123,7 @@ function pathOf(req: FsOpRequest): string | undefined {
|
||||
|
||||
export class FsProxy {
|
||||
readonly cfg: Required<Omit<FsProxyConfig, "workerCommand" | "mode">> & Pick<FsProxyConfig, "workerCommand">;
|
||||
/** 子进程怎么起:script = deno run fsworker.ts;self = 编译二进制 --role fsworker;custom = 显式 workerCommand */
|
||||
/** 子进程怎么起:script = bun fsworker.ts;self = 编译二进制 --role fsworker;custom = 显式 workerCommand */
|
||||
readonly workerKind: "script" | "self" | "custom" | null;
|
||||
#mode: "subprocess" | "inprocess";
|
||||
#forced: boolean;
|
||||
@@ -177,9 +146,9 @@ export class FsProxy {
|
||||
if (mode === "inprocess") this.#mode = "inprocess";
|
||||
else if (wc) this.#mode = "subprocess";
|
||||
else {
|
||||
if (mode === "subprocess") throw new Error("fsproxy: 当前运行时无法启动 deno 子进程,且未提供 workerCommand");
|
||||
if (mode === "subprocess") throw new Error("fsproxy: 当前运行时无法启动 bun 子进程,且未提供 workerCommand");
|
||||
this.#mode = "inprocess";
|
||||
this.#fallbackReason = "no_deno_cli";
|
||||
this.#fallbackReason = "no_bun_cli";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,7 +220,8 @@ export class FsProxy {
|
||||
try {
|
||||
return await this.#runSubprocess(req, call);
|
||||
} catch (e) {
|
||||
if (!(e instanceof SpawnError) || this.#forced) throw e instanceof SpawnError ? e.cause : e;
|
||||
if (!(e instanceof SpawnError)) throw e;
|
||||
if (this.#forced) throw new FsError("IO", `文件 worker 无法启动:${e.message}`, { cause: e.cause });
|
||||
this.#mode = "inprocess";
|
||||
this.#fallbackReason = e.message;
|
||||
}
|
||||
@@ -310,144 +280,72 @@ export class FsProxy {
|
||||
}
|
||||
}
|
||||
|
||||
async #runSubprocess(req: FsOpRequest, call: FsCallOptions): Promise<unknown> {
|
||||
const D = (globalThis as unknown as { Deno: DenoLike }).Deno;
|
||||
const cmd = this.cfg.workerCommand!;
|
||||
let child: DenoChild;
|
||||
async #runSubprocess(req: FsOpRequest, call: FsCallOptions = {}): Promise<unknown> {
|
||||
let connection: WorkerConnection;
|
||||
try {
|
||||
child = new D.Command(cmd.cmd, {
|
||||
args: cmd.args,
|
||||
stdin: "piped",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
env: { UMT_FS_PROGRESS_INTERVAL_MS: String(this.cfg.progressIntervalMs) },
|
||||
}).spawn();
|
||||
} catch (e) {
|
||||
throw new SpawnError(`spawn_failed:${(e as Error)?.name ?? "Error"}`, e);
|
||||
connection = new WorkerConnection(this.cfg.workerCommand!, {
|
||||
values: { UMT_FS_PROGRESS_INTERVAL_MS: String(this.cfg.progressIntervalMs) },
|
||||
});
|
||||
} catch (error) {
|
||||
throw new SpawnError(`spawn_failed:${(error as Error).name}`, error);
|
||||
}
|
||||
this.#stats.spawned += 1;
|
||||
const limit = this.#limitFor(req);
|
||||
const w = child.stdin.getWriter();
|
||||
let last = Date.now();
|
||||
let lastKey = "";
|
||||
const terminate = (error: FsError) => connection.terminate(error, this.cfg.killGraceMs, () => this.#stats.killed++);
|
||||
const onAbort = () => terminate(new FsError("ABORTED", "aborted", { path: pathOf(req) }));
|
||||
const watch = setInterval(
|
||||
() => {
|
||||
if (Date.now() - last > limit.ms) {
|
||||
this.#stats.timeouts += 1;
|
||||
clearInterval(watch);
|
||||
terminate(timeoutError(req.op, limit.why, pathOf(req)));
|
||||
}
|
||||
},
|
||||
Math.max(10, Math.min(1000, limit.ms / 4)),
|
||||
);
|
||||
call.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
const hook = call._test ?? (DATA_OPS.has(req.op) ? envTestHook() : undefined);
|
||||
await w.write(new TextEncoder().encode(JSON.stringify(hook ? { ...req, _test: hook } : req)));
|
||||
await w.close().catch(() => undefined);
|
||||
|
||||
let stderr = "";
|
||||
const errPump = (async () => {
|
||||
const dec = new TextDecoder();
|
||||
try {
|
||||
for await (const c of child.stderr) stderr = (stderr + dec.decode(c, { stream: true })).slice(-2000);
|
||||
} catch {
|
||||
// 被 kill 时流会出错
|
||||
}
|
||||
})();
|
||||
|
||||
return await new Promise<unknown>((resolve, reject) => {
|
||||
let settled = false;
|
||||
let last = Date.now();
|
||||
let lastKey = "";
|
||||
let killTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
/** 已判超时/已取消:之后 worker 自己报的结果(如 SIGTERM 后的 ABORTED)一律以此为准 */
|
||||
let terminating: FsError | null = null;
|
||||
const finish = (fn: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearInterval(watch);
|
||||
call.signal?.removeEventListener("abort", onAbort);
|
||||
fn();
|
||||
};
|
||||
const terminate = (err: FsError) => {
|
||||
if (settled || terminating) return;
|
||||
terminating = err;
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
// 已退出
|
||||
}
|
||||
clearInterval(watch);
|
||||
killTimer = setTimeout(() => {
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
this.#stats.killed += 1;
|
||||
} catch {
|
||||
// SIGTERM 后已退出
|
||||
}
|
||||
// D 状态的子进程可能连 SIGKILL 都要等系统调用返回:不等它,unref 后直接报错
|
||||
child.unref();
|
||||
finish(() => reject(err));
|
||||
}, this.cfg.killGraceMs);
|
||||
};
|
||||
const onAbort = () =>
|
||||
terminate(new FsError("ABORTED", "aborted", { ...(pathOf(req) ? { path: pathOf(req) } : {}) }));
|
||||
call.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
const watch = setInterval(
|
||||
() => {
|
||||
if (Date.now() - last > limit.ms) {
|
||||
this.#stats.timeouts += 1;
|
||||
terminate(timeoutError(req.op, limit.why, pathOf(req)));
|
||||
}
|
||||
try {
|
||||
const pending = connection.request(
|
||||
{
|
||||
id: "1",
|
||||
task: req.op,
|
||||
input: hook ? { ...req, _test: hook } : req,
|
||||
},
|
||||
Math.max(10, Math.min(1000, limit.ms / 4)),
|
||||
);
|
||||
|
||||
(async () => {
|
||||
const dec = new TextDecoder();
|
||||
let buf = "";
|
||||
let final: WorkerFinal | null = null;
|
||||
try {
|
||||
for await (const c of child.stdout) {
|
||||
buf += dec.decode(c, { stream: true });
|
||||
let i = buf.indexOf("\n");
|
||||
while (i >= 0) {
|
||||
const line = buf.slice(0, i);
|
||||
buf = buf.slice(i + 1);
|
||||
i = buf.indexOf("\n");
|
||||
if (!line.trim()) continue;
|
||||
const msg = JSON.parse(line) as WorkerFinal & FsProgress;
|
||||
if (msg.t === "hb") {
|
||||
const key = `${msg.phase}:${msg.bytes}`;
|
||||
if (!limit.stall || key !== lastKey) {
|
||||
lastKey = key;
|
||||
if (limit.stall) last = Date.now();
|
||||
}
|
||||
call.onProgress?.({ phase: msg.phase, bytes: msg.bytes, total: msg.total });
|
||||
} else final = msg;
|
||||
{
|
||||
maxRequestBytes: FS_MESSAGE_BYTES,
|
||||
maxResponseBytes: FS_MESSAGE_BYTES,
|
||||
onProgress: (value) => {
|
||||
const p = value as FsProgress;
|
||||
if (!p || typeof p.phase !== "string" || !Number.isFinite(p.bytes) || !Number.isFinite(p.total))
|
||||
throw new Error("文件 worker 进度格式无效");
|
||||
const key = `${p.phase}:${p.bytes}`;
|
||||
if (limit.stall && key !== lastKey) {
|
||||
lastKey = key;
|
||||
last = Date.now();
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// kill 时 stdout 断开
|
||||
}
|
||||
const status = await child.status;
|
||||
await errPump;
|
||||
if (terminating) {
|
||||
// SIGTERM 生效(worker 清理后退出)就不必等到宽限期满
|
||||
clearTimeout(killTimer);
|
||||
const err = terminating;
|
||||
return finish(() => reject(err));
|
||||
}
|
||||
if (final?.t === "ok") return finish(() => resolve(final!.result));
|
||||
if (final?.t === "err")
|
||||
return finish(() =>
|
||||
reject(
|
||||
new FsError(final!.code ?? "IO", final!.message ?? "fs worker error", {
|
||||
...(final!.path ? { path: final!.path } : {}),
|
||||
}),
|
||||
),
|
||||
);
|
||||
finish(() =>
|
||||
reject(
|
||||
new FsError("IO", `fs worker 异常退出(code=${status.code} signal=${status.signal}):${stderr.trim()}`, {
|
||||
...(pathOf(req) ? { path: pathOf(req) } : {}),
|
||||
}),
|
||||
),
|
||||
);
|
||||
})();
|
||||
});
|
||||
call.onProgress?.(p);
|
||||
},
|
||||
},
|
||||
);
|
||||
if (call.signal?.aborted) onAbort();
|
||||
const reply = await pending;
|
||||
if (reply.type === "result") return reply.result;
|
||||
throw new FsError(reply.error.code as FsErrorCode, reply.error.message, reply.error.detail);
|
||||
} catch (error) {
|
||||
if (error instanceof FsError) throw error;
|
||||
if (error instanceof WorkerTransportError && error.kind === "spawn")
|
||||
throw new SpawnError(`spawn_failed:${(error.cause as Error)?.name ?? "Error"}`, error.cause);
|
||||
throw new FsError("IO", (error as Error).message, { path: pathOf(req), cause: error });
|
||||
} finally {
|
||||
clearInterval(watch);
|
||||
call.signal?.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type WorkerFinal = { t: string; result?: unknown; code?: FsErrorCode; message?: string; path?: string };
|
||||
|
||||
class SpawnError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
@@ -457,20 +355,6 @@ class SpawnError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
type DenoLike = {
|
||||
execPath(): string;
|
||||
Command: new (
|
||||
cmd: string,
|
||||
opts: {
|
||||
args: string[];
|
||||
stdin: "piped";
|
||||
stdout: "piped";
|
||||
stderr: "piped";
|
||||
env?: Record<string, string>;
|
||||
},
|
||||
) => { spawn(): DenoChild };
|
||||
};
|
||||
|
||||
let shared: FsProxy | null = null;
|
||||
/** 进程级缺省代理(配置由 server 启动时 configureFsProxy 注入;未配置时用缺省阈值) */
|
||||
export function fsProxy(): FsProxy {
|
||||
|
||||
@@ -1,34 +1,25 @@
|
||||
// fsproxy 执行子进程:从 stdin 读一条 JSON 请求,执行单个文件操作,stdout 逐行 JSON 上报
|
||||
// {t:"hb",phase,bytes,total}(按 progressIntervalMs 节流,阶段切换与阶段完成立即报)/ {t:"ok",result} / {t:"err",code,message}。
|
||||
// SIGTERM → 中止当前操作(copy 会清掉自己的 partial)后退出;挂死在系统调用里时由父进程 SIGKILL 回收。
|
||||
// 只依赖 node 内置模块,`deno run --no-config` 毫秒级启动。编译二进制可在入口按 --role fsworker 调 fsWorkerMain()。
|
||||
import { decodeWorkerRequest, readWorkerRequestText, writeWorkerMessage } from "@umt/runtime/worker";
|
||||
import { toFsError } from "./errors.ts";
|
||||
import { type FsOpRequest, type FsProgress, runFsOp } from "./fsops.ts";
|
||||
|
||||
const enc = new TextEncoder();
|
||||
|
||||
function send(obj: unknown): void {
|
||||
try {
|
||||
Deno.stdout.writeSync(enc.encode(`${JSON.stringify(obj)}\n`));
|
||||
} catch {
|
||||
// 父进程已退出:无处可报
|
||||
}
|
||||
}
|
||||
import { FS_MESSAGE_BYTES, type FsOpRequest, type FsProgress, runFsOp } from "./fsops.ts";
|
||||
|
||||
// 一次性文件 worker;固定操作分派、进度停滞策略与文件清理语义保留在文件模块。
|
||||
/** 阻塞 JS 线程(测试用,模拟卡死的系统调用:信号监听器也不会运行) */
|
||||
function blockForever(): never {
|
||||
const a = new Int32Array(new SharedArrayBuffer(4));
|
||||
for (;;) Atomics.wait(a, 0, 0);
|
||||
}
|
||||
|
||||
export async function fsWorkerMain(progressIntervalMs = Number(Deno.env.get("UMT_FS_PROGRESS_INTERVAL_MS") ?? 1000)) {
|
||||
const req = JSON.parse(await new Response(Deno.stdin.readable).text()) as FsOpRequest;
|
||||
export async function fsWorkerMain(progressIntervalMs = Number(process.env.UMT_FS_PROGRESS_INTERVAL_MS ?? 1000)) {
|
||||
const request = decodeWorkerRequest(await readWorkerRequestText(FS_MESSAGE_BYTES));
|
||||
const req = request.input as FsOpRequest;
|
||||
if (!req || req.op !== request.task) throw new Error("文件 worker 任务与操作不一致");
|
||||
const send = (value: FsProgress) => writeWorkerMessage({ id: request.id, type: "progress", value }, FS_MESSAGE_BYTES);
|
||||
const ac = new AbortController();
|
||||
Deno.addSignalListener("SIGTERM", () => {
|
||||
process.on("SIGTERM", () => {
|
||||
if (req._test?.ignoreTerm) return;
|
||||
ac.abort();
|
||||
// 操作卡在系统调用里时 abort 解不开:给清理一点时间后直接退出
|
||||
setTimeout(() => Deno.exit(143), 500);
|
||||
setTimeout(() => process.exit(143), 500);
|
||||
});
|
||||
let lastAt = 0;
|
||||
let lastPhase = "";
|
||||
@@ -45,28 +36,29 @@ export async function fsWorkerMain(progressIntervalMs = Number(Deno.env.get("UMT
|
||||
) {
|
||||
lastAt = now;
|
||||
lastPhase = p.phase;
|
||||
send({ t: "hb", ...p });
|
||||
send(p);
|
||||
}
|
||||
if (hang !== undefined && p.bytes >= hang) blockForever();
|
||||
};
|
||||
if (req._test?.hangAfterBytes === 0) {
|
||||
send({ t: "hb", phase: "start", bytes: 0, total: 0 });
|
||||
send({ phase: "start", bytes: 0, total: 0 });
|
||||
blockForever();
|
||||
}
|
||||
try {
|
||||
const result = await runFsOp(req, { signal: ac.signal, progress });
|
||||
send({ t: "ok", result });
|
||||
writeWorkerMessage({ id: request.id, type: "result", result }, FS_MESSAGE_BYTES);
|
||||
} catch (e) {
|
||||
const fe = toFsError(e);
|
||||
send({
|
||||
t: "err",
|
||||
code: ac.signal.aborted ? "ABORTED" : fe.code,
|
||||
message: fe.message,
|
||||
...(fe.detail.path ? { path: fe.detail.path } : {}),
|
||||
...(fe.detail.syscall ? { syscall: fe.detail.syscall } : {}),
|
||||
});
|
||||
writeWorkerMessage(
|
||||
{
|
||||
id: request.id,
|
||||
type: "error",
|
||||
error: { code: ac.signal.aborted ? "ABORTED" : fe.code, message: fe.message, detail: fe.detail },
|
||||
},
|
||||
FS_MESSAGE_BYTES,
|
||||
);
|
||||
}
|
||||
Deno.exit(0);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (import.meta.main) await fsWorkerMain();
|
||||
@@ -44,7 +44,7 @@ export interface MetaRequestCache {
|
||||
}
|
||||
|
||||
export interface MetaClientOptions {
|
||||
fetch?: typeof globalThis.fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
/** 缺省 15s */
|
||||
timeoutMs?: number;
|
||||
/** API 地址(bgm api.bgm.tv / 豆瓣 rexxar) */
|
||||
|
||||
@@ -39,7 +39,7 @@ export interface QbInstanceConfig {
|
||||
sequentialDownload?: boolean;
|
||||
firstLastPiecePrio?: boolean;
|
||||
/** 注入 fetch(测试/代理) */
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
}
|
||||
|
||||
export type QbAuthMode = "password" | "api_key" | "none";
|
||||
@@ -314,7 +314,7 @@ export class QbClient implements DownloaderClient {
|
||||
private readonly retryBaseMs: number;
|
||||
private readonly lookup: { attempts: number; intervalMs: number };
|
||||
private readonly probePrefix: string;
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
private sid: string | null = null;
|
||||
private loginInFlight: Promise<void> | null = null;
|
||||
/** qB 5.x 把 pause/resume 改名为 stop/start;探测到后缓存 */
|
||||
|
||||
@@ -7,7 +7,7 @@ import { redactText, redactUrl } from "./redact.ts";
|
||||
import { FeedError, type FeedFetchResult, type FeedFetchState } from "./types.ts";
|
||||
|
||||
export interface RssFetcherOptions {
|
||||
fetch?: typeof globalThis.fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
now?: () => Date;
|
||||
/** 单次请求超时,默认 30s(MP `timeout or 30`) */
|
||||
timeoutMs?: number;
|
||||
@@ -31,7 +31,7 @@ export interface FetchFeedOptions {
|
||||
/** 本次请求的 UA(覆盖 RssFetcherOptions.userAgent;BR-4 按站点账号取) */
|
||||
userAgent?: string;
|
||||
/** 本次请求用的 fetch(覆盖 RssFetcherOptions.fetch;BR-4 站点代理 = 与站点其它请求同一出口) */
|
||||
fetch?: typeof globalThis.fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
}
|
||||
|
||||
export function initialFeedState(): FeedFetchState {
|
||||
|
||||
@@ -22,7 +22,7 @@ const CF_JS_PAGE = `<html><head><title>Just a moment...</title></head><body>
|
||||
<script>window._cf_chl_opt={cType: 'non-interactive'}</script></body></html>`;
|
||||
const NORMAL = `<html><head><title>种子 - 观众</title><script src="/cdn-cgi/challenge-platform/scripts/jsd/main.js"></script></head><body>ok</body></html>`;
|
||||
|
||||
function ctx(fetchImpl: typeof fetch) {
|
||||
function ctx(fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>) {
|
||||
return {
|
||||
siteId: "s1",
|
||||
fetch: fetchImpl,
|
||||
@@ -32,7 +32,10 @@ function ctx(fetchImpl: typeof fetch) {
|
||||
};
|
||||
}
|
||||
const reply = (status: number, body: string, headers: Record<string, string> = {}) =>
|
||||
(async () => new Response(body, { status, headers: { "content-type": "text/html", ...headers } })) as typeof fetch;
|
||||
(async () => new Response(body, { status, headers: { "content-type": "text/html", ...headers } })) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>;
|
||||
|
||||
describe("challengeRuleOfHtml", () => {
|
||||
it("挑战页标题 / 脚本特征命中;正常页面里注入的 challenge-platform 脚本不算", () => {
|
||||
@@ -62,7 +65,7 @@ describe("detectChallenge", () => {
|
||||
|
||||
describe("siteFetch 挑战分类", () => {
|
||||
const url = "https://audiences.example/torrents.php";
|
||||
const kind = async (f: typeof fetch) => {
|
||||
const kind = async (f: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>) => {
|
||||
try {
|
||||
await siteFetch(ctx(f), url);
|
||||
return "ok";
|
||||
|
||||
@@ -130,7 +130,7 @@ async function peekBody(
|
||||
res: Response,
|
||||
max: number,
|
||||
): Promise<{ head: Uint8Array<ArrayBuffer>; complete: boolean; res: Response }> {
|
||||
const reader = (res.body as ReadableStream<Uint8Array>).getReader();
|
||||
const reader = (res.body as unknown as ReadableStream<Uint8Array>).getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let total = 0;
|
||||
let complete = false;
|
||||
|
||||
@@ -5,7 +5,7 @@ import type { SecretResolver } from "./types.ts";
|
||||
|
||||
export interface SiteAdapterDeps {
|
||||
secrets: SecretResolver;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
/** 同站共享的限流器;不传则按站点定义新建(列表/搜索/详情/下载共享) */
|
||||
limiter?: SiteRateLimiter;
|
||||
now?: () => Date;
|
||||
|
||||
@@ -6,7 +6,7 @@ import { SiteError } from "./types.ts";
|
||||
|
||||
export interface SiteHttpContext {
|
||||
siteId: string;
|
||||
fetch: typeof globalThis.fetch;
|
||||
fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
limiter: SiteRateLimiter;
|
||||
timeoutMs: number;
|
||||
userAgent?: string;
|
||||
|
||||
@@ -53,7 +53,7 @@ export interface NexusLoginInput {
|
||||
/** base32 TOTP 密钥 */
|
||||
totpSecret?: string | null;
|
||||
userAgent: string;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
limiter?: { acquire(signal?: AbortSignal): Promise<void> };
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
|
||||
@@ -38,7 +38,7 @@ function adapter(body: string, status = 200, headers: Record<string, string> = {
|
||||
new Response(body, {
|
||||
status,
|
||||
headers: { "content-type": "text/html; charset=utf-8", ...headers },
|
||||
})) as typeof fetch;
|
||||
})) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
const a = createSpiderSiteAdapter(audiencesDefinition, {
|
||||
baseUrl: "https://aud.test/",
|
||||
secrets: staticSecretResolver({ [audiencesDefinition.id]: { cookie: "c_secure_uid=1; c_secure_pass=abc" } }),
|
||||
|
||||
@@ -176,7 +176,7 @@ describe("详情页扩展内容(憨憨 2026 真实结构:分节 div,无 #kdescr)
|
||||
paths.push(`${u.pathname}${u.search}`);
|
||||
const body = u.pathname === "/viewfilelist.php" ? HHANCLUB_FILELIST_FRAGMENT : HHANCLUB_DETAIL_PAGE_2026;
|
||||
return new Response(body, { status: 200, headers: { "content-type": "text/html; charset=utf-8" } });
|
||||
}) as typeof globalThis.fetch;
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
const limiter = createSiteRateLimiter({ minIntervalMs: 0 });
|
||||
const a = createSpiderSiteAdapter(hhanclubDefinition, {
|
||||
fetch,
|
||||
|
||||
@@ -30,7 +30,7 @@ export function testDef(
|
||||
export function adapterWith(
|
||||
def: SpecializedSiteDefinition,
|
||||
spider: SpecializedSpider,
|
||||
fetchImpl: typeof globalThis.fetch,
|
||||
fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
|
||||
secrets: Partial<Record<SiteCredentialKind, string>> = { cookie: "c=COOKIESECRET", apiKey: "APIKEYSECRET" },
|
||||
): SiteAdapter {
|
||||
return createSpecializedSiteAdapter(def, spider, {
|
||||
@@ -42,10 +42,14 @@ export function adapterWith(
|
||||
|
||||
/** 包一层 fetch:命中 predicate 的请求返回合法 .torrent 字节 */
|
||||
export function withTorrent(
|
||||
inner: typeof globalThis.fetch,
|
||||
inner: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
|
||||
predicate: (url: URL) => boolean,
|
||||
name = "demo",
|
||||
): { fetch: typeof globalThis.fetch; infoHash: string; hits: Array<{ url: URL; headers: Headers }> } {
|
||||
): {
|
||||
fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
infoHash: string;
|
||||
hits: Array<{ url: URL; headers: Headers }>;
|
||||
} {
|
||||
const t = buildFakeTorrent({ name });
|
||||
const hits: Array<{ url: URL; headers: Headers }> = [];
|
||||
const f = (async (input: string | URL | Request, init?: RequestInit) => {
|
||||
@@ -58,7 +62,7 @@ export function withTorrent(
|
||||
});
|
||||
}
|
||||
return inner(input, init);
|
||||
}) as typeof globalThis.fetch;
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
return { fetch: f, infoHash: t.infoHash, hits };
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import type { SiteUserInfo, SiteUserParserOptions } from "./types.ts";
|
||||
|
||||
export async function runParser(
|
||||
cls: SiteParserClass,
|
||||
opts: Partial<SiteUserParserOptions> & { fetch: typeof globalThis.fetch },
|
||||
opts: Partial<SiteUserParserOptions> & { fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> },
|
||||
): Promise<SiteUserInfo> {
|
||||
const p = new cls({ siteName: "demo", url: "https://pt.example.com/", ...opts });
|
||||
await p.parse();
|
||||
|
||||
@@ -92,7 +92,7 @@ export interface SiteUserParserOptions {
|
||||
apiUrl?: string | null;
|
||||
/** 是否读取未读消息正文(MP 全局设置 `SITE_MESSAGE`);默认 false。注意 MTorrent 读取后会标记已读 */
|
||||
fetchMessages?: boolean;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
/** 单请求超时(MP 固定 60s) */
|
||||
timeoutMs?: number;
|
||||
/** 共享站点限流器(与 list/search 共用) */
|
||||
|
||||
@@ -119,7 +119,7 @@ export interface TmdbConfig {
|
||||
* 语义对照 MP @cached(skip_none、业务失败不缓存、results 为空按 emptyTtlMs)。
|
||||
*/
|
||||
sharedCache?: TmdbSharedCache;
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
@@ -185,7 +185,7 @@ export class TmdbClient {
|
||||
private readonly langs: string[];
|
||||
private readonly cache: TtlLruCache<unknown>;
|
||||
private readonly limiter: RequestLimiter;
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly fetchImpl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
|
||||
constructor(private readonly cfg: TmdbConfig) {
|
||||
if (!cfg.apiKey && !cfg.accessToken) throw new TypeError("TMDB apiKey or accessToken required");
|
||||
|
||||
@@ -219,7 +219,7 @@ describe("TmdbClient 缓存/限流/错误分类", () => {
|
||||
const e1 = await dead.details("movie", 1).catch((x) => x);
|
||||
expect(e1.kind).toBe("network");
|
||||
// 模拟把 URL(含 api_key 参数与编码后的明文)拼进信息的 fetch 实现
|
||||
const leaky: typeof fetch = async (input) => {
|
||||
const leaky: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response> = async (input) => {
|
||||
throw new TypeError(`error sending request for url (${String(input)}): key ${encodeURIComponent(KEY)}`);
|
||||
};
|
||||
const e2 = await createTmdbClient({ baseUrl: "http://tmdb.local/3", apiKey: KEY, retries: 0, fetch: leaky })
|
||||
|
||||
@@ -16,7 +16,7 @@ export interface BraveClientOptions {
|
||||
baseUrl?: string;
|
||||
/** 缺省 10s */
|
||||
timeoutMs?: number;
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
}
|
||||
|
||||
export interface BraveSearchParams {
|
||||
|
||||
@@ -18,7 +18,7 @@ export interface FetchPageOptions {
|
||||
/** 最多跟随几次重定向,缺省 3 */
|
||||
maxRedirects?: number;
|
||||
userAgent?: string;
|
||||
fetch?: typeof fetch;
|
||||
fetch?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
resolve?: HostResolver;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ describe("Brave 客户端", () => {
|
||||
fetch: (async (url: string, init: RequestInit) => {
|
||||
seen = { url, headers: new Headers(init.headers) };
|
||||
return jsonRes(BRAVE_BODY);
|
||||
}) as typeof fetch,
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
|
||||
});
|
||||
const r = await c.search({
|
||||
q: "MoviePilot v3",
|
||||
@@ -85,8 +85,10 @@ describe("Brave 客户端", () => {
|
||||
])("HTTP %i → %s,错误信息不含 Key", async (status, kind) => {
|
||||
const c = createBraveClient({
|
||||
apiKey: KEY,
|
||||
fetch: (async () =>
|
||||
jsonRes({ error: `bad token ${KEY}` }, status, { "x-ratelimit-reset": "1, 1000" })) as typeof fetch,
|
||||
fetch: (async () => jsonRes({ error: `bad token ${KEY}` }, status, { "x-ratelimit-reset": "1, 1000" })) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
});
|
||||
const err = (await c.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError;
|
||||
expect(err).toBeInstanceOf(WebSearchError);
|
||||
@@ -100,7 +102,7 @@ describe("Brave 客户端", () => {
|
||||
apiKey: KEY,
|
||||
fetch: (async () => {
|
||||
throw new TypeError(`connect fail ${KEY}`);
|
||||
}) as typeof fetch,
|
||||
}) as (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
|
||||
});
|
||||
const e1 = (await net.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError;
|
||||
expect(e1.kind).toBe("network");
|
||||
@@ -109,9 +111,10 @@ describe("Brave 客户端", () => {
|
||||
apiKey: KEY,
|
||||
timeoutMs: 50,
|
||||
fetch: ((_u: string, init: RequestInit) =>
|
||||
new Promise((_r, rej) =>
|
||||
init.signal?.addEventListener("abort", () => rej(new Error("aborted"))),
|
||||
)) as typeof fetch,
|
||||
new Promise((_r, rej) => init.signal?.addEventListener("abort", () => rej(new Error("aborted"))))) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
});
|
||||
expect(((await slow.search({ q: "x", count: 1 }).catch((e) => e)) as WebSearchError).kind).toBe("timeout");
|
||||
});
|
||||
@@ -164,8 +167,10 @@ describe("读取网页", () => {
|
||||
it("HTML 转纯文本:去脚本样式表单,保留标题与段落,URL 凭证脱敏", async () => {
|
||||
const p = await fetchPage("https://example.org/notice", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () =>
|
||||
new Response(html, { headers: { "content-type": "text/html; charset=utf-8" } })) as typeof fetch,
|
||||
fetch: (async () => new Response(html, { headers: { "content-type": "text/html; charset=utf-8" } })) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
});
|
||||
expect(p.title).toBe("站点公告");
|
||||
expect(p.text).toContain("9 月 20 日维护。");
|
||||
@@ -180,7 +185,10 @@ describe("读取网页", () => {
|
||||
const big = "a".repeat(600 * 1024);
|
||||
const p = await fetchPage("https://example.org/big.txt", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () => new Response(big, { headers: { "content-type": "text/plain" } })) as typeof fetch,
|
||||
fetch: (async () => new Response(big, { headers: { "content-type": "text/plain" } })) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
});
|
||||
expect(p.bytes).toBe(512 * 1024);
|
||||
expect(p.truncated).toBe(true);
|
||||
@@ -190,22 +198,28 @@ describe("读取网页", () => {
|
||||
it("非文本类型拒绝;重定向到内网拒绝;超时", async () => {
|
||||
const pdf = await fetchPage("https://example.org/a.pdf", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () => new Response("%PDF", { headers: { "content-type": "application/pdf" } })) as typeof fetch,
|
||||
fetch: (async () => new Response("%PDF", { headers: { "content-type": "application/pdf" } })) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
}).catch((e) => e);
|
||||
expect((pdf as WebSearchError).kind).toBe("unsupported");
|
||||
const redir = await fetchPage("https://example.org/r", {
|
||||
resolve: publicDns,
|
||||
fetch: (async () =>
|
||||
new Response(null, { status: 302, headers: { location: "http://192.168.1.1/admin" } })) as typeof fetch,
|
||||
fetch: (async () => new Response(null, { status: 302, headers: { location: "http://192.168.1.1/admin" } })) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
}).catch((e) => e);
|
||||
expect((redir as WebSearchError).kind).toBe("blocked");
|
||||
const slow = await fetchPage("https://example.org/slow", {
|
||||
resolve: publicDns,
|
||||
timeoutMs: 50,
|
||||
fetch: ((_u: string, init: RequestInit) =>
|
||||
new Promise((_r, rej) =>
|
||||
init.signal?.addEventListener("abort", () => rej(new Error("aborted"))),
|
||||
)) as typeof fetch,
|
||||
new Promise((_r, rej) => init.signal?.addEventListener("abort", () => rej(new Error("aborted"))))) as (
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => Promise<Response>,
|
||||
}).catch((e) => e);
|
||||
expect((slow as WebSearchError).kind).toBe("timeout");
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user