feat(sandbox): 不可信字节在沙箱子进程里解析(撤销权限、超时与内存强杀);种子、文件预览、字幕解析迁入
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
1a76d2d700
commit
eeec91e892
21 files changed
+632
-178
No files matched your search
@@ -0,0 +1,162 @@
|
||||
// 沙箱调用(docs/adr/ADR-005-sandbox.md):每次一个子进程,权限全部撤销;超时 SIGKILL,子进程实际内存(RSS)超限 SIGKILL。
|
||||
// 起子进程方式同 fsproxy:deno CLI → `deno run worker.ts`(开发 / 测试);编译二进制 → `<自身> --role sandbox`。
|
||||
// 起不来(权限 / 运行时不支持)时退回进程内执行并记日志:仍有超时,但卡死的同步代码无法收回(只在本机调试时出现)。
|
||||
import { basename } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { SANDBOX_JOBS, type SandboxInput, type SandboxJobName, type SandboxOutput } from "./jobs.ts";
|
||||
|
||||
export type SandboxErrorKind = "timeout" | "memory" | "crash" | "job" | "spawn";
|
||||
|
||||
export class SandboxError extends Error {
|
||||
constructor(
|
||||
readonly kind: SandboxErrorKind,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "SandboxError";
|
||||
}
|
||||
}
|
||||
|
||||
export interface SandboxOptions {
|
||||
/** 缺省 20 s */
|
||||
timeoutMs?: number;
|
||||
/** 子进程内存上限,缺省 512 MB */
|
||||
memoryMb?: number;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
const WORKER_PATH = fileURLToPath(new URL("./worker.ts", import.meta.url));
|
||||
let inProcessReason: string | null = null;
|
||||
const stats = { spawned: 0, timeouts: 0, memoryKills: 0, crashes: 0 };
|
||||
|
||||
export function sandboxStats() {
|
||||
return { ...stats, inProcessReason };
|
||||
}
|
||||
|
||||
function command(memoryMb: number): { cmd: string; args: string[] } {
|
||||
const exe = Deno.execPath();
|
||||
if (/^deno(\.exe)?$/.test(basename(exe)))
|
||||
return {
|
||||
cmd: exe,
|
||||
args: [
|
||||
"run",
|
||||
"-A",
|
||||
"--quiet",
|
||||
"--no-prompt", // V8 堆上限只作兜底(设为两倍):主要靠下面的 RSS 看门狗,编译产物不认 V8 参数,两种模式一致
|
||||
`--v8-flags=--max-old-space-size=${memoryMb * 2}`,
|
||||
WORKER_PATH,
|
||||
],
|
||||
};
|
||||
return { cmd: exe, args: ["--role", "sandbox"] };
|
||||
}
|
||||
|
||||
/** 子进程实际内存(KB);读不到返回 null */
|
||||
async function rssKb(pid: number): Promise<number | null> {
|
||||
try {
|
||||
const s = await Deno.readTextFile(`/proc/${pid}/status`);
|
||||
const m = /VmRSS:\s+(\d+)\s+kB/.exec(s);
|
||||
return m ? Number(m[1]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function runSandbox<J extends SandboxJobName>(
|
||||
job: J,
|
||||
input: SandboxInput<J>,
|
||||
opts: SandboxOptions = {},
|
||||
): Promise<SandboxOutput<J>> {
|
||||
const timeoutMs = opts.timeoutMs ?? 20_000;
|
||||
const memoryMb = opts.memoryMb ?? 512;
|
||||
opts.signal?.throwIfAborted();
|
||||
if (inProcessReason) return runInProcess(job, input, timeoutMs) as Promise<SandboxOutput<J>>;
|
||||
const { cmd, args } = command(memoryMb);
|
||||
let child: Deno.ChildProcess;
|
||||
try {
|
||||
child = new Deno.Command(cmd, {
|
||||
args,
|
||||
stdin: "piped",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
env: {},
|
||||
clearEnv: true,
|
||||
}).spawn();
|
||||
} catch (e) {
|
||||
inProcessReason = (e as Error).message;
|
||||
console.warn(`[sandbox] 起不来子进程,退回进程内执行:${inProcessReason}`);
|
||||
return runInProcess(job, input, timeoutMs) as Promise<SandboxOutput<J>>;
|
||||
}
|
||||
stats.spawned += 1;
|
||||
let killedFor: SandboxErrorKind | null = null;
|
||||
const kill = (why: SandboxErrorKind) => {
|
||||
if (killedFor) return;
|
||||
killedFor = why;
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
// 已退出
|
||||
}
|
||||
};
|
||||
const timer = setTimeout(() => kill("timeout"), timeoutMs);
|
||||
const watch = setInterval(async () => {
|
||||
const kb = await rssKb(child.pid);
|
||||
if (kb !== null && kb > memoryMb * 1024) kill("memory");
|
||||
}, 100);
|
||||
const onAbort = () => kill("crash");
|
||||
opts.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
try {
|
||||
const w = child.stdin.getWriter();
|
||||
await w.write(new TextEncoder().encode(JSON.stringify({ job, input }))).catch(() => undefined);
|
||||
await w.close().catch(() => undefined);
|
||||
const [status, stdout, stderr] = await Promise.all([
|
||||
child.status,
|
||||
new Response(child.stdout).text(),
|
||||
new Response(child.stderr).text(),
|
||||
]);
|
||||
if (killedFor === "timeout") {
|
||||
stats.timeouts += 1;
|
||||
throw new SandboxError("timeout", `沙箱任务 ${job} 超过 ${timeoutMs} ms 未完成，已强制结束`);
|
||||
}
|
||||
if (killedFor === "memory" || /out of memory|heap limit/i.test(stderr)) {
|
||||
stats.memoryKills += 1;
|
||||
throw new SandboxError("memory", `沙箱任务 ${job} 内存超过 ${memoryMb} MB，已强制结束`);
|
||||
}
|
||||
opts.signal?.throwIfAborted();
|
||||
const line = stdout.trim().split("\n").pop() ?? "";
|
||||
let msg: { ok: boolean; result?: unknown; error?: string };
|
||||
try {
|
||||
msg = JSON.parse(line);
|
||||
} catch {
|
||||
stats.crashes += 1;
|
||||
throw new SandboxError("crash", `沙箱任务 ${job} 异常退出(code ${status.code}):${stderr.trim().slice(-300)}`);
|
||||
}
|
||||
if (!msg.ok) throw new SandboxError("job", msg.error ?? "沙箱任务失败");
|
||||
return msg.result as SandboxOutput<J>;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
clearInterval(watch);
|
||||
opts.signal?.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
|
||||
async function runInProcess<J extends SandboxJobName>(
|
||||
job: J,
|
||||
input: SandboxInput<J>,
|
||||
timeoutMs: number,
|
||||
): Promise<unknown> {
|
||||
const fn = SANDBOX_JOBS[job] as (i: SandboxInput<J>) => unknown;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
Promise.resolve().then(() => fn(input)),
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new SandboxError("timeout", `沙箱任务 ${job} 超时(进程内)`)), timeoutMs);
|
||||
}),
|
||||
]);
|
||||
} catch (e) {
|
||||
if (e instanceof SandboxError) throw e;
|
||||
throw new SandboxError("job", `${(e as Error).name}: ${(e as Error).message}`);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { BencodeError, setTorrentParser, type TorrentMeta } from "@umt/adapters";
|
||||
import { runSandbox, SandboxError } from "./client.ts";
|
||||
import { b64encode } from "./jobs.ts";
|
||||
|
||||
/** 常驻角色启动时调用:adapters 里的不可信字节解析改走沙箱子进程(docs/adr/ADR-005-sandbox.md) */
|
||||
export function installSandboxParsers(): void {
|
||||
setTorrentParser(async (bytes) => {
|
||||
try {
|
||||
return (await runSandbox("torrent.parse", { base64: b64encode(bytes) }, { timeoutMs: 15_000 })) as TorrentMeta;
|
||||
} catch (e) {
|
||||
// 解析失败、超时、内存超限都按「不是有效种子」处理,调用方据此拒收
|
||||
if (e instanceof SandboxError) throw new BencodeError(e.message);
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// 沙箱任务表(docs/adr/ADR-005-sandbox.md):只放「外部不可信字节 → 结构化数据」的纯函数。
|
||||
// 子进程加载完本表后撤销全部权限(文件 / 网络 / 环境变量 / 子进程),所以任务里不能有任何 I/O;
|
||||
// 输入输出必须能 JSON 化(二进制用 base64)。新增任务:在这里加一项,调用方用 runSandbox(名字, 输入)。
|
||||
import { parseTorrent } from "@umt/adapters";
|
||||
import { decodeSubtitle, detectSubtitleLanguage, subtitleEpisode, subtitleExtOf } from "@umt/domain";
|
||||
import { unzipSync } from "fflate";
|
||||
import { buildPreview } from "../modules/file-manager/preview-build.ts";
|
||||
|
||||
const MAX_SUBTITLE_BYTES = 10 * 1024 * 1024;
|
||||
const MAX_ARCHIVE_ENTRIES = 300;
|
||||
|
||||
export const b64encode = (bytes: Uint8Array) => {
|
||||
let s = "";
|
||||
for (let i = 0; i < bytes.length; i += 0x8000) s += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
|
||||
return btoa(s);
|
||||
};
|
||||
export const b64decode = (s: string) => Uint8Array.from(atob(s), (c) => c.charCodeAt(0));
|
||||
|
||||
export interface ParsedSubtitleFile {
|
||||
name: string;
|
||||
archive: string | null;
|
||||
ext: string;
|
||||
language: string;
|
||||
season: number | null;
|
||||
seasonInferred: boolean;
|
||||
episode: number | null;
|
||||
encoding: string;
|
||||
sizeBytes: number;
|
||||
/** 转成 UTF-8 后的内容(base64) */
|
||||
content: string;
|
||||
}
|
||||
|
||||
/** 字幕 / zip:解压、解码转 UTF-8、识别语言与季集 */
|
||||
function subtitleParse(input: { files: { name: string; base64: string }[] }) {
|
||||
const items: ParsedSubtitleFile[] = [];
|
||||
const skipped: { name: string; reason: string }[] = [];
|
||||
const one = (name: string, bytes: Uint8Array, archive: string | null) => {
|
||||
const ext = subtitleExtOf(name);
|
||||
if (!ext) return skipped.push({ name, reason: "不是字幕文件" });
|
||||
if (bytes.length > MAX_SUBTITLE_BYTES) return skipped.push({ name, reason: "文件超过 10 MB" });
|
||||
const { text, encoding } = decodeSubtitle(bytes);
|
||||
const ep = subtitleEpisode(name);
|
||||
items.push({
|
||||
name,
|
||||
archive,
|
||||
ext,
|
||||
language: detectSubtitleLanguage(name, text),
|
||||
season: ep.season,
|
||||
seasonInferred: ep.seasonInferred,
|
||||
episode: ep.episode,
|
||||
encoding,
|
||||
sizeBytes: bytes.length,
|
||||
content: b64encode(new TextEncoder().encode(text)),
|
||||
});
|
||||
};
|
||||
for (const f of input.files) {
|
||||
let bytes: Uint8Array;
|
||||
try {
|
||||
bytes = b64decode(f.base64);
|
||||
} catch {
|
||||
skipped.push({ name: f.name, reason: "内容不是有效的 base64" });
|
||||
continue;
|
||||
}
|
||||
if (/\.zip$/i.test(f.name)) {
|
||||
let entries: Record<string, Uint8Array>;
|
||||
try {
|
||||
let n = 0;
|
||||
entries = unzipSync(bytes, {
|
||||
filter: (e) =>
|
||||
!e.name.endsWith("/") &&
|
||||
!!subtitleExtOf(e.name) &&
|
||||
e.originalSize <= MAX_SUBTITLE_BYTES &&
|
||||
++n <= MAX_ARCHIVE_ENTRIES,
|
||||
});
|
||||
} catch {
|
||||
skipped.push({ name: f.name, reason: "压缩包损坏或加密" });
|
||||
continue;
|
||||
}
|
||||
const names = Object.keys(entries);
|
||||
if (!names.length) skipped.push({ name: f.name, reason: "压缩包里没有字幕文件" });
|
||||
for (const n of names) one(n, entries[n]!, f.name);
|
||||
} else if (/\.(rar|7z)$/i.test(f.name)) {
|
||||
skipped.push({ name: f.name, reason: "暂不支持 rar / 7z，请解压后上传或改用 zip" });
|
||||
} else one(f.name, bytes, null);
|
||||
}
|
||||
return { items, skipped };
|
||||
}
|
||||
|
||||
/** 种子文件:info hash、名称、文件清单(TorrentMeta,可 JSON 化) */
|
||||
function torrentParse(input: { base64: string }) {
|
||||
return parseTorrent(b64decode(input.base64));
|
||||
}
|
||||
|
||||
/** 测试用:按指令卡死 / 吃内存 / 抛错,验证父进程的超时强杀与内存上限 */
|
||||
function selfTest(input: { mode: "ok" | "hang" | "memory" | "throw" | "io" }) {
|
||||
if (input.mode === "hang") for (;;) {}
|
||||
if (input.mode === "memory") {
|
||||
const keep: Uint8Array[] = [];
|
||||
for (;;) keep.push(new Uint8Array(16 * 1024 * 1024).fill(1));
|
||||
}
|
||||
if (input.mode === "throw") throw new Error("任务自己抛的错");
|
||||
if (input.mode === "io") return { read: Deno.readTextFileSync("/etc/hostname") };
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
export const SANDBOX_JOBS = {
|
||||
"subtitle.parse": subtitleParse,
|
||||
"torrent.parse": torrentParse,
|
||||
"file.preview": buildPreview,
|
||||
"self.test": selfTest,
|
||||
} as const;
|
||||
|
||||
export type SandboxJobName = keyof typeof SANDBOX_JOBS;
|
||||
export type SandboxInput<J extends SandboxJobName> = Parameters<(typeof SANDBOX_JOBS)[J]>[0];
|
||||
export type SandboxOutput<J extends SandboxJobName> = Awaited<ReturnType<(typeof SANDBOX_JOBS)[J]>>;
|
||||
@@ -0,0 +1,32 @@
|
||||
// 沙箱子进程入口:先加载任务表,再撤销全部权限,然后从 stdin 读一条 {job, input},把 {ok, result | error} 写到 stdout。
|
||||
// 开发 / 测试由 deno run 直接跑本文件;编译二进制以 `<自身> --role sandbox` 拉起(main.ts 在加载其余模块前分派)。
|
||||
import { SANDBOX_JOBS } from "./jobs.ts";
|
||||
|
||||
const PERMISSIONS = ["read", "write", "net", "env", "run", "ffi", "sys"] as const;
|
||||
|
||||
async function writeAll(bytes: Uint8Array) {
|
||||
let off = 0;
|
||||
while (off < bytes.length) off += await Deno.stdout.write(bytes.subarray(off));
|
||||
}
|
||||
|
||||
export async function sandboxWorkerMain(): Promise<void> {
|
||||
for (const name of PERMISSIONS) {
|
||||
try {
|
||||
Deno.permissions.revokeSync({ name } as Deno.PermissionDescriptor);
|
||||
} catch {
|
||||
// 运行时不认识的权限名:跳过
|
||||
}
|
||||
}
|
||||
let out: unknown;
|
||||
try {
|
||||
const req = JSON.parse(await new Response(Deno.stdin.readable).text()) as { job: string; input: unknown };
|
||||
const fn = (SANDBOX_JOBS as Record<string, (i: unknown) => unknown>)[req.job];
|
||||
if (!fn) throw new Error(`没有这个沙箱任务:${req.job}`);
|
||||
out = { ok: true, result: await fn(req.input) };
|
||||
} catch (e) {
|
||||
out = { ok: false, error: `${(e as Error).name}: ${(e as Error).message}`.slice(0, 2000) };
|
||||
}
|
||||
await writeAll(new TextEncoder().encode(`${JSON.stringify(out)}\n`));
|
||||
}
|
||||
|
||||
if (import.meta.main) await sandboxWorkerMain();
|
||||
Reference in new issue
Block a user