feat(sandbox): 不可信字节在沙箱子进程里解析(撤销权限、超时与内存强杀);种子、文件预览、字幕解析迁入
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
1a76d2d700
commit
eeec91e892
21 files changed
+632
-178
No files matched your search
@@ -12,6 +12,8 @@ import type { MappingExport } from "./ops.ts";
|
||||
// (逐行事务、幂等可重跑;步骤与回滚见 docs/runbook/auth.md)
|
||||
// fsworker fsproxy 执行子进程(编译二进制里由 fsproxy 以
|
||||
// `<自身> --role fsworker` 拉起,stdin 一条请求)
|
||||
// sandbox 沙箱子进程(src/sandbox:撤销全部权限后解析不可信
|
||||
// 字节,stdin 一条请求,stdout 一条结果)
|
||||
const Roles = [
|
||||
"api",
|
||||
"worker",
|
||||
@@ -24,6 +26,7 @@ const Roles = [
|
||||
"read-only-on",
|
||||
"rotate-master-key",
|
||||
"fsworker",
|
||||
"sandbox",
|
||||
] as const;
|
||||
type Role = (typeof Roles)[number];
|
||||
const OneShot = ["migrate", "preflight", "export-mappings", "read-only-on", "rotate-master-key"] as const;
|
||||
@@ -53,6 +56,13 @@ if (role === "fsworker") {
|
||||
Deno.exit(0);
|
||||
}
|
||||
|
||||
// 沙箱子进程:同样在加载其余模块前分派,只加载沙箱任务表
|
||||
if (role === "sandbox") {
|
||||
const { sandboxWorkerMain } = await import("./sandbox/worker.ts");
|
||||
await sandboxWorkerMain();
|
||||
Deno.exit(0);
|
||||
}
|
||||
|
||||
// 健康探测:只打本机回环端口,不需要完整配置(HEALTHCHECK 用,镜像里不装 curl)
|
||||
if (role === "probe") {
|
||||
const port = Deno.env.get("PORT") ?? "4380";
|
||||
@@ -78,6 +88,9 @@ const { diffCounts, enableReadOnlyReconcile, exportMappings, writeMappingExport
|
||||
const { auditStorageWarnings, checkStorage, loadMountedPaths, runPreflight } = await import("./preflight.ts");
|
||||
const { startWorker } = await import("./worker/index.ts");
|
||||
const { laneOverridesFromEnv } = await import("./worker/lanes.ts");
|
||||
const { installSandboxParsers } = await import("./sandbox/install.ts");
|
||||
// 不可信字节(种子等)的解析改走沙箱子进程;一次性子命令也装上,无副作用
|
||||
installSandboxParsers();
|
||||
|
||||
let config: Config;
|
||||
try {
|
||||
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
InMemoryProfileLock,
|
||||
imageDimensions,
|
||||
type PageContent,
|
||||
parseTorrent,
|
||||
parseTorrentIsolated,
|
||||
prepareImage,
|
||||
profileKeyFor,
|
||||
type TabRef,
|
||||
@@ -2237,7 +2237,7 @@ export class BrowserService {
|
||||
};
|
||||
try {
|
||||
const bytes = new Uint8Array(await readFile(this.stagedPath(artifact.bytes.ref)));
|
||||
const meta = parseTorrent(bytes);
|
||||
const meta = await parseTorrentIsolated(bytes);
|
||||
verification = {
|
||||
valid: true,
|
||||
infohashV1: meta.infohashV1,
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
// 文件预览的纯计算(沙箱任务 file.preview):文件头字节 → 预览结构;读文件在主进程(preview.ts)。
|
||||
import { type BencodeValue, bdecode, parseTorrent } from "@umt/adapters";
|
||||
import type { FilePreview } from "@umt/contracts";
|
||||
import {
|
||||
CUE_LIMIT,
|
||||
decodeText,
|
||||
IMAGE_MIME,
|
||||
imageSize,
|
||||
maskTracker,
|
||||
parseAss,
|
||||
parseSrtLike,
|
||||
parseSup,
|
||||
} from "./preview-parse.ts";
|
||||
|
||||
export const TORRENT_FILES = 500;
|
||||
/** 不是有效 .torrent 时抛出的错误信息(主进程据此转成 400) */
|
||||
export const BAD_TORRENT = "BAD_TORRENT";
|
||||
|
||||
const str = (v: BencodeValue | undefined) => (v instanceof Uint8Array ? new TextDecoder().decode(v) : null);
|
||||
|
||||
function trackersOf(root: { [k: string]: BencodeValue }): string[] {
|
||||
const out: string[] = [];
|
||||
const tiers = root["announce-list"];
|
||||
if (Array.isArray(tiers))
|
||||
for (const tier of tiers) for (const t of Array.isArray(tier) ? tier : [tier]) out.push(str(t) ?? "");
|
||||
out.push(str(root.announce) ?? "");
|
||||
return [...new Set(out.filter(Boolean).map(maskTracker))];
|
||||
}
|
||||
|
||||
export interface PreviewInput {
|
||||
kind: "image" | "torrent" | "subtitle" | "text";
|
||||
ext: string;
|
||||
/** 文件头字节(base64) */
|
||||
base64: string;
|
||||
/** 文件实际大小 */
|
||||
size: number;
|
||||
}
|
||||
|
||||
export function buildPreview(input: PreviewInput): FilePreview {
|
||||
const { kind, ext, size } = input;
|
||||
const buf = Uint8Array.from(atob(input.base64), (c) => c.charCodeAt(0));
|
||||
if (kind === "image") {
|
||||
const dim = imageSize(buf);
|
||||
return {
|
||||
type: "image",
|
||||
size,
|
||||
mime: IMAGE_MIME[ext] ?? "application/octet-stream",
|
||||
width: dim?.width ?? null,
|
||||
height: dim?.height ?? null,
|
||||
};
|
||||
}
|
||||
if (kind === "torrent") {
|
||||
let meta: ReturnType<typeof parseTorrent>;
|
||||
let root: { [k: string]: BencodeValue };
|
||||
try {
|
||||
meta = parseTorrent(buf);
|
||||
root = bdecode(buf) as { [k: string]: BencodeValue };
|
||||
} catch {
|
||||
throw new Error(BAD_TORRENT);
|
||||
}
|
||||
const info = root.info as { [k: string]: BencodeValue } | undefined;
|
||||
const created = typeof root["creation date"] === "number" ? root["creation date"] : null;
|
||||
return {
|
||||
type: "torrent",
|
||||
size,
|
||||
name: meta.name,
|
||||
hash: meta.hash,
|
||||
infohashV1: meta.infohashV1,
|
||||
infohashV2: meta.infohashV2,
|
||||
private: meta.private,
|
||||
totalSize: meta.totalSize,
|
||||
fileCount: meta.files.length,
|
||||
files: meta.files.slice(0, TORRENT_FILES).map((f) => ({ path: f.path, length: Math.max(0, f.length) })),
|
||||
trackers: trackersOf(root),
|
||||
comment: str(root["comment.utf-8"]) ?? str(root.comment),
|
||||
createdBy: str(root["created by"]),
|
||||
createdAt: created && created > 0 ? new Date(created * 1000).toISOString() : null,
|
||||
pieceLength: typeof info?.["piece length"] === "number" ? info["piece length"] : null,
|
||||
};
|
||||
}
|
||||
const truncated = size > buf.length;
|
||||
if (kind === "subtitle") {
|
||||
if (ext === "sup") {
|
||||
const all = parseSup(buf);
|
||||
return {
|
||||
type: "subtitle",
|
||||
size,
|
||||
format: "sup",
|
||||
encoding: null,
|
||||
cues: all.slice(0, CUE_LIMIT),
|
||||
parsed: all.length,
|
||||
truncated,
|
||||
title: null,
|
||||
};
|
||||
}
|
||||
const { text, encoding } = decodeText(buf, truncated);
|
||||
const ass = ext === "ass" || ext === "ssa";
|
||||
const parsed = ass ? parseAss(text) : { cues: parseSrtLike(text), title: null };
|
||||
const cues = [...parsed.cues].sort((a, b) => a.start - b.start);
|
||||
return {
|
||||
type: "subtitle",
|
||||
size,
|
||||
format: ext as "srt" | "vtt" | "ass" | "ssa",
|
||||
encoding,
|
||||
cues: cues.slice(0, CUE_LIMIT),
|
||||
parsed: cues.length,
|
||||
truncated,
|
||||
title: parsed.title,
|
||||
};
|
||||
}
|
||||
const { text, encoding } = decodeText(buf, truncated);
|
||||
return { type: "text", size, format: ext, encoding, text, truncated };
|
||||
}
|
||||
@@ -1,21 +1,12 @@
|
||||
import { open } from "node:fs/promises";
|
||||
import { basename } from "node:path";
|
||||
import { BencodeError, type BencodeValue, bdecode, parseTorrent } from "@umt/adapters";
|
||||
import type { FilePreview, FilesPreviewQuery } from "@umt/contracts";
|
||||
import type { Principal } from "../../http/auth.ts";
|
||||
import { runSandbox, SandboxError } from "../../sandbox/client.ts";
|
||||
import { b64encode } from "../../sandbox/jobs.ts";
|
||||
import { FilesError, type FmDeps, isAdminSession, recheck, resolveIn } from "./common.ts";
|
||||
import {
|
||||
CUE_LIMIT,
|
||||
decodeText,
|
||||
extOf,
|
||||
IMAGE_MIME,
|
||||
imageSize,
|
||||
maskTracker,
|
||||
parseAss,
|
||||
parseSrtLike,
|
||||
parseSup,
|
||||
previewKindOf,
|
||||
} from "./preview-parse.ts";
|
||||
import { BAD_TORRENT } from "./preview-build.ts";
|
||||
import { extOf, previewKindOf } from "./preview-parse.ts";
|
||||
|
||||
// GET /files/preview(FM-2):仅管理员会话;路径守卫同下载(realpath 在根内、符号链接目标也须在根内,打开前复核)。
|
||||
// 只读文件头:文本/字幕 ≤ 1 MiB、图片头 512 KiB;.torrent 整个读但上限 4 MiB。只回 JSON,不回原始内容。
|
||||
@@ -23,7 +14,6 @@ import {
|
||||
export const TEXT_PREVIEW_MAX = 1024 * 1024;
|
||||
const IMAGE_HEAD = 512 * 1024;
|
||||
const TORRENT_MAX = 4 * 1024 * 1024;
|
||||
const TORRENT_FILES = 500;
|
||||
|
||||
async function readHead(abs: string, max: number): Promise<{ buf: Uint8Array; size: number }> {
|
||||
const fh = await open(abs, "r");
|
||||
@@ -44,17 +34,6 @@ async function readHead(abs: string, max: number): Promise<{ buf: Uint8Array; si
|
||||
}
|
||||
}
|
||||
|
||||
const str = (v: BencodeValue | undefined) => (v instanceof Uint8Array ? new TextDecoder().decode(v) : null);
|
||||
|
||||
function trackersOf(root: { [k: string]: BencodeValue }): string[] {
|
||||
const out: string[] = [];
|
||||
const tiers = root["announce-list"];
|
||||
if (Array.isArray(tiers))
|
||||
for (const tier of tiers) for (const t of Array.isArray(tier) ? tier : [tier]) out.push(str(t) ?? "");
|
||||
out.push(str(root.announce) ?? "");
|
||||
return [...new Set(out.filter(Boolean).map(maskTracker))];
|
||||
}
|
||||
|
||||
export async function previewFile(
|
||||
d: FmDeps,
|
||||
principal: Principal | undefined,
|
||||
@@ -69,82 +48,21 @@ export async function previewFile(
|
||||
if (!kind) throw new FilesError("forbidden", `不支持预览 .${ext || "无扩展名"} 文件`);
|
||||
const real = await recheck(r.abs, r.rootReal);
|
||||
|
||||
if (kind === "image") {
|
||||
const { buf, size } = await readHead(real, IMAGE_HEAD);
|
||||
const dim = imageSize(buf);
|
||||
return {
|
||||
type: "image",
|
||||
size,
|
||||
mime: IMAGE_MIME[ext] ?? "application/octet-stream",
|
||||
width: dim?.width ?? null,
|
||||
height: dim?.height ?? null,
|
||||
};
|
||||
const head = kind === "image" ? IMAGE_HEAD : kind === "torrent" ? TORRENT_MAX : TEXT_PREVIEW_MAX;
|
||||
const { buf, size } = await readHead(real, head);
|
||||
if (kind === "torrent" && size > TORRENT_MAX)
|
||||
throw new FilesError("too_large", `.torrent 文件 ${size} 字节,超过预览上限 4 MiB`);
|
||||
// 文件内容不可信:解析放进沙箱子进程(docs/adr/ADR-005-sandbox.md)
|
||||
try {
|
||||
return (await runSandbox(
|
||||
"file.preview",
|
||||
{ kind, ext, base64: b64encode(buf), size },
|
||||
{ timeoutMs: 15_000 },
|
||||
)) as FilePreview;
|
||||
} catch (e) {
|
||||
if (e instanceof SandboxError && e.kind === "job" && e.message.includes(BAD_TORRENT))
|
||||
throw new FilesError("bad_request", "不是有效的 .torrent 文件");
|
||||
if (e instanceof SandboxError) throw new FilesError("bad_request", `预览失败:${e.message}`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (kind === "torrent") {
|
||||
const { buf, size } = await readHead(real, TORRENT_MAX);
|
||||
if (size > TORRENT_MAX) throw new FilesError("too_large", `.torrent 文件 ${size} 字节,超过预览上限 4 MiB`);
|
||||
try {
|
||||
const meta = parseTorrent(buf);
|
||||
const root = bdecode(buf) as { [k: string]: BencodeValue };
|
||||
const info = root.info as { [k: string]: BencodeValue } | undefined;
|
||||
const created = typeof root["creation date"] === "number" ? root["creation date"] : null;
|
||||
return {
|
||||
type: "torrent",
|
||||
size,
|
||||
name: meta.name,
|
||||
hash: meta.hash,
|
||||
infohashV1: meta.infohashV1,
|
||||
infohashV2: meta.infohashV2,
|
||||
private: meta.private,
|
||||
totalSize: meta.totalSize,
|
||||
fileCount: meta.files.length,
|
||||
files: meta.files.slice(0, TORRENT_FILES).map((f) => ({ path: f.path, length: Math.max(0, f.length) })),
|
||||
trackers: trackersOf(root),
|
||||
comment: str(root["comment.utf-8"]) ?? str(root.comment),
|
||||
createdBy: str(root["created by"]),
|
||||
createdAt: created && created > 0 ? new Date(created * 1000).toISOString() : null,
|
||||
pieceLength: typeof info?.["piece length"] === "number" ? info["piece length"] : null,
|
||||
};
|
||||
} catch (e) {
|
||||
if (e instanceof BencodeError) throw new FilesError("bad_request", "不是有效的 .torrent 文件");
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
const { buf, size } = await readHead(real, TEXT_PREVIEW_MAX);
|
||||
const truncated = size > buf.length;
|
||||
|
||||
if (kind === "subtitle") {
|
||||
if (ext === "sup") {
|
||||
const all = parseSup(buf);
|
||||
return {
|
||||
type: "subtitle",
|
||||
size,
|
||||
format: "sup",
|
||||
encoding: null,
|
||||
cues: all.slice(0, CUE_LIMIT),
|
||||
parsed: all.length,
|
||||
truncated,
|
||||
title: null,
|
||||
};
|
||||
}
|
||||
const { text, encoding } = decodeText(buf, truncated);
|
||||
const ass = ext === "ass" || ext === "ssa";
|
||||
const parsed = ass ? parseAss(text) : { cues: parseSrtLike(text), title: null };
|
||||
const cues = [...parsed.cues].sort((a, b) => a.start - b.start);
|
||||
return {
|
||||
type: "subtitle",
|
||||
size,
|
||||
format: ext as "srt" | "vtt" | "ass" | "ssa",
|
||||
encoding,
|
||||
cues: cues.slice(0, CUE_LIMIT),
|
||||
parsed: cues.length,
|
||||
truncated,
|
||||
title: parsed.title,
|
||||
};
|
||||
}
|
||||
|
||||
const { text, encoding } = decodeText(buf, truncated);
|
||||
return { type: "text", size, format: ext, encoding, text, truncated };
|
||||
}
|
||||
@@ -14,32 +14,22 @@ import type {
|
||||
import type { SubtitleWriteTable } from "@umt/db";
|
||||
import {
|
||||
type DefaultSubtitleLang,
|
||||
decodeSubtitle,
|
||||
detectSubtitleLanguage,
|
||||
isSidecarOf,
|
||||
pairSubtitle,
|
||||
type SubtitleExt,
|
||||
subtitleEpisode,
|
||||
subtitleExtOf,
|
||||
subtitleFileName,
|
||||
} from "@umt/domain";
|
||||
import { unzipSync } from "fflate";
|
||||
import type { Selectable } from "kysely";
|
||||
import { dataPaths } from "../../config.ts";
|
||||
import { ApiException } from "../../http/errors.ts";
|
||||
import { runSandbox, SandboxError } from "../../sandbox/client.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
// 补字幕(docs/plan/subtitles.md;对照 MP 插件「海拉鲁字幕大师」upload / matching / subtitle_writer):
|
||||
// 解析(编码转 UTF-8、语言、季集)→ 配对到作品的视频 → 写到视频同目录(主干名 + 语言后缀)→ 登记字幕资产 + 历史,可撤销。
|
||||
|
||||
const MAX_FILE_BYTES = 10 * 1024 * 1024;
|
||||
const MAX_ARCHIVE_ENTRIES = 300;
|
||||
|
||||
const b64 = (bytes: Uint8Array) => {
|
||||
let s = "";
|
||||
for (let i = 0; i < bytes.length; i += 0x8000) s += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
|
||||
return btoa(s);
|
||||
};
|
||||
const unb64 = (s: string) => Uint8Array.from(atob(s), (c) => c.charCodeAt(0));
|
||||
const fileName = (p: string) => p.replace(/\\/g, "/").split("/").pop() ?? p;
|
||||
|
||||
@@ -133,60 +123,25 @@ export async function parseSubtitles(
|
||||
episode: t.episode,
|
||||
}))
|
||||
: [];
|
||||
const items: SubtitleParsedItem[] = [];
|
||||
const skipped: SubtitleParseResponse["skipped"] = [];
|
||||
const one = (name: string, bytes: Uint8Array, archive: string | null) => {
|
||||
const ext = subtitleExtOf(name);
|
||||
if (!ext) return skipped.push({ name, reason: "不是字幕文件" });
|
||||
if (bytes.length > MAX_FILE_BYTES) return skipped.push({ name, reason: "文件超过 10 MB" });
|
||||
const { text, encoding } = decodeSubtitle(bytes);
|
||||
const ep = subtitleEpisode(name);
|
||||
items.push({
|
||||
id: crypto.randomUUID(),
|
||||
name,
|
||||
archive,
|
||||
ext,
|
||||
language: detectSubtitleLanguage(name, text) as SubtitleLanguage,
|
||||
season: ep.season,
|
||||
seasonInferred: ep.seasonInferred,
|
||||
episode: ep.episode,
|
||||
encoding,
|
||||
sizeBytes: bytes.length,
|
||||
content: b64(new TextEncoder().encode(text)),
|
||||
targetLocationId: targets.length ? pairSubtitle(ep, targets) : null,
|
||||
});
|
||||
};
|
||||
for (const f of files) {
|
||||
let bytes: Uint8Array;
|
||||
try {
|
||||
bytes = unb64(f.base64);
|
||||
} catch {
|
||||
skipped.push({ name: f.name, reason: "内容不是有效的 base64" });
|
||||
continue;
|
||||
}
|
||||
if (/\.zip$/i.test(f.name)) {
|
||||
let entries: Record<string, Uint8Array>;
|
||||
try {
|
||||
let n = 0;
|
||||
entries = unzipSync(bytes, {
|
||||
filter: (e) =>
|
||||
!e.name.endsWith("/") &&
|
||||
!!subtitleExtOf(e.name) &&
|
||||
e.originalSize <= MAX_FILE_BYTES &&
|
||||
++n <= MAX_ARCHIVE_ENTRIES,
|
||||
});
|
||||
} catch {
|
||||
skipped.push({ name: f.name, reason: "压缩包损坏或加密" });
|
||||
continue;
|
||||
}
|
||||
const names = Object.keys(entries);
|
||||
if (!names.length) skipped.push({ name: f.name, reason: "压缩包里没有字幕文件" });
|
||||
for (const n of names) one(n, entries[n]!, f.name);
|
||||
} else if (/\.(rar|7z)$/i.test(f.name)) {
|
||||
skipped.push({ name: f.name, reason: "暂不支持 rar / 7z,请解压后上传或改用 zip" });
|
||||
} else one(f.name, bytes, null);
|
||||
// 上传 / 下载的字幕与压缩包不可信:解压、解码、识别都在沙箱子进程里做(docs/adr/ADR-005-sandbox.md)
|
||||
let parsed: Awaited<ReturnType<typeof runSandbox<"subtitle.parse">>>;
|
||||
try {
|
||||
parsed = await runSandbox("subtitle.parse", { files }, { timeoutMs: 30_000 });
|
||||
} catch (e) {
|
||||
if (e instanceof SandboxError) throw new ApiException("bad_request", `字幕解析失败:${e.message}`);
|
||||
throw e;
|
||||
}
|
||||
return { items, skipped };
|
||||
const items: SubtitleParsedItem[] = parsed.items.map((it) => {
|
||||
const ep = { season: it.season, seasonInferred: it.seasonInferred, episode: it.episode };
|
||||
return {
|
||||
...it,
|
||||
id: crypto.randomUUID(),
|
||||
ext: it.ext as SubtitleExt,
|
||||
language: it.language as SubtitleLanguage,
|
||||
targetLocationId: targets.length ? pairSubtitle(ep, targets) : null,
|
||||
};
|
||||
});
|
||||
return { items, skipped: parsed.skipped };
|
||||
}
|
||||
|
||||
// ── 写入 ──
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
// 沙箱调用(docs/adr/ADR-005-sandbox.md):每次一个子进程,权限全部撤销;超时 SIGKILL,子进程实际内存(RSS)超限 SIGKILL。
|
||||
// 起子进程方式同 fsproxy:deno CLI → `deno run worker.ts`(开发 / 测试);编译二进制 → `<自身> --role sandbox`。
|
||||
// 起不来(权限 / 运行时不支持)时退回进程内执行并记日志:仍有超时,但卡死的同步代码无法收回(只在本机调试时出现)。
|
||||
import { basename } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { SANDBOX_JOBS, type SandboxInput, type SandboxJobName, type SandboxOutput } from "./jobs.ts";
|
||||
|
||||
export type SandboxErrorKind = "timeout" | "memory" | "crash" | "job" | "spawn";
|
||||
|
||||
export class SandboxError extends Error {
|
||||
constructor(
|
||||
readonly kind: SandboxErrorKind,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "SandboxError";
|
||||
}
|
||||
}
|
||||
|
||||
export interface SandboxOptions {
|
||||
/** 缺省 20 s */
|
||||
timeoutMs?: number;
|
||||
/** 子进程内存上限,缺省 512 MB */
|
||||
memoryMb?: number;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
const WORKER_PATH = fileURLToPath(new URL("./worker.ts", import.meta.url));
|
||||
let inProcessReason: string | null = null;
|
||||
const stats = { spawned: 0, timeouts: 0, memoryKills: 0, crashes: 0 };
|
||||
|
||||
export function sandboxStats() {
|
||||
return { ...stats, inProcessReason };
|
||||
}
|
||||
|
||||
function command(memoryMb: number): { cmd: string; args: string[] } {
|
||||
const exe = Deno.execPath();
|
||||
if (/^deno(\.exe)?$/.test(basename(exe)))
|
||||
return {
|
||||
cmd: exe,
|
||||
args: [
|
||||
"run",
|
||||
"-A",
|
||||
"--quiet",
|
||||
"--no-prompt", // V8 堆上限只作兜底(设为两倍):主要靠下面的 RSS 看门狗,编译产物不认 V8 参数,两种模式一致
|
||||
`--v8-flags=--max-old-space-size=${memoryMb * 2}`,
|
||||
WORKER_PATH,
|
||||
],
|
||||
};
|
||||
return { cmd: exe, args: ["--role", "sandbox"] };
|
||||
}
|
||||
|
||||
/** 子进程实际内存(KB);读不到返回 null */
|
||||
async function rssKb(pid: number): Promise<number | null> {
|
||||
try {
|
||||
const s = await Deno.readTextFile(`/proc/${pid}/status`);
|
||||
const m = /VmRSS:\s+(\d+)\s+kB/.exec(s);
|
||||
return m ? Number(m[1]) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function runSandbox<J extends SandboxJobName>(
|
||||
job: J,
|
||||
input: SandboxInput<J>,
|
||||
opts: SandboxOptions = {},
|
||||
): Promise<SandboxOutput<J>> {
|
||||
const timeoutMs = opts.timeoutMs ?? 20_000;
|
||||
const memoryMb = opts.memoryMb ?? 512;
|
||||
opts.signal?.throwIfAborted();
|
||||
if (inProcessReason) return runInProcess(job, input, timeoutMs) as Promise<SandboxOutput<J>>;
|
||||
const { cmd, args } = command(memoryMb);
|
||||
let child: Deno.ChildProcess;
|
||||
try {
|
||||
child = new Deno.Command(cmd, {
|
||||
args,
|
||||
stdin: "piped",
|
||||
stdout: "piped",
|
||||
stderr: "piped",
|
||||
env: {},
|
||||
clearEnv: true,
|
||||
}).spawn();
|
||||
} catch (e) {
|
||||
inProcessReason = (e as Error).message;
|
||||
console.warn(`[sandbox] 起不来子进程,退回进程内执行:${inProcessReason}`);
|
||||
return runInProcess(job, input, timeoutMs) as Promise<SandboxOutput<J>>;
|
||||
}
|
||||
stats.spawned += 1;
|
||||
let killedFor: SandboxErrorKind | null = null;
|
||||
const kill = (why: SandboxErrorKind) => {
|
||||
if (killedFor) return;
|
||||
killedFor = why;
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
// 已退出
|
||||
}
|
||||
};
|
||||
const timer = setTimeout(() => kill("timeout"), timeoutMs);
|
||||
const watch = setInterval(async () => {
|
||||
const kb = await rssKb(child.pid);
|
||||
if (kb !== null && kb > memoryMb * 1024) kill("memory");
|
||||
}, 100);
|
||||
const onAbort = () => kill("crash");
|
||||
opts.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
try {
|
||||
const w = child.stdin.getWriter();
|
||||
await w.write(new TextEncoder().encode(JSON.stringify({ job, input }))).catch(() => undefined);
|
||||
await w.close().catch(() => undefined);
|
||||
const [status, stdout, stderr] = await Promise.all([
|
||||
child.status,
|
||||
new Response(child.stdout).text(),
|
||||
new Response(child.stderr).text(),
|
||||
]);
|
||||
if (killedFor === "timeout") {
|
||||
stats.timeouts += 1;
|
||||
throw new SandboxError("timeout", `沙箱任务 ${job} 超过 ${timeoutMs} ms 未完成,已强制结束`);
|
||||
}
|
||||
if (killedFor === "memory" || /out of memory|heap limit/i.test(stderr)) {
|
||||
stats.memoryKills += 1;
|
||||
throw new SandboxError("memory", `沙箱任务 ${job} 内存超过 ${memoryMb} MB,已强制结束`);
|
||||
}
|
||||
opts.signal?.throwIfAborted();
|
||||
const line = stdout.trim().split("\n").pop() ?? "";
|
||||
let msg: { ok: boolean; result?: unknown; error?: string };
|
||||
try {
|
||||
msg = JSON.parse(line);
|
||||
} catch {
|
||||
stats.crashes += 1;
|
||||
throw new SandboxError("crash", `沙箱任务 ${job} 异常退出(code ${status.code}):${stderr.trim().slice(-300)}`);
|
||||
}
|
||||
if (!msg.ok) throw new SandboxError("job", msg.error ?? "沙箱任务失败");
|
||||
return msg.result as SandboxOutput<J>;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
clearInterval(watch);
|
||||
opts.signal?.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
|
||||
async function runInProcess<J extends SandboxJobName>(
|
||||
job: J,
|
||||
input: SandboxInput<J>,
|
||||
timeoutMs: number,
|
||||
): Promise<unknown> {
|
||||
const fn = SANDBOX_JOBS[job] as (i: SandboxInput<J>) => unknown;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
Promise.resolve().then(() => fn(input)),
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new SandboxError("timeout", `沙箱任务 ${job} 超时(进程内)`)), timeoutMs);
|
||||
}),
|
||||
]);
|
||||
} catch (e) {
|
||||
if (e instanceof SandboxError) throw e;
|
||||
throw new SandboxError("job", `${(e as Error).name}: ${(e as Error).message}`);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { BencodeError, setTorrentParser, type TorrentMeta } from "@umt/adapters";
|
||||
import { runSandbox, SandboxError } from "./client.ts";
|
||||
import { b64encode } from "./jobs.ts";
|
||||
|
||||
/** 常驻角色启动时调用:adapters 里的不可信字节解析改走沙箱子进程(docs/adr/ADR-005-sandbox.md) */
|
||||
export function installSandboxParsers(): void {
|
||||
setTorrentParser(async (bytes) => {
|
||||
try {
|
||||
return (await runSandbox("torrent.parse", { base64: b64encode(bytes) }, { timeoutMs: 15_000 })) as TorrentMeta;
|
||||
} catch (e) {
|
||||
// 解析失败、超时、内存超限都按「不是有效种子」处理,调用方据此拒收
|
||||
if (e instanceof SandboxError) throw new BencodeError(e.message);
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// 沙箱任务表(docs/adr/ADR-005-sandbox.md):只放「外部不可信字节 → 结构化数据」的纯函数。
|
||||
// 子进程加载完本表后撤销全部权限(文件 / 网络 / 环境变量 / 子进程),所以任务里不能有任何 I/O;
|
||||
// 输入输出必须能 JSON 化(二进制用 base64)。新增任务:在这里加一项,调用方用 runSandbox(名字, 输入)。
|
||||
import { parseTorrent } from "@umt/adapters";
|
||||
import { decodeSubtitle, detectSubtitleLanguage, subtitleEpisode, subtitleExtOf } from "@umt/domain";
|
||||
import { unzipSync } from "fflate";
|
||||
import { buildPreview } from "../modules/file-manager/preview-build.ts";
|
||||
|
||||
const MAX_SUBTITLE_BYTES = 10 * 1024 * 1024;
|
||||
const MAX_ARCHIVE_ENTRIES = 300;
|
||||
|
||||
export const b64encode = (bytes: Uint8Array) => {
|
||||
let s = "";
|
||||
for (let i = 0; i < bytes.length; i += 0x8000) s += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
|
||||
return btoa(s);
|
||||
};
|
||||
export const b64decode = (s: string) => Uint8Array.from(atob(s), (c) => c.charCodeAt(0));
|
||||
|
||||
export interface ParsedSubtitleFile {
|
||||
name: string;
|
||||
archive: string | null;
|
||||
ext: string;
|
||||
language: string;
|
||||
season: number | null;
|
||||
seasonInferred: boolean;
|
||||
episode: number | null;
|
||||
encoding: string;
|
||||
sizeBytes: number;
|
||||
/** 转成 UTF-8 后的内容(base64) */
|
||||
content: string;
|
||||
}
|
||||
|
||||
/** 字幕 / zip:解压、解码转 UTF-8、识别语言与季集 */
|
||||
function subtitleParse(input: { files: { name: string; base64: string }[] }) {
|
||||
const items: ParsedSubtitleFile[] = [];
|
||||
const skipped: { name: string; reason: string }[] = [];
|
||||
const one = (name: string, bytes: Uint8Array, archive: string | null) => {
|
||||
const ext = subtitleExtOf(name);
|
||||
if (!ext) return skipped.push({ name, reason: "不是字幕文件" });
|
||||
if (bytes.length > MAX_SUBTITLE_BYTES) return skipped.push({ name, reason: "文件超过 10 MB" });
|
||||
const { text, encoding } = decodeSubtitle(bytes);
|
||||
const ep = subtitleEpisode(name);
|
||||
items.push({
|
||||
name,
|
||||
archive,
|
||||
ext,
|
||||
language: detectSubtitleLanguage(name, text),
|
||||
season: ep.season,
|
||||
seasonInferred: ep.seasonInferred,
|
||||
episode: ep.episode,
|
||||
encoding,
|
||||
sizeBytes: bytes.length,
|
||||
content: b64encode(new TextEncoder().encode(text)),
|
||||
});
|
||||
};
|
||||
for (const f of input.files) {
|
||||
let bytes: Uint8Array;
|
||||
try {
|
||||
bytes = b64decode(f.base64);
|
||||
} catch {
|
||||
skipped.push({ name: f.name, reason: "内容不是有效的 base64" });
|
||||
continue;
|
||||
}
|
||||
if (/\.zip$/i.test(f.name)) {
|
||||
let entries: Record<string, Uint8Array>;
|
||||
try {
|
||||
let n = 0;
|
||||
entries = unzipSync(bytes, {
|
||||
filter: (e) =>
|
||||
!e.name.endsWith("/") &&
|
||||
!!subtitleExtOf(e.name) &&
|
||||
e.originalSize <= MAX_SUBTITLE_BYTES &&
|
||||
++n <= MAX_ARCHIVE_ENTRIES,
|
||||
});
|
||||
} catch {
|
||||
skipped.push({ name: f.name, reason: "压缩包损坏或加密" });
|
||||
continue;
|
||||
}
|
||||
const names = Object.keys(entries);
|
||||
if (!names.length) skipped.push({ name: f.name, reason: "压缩包里没有字幕文件" });
|
||||
for (const n of names) one(n, entries[n]!, f.name);
|
||||
} else if (/\.(rar|7z)$/i.test(f.name)) {
|
||||
skipped.push({ name: f.name, reason: "暂不支持 rar / 7z,请解压后上传或改用 zip" });
|
||||
} else one(f.name, bytes, null);
|
||||
}
|
||||
return { items, skipped };
|
||||
}
|
||||
|
||||
/** 种子文件:info hash、名称、文件清单(TorrentMeta,可 JSON 化) */
|
||||
function torrentParse(input: { base64: string }) {
|
||||
return parseTorrent(b64decode(input.base64));
|
||||
}
|
||||
|
||||
/** 测试用:按指令卡死 / 吃内存 / 抛错,验证父进程的超时强杀与内存上限 */
|
||||
function selfTest(input: { mode: "ok" | "hang" | "memory" | "throw" | "io" }) {
|
||||
if (input.mode === "hang") for (;;) {}
|
||||
if (input.mode === "memory") {
|
||||
const keep: Uint8Array[] = [];
|
||||
for (;;) keep.push(new Uint8Array(16 * 1024 * 1024).fill(1));
|
||||
}
|
||||
if (input.mode === "throw") throw new Error("任务自己抛的错");
|
||||
if (input.mode === "io") return { read: Deno.readTextFileSync("/etc/hostname") };
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
export const SANDBOX_JOBS = {
|
||||
"subtitle.parse": subtitleParse,
|
||||
"torrent.parse": torrentParse,
|
||||
"file.preview": buildPreview,
|
||||
"self.test": selfTest,
|
||||
} as const;
|
||||
|
||||
export type SandboxJobName = keyof typeof SANDBOX_JOBS;
|
||||
export type SandboxInput<J extends SandboxJobName> = Parameters<(typeof SANDBOX_JOBS)[J]>[0];
|
||||
export type SandboxOutput<J extends SandboxJobName> = Awaited<ReturnType<(typeof SANDBOX_JOBS)[J]>>;
|
||||
@@ -0,0 +1,32 @@
|
||||
// 沙箱子进程入口:先加载任务表,再撤销全部权限,然后从 stdin 读一条 {job, input},把 {ok, result | error} 写到 stdout。
|
||||
// 开发 / 测试由 deno run 直接跑本文件;编译二进制以 `<自身> --role sandbox` 拉起(main.ts 在加载其余模块前分派)。
|
||||
import { SANDBOX_JOBS } from "./jobs.ts";
|
||||
|
||||
const PERMISSIONS = ["read", "write", "net", "env", "run", "ffi", "sys"] as const;
|
||||
|
||||
async function writeAll(bytes: Uint8Array) {
|
||||
let off = 0;
|
||||
while (off < bytes.length) off += await Deno.stdout.write(bytes.subarray(off));
|
||||
}
|
||||
|
||||
export async function sandboxWorkerMain(): Promise<void> {
|
||||
for (const name of PERMISSIONS) {
|
||||
try {
|
||||
Deno.permissions.revokeSync({ name } as Deno.PermissionDescriptor);
|
||||
} catch {
|
||||
// 运行时不认识的权限名:跳过
|
||||
}
|
||||
}
|
||||
let out: unknown;
|
||||
try {
|
||||
const req = JSON.parse(await new Response(Deno.stdin.readable).text()) as { job: string; input: unknown };
|
||||
const fn = (SANDBOX_JOBS as Record<string, (i: unknown) => unknown>)[req.job];
|
||||
if (!fn) throw new Error(`没有这个沙箱任务:${req.job}`);
|
||||
out = { ok: true, result: await fn(req.input) };
|
||||
} catch (e) {
|
||||
out = { ok: false, error: `${(e as Error).name}: ${(e as Error).message}`.slice(0, 2000) };
|
||||
}
|
||||
await writeAll(new TextEncoder().encode(`${JSON.stringify(out)}\n`));
|
||||
}
|
||||
|
||||
if (import.meta.main) await sandboxWorkerMain();
|
||||
@@ -0,0 +1,51 @@
|
||||
import { buildFakeTorrent } from "@umt/testkit";
|
||||
import { zipSync } from "fflate";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { runSandbox, SandboxError } from "../../src/sandbox/client.ts";
|
||||
import { b64encode } from "../../src/sandbox/jobs.ts";
|
||||
|
||||
// 沙箱(docs/adr/ADR-005-sandbox.md):子进程撤销全部权限;卡死超时强杀;内存超限强杀;任务错误原样带回
|
||||
const enc = (s: string) => new TextEncoder().encode(s);
|
||||
|
||||
describe("沙箱子进程", () => {
|
||||
it("正常返回;子进程里读文件被拒绝", async () => {
|
||||
expect(await runSandbox("self.test", { mode: "ok" })).toEqual({ ok: true });
|
||||
await expect(runSandbox("self.test", { mode: "io" })).rejects.toMatchObject({ kind: "job", message: /NotCapable/ });
|
||||
});
|
||||
|
||||
it("卡死:超时后强杀,不拖住调用方", async () => {
|
||||
const t0 = Date.now();
|
||||
const e = await runSandbox("self.test", { mode: "hang" }, { timeoutMs: 800 }).catch((x) => x);
|
||||
expect(e).toBeInstanceOf(SandboxError);
|
||||
expect(e.kind).toBe("timeout");
|
||||
expect(Date.now() - t0).toBeLessThan(5000);
|
||||
});
|
||||
|
||||
it("吃内存:超过上限强杀", async () => {
|
||||
const e = await runSandbox("self.test", { mode: "memory" }, { memoryMb: 128, timeoutMs: 20_000 }).catch((x) => x);
|
||||
expect(e).toBeInstanceOf(SandboxError);
|
||||
expect(e.kind).toBe("memory");
|
||||
});
|
||||
|
||||
it("任务自己抛错:kind = job,带原因", async () => {
|
||||
await expect(runSandbox("self.test", { mode: "throw" })).rejects.toMatchObject({
|
||||
kind: "job",
|
||||
message: /任务自己抛的错/,
|
||||
});
|
||||
});
|
||||
|
||||
it("字幕:zip 解压、语言与季集;种子:文件清单", async () => {
|
||||
const zip = zipSync({ "a/Show.S01E03.chs.srt": enc("1\n00:00:01,000 --> 00:00:02,000\n你好\n") });
|
||||
const r = await runSandbox("subtitle.parse", { files: [{ name: "x.zip", base64: b64encode(zip) }] });
|
||||
expect(r.items[0]).toMatchObject({
|
||||
name: "a/Show.S01E03.chs.srt",
|
||||
archive: "x.zip",
|
||||
language: "zh-Hans",
|
||||
episode: 3,
|
||||
});
|
||||
const t = buildFakeTorrent({ name: "Show.S01E01.mkv" });
|
||||
const f = await runSandbox("torrent.parse", { base64: b64encode(t.bytes) });
|
||||
expect(f.files.map((x) => x.path)).toEqual(["Show.S01E01.mkv"]);
|
||||
expect(f.hash).toBe(t.infoHash);
|
||||
});
|
||||
});
|
||||
@@ -24,6 +24,8 @@ const UNIT = [
|
||||
];
|
||||
|
||||
const SERIAL = [
|
||||
// 起真实沙箱子进程(内存 / 超时强杀)
|
||||
"test/sandbox/**",
|
||||
"test/e2e/**/*.test.ts",
|
||||
"test/auth-settings/admin-api.test.ts",
|
||||
"test/auth-settings/engine-retry.test.ts",
|
||||
|
||||
Reference in new issue
Block a user