Merge branch 'scr2'

Co-Authored-By: Claude Fable 5.1 <[email protected]>

# Conflicts:
#	CHANGELOG.md
#	apps/server/src/modules/file-manager/routes.ts
This commit is contained in:
root committed 2026-10-01 00:09:01 +08:00
commit 44f7c88be6
41 files changed
+2810 -826

No files matched your search

+26 -4
View File
@@ -22,6 +22,7 @@ import { EventBus } from "../../src/http/sse.ts";
import { createLogger } from "../../src/logger.ts";
import { fakeEngine } from "../../src/modules/acquisition-organize/__tests__/harness.ts";
import { executeRecord, type FrozenPlan } from "../../src/modules/acquisition-organize/services/organize.ts";
import { runScrapeRecord } from "../../src/modules/acquisition-organize/services/scrape.ts";
import { createCapabilityRegistry } from "../../src/modules/agent/capability-registry.ts";
import { createFileManagerModule, type FmOverrides } from "../../src/modules/file-manager/index.ts";
import type { ServerContext } from "../../src/modules/types.ts";
@@ -57,6 +58,17 @@ export async function pgAvailable() {
// biome-ignore lint/suspicious/noExplicitAny: 测试断言直接读响应 JSON
type Loose = any;
/** catalog.lookup 远端候选(测试用) */
export interface RemoteHit {
provider: string;
entityType: "movie" | "tv";
externalId: string;
title: string;
originalTitle?: string | null;
year: number | null;
posterPath?: string | null;
}
export interface FmHarness {
db: Kysely<DB>;
engine: ReturnType<typeof fakeEngine>;
@@ -80,7 +92,11 @@ export async function createFmHarness(
name: string,
overrides: FmOverrides = {},
/** runTransfers(默认 true):假引擎收到 /files 移动/复制的 verifyAndImport 时就地执行(模拟 worker) */
opts: { catalog?: boolean; runTransfers?: boolean } = {},
opts: {
/** true = 内置假 catalog.lookup("Remote Film");函数 = 自定义远端候选(按查询词返回) */
catalog?: boolean | ((query: string) => RemoteHit[]);
runTransfers?: boolean;
} = {},
): Promise<FmHarness> {
const schema = `fm_test_${name}_${process.pid.toString(36)}${Math.random().toString(36).slice(2, 7)}`;
const admin = new pg.Client({ connectionString: DB_URL });
@@ -124,9 +140,12 @@ export async function createFmHarness(
if (opts.catalog)
capabilities.register(capabilitySpec("catalog.lookup"), async (input) => ({
local: [],
remote: String(input.query).includes("Remote")
? [{ provider: "tmdb", entityType: "movie" as const, externalId: "999", title: "Remote Film", year: 2019 }]
: [],
remote:
typeof opts.catalog === "function"
? opts.catalog(String(input.query))
: String(input.query).includes("Remote")
? [{ provider: "tmdb", entityType: "movie" as const, externalId: "999", title: "Remote Film", year: 2019 }]
: [],
}));
const ctx: ServerContext = {
role: "all",
@@ -173,6 +192,9 @@ export async function createFmHarness(
engine.enqueue = async (task, input, o) => {
const r = await enqueue(task, input, o);
const recordId = (input as { recordId?: string }).recordId;
// SCR-2 刮削:模拟 worker 就地执行 scrapeMedia
if (task === "scrapeMedia" && recordId)
setTimeout(() => void runScrapeRecord(module.deps().ao, recordId).catch(() => {}), 0);
if (task === "verifyAndImport" && recordId)
setTimeout(async () => {
try {
@@ -14,7 +14,7 @@ import {
} from "./harness.ts";
// identify(只读,本地 + 远端召回前 5,不落库)、organize(按作品分组走 planImport,dryRun→execute 一致与源变更)、
// probe(已登记写回 media_info)、scrape(仅已登记作品,入队 refreshMetadata;NFO 待 W5)。
// probe(已登记写回 media_info)、刷新元数据(scrape-batch,入队 refreshMetadata)。
const skip = !(await pgAvailable());
@@ -278,7 +278,7 @@ describe.skipIf(skip)("files 识别 / 整理 / 探测 / 刮削", () => {
expect(r.status).toBeLessThan(500);
});
it("probe:已登记资产写回 media_info;scrape 仅已登记作品", async () => {
it("probe:已登记资产写回 media_info;刷新元数据按作品入队", async () => {
await mkdir(join(h.lib, "Blue Film probe"));
await writeFile(join(h.lib, "Blue Film probe", "Blue.Film.2021.mkv"), "x");
const a = await seedAsset(h.db, h, { entityId: film, rel: "Blue Film probe/Blue.Film.2021.mkv" });
@@ -303,21 +303,10 @@ describe.skipIf(skip)("files 识别 / 整理 / 探测 / 刮削", () => {
const u2 = await h.api("POST", "/files/probe", { items: [{ root: h.dlRoot, path: "Remote.Film.2019.1080p.mkv" }] });
expect(u2.body.data.items[0].cached).toBe(true);
const s = await h.api("POST", "/files/scrape", { idempotencyKey: key(), mediaEntityId: film, writeNfo: true });
expect(s.status).toBe(200);
// SCR-1:writeNfo 另排 scrapeMedia(写 NFO 与图片到媒体库目录)
expect(s.body.data).toMatchObject({ mediaEntityId: film, nfo: "queued", images: "not_requested" });
expect(s.body.data.scrapeTaskId).toBeTruthy();
expect(h.engine.enqueued.some((e) => e.task === "refreshMetadata")).toBe(true);
expect(h.engine.enqueued.some((e) => e.task === "scrapeMedia")).toBe(true);
const b = await h.api("POST", "/files/scrape-batch", {
idempotencyKey: key(),
mediaEntityIds: [film],
write: true,
});
// 刷新元数据(scrape-batch):按作品入队 refreshMetadata;刮削(写 NFO)见 scrape.test.ts
const b = await h.api("POST", "/files/scrape-batch", { idempotencyKey: key(), mediaEntityIds: [film] });
expect(b.status).toBe(200);
expect(b.body.data.submitted).toHaveLength(1);
const none = await h.api("POST", "/files/scrape", { idempotencyKey: key(), mediaEntityId: show });
expect(none.status).toBe(400);
expect(h.engine.enqueued.some((e) => e.task === "refreshMetadata")).toBe(true);
});
});
@@ -0,0 +1,443 @@
import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
import { join, relative } from "node:path";
import type { TmdbRichDetail, TmdbRichSeasonDetail } from "@umt/adapters";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { latestEntityScrape, scrapeEntity } from "../../src/modules/acquisition-organize/services/scrape.ts";
import {
createFmHarness,
EXEC_KEY,
type FmHarness,
key,
pgAvailable,
type RemoteHit,
seedAsset,
seedEntity,
} from "./harness.ts";
// SCR-2 刮削照 MP:文件管理任意目录 / 视频文件 → 按路径识别作品(未登记走 TMDB,认不准返回候选)→ 就地写 NFO 与图片。
// 场景:未登记的《人生切割术》S01 两集 + 《星际穿越》目录一次刮削;歧义候选(Solaris 1972 / 2002)选定后刮削并加入媒体库;
// 只补缺失 / 全部覆盖;季目录与单集文件的范围;已登记目录直接用作品;只登记目录与非视频文件跳过。
const skip = !(await pgAvailable());
const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00]);
function detail(
p: Partial<TmdbRichDetail> & Pick<TmdbRichDetail, "entityType" | "externalId" | "title">,
): TmdbRichDetail {
return {
titleLanguage: "zh-CN",
titleZh: p.title,
titleZhLanguage: "zh-CN",
originalTitle: null,
originalLanguage: "en",
overview: null,
overviewLanguage: null,
tagline: null,
year: null,
releaseDate: null,
lastAirDate: null,
genres: [],
voteAverage: null,
voteCount: null,
status: null,
runtime: null,
seasonCount: null,
episodeCount: null,
networks: [],
originCountry: [],
posterPath: null,
backdropPath: null,
cast: [],
crew: [],
externalIds: { imdb: null, tvdb: null, wikidata: null },
translations: [],
alternativeTitles: [],
seasons: [],
languages: ["zh-CN"],
...p,
};
}
const DETAILS: Record<string, TmdbRichDetail> = {
"tv:95396": detail({
entityType: "tv",
externalId: "95396",
title: "人生切割术",
originalTitle: "Severance",
year: 2022,
releaseDate: "2022-02-17",
overview: "马克领导着一个办公室团队。",
overviewLanguage: "zh-CN",
genres: ["剧情", "悬疑"],
voteAverage: 8.4,
networks: ["Apple TV+"],
originCountry: ["US"],
posterPath: "/sev.jpg",
backdropPath: "/sev-bg.jpg",
externalIds: { imdb: "tt11280740", tvdb: "371980", wikidata: null },
}),
"movie:157336": detail({
entityType: "movie",
externalId: "157336",
title: "星际穿越",
originalTitle: "Interstellar",
year: 2014,
releaseDate: "2014-11-05",
runtime: 169,
posterPath: "/inter.jpg",
backdropPath: "/inter-bg.jpg",
crew: [{ id: 525, name: "克里斯托弗·诺兰", originalName: null, role: "Director", profilePath: null }],
}),
"movie:593": detail({
entityType: "movie",
externalId: "593",
title: "飞向太空",
originalTitle: "Solaris",
year: 1972,
}),
"movie:2103": detail({
entityType: "movie",
externalId: "2103",
title: "索拉里斯星",
originalTitle: "Solaris",
year: 2002,
posterPath: "/solaris.jpg",
}),
};
const SEASON1: TmdbRichSeasonDetail = {
tvId: "95396",
seasonNumber: 1,
name: "第 1 季",
nameLanguage: "zh-CN",
overview: null,
overviewLanguage: null,
airDate: "2022-02-17",
posterPath: "/s1.jpg",
fetchedLanguages: ["zh-CN"],
episodes: [
[1, "有关地狱的好消息", "/e1.jpg"],
[2, "半循环", "/e2.jpg"],
].map(([n, name, still]) => ({
seasonNumber: 1,
episodeNumber: n as number,
name: name as string,
nameLanguage: "zh-CN",
overview: `第 ${n} 集简介`,
overviewLanguage: "zh-CN",
airDate: "2022-02-17",
runtime: 55,
stillPath: still as string,
voteAverage: 8,
voteCount: 10,
})),
};
const REMOTE: Record<string, RemoteHit[]> = {
人生切割术: [
{
provider: "tmdb",
entityType: "tv",
externalId: "95396",
title: "人生切割术",
originalTitle: "Severance",
year: 2022,
},
],
星际穿越: [
{
provider: "tmdb",
entityType: "movie",
externalId: "157336",
title: "星际穿越",
originalTitle: "Interstellar",
year: 2014,
},
],
Solaris: [
{
provider: "tmdb",
entityType: "movie",
externalId: "593",
title: "飞向太空",
originalTitle: "Solaris",
year: 1972,
},
{
provider: "tmdb",
entityType: "movie",
externalId: "2103",
title: "索拉里斯星",
originalTitle: "Solaris",
year: 2002,
},
],
};
async function tree(root: string, under = ""): Promise<string[]> {
const out: string[] = [];
const walk = async (d: string) => {
for (const n of (await readdir(d)).sort()) {
if (n.startsWith(".")) continue;
const p = join(d, n);
if ((await stat(p)).isDirectory()) await walk(p);
else out.push(relative(root, p));
}
};
await walk(join(root, under));
return out.sort();
}
const SHOW = "人生切割术 (2022)";
const EP1 = `${SHOW}/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.mkv`;
const EP2 = `${SHOW}/Season 1/人生切割术 - S01E02 - 半循环.mkv`;
const MOVIE = "星际穿越 (2014)";
const MOVIE_FILE = `${MOVIE}/星际穿越 (2014) - 1080p.mkv`;
describe.skipIf(skip)("files 刮削(SCR-2:任意目录就地刮削)", () => {
let h: FmHarness;
const images: string[] = [];
const tmdbCalls: string[] = [];
beforeAll(async () => {
h = await createFmHarness(
"scrape2",
{
ao: {
async scrapeImage(p) {
images.push(p);
return { content: JPEG, mime: "image/jpeg" };
},
async scrapeTmdbDetail(type, id) {
tmdbCalls.push(`${type}:${id}`);
const d = DETAILS[`${type}:${id}`];
if (!d) throw new Error("TMDB 404");
return d;
},
async scrapeTmdbSeason(tvId, n) {
tmdbCalls.push(`season:${tvId}:${n}`);
return tvId === "95396" && n === 1 ? SEASON1 : null;
},
async entityFromTmdb(ref) {
const d = DETAILS[`${ref.type}:${ref.id}`]!;
const e = await h.db
.insertInto("media_entity")
.values({
kind: ref.type === "movie" ? "movie" : "series",
title: d.title,
year: d.year,
status: "active",
})
.returning("id")
.executeTakeFirstOrThrow();
await h.db
.insertInto("external_identity")
.values({
media_entity_id: e.id,
provider: "tmdb",
entity_type: ref.type,
external_id: ref.id,
status: "accepted",
})
.execute();
return e.id;
},
},
},
{
catalog: (q) =>
Object.entries(REMOTE)
.filter(([k]) => q.includes(k) || k.includes(q))
.flatMap(([, v]) => v),
},
);
await mkdir(join(h.lib, SHOW, "Season 1"), { recursive: true });
await mkdir(join(h.lib, MOVIE));
await mkdir(join(h.lib, "Solaris"));
for (const f of [EP1, EP2, MOVIE_FILE, "Solaris/Solaris.1080p.BluRay.x264-GRP.mkv"])
await writeFile(join(h.lib, f), "video-bytes");
await writeFile(join(h.lib, MOVIE, "readme.txt"), "x");
});
afterAll(async () => h?.close());
const scrape = (items: unknown[], extra: Record<string, unknown> = {}, k = EXEC_KEY) =>
h.api("POST", "/files/scrape", { idempotencyKey: key(), items, ...extra }, k);
it("未登记的剧集目录 + 电影目录:按路径识别 TMDB,就地写 NFO 与图片,不建作品", async () => {
const r = await scrape([
{ root: h.libRoot, path: SHOW },
{ root: h.libRoot, path: MOVIE },
]);
expect(r.status).toBe(200);
const data = r.body.data;
expect(data.needsChoice).toEqual([]);
expect(data.pending).toBe(false);
expect(data.records).toHaveLength(2);
const [show, movie] = data.records;
expect(show).toMatchObject({
status: "succeeded",
path: SHOW,
mode: "missing",
trigger: "manual",
match: {
mediaEntityId: null,
provider: "tmdb",
externalId: "95396",
kind: "tv",
source: "auto",
title: "人生切割术",
},
});
expect(movie).toMatchObject({ status: "succeeded", match: { externalId: "157336", kind: "movie" } });
const files = await tree(h.lib);
expect(files).toEqual(
[
`${SHOW}/Season 1/poster.jpg`,
`${SHOW}/Season 1/season.nfo`,
EP1,
EP1.replace(/\.mkv$/, "-thumb.jpg"),
EP1.replace(/\.mkv$/, ".nfo"),
EP2,
EP2.replace(/\.mkv$/, "-thumb.jpg"),
EP2.replace(/\.mkv$/, ".nfo"),
`${SHOW}/fanart.jpg`,
`${SHOW}/poster.jpg`,
`${SHOW}/season01-poster.jpg`,
`${SHOW}/tvshow.nfo`,
`${MOVIE}/fanart.jpg`,
`${MOVIE}/poster.jpg`,
`${MOVIE}/readme.txt`,
MOVIE_FILE,
MOVIE_FILE.replace(/\.mkv$/, ".nfo"),
"Solaris/Solaris.1080p.BluRay.x264-GRP.mkv",
].sort(),
);
// 未登记:不建作品,字段直接取 TMDB
expect(await h.db.selectFrom("media_entity").select("id").execute()).toHaveLength(0);
const tvshow = await readFile(join(h.lib, SHOW, "tvshow.nfo"), "utf8");
expect(tvshow).toContain("<title>人生切割术</title>");
expect(tvshow).toContain('<uniqueid type="imdb" default="true">tt11280740</uniqueid>');
const ep = await readFile(join(h.lib, EP1.replace(/\.mkv$/, ".nfo")), "utf8");
expect(ep).toContain("<title>有关地狱的好消息</title>");
expect(ep).toMatch(/<season>1<\/season>\n {2}<episode>1<\/episode>/);
const nfo = await readFile(join(h.lib, MOVIE_FILE.replace(/\.mkv$/, ".nfo")), "utf8");
expect(nfo).toContain('<director tmdbid="525">克里斯托弗·诺兰</director>');
expect(tmdbCalls).toEqual(expect.arrayContaining(["tv:95396", "season:95396:1", "movie:157336"]));
// 记录可按 id 回查;审计按记录
const back = await h.api("GET", `/files/scrape-records?ids=${show.id},${movie.id}`);
expect(back.body.data.records.map((x: { id: string }) => x.id)).toEqual([show.id, movie.id]);
expect(back.body.data.records[0].files.filter((f: { action: string }) => f.action === "written")).toHaveLength(10);
expect(h.events.some((e) => e.type === "files.changed" && e.payload.rootId === h.libRoot)).toBe(true);
});
it("歧义:返回候选;选定 TMDB 条目后刮削,并按需加入媒体库", async () => {
const first = await scrape([{ root: h.libRoot, path: "Solaris" }]);
expect(first.status).toBe(200);
expect(first.body.data.records).toEqual([]);
expect(first.body.data.needsChoice).toHaveLength(1);
const nc = first.body.data.needsChoice[0];
expect(nc).toMatchObject({ root: h.libRoot, path: "Solaris", title: "Solaris" });
expect(nc.candidates.map((c: { externalId: string }) => c.externalId).sort()).toEqual(["2103", "593"]);
expect(await tree(h.lib, "Solaris")).toEqual(["Solaris/Solaris.1080p.BluRay.x264-GRP.mkv"]);
const second = await scrape(
[{ root: h.libRoot, path: "Solaris", external: { provider: "tmdb", externalId: "2103", kind: "movie" } }],
{ addToLibrary: true },
);
expect(second.status).toBe(200);
const rec = second.body.data.records[0];
expect(rec).toMatchObject({
status: "succeeded",
match: { provider: "tmdb", externalId: "2103", source: "chosen", title: "索拉里斯星", year: 2002 },
});
expect(second.body.data.entities).toMatchObject([{ title: "索拉里斯星", created: true }]);
expect(rec.match.mediaEntityId).toBe(second.body.data.entities[0].mediaEntityId);
expect(await tree(h.lib, "Solaris")).toEqual([
"Solaris/Solaris.1080p.BluRay.x264-GRP.mkv",
"Solaris/Solaris.1080p.BluRay.x264-GRP.nfo",
"Solaris/poster.jpg",
]);
expect(await readFile(join(h.lib, "Solaris/Solaris.1080p.BluRay.x264-GRP.nfo"), "utf8")).toContain(
"<title>索拉里斯星</title>",
);
// 作品详情取的是该作品最近一条刮削记录
expect((await latestEntityScrape(h.module.deps().ao, rec.match.mediaEntityId))?.id).toBe(rec.id);
});
it("只补缺失跳过已有文件(包括别人写的);全部覆盖改写", async () => {
const show = join(h.lib, SHOW, "tvshow.nfo");
await writeFile(show, "<tvshow>别的工具写的</tvshow>");
const miss = await scrape([{ root: h.libRoot, path: SHOW }]);
const rec = miss.body.data.records[0];
expect(rec).toMatchObject({ status: "succeeded", written: 0 });
expect(rec.skipped).toBeGreaterThan(0);
expect(await readFile(show, "utf8")).toBe("<tvshow>别的工具写的</tvshow>");
const over = await scrape([{ root: h.libRoot, path: SHOW }], { mode: "overwrite" });
expect(over.body.data.records[0]).toMatchObject({ status: "succeeded", mode: "overwrite", skipped: 0 });
expect(await readFile(show, "utf8")).toContain("<title>人生切割术</title>");
});
it("季目录只写季与各集;单集文件只写该集", async () => {
const season = await scrape([{ root: h.libRoot, path: `${SHOW}/Season 1` }], { mode: "overwrite" });
const roles = season.body.data.records[0].files.map((f: { path: string; role: string }) => `${f.role}:${f.path}`);
expect(roles.sort()).toEqual(
[
`season_nfo:${SHOW}/Season 1/season.nfo`,
`season_poster:${SHOW}/season01-poster.jpg`,
`season_poster:${SHOW}/Season 1/poster.jpg`,
`episode_nfo:${EP1.replace(/\.mkv$/, ".nfo")}`,
`episode_nfo:${EP2.replace(/\.mkv$/, ".nfo")}`,
`episode_thumb:${EP1.replace(/\.mkv$/, "-thumb.jpg")}`,
`episode_thumb:${EP2.replace(/\.mkv$/, "-thumb.jpg")}`,
].sort(),
);
const one = await scrape([{ root: h.libRoot, path: EP2 }], { mode: "overwrite" });
expect(one.body.data.records[0].files.map((f: { role: string }) => f.role).sort()).toEqual([
"episode_nfo",
"episode_thumb",
]);
});
it("已登记的目录直接用作品;作品详情「重新刮削」对作品目录刮削", async () => {
const film = await seedEntity(h.db, { kind: "movie", title: "星际穿越", year: 2014 });
await h.db
.insertInto("external_identity")
.values({
media_entity_id: film,
provider: "tmdb",
entity_type: "movie",
external_id: "157336",
status: "accepted",
})
.execute();
await seedAsset(h.db, h, { entityId: film, rel: MOVIE_FILE });
const r = await scrape([{ root: h.libRoot, path: MOVIE }]);
expect(r.body.data.records[0].match).toMatchObject({
mediaEntityId: film,
source: "library",
externalId: "157336",
});
// 作品详情「重新刮削」(POST /library/{id}/scrape)的服务端做法
const again = await scrapeEntity(h.module.deps().ao, film, { trigger: "manual", mode: "overwrite" });
expect(again).toHaveLength(1);
const recs = await h.api("GET", `/files/scrape-records?ids=${again[0]!.recordId}`);
expect(recs.body.data.records[0]).toMatchObject({ path: MOVIE, root: h.libRoot, mode: "overwrite" });
});
it("只登记目录与非视频文件跳过;没有视频的目录跳过", async () => {
await mkdir(join(h.lib, "空目录"));
const r = await scrape([
{ root: h.libRoot, path: `${MOVIE}/readme.txt` },
{ root: h.libRoot, path: "空目录" },
]);
expect(r.body.data.skipped).toEqual([
{ root: h.libRoot, path: `${MOVIE}/readme.txt`, reason: "scrape_not_video" },
{ root: h.libRoot, path: "空目录", reason: "scrape_no_videos" },
]);
await h.db.updateTable("library_root").set({ management_mode: "register_only" }).execute();
const ro = await scrape([{ root: h.libRoot, path: MOVIE }]);
expect(ro.body.data.skipped).toEqual([{ root: h.libRoot, path: MOVIE, reason: "scrape_root_read_only" }]);
await h.db.updateTable("library_root").set({ management_mode: "managed" }).execute();
});
});
+3 -3
View File
@@ -277,7 +277,7 @@ describe.skipIf(skip)("files v2(W9-H)", () => {
expect(src.body.data.mediaEntityId).toBe(film);
});
it("G5:批量刷新元数据按作品去重,未登记作品跳过", async () => {
it("G5:批量刷新元数据按作品去重(不要求有已入库文件)", async () => {
const reg = await seedEntity(h.db, { kind: "movie", title: "Registered Film", year: 2018 });
await mkdir(join(h.lib, "Registered Film (2018)"), { recursive: true });
await writeFile(join(h.lib, "Registered Film (2018)", "a.mkv"), big);
@@ -299,8 +299,8 @@ describe.skipIf(skip)("files v2(W9-H)", () => {
EXEC_KEY,
);
expect(r.status).toBe(200);
expect(r.body.data.submitted.map((s: Loose) => s.mediaEntityId)).toEqual([reg]);
expect(r.body.data.skipped).toEqual([{ ref: bare, reason: "not_registered" }]);
expect(r.body.data.submitted.map((s: Loose) => s.mediaEntityId)).toEqual([reg, bare]);
expect(r.body.data.skipped).toEqual([]);
});
it("G8:目录摘要 — 上一级、子目录是否有下级、视频/已登记计数", async () => {