feat(scrape): 覆盖方式加「更新」并设为缺省,季集缺元数据不写占位,作品类型判定统一
- 每次写入在 scrape_record.files[].hash 记 sha256;update 模式只覆盖本工具写过且没被改过的文件, 被改过的跳过(scrape_user_modified),内容没变不重写(scrape_unchanged),旧记录无 hash 按修改时间兜底; 迁移 0070 给 scrape_record.mode 加 update,设置缺省改为 update(旧值照旧有效) - 季 / 集单元缺元数据时按 TMDB id 现取季详情补齐;仍缺跳过(scrape_metadata_missing)并记 partial,不写占位 NFO - scrapeKindOf 统一作品类型判定,合集 / 其他类作品 NFO 根元素为 <movie>;TMDB 映射按作品类型过滤 entity_type - 兜底异常记 scrape_internal_error 并保留已写文件结果 - 新增 scheduleEntityRescrape(刮削过的作品延后排 update 重刮,排队记录合批防抖) Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
00045a24cb
commit
dbbf39eda4
13 files changed
+607
-54
No files matched your search
@@ -0,0 +1,327 @@
|
||||
import { readFile, stat, utimes, writeFile } from "node:fs/promises";
|
||||
import type { TmdbRichSeasonDetail } from "@umt/adapters";
|
||||
import type { ScrapeFileResult, ScrapeMatch } from "@umt/contracts";
|
||||
import { createTempTree, type TempTree } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { queueScrape, runScrapeRecord } from "../../src/modules/acquisition-organize/services/scrape.ts";
|
||||
import { nfoWork } from "../../src/modules/acquisition-organize/services/scrape-input.ts";
|
||||
import { createHarness, type Harness, pgAvailable, seedBase } from "./harness.ts";
|
||||
|
||||
// 刮削准确度:
|
||||
// - 「更新」模式(缺省):不存在的写入;本工具上次写入且没被改过的按新内容覆盖(内容没变不重写);被人改过的跳过
|
||||
// (scrape_user_modified);不是本工具写的不动;旧版本没有 hash 的记录按修改时间兜底;
|
||||
// - 集 / 季缺元数据:先按 TMDB id 现取季详情补齐;取不到跳过(scrape_metadata_missing)、记录 partial,不写占位 NFO;
|
||||
// - 合集 / 其他类作品按电影刮,NFO 根元素是 <movie>;
|
||||
// - 兜底异常记 scrape_internal_error。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const at = new Date().toISOString();
|
||||
const f = (value: unknown, language: string | null = null) => ({ value, source: "tmdb", language, at, locked: false });
|
||||
const enc = (s: string) => new TextEncoder().encode(s);
|
||||
|
||||
function season1(names: Record<number, string>): TmdbRichSeasonDetail {
|
||||
return {
|
||||
tvId: "95396",
|
||||
seasonNumber: 1,
|
||||
name: "第 1 季",
|
||||
nameLanguage: "zh-CN",
|
||||
overview: null,
|
||||
overviewLanguage: null,
|
||||
airDate: "2022-02-17",
|
||||
posterPath: null,
|
||||
fetchedLanguages: ["zh-CN"],
|
||||
episodes: Object.entries(names).map(([n, name]) => ({
|
||||
seasonNumber: 1,
|
||||
episodeNumber: Number(n),
|
||||
name,
|
||||
nameLanguage: "zh-CN",
|
||||
overview: `${name}的简介`,
|
||||
overviewLanguage: "zh-CN",
|
||||
airDate: "2022-02-25",
|
||||
runtime: 50,
|
||||
stillPath: null,
|
||||
voteAverage: null,
|
||||
voteCount: null,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
describe.skipIf(!hasPg)("刮削准确度:更新模式、缺元数据不写占位、作品类型", () => {
|
||||
let h: Harness;
|
||||
let lib: TempTree;
|
||||
let rootId = "";
|
||||
let seasonDetail: (n: number) => Promise<TmdbRichSeasonDetail | null> = async () => null;
|
||||
const seasonCalls: number[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
lib = await createTempTree(
|
||||
{
|
||||
"Movies/星际穿越 (2014)/Interstellar.2014.mkv": 100,
|
||||
"Movies/星际穿越 (2014)/fanart.jpg": "别的工具放的背景图",
|
||||
"TV/人生切割术 (2022)/Season 1/Severance.S01E01.mkv": 100,
|
||||
"TV/人生切割术 (2022)/Season 1/Severance.S01E02.mkv": 100,
|
||||
"Movies/合集 (2020)/Collection.2020.mkv": 100,
|
||||
"Movies/其他 (2021)/Other.2021.mkv": 100,
|
||||
},
|
||||
{ sentinel: true, prefix: "umt-scracc-" },
|
||||
);
|
||||
h = await createHarness("scrape_acc", {
|
||||
scrapeImage: async (p) => ({ content: enc(p), mime: "image/jpeg" }),
|
||||
scrapeTmdbSeason: async (_tv, n) => {
|
||||
seasonCalls.push(n);
|
||||
return seasonDetail(n);
|
||||
},
|
||||
});
|
||||
rootId = (await seedBase(h.db, { libraryPath: lib.root, cachePath: lib.root, qbUrl: "http://127.0.0.1:9" })).rootId;
|
||||
}, 60_000);
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await lib?.cleanup();
|
||||
});
|
||||
|
||||
async function entity(kind: "movie" | "series" | "collection" | "other", title: string, meta: object) {
|
||||
return (
|
||||
await h.db
|
||||
.insertInto("media_entity")
|
||||
.values({ kind, title, year: 2020, status: "active", metadata: JSON.stringify(meta) })
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
}
|
||||
async function scrape(
|
||||
entityId: string,
|
||||
kind: "movie" | "tv",
|
||||
path: string,
|
||||
mode?: "missing" | "overwrite" | "update",
|
||||
externalId: string | null = null,
|
||||
) {
|
||||
const match: ScrapeMatch = {
|
||||
mediaEntityId: entityId,
|
||||
provider: externalId ? "tmdb" : null,
|
||||
externalId,
|
||||
kind,
|
||||
title: "x",
|
||||
year: null,
|
||||
source: "library",
|
||||
};
|
||||
const q = await queueScrape(h.d, {
|
||||
rootId: `lib:${rootId}`,
|
||||
rootPath: lib.root,
|
||||
libraryRootId: rootId,
|
||||
path,
|
||||
match,
|
||||
mode,
|
||||
trigger: "manual",
|
||||
});
|
||||
return runScrapeRecord(h.d, q.recordId);
|
||||
}
|
||||
const fileOf = (files: ScrapeFileResult[], role: string) => files.find((x) => x.role === role);
|
||||
const setTitle = (id: string, title: string) =>
|
||||
h.db
|
||||
.updateTable("media_entity")
|
||||
.set({ metadata: JSON.stringify({ title: f(title), posterPath: f("/inter.jpg") }) })
|
||||
.where("id", "=", id)
|
||||
.execute();
|
||||
|
||||
const MOVIE_DIR = "Movies/星际穿越 (2014)";
|
||||
const MOVIE_NFO = `${MOVIE_DIR}/Interstellar.2014.nfo`;
|
||||
let movie = "";
|
||||
|
||||
it("更新模式:不存在的写入并记 hash;不是本工具写的不动", async () => {
|
||||
movie = await entity("movie", "星际穿越", { title: f("星际穿越"), posterPath: f("/inter.jpg") });
|
||||
const r = await scrape(movie, "movie", MOVIE_DIR);
|
||||
expect(r).toMatchObject({ status: "succeeded", mode: "update" });
|
||||
expect(fileOf(r.files, "movie_nfo")).toMatchObject({
|
||||
action: "written",
|
||||
hash: expect.stringMatching(/^[0-9a-f]{64}$/),
|
||||
});
|
||||
expect(fileOf(r.files, "poster")).toMatchObject({ action: "written" });
|
||||
expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toContain("<title>星际穿越</title>");
|
||||
// fanart.jpg 是别的工具放的:元数据没有背景图,不在目标里;加上背景图后更新模式也不覆盖它
|
||||
await h.db
|
||||
.updateTable("media_entity")
|
||||
.set({
|
||||
metadata: JSON.stringify({ title: f("星际穿越"), posterPath: f("/inter.jpg"), backdropPath: f("/bg.jpg") }),
|
||||
})
|
||||
.where("id", "=", movie)
|
||||
.execute();
|
||||
const again = await scrape(movie, "movie", MOVIE_DIR, "update");
|
||||
expect(fileOf(again.files, "fanart")).toMatchObject({ action: "skipped", reason: "scrape_exists" });
|
||||
expect(await readFile(lib.path(`${MOVIE_DIR}/fanart.jpg`), "utf8")).toBe("别的工具放的背景图");
|
||||
// 内容没变的不重写
|
||||
expect(fileOf(again.files, "movie_nfo")).toMatchObject({ action: "skipped", reason: "scrape_unchanged" });
|
||||
expect(fileOf(again.files, "poster")).toMatchObject({ action: "skipped", reason: "scrape_unchanged" });
|
||||
expect(again.status).toBe("succeeded");
|
||||
});
|
||||
|
||||
it("更新模式:本工具写的、没被改过的按新内容覆盖;被人改过的跳过并记原因", async () => {
|
||||
await setTitle(movie, "星际穿越(新译名)");
|
||||
const upd = await scrape(movie, "movie", MOVIE_DIR, "update");
|
||||
expect(fileOf(upd.files, "movie_nfo")).toMatchObject({ action: "written" });
|
||||
expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toContain("<title>星际穿越(新译名)</title>");
|
||||
|
||||
await writeFile(lib.path(MOVIE_NFO), "<movie><title>我手改的</title></movie>");
|
||||
await setTitle(movie, "星际穿越(再改)");
|
||||
const kept = await scrape(movie, "movie", MOVIE_DIR, "update");
|
||||
expect(fileOf(kept.files, "movie_nfo")).toMatchObject({ action: "skipped", reason: "scrape_user_modified" });
|
||||
expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toBe("<movie><title>我手改的</title></movie>");
|
||||
expect(kept.status).toBe("succeeded");
|
||||
// 之后每次更新都还认得出是人改过的(跳过的结果不带 hash,不会把人改的内容当成本工具写的)
|
||||
const still = await scrape(movie, "movie", MOVIE_DIR, "update");
|
||||
expect(fileOf(still.files, "movie_nfo")).toMatchObject({ action: "skipped", reason: "scrape_user_modified" });
|
||||
// 全部覆盖照旧改写
|
||||
const over = await scrape(movie, "movie", MOVIE_DIR, "overwrite");
|
||||
expect(fileOf(over.files, "movie_nfo")).toMatchObject({ action: "written" });
|
||||
expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toContain("<title>星际穿越(再改)</title>");
|
||||
});
|
||||
|
||||
it("更新模式:旧版本记录没有 hash 时按修改时间判断", async () => {
|
||||
// 模拟旧版本:去掉所有记录里的 hash
|
||||
const rows = await h.db.selectFrom("scrape_record").select(["id", "files", "finished_at"]).execute();
|
||||
for (const r of rows)
|
||||
await h.db
|
||||
.updateTable("scrape_record")
|
||||
.set({
|
||||
files: JSON.stringify((r.files as ScrapeFileResult[]).map(({ hash: _h, ...x }) => x)),
|
||||
})
|
||||
.where("id", "=", r.id)
|
||||
.execute();
|
||||
const last = rows
|
||||
.filter((r) => r.finished_at)
|
||||
.sort((a, b) => +new Date(b.finished_at!) - +new Date(a.finished_at!))[0]!;
|
||||
const past = new Date(+new Date(last.finished_at!) - 60_000);
|
||||
await utimes(lib.path(MOVIE_NFO), past, past);
|
||||
await setTitle(movie, "星际穿越(旧记录)");
|
||||
const r = await scrape(movie, "movie", MOVIE_DIR, "update");
|
||||
expect(fileOf(r.files, "movie_nfo")).toMatchObject({ action: "written" });
|
||||
expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toContain("<title>星际穿越(旧记录)</title>");
|
||||
|
||||
// 改动时间晚于那次刮削:当作人改过
|
||||
const rows2 = await h.db.selectFrom("scrape_record").select(["id", "files"]).execute();
|
||||
for (const x of rows2)
|
||||
await h.db
|
||||
.updateTable("scrape_record")
|
||||
.set({ files: JSON.stringify((x.files as ScrapeFileResult[]).map(({ hash: _h, ...y }) => y)) })
|
||||
.where("id", "=", x.id)
|
||||
.execute();
|
||||
const future = new Date(Date.now() + 60_000);
|
||||
await utimes(lib.path(MOVIE_NFO), future, future);
|
||||
expect((await stat(lib.path(MOVIE_NFO))).mtimeMs).toBeGreaterThan(Date.now());
|
||||
await setTitle(movie, "星际穿越(不该写进去)");
|
||||
const kept = await scrape(movie, "movie", MOVIE_DIR, "update");
|
||||
expect(fileOf(kept.files, "movie_nfo")).toMatchObject({ action: "skipped", reason: "scrape_user_modified" });
|
||||
});
|
||||
|
||||
const SHOW = "TV/人生切割术 (2022)";
|
||||
let series = "";
|
||||
|
||||
it("集单元没有元数据:按 TMDB id 现取季详情补齐,写出正确的集 NFO", async () => {
|
||||
series = await entity("series", "人生切割术", {
|
||||
title: f("人生切割术"),
|
||||
overview: f("马克领导着一个办公室团队。"),
|
||||
});
|
||||
const season = await h.db
|
||||
.insertInto("content_unit")
|
||||
.values({
|
||||
media_entity_id: series,
|
||||
kind: "season",
|
||||
season_number: 1,
|
||||
status: "confirmed",
|
||||
title: "第 1 季",
|
||||
metadata: JSON.stringify({ name: f("第 1 季") }),
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow();
|
||||
await h.db
|
||||
.insertInto("content_unit")
|
||||
.values({
|
||||
media_entity_id: series,
|
||||
parent_unit_id: season.id,
|
||||
kind: "episode",
|
||||
season_number: 1,
|
||||
episode_number: 1,
|
||||
status: "confirmed",
|
||||
title: "有关地狱的好消息",
|
||||
metadata: JSON.stringify({ name: f("有关地狱的好消息"), overview: f("第一集简介") }),
|
||||
})
|
||||
.execute();
|
||||
// 第 2 集是整理懒建的空单元
|
||||
await h.db
|
||||
.insertInto("content_unit")
|
||||
.values({
|
||||
media_entity_id: series,
|
||||
parent_unit_id: season.id,
|
||||
kind: "episode",
|
||||
season_number: 1,
|
||||
episode_number: 2,
|
||||
status: "provisional",
|
||||
})
|
||||
.execute();
|
||||
seasonDetail = async (n) => (n === 1 ? season1({ 1: "TMDB 第一集", 2: "半循环" }) : null);
|
||||
const r = await scrape(series, "tv", SHOW, "update", "95396");
|
||||
expect(seasonCalls).toEqual([1]);
|
||||
expect(r.status).toBe("succeeded");
|
||||
const ep2 = await readFile(lib.path(`${SHOW}/Season 1/Severance.S01E02.nfo`), "utf8");
|
||||
expect(ep2).toContain("<title>半循环</title>");
|
||||
expect(ep2).toContain("<plot><![CDATA[半循环的简介]]></plot>");
|
||||
// 本地已有的集字段不被 TMDB 现取的值替换
|
||||
expect(await readFile(lib.path(`${SHOW}/Season 1/Severance.S01E01.nfo`), "utf8")).toContain(
|
||||
"<title>有关地狱的好消息</title>",
|
||||
);
|
||||
});
|
||||
|
||||
it("TMDB 季详情失败:缺元数据的集跳过(scrape_metadata_missing)、记录 partial,不写占位 NFO", async () => {
|
||||
await writeFile(lib.path(`${SHOW}/Season 1/Severance.S01E03.mkv`), "video");
|
||||
seasonDetail = async () => {
|
||||
throw new Error("TMDB 超时");
|
||||
};
|
||||
const r = await scrape(series, "tv", SHOW, "update", "95396");
|
||||
expect(r).toMatchObject({ status: "partial", reason: "scrape_metadata_missing" });
|
||||
expect(r.files.find((x) => x.path.endsWith("S01E03.nfo"))).toMatchObject({
|
||||
action: "skipped",
|
||||
reason: "scrape_metadata_missing",
|
||||
});
|
||||
await expect(stat(lib.path(`${SHOW}/Season 1/Severance.S01E03.nfo`))).rejects.toThrow();
|
||||
|
||||
// 季单元也没有元数据时同样不写「季 N」占位
|
||||
await h.db
|
||||
.updateTable("content_unit")
|
||||
.set({ title: null, metadata: JSON.stringify({}) })
|
||||
.where("media_entity_id", "=", series)
|
||||
.where("kind", "=", "season")
|
||||
.execute();
|
||||
const r2 = await scrape(series, "tv", SHOW, "overwrite", "95396");
|
||||
expect(fileOf(r2.files, "season_nfo")).toMatchObject({ action: "skipped", reason: "scrape_metadata_missing" });
|
||||
expect(r2.status).toBe("partial");
|
||||
});
|
||||
|
||||
it("合集 / 其他类作品按电影刮削,NFO 根元素是 <movie>", async () => {
|
||||
expect(nfoWork({ kind: "collection", title: "x", original_title: null, year: null, metadata: {} }, {}).kind).toBe(
|
||||
"movie",
|
||||
);
|
||||
for (const [kind, dir, nfo] of [
|
||||
["collection", "Movies/合集 (2020)", "Collection.2020.nfo"],
|
||||
["other", "Movies/其他 (2021)", "Other.2021.nfo"],
|
||||
] as const) {
|
||||
const id = await entity(kind, `${kind} 作品`, { title: f(`${kind} 作品`) });
|
||||
const r = await scrape(id, "movie", dir);
|
||||
expect(r.status).toBe("succeeded");
|
||||
const xml = await readFile(lib.path(`${dir}/${nfo}`), "utf8");
|
||||
expect(xml).toContain("\n<movie>");
|
||||
expect(xml).not.toContain("<tvshow>");
|
||||
}
|
||||
});
|
||||
|
||||
it("兜底异常记 scrape_internal_error(不是写文件失败)", async () => {
|
||||
h.d.ports.scrapeTmdbSeason = (() => {
|
||||
throw new Error("意外错误");
|
||||
}) as never;
|
||||
await h.db
|
||||
.updateTable("content_unit")
|
||||
.set({ title: null, metadata: JSON.stringify({}) })
|
||||
.where("media_entity_id", "=", series)
|
||||
.execute();
|
||||
const r = await scrape(series, "tv", SHOW, "update", "95396");
|
||||
expect(r).toMatchObject({ status: "failed", reason: "scrape_internal_error" });
|
||||
});
|
||||
});
|
||||
@@ -96,7 +96,7 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => {
|
||||
}
|
||||
|
||||
/** 作品详情「重新刮削」的服务端做法:对作品目录排刮削并执行 */
|
||||
async function rescrape(entity: string, mode?: "missing" | "overwrite") {
|
||||
async function rescrape(entity: string, mode?: "missing" | "overwrite" | "update") {
|
||||
const q = await scrapeEntity(h.d, entity, { trigger: "manual", mode });
|
||||
expect(q.length).toBe(1);
|
||||
return runScrapeRecord(h.d, q[0]!.recordId);
|
||||
@@ -193,7 +193,7 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => {
|
||||
expect(first).toMatchObject({
|
||||
media_entity_id: series,
|
||||
trigger: "auto",
|
||||
mode: "missing",
|
||||
mode: "update", // 缺省覆盖方式:更新
|
||||
organization_record_id: rec,
|
||||
});
|
||||
|
||||
|
||||
@@ -124,6 +124,12 @@ describe.skipIf(skip)("file manager in-place import", () => {
|
||||
.where("organization_record_id", "=", r.recordId)
|
||||
.execute();
|
||||
expect(recs).toMatchObject([{ path: title, videos: [ep1] }]);
|
||||
// 季 / 集单元缺元数据时不写占位 NFO:模拟刷新元数据已填过整理懒建的单元
|
||||
await h.db
|
||||
.updateTable("content_unit")
|
||||
.set({ metadata: JSON.stringify({ name: { value: "名称", source: "tmdb", language: null, at, locked: false } }) })
|
||||
.where("media_entity_id", "=", entity)
|
||||
.execute();
|
||||
await runScrapeRecord(h.module.deps().ao, recs[0]!.id);
|
||||
const show = await readdir(join(h.lib, title));
|
||||
const season = await readdir(join(h.lib, title, "Season 1"));
|
||||
|
||||
@@ -277,7 +277,7 @@ describe.skipIf(skip)("files 刮削(SCR-2:任意目录就地刮削)", () => {
|
||||
expect(show).toMatchObject({
|
||||
status: "succeeded",
|
||||
path: SHOW,
|
||||
mode: "missing",
|
||||
mode: "update",
|
||||
trigger: "manual",
|
||||
match: {
|
||||
mediaEntityId: null,
|
||||
|
||||
Reference in new issue
Block a user