merge: fix/e-scrape

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-10-03 21:17:16 +08:00
commit b3555c04b5
24 files changed
+1039 -79

No files matched your search

@@ -1,5 +1,5 @@
import type { TmdbRichDetail, TmdbRichSeasonDetail } from "@umt/adapters"; import type { TmdbRichDetail, TmdbRichSeasonDetail } from "@umt/adapters";
import type { NfoEpisode, NfoIds, NfoPerson, NfoSeason, NfoWork } from "@umt/domain"; import { type NfoEpisode, type NfoIds, type NfoPerson, type NfoSeason, type NfoWork, scrapeKindOf } from "@umt/domain";
import { richEntityFields } from "../../sources-identity/service/metadata-rich.ts"; import { richEntityFields } from "../../sources-identity/service/metadata-rich.ts";
// SCR-1:本地作品 / 季 / 集的最终字段(media_entity.metadata、content_unit.metadata,字段级 {value, …})→ NFO 输入。 // SCR-1:本地作品 / 季 / 集的最终字段(media_entity.metadata、content_unit.metadata,字段级 {value, …})→ NFO 输入。
@@ -71,7 +71,7 @@ export function nfoWork(e: EntityForNfo, ids: IdentityMap): NfoWork {
douban: ids.douban ?? null, douban: ids.douban ?? null,
}; };
const crew = credits(val(m, "crew")); const crew = credits(val(m, "crew"));
const isTv = e.kind !== "movie"; const isTv = scrapeKindOf(e.kind) === "tv";
return { return {
kind: isTv ? "tvshow" : "movie", kind: isTv ? "tvshow" : "movie",
title: str(val(m, "title")) ?? e.title, title: str(val(m, "title")) ?? e.title,
@@ -136,6 +136,22 @@ export function nfoEpisode(season: number, episode: number, u: UnitForNfo | unde
}; };
} }
/** 单元被元数据来源填过(名称 / 简介 / 日期至少有一个字段,值为空也算:TMDB 就是没有,刷新也补不出来) */
const filled = (m: unknown) => !!m && typeof m === "object" && ["name", "overview", "airDate"].some((k) => k in m);
/**
* 季单元缺元数据(没有单元,或从没被元数据来源填过、也没有名称):不写「季 N」占位 NFO。
* 「缺」指还没取到(整理懒建、改身份后清空),不是 TMDB 本身没写简介。
*/
export function seasonMetadataMissing(u: UnitForNfo | undefined): boolean {
return !u || (!filled(u.metadata) && !str(u.title));
}
/** 集单元缺元数据(没有单元,或从没被填过、也没有标题和播出日期):不写「第 N 集」占位 NFO */
export function episodeMetadataMissing(u: UnitForNfo | undefined): boolean {
return !u || (!filled(u.metadata) && !str(u.title) && !u.air_date);
}
/** 图片路径:作品海报 / 背景图,季海报,集剧照 */ /** 图片路径:作品海报 / 背景图,季海报,集剧照 */
export function imagePathsOf(entityMeta: unknown) { export function imagePathsOf(entityMeta: unknown) {
return { return {
@@ -1,5 +1,5 @@
import { randomUUID } from "node:crypto"; import { createHash, randomUUID } from "node:crypto";
import { lstat, mkdir, readdir, realpath, rename, rm, writeFile } from "node:fs/promises"; import { lstat, mkdir, readdir, readFile, realpath, rename, rm, writeFile } from "node:fs/promises";
import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import { dirname, isAbsolute, join, relative, resolve } from "node:path";
import { checkRoot, DEFAULT_ROOT_SENTINEL } from "@umt/adapters"; import { checkRoot, DEFAULT_ROOT_SENTINEL } from "@umt/adapters";
import { import {
@@ -19,6 +19,7 @@ import {
planScrapeTargets, planScrapeTargets,
type ScrapeTarget, type ScrapeTarget,
type ScrapeVideo, type ScrapeVideo,
scrapeKindOf,
seasonNfo, seasonNfo,
seasonOfDirName, seasonOfDirName,
workNfo, workNfo,
@@ -31,11 +32,13 @@ import { errMessage, jsonb } from "../util.ts";
import type { AoDeps } from "./common.ts"; import type { AoDeps } from "./common.ts";
import { import {
type EntityForNfo, type EntityForNfo,
episodeMetadataMissing,
type IdentityMap, type IdentityMap,
imagePathsOf, imagePathsOf,
nfoEpisode, nfoEpisode,
nfoSeason, nfoSeason,
nfoWork, nfoWork,
seasonMetadataMissing,
tmdbEntity, tmdbEntity,
tmdbSeasonUnits, tmdbSeasonUnits,
type UnitForNfo, type UnitForNfo,
@@ -51,7 +54,10 @@ import {
// 剧集目录 → tvshow.nfo + 海报 / 背景图 + 季目录(Season 1 / S01 / Specials)的 season.nfo 与季海报 + 每集 NFO 与剧照; // 剧集目录 → tvshow.nfo + 海报 / 背景图 + 季目录(Season 1 / S01 / Specials)的 season.nfo 与季海报 + 每集 NFO 与剧照;
// 季目录 → season.nfo + 季海报(季目录 poster.jpg 与上级 seasonNN-poster.jpg)+ 每集;单集文件 → 该集 NFO 与剧照。 // 季目录 → season.nfo + 季海报(季目录 poster.jpg 与上级 seasonNN-poster.jpg)+ 每集;单集文件 → 该集 NFO 与剧照。
// - 字段:有本地作品且有元数据时用最终字段(字段锁、人工覆盖优先);否则按 TMDB id 直接取 TMDB(与刷新元数据同一映射)。 // - 字段:有本地作品且有元数据时用最终字段(字段锁、人工覆盖优先);否则按 TMDB id 直接取 TMDB(与刷新元数据同一映射)。
// - 覆盖:missing 只补缺失 / overwrite 全部覆盖;写入走同目录临时文件 + rename;目标必须在登记目录内。 // - 覆盖:update(缺省)只覆盖本工具上次写入且没被改过的文件(逐文件 sha256 记在 scrape_record.files,
// 改过的记 scrape_user_modified)/ missing 只补缺失 / overwrite 全部覆盖;写入走同目录临时文件 + rename;
// 目标必须在登记目录内。
// - 季 / 集缺元数据时先按 TMDB id 现取季详情补齐,仍缺就跳过(scrape_metadata_missing,记录 partial),不写占位 NFO。
// - 每次刮削一条 scrape_record(目录、识别结果、逐文件结果、原因);审计 organize.scrape;SSE files.changed。 // - 每次刮削一条 scrape_record(目录、识别结果、逐文件结果、原因);审计 organize.scrape;SSE files.changed。
export const SCRAPE_TASK = "scrapeMedia"; export const SCRAPE_TASK = "scrapeMedia";
@@ -59,7 +65,10 @@ const IMAGE_CONCURRENCY = 4;
const DEDUPE_MS = 30 * 60_000; const DEDUPE_MS = 30 * 60_000;
const TEMP_RE = /(\.uhd-partial|\.!qB|\.part)$/i; const TEMP_RE = /(\.uhd-partial|\.!qB|\.part)$/i;
export async function scrapeSettings(d: AoDeps): Promise<OrganizeSettings> { /** 排队一侧只用到 ctx(元数据刷新后自动重刮从 sources-identity 调用) */
type QueueDeps = Pick<AoDeps, "ctx">;
export async function scrapeSettings(d: QueueDeps): Promise<OrganizeSettings> {
const s = await d.ctx.settings.get("organize").catch(() => DEFAULT_SETTINGS.organize); const s = await d.ctx.settings.get("organize").catch(() => DEFAULT_SETTINGS.organize);
return { ...DEFAULT_SETTINGS.organize, ...s }; return { ...DEFAULT_SETTINGS.organize, ...s };
} }
@@ -83,6 +92,8 @@ export interface QueueScrapeInput extends ScrapeLocation {
idempotencyKey?: string | undefined; idempotencyKey?: string | undefined;
/** 只刮这些视频(库根相对路径;整理后刮削用,作品 / 季目录元数据照常初始化);缺省 = 目录下全部 */ /** 只刮这些视频(库根相对路径;整理后刮削用,作品 / 季目录元数据照常初始化);缺省 = 目录下全部 */
videos?: readonly string[] | undefined; videos?: readonly string[] | undefined;
/** 任务最早开始时间(元数据刷新后的自动重刮延后执行,期间再来的刷新合进同一条排队记录) */
availableAt?: Date | undefined;
} }
type Row = { type Row = {
@@ -158,7 +169,7 @@ export async function latestEntityScrape(d: AoDeps, entityId: string): Promise<S
* 建一条刮削记录并排任务 scrapeMedia。同一目录已有排队中(还没开始,30 分钟内)的同方式、同作品刮削时直接复用。 * 建一条刮削记录并排任务 scrapeMedia。同一目录已有排队中(还没开始,30 分钟内)的同方式、同作品刮削时直接复用。
*/ */
export async function queueScrape( export async function queueScrape(
d: AoDeps, d: QueueDeps,
input: QueueScrapeInput, input: QueueScrapeInput,
): Promise<{ recordId: string; taskId: string | null; deduplicated: boolean }> { ): Promise<{ recordId: string; taskId: string | null; deduplicated: boolean }> {
const db = d.ctx.db; const db = d.ctx.db;
@@ -212,7 +223,11 @@ export async function queueScrape(
queued = await d.ctx.engine.enqueue( queued = await d.ctx.engine.enqueue(
SCRAPE_TASK, SCRAPE_TASK,
{ recordId: rec.id }, { recordId: rec.id },
{ idempotencyKey: input.idempotencyKey ?? `${SCRAPE_TASK}:${rec.id}`, traceId: randomUUID() }, {
idempotencyKey: input.idempotencyKey ?? `${SCRAPE_TASK}:${rec.id}`,
traceId: randomUUID(),
...(input.availableAt ? { availableAt: input.availableAt } : {}),
},
); );
} catch (err) { } catch (err) {
// 只有新记录的入队失败才标失败;复用的记录可能已有接受但未关联的 run。 // 只有新记录的入队失败才标失败;复用的记录可能已有接受但未关联的 run。
@@ -259,11 +274,9 @@ export async function queueScrape(
// ── 作品 → 目录(作品详情「重新刮削」、入库后自动刮削) ── // ── 作品 → 目录(作品详情「重新刮削」、入库后自动刮削) ──
const isMovieKind = (k: string) => k === "movie" || k === "collection" || k === "other"; /** 本地作品 → 识别结果(带已接受的、与作品类型一致的 TMDB id) */
/** 本地作品 → 识别结果(带已接受的 TMDB id) */
export async function entityScrapeMatch( export async function entityScrapeMatch(
d: AoDeps, d: QueueDeps,
entityId: string, entityId: string,
source: ScrapeMatch["source"], source: ScrapeMatch["source"],
): Promise<ScrapeMatch | null> { ): Promise<ScrapeMatch | null> {
@@ -273,12 +286,15 @@ export async function entityScrapeMatch(
.where("id", "=", entityId) .where("id", "=", entityId)
.executeTakeFirst(); .executeTakeFirst();
if (!e || e.status === "merged") return null; if (!e || e.status === "merged") return null;
const kind = scrapeKindOf(e.kind);
// 同一作品可同时有 movie 与 tv 两个 accepted 映射:只取与作品类型一致的那个
const tmdb = await d.ctx.db const tmdb = await d.ctx.db
.selectFrom("external_identity") .selectFrom("external_identity")
.select(["external_id"]) .select(["external_id"])
.where("media_entity_id", "=", entityId) .where("media_entity_id", "=", entityId)
.where("content_unit_id", "is", null) .where("content_unit_id", "is", null)
.where("provider", "=", "tmdb") .where("provider", "=", "tmdb")
.where("entity_type", "=", kind)
.where("status", "=", "accepted") .where("status", "=", "accepted")
.orderBy("updated_at", "desc") .orderBy("updated_at", "desc")
.executeTakeFirst(); .executeTakeFirst();
@@ -286,7 +302,7 @@ export async function entityScrapeMatch(
mediaEntityId: e.id, mediaEntityId: e.id,
provider: tmdb ? "tmdb" : null, provider: tmdb ? "tmdb" : null,
externalId: tmdb?.external_id ?? null, externalId: tmdb?.external_id ?? null,
kind: isMovieKind(e.kind) ? "movie" : "tv", kind,
title: e.title, title: e.title,
year: e.year, year: e.year,
source, source,
@@ -298,7 +314,7 @@ export async function entityScrapeMatch(
* 推不出(视频直接在库根)时对视频文件本身刮削。只取启用的库根里 active 视频的位置。 * 推不出(视频直接在库根)时对视频文件本身刮削。只取启用的库根里 active 视频的位置。
*/ */
export async function entityScrapeLocations( export async function entityScrapeLocations(
d: AoDeps, d: QueueDeps,
entityId: string, entityId: string,
kind: "movie" | "tv", kind: "movie" | "tv",
): Promise<ScrapeLocation[]> { ): Promise<ScrapeLocation[]> {
@@ -348,13 +364,14 @@ export async function entityScrapeLocations(
/** 对作品的目录刮削(每个目录一条记录);没有已入库文件时返回空 */ /** 对作品的目录刮削(每个目录一条记录);没有已入库文件时返回空 */
export async function scrapeEntity( export async function scrapeEntity(
d: AoDeps, d: QueueDeps,
entityId: string, entityId: string,
opts: { opts: {
mode?: ScrapeMode | undefined; mode?: ScrapeMode | undefined;
trigger: "auto" | "manual"; trigger: "auto" | "manual";
organizationRecordId?: string | undefined; organizationRecordId?: string | undefined;
actor?: { type: string; id: string | null } | undefined; actor?: { type: string; id: string | null } | undefined;
availableAt?: Date | undefined;
}, },
): Promise<{ recordId: string; taskId: string | null; deduplicated: boolean }[]> { ): Promise<{ recordId: string; taskId: string | null; deduplicated: boolean }[]> {
const match = await entityScrapeMatch(d, entityId, "library"); const match = await entityScrapeMatch(d, entityId, "library");
@@ -370,6 +387,7 @@ export async function scrapeEntity(
trigger: opts.trigger, trigger: opts.trigger,
organizationRecordId: opts.organizationRecordId, organizationRecordId: opts.organizationRecordId,
actor: opts.actor, actor: opts.actor,
availableAt: opts.availableAt,
idempotencyKey: opts.organizationRecordId idempotencyKey: opts.organizationRecordId
? `${SCRAPE_TASK}:record:${opts.organizationRecordId}:${i}` ? `${SCRAPE_TASK}:record:${opts.organizationRecordId}:${i}`
: undefined, : undefined,
@@ -378,6 +396,31 @@ export async function scrapeEntity(
return out; return out;
} }
/** 元数据刷新后自动重刮的延迟:这段时间内再来的刷新合进同一条排队记录(防抖) */
export const RESCRAPE_DELAY_MS = 2 * 60_000;
/**
* 元数据刷新(或改身份后的刷新)改变了作品的 NFO 字段:对刮削过的作品(有成功 / 部分成功的刮削记录)排一次
* 「更新」模式重刮,延后 RESCRAPE_DELAY_MS 执行;还没开始的同目录更新刮削直接复用(queueScrape 合批)。
* 返回排上(含复用)的记录数;没刮削过的作品不排。
*/
export async function scheduleEntityRescrape(d: QueueDeps, entityId: string): Promise<number> {
const scraped = await d.ctx.db
.selectFrom("scrape_record")
.select("id")
.where("media_entity_id", "=", entityId)
.where("status", "in", ["succeeded", "partial"])
.limit(1)
.executeTakeFirst();
if (!scraped) return 0;
const out = await scrapeEntity(d, entityId, {
mode: "update",
trigger: "auto",
availableAt: new Date(Date.now() + RESCRAPE_DELAY_MS),
});
return out.length;
}
/** 整理入库成功后(设置打开或本次要求刮削):照 MP 对作品目录刮削,集只刮本次整理的文件。 */ /** 整理入库成功后(设置打开或本次要求刮削):照 MP 对作品目录刮削,集只刮本次整理的文件。 */
export async function scrapeAfterImport( export async function scrapeAfterImport(
d: AoDeps, d: AoDeps,
@@ -552,6 +595,50 @@ async function coverageOf(
return out; return out;
} }
const sha256 = (c: Uint8Array | string) => createHash("sha256").update(c).digest("hex");
/** 旧版本写的文件没有 hash:修改时间不晚于那次刮削结束(加这点余量)就当没被改过 */
const LEGACY_MTIME_SLACK_MS = 2_000;
const missingMeta = (t: ScrapeTarget): ScrapeFileResult => ({
path: t.path,
kind: t.kind,
role: t.role,
action: "skipped",
reason: "scrape_metadata_missing",
});
interface PrevWrite {
/** 写入时的 sha256;旧版本记录没有 */
hash: string | null;
/** 那次刮削结束时间(毫秒) */
at: number;
}
/**
* 同一登记目录里以前的刮削记录 → 本工具最近一次写入(或核对未变)的文件:路径 → hash。
* 从新到旧取每个路径第一条带 hash 的结果;旧版本只有 written、没有 hash 的按结束时间兜底。
*/
async function previousWrites(d: AoDeps, rootId: string, exceptId: string): Promise<Map<string, PrevWrite>> {
const rows = await d.ctx.db
.selectFrom("scrape_record")
.select(["files", "finished_at"])
.where("root_id", "=", rootId)
.where("id", "<>", exceptId)
.where("finished_at", "is not", null)
.orderBy("finished_at", "desc")
.limit(500)
.execute();
const out = new Map<string, PrevWrite>();
for (const r of rows) {
const at = new Date(r.finished_at!).getTime();
for (const f of (r.files ?? []) as ScrapeFileResult[]) {
if (out.has(f.path)) continue;
if (f.hash) out.set(f.path, { hash: f.hash, at });
else if (f.action === "written") out.set(f.path, { hash: null, at });
}
}
return out;
}
interface Fields { interface Fields {
entity: EntityForNfo; entity: EntityForNfo;
ids: IdentityMap; ids: IdentityMap;
@@ -579,6 +666,8 @@ export async function runScrapeRecord(
const mode = rec.mode as ScrapeMode; const mode = rec.mode as ScrapeMode;
let match = (rec.match ?? null) as ScrapeMatch | null; let match = (rec.match ?? null) as ScrapeMatch | null;
const dirs = new Set<string>(); const dirs = new Set<string>();
/** 逐文件结果;兜底异常时也保留已经写了的 */
const files: ScrapeFileResult[] = [];
const finish = async (r: { const finish = async (r: {
status: ScrapeRecord["status"]; status: ScrapeRecord["status"];
@@ -681,7 +770,7 @@ export async function runScrapeRecord(
const ids: IdentityMap = {}; const ids: IdentityMap = {};
const idRows = await db const idRows = await db
.selectFrom("external_identity") .selectFrom("external_identity")
.select(["provider", "external_id"]) .select(["provider", "external_id", "entity_type"])
.where("media_entity_id", "=", e.id) .where("media_entity_id", "=", e.id)
.where("content_unit_id", "is", null) .where("content_unit_id", "is", null)
.where("status", "=", "accepted") .where("status", "=", "accepted")
@@ -689,6 +778,8 @@ export async function runScrapeRecord(
.execute(); .execute();
for (const r of idRows) { for (const r of idRows) {
const p = r.provider.toLowerCase() as keyof IdentityMap; const p = r.provider.toLowerCase() as keyof IdentityMap;
// TMDB 映射只认与刮削类型一致的(电影 / 剧集各一个时不串)
if (p === "tmdb" && r.entity_type !== kind) continue;
if (["tmdb", "imdb", "tvdb", "douban"].includes(p) && !ids[p]) ids[p] = r.external_id; if (["tmdb", "imdb", "tvdb", "douban"].includes(p) && !ids[p]) ids[p] = r.external_id;
} }
if (!match.externalId && ids.tmdb) match = { ...match, provider: "tmdb", externalId: ids.tmdb }; if (!match.externalId && ids.tmdb) match = { ...match, provider: "tmdb", externalId: ids.tmdb };
@@ -757,6 +848,7 @@ export async function runScrapeRecord(
} }
// ── 没有本地字段:按 TMDB 条目直接取 ── // ── 没有本地字段:按 TMDB 条目直接取 ──
let fromTmdb = false;
if (!fields) { if (!fields) {
if (!match.externalId) return await fail("scrape_no_metadata"); if (!match.externalId) return await fail("scrape_no_metadata");
let detail: Awaited<ReturnType<typeof d.ports.scrapeTmdbDetail>>; let detail: Awaited<ReturnType<typeof d.ports.scrapeTmdbDetail>>;
@@ -770,22 +862,38 @@ export async function runScrapeRecord(
const t = tmdbEntity(detail); const t = tmdbEntity(detail);
fields = { entity: t.entity, ids: t.ids, seasonUnit: new Map(), episodeUnit: new Map() }; fields = { entity: t.entity, ids: t.ids, seasonUnit: new Map(), episodeUnit: new Map() };
match = { ...match, provider: "tmdb", title: detail.title, year: detail.year }; match = { ...match, provider: "tmdb", title: detail.title, year: detail.year };
if (kind === "tv") { fromTmdb = true;
const seasons = [...new Set(videos.map((v) => v.season).filter((s): s is number => typeof s === "number"))]; }
for (const n of seasons) {
const sd = await d.ports.scrapeTmdbSeason(match.externalId!, n).catch((err) => { // ── 季 / 集元数据:TMDB 分支取每个涉及的季;本地分支只在季 / 集单元缺元数据时按 TMDB id 现取补齐 ──
d.ctx.logger.warn({ recordId, season: n, err: { message: errMessage(err) } }, "刮削取 TMDB 季详情失败"); const { seasonUnit, episodeUnit } = fields;
return null; if (kind === "tv") {
}); const tmdbId = match.externalId ?? fields.ids.tmdb ?? null;
if (!sd) continue; const seasons = [...new Set(videos.map((v) => v.season).filter((s): s is number => typeof s === "number"))];
const u = tmdbSeasonUnits(sd); for (const n of seasons.sort((a, b) => a - b)) {
fields.seasonUnit.set(n, u.season); const lacking =
for (const [ep, eu] of u.episodes) fields.episodeUnit.set(`${n}:${ep}`, eu); (lay.parts.season && seasonMetadataMissing(seasonUnit.get(n))) ||
videos.some(
(v) =>
v.season === n &&
typeof v.episode === "number" &&
episodeMetadataMissing(episodeUnit.get(`${n}:${v.episode}`)),
);
if (!tmdbId || (!fromTmdb && !lacking)) continue;
const sd = await d.ports.scrapeTmdbSeason(tmdbId, n).catch((err) => {
d.ctx.logger.warn({ recordId, season: n, err: { message: errMessage(err) } }, "刮削取 TMDB 季详情失败");
return null;
});
if (!sd) continue;
const u = tmdbSeasonUnits(sd);
// 本地已有的字段(含人工覆盖)不动,只补缺元数据的单元
if (fromTmdb || seasonMetadataMissing(seasonUnit.get(n))) seasonUnit.set(n, u.season);
for (const [ep, eu] of u.episodes) {
const k = `${n}:${ep}`;
if (fromTmdb || episodeMetadataMissing(episodeUnit.get(k))) episodeUnit.set(k, eu);
} }
} }
} }
const { seasonUnit, episodeUnit } = fields;
const work = nfoWork(fields.entity, fields.ids); const work = nfoWork(fields.entity, fields.ids);
const seasonPosters = new Map<number, string | null>(); const seasonPosters = new Map<number, string | null>();
for (const [n, u] of seasonUnit) seasonPosters.set(n, unitPosterPath(u.metadata)); for (const [n, u] of seasonUnit) seasonPosters.set(n, unitPosterPath(u.metadata));
@@ -800,7 +908,9 @@ export async function runScrapeRecord(
options: { images: settings.scrapeImages, episodeThumbs: settings.scrapeEpisodeThumbs }, options: { images: settings.scrapeImages, episodeThumbs: settings.scrapeEpisodeThumbs },
parts: lay.parts, parts: lay.parts,
}); });
const files: ScrapeFileResult[] = [...noEpisode]; files.push(...noEpisode);
// 「更新」模式:本工具以前写过的文件(路径 → 写入时的 hash / 时间)
const ours = mode === "update" ? await previousWrites(d, rec.root_id, recordId) : new Map<string, PrevWrite>();
const imageCache = new Map<string, Promise<Uint8Array>>(); const imageCache = new Map<string, Promise<Uint8Array>>();
const fetchImage = (p: string) => { const fetchImage = (p: string) => {
let hit = imageCache.get(p); let hit = imageCache.get(p);
@@ -810,21 +920,37 @@ export async function runScrapeRecord(
} }
return hit; return hit;
}; };
const decide = async (t: ScrapeTarget): Promise<{ abs: string } | ScrapeFileResult> => { type Decision = { abs: string; prevHash: string | null } | ScrapeFileResult;
const decide = async (t: ScrapeTarget): Promise<Decision> => {
const target = resolve(rootReal, t.path); const target = resolve(rootReal, t.path);
const base = { path: t.path, kind: t.kind, role: t.role }; const base = { path: t.path, kind: t.kind, role: t.role };
if (!inside(rootReal, target)) return { ...base, action: "failed", reason: "path_escape" }; if (!inside(rootReal, target)) return { ...base, action: "failed", reason: "path_escape" };
const ex = await lstat(target).catch(() => null); const ex = await lstat(target).catch(() => null);
if (ex && (!ex.isFile() || ex.isSymbolicLink())) return { ...base, action: "skipped", reason: "scrape_exists" }; if (!ex) return { abs: target, prevHash: null };
if (ex && mode === "missing") return { ...base, action: "skipped", reason: "scrape_exists" }; if (!ex.isFile() || ex.isSymbolicLink()) return { ...base, action: "skipped", reason: "scrape_exists" };
return { abs: target }; if (mode === "missing") return { ...base, action: "skipped", reason: "scrape_exists" };
if (mode === "overwrite") return { abs: target, prevHash: null };
// update:只覆盖本工具写过且之后没被改过的
const prev = ours.get(t.path);
if (!prev) return { ...base, action: "skipped", reason: "scrape_exists" };
const cur = sha256(await readFile(target));
const untouched = prev.hash ? prev.hash === cur : ex.mtimeMs <= prev.at + LEGACY_MTIME_SLACK_MS;
if (!untouched) return { ...base, action: "skipped", reason: "scrape_user_modified" };
return { abs: target, prevHash: cur };
}; };
const write = async (t: ScrapeTarget, target: string, content: Uint8Array | string): Promise<ScrapeFileResult> => { const write = async (
t: ScrapeTarget,
dec: { abs: string; prevHash: string | null },
content: Uint8Array | string,
): Promise<ScrapeFileResult> => {
const base = { path: t.path, kind: t.kind, role: t.role }; const base = { path: t.path, kind: t.kind, role: t.role };
const hash = sha256(content);
// 更新模式下内容没变就不重写(不打扰媒体服务器)
if (dec.prevHash === hash) return { ...base, action: "skipped", reason: "scrape_unchanged", hash };
try { try {
await atomicWrite(target, content); await atomicWrite(dec.abs, content);
dirs.add(dirOf(t.path)); dirs.add(dirOf(t.path));
return { ...base, action: "written", reason: null }; return { ...base, action: "written", reason: null, hash };
} catch (err) { } catch (err) {
d.ctx.logger.warn({ recordId, path: t.path, err: { message: errMessage(err) } }, "刮削文件写入失败"); d.ctx.logger.warn({ recordId, path: t.path, err: { message: errMessage(err) } }, "刮削文件写入失败");
return { ...base, action: "failed", reason: "scrape_write_failed" }; return { ...base, action: "failed", reason: "scrape_write_failed" };
@@ -841,9 +967,22 @@ export async function runScrapeRecord(
} }
let xml: string; let xml: string;
if (t.role === "movie_nfo" || t.role === "tvshow_nfo") xml = workNfo(work); if (t.role === "movie_nfo" || t.role === "tvshow_nfo") xml = workNfo(work);
else if (t.role === "season_nfo") xml = seasonNfo(nfoSeason(t.season!, seasonUnit.get(t.season!))); else if (t.role === "season_nfo") {
else xml = episodeNfo(nfoEpisode(t.season!, t.episode!, episodeUnit.get(`${t.season}:${t.episode}`))); const u = seasonUnit.get(t.season!);
files.push(await write(t, dec.abs, xml)); if (seasonMetadataMissing(u)) {
files.push(missingMeta(t));
continue;
}
xml = seasonNfo(nfoSeason(t.season!, u));
} else {
const u = episodeUnit.get(`${t.season}:${t.episode}`);
if (episodeMetadataMissing(u)) {
files.push(missingMeta(t));
continue;
}
xml = episodeNfo(nfoEpisode(t.season!, t.episode!, u));
}
files.push(await write(t, dec, xml));
} }
// 图片:限并发,失败不影响 NFO // 图片:限并发,失败不影响 NFO
const imgs = targets.filter((x) => x.kind === "image" && x.imagePath); const imgs = targets.filter((x) => x.kind === "image" && x.imagePath);
@@ -859,23 +998,37 @@ export async function runScrapeRecord(
d.ctx.logger.warn({ recordId, image: t.imagePath, err: { message: errMessage(err) } }, "刮削图片下载失败"); d.ctx.logger.warn({ recordId, image: t.imagePath, err: { message: errMessage(err) } }, "刮削图片下载失败");
return { path: t.path, kind: t.kind, role: t.role, action: "failed", reason: "scrape_image_failed" }; return { path: t.path, kind: t.kind, role: t.role, action: "failed", reason: "scrape_image_failed" };
} }
return write(t, dec.abs, bytes); return write(t, dec, bytes);
})), })),
); );
const written = files.filter((f) => f.action === "written").length; const written = files.filter((f) => f.action === "written").length;
const failed = files.filter((f) => f.action === "failed").length; const failed = files.filter((f) => f.action === "failed").length;
// 缺元数据跳过的季 / 集不算成功(要刷新元数据后再刮)
const lacking = files.some((f) => f.reason === "scrape_metadata_missing");
const status: ScrapeRecord["status"] = const status: ScrapeRecord["status"] =
failed === 0 ? "succeeded" : written > 0 || files.some((f) => f.action === "skipped") ? "partial" : "failed"; failed === 0
? lacking
? "partial"
: "succeeded"
: written > 0 || files.some((f) => f.action === "skipped")
? "partial"
: "failed";
return await finish({ return await finish({
status, status,
reason: failed ? (files.find((f) => f.action === "failed")?.reason ?? null) : null, reason: failed
? (files.find((f) => f.action === "failed")?.reason ?? null)
: lacking
? "scrape_metadata_missing"
: null,
files, files,
}); });
} catch (err) { } catch (err) {
if (signal?.aborted) throw err; if (signal?.aborted) throw err;
d.ctx.logger.warn({ recordId, err: { message: errMessage(err) } }, "刮削失败"); d.ctx.logger.warn({ recordId, err: { message: errMessage(err) } }, "刮削失败");
return fail("scrape_write_failed"); // 不是写文件的错(如查库失败):记内部错误,已写的文件结果照留
const anyWritten = files.some((f) => f.action === "written");
return finish({ status: anyWritten ? "partial" : "failed", reason: "scrape_internal_error", files });
} }
} }
@@ -29,6 +29,7 @@ import {
settleEpisodeDecision, settleEpisodeDecision,
} from "./episode-resolution.ts"; } from "./episode-resolution.ts";
import { detachFromIdentityDecisions } from "./identity-decisions.ts"; import { detachFromIdentityDecisions } from "./identity-decisions.ts";
import { clearTmdbMetadata } from "./metadata.ts";
import { metadataLanguages, metadataTextOptions } from "./metadata-rich.ts"; import { metadataLanguages, metadataTextOptions } from "./metadata-rich.ts";
import { recordAttempt } from "./resolve.ts"; import { recordAttempt } from "./resolve.ts";
import { backfillRecognizeCache } from "./tmdb-recognize-cache.ts"; import { backfillRecognizeCache } from "./tmdb-recognize-cache.ts";
@@ -271,10 +272,12 @@ async function retireIdentity(exec: Exec, row: ExternalIdentityRow, actor: Actor
/** /**
* 改作品的外部映射:同 provider+类型的旧 accepted 映射退役(留修订并锁定,防止自动匹配改回), * 改作品的外部映射:同 provider+类型的旧 accepted 映射退役(留修订并锁定,防止自动匹配改回),
* 新映射 accepted + 人工锁。目标身份已被其它作品接受时 409(应走合并)。 * 新映射 accepted + 人工锁。目标身份已被其它作品接受时 409(应走合并)。
* TMDB 映射变了:同一事务里清掉该作品(及季 / 集单元)tmdb 来源的元数据(人工覆盖 / 锁定保留),提交后排一次
* full 刷新;刷新写回新身份的字段后,刮削过的作品自动排「更新」模式重刮(refreshMetadata → scheduleEntityRescrape)。
*/ */
export async function setEntityIdentity(rt: SourcesRuntime, entityId: string, input: EntityIdentitySet, actor: Actor) { export async function setEntityIdentity(rt: SourcesRuntime, entityId: string, input: EntityIdentitySet, actor: Actor) {
const { ctx } = rt; const { ctx } = rt;
return ctx.db.transaction().execute(async (trx) => { const r = await ctx.db.transaction().execute(async (trx) => {
const id = await resolveEntityId(trx, entityId); const id = await resolveEntityId(trx, entityId);
if (!id) throw new ApiException("not_found", "作品不存在"); if (!id) throw new ApiException("not_found", "作品不存在");
const other = await entityByExternal(trx, input); const other = await entityByExternal(trx, input);
@@ -297,6 +300,21 @@ export async function setEntityIdentity(rt: SourcesRuntime, entityId: string, in
reason: input.reason ?? null, reason: input.reason ?? null,
}); });
if (!newId) throw new ApiException("conflict", "外部身份唯一约束冲突"); if (!newId) throw new ApiException("conflict", "外部身份唯一约束冲突");
if (input.provider === "tmdb") {
// 旧 TMDB 带出来的 IMDb 映射(没人工锁的)一并退役:NFO 里 IMDb 是 default uniqueid,留着会被认成旧作品;
// 刷新后 NFO 取新详情里的 IMDb
const derived = await trx
.selectFrom("external_identity")
.selectAll()
.where("media_entity_id", "=", id)
.where("content_unit_id", "is", null)
.where("provider", "=", "imdb")
.where("status", "=", "accepted")
.where("locked", "=", false)
.execute();
for (const o of derived) await retireIdentity(trx, o, actor, "tmdb_mapping_changed");
await clearTmdbMetadata(trx, id, rt.now());
}
await audit( await audit(
trx, trx,
actor, actor,
@@ -310,6 +328,20 @@ export async function setEntityIdentity(rt: SourcesRuntime, entityId: string, in
); );
return { mediaEntityId: id, changed: true, identityId: newId, retired: olds.map((o) => o.id) }; return { mediaEntityId: id, changed: true, identityId: newId, retired: olds.map((o) => o.id) };
}); });
if (r.changed && input.provider === "tmdb" && r.identityId)
await ctx.engine
.enqueue(
"refreshMetadata",
{ mediaEntityId: r.mediaEntityId, full: true },
{ idempotencyKey: `refreshMetadata:identity:${r.identityId}`, traceId: r.mediaEntityId },
)
.catch((err: unknown) =>
ctx.logger.warn(
{ entityId: r.mediaEntityId, err: { message: (err as Error).message } },
"改映射后元数据刷新没排上",
),
);
return r;
} }
/** 外部身份字段修订(乐观并发 expectedRevision);每次变更写 identity_revision */ /** 外部身份字段修订(乐观并发 expectedRevision);每次变更写 identity_revision */
@@ -315,8 +315,20 @@ export interface SeasonMetadataResult {
changedUnits: number; changedUnits: number;
/** DH-1:临时转正式的集单元数(季单元转正不计) */ /** DH-1:临时转正式的集单元数(季单元转正不计) */
promoted: number; promoted: number;
/** 写进 NFO 的字段(名称、简介、日期、时长、剧照、季海报)有变化的单元数;刷新后据此决定要不要自动重刮 */
nfoChangedUnits?: number;
} }
/** 内容单元里写进 NFO 的字段(评分 / 票数变化不触发自动重刮) */
export const UNIT_NFO_FIELDS: ReadonlySet<string> = new Set([
"name",
"overview",
"airDate",
"runtime",
"stillPath",
"posterPath",
]);
/** /**
* 季详情 → 季单元与集单元的字段级元数据。集单元按(作品, 季, 集号)找最早建的未退役单元,没有时: * 季详情 → 季单元与集单元的字段级元数据。集单元按(作品, 季, 集号)找最早建的未退役单元,没有时:
* create = true(full 刷新 / 追踪中的季)建 confirmed 单元;否则只更新已有单元(集单元仍由发布 / 文件懒建)。 * create = true(full 刷新 / 追踪中的季)建 confirmed 单元;否则只更新已有单元(集单元仍由发布 / 文件懒建)。
@@ -331,7 +343,14 @@ export async function applySeasonMetadata(
opts: { create?: boolean } = {}, opts: { create?: boolean } = {},
): Promise<SeasonMetadataResult> { ): Promise<SeasonMetadataResult> {
const create = opts.create ?? true; const create = opts.create ?? true;
const out: SeasonMetadataResult = { season: sd.seasonNumber, episodes: 0, created: 0, changedUnits: 0, promoted: 0 }; const out: SeasonMetadataResult = {
season: sd.seasonNumber,
episodes: 0,
created: 0,
changedUnits: 0,
promoted: 0,
nfoChangedUnits: 0,
};
if (sd.seasonNumber < 0) return out; if (sd.seasonNumber < 0) return out;
const seasonUnit = await ensureSeason(exec, entityId, sd.seasonNumber, sd.name); const seasonUnit = await ensureSeason(exec, entityId, sd.seasonNumber, sd.name);
const units = await exec const units = await exec
@@ -399,6 +418,7 @@ export async function applySeasonMetadata(
if (written.length || t.fresh) { if (written.length || t.fresh) {
const r = await projectUnit(exec, t.unitId, now); const r = await projectUnit(exec, t.unitId, now);
if (r.changedFields.length) out.changedUnits++; if (r.changedFields.length) out.changedUnits++;
if (r.changedFields.some((f) => UNIT_NFO_FIELDS.has(f))) out.nfoChangedUnits = (out.nfoChangedUnits ?? 0) + 1;
} }
} }
out.episodes = eps.length; out.episodes = eps.length;
@@ -17,6 +17,7 @@ import type {
} from "@umt/contracts"; } from "@umt/contracts";
import { sql } from "@umt/db"; import { sql } from "@umt/db";
import { ApiException } from "../../../http/errors.ts"; import { ApiException } from "../../../http/errors.ts";
import { scheduleEntityRescrape } from "../../acquisition-organize/services/scrape.ts";
import { jsonb } from "../rows.ts"; import { jsonb } from "../rows.ts";
import type { SourcesRuntime } from "../runtime.ts"; import type { SourcesRuntime } from "../runtime.ts";
import { type Actor, audit, type Exec } from "./common.ts"; import { type Actor, audit, type Exec } from "./common.ts";
@@ -118,6 +119,102 @@ export async function writeEnrichmentFields(
} }
} }
/** 作品里写进 NFO 的字段:刷新后这些字段有变化才自动重刮(评分、票数、播出状态等不算) */
const ENTITY_NFO_FIELDS: ReadonlySet<string> = new Set([
"title",
"originalTitle",
"year",
"overview",
"tagline",
"releaseDate",
"genres",
"originCountry",
"platform",
"runtime",
"cast",
"crew",
"externalIds",
"posterPath",
"backdropPath",
]);
/**
* 改作品的 TMDB 映射后:清掉该作品及其季 / 集单元的 tmdb 来源字段(metadata_field 与投影出的最终值),
* 人工覆盖与锁定字段保留;单元的 title / air_date 列随 name / airDate 一起清空。之后由 full 刷新按新身份重新填,
* 避免新身份的 NFO 混进旧作品的标题、简介、IMDb、海报(海报 / 背景图「TMDB 没有时保留旧值」不能保留旧作品的)。
*/
export async function clearTmdbMetadata(trx: Exec, entityId: string, now: Date): Promise<void> {
const keep = (meta: Record<string, FinalFieldValue>, locks: Set<string>, overridden: Set<string>) => {
const out: Record<string, FinalFieldValue> = {};
for (const [k, v] of Object.entries(meta))
if (v?.source !== "tmdb" || locks.has(k) || overridden.has(k) || v.locked) out[k] = v;
return out;
};
const e = await trx
.selectFrom("media_entity")
.select(["metadata", "field_locks"])
.where("id", "=", entityId)
.forUpdate()
.executeTakeFirstOrThrow();
const entityOverrides = await trx
.selectFrom("field_override")
.select("field")
.where("media_entity_id", "=", entityId)
.execute();
await trx
.deleteFrom("metadata_field")
.where("media_entity_id", "=", entityId)
.where("content_unit_id", "is", null)
.where("source", "=", "tmdb")
.execute();
await trx
.updateTable("media_entity")
.set({
metadata: jsonb(
keep(
(e.metadata ?? {}) as Record<string, FinalFieldValue>,
new Set(e.field_locks),
new Set(entityOverrides.map((o) => o.field)),
),
),
updated_at: now,
})
.where("id", "=", entityId)
.execute();
// 其它来源(web / agent 补全)按原规则重新投影
await project(trx, entityId, now);
const units = await trx
.selectFrom("content_unit")
.select(["id", "metadata", "field_locks"])
.where("media_entity_id", "=", entityId)
.execute();
if (!units.length) return;
const ids = units.map((u) => u.id);
const unitOverrides = await trx
.selectFrom("field_override")
.select(["content_unit_id", "field"])
.where("content_unit_id", "in", ids)
.execute();
await trx.deleteFrom("metadata_field").where("content_unit_id", "in", ids).where("source", "=", "tmdb").execute();
for (const u of units) {
const meta = (u.metadata ?? {}) as Record<string, FinalFieldValue>;
const overridden = new Set(unitOverrides.filter((o) => o.content_unit_id === u.id).map((o) => o.field));
const next = keep(meta, new Set(u.field_locks), overridden);
if (Object.keys(next).length === Object.keys(meta).length) continue;
await trx
.updateTable("content_unit")
.set({
metadata: jsonb(next),
...(next.name ? {} : { title: null }),
...(next.airDate ? {} : { air_date: null }),
updated_at: now,
})
.where("id", "=", u.id)
.execute();
}
}
export interface RefreshResult { export interface RefreshResult {
status: "refreshed" | "skipped" | "failed"; status: "refreshed" | "skipped" | "failed";
reason?: string; reason?: string;
@@ -240,7 +337,7 @@ export async function refreshMetadata(
} }
const now = rt.now(); const now = rt.now();
const rev = `tmdb:${ident.entity_type}/${ident.external_id}`; const rev = `tmdb:${ident.entity_type}/${ident.external_id}`;
return ctx.db.transaction().execute(async (trx) => { const result = await ctx.db.transaction().execute(async (trx) => {
if (richDetail) await writeRichEntityFields(trx, id, richDetail, rev, now); if (richDetail) await writeRichEntityFields(trx, id, richDetail, rev, now);
else await writeTmdbFields(trx, id, detail, rev, now); else await writeTmdbFields(trx, id, detail, rev, now);
await trx await trx
@@ -282,6 +379,15 @@ export async function refreshMetadata(
: {}), : {}),
}; };
}); });
// 写进 NFO 的字段有变化:刮削过的作品延后排一次「更新」模式重刮(期间再刷新合进同一条);失败只记日志
const nfoChanged =
result.changedFields.some((f) => ENTITY_NFO_FIELDS.has(f)) ||
(result.episodes ?? []).some((x) => (x.nfoChangedUnits ?? 0) > 0);
if (nfoChanged)
await scheduleEntityRescrape({ ctx }, id).catch((err: unknown) =>
ctx.logger.warn({ entityId: id, err: { message: (err as Error).message } }, "元数据刷新后重刮没排上"),
);
return result;
} }
/** /**
@@ -0,0 +1,327 @@
import { readFile, stat, utimes, writeFile } from "node:fs/promises";
import type { TmdbRichSeasonDetail } from "@umt/adapters";
import type { ScrapeFileResult, ScrapeMatch } from "@umt/contracts";
import { createTempTree, type TempTree } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { queueScrape, runScrapeRecord } from "../../src/modules/acquisition-organize/services/scrape.ts";
import { nfoWork } from "../../src/modules/acquisition-organize/services/scrape-input.ts";
import { createHarness, type Harness, pgAvailable, seedBase } from "./harness.ts";
// 刮削准确度:
// - 「更新」模式(缺省):不存在的写入;本工具上次写入且没被改过的按新内容覆盖(内容没变不重写);被人改过的跳过
// (scrape_user_modified);不是本工具写的不动;旧版本没有 hash 的记录按修改时间兜底;
// - 集 / 季缺元数据:先按 TMDB id 现取季详情补齐;取不到跳过(scrape_metadata_missing)、记录 partial,不写占位 NFO;
// - 合集 / 其他类作品按电影刮,NFO 根元素是 <movie>;
// - 兜底异常记 scrape_internal_error。
const hasPg = await pgAvailable();
const at = new Date().toISOString();
const f = (value: unknown, language: string | null = null) => ({ value, source: "tmdb", language, at, locked: false });
const enc = (s: string) => new TextEncoder().encode(s);
function season1(names: Record<number, string>): TmdbRichSeasonDetail {
return {
tvId: "95396",
seasonNumber: 1,
name: "第 1 季",
nameLanguage: "zh-CN",
overview: null,
overviewLanguage: null,
airDate: "2022-02-17",
posterPath: null,
fetchedLanguages: ["zh-CN"],
episodes: Object.entries(names).map(([n, name]) => ({
seasonNumber: 1,
episodeNumber: Number(n),
name,
nameLanguage: "zh-CN",
overview: `${name}的简介`,
overviewLanguage: "zh-CN",
airDate: "2022-02-25",
runtime: 50,
stillPath: null,
voteAverage: null,
voteCount: null,
})),
};
}
describe.skipIf(!hasPg)("刮削准确度:更新模式、缺元数据不写占位、作品类型", () => {
let h: Harness;
let lib: TempTree;
let rootId = "";
let seasonDetail: (n: number) => Promise<TmdbRichSeasonDetail | null> = async () => null;
const seasonCalls: number[] = [];
beforeAll(async () => {
lib = await createTempTree(
{
"Movies/星际穿越 (2014)/Interstellar.2014.mkv": 100,
"Movies/星际穿越 (2014)/fanart.jpg": "别的工具放的背景图",
"TV/人生切割术 (2022)/Season 1/Severance.S01E01.mkv": 100,
"TV/人生切割术 (2022)/Season 1/Severance.S01E02.mkv": 100,
"Movies/合集 (2020)/Collection.2020.mkv": 100,
"Movies/其他 (2021)/Other.2021.mkv": 100,
},
{ sentinel: true, prefix: "umt-scracc-" },
);
h = await createHarness("scrape_acc", {
scrapeImage: async (p) => ({ content: enc(p), mime: "image/jpeg" }),
scrapeTmdbSeason: async (_tv, n) => {
seasonCalls.push(n);
return seasonDetail(n);
},
});
rootId = (await seedBase(h.db, { libraryPath: lib.root, cachePath: lib.root, qbUrl: "http://127.0.0.1:9" })).rootId;
}, 60_000);
afterAll(async () => {
await h?.close();
await lib?.cleanup();
});
async function entity(kind: "movie" | "series" | "collection" | "other", title: string, meta: object) {
return (
await h.db
.insertInto("media_entity")
.values({ kind, title, year: 2020, status: "active", metadata: JSON.stringify(meta) })
.returning("id")
.executeTakeFirstOrThrow()
).id;
}
async function scrape(
entityId: string,
kind: "movie" | "tv",
path: string,
mode?: "missing" | "overwrite" | "update",
externalId: string | null = null,
) {
const match: ScrapeMatch = {
mediaEntityId: entityId,
provider: externalId ? "tmdb" : null,
externalId,
kind,
title: "x",
year: null,
source: "library",
};
const q = await queueScrape(h.d, {
rootId: `lib:${rootId}`,
rootPath: lib.root,
libraryRootId: rootId,
path,
match,
mode,
trigger: "manual",
});
return runScrapeRecord(h.d, q.recordId);
}
const fileOf = (files: ScrapeFileResult[], role: string) => files.find((x) => x.role === role);
const setTitle = (id: string, title: string) =>
h.db
.updateTable("media_entity")
.set({ metadata: JSON.stringify({ title: f(title), posterPath: f("/inter.jpg") }) })
.where("id", "=", id)
.execute();
const MOVIE_DIR = "Movies/星际穿越 (2014)";
const MOVIE_NFO = `${MOVIE_DIR}/Interstellar.2014.nfo`;
let movie = "";
it("更新模式:不存在的写入并记 hash;不是本工具写的不动", async () => {
movie = await entity("movie", "星际穿越", { title: f("星际穿越"), posterPath: f("/inter.jpg") });
const r = await scrape(movie, "movie", MOVIE_DIR);
expect(r).toMatchObject({ status: "succeeded", mode: "update" });
expect(fileOf(r.files, "movie_nfo")).toMatchObject({
action: "written",
hash: expect.stringMatching(/^[0-9a-f]{64}$/),
});
expect(fileOf(r.files, "poster")).toMatchObject({ action: "written" });
expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toContain("<title>星际穿越</title>");
// fanart.jpg 是别的工具放的:元数据没有背景图,不在目标里;加上背景图后更新模式也不覆盖它
await h.db
.updateTable("media_entity")
.set({
metadata: JSON.stringify({ title: f("星际穿越"), posterPath: f("/inter.jpg"), backdropPath: f("/bg.jpg") }),
})
.where("id", "=", movie)
.execute();
const again = await scrape(movie, "movie", MOVIE_DIR, "update");
expect(fileOf(again.files, "fanart")).toMatchObject({ action: "skipped", reason: "scrape_exists" });
expect(await readFile(lib.path(`${MOVIE_DIR}/fanart.jpg`), "utf8")).toBe("别的工具放的背景图");
// 内容没变的不重写
expect(fileOf(again.files, "movie_nfo")).toMatchObject({ action: "skipped", reason: "scrape_unchanged" });
expect(fileOf(again.files, "poster")).toMatchObject({ action: "skipped", reason: "scrape_unchanged" });
expect(again.status).toBe("succeeded");
});
it("更新模式:本工具写的、没被改过的按新内容覆盖;被人改过的跳过并记原因", async () => {
await setTitle(movie, "星际穿越(新译名)");
const upd = await scrape(movie, "movie", MOVIE_DIR, "update");
expect(fileOf(upd.files, "movie_nfo")).toMatchObject({ action: "written" });
expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toContain("<title>星际穿越(新译名)</title>");
await writeFile(lib.path(MOVIE_NFO), "<movie><title>我手改的</title></movie>");
await setTitle(movie, "星际穿越(再改)");
const kept = await scrape(movie, "movie", MOVIE_DIR, "update");
expect(fileOf(kept.files, "movie_nfo")).toMatchObject({ action: "skipped", reason: "scrape_user_modified" });
expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toBe("<movie><title>我手改的</title></movie>");
expect(kept.status).toBe("succeeded");
// 之后每次更新都还认得出是人改过的(跳过的结果不带 hash,不会把人改的内容当成本工具写的)
const still = await scrape(movie, "movie", MOVIE_DIR, "update");
expect(fileOf(still.files, "movie_nfo")).toMatchObject({ action: "skipped", reason: "scrape_user_modified" });
// 全部覆盖照旧改写
const over = await scrape(movie, "movie", MOVIE_DIR, "overwrite");
expect(fileOf(over.files, "movie_nfo")).toMatchObject({ action: "written" });
expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toContain("<title>星际穿越(再改)</title>");
});
it("更新模式:旧版本记录没有 hash 时按修改时间判断", async () => {
// 模拟旧版本:去掉所有记录里的 hash
const rows = await h.db.selectFrom("scrape_record").select(["id", "files", "finished_at"]).execute();
for (const r of rows)
await h.db
.updateTable("scrape_record")
.set({
files: JSON.stringify((r.files as ScrapeFileResult[]).map(({ hash: _h, ...x }) => x)),
})
.where("id", "=", r.id)
.execute();
const last = rows
.filter((r) => r.finished_at)
.sort((a, b) => +new Date(b.finished_at!) - +new Date(a.finished_at!))[0]!;
const past = new Date(+new Date(last.finished_at!) - 60_000);
await utimes(lib.path(MOVIE_NFO), past, past);
await setTitle(movie, "星际穿越(旧记录)");
const r = await scrape(movie, "movie", MOVIE_DIR, "update");
expect(fileOf(r.files, "movie_nfo")).toMatchObject({ action: "written" });
expect(await readFile(lib.path(MOVIE_NFO), "utf8")).toContain("<title>星际穿越(旧记录)</title>");
// 改动时间晚于那次刮削:当作人改过
const rows2 = await h.db.selectFrom("scrape_record").select(["id", "files"]).execute();
for (const x of rows2)
await h.db
.updateTable("scrape_record")
.set({ files: JSON.stringify((x.files as ScrapeFileResult[]).map(({ hash: _h, ...y }) => y)) })
.where("id", "=", x.id)
.execute();
const future = new Date(Date.now() + 60_000);
await utimes(lib.path(MOVIE_NFO), future, future);
expect((await stat(lib.path(MOVIE_NFO))).mtimeMs).toBeGreaterThan(Date.now());
await setTitle(movie, "星际穿越(不该写进去)");
const kept = await scrape(movie, "movie", MOVIE_DIR, "update");
expect(fileOf(kept.files, "movie_nfo")).toMatchObject({ action: "skipped", reason: "scrape_user_modified" });
});
const SHOW = "TV/人生切割术 (2022)";
let series = "";
it("集单元没有元数据:按 TMDB id 现取季详情补齐,写出正确的集 NFO", async () => {
series = await entity("series", "人生切割术", {
title: f("人生切割术"),
overview: f("马克领导着一个办公室团队。"),
});
const season = await h.db
.insertInto("content_unit")
.values({
media_entity_id: series,
kind: "season",
season_number: 1,
status: "confirmed",
title: "第 1 季",
metadata: JSON.stringify({ name: f("第 1 季") }),
})
.returning("id")
.executeTakeFirstOrThrow();
await h.db
.insertInto("content_unit")
.values({
media_entity_id: series,
parent_unit_id: season.id,
kind: "episode",
season_number: 1,
episode_number: 1,
status: "confirmed",
title: "有关地狱的好消息",
metadata: JSON.stringify({ name: f("有关地狱的好消息"), overview: f("第一集简介") }),
})
.execute();
// 第 2 集是整理懒建的空单元
await h.db
.insertInto("content_unit")
.values({
media_entity_id: series,
parent_unit_id: season.id,
kind: "episode",
season_number: 1,
episode_number: 2,
status: "provisional",
})
.execute();
seasonDetail = async (n) => (n === 1 ? season1({ 1: "TMDB 第一集", 2: "半循环" }) : null);
const r = await scrape(series, "tv", SHOW, "update", "95396");
expect(seasonCalls).toEqual([1]);
expect(r.status).toBe("succeeded");
const ep2 = await readFile(lib.path(`${SHOW}/Season 1/Severance.S01E02.nfo`), "utf8");
expect(ep2).toContain("<title>半循环</title>");
expect(ep2).toContain("<plot><![CDATA[半循环的简介]]></plot>");
// 本地已有的集字段不被 TMDB 现取的值替换
expect(await readFile(lib.path(`${SHOW}/Season 1/Severance.S01E01.nfo`), "utf8")).toContain(
"<title>有关地狱的好消息</title>",
);
});
it("TMDB 季详情失败:缺元数据的集跳过(scrape_metadata_missing)、记录 partial,不写占位 NFO", async () => {
await writeFile(lib.path(`${SHOW}/Season 1/Severance.S01E03.mkv`), "video");
seasonDetail = async () => {
throw new Error("TMDB 超时");
};
const r = await scrape(series, "tv", SHOW, "update", "95396");
expect(r).toMatchObject({ status: "partial", reason: "scrape_metadata_missing" });
expect(r.files.find((x) => x.path.endsWith("S01E03.nfo"))).toMatchObject({
action: "skipped",
reason: "scrape_metadata_missing",
});
await expect(stat(lib.path(`${SHOW}/Season 1/Severance.S01E03.nfo`))).rejects.toThrow();
// 季单元也没有元数据时同样不写「季 N」占位
await h.db
.updateTable("content_unit")
.set({ title: null, metadata: JSON.stringify({}) })
.where("media_entity_id", "=", series)
.where("kind", "=", "season")
.execute();
const r2 = await scrape(series, "tv", SHOW, "overwrite", "95396");
expect(fileOf(r2.files, "season_nfo")).toMatchObject({ action: "skipped", reason: "scrape_metadata_missing" });
expect(r2.status).toBe("partial");
});
it("合集 / 其他类作品按电影刮削,NFO 根元素是 <movie>", async () => {
expect(nfoWork({ kind: "collection", title: "x", original_title: null, year: null, metadata: {} }, {}).kind).toBe(
"movie",
);
for (const [kind, dir, nfo] of [
["collection", "Movies/合集 (2020)", "Collection.2020.nfo"],
["other", "Movies/其他 (2021)", "Other.2021.nfo"],
] as const) {
const id = await entity(kind, `${kind} 作品`, { title: f(`${kind} 作品`) });
const r = await scrape(id, "movie", dir);
expect(r.status).toBe("succeeded");
const xml = await readFile(lib.path(`${dir}/${nfo}`), "utf8");
expect(xml).toContain("\n<movie>");
expect(xml).not.toContain("<tvshow>");
}
});
it("兜底异常记 scrape_internal_error(不是写文件失败)", async () => {
h.d.ports.scrapeTmdbSeason = (() => {
throw new Error("意外错误");
}) as never;
await h.db
.updateTable("content_unit")
.set({ title: null, metadata: JSON.stringify({}) })
.where("media_entity_id", "=", series)
.execute();
const r = await scrape(series, "tv", SHOW, "update", "95396");
expect(r).toMatchObject({ status: "failed", reason: "scrape_internal_error" });
});
});
@@ -96,7 +96,7 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => {
} }
/** 作品详情「重新刮削」的服务端做法:对作品目录排刮削并执行 */ /** 作品详情「重新刮削」的服务端做法:对作品目录排刮削并执行 */
async function rescrape(entity: string, mode?: "missing" | "overwrite") { async function rescrape(entity: string, mode?: "missing" | "overwrite" | "update") {
const q = await scrapeEntity(h.d, entity, { trigger: "manual", mode }); const q = await scrapeEntity(h.d, entity, { trigger: "manual", mode });
expect(q.length).toBe(1); expect(q.length).toBe(1);
return runScrapeRecord(h.d, q[0]!.recordId); return runScrapeRecord(h.d, q[0]!.recordId);
@@ -193,7 +193,7 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => {
expect(first).toMatchObject({ expect(first).toMatchObject({
media_entity_id: series, media_entity_id: series,
trigger: "auto", trigger: "auto",
mode: "missing", mode: "update", // 缺省覆盖方式:更新
organization_record_id: rec, organization_record_id: rec,
}); });
@@ -124,6 +124,12 @@ describe.skipIf(skip)("file manager in-place import", () => {
.where("organization_record_id", "=", r.recordId) .where("organization_record_id", "=", r.recordId)
.execute(); .execute();
expect(recs).toMatchObject([{ path: title, videos: [ep1] }]); expect(recs).toMatchObject([{ path: title, videos: [ep1] }]);
// 季 / 集单元缺元数据时不写占位 NFO:模拟刷新元数据已填过整理懒建的单元
await h.db
.updateTable("content_unit")
.set({ metadata: JSON.stringify({ name: { value: "名称", source: "tmdb", language: null, at, locked: false } }) })
.where("media_entity_id", "=", entity)
.execute();
await runScrapeRecord(h.module.deps().ao, recs[0]!.id); await runScrapeRecord(h.module.deps().ao, recs[0]!.id);
const show = await readdir(join(h.lib, title)); const show = await readdir(join(h.lib, title));
const season = await readdir(join(h.lib, title, "Season 1")); const season = await readdir(join(h.lib, title, "Season 1"));
+1 -1
View File
@@ -277,7 +277,7 @@ describe.skipIf(skip)("files 刮削(SCR-2:任意目录就地刮削)", () => {
expect(show).toMatchObject({ expect(show).toMatchObject({
status: "succeeded", status: "succeeded",
path: SHOW, path: SHOW,
mode: "missing", mode: "update",
trigger: "manual", trigger: "manual",
match: { match: {
mediaEntityId: null, mediaEntityId: null,
@@ -0,0 +1,179 @@
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { createMockTmdb, createTempTree, type MockTmdb, type TempTree, TMDB_FIXTURES } from "@umt/testkit";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import type { AoPorts } from "../../src/modules/acquisition-organize/ports.ts";
import type { AoDeps } from "../../src/modules/acquisition-organize/services/common.ts";
import {
RESCRAPE_DELAY_MS,
runScrapeRecord,
SCRAPE_TASK,
scrapeEntity,
} from "../../src/modules/acquisition-organize/services/scrape.ts";
import { createEntityFromTmdbCandidate } from "../../src/modules/sources-identity/service/entity-from-candidate.ts";
import { setEntityIdentity } from "../../src/modules/sources-identity/service/identity-admin.ts";
import { refreshMetadata, setFieldOverride } from "../../src/modules/sources-identity/service/metadata.ts";
import { createHarness, type Harness, pgAvailable } from "./harness.ts";
// 改作品的 TMDB 映射后:清掉旧作品的 tmdb 来源字段(人工锁定的保留)→ full 刷新 → 刮削过的作品延后排一次「更新」模式重刮;
// 重刮后的 NFO 标题 / 简介 / IMDb / 海报都是新作品的,不残留旧作品;刷新期间再刷新合进同一条排队记录(防抖)。
const up = await pgAvailable();
const enc = (s: string) => new TextEncoder().encode(s);
const VIDEO = "Movies/奥本海默 (2023)/Oppenheimer.2023.1080p.mkv";
const DIR = "Movies/奥本海默 (2023)";
const NFO = `${DIR}/Oppenheimer.2023.1080p.nfo`;
type Final = { value: unknown; source: string; locked: boolean };
describe.skipIf(!up)("改映射后不再新旧混写:清字段 → 刷新 → 更新模式重刮", () => {
let h: Harness;
let tmdb: MockTmdb;
let lib: TempTree;
let d: AoDeps;
let id = "";
const actor = { type: "user" as const, id: "tester" };
const enqueueOpts: { task: string; availableAt?: Date | undefined }[] = [];
beforeAll(async () => {
// 沙丘 2 补一张海报,验证改映射后海报换成新作品的
const fixtures = TMDB_FIXTURES.map((f) => (f.id === 693134 ? { ...f, poster_path: "/dune.jpg" } : f));
tmdb = await createMockTmdb({ fixtures });
h = await createHarness("idrescrape", { tmdbUrl: tmdb.url });
lib = await createTempTree({ [VIDEO]: 1000 }, { sentinel: true, prefix: "umt-idscr-" });
const orig = h.engine.enqueue.bind(h.engine);
h.engine.enqueue = async (task, input, o) => {
enqueueOpts.push({ task, availableAt: o.availableAt });
return orig(task, input, o);
};
d = {
ctx: h.ctx,
ports: {
now: () => new Date(),
scrapeImage: async (p: string) => ({ content: enc(p), mime: "image/jpeg" }),
scrapeTmdbDetail: async () => null,
scrapeTmdbSeason: async () => null,
} as unknown as AoPorts,
};
}, 60_000);
afterAll(async () => {
await h?.close();
await tmdb?.close();
await lib?.cleanup();
});
const queuedUpdates = () =>
h.db
.selectFrom("scrape_record")
.select(["id", "mode", "trigger", "status"])
.where("media_entity_id", "=", id)
.where("status", "=", "queued")
.execute();
it("先按奥本海默建档、刷新、刮削;没刮削过时刷新不排重刮", async () => {
const r = await createEntityFromTmdbCandidate(
h.db,
() => h.rt.tmdb(),
{ externalId: "872585", kind: "movie" },
actor,
);
id = r.mediaEntityId;
expect((await refreshMetadata(h.rt, id, { full: true })).status).toBe("refreshed");
expect(h.engine.enqueued.some((x) => x.task === SCRAPE_TASK)).toBe(false);
await setFieldOverride(h.rt, id, "tagline", { value: "我的标语", lock: true }, actor);
const root = await h.db
.insertInto("library_root")
.values({ name: "lib-idscr", media_kind: "mixed", local_path: lib.root, management_mode: "managed" })
.returning("id")
.executeTakeFirstOrThrow();
const asset = await h.db
.insertInto("media_asset")
.values({ media_entity_id: id, kind: "video" })
.returning("id")
.executeTakeFirstOrThrow();
await h.db
.insertInto("asset_location")
.values({ media_asset_id: asset.id, library_root_id: root.id, relative_path: VIDEO })
.execute();
const q = await scrapeEntity(d, id, { trigger: "manual" });
expect(q).toHaveLength(1);
const rec = await runScrapeRecord(d, q[0]!.recordId);
expect(rec).toMatchObject({ status: "succeeded", mode: "update" });
expect(rec.files.every((f) => f.action !== "written" || /^[0-9a-f]{64}$/.test(f.hash ?? ""))).toBe(true);
const nfo = await readFile(join(lib.root, NFO), "utf8");
expect(nfo).toContain("<title>奥本海默</title>");
expect(nfo).toContain("tt15398776");
expect(nfo).toContain("<tagline>我的标语</tagline>");
expect(await readFile(join(lib.root, DIR, "poster.jpg"), "utf8")).toBe("/oppenheimer.jpg");
}, 60_000);
it("改映射到沙丘 2:tmdb 来源字段清空,锁定字段保留,排 full 刷新", async () => {
const before = h.engine.enqueued.length;
const r = await setEntityIdentity(
h.rt,
id,
{ provider: "tmdb", entityType: "movie", externalId: "693134", lock: true },
actor,
);
expect(r.changed).toBe(true);
const e = await h.db.selectFrom("media_entity").select(["metadata"]).where("id", "=", id).executeTakeFirstOrThrow();
const meta = e.metadata as Record<string, Final>;
expect(meta.overview).toBeUndefined();
expect(meta.posterPath).toBeUndefined();
expect(meta.externalIds).toBeUndefined();
expect(meta.tagline).toMatchObject({ value: "我的标语", locked: true });
const tmdbRows = await h.db
.selectFrom("metadata_field")
.select("field")
.where("media_entity_id", "=", id)
.where("source", "=", "tmdb")
.execute();
expect(tmdbRows).toEqual([]);
expect(h.engine.enqueued.slice(before)).toEqual([
expect.objectContaining({ task: "refreshMetadata", input: { mediaEntityId: id, full: true } }),
]);
});
it("刷新后延后排一次更新模式重刮;期间再刷新合进同一条", async () => {
const t0 = Date.now();
expect((await refreshMetadata(h.rt, id, { full: true })).status).toBe("refreshed");
const queued = await queuedUpdates();
expect(queued).toEqual([expect.objectContaining({ mode: "update", trigger: "auto" })]);
const opt = enqueueOpts.filter((x) => x.task === SCRAPE_TASK).at(-1);
expect(opt?.availableAt?.getTime()).toBeGreaterThanOrEqual(t0 + RESCRAPE_DELAY_MS - 1000);
// 防抖:还没开始时再来一次刷新(哪怕字段又变了)不另建记录、不另排任务
await h.db
.updateTable("metadata_field")
.set({ value: JSON.stringify("旧值") })
.where("media_entity_id", "=", id)
.where("field", "=", "overview")
.execute();
const scrapes = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK).length;
await refreshMetadata(h.rt, id, { full: true });
expect(await queuedUpdates()).toHaveLength(1);
expect(h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK).length).toBe(scrapes);
}, 60_000);
it("重刮后 NFO 的标题 / 简介 / IMDb / 海报都是沙丘 2,不残留奥本海默;锁定字段照写", async () => {
const [q] = await queuedUpdates();
const rec = await runScrapeRecord(d, q!.id);
expect(rec.status).toBe("succeeded");
expect(rec.files.find((f) => f.role === "movie_nfo")).toMatchObject({ action: "written" });
const nfo = await readFile(join(lib.root, NFO), "utf8");
expect(nfo).toContain("<title>沙丘2</title>");
expect(nfo).toContain("<tmdbid>693134</tmdbid>");
expect(nfo).toContain("tt15239678");
expect(nfo).not.toContain("tt15398776");
expect(nfo).not.toContain("奥本海默");
expect(nfo).not.toContain("原子弹之父");
expect(nfo).toContain("<tagline>我的标语</tagline>");
expect(await readFile(join(lib.root, DIR, "poster.jpg"), "utf8")).toBe("/dune.jpg");
// 字段没变的刷新不再排重刮
const scrapes = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK).length;
await refreshMetadata(h.rt, id, { full: true });
expect(h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK).length).toBe(scrapes);
}, 60_000);
});
+3 -2
View File
@@ -1093,8 +1093,8 @@ const SKIP_TEXT: Record<string, string> = {
export function ScrapeDialog({ items, root, onClose }: { items: FileEntry[]; root: FileRoot; onClose: () => void }) { export function ScrapeDialog({ items, root, onClose }: { items: FileEntry[]; root: FileRoot; onClose: () => void }) {
const settings = useSettings(); const settings = useSettings();
const scrape = useScrape(); const scrape = useScrape();
const [mode, setMode] = useState<"missing" | "overwrite" | null>(null); const [mode, setMode] = useState<"missing" | "overwrite" | "update" | null>(null);
const curMode = mode ?? settings.data?.organize.scrapeMode ?? "missing"; const curMode = mode ?? settings.data?.organize.scrapeMode ?? "update";
const [recordIds, setRecordIds] = useState<string[]>([]); const [recordIds, setRecordIds] = useState<string[]>([]);
const [choices, setChoices] = useState<FilesScrapeNeedsChoice[]>([]); const [choices, setChoices] = useState<FilesScrapeNeedsChoice[]>([]);
const [picked, setPicked] = useState<Record<string, WizardEntity | null>>({}); const [picked, setPicked] = useState<Record<string, WizardEntity | null>>({});
@@ -1157,6 +1157,7 @@ export function ScrapeDialog({ items, root, onClose }: { items: FileEntry[]; roo
<fieldset aria-label="已有文件" className="m-0 flex flex-wrap gap-1.5 border-0 p-0"> <fieldset aria-label="已有文件" className="m-0 flex flex-wrap gap-1.5 border-0 p-0">
{( {(
[ [
["update", "更新"],
["missing", "只补缺失"], ["missing", "只补缺失"],
["overwrite", "全部覆盖"], ["overwrite", "全部覆盖"],
] as const ] as const
+1 -1
View File
@@ -704,7 +704,7 @@ let seq = 0;
const SCRAPES = new Map<string, ScrapeRecord>(); const SCRAPES = new Map<string, ScrapeRecord>();
function mockScrape(b: { function mockScrape(b: {
items: { root: string; path: string; mediaEntityId?: string; external?: { externalId: string } }[]; items: { root: string; path: string; mediaEntityId?: string; external?: { externalId: string } }[];
mode?: "missing" | "overwrite"; mode?: "missing" | "overwrite" | "update";
}) { }) {
const records: ScrapeRecord[] = []; const records: ScrapeRecord[] = [];
const needsChoice: unknown[] = []; const needsChoice: unknown[] = [];
@@ -70,7 +70,7 @@ function FileRow({ f }: { f: ScrapeFileResult }) {
)} )}
</span> </span>
<span className="text-meta text-muted-foreground">{ROLE[f.role] ?? f.role}</span> <span className="text-meta text-muted-foreground">{ROLE[f.role] ?? f.role}</span>
{f.action !== "written" && f.reason && f.reason !== "scrape_exists" ? ( {f.action !== "written" && f.reason && !QUIET_SKIPS.has(f.reason) ? (
<span className={f.action === "failed" ? "text-meta text-destructive" : "text-meta text-muted-foreground"}> <span className={f.action === "failed" ? "text-meta text-destructive" : "text-meta text-muted-foreground"}>
{reasonLine(f.reason, "写入失败")} {reasonLine(f.reason, "写入失败")}
</span> </span>
@@ -80,11 +80,14 @@ function FileRow({ f }: { f: ScrapeFileResult }) {
); );
} }
/** 不必逐条列出的跳过:已存在(不是本工具写的 / 只补缺失)、内容没变 */
const QUIET_SKIPS = new Set(["scrape_exists", "scrape_unchanged"]);
/** 一条刮削记录:状态、目录与识别结果、计数、逐文件结果(缺省不列"已存在"跳过的) */ /** 一条刮削记录:状态、目录与识别结果、计数、逐文件结果(缺省不列"已存在"跳过的) */
export function ScrapeRecordView({ record: s, showPath = false }: { record: ScrapeRecord; showPath?: boolean }) { export function ScrapeRecordView({ record: s, showPath = false }: { record: ScrapeRecord; showPath?: boolean }) {
const [showAll, setShowAll] = useState(false); const [showAll, setShowAll] = useState(false);
const files = s.files; const files = s.files;
const important = files.filter((f) => f.action !== "skipped" || f.reason !== "scrape_exists"); const important = files.filter((f) => f.action !== "skipped" || !QUIET_SKIPS.has(f.reason ?? ""));
const shown = showAll ? files : important.slice(0, 12); const shown = showAll ? files : important.slice(0, 12);
const hidden = files.length - shown.length; const hidden = files.length - shown.length;
const done = s.status !== "queued" && s.status !== "running"; const done = s.status !== "queued" && s.status !== "running";
+6 -2
View File
@@ -57,10 +57,11 @@ const SCRAPE_CARD: CardSpec = {
label: "已有文件", label: "已有文件",
kind: "select", kind: "select",
options: [ options: [
{ value: "update", label: "更新" },
{ value: "missing", label: "只补缺失" }, { value: "missing", label: "只补缺失" },
{ value: "overwrite", label: "全部覆盖" }, { value: "overwrite", label: "全部覆盖" },
], ],
hint: "全部覆盖会替换目录里已有的 NFO 和图片,包括其他工具写的", hint: "更新只替换本工具写过、之后没被改过的文件;全部覆盖会替换目录里已有的 NFO 和图片,包括其他工具写的",
}, },
{ {
name: "organize.scrapeImages", name: "organize.scrapeImages",
@@ -84,13 +85,16 @@ const SCRAPE_CARD: CardSpec = {
toPatch: (v) => ({ toPatch: (v) => ({
organize: { organize: {
scrapeAfterImport: v["organize.scrapeAfterImport"] === true, scrapeAfterImport: v["organize.scrapeAfterImport"] === true,
scrapeMode: str(v["organize.scrapeMode"]) === "overwrite" ? "overwrite" : "missing", scrapeMode: scrapeModeOf(str(v["organize.scrapeMode"])),
scrapeImages: v["organize.scrapeImages"] === true, scrapeImages: v["organize.scrapeImages"] === true,
scrapeEpisodeThumbs: v["organize.scrapeEpisodeThumbs"] === true, scrapeEpisodeThumbs: v["organize.scrapeEpisodeThumbs"] === true,
}, },
}), }),
}; };
const scrapeModeOf = (v: string | null | undefined): "missing" | "overwrite" | "update" =>
v === "overwrite" || v === "missing" ? v : "update";
export function LibrarySection() { export function LibrarySection() {
return ( return (
<div className="grid grid-cols-1 gap-8"> <div className="grid grid-cols-1 gap-8">
+1 -1
View File
@@ -25,7 +25,7 @@
| AcquisitionPlan(下载计划,DL-3 起界面不单独命名) | 下载;还没提交到下载器的按在等什么写状态:「等你批准」「等磁盘空间」「等开启自动化」「准备提交」;没提交就结束的写「没提交成功 / 已取消 / 已过期」+ 原因;来由写「为什么下这个」;智能体动作计数写"准备下载 N 个资源" | 下载计划、草稿、已预留(界面) | | AcquisitionPlan(下载计划,DL-3 起界面不单独命名) | 下载;还没提交到下载器的按在等什么写状态:「等你批准」「等磁盘空间」「等开启自动化」「准备提交」;没提交就结束的写「没提交成功 / 已取消 / 已过期」+ 原因;来由写「为什么下这个」;智能体动作计数写"准备下载 N 个资源" | 下载计划、草稿、已预留(界面) |
| DownloadTask | 下载任务 | 下载单 | | DownloadTask | 下载任务 | 下载单 |
| Organization record | 整理记录;动作叫"整理",结果状态叫"已入库" | 转移、入库记录 | | Organization record | 整理记录;动作叫"整理",结果状态叫"已入库" | 转移、入库记录 |
| 在目录里就地写 NFO 与图片(scrapeMedia,SCR-1 / SCR-2) | 刮削;文件管理菜单「刮削…」,作品详情「重新刮削」「覆盖重写」;覆盖方式叫「只补缺失 / 全部覆盖」;认不准作品时"选一个再刮削";只更新本工具字段的叫「刷新元数据」 | 投影、NFO 投影、写元数据(界面) | | 在目录里就地写 NFO 与图片(scrapeMedia,SCR-1 / SCR-2) | 刮削;文件管理菜单「刮削…」,作品详情「重新刮削」「覆盖重写」;覆盖方式叫「更新 / 只补缺失 / 全部覆盖」(更新 = 只替换本工具写过且没被改过的文件);认不准作品时"选一个再刮削";只更新本工具字段的叫「刷新元数据」 | 投影、NFO 投影、写元数据(界面) |
| 整理方式 | 复制 / 移动 / 硬链接 | Copy / Move | | 整理方式 | 复制 / 移动 / 硬链接 | Copy / Move |
| Library root | 媒体库目录 | 库根、媒体库根、根目录 | | Library root | 媒体库目录 | 库根、媒体库根、根目录 |
| 文件管理可浏览的目录(库根 + 下载器路径映射) | 登记目录 | 登记根、根 | | 文件管理可浏览的目录(库根 + 下载器路径映射) | 登记目录 | 登记根、根 |
@@ -426,13 +426,17 @@ export interface TmdbPort { searchMovie(q: string, year?: number): Promise<TmdbS
## 10. 刮削(NFO、图片) ## 10. 刮削(NFO、图片)
| 上游 | 行为 | 本项目 | 按 2026-10 的实现改写;细节与排查见 `docs/runbook/scrape.md`。
| 上游 | 行为 | 本项目(现状) |
| --- | --- | --- | | --- | --- | --- |
| `app/chain/scraping.py:148 ScrapingOption`、`:243 get_default_config` | 按 `(movie|tv|season|episode)_(nfo|poster|backdrop|logo|disc|banner|thumb|clearart|landscape)` 逐项配置策略 `skip/missingonly/overwrite/upgrade`,默认 `missingonly` | 保留"逐项策略"的结构,放在 `policy_revision` 下的目录策略中(REQ §14.1.3:"NFO/海报写入受目录策略控制") | | `app/chain/scraping.py:148 ScrapingOption`、`:243 get_default_config` | 按 `(movie\|tv\|season\|episode)_(nfo\|poster\|backdrop\|…)` 逐项配置策略 `skip/missingonly/overwrite/upgrade`,默认 `missingonly` | 没有逐项策略,只有全局覆盖方式 `organize.scrapeMode`:`update` 更新(缺省)/ `missing` 只补缺失 / `overwrite` 全部覆盖;图片与剧照各有一个开关 |
| `:320 _should_scrape` | skip 时不写;文件不存在时写;文件存在时,只有 overwrite 或全局 overwrite 才写 | 保留;另外增加一条:**用户或其他工具已修改的 NFO(含 `<lockdata>true` 或 hash 与上次写入不一致)不覆盖** | | `:320 _should_scrape` | skip 时不写;文件不存在时写;文件存在时,只有 overwrite 或全局 overwrite 才写 | `missing` / `overwrite` 同 MP。新增 `update`:每次写入在 `scrape_record.files[].hash` 记内容 sha256,目标存在且与本工具上次写入的 hash 一致(没被改过)才用新内容覆盖,内容没变不重写;被改过的跳过(`scrape_user_modified`),不是本工具写的跳过(`scrape_exists`);旧版本没有 hash 的记录按修改时间兜底。**不读 `<lockdata>`,也不写它** |
| `:356 _save_file` | 先写临时文件,chmod `0666 & ~umask`,再经存储链上传 | 保留"临时文件加原子替换" | | `:356 _save_file` | 先写临时文件,chmod `0666 & ~umask`,再经存储链上传 | 同目录临时文件 + rename(`services/scrape.ts` `atomicWrite`),目标必须在登记目录内 |
| `app/modules/themoviedb/scraper.py:35 get_metadata_nfo` | 电影、剧、季、集四种 NFO;季和集走 `get_tv_season_detail`,有剧集组时走 `get_tv_group_detail`;标签包括 `uniqueid(tmdb default, imdb, tvdb)`、`plot`、`genre`、`actor`、`director`、`studio`、`country`、`mpaa`、`rating`、`runtime` | `packages/adapters/src/nfo/writer.ts` 输出 Kodi/Emby 兼容 NFO;**另写一个 `uniqueid type="uhd"`,内容为本地 UUID** | | `app/modules/themoviedb/scraper.py:35 get_metadata_nfo` | 电影、剧、季、集四种 NFO;标签包括 `uniqueid(tmdb default, imdb, tvdb)`、`plot`、`genre`、`actor`、`director`、`studio`、`country`、`mpaa`、`rating`、`runtime` | `packages/domain/src/scrape/nfo.ts`(纯函数,与 MP 元素顺序一致,金样差分 `apps/server/test/scrape/nfo-mp-diff.test.ts`,偏离登记 `nfo-deviations.json`);`uniqueid` 只有 tmdb / imdb / tvdb / douban,**没有 `type="uhd"`**;不写 `mpaa`、`episodeguide`、`lockdata`。合集 / 其他类作品按电影写(根元素 `<movie>`,判定函数 `scrapeKindOf`) |
| 失败语义 | 刮削在整理后执行,失败只记日志,不回滚转移 | 保留"不重做搬运";改为**独立任务 `refreshMetadata`,有自己的状态和重试**,不能伪报全部步骤成功(REQ §14.1.3) | | 季 / 集元数据 | `get_tv_season_detail` 现取;集名缺时写占位 | 本地作品用投影后的季 / 集字段;单元缺元数据时按 TMDB id 现取季详情补齐,仍缺则该文件跳过(`scrape_metadata_missing`)、记录 `partial`,**不写「季 N」「第 N 集」占位** |
| 元数据变化后 | 不会自动重刮 | 刷新元数据后 NFO 相关字段有变化、且作品刮削过 → 延后 2 分钟排一次 `update` 模式重刮(同目录排队记录合批防抖);改 TMDB 映射时先清旧作品的 tmdb 来源字段与没锁的 IMDb 映射,再 full 刷新,刷新后按同一规则重刮 |
| 失败语义 | 刮削在整理后执行,失败只记日志,不回滚转移 | 不重做搬运;刮削是独立任务 `scrapeMedia`(不重试,失败的重新刮削),每次一条 `scrape_record`(逐文件结果与原因);非写文件的意外错误记 `scrape_internal_error` 并保留已写文件结果 |
--- ---
+66 -8
View File
@@ -18,6 +18,8 @@ SCR-2(用户 2026-09-30:"已经有的也可以重新刮削……如果是直接
| `apps/server/src/modules/file-manager/scrape.ts` | 文件管理「刮削」:按路径识别(复用 `identifyCore` + `resolveIdentity`)、候选、排任务、请求内等结果 | | `apps/server/src/modules/file-manager/scrape.ts` | 文件管理「刮削」:按路径识别(复用 `identifyCore` + `resolveIdentity`)、候选、排任务、请求内等结果 |
| `handlers.ts` 任务 `scrapeMedia`(v2,输入 `{recordId}`) | 文件道;不重试(失败的重新刮削) | | `handlers.ts` 任务 `scrapeMedia`(v2,输入 `{recordId}`) | 文件道;不重试(失败的重新刮削) |
| 迁移 `0058_scrape_record` | 每次刮削一条记录;替代 0057 的 `media_scrape`(一部作品一行) | | 迁移 `0058_scrape_record` | 每次刮削一条记录;替代 0057 的 `media_scrape`(一部作品一行) |
| 迁移 `0070_scrape_update_mode` | `scrape_record.mode` 增加 `update`;逐文件 hash 记在 `files` jsonb 里,不加列 |
| `sources-identity/service/metadata.ts` `refreshMetadata` / `clearTmdbMetadata` | 刷新后 NFO 字段有变化 → `scheduleEntityRescrape`;改 TMDB 映射时清旧字段(`identity-admin.ts` `setEntityIdentity` 调用) |
| 端口 `AoPorts.scrapeImage` / `scrapeTmdbDetail` / `scrapeTmdbSeason` | 图片字节(图片缓存 `DATA_DIR/cache/images` + 出口代理);未登记目录直接取 TMDB 富详情 / 季详情(元数据语言链,经 TMDB 缓存) | | 端口 `AoPorts.scrapeImage` / `scrapeTmdbDetail` / `scrapeTmdbSeason` | 图片字节(图片缓存 `DATA_DIR/cache/images` + 出口代理);未登记目录直接取 TMDB 富详情 / 季详情(元数据语言链,经 TMDB 缓存) |
## 触发 ## 触发
@@ -27,9 +29,58 @@ SCR-2(用户 2026-09-30:"已经有的也可以重新刮削……如果是直接
| 文件管理「刮削…」(右键 / ⋯ 菜单、多选批量栏、详情按钮) | `POST /files/scrape {items[{root,path,mediaEntityId?,external?}], mode?, addToLibrary?}`;任意视频文件或目录,每项一条记录,请求内最多等 `FILES_TRANSFER_WAIT_SEC`(缺省 20 s),没刮完的 `pending=true`,前端按 `GET /files/scrape-records?ids=` 轮询 | | 文件管理「刮削…」(右键 / ⋯ 菜单、多选批量栏、详情按钮) | `POST /files/scrape {items[{root,path,mediaEntityId?,external?}], mode?, addToLibrary?}`;任意视频文件或目录,每项一条记录,请求内最多等 `FILES_TRANSFER_WAIT_SEC`(缺省 20 s),没刮完的 `pending=true`,前端按 `GET /files/scrape-records?ids=` 轮询 |
| 作品详情「重新刮削」「覆盖重写」 | `POST /library/{id}/scrape {mode?}`(202):对作品目录刮削,每个目录一条记录;没有已入库文件 412(到文件管理里对目录刮削) | | 作品详情「重新刮削」「覆盖重写」 | `POST /library/{id}/scrape {mode?}`(202):对作品目录刮削,每个目录一条记录;没有已入库文件 412(到文件管理里对目录刮削) |
| 整理入库成功(`finalizeRecord` 结果为已入库 / 部分入库) | 设置 `organize.scrapeAfterImport` 打开时对作品目录排刮削,幂等键 `scrapeMedia:record:<整理记录 id>:<序号>` | | 整理入库成功(`finalizeRecord` 结果为已入库 / 部分入库) | 设置 `organize.scrapeAfterImport` 打开时对作品目录排刮削,幂等键 `scrapeMedia:record:<整理记录 id>:<序号>` |
| 元数据刷新后 NFO 字段有变化 | 只对刮削过的作品(有成功 / 部分成功的记录):对作品目录排「更新」模式刮削(`trigger=auto`),任务延后 2 分钟开始;见下「刷新后自动重刮」 |
| 改作品的 TMDB 映射 | 清掉旧作品的 tmdb 来源字段 → 排 full 刷新(幂等键 `refreshMetadata:identity:<新身份 id>`)→ 刷新写回新字段后按上一行自动重刮 |
同一目录已有排队中(未开始,30 分钟内)、同覆盖方式、同作品的刮削时直接复用(`deduplicated`)。 同一目录已有排队中(未开始,30 分钟内)、同覆盖方式、同作品的刮削时直接复用(`deduplicated`)。
### 覆盖方式
| 方式 | 目标不存在 | 目标存在 |
| --- | --- | --- |
| `update` 更新(缺省) | 写 | 本工具上次写入、之后没被改过 → 按新内容覆盖(内容没变就不重写,记 `scrape_unchanged`);被改过 → 跳过 `scrape_user_modified`;不是本工具写的 → 跳过 `scrape_exists` |
| `missing` 只补缺失 | 写 | 跳过 `scrape_exists` |
| `overwrite` 全部覆盖 | 写 | 覆盖(包括其他工具写的、人改过的) |
- 每次写入(任何方式)在该记录的逐文件结果里记内容的 sha256(`files[].hash`);「更新」时取同一登记目录里以前的记录,
每个路径按从新到旧第一条带 hash 的结果,与磁盘上现在的内容比对。跳过的结果不带 hash,人改过的文件以后每次都还是跳过,
要替换就用「全部覆盖」。
- 旧版本写的文件没有 hash:修改时间不晚于那次刮削结束(留 2 秒余量)就当没被改过。
- 已存在的目标是目录 / 符号链接时三种方式都跳过。
- 更新不删除文件:改身份后新作品没有背景图时,旧的 `fanart.jpg` 留在目录里(元数据里旧的背景图路径已清掉,不会再写旧图)。
### 刷新后自动重刮
- `refreshMetadata` 结束后,作品写进 NFO 的字段(标题、原名、年份、简介、标语、首映、类型、制片国、平台、时长、演职员、外部 ID、海报、背景图)
或季 / 集的名称、简介、日期、时长、剧照、季海报有变化时,调 `scheduleEntityRescrape`。评分、票数、播出状态的变化不触发(避免每天刷新都改写 NFO,
下次因为别的原因重刮时会带上新评分)。
- 没刮削过的作品不排(要先手动或入库后刮一次)。
- 防抖:任务带 `availableAt = 现在 + 2 分钟`;这段时间里再来的刷新命中同目录、同作品、`update` 方式的排队记录直接复用,不另排。
- 排上的记录在作品详情「刮削」卡片里显示为「入库后自动」那一类(`trigger=auto`)、方式 `update`。
### 改 TMDB 映射(纠错)后
`setEntityIdentity`(作品页改映射、`POST /media-entities/{id}/identities`)在同一事务里:
1. 旧映射退役、新映射接受(原有逻辑);
2. 旧 TMDB 带出来的 IMDb 映射(没人工锁的)一并退役——NFO 里 IMDb 是 default uniqueid,留着媒体服务器会认成旧作品;
3. 清掉该作品及其季 / 集单元的 tmdb 来源字段(`metadata_field` 与投影出的最终值;单元的 `title` / `air_date` 列跟着清),
人工覆盖、锁定字段和 web / agent 补全字段保留;海报 / 背景图一并清掉,不再按「TMDB 没有时保留旧值」留旧作品的图。
提交后排 full 刷新;刷新填回新作品字段后自动重刮(上一节),本工具写过且没被改过的 NFO 和图片换成新作品的。
刷新完成前若刚好有刮削在跑,季 / 集缺元数据的部分会按下一节跳过,不会写进旧作品的内容。
### 季 / 集缺元数据
本地作品有元数据时用最终字段;季单元或要刮的集单元**缺元数据**(没有单元,或从没被元数据来源填过、也没有标题 / 播出日期,
典型是整理懒建的集、改身份后被清空的单元)时,先按作品的 TMDB id 现取该季详情补齐(映射同刷新元数据,只补缺的单元,本地已有的字段不动,
不落库)。仍然缺(没有 TMDB id、季详情失败、TMDB 没有这一集)时:
- 该季 `season.nfo` / 该集 NFO 跳过,原因 `scrape_metadata_missing`,**不写「季 N」「第 N 集」占位**;
- 记录状态是 `partial`(原因 `scrape_metadata_missing`),不再报成功;刷新元数据后会自动重刮(上一节),或手动重新刮削。
TMDB 有这一集但没有译名 / 简介(只有占位名和播出日期)不算缺,照常写(和 MP 一样)。未登记目录直接取 TMDB 时同一规则:季详情失败的季跳过。
### 识别(文件管理) ### 识别(文件管理)
顺序:请求指定 > 目录里已登记文件所属的唯一作品(`source=library`)> 按路径识别(`source=auto`)。 顺序:请求指定 > 目录里已登记文件所属的唯一作品(`source=library`)> 按路径识别(`source=auto`)。
@@ -53,6 +104,8 @@ SCR-2(用户 2026-09-30:"已经有的也可以重新刮削……如果是直接
- 集号:已登记视频按覆盖的集单元;其余按路径解析(季号缺失时取季目录名,再缺按第 1 季;同 MP 对每集重新 `MetaInfoPath`)。 - 集号:已登记视频按覆盖的集单元;其余按路径解析(季号缺失时取季目录名,再缺按第 1 季;同 MP 对每集重新 `MetaInfoPath`)。
认不出集号的视频记 `scrape_no_episode` 跳过。多集文件按首集。 认不出集号的视频记 `scrape_no_episode` 跳过。多集文件按首集。
- 字段:本地作品有元数据时用最终字段(字段锁、人工覆盖优先,见 `metadata.md`);否则按 TMDB id 直接取 TMDB(与刷新元数据同一映射)。 - 字段:本地作品有元数据时用最终字段(字段锁、人工覆盖优先,见 `metadata.md`);否则按 TMDB id 直接取 TMDB(与刷新元数据同一映射)。
TMDB id 只认与刮削类型一致的映射(同一作品同时有 movie / tv 两个 accepted 映射时不串)。
- 作品类型:剧集 / 综艺按剧集刮,电影 / 合集 / 其他按电影刮(`<视频名>.nfo`,根元素 `<movie>`),范围判定与 NFO 生成同一个函数 `scrapeKindOf`。
- 登记目录:库根只写 `managed`(只登记模式跳过 `scrape_root_read_only`),库根离线 / 标记文件不对 `scrape_root_offline`; - 登记目录:库根只写 `managed`(只登记模式跳过 `scrape_root_read_only`),库根离线 / 标记文件不对 `scrape_root_offline`;
下载根也能刮削(同 MP)。目标路径必须在登记目录内;已存在的目标是目录 / 符号链接时跳过。 下载根也能刮削(同 MP)。目标路径必须在登记目录内;已存在的目标是目录 / 符号链接时跳过。
@@ -97,7 +150,8 @@ SCR-2(用户 2026-09-30:"已经有的也可以重新刮削……如果是直接
| 项 | MP | 我们 | 理由 | | 项 | MP | 我们 | 理由 |
| --- | --- | --- | --- | | --- | --- | --- | --- |
| 已有文件 | 手动刮削一律覆盖(`overwrite=True`),自动刮削按每类文件的策略 | 缺省按设置「只补缺失 / 全部覆盖」,刮削对话框里可临时改 | 用户要求覆盖方式沿用设置、菜单里可临时选;缺省不动其他工具写的文件 | | 已有文件 | 手动刮削一律覆盖(`overwrite=True`),自动刮削按每类文件的策略 | 缺省「更新」(只覆盖本工具写过且没被改过的,按 hash 判断),也可选「只补缺失 / 全部覆盖」,刮削对话框里可临时改 | 元数据纠正后已有 NFO 要能跟着更新,又不能覆盖用户手改的、其他工具写的 |
| 季 / 集缺元数据 | 写占位(「第 N 集」) | 现取季详情补齐,仍缺跳过并记 partial | 占位 NFO 在「只补缺失」下永远不会被替换 |
| 识别 | 按路径识别,认不出直接失败;对话框可手填 TMDB / 豆瓣 id | 按路径识别,认不准时返回前 5 候选让用户选(也可搜索);已登记的目录直接用作品 | 选一次比手填 id 省事;已登记的不必再识别 | | 识别 | 按路径识别,认不出直接失败;对话框可手填 TMDB / 豆瓣 id | 按路径识别,认不准时返回前 5 候选让用户选(也可搜索);已登记的目录直接用作品 | 选一次比手填 id 省事;已登记的不必再识别 |
| 执行 | 请求内同步 | 后台任务 `scrapeMedia`,请求内最多等 20 s | 大季包图片多;超时不丢 | | 执行 | 请求内同步 | 后台任务 `scrapeMedia`,请求内最多等 20 s | 大季包图片多;超时不丢 |
| 记录 | 只有日志 | 每次刮削一条记录(目录、识别结果、逐文件结果、原因),作品详情显示最近一条 | 可回看、可排查 | | 记录 | 只有日志 | 每次刮削一条记录(目录、识别结果、逐文件结果、原因),作品详情显示最近一条 | 可回看、可排查 |
@@ -110,7 +164,7 @@ SCR-2(用户 2026-09-30:"已经有的也可以重新刮削……如果是直接
| 字段 | 缺省 | 说明 | | 字段 | 缺省 | 说明 |
| --- | --- | --- | | --- | --- | --- |
| `organize.scrapeAfterImport` | 关 | 入库后自动刮削(MP 目录「刮削」缺省同样关) | | `organize.scrapeAfterImport` | 关 | 入库后自动刮削(MP 目录「刮削」缺省同样关) |
| `organize.scrapeMode` | `missing` | `missing` 只补缺失(MP「仅缺失」)/ `overwrite` 全部覆盖;手动刮削时对话框可临时改 | | `organize.scrapeMode` | `update` | `update` 更新 / `missing` 只补缺失(MP「仅缺失」)/ `overwrite` 全部覆盖,见上「覆盖方式」;手动刮削时对话框可临时改。以前保存过的 `missing` / `overwrite` 照旧生效,要用更新到设置里改 |
| `organize.scrapeImages` | 开 | 海报、背景图、季海报 | | `organize.scrapeImages` | 开 | 海报、背景图、季海报 |
| `organize.scrapeEpisodeThumbs` | 开 | 每集剧照;`scrapeImages` 关时不生效 | | `organize.scrapeEpisodeThumbs` | 开 | 每集剧照;`scrapeImages` 关时不生效 |
@@ -121,8 +175,9 @@ SCR-2(用户 2026-09-30:"已经有的也可以重新刮削……如果是直接
- 公共(电影 / 剧集):`tmdbid` + `uniqueid type=tmdb`;有 TVDB / IMDb 时追加(有 IMDb 时 IMDb 为 `default`);`plot` / `outline`(CDATA); - 公共(电影 / 剧集):`tmdbid` + `uniqueid type=tmdb`;有 TVDB / IMDb 时追加(有 IMDb 时 IMDb 为 `default`);`plot` / `outline`(CDATA);
`director`(带 `tmdbid`);`actor{name,type,role,tmdbid,thumb,profile}`;`genre`;`rating`(一位小数);`studio`;`country`;`runtime`; `director`(带 `tmdbid`);`actor{name,type,role,tmdbid,thumb,profile}`;`genre`;`rating`(一位小数);`studio`;`country`;`runtime`;
`title`、`originaltitle`、`premiered`、`year`;剧集另有 `season=-1`、`episode=-1`。 `title`、`originaltitle`、`premiered`、`year`;剧集另有 `season=-1`、`episode=-1`。
- 季:`plot`、`outline`、`title`(没有时「季 N」)、`premiered`、`releasedate`、`year`、`seasonnumber`。 - 季:`plot`、`outline`、`title`(单元有简介 / 日期但没名称时「季 N」)、`premiered`、`releasedate`、`year`、`seasonnumber`。
- 集:`title`(没有或 TMDB 占位名时「第 N 集」)、`plot`、`outline`、`aired`、`year`、`season`、`episode`、`rating`。 - 集:`title`(TMDB 占位名时「第 N 集」)、`plot`、`outline`、`aired`、`year`、`season`、`episode`、`rating`。
季 / 集单元整个缺元数据时不写(见「季 / 集缺元数据」)。
- 取值:标题 / 简介 / 标语按元数据语言链(见 `metadata.md`);字段锁与人工覆盖优先(用的是投影后的最终值)。 - 取值:标题 / 简介 / 标语按元数据语言链(见 `metadata.md`);字段锁与人工覆盖优先(用的是投影后的最终值)。
与 MP 的差异(差分登记 `apps/server/test/scrape/nfo-deviations.json`): 与 MP 的差异(差分登记 `apps/server/test/scrape/nfo-deviations.json`):
@@ -153,17 +208,20 @@ cd /opt/uhd_media_tool && deno run -A npm:@biomejs/biome check --write apps/serv
- 作品详情「刮削」卡片:该作品最近一条刮削记录;排队或进行中时每 3 秒刷新。 - 作品详情「刮削」卡片:该作品最近一条刮削记录;排队或进行中时每 3 秒刷新。
- `GET /files/scrape-records?ids=`、`GET /library/{id}` 的 `scrape`:`id`、`root`、`path`、`status`、`trigger`、`mode`、 - `GET /files/scrape-records?ids=`、`GET /library/{id}` 的 `scrape`:`id`、`root`、`path`、`status`、`trigger`、`mode`、
`match{mediaEntityId, provider, externalId, kind, title, year, source}`、`written` / `skipped` / `failed`、`reason`、 `match{mediaEntityId, provider, externalId, kind, title, year, source}`、`written` / `skipped` / `failed`、`reason`、
`files[{path, kind, role, action, reason}]`(path 为登记目录内相对路径)、`organizationRecordId`、`taskId`。 `files[{path, kind, role, action, reason, hash}]`(path 为登记目录内相对路径;hash 为写入时内容的 sha256)、`organizationRecordId`、`taskId`。
- 审计 `organize.scrape`(目标 `scrape_record`;载荷含目录、识别结果、写入与失败的文件,最多 200 条);写入后发 `files.changed`,文件管理自动刷新。 - 审计 `organize.scrape`(目标 `scrape_record`;载荷含目录、识别结果、写入与失败的文件,最多 200 条);写入后发 `files.changed`,文件管理自动刷新。
- 原因码(`packages/contracts/src/reasons.ts`):`scrape_no_metadata`、`scrape_root_read_only`、`scrape_root_offline`、`scrape_not_video`、 - 原因码(`packages/contracts/src/reasons.ts`):`scrape_no_metadata`、`scrape_root_read_only`、`scrape_root_offline`、`scrape_not_video`、
`scrape_no_videos`、`scrape_path_missing`、`scrape_tmdb_unavailable`、`scrape_no_episode`、`scrape_exists`、`scrape_write_failed`、 `scrape_no_videos`、`scrape_path_missing`、`scrape_tmdb_unavailable`、`scrape_no_episode`、`scrape_exists`、`scrape_unchanged`、
`scrape_image_failed`、`path_escape`、`task_enqueue_failed`。 `scrape_user_modified`、`scrape_metadata_missing`、`scrape_write_failed`、`scrape_image_failed`、`scrape_internal_error`(查库等非写文件错误,
已写的文件结果保留)、`path_escape`、`task_enqueue_failed`。
| 现象 | 看哪里 | | 现象 | 看哪里 |
| --- | --- | | --- | --- |
| 入库后没有 NFO | 设置里「入库后自动刮削」是否打开;作品详情「刮削」是否有记录;引擎运行 `scrapeMedia` | | 入库后没有 NFO | 设置里「入库后自动刮削」是否打开;作品详情「刮削」是否有记录;引擎运行 `scrapeMedia` |
| 刮削对话框一直让选作品 | 文件名解析不出标题,或 TMDB 有多个同名候选;选一次即可,打开「同时加入媒体库」后下次按已入库作品识别 | | 刮削对话框一直让选作品 | 文件名解析不出标题,或 TMDB 有多个同名候选;选一次即可,打开「同时加入媒体库」后下次按已入库作品识别 |
| 全部「跳过」 | 覆盖方式是只补缺失;要替换选「全部覆盖」或作品详情「覆盖重写」 | | 全部「跳过」 | 覆盖方式是只补缺失,或更新时文件不是本工具写的 / 被人改过(`scrape_exists` / `scrape_user_modified`);要替换选「全部覆盖」或作品详情「覆盖重写」 |
| 改了映射 / 刷新了元数据,NFO 没变 | 作品以前没刮削过(不自动重刮);重刮任务要等 2 分钟;NFO 被人改过(`scrape_user_modified`) |
| 记录「部分成功」、原因缺元数据 | 季 / 集还没取到元数据且 TMDB 季详情取不到;刷新元数据(作品详情「刷新」带 full)后会自动重刮 |
| 图片失败、NFO 正常 | TMDB 图片访问(出口代理);稍后重新刮削,只补缺失的图片 | | 图片失败、NFO 正常 | TMDB 图片访问(出口代理);稍后重新刮削,只补缺失的图片 |
| 「没有可用的元数据」 | 作品没有 TMDB 字段也没有 TMDB id;在文件管理里刮削并指定 TMDB 条目 | | 「没有可用的元数据」 | 作品没有 TMDB 字段也没有 TMDB id;在文件管理里刮削并指定 TMDB 条目 |
| 「TMDB 不可用」 | 设置里 TMDB API Key;未登记的目录要直接取 TMDB | | 「TMDB 不可用」 | 设置里 TMDB API Key;未登记的目录要直接取 TMDB |
+5 -1
View File
@@ -108,7 +108,11 @@ const REASONS: Record<string, ReasonText> = {
task_enqueue_failed: { text: "后台任务没排上", action: "稍后重试" }, task_enqueue_failed: { text: "后台任务没排上", action: "稍后重试" },
scrape_no_videos: { text: "目录里没有视频", action: "对作品或季目录刮削" }, scrape_no_videos: { text: "目录里没有视频", action: "对作品或季目录刮削" },
scrape_tmdb_unavailable: { text: "TMDB 不可用", action: "检查 TMDB 设置后重新刮削" }, scrape_tmdb_unavailable: { text: "TMDB 不可用", action: "检查 TMDB 设置后重新刮削" },
scrape_exists: { text: "文件已存在,只补缺失时不改" }, scrape_exists: { text: "文件已存在,不是本工具写的或只补缺失时不改" },
scrape_unchanged: { text: "内容没有变化" },
scrape_user_modified: { text: "文件写入后被改过,更新时不覆盖", action: "要替换就选全部覆盖重新刮削" },
scrape_metadata_missing: { text: "缺少这一季或这一集的元数据,没有写占位文件", action: "刷新元数据后重新刮削" },
scrape_internal_error: { text: "刮削过程中出错", action: "查看日志后重新刮削" },
scrape_write_failed: { text: "写入目录失败", action: "检查目录权限后重新刮削" }, scrape_write_failed: { text: "写入目录失败", action: "检查目录权限后重新刮削" },
scrape_image_failed: { text: "图片没下载成功", action: "稍后重新刮削" }, scrape_image_failed: { text: "图片没下载成功", action: "稍后重新刮削" },
scrape_no_episode: { text: "文件名里认不出集号" }, scrape_no_episode: { text: "文件名里认不出集号" },
+7 -2
View File
@@ -14,11 +14,16 @@ export const ScrapeFileResult = z.object({
action: z.enum(["written", "skipped", "failed"]), action: z.enum(["written", "skipped", "failed"]),
/** 跳过 / 失败原因码(reasons.ts:scrape_exists / scrape_write_failed / scrape_image_failed …) */ /** 跳过 / 失败原因码(reasons.ts:scrape_exists / scrape_write_failed / scrape_image_failed …) */
reason: z.string().nullable(), reason: z.string().nullable(),
/** 本工具写入(或核对未变)时文件内容的 sha256;「更新」模式据此判断文件之后有没有被人改过 */
hash: z.string().nullable().optional(),
}); });
export type ScrapeFileResult = z.infer<typeof ScrapeFileResult>; export type ScrapeFileResult = z.infer<typeof ScrapeFileResult>;
/** missing = 只补缺失;overwrite = 全部覆盖 */ /**
export const ScrapeMode = z.enum(["missing", "overwrite"]); * missing = 只补缺失;overwrite = 全部覆盖;
* update = 更新(缺省):不存在的写入,本工具上次写入且没被改过的用新内容覆盖,被改过 / 不是本工具写的不动
*/
export const ScrapeMode = z.enum(["missing", "overwrite", "update"]);
export type ScrapeMode = z.infer<typeof ScrapeMode>; export type ScrapeMode = z.infer<typeof ScrapeMode>;
export const ScrapeStatus = z.enum(["queued", "running", "succeeded", "partial", "failed"]); export const ScrapeStatus = z.enum(["queued", "running", "succeeded", "partial", "failed"]);
+6 -3
View File
@@ -78,8 +78,11 @@ export const OrganizeSettings = z.object({
minSizeBytes: z.number().int().nonnegative(), minSizeBytes: z.number().int().nonnegative(),
/** SCR-1:整理入库成功后自动刮削(往媒体库目录写 NFO 与图片);缺省关(同 MP 目录「刮削」缺省) */ /** SCR-1:整理入库成功后自动刮削(往媒体库目录写 NFO 与图片);缺省关(同 MP 目录「刮削」缺省) */
scrapeAfterImport: z.boolean(), scrapeAfterImport: z.boolean(),
/** SCR-1:目标文件已存在时 missing = 只补缺失(MP「仅缺失」)/ overwrite = 全部覆盖 */ /**
scrapeMode: z.enum(["missing", "overwrite"]), * 目标文件已存在时:update = 更新(缺省;只覆盖本工具写过且没被改过的文件)/ missing = 只补缺失(MP「仅缺失」)/
* overwrite = 全部覆盖
*/
scrapeMode: z.enum(["missing", "overwrite", "update"]),
/** SCR-1:下载海报、背景图、季海报 */ /** SCR-1:下载海报、背景图、季海报 */
scrapeImages: z.boolean(), scrapeImages: z.boolean(),
/** SCR-1:下载每集剧照(`<视频名>-thumb.jpg`) */ /** SCR-1:下载每集剧照(`<视频名>-thumb.jpg`) */
@@ -642,7 +645,7 @@ export const DEFAULT_SETTINGS: GlobalSettings = {
allowCrossVolumeMove: true, allowCrossVolumeMove: true,
minSizeBytes: 50 * 1024 * 1024, minSizeBytes: 50 * 1024 * 1024,
scrapeAfterImport: false, scrapeAfterImport: false,
scrapeMode: "missing", scrapeMode: "update",
scrapeImages: true, scrapeImages: true,
scrapeEpisodeThumbs: true, scrapeEpisodeThumbs: true,
}, },
@@ -0,0 +1,23 @@
import type { Kysely } from "kysely";
import { execDdl, oneOf } from "../src/migration-helpers.ts";
// 刮削覆盖策略加「更新」(update):只覆盖本工具上次写入且之后没被改过的文件(按 scrape_record.files 里记的 hash 判断),
// 被人改过的跳过(scrape_user_modified)。scrape_record.mode 的 CHECK 增加 update;逐文件 hash 放在 files jsonb 里,不加列。
const OLD = ["missing", "overwrite"];
const NEW = [...OLD, "update"];
const swap = (values: readonly string[]) => `
alter table scrape_record drop constraint if exists scrape_record_mode_check;
alter table scrape_record add constraint scrape_record_mode_check ${oneOf("mode", values)};
`;
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
export async function up(db: Kysely<any>): Promise<void> {
await execDdl(db, swap(NEW));
}
// biome-ignore lint/suspicious/noExplicitAny: 同上
export async function down(db: Kysely<any>): Promise<void> {
await execDdl(db, `update scrape_record set mode = 'missing' where mode = 'update'; ${swap(OLD)}`);
}
+1 -1
View File
@@ -233,7 +233,7 @@ export type ScheduledJobRunTrigger = "schedule" | "manual" | "skip";
export type ScheduledJobRunStatus = "ok" | "failed" | "skipped" | "running"; export type ScheduledJobRunStatus = "ok" | "failed" | "skipped" | "running";
export type ScrapeRecordStatus = "queued" | "running" | "succeeded" | "partial" | "failed"; export type ScrapeRecordStatus = "queued" | "running" | "succeeded" | "partial" | "failed";
export type ScrapeRecordTrigger = "auto" | "manual"; export type ScrapeRecordTrigger = "auto" | "manual";
export type ScrapeRecordMode = "missing" | "overwrite"; export type ScrapeRecordMode = "missing" | "overwrite" | "update";
export type SecretPurpose = export type SecretPurpose =
| "site_cookie" | "site_cookie"
| "site_passkey" | "site_passkey"
+8
View File
@@ -10,6 +10,14 @@
// 视频直接在库根(没有作品目录)时,电影图片改用 `<视频名>-poster.jpg` / `<视频名>-fanart.jpg`, // 视频直接在库根(没有作品目录)时,电影图片改用 `<视频名>-poster.jpg` / `<视频名>-fanart.jpg`,
// 剧集不写 tvshow / 季文件(不能往库根里放 tvshow.nfo)。 // 剧集不写 tvshow / 季文件(不能往库根里放 tvshow.nfo)。
/**
* 作品类型 → 刮削布局与 NFO 根元素:剧集 / 综艺按剧集(tvshow.nfo + 季 / 集),其余(电影、合集、其他)都按电影
* (`<视频名>.nfo`,根元素 `<movie>`)。刮削的范围判定与 NFO 生成都用这一个口径。
*/
export function scrapeKindOf(entityKind: string): "movie" | "tv" {
return entityKind === "series" || entityKind === "variety" || entityKind === "tv" ? "tv" : "movie";
}
/** 纯字符串的 POSIX dirname / basename / extname(domain 不依赖 node:path,前端也能打包) */ /** 纯字符串的 POSIX dirname / basename / extname(domain 不依赖 node:path,前端也能打包) */
const posix = { const posix = {
dirname(p: string): string { dirname(p: string): string {
+8
View File
@@ -4,6 +4,7 @@ import {
inferWorkDir, inferWorkDir,
isSeasonDirName, isSeasonDirName,
planScrapeTargets, planScrapeTargets,
scrapeKindOf,
seasonNfo, seasonNfo,
seasonOfDirName, seasonOfDirName,
seasonPosterName, seasonPosterName,
@@ -247,3 +248,10 @@ describe("刮削目标布局", () => {
expect(ep.map((x) => x.role)).toEqual(["episode_nfo", "episode_thumb"]); expect(ep.map((x) => x.role)).toEqual(["episode_nfo", "episode_thumb"]);
}); });
}); });
describe("作品类型 → 刮削类型", () => {
it("剧集 / 综艺按剧集,电影 / 合集 / 其他按电影", () => {
expect(["series", "variety", "tv"].map(scrapeKindOf)).toEqual(["tv", "tv", "tv"]);
expect(["movie", "collection", "other"].map(scrapeKindOf)).toEqual(["movie", "movie", "movie"]);
});
});