feat(scrape): 任意目录就地刮削,按路径识别作品,每次刮削一条记录(SCR-2)
- POST /files/scrape 改为对目录 / 视频文件刮削:已登记用作品,否则按路径识别(本地 + TMDB), 认不准返回候选,选定后再提交;未登记直接用 TMDB 字段,addToLibrary 顺手建作品 - 作品详情重新刮削、入库后自动刮削改为对作品目录刮削,同一实现 - 迁移 0058_scrape_record 替代 media_scrape;GET /files/scrape-records - 与 MP ScrapingChain 的文件布局差分(7 个场景,3 处登记偏离) Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
f60254e265
commit
87bf33b42b
29 files changed
+2272
-653
No files matched your search
@@ -0,0 +1,211 @@
|
||||
import type {
|
||||
FileIdentifyCandidate,
|
||||
FilesScrapeItem,
|
||||
FilesScrapeNeedsChoice,
|
||||
FilesScrapeRequest,
|
||||
FilesScrapeResult,
|
||||
ScrapeMatch,
|
||||
} from "@umt/contracts";
|
||||
import type { Principal } from "../../http/auth.ts";
|
||||
import {
|
||||
entityScrapeMatch,
|
||||
loadScrapeRecords,
|
||||
queueScrape,
|
||||
scrapeSettings,
|
||||
waitScrapeRecords,
|
||||
} from "../acquisition-organize/services/scrape.ts";
|
||||
import { actorOfP, FilesError, type FmDeps, resolveIn } from "./common.ts";
|
||||
import { candidateKey, identifyCore } from "./identify.ts";
|
||||
import { audit } from "./ledger.ts";
|
||||
import { kindOfName } from "./list.ts";
|
||||
import { entitiesUnder } from "./records.ts";
|
||||
|
||||
// SCR-2 文件管理「刮削」(照 MP 文件管理的刮削):任意目录 / 视频文件 → 识别作品 → 就地写 NFO 与图片。
|
||||
// 识别顺序:请求指定(选定的候选)> 目录里已登记文件所属的唯一作品 > 按路径识别(本地召回 + TMDB,resolveIdentity
|
||||
// 能自动确认且各视频一致时自动用)。认不准的项进 needsChoice(前 5 候选),选定后带 mediaEntityId / external 再提交。
|
||||
// 未登记也能刮削(直接用 TMDB 字段);addToLibrary 时顺手把 TMDB 条目加入媒体库。
|
||||
// 每项一条刮削记录,排 scrapeMedia;请求内最多等 transferWaitMs,没刮完的 pending=true,前端按 id 轮询。
|
||||
|
||||
const RECOGNIZE_TIMEOUT_MS = 15_000;
|
||||
|
||||
type Actor = { type: string; id: string | null };
|
||||
const actorOf = (p: Principal | undefined): Actor => ({
|
||||
type: p?.kind === "api_key" ? "api_key" : "user",
|
||||
id: p?.id ?? null,
|
||||
});
|
||||
|
||||
/** 路径识别:各视频都能自动确认且是同一作品时返回它;否则返回候选(按出现次数与等级合并,前 5) */
|
||||
async function recognize(
|
||||
d: FmDeps,
|
||||
principal: Principal | undefined,
|
||||
it: FilesScrapeItem,
|
||||
): Promise<{ pick: FileIdentifyCandidate | null; choice: Omit<FilesScrapeNeedsChoice, "root" | "path"> }> {
|
||||
const r = await identifyCore(
|
||||
d,
|
||||
{ items: [{ root: it.root, path: it.path }], minSizeBytes: 0 },
|
||||
{ mode: "remote", actorId: actorOfP(principal), signal: AbortSignal.timeout(RECOGNIZE_TIMEOUT_MS) },
|
||||
);
|
||||
const items = r.result.items;
|
||||
const picks = items.map((i) => r.autoPicks.get(`${i.root}\u0000${i.path}`) ?? null);
|
||||
const agreed = picks.length > 0 && picks.every((p) => p && p === picks[0]) ? picks[0] : null;
|
||||
const LV = { high: 3, medium: 2, low: 1 } as const;
|
||||
const merged = new Map<string, { c: FileIdentifyCandidate; n: number }>();
|
||||
for (const i of items)
|
||||
for (const c of i.candidates) {
|
||||
const k = candidateKey(c);
|
||||
const cur = merged.get(k);
|
||||
if (!cur) merged.set(k, { c, n: 1 });
|
||||
else {
|
||||
cur.n += 1;
|
||||
if (LV[c.level] > LV[cur.c.level]) cur.c = { ...c, posterUrl: c.posterUrl ?? cur.c.posterUrl };
|
||||
}
|
||||
}
|
||||
const ranked = [...merged.values()].sort((a, b) => b.n - a.n || LV[b.c.level] - LV[a.c.level]).map((x) => x.c);
|
||||
const pick = agreed ? (ranked.find((c) => candidateKey(c) === agreed) ?? null) : null;
|
||||
const first = items[0]?.parsed;
|
||||
return {
|
||||
pick,
|
||||
choice: {
|
||||
title: first?.title ?? null,
|
||||
candidates: ranked.slice(0, 5),
|
||||
warnings: [...new Set(items.flatMap((i) => i.warnings))].slice(0, 5),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** TMDB 条目 → 识别结果:已有本地作品(已接受的 TMDB 映射)就用它;addToLibrary 时建作品 */
|
||||
async function externalMatch(
|
||||
d: FmDeps,
|
||||
ext: { externalId: string; kind: "movie" | "tv"; title?: string; year?: number | null },
|
||||
source: ScrapeMatch["source"],
|
||||
addToLibrary: boolean,
|
||||
actor: Actor,
|
||||
entities: FilesScrapeResult["entities"],
|
||||
): Promise<ScrapeMatch> {
|
||||
const local = await d.ctx.db
|
||||
.selectFrom("external_identity")
|
||||
.select(["media_entity_id"])
|
||||
.where("provider", "=", "tmdb")
|
||||
.where("entity_type", "=", ext.kind)
|
||||
.where("external_id", "=", ext.externalId)
|
||||
.where("status", "=", "accepted")
|
||||
.where("content_unit_id", "is", null)
|
||||
.executeTakeFirst();
|
||||
let entityId = local?.media_entity_id ?? null;
|
||||
if (!entityId && addToLibrary) {
|
||||
entityId = await d.ao.ports.entityFromTmdb({ type: ext.kind, id: ext.externalId }, actor);
|
||||
const e = await d.ctx.db.selectFrom("media_entity").select(["title"]).where("id", "=", entityId).executeTakeFirst();
|
||||
entities.push({ mediaEntityId: entityId, title: e?.title ?? ext.title ?? ext.externalId, created: true });
|
||||
}
|
||||
if (entityId) {
|
||||
const m = await entityScrapeMatch(d.ao, entityId, source);
|
||||
if (m) return { ...m, provider: "tmdb", externalId: ext.externalId };
|
||||
}
|
||||
return {
|
||||
mediaEntityId: null,
|
||||
provider: "tmdb",
|
||||
externalId: ext.externalId,
|
||||
kind: ext.kind,
|
||||
title: ext.title ?? `TMDB ${ext.externalId}`,
|
||||
year: ext.year ?? null,
|
||||
source,
|
||||
};
|
||||
}
|
||||
|
||||
export async function scrapeFiles(
|
||||
d: FmDeps,
|
||||
principal: Principal | undefined,
|
||||
body: FilesScrapeRequest,
|
||||
): Promise<FilesScrapeResult> {
|
||||
const db = d.ctx.db;
|
||||
const actor = actorOf(principal);
|
||||
const mode = body.mode ?? (await scrapeSettings(d.ao)).scrapeMode;
|
||||
const recordIds: string[] = [];
|
||||
const needsChoice: FilesScrapeResult["needsChoice"] = [];
|
||||
const skipped: FilesScrapeResult["skipped"] = [];
|
||||
const entities: FilesScrapeResult["entities"] = [];
|
||||
for (const it of body.items) {
|
||||
if (it.mediaEntityId && it.external) throw new FilesError("bad_request", "mediaEntityId 与 external 只能给一个");
|
||||
const r = await resolveIn(d, it.root, it.path);
|
||||
const ref = { root: it.root, path: r.rel };
|
||||
if (r.def.kind === "library" && !r.def.modifiable) {
|
||||
skipped.push({ ...ref, reason: "scrape_root_read_only" });
|
||||
continue;
|
||||
}
|
||||
const isDir = !!r.lst?.isDirectory();
|
||||
if (!isDir && !(r.lst?.isFile() && kindOfName(r.abs) === "video")) {
|
||||
skipped.push({ ...ref, reason: "scrape_not_video" });
|
||||
continue;
|
||||
}
|
||||
let match: ScrapeMatch | null = null;
|
||||
if (it.mediaEntityId) {
|
||||
match = await entityScrapeMatch(d.ao, it.mediaEntityId, "chosen");
|
||||
if (!match) throw new FilesError("not_found", `作品不存在：${it.mediaEntityId}`);
|
||||
} else if (it.external) {
|
||||
match = await externalMatch(d, it.external, "chosen", body.addToLibrary, actor, entities);
|
||||
} else {
|
||||
// 目录里的文件已登记到唯一作品:直接用(重新刮削已入库的)
|
||||
const owners = r.def.libraryRootId ? await entitiesUnder(db, r.def.libraryRootId, r.rel, isDir) : [];
|
||||
if (owners.length === 1) match = await entityScrapeMatch(d.ao, owners[0]!, "library");
|
||||
if (!match) {
|
||||
let rec: Awaited<ReturnType<typeof recognize>>;
|
||||
try {
|
||||
rec = await recognize(d, principal, { ...it, path: r.rel });
|
||||
} catch (e) {
|
||||
if (e instanceof FilesError && e.reason === "bad_request") {
|
||||
skipped.push({ ...ref, reason: "scrape_no_videos" });
|
||||
continue;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
const c = rec.pick;
|
||||
if (c?.mediaEntityId) match = await entityScrapeMatch(d.ao, c.mediaEntityId, "auto");
|
||||
else if (c && c.provider === "tmdb" && c.externalId)
|
||||
match = await externalMatch(
|
||||
d,
|
||||
{ externalId: c.externalId, kind: c.kind === "movie" ? "movie" : "tv", title: c.title, year: c.year },
|
||||
"auto",
|
||||
body.addToLibrary,
|
||||
actor,
|
||||
entities,
|
||||
);
|
||||
if (!match) {
|
||||
needsChoice.push({ ...ref, ...rec.choice });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
const q = await queueScrape(d.ao, {
|
||||
rootId: r.def.id,
|
||||
rootPath: r.def.hostPath,
|
||||
libraryRootId: r.def.libraryRootId,
|
||||
path: r.rel,
|
||||
match,
|
||||
mode,
|
||||
trigger: "manual",
|
||||
actor,
|
||||
});
|
||||
recordIds.push(q.recordId);
|
||||
}
|
||||
const records = recordIds.length ? await waitScrapeRecords(d.ao, recordIds, d.options.transferWaitMs) : [];
|
||||
await audit(d, principal, "scrape", {
|
||||
items: body.items.map((i) => `${i.root}:${i.path}`).slice(0, 100),
|
||||
mode,
|
||||
recordIds,
|
||||
needsChoice: needsChoice.map((n) => `${n.root}:${n.path}`),
|
||||
skipped,
|
||||
});
|
||||
return {
|
||||
records,
|
||||
needsChoice,
|
||||
skipped,
|
||||
pending: records.some((r) => r.status === "queued" || r.status === "running"),
|
||||
entities,
|
||||
};
|
||||
}
|
||||
|
||||
export async function scrapeRecords(d: FmDeps, idsParam: string) {
|
||||
const ids = [...new Set(idsParam.split(",").map((s) => s.trim()))].filter((s) => /^[0-9a-f-]{36}$/i.test(s));
|
||||
if (!ids.length || ids.length > 100) throw new FilesError("bad_request", "ids 需要 1–100 个刮削记录 id");
|
||||
return { records: await loadScrapeRecords(d.ao, ids) };
|
||||
}
|
||||
Reference in new issue
Block a user