feat(scrape): 任意目录就地刮削,按路径识别作品,每次刮削一条记录(SCR-2)

- POST /files/scrape 改为对目录 / 视频文件刮削:已登记用作品,否则按路径识别(本地 + TMDB), 认不准返回候选,选定后再提交;未登记直接用 TMDB 字段,addToLibrary 顺手建作品 - 作品详情重新刮削、入库后自动刮削改为对作品目录刮削,同一实现 - 迁移 0058_scrape_record 替代 media_scrape;GET /files/scrape-records - 与 MP ScrapingChain 的文件布局差分(7 个场景,3 处登记偏离) Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-01 00:08:17 +08:00
1 parent f60254e265
commit 87bf33b42b
@@ -11,7 +11,14 @@ import {
  type OrganizationStatus,
 } from "@umt/contracts";
 import { sql } from "@umt/db";
-import { type IdentityCandidate, type ParsedRelease, parseReleasePath, queryTitles, rankCandidates } from "@umt/domain";
+import {
+  type IdentityCandidate,
+  type ParsedRelease,
+  parseReleasePath,
+  queryTitles,
+  rankCandidates,
+  resolveIdentity,
+} from "@umt/domain";
 import { localIdentityCandidate } from "@umt/mapping";
 import type { Principal } from "../../http/auth.ts";
 import { EXTERNAL_INVOKE_TIMEOUT_MS, sharedInvokeLimiter } from "../agent/invoke-limits.ts";
@@ -177,13 +184,24 @@ export async function runIdentifyRemote(
  return payload;
 }

-async function identifyCore(
+/** 候选键:本地作品 id 或 `provider:externalId`(与 FileIdentifyCandidate 对应) */
+export const candidateKey = (c: { mediaEntityId: string | null; provider: string; externalId: string | null }) =>
+  c.mediaEntityId ?? `${c.provider}:${c.externalId}`;
+
+export async function identifyCore(
  d: FmDeps,
  body: FilesIdentifyRequest,
  opts: CoreOpts,
-): Promise<{ result: FilesIdentifyResult; remotePending: number; pendingPaths: Set<string> }> {
+): Promise<{
+  result: FilesIdentifyResult;
+  remotePending: number;
+  pendingPaths: Set<string>;
+  /** SCR-2:每个视频按 resolveIdentity 能否自动确认(root\0path → 候选键;不能为 null) */
+  autoPicks: Map<string, string | null>;
+}> {
  const db = d.ctx.db;
  const pendingPaths = new Set<string>();
+  const autoPicks = new Map<string, string | null>();
  const { videos, truncated } = await collectVideos(d, body);
  // G2:最小体积(请求/向导 > settings.organize.minSizeBytes,缺省 50 MiB;设置读不到时不过滤)
  const orgSettings = (await d.ctx.settings.get("organize").catch(() => null)) as { minSizeBytes?: unknown } | null;
@@ -411,6 +429,14 @@ async function identifyCore(
        ),
      };
    });
+    const res = resolveIdentity(parsed, all);
+    const pick = res.status === "auto" && res.best ? res.best.candidate : null;
+    autoPicks.set(
+      `${v.root}\u0000${v.rel}`,
+      pick && (!kindHint || (kindHint === "movie") === (pick.entityType === "movie"))
+        ? (pick.localEntityId ?? `${pick.provider}:${pick.externalId}`)
+        : null,
+    );
    const best = candidates[0];
    let contentUnitId: string | null = null;
    let contentUnit: string | null = null;
@@ -454,7 +480,7 @@ async function identifyCore(
  }
  if (videos.length === 0 && body.items.length > 0 && !truncated)
    throw new FilesError("bad_request", "所选项中没有视频文件");
-  return { result: { items, truncated, excludedBySize }, remotePending: remoteQueries, pendingPaths };
+  return { result: { items, truncated, excludedBySize }, remotePending: remoteQueries, pendingPaths, autoPicks };
 }

 /** TMDB 图片 CDN 缩略图(w154);已是完整 URL 的原样返回 */
@@ -8,21 +8,17 @@ import type {
  FilesProbeResult,
  FilesScrapeBatchRequest,
  FilesScrapeBatchResult,
-  FilesScrapeRequest,
-  FilesScrapeResult,
 } from "@umt/contracts";
 import { sql } from "@umt/db";
 import type { Principal } from "../../http/auth.ts";
-import { enqueueScrape } from "../acquisition-organize/services/scrape.ts";
 import { FilesError, type FmDeps, jsonb, resolveIn } from "./common.ts";
 import { audit } from "./ledger.ts";
 import { kindOfName, mediaSummary } from "./list.ts";
 import { entitiesUnder } from "./records.ts";

 // probe:ffprobe 摘要;已登记资产写回 media_asset.media_info.probe,未登记只缓存 24h(DATA_DIR/cache/probe)。
-// scrape:只对已登记作品,等价 POST /media-entities/{id}/refresh(入队 refreshMetadata,字段锁优先、不动路径);
-// writeNfo / images(SCR-1)另排 scrapeMedia,把 NFO 与图片写进媒体库目录(与作品详情「重新刮削」同一任务)。
-// scrape-batch write=true:文件管理「刮削」,按作品去重各排一次 scrapeMedia(不刷新元数据)。
+// scrape-batch:文件管理「刷新元数据」,按作品去重各排一次 refreshMetadata(字段锁优先、不动路径、不写文件)。
+// 刮削(写 NFO 与图片)见 scrape.ts。

 const CACHE_TTL_MS = 24 * 3600_000;

@@ -104,62 +100,8 @@ export async function probeFiles(d: FmDeps, body: FilesProbeRequest): Promise<Fi
  return { items };
 }

-export async function scrape(
-  d: FmDeps,
-  principal: Principal | undefined,
-  body: FilesScrapeRequest,
-): Promise<FilesScrapeResult> {
-  const db = d.ctx.db;
-  const e = await db
-    .selectFrom("media_entity")
-    .select(["id", "status", "field_locks"])
-    .where("id", "=", body.mediaEntityId)
-    .executeTakeFirst();
-  if (!e) throw new FilesError("not_found", "作品不存在");
-  const registered = await db
-    .selectFrom("media_asset")
-    .select(["id"])
-    .where("media_entity_id", "=", e.id)
-    .where("status", "=", "active")
-    .executeTakeFirst();
-  if (!registered) throw new FilesError("bad_request", "作品没有已入库文件。先识别并整理入库，再刮削");
-  const { runId } = await d.ctx.engine.enqueue(
-    "refreshMetadata",
-    { mediaEntityId: e.id },
-    { idempotencyKey: `refreshMetadata:files:${body.idempotencyKey}`, traceId: randomUUID() },
-  );
-  const overrides = await db
-    .selectFrom("field_override")
-    .select(["field"])
-    .where("media_entity_id", "=", e.id)
-    .where("locked", "=", true)
-    .execute();
-  const lockedFields = [...new Set([...(e.field_locks ?? []), ...overrides.map((o) => o.field)])].sort();
-  const scrapeTaskId =
-    body.writeNfo || body.images
-      ? (
-          await enqueueScrape(d.ao, {
-            mediaEntityId: e.id,
-            trigger: "manual",
-            actor: { type: principal?.kind === "api_key" ? "api_key" : "user", id: principal?.id ?? null },
-          })
-        ).taskId
-      : null;
-  const result: FilesScrapeResult = {
-    mediaEntityId: e.id,
-    taskId: runId,
-    lockedFields,
-    nfo: body.writeNfo ? "queued" : "not_requested",
-    images: body.images ? "queued" : "not_requested",
-    scrapeTaskId,
-  };
-  await audit(d, principal, "scrape", { ...result });
-  return result;
-}
-
 /**
 * G5 批量刷新元数据:文件/目录归到其下已登记资产的作品,与显式作品合并去重,逐个入队 refreshMetadata。
- * 只对有 active 资产的作品提交;其余进 skipped(前端汇总"已提交 n 个作品,锁定字段不变")。
 */
 export async function scrapeBatch(
  d: FmDeps,
@@ -183,18 +125,6 @@ export async function scrapeBatch(
  const ents = list.length
    ? await db.selectFrom("media_entity").select(["id", "title", "field_locks"]).where("id", "in", list).execute()
    : [];
-  const active = new Set(
-    list.length
-      ? (
-          await db
-            .selectFrom("media_asset")
-            .select(["media_entity_id"])
-            .where("media_entity_id", "in", list)
-            .where("status", "=", "active")
-            .execute()
-        ).map((a) => a.media_entity_id)
-      : [],
-  );
  const locks = list.length
    ? await db
        .selectFrom("field_override")
@@ -210,34 +140,16 @@ export async function scrapeBatch(
      skipped.push({ ref: id, reason: "not_found" });
      continue;
    }
-    if (!active.has(id)) {
-      skipped.push({ ref: id, reason: "not_registered" });
-      continue;
-    }
-    const runId = body.write
-      ? (
-          await enqueueScrape(d.ao, {
-            mediaEntityId: id,
-            trigger: "manual",
-            actor: { type: principal?.kind === "api_key" ? "api_key" : "user", id: principal?.id ?? null },
-          })
-        ).taskId
-      : (
-          await d.ctx.engine.enqueue(
-            "refreshMetadata",
-            { mediaEntityId: id },
-            { idempotencyKey: `refreshMetadata:files-batch:${body.idempotencyKey}:${id}`, traceId: randomUUID() },
-          )
-        ).runId;
+    const { runId } = await d.ctx.engine.enqueue(
+      "refreshMetadata",
+      { mediaEntityId: id },
+      { idempotencyKey: `refreshMetadata:files-batch:${body.idempotencyKey}:${id}`, traceId: randomUUID() },
+    );
    const lockedFields = [
      ...new Set([...(e.field_locks ?? []), ...locks.filter((l) => l.media_entity_id === id).map((l) => l.field)]),
    ].sort();
    submitted.push({ mediaEntityId: id, title: e.title, taskId: runId, lockedFields });
  }
-  await audit(d, principal, "scrape_batch", {
-    write: body.write,
-    submitted: submitted.map((s) => s.mediaEntityId),
-    skipped,
-  });
+  await audit(d, principal, "scrape_batch", { submitted: submitted.map((s) => s.mediaEntityId), skipped });
  return { submitted, skipped };
 }
@@ -15,8 +15,9 @@ import { listDir, listRoots, statEntry } from "./list.ts";
 import { mkdirIn, renameEntry } from "./mutate.ts";
 import { organizeFiles } from "./organize.ts";
 import { previewFile } from "./preview.ts";
-import { probeFiles, scrape, scrapeBatch } from "./probe.ts";
+import { probeFiles, scrapeBatch } from "./probe.ts";
 import { findRecords } from "./records.ts";
+import { scrapeFiles, scrapeRecords } from "./scrape.ts";
 import { suggestName } from "./suggest-name.ts";
 import { transfer } from "./transfer.ts";
 import { deleteFiles, listTrash, purgeTrash, restoreTrash } from "./trash.ts";
@@ -110,7 +111,11 @@ export function registerFileRoutes(app: ModuleApp, d: FmDeps) {
  mount(app, F.identify, async (c, { body }) =>
    identify(d, c.get("principal"), body, { wait: /^(1|true)$/i.test(c.req.query("wait") ?? "") }),
  );
-  mount(app, F.scrape, async (c, { body }) => idem(d, c, "scrape", body, () => scrape(d, c.get("principal"), body)));
+  // SCR-2:任意目录 / 视频文件就地刮削(识别 → 写 NFO 与图片);认不准的项返回候选
+  mount(app, F.scrape, async (c, { body }) =>
+    idem(d, c, "scrape", body, () => scrapeFiles(d, c.get("principal"), body)),
+  );
+  mount(app, F.scrapeRecords, async (_c, { query }) => scrapeRecords(d, query.ids));
  mount(app, F.probe, async (_c, { body }) => probeFiles(d, body));
  mount(app, F.records, async (_c, { query }) => findRecords(d, query));
  mount(app, F.download, async (c, { query }) => downloadFile(d, c.get("principal"), query), true);
@@ -0,0 +1,211 @@
+import type {
+  FileIdentifyCandidate,
+  FilesScrapeItem,
+  FilesScrapeNeedsChoice,
+  FilesScrapeRequest,
+  FilesScrapeResult,
+  ScrapeMatch,
+} from "@umt/contracts";
+import type { Principal } from "../../http/auth.ts";
+import {
+  entityScrapeMatch,
+  loadScrapeRecords,
+  queueScrape,
+  scrapeSettings,
+  waitScrapeRecords,
+} from "../acquisition-organize/services/scrape.ts";
+import { actorOfP, FilesError, type FmDeps, resolveIn } from "./common.ts";
+import { candidateKey, identifyCore } from "./identify.ts";
+import { audit } from "./ledger.ts";
+import { kindOfName } from "./list.ts";
+import { entitiesUnder } from "./records.ts";
+
+// SCR-2 文件管理「刮削」(照 MP 文件管理的刮削):任意目录 / 视频文件 → 识别作品 → 就地写 NFO 与图片。
+// 识别顺序:请求指定(选定的候选)> 目录里已登记文件所属的唯一作品 > 按路径识别(本地召回 + TMDB,resolveIdentity
+// 能自动确认且各视频一致时自动用)。认不准的项进 needsChoice(前 5 候选),选定后带 mediaEntityId / external 再提交。
+// 未登记也能刮削(直接用 TMDB 字段);addToLibrary 时顺手把 TMDB 条目加入媒体库。
+// 每项一条刮削记录,排 scrapeMedia;请求内最多等 transferWaitMs,没刮完的 pending=true,前端按 id 轮询。
+
+const RECOGNIZE_TIMEOUT_MS = 15_000;
+
+type Actor = { type: string; id: string | null };
+const actorOf = (p: Principal | undefined): Actor => ({
+  type: p?.kind === "api_key" ? "api_key" : "user",
+  id: p?.id ?? null,
+});
+
+/** 路径识别:各视频都能自动确认且是同一作品时返回它;否则返回候选(按出现次数与等级合并,前 5) */
+async function recognize(
+  d: FmDeps,
+  principal: Principal | undefined,
+  it: FilesScrapeItem,
+): Promise<{ pick: FileIdentifyCandidate | null; choice: Omit<FilesScrapeNeedsChoice, "root" | "path"> }> {
+  const r = await identifyCore(
+    d,
+    { items: [{ root: it.root, path: it.path }], minSizeBytes: 0 },
+    { mode: "remote", actorId: actorOfP(principal), signal: AbortSignal.timeout(RECOGNIZE_TIMEOUT_MS) },
+  );
+  const items = r.result.items;
+  const picks = items.map((i) => r.autoPicks.get(`${i.root}\u0000${i.path}`) ?? null);
+  const agreed = picks.length > 0 && picks.every((p) => p && p === picks[0]) ? picks[0] : null;
+  const LV = { high: 3, medium: 2, low: 1 } as const;
+  const merged = new Map<string, { c: FileIdentifyCandidate; n: number }>();
+  for (const i of items)
+    for (const c of i.candidates) {
+      const k = candidateKey(c);
+      const cur = merged.get(k);
+      if (!cur) merged.set(k, { c, n: 1 });
+      else {
+        cur.n += 1;
+        if (LV[c.level] > LV[cur.c.level]) cur.c = { ...c, posterUrl: c.posterUrl ?? cur.c.posterUrl };
+      }
+    }
+  const ranked = [...merged.values()].sort((a, b) => b.n - a.n || LV[b.c.level] - LV[a.c.level]).map((x) => x.c);
+  const pick = agreed ? (ranked.find((c) => candidateKey(c) === agreed) ?? null) : null;
+  const first = items[0]?.parsed;
+  return {
+    pick,
+    choice: {
+      title: first?.title ?? null,
+      candidates: ranked.slice(0, 5),
+      warnings: [...new Set(items.flatMap((i) => i.warnings))].slice(0, 5),
+    },
+  };
+}
+
+/** TMDB 条目 → 识别结果:已有本地作品(已接受的 TMDB 映射)就用它;addToLibrary 时建作品 */
+async function externalMatch(
+  d: FmDeps,
+  ext: { externalId: string; kind: "movie" | "tv"; title?: string; year?: number | null },
+  source: ScrapeMatch["source"],
+  addToLibrary: boolean,
+  actor: Actor,
+  entities: FilesScrapeResult["entities"],
+): Promise<ScrapeMatch> {
+  const local = await d.ctx.db
+    .selectFrom("external_identity")
+    .select(["media_entity_id"])
+    .where("provider", "=", "tmdb")
+    .where("entity_type", "=", ext.kind)
+    .where("external_id", "=", ext.externalId)
+    .where("status", "=", "accepted")
+    .where("content_unit_id", "is", null)
+    .executeTakeFirst();
+  let entityId = local?.media_entity_id ?? null;
+  if (!entityId && addToLibrary) {
+    entityId = await d.ao.ports.entityFromTmdb({ type: ext.kind, id: ext.externalId }, actor);
+    const e = await d.ctx.db.selectFrom("media_entity").select(["title"]).where("id", "=", entityId).executeTakeFirst();
+    entities.push({ mediaEntityId: entityId, title: e?.title ?? ext.title ?? ext.externalId, created: true });
+  }
+  if (entityId) {
+    const m = await entityScrapeMatch(d.ao, entityId, source);
+    if (m) return { ...m, provider: "tmdb", externalId: ext.externalId };
+  }
+  return {
+    mediaEntityId: null,
+    provider: "tmdb",
+    externalId: ext.externalId,
+    kind: ext.kind,
+    title: ext.title ?? `TMDB ${ext.externalId}`,
+    year: ext.year ?? null,
+    source,
+  };
+}
+
+export async function scrapeFiles(
+  d: FmDeps,
+  principal: Principal | undefined,
+  body: FilesScrapeRequest,
+): Promise<FilesScrapeResult> {
+  const db = d.ctx.db;
+  const actor = actorOf(principal);
+  const mode = body.mode ?? (await scrapeSettings(d.ao)).scrapeMode;
+  const recordIds: string[] = [];
+  const needsChoice: FilesScrapeResult["needsChoice"] = [];
+  const skipped: FilesScrapeResult["skipped"] = [];
+  const entities: FilesScrapeResult["entities"] = [];
+  for (const it of body.items) {
+    if (it.mediaEntityId && it.external) throw new FilesError("bad_request", "mediaEntityId 与 external 只能给一个");
+    const r = await resolveIn(d, it.root, it.path);
+    const ref = { root: it.root, path: r.rel };
+    if (r.def.kind === "library" && !r.def.modifiable) {
+      skipped.push({ ...ref, reason: "scrape_root_read_only" });
+      continue;
+    }
+    const isDir = !!r.lst?.isDirectory();
+    if (!isDir && !(r.lst?.isFile() && kindOfName(r.abs) === "video")) {
+      skipped.push({ ...ref, reason: "scrape_not_video" });
+      continue;
+    }
+    let match: ScrapeMatch | null = null;
+    if (it.mediaEntityId) {
+      match = await entityScrapeMatch(d.ao, it.mediaEntityId, "chosen");
+      if (!match) throw new FilesError("not_found", `作品不存在：${it.mediaEntityId}`);
+    } else if (it.external) {
+      match = await externalMatch(d, it.external, "chosen", body.addToLibrary, actor, entities);
+    } else {
+      // 目录里的文件已登记到唯一作品:直接用(重新刮削已入库的)
+      const owners = r.def.libraryRootId ? await entitiesUnder(db, r.def.libraryRootId, r.rel, isDir) : [];
+      if (owners.length === 1) match = await entityScrapeMatch(d.ao, owners[0]!, "library");
+      if (!match) {
+        let rec: Awaited<ReturnType<typeof recognize>>;
+        try {
+          rec = await recognize(d, principal, { ...it, path: r.rel });
+        } catch (e) {
+          if (e instanceof FilesError && e.reason === "bad_request") {
+            skipped.push({ ...ref, reason: "scrape_no_videos" });
+            continue;
+          }
+          throw e;
+        }
+        const c = rec.pick;
+        if (c?.mediaEntityId) match = await entityScrapeMatch(d.ao, c.mediaEntityId, "auto");
+        else if (c && c.provider === "tmdb" && c.externalId)
+          match = await externalMatch(
+            d,
+            { externalId: c.externalId, kind: c.kind === "movie" ? "movie" : "tv", title: c.title, year: c.year },
+            "auto",
+            body.addToLibrary,
+            actor,
+            entities,
+          );
+        if (!match) {
+          needsChoice.push({ ...ref, ...rec.choice });
+          continue;
+        }
+      }
+    }
+    const q = await queueScrape(d.ao, {
+      rootId: r.def.id,
+      rootPath: r.def.hostPath,
+      libraryRootId: r.def.libraryRootId,
+      path: r.rel,
+      match,
+      mode,
+      trigger: "manual",
+      actor,
+    });
+    recordIds.push(q.recordId);
+  }
+  const records = recordIds.length ? await waitScrapeRecords(d.ao, recordIds, d.options.transferWaitMs) : [];
+  await audit(d, principal, "scrape", {
+    items: body.items.map((i) => `${i.root}:${i.path}`).slice(0, 100),
+    mode,
+    recordIds,
+    needsChoice: needsChoice.map((n) => `${n.root}:${n.path}`),
+    skipped,
+  });
+  return {
+    records,
+    needsChoice,
+    skipped,
+    pending: records.some((r) => r.status === "queued" || r.status === "running"),
+    entities,
+  };
+}
+
+export async function scrapeRecords(d: FmDeps, idsParam: string) {
+  const ids = [...new Set(idsParam.split(",").map((s) => s.trim()))].filter((s) => /^[0-9a-f-]{36}$/i.test(s));
+  if (!ids.length || ids.length > 100) throw new FilesError("bad_request", "ids 需要 1–100 个刮削记录 id");
+  return { records: await loadScrapeRecords(d.ao, ids) };
+}