UHD_Media_Tool/apps/server/test/file-manager/scrape.test.ts

import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
import { join, relative } from "node:path";
import type { TmdbRichDetail, TmdbRichSeasonDetail } from "@umt/adapters";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { latestEntityScrape, scrapeEntity } from "../../src/modules/acquisition-organize/services/scrape.ts";
import {
  createFmHarness,
  EXEC_KEY,
  type FmHarness,
  key,
  pgAvailable,
  type RemoteHit,
  seedAsset,
  seedEntity,
} from "./harness.ts";

// SCR-2 刮削照 MP:文件管理任意目录 / 视频文件 → 按路径识别作品(未登记走 TMDB,认不准返回候选)→ 就地写 NFO 与图片。
// 场景:未登记的《人生切割术》S01 两集 + 《星际穿越》目录一次刮削;歧义候选(Solaris 1972 / 2002)选定后刮削并加入媒体库;
// 只补缺失 / 全部覆盖;季目录与单集文件的范围;已登记目录直接用作品;只登记目录与非视频文件跳过。

const skip = !(await pgAvailable());
const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00]);

function detail(
  p: Partial<TmdbRichDetail> & Pick<TmdbRichDetail, "entityType" | "externalId" | "title">,
): TmdbRichDetail {
  return {
    titleLanguage: "zh-CN",
    titleZh: p.title,
    titleZhLanguage: "zh-CN",
    originalTitle: null,
    originalLanguage: "en",
    overview: null,
    overviewLanguage: null,
    tagline: null,
    taglineLanguage: null,
    year: null,
    releaseDate: null,
    lastAirDate: null,
    genres: [],
    voteAverage: null,
    voteCount: null,
    status: null,
    runtime: null,
    seasonCount: null,
    episodeCount: null,
    networks: [],
    originCountry: [],
    posterPath: null,
    backdropPath: null,
    cast: [],
    crew: [],
    externalIds: { imdb: null, tvdb: null, wikidata: null },
    translations: [],
    alternativeTitles: [],
    seasons: [],
    languages: ["zh-CN"],
    ...p,
  };
}

const DETAILS: Record<string, TmdbRichDetail> = {
  "tv:95396": detail({
    entityType: "tv",
    externalId: "95396",
    title: "人生切割术",
    originalTitle: "Severance",
    year: 2022,
    releaseDate: "2022-02-17",
    overview: "马克领导着一个办公室团队。",
    overviewLanguage: "zh-CN",
    genres: ["剧情", "悬疑"],
    voteAverage: 8.4,
    networks: ["Apple TV+"],
    originCountry: ["US"],
    posterPath: "/sev.jpg",
    backdropPath: "/sev-bg.jpg",
    externalIds: { imdb: "tt11280740", tvdb: "371980", wikidata: null },
  }),
  "movie:157336": detail({
    entityType: "movie",
    externalId: "157336",
    title: "星际穿越",
    originalTitle: "Interstellar",
    year: 2014,
    releaseDate: "2014-11-05",
    runtime: 169,
    posterPath: "/inter.jpg",
    backdropPath: "/inter-bg.jpg",
    crew: [{ id: 525, name: "克里斯托弗·诺兰", originalName: null, role: "Director", profilePath: null }],
  }),
  "movie:593": detail({
    entityType: "movie",
    externalId: "593",
    title: "飞向太空",
    originalTitle: "Solaris",
    year: 1972,
  }),
  "movie:2103": detail({
    entityType: "movie",
    externalId: "2103",
    title: "索拉里斯星",
    originalTitle: "Solaris",
    year: 2002,
    posterPath: "/solaris.jpg",
  }),
};

const SEASON1: TmdbRichSeasonDetail = {
  tvId: "95396",
  seasonNumber: 1,
  name: "第 1 季",
  nameLanguage: "zh-CN",
  overview: null,
  overviewLanguage: null,
  airDate: "2022-02-17",
  posterPath: "/s1.jpg",
  fetchedLanguages: ["zh-CN"],
  episodes: [
    [1, "有关地狱的好消息", "/e1.jpg"],
    [2, "半循环", "/e2.jpg"],
  ].map(([n, name, still]) => ({
    seasonNumber: 1,
    episodeNumber: n as number,
    name: name as string,
    nameLanguage: "zh-CN",
    overview: `第 ${n} 集简介`,
    overviewLanguage: "zh-CN",
    airDate: "2022-02-17",
    runtime: 55,
    stillPath: still as string,
    voteAverage: 8,
    voteCount: 10,
  })),
};

const REMOTE: Record<string, RemoteHit[]> = {
  人生切割术: [
    {
      provider: "tmdb",
      entityType: "tv",
      externalId: "95396",
      title: "人生切割术",
      originalTitle: "Severance",
      year: 2022,
    },
  ],
  星际穿越: [
    {
      provider: "tmdb",
      entityType: "movie",
      externalId: "157336",
      title: "星际穿越",
      originalTitle: "Interstellar",
      year: 2014,
    },
  ],
  Solaris: [
    {
      provider: "tmdb",
      entityType: "movie",
      externalId: "593",
      title: "飞向太空",
      originalTitle: "Solaris",
      year: 1972,
    },
    {
      provider: "tmdb",
      entityType: "movie",
      externalId: "2103",
      title: "索拉里斯星",
      originalTitle: "Solaris",
      year: 2002,
    },
  ],
};

async function tree(root: string, under = ""): Promise<string[]> {
  const out: string[] = [];
  const walk = async (d: string) => {
    for (const n of (await readdir(d)).sort()) {
      if (n.startsWith(".")) continue;
      const p = join(d, n);
      if ((await stat(p)).isDirectory()) await walk(p);
      else out.push(relative(root, p));
    }
  };
  await walk(join(root, under));
  return out.sort();
}

const SHOW = "人生切割术 (2022)";
const EP1 = `${SHOW}/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.mkv`;
const EP2 = `${SHOW}/Season 1/人生切割术 - S01E02 - 半循环.mkv`;
const MOVIE = "星际穿越 (2014)";
const MOVIE_FILE = `${MOVIE}/星际穿越 (2014) - 1080p.mkv`;

describe.skipIf(skip)("files 刮削(SCR-2:任意目录就地刮削)", () => {
  let h: FmHarness;
  const images: string[] = [];
  const tmdbCalls: string[] = [];

  beforeAll(async () => {
    h = await createFmHarness(
      "scrape2",
      {
        ao: {
          async scrapeImage(p) {
            images.push(p);
            return { content: JPEG, mime: "image/jpeg" };
          },
          async scrapeTmdbDetail(type, id) {
            tmdbCalls.push(`${type}:${id}`);
            const d = DETAILS[`${type}:${id}`];
            if (!d) throw new Error("TMDB 404");
            return d;
          },
          async scrapeTmdbSeason(tvId, n) {
            tmdbCalls.push(`season:${tvId}:${n}`);
            return tvId === "95396" && n === 1 ? SEASON1 : null;
          },
          async entityFromTmdb(ref) {
            const d = DETAILS[`${ref.type}:${ref.id}`]!;
            const e = await h.db
              .insertInto("media_entity")
              .values({
                kind: ref.type === "movie" ? "movie" : "series",
                title: d.title,
                year: d.year,
                status: "active",
              })
              .returning("id")
              .executeTakeFirstOrThrow();
            await h.db
              .insertInto("external_identity")
              .values({
                media_entity_id: e.id,
                provider: "tmdb",
                entity_type: ref.type,
                external_id: ref.id,
                status: "accepted",
              })
              .execute();
            return e.id;
          },
        },
      },
      {
        catalog: (q) =>
          Object.entries(REMOTE)
            .filter(([k]) => q.includes(k) || k.includes(q))
            .flatMap(([, v]) => v),
      },
    );
    await mkdir(join(h.lib, SHOW, "Season 1"), { recursive: true });
    await mkdir(join(h.lib, MOVIE));
    await mkdir(join(h.lib, "Solaris"));
    for (const f of [EP1, EP2, MOVIE_FILE, "Solaris/Solaris.1080p.BluRay.x264-GRP.mkv"])
      await writeFile(join(h.lib, f), "video-bytes");
    await writeFile(join(h.lib, MOVIE, "readme.txt"), "x");
  });
  afterAll(async () => h?.close());

  const scrape = (items: unknown[], extra: Record<string, unknown> = {}, k = EXEC_KEY) =>
    h.api("POST", "/files/scrape", { idempotencyKey: key(), items, ...extra }, k);

  it("未登记的剧集目录 + 电影目录:按路径识别 TMDB,就地写 NFO 与图片,不建作品", async () => {
    const r = await scrape([
      { root: h.libRoot, path: SHOW },
      { root: h.libRoot, path: MOVIE },
    ]);
    expect(r.status).toBe(200);
    const data = r.body.data;
    expect(data.needsChoice).toEqual([]);
    expect(data.pending).toBe(false);
    expect(data.records).toHaveLength(2);
    const [show, movie] = data.records;
    expect(show).toMatchObject({
      status: "succeeded",
      path: SHOW,
      mode: "update",
      trigger: "manual",
      match: {
        mediaEntityId: null,
        provider: "tmdb",
        externalId: "95396",
        kind: "tv",
        source: "auto",
        title: "人生切割术",
      },
    });
    expect(movie).toMatchObject({ status: "succeeded", match: { externalId: "157336", kind: "movie" } });

    const files = await tree(h.lib);
    expect(files).toEqual(
      [
        `${SHOW}/Season 1/poster.jpg`,
        `${SHOW}/Season 1/season.nfo`,
        EP1,
        EP1.replace(/\.mkv$/, "-thumb.jpg"),
        EP1.replace(/\.mkv$/, ".nfo"),
        EP2,
        EP2.replace(/\.mkv$/, "-thumb.jpg"),
        EP2.replace(/\.mkv$/, ".nfo"),
        `${SHOW}/fanart.jpg`,
        `${SHOW}/poster.jpg`,
        `${SHOW}/season01-poster.jpg`,
        `${SHOW}/tvshow.nfo`,
        `${MOVIE}/fanart.jpg`,
        `${MOVIE}/poster.jpg`,
        `${MOVIE}/readme.txt`,
        MOVIE_FILE,
        MOVIE_FILE.replace(/\.mkv$/, ".nfo"),
        "Solaris/Solaris.1080p.BluRay.x264-GRP.mkv",
      ].sort(),
    );
    // 未登记:不建作品,字段直接取 TMDB
    expect(await h.db.selectFrom("media_entity").select("id").execute()).toHaveLength(0);
    const tvshow = await readFile(join(h.lib, SHOW, "tvshow.nfo"), "utf8");
    expect(tvshow).toContain("<title>人生切割术</title>");
    expect(tvshow).toContain('<uniqueid type="imdb" default="true">tt11280740</uniqueid>');
    const ep = await readFile(join(h.lib, EP1.replace(/\.mkv$/, ".nfo")), "utf8");
    expect(ep).toContain("<title>有关地狱的好消息</title>");
    expect(ep).toMatch(/<season>1<\/season>\n {2}<episode>1<\/episode>/);
    const nfo = await readFile(join(h.lib, MOVIE_FILE.replace(/\.mkv$/, ".nfo")), "utf8");
    expect(nfo).toContain('<director tmdbid="525">克里斯托弗·诺兰</director>');
    expect(tmdbCalls).toEqual(expect.arrayContaining(["tv:95396", "season:95396:1", "movie:157336"]));
    // 记录可按 id 回查;审计按记录
    const back = await h.api("GET", `/files/scrape-records?ids=${show.id},${movie.id}`);
    expect(back.body.data.records.map((x: { id: string }) => x.id)).toEqual([show.id, movie.id]);
    expect(back.body.data.records[0].files.filter((f: { action: string }) => f.action === "written")).toHaveLength(10);
    expect(h.events.some((e) => e.type === "files.changed" && e.payload.rootId === h.libRoot)).toBe(true);
  });

  it("歧义:返回候选;选定 TMDB 条目后刮削,并按需加入媒体库", async () => {
    const first = await scrape([{ root: h.libRoot, path: "Solaris" }]);
    expect(first.status).toBe(200);
    expect(first.body.data.records).toEqual([]);
    expect(first.body.data.needsChoice).toHaveLength(1);
    const nc = first.body.data.needsChoice[0];
    expect(nc).toMatchObject({ root: h.libRoot, path: "Solaris", title: "Solaris" });
    expect(nc.candidates.map((c: { externalId: string }) => c.externalId).sort()).toEqual(["2103", "593"]);
    expect(await tree(h.lib, "Solaris")).toEqual(["Solaris/Solaris.1080p.BluRay.x264-GRP.mkv"]);

    const second = await scrape(
      [{ root: h.libRoot, path: "Solaris", external: { provider: "tmdb", externalId: "2103", kind: "movie" } }],
      { addToLibrary: true },
    );
    expect(second.status).toBe(200);
    const rec = second.body.data.records[0];
    expect(rec).toMatchObject({
      status: "succeeded",
      match: { provider: "tmdb", externalId: "2103", source: "chosen", title: "索拉里斯星", year: 2002 },
    });
    expect(second.body.data.entities).toMatchObject([{ title: "索拉里斯星", created: true }]);
    expect(rec.match.mediaEntityId).toBe(second.body.data.entities[0].mediaEntityId);
    expect(await tree(h.lib, "Solaris")).toEqual([
      "Solaris/Solaris.1080p.BluRay.x264-GRP.mkv",
      "Solaris/Solaris.1080p.BluRay.x264-GRP.nfo",
      "Solaris/poster.jpg",
    ]);
    expect(await readFile(join(h.lib, "Solaris/Solaris.1080p.BluRay.x264-GRP.nfo"), "utf8")).toContain(
      "<title>索拉里斯星</title>",
    );
    // 作品详情取的是该作品最近一条刮削记录
    expect((await latestEntityScrape(h.module.deps().ao, rec.match.mediaEntityId))?.id).toBe(rec.id);
  });

  it("只补缺失跳过已有文件(包括别人写的);全部覆盖改写", async () => {
    const show = join(h.lib, SHOW, "tvshow.nfo");
    await writeFile(show, "<tvshow>别的工具写的</tvshow>");
    const miss = await scrape([{ root: h.libRoot, path: SHOW }]);
    const rec = miss.body.data.records[0];
    expect(rec).toMatchObject({ status: "succeeded", written: 0 });
    expect(rec.skipped).toBeGreaterThan(0);
    expect(await readFile(show, "utf8")).toBe("<tvshow>别的工具写的</tvshow>");
    const over = await scrape([{ root: h.libRoot, path: SHOW }], { mode: "overwrite" });
    expect(over.body.data.records[0]).toMatchObject({ status: "succeeded", mode: "overwrite", skipped: 0 });
    expect(await readFile(show, "utf8")).toContain("<title>人生切割术</title>");
  });

  it("季目录只写季与各集;单集文件只写该集", async () => {
    const season = await scrape([{ root: h.libRoot, path: `${SHOW}/Season 1` }], { mode: "overwrite" });
    const roles = season.body.data.records[0].files.map((f: { path: string; role: string }) => `${f.role}:${f.path}`);
    expect(roles.sort()).toEqual(
      [
        `season_nfo:${SHOW}/Season 1/season.nfo`,
        `season_poster:${SHOW}/season01-poster.jpg`,
        `season_poster:${SHOW}/Season 1/poster.jpg`,
        `episode_nfo:${EP1.replace(/\.mkv$/, ".nfo")}`,
        `episode_nfo:${EP2.replace(/\.mkv$/, ".nfo")}`,
        `episode_thumb:${EP1.replace(/\.mkv$/, "-thumb.jpg")}`,
        `episode_thumb:${EP2.replace(/\.mkv$/, "-thumb.jpg")}`,
      ].sort(),
    );
    const one = await scrape([{ root: h.libRoot, path: EP2 }], { mode: "overwrite" });
    expect(one.body.data.records[0].files.map((f: { role: string }) => f.role).sort()).toEqual([
      "episode_nfo",
      "episode_thumb",
    ]);
  });

  it("已登记的目录直接用作品;作品详情「重新刮削」对作品目录刮削", async () => {
    const film = await seedEntity(h.db, { kind: "movie", title: "星际穿越", year: 2014 });
    await h.db
      .insertInto("external_identity")
      .values({
        media_entity_id: film,
        provider: "tmdb",
        entity_type: "movie",
        external_id: "157336",
        status: "accepted",
      })
      .execute();
    await seedAsset(h.db, h, { entityId: film, rel: MOVIE_FILE });
    const r = await scrape([{ root: h.libRoot, path: MOVIE }]);
    expect(r.body.data.records[0].match).toMatchObject({
      mediaEntityId: film,
      source: "library",
      externalId: "157336",
    });
    // 作品详情「重新刮削」(POST /library/{id}/scrape)的服务端做法
    const again = await scrapeEntity(h.module.deps().ao, film, { trigger: "manual", mode: "overwrite" });
    expect(again).toHaveLength(1);
    const recs = await h.api("GET", `/files/scrape-records?ids=${again[0]!.recordId}`);
    expect(recs.body.data.records[0]).toMatchObject({ path: MOVIE, root: h.libRoot, mode: "overwrite" });
  });

  it("只登记目录与非视频文件跳过;没有视频的目录跳过", async () => {
    await mkdir(join(h.lib, "空目录"));
    const r = await scrape([
      { root: h.libRoot, path: `${MOVIE}/readme.txt` },
      { root: h.libRoot, path: "空目录" },
    ]);
    expect(r.body.data.skipped).toEqual([
      { root: h.libRoot, path: `${MOVIE}/readme.txt`, reason: "scrape_not_video" },
      { root: h.libRoot, path: "空目录", reason: "scrape_no_videos" },
    ]);
    await h.db.updateTable("library_root").set({ management_mode: "register_only" }).execute();
    const ro = await scrape([{ root: h.libRoot, path: MOVIE }]);
    expect(ro.body.data.skipped).toEqual([{ root: h.libRoot, path: MOVIE, reason: "scrape_root_read_only" }]);
    await h.db.updateTable("library_root").set({ management_mode: "managed" }).execute();
  });
});