feat(server): 发现页接口——TMDB / 豆瓣 / Bangumi 列表、本地状态、确定匹配记忆

- GET /discover/{tmdb,douban,bangumi},bgm 趋势页在沙箱任务 bangumi.trends 里解析
- 豆瓣 / bgm 每个列表 / 搜索请求走 douban.request / bangumi.request 缓存(TTL 取 storage.cacheTtl),每源限流
- 条目按 TMDB 身份标已订阅 / 媒体库已有 / 下载中;豆瓣 / bgm 条目确定匹配后记在 discover.match

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei

apps/server/src/modules/search/discover.ts

@@ -1,17 +1,52 @@
-import type { MetadataCandidate } from "@umt/adapters";
-import { type DiscoverMatchRequest, DiscoverMatchResult } from "@umt/contracts";
+import {
+  type MetadataCandidate,
+  type MetaSubject,
+  type parseBangumiBrowser,
+  TmdbError,
+  type TmdbListItem,
+  type TmdbListPage,
+  tmdbChart,
+  tmdbDiscover,
+  tmdbSearchPage,
+  tmdbTrending,
+} from "@umt/adapters";
+import {
+  DISCOVER_PAGE_SIZE,
+  type DiscoverBangumiQuery,
+  type DiscoverCalendar,
+  type DiscoverDoubanQuery,
+  type DiscoverItem,
+  type DiscoverLocal,
+  type DiscoverMatchCandidate,
+  type DiscoverMatchRequest,
+  DiscoverMatchResult,
+  type DiscoverPage,
+  type DiscoverTmdbQuery,
+  type InventoryEntry,
+} from "@umt/contracts";
 import { pgCacheFor } from "@umt/db";
 import { type IdentityCandidate, parseRelease, resolveIdentity } from "@umt/domain";
 import { ApiException } from "../../http/errors.ts";
+import { runSandbox } from "../../sandbox/client.ts";
 import type { SourcesRuntime } from "../sources-identity/runtime.ts";
 import { subjectRecall } from "../sources-identity/service/id-escalation.ts";
 import { metaClients, toCached } from "../sources-identity/service/meta-subjects.ts";
 import { tmdbToCandidate } from "../sources-identity/service/tmdb-candidate.ts";
+import { localEntities } from "./media.ts";
 
-// 发现(对照 MP 推荐页的 Bangumi 每日放送、豆瓣热门):列表来自 bgm.tv / 豆瓣,点开时按名称 + 年份匹配 TMDB 作品。
+// 发现(对照 MP 推荐 / 探索,docs/porting/moviepilot-discover.md):
+// - TMDB:流行趋势、榜单、按条件筛选、搜索,走 TMDB 请求缓存(列表类 6 小时,失败不缓存);
+// - 豆瓣 / bgm:榜单、标签、筛选、搜索、每日放送、热门趋势,走 douban.request / bangumi.request 请求缓存(meta-subjects.ts);
+// - bgm 热门趋势没有 API,网页在沙箱里解析(ADR-005);
+// - 每条结果按 TMDB 身份标本地状态(已订阅 / 媒体库已有 / 下载中);豆瓣 / bgm 条目匹配过 TMDB 的才知道(discover.match)。
+// 本地状态每次现查,不进缓存。
 
-const REGION = "discover.list";
-const TTL_SEC = 3600;
+/** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配(点开时写入,列表据此标本地状态) */
+const MATCH_REGION = "discover.match";
+const MATCH_TTL_SEC = 30 * 86_400;
+const MATCH_MAX = 20_000;
+
+export type InventoryLookup = (ids: string[]) => Promise<Map<string, InventoryEntry>>;
 
 function clients(rt: SourcesRuntime) {
   const c = metaClients(rt);
@@ -19,29 +54,201 @@ function clients(rt: SourcesRuntime) {
   return c;
 }
 
-async function cached<T>(rt: SourcesRuntime, key: string, load: () => Promise<T>): Promise<T> {
-  const cache = pgCacheFor(rt.ctx.db);
-  const hit = await cache.get<T>(REGION, key).catch(() => null);
-  if (hit) return hit;
-  let v: T;
+async function upstream<T>(what: string, fn: () => Promise<T>): Promise<T> {
   try {
-    v = await load();
+    return await fn();
   } catch (e) {
-    throw new ApiException("upstream_unavailable", (e as Error).message);
+    if (e instanceof ApiException) throw e;
+    if (e instanceof TmdbError && e.kind === "auth")
+      throw new ApiException("precondition_failed", "TMDB 拒绝了 API Key。到设置的元数据分区检查");
+    throw new ApiException(
+      "upstream_unavailable",
+      `${what}查询失败：${e instanceof TmdbError ? e.kind : (e as Error).message}`,
+    );
   }
-  await cache.set(REGION, key, v, { ttl: TTL_SEC, maxsize: 50 }).catch(() => {});
-  return v;
 }
 
-export function bangumiCalendar(rt: SourcesRuntime) {
-  return cached(rt, "bangumi:calendar", async () => ({ days: await clients(rt).bangumi.calendar() }));
+const fromMeta = (s: MetaSubject): DiscoverItem => ({
+  source: s.source,
+  id: s.id,
+  title: s.title,
+  aliases: s.aliases,
+  year: s.year,
+  kind: s.kind,
+  cover: s.cover,
+  rating: s.rating,
+  date: s.date,
+  episodes: s.episodes,
+  url: s.url,
+  genres: s.genres ?? [],
+  posterPath: null,
+  tmdb: null,
+  local: null,
+});
+
+const fromTmdb = (t: TmdbListItem): DiscoverItem => ({
+  source: "tmdb",
+  id: `${t.type}:${t.id}`,
+  title: t.title,
+  aliases: t.originalTitle && t.originalTitle !== t.title ? [t.originalTitle] : [],
+  year: t.year,
+  kind: t.type,
+  cover: null,
+  rating: t.voteAverage,
+  date: t.date,
+  episodes: null,
+  url: `https://www.themoviedb.org/${t.type}/${t.id}`,
+  genres: t.genres,
+  posterPath: t.posterPath,
+  tmdb: { type: t.type, id: t.id },
+  local: null,
+});
+
+// ── 本地状态 ──
+
+/** 豆瓣 / bgm 条目补上记住的 TMDB 匹配,再按 TMDB 身份查本地作品与库存 */
+export async function annotate(rt: SourcesRuntime, items: DiscoverItem[], inventory: InventoryLookup) {
+  const cache = pgCacheFor(rt.ctx.db);
+  await Promise.all(
+    items
+      .filter((it) => !it.tmdb && it.source !== "tmdb")
+      .map(async (it) => {
+        const hit = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, `${it.source}:${it.id}`).catch(() => null);
+        if (hit) it.tmdb = { type: hit.type, id: hit.id };
+      }),
+  );
+  const refs = items.flatMap((it) => (it.tmdb ? [{ entityType: it.tmdb.type, externalId: it.tmdb.id }] : []));
+  if (!refs.length) return items;
+  const local = await localEntities(rt.ctx, refs);
+  const ids = [...new Set(local.values())];
+  const inv = ids.length ? await inventory(ids) : new Map<string, InventoryEntry>();
+  for (const it of items) {
+    const entityId = it.tmdb ? local.get(`${it.tmdb.type}:${it.tmdb.id}`) : undefined;
+    if (!entityId) continue;
+    const e = inv.get(entityId);
+    it.local = {
+      entityId,
+      subscribed: e?.subscribed ?? false,
+      versionCount: e?.versions.length ?? 0,
+      downloadingCount: e?.downloading.length ?? 0,
+    } satisfies DiscoverLocal;
+  }
+  return items;
 }
 
-export function doubanHot(rt: SourcesRuntime, kind: "movie" | "tv") {
-  return cached(rt, `douban:hot:${kind}`, async () => ({ items: await clients(rt).douban.hot(kind, 30) }));
+// ── TMDB ──
+
+const tmdbPage = (p: TmdbListPage, page: number): DiscoverPage => ({
+  items: p.items.map(fromTmdb),
+  page,
+  hasMore: page < Math.min(p.totalPages, 500),
+});
+
+export async function tmdbList(rt: SourcesRuntime, q: DiscoverTmdbQuery): Promise<DiscoverPage> {
+  const client = await rt.tmdb();
+  if (!client) throw new ApiException("precondition_failed", "未配置 TMDB。到设置的元数据分区填写 API Key");
+  const type = q.type === "all" ? "movie" : q.type;
+  const page = q.page;
+  const p = await upstream("TMDB ", () => {
+    switch (q.list) {
+      case "trending":
+        return tmdbTrending(client, { mediaType: q.type, window: q.window, page });
+      case "search":
+        return tmdbSearchPage(client, { type: q.type, query: q.q ?? "", page });
+      case "discover":
+        return tmdbDiscover(client, {
+          type,
+          ...(q.sort ? { sort: q.sort } : {}),
+          ...(q.genre ? { genre: q.genre } : {}),
+          ...(q.language ? { language: q.language } : {}),
+          ...(q.region ? { region: q.region } : {}),
+          ...(q.year ? { year: q.year } : {}),
+          ...(q.minRating ? { minRating: q.minRating } : {}),
+          page,
+          today: rt.now().toISOString().slice(0, 10),
+        });
+      default:
+        return tmdbChart(client, { type, chart: q.list, page });
+    }
+  });
+  return tmdbPage(p, page);
 }
 
-const view = (m: MetadataCandidate) => ({
+// ── 豆瓣 ──
+
+export async function doubanList(rt: SourcesRuntime, q: DiscoverDoubanQuery): Promise<DiscoverPage> {
+  const { douban } = clients(rt);
+  const size = DISCOVER_PAGE_SIZE;
+  const start = (q.page - 1) * size;
+  return upstream("豆瓣", async () => {
+    if (q.list === "search") {
+      // 联想接口不分页
+      const items = q.page === 1 ? await douban.suggest(q.q ?? "") : [];
+      return { items: items.map(fromMeta), page: q.page, hasMore: false };
+    }
+    if (q.list === "tag") {
+      const items = await douban.tagSubjects(q.type, q.tag ?? "热门", start, size);
+      return { items: items.map(fromMeta), page: q.page, hasMore: items.length >= size };
+    }
+    const r =
+      q.list === "explore"
+        ? await douban.explore(
+            q.type,
+            { sort: q.sort ?? "U", tags: [q.genre, q.region, q.year].filter((x): x is string => !!x) },
+            start,
+            size,
+          )
+        : await douban.collection(q.list, start, size);
+    const hasMore = r.total !== null ? start + size < r.total : r.items.length >= size;
+    return { items: r.items.map(fromMeta), page: q.page, hasMore };
+  });
+}
+
+// ── bgm ──
+
+/** 趋势页 HTML 解析(外部字节走沙箱,ADR-005);测试可替换 */
+export let parseTrendsPage = (html: string) =>
+  runSandbox("bangumi.trends", { html }, { timeoutMs: 15_000 }) as Promise<ReturnType<typeof parseBangumiBrowser>>;
+export function setTrendsParser(fn: typeof parseTrendsPage) {
+  parseTrendsPage = fn;
+}
+
+export async function bangumiList(rt: SourcesRuntime, q: DiscoverBangumiQuery): Promise<DiscoverPage> {
+  const { bangumi } = clients(rt);
+  const size = DISCOVER_PAGE_SIZE;
+  const offset = (q.page - 1) * size;
+  return upstream("Bangumi ", async () => {
+    if (q.list === "trends") {
+      const r = await parseTrendsPage(await bangumi.trendsHtml(q.page));
+      return { items: r.items.map(fromMeta), page: q.page, hasMore: r.hasMore };
+    }
+    const r =
+      q.list === "search"
+        ? await bangumi.searchPage(q.q ?? "", size, offset)
+        : await bangumi.browse({
+            sort: q.list === "rank" ? "rank" : "date",
+            ...(q.year ? { year: q.year } : {}),
+            limit: size,
+            offset,
+          });
+    return { items: r.items.map(fromMeta), page: q.page, hasMore: offset + size < r.total };
+  });
+}
+
+export async function bangumiCalendar(rt: SourcesRuntime, inventory: InventoryLookup): Promise<DiscoverCalendar> {
+  const days = await upstream("Bangumi ", () => clients(rt).bangumi.calendar());
+  const out = days.map((d) => ({ ...d, items: d.items.map(fromMeta) }));
+  await annotate(
+    rt,
+    out.flatMap((d) => d.items),
+    inventory,
+  );
+  return { days: out };
+}
+
+// ── 匹配 TMDB ──
+
+const view = (m: MetadataCandidate): DiscoverMatchCandidate => ({
   type: m.entityType as "movie" | "tv",
   id: m.externalId,
   title: m.title,
@@ -50,6 +257,10 @@ const view = (m: MetadataCandidate) => ({
 });
 
 export async function matchTmdb(rt: SourcesRuntime, req: DiscoverMatchRequest): Promise<DiscoverMatchResult> {
+  const cache = pgCacheFor(rt.ctx.db);
+  const memoKey = `${req.source}:${req.id}`;
+  const memo = await cache.get<DiscoverMatchCandidate>(MATCH_REGION, memoKey).catch(() => null);
+  if (memo) return { match: memo, candidates: [memo] };
   const subject = toCached({ ...req, cover: null, rating: null, date: null, episodes: null, url: "" }, rt.now());
   const base = parseRelease(req.title);
   if (req.kind === "tv") base.mediaTypeHint = "tv";
@@ -65,5 +276,8 @@ export async function matchTmdb(rt: SourcesRuntime, req: DiscoverMatchRequest):
     .flatMap((r) => byCand.get(r.candidate) ?? [])
     .slice(0, 6);
   const best = res.status === "auto" && res.best ? byCand.get(res.best.candidate) : undefined;
-  return DiscoverMatchResult.parse({ match: best ? view(best) : null, candidates: ranked.map(view) });
+  const match = best ? view(best) : null;
+  // 只记确定的匹配;不确定 / 失败不记,下次重算
+  if (match) await cache.set(MATCH_REGION, memoKey, match, { ttl: MATCH_TTL_SEC, maxsize: MATCH_MAX }).catch(() => {});
+  return DiscoverMatchResult.parse({ match, candidates: ranked.map(view) });
 }

apps/server/src/modules/search/routes.ts

@@ -11,8 +11,10 @@ import {
   type ApiScope,
   apiResponse,
   DEFAULT_SETTINGS,
-  type DiscoverHotQuery,
+  type DiscoverBangumiQuery,
+  type DiscoverDoubanQuery,
   type DiscoverMatchRequest,
+  type DiscoverTmdbQuery,
   type EndpointDef,
   type InventoryEntry,
   type InventoryResponse,
@@ -40,7 +42,7 @@ import { metadataLanguages, metadataTextOptions } from "../sources-identity/serv
 import { upsertSiteRelease } from "../sources-identity/service/snapshot.ts";
 import type { ModuleApp, ServerContext } from "../types.ts";
 import { matchActions } from "./actions.ts";
-import { bangumiCalendar, doubanHot, matchTmdb } from "./discover.ts";
+import { annotate, bangumiCalendar, bangumiList, doubanList, matchTmdb, tmdbList } from "./discover.ts";
 import { InternalDispatcher } from "./dispatch.ts";
 import { localEntities, raceAbort, searchLocalMedia, TmdbSearch, withInventory } from "./media.ts";
 import { decorate, facets, quickFilter, ReleaseFanout, sortReleases } from "./releases.ts";
@@ -383,8 +385,22 @@ export function registerSearchRoutes(app: ModuleApp, ctx: ServerContext, d: Sear
     });
   });
 
-  route(app, Api.search.discoverCalendar, () => bangumiCalendar(d.runtime()));
-  route(app, Api.search.discoverHot, ({ query }) => doubanHot(d.runtime(), (query as DiscoverHotQuery).kind));
+  route(app, Api.search.discoverCalendar, ({ c }) => bangumiCalendar(d.runtime(), inventory(c)));
+  const withLocal =
+    (c: Context<AuthEnv>) =>
+    async <P extends { items: Parameters<typeof annotate>[1] }>(p: P): Promise<P> => {
+      await annotate(d.runtime(), p.items, inventory(c));
+      return p;
+    };
+  route(app, Api.search.discoverTmdb, async ({ query, c }) =>
+    withLocal(c)(await tmdbList(d.runtime(), query as DiscoverTmdbQuery)),
+  );
+  route(app, Api.search.discoverDouban, async ({ query, c }) =>
+    withLocal(c)(await doubanList(d.runtime(), query as DiscoverDoubanQuery)),
+  );
+  route(app, Api.search.discoverBangumi, async ({ query, c }) =>
+    withLocal(c)(await bangumiList(d.runtime(), query as DiscoverBangumiQuery)),
+  );
   route(app, Api.search.discoverMatch, ({ body }) => matchTmdb(d.runtime(), body as DiscoverMatchRequest));
 
   route(app, Api.search.tmdbDetail, async ({ params, query, c }) => {

apps/server/src/modules/sources-identity/service/meta-subjects.ts

@@ -3,10 +3,13 @@ import {
   createBangumiClient,
   createDoubanClient,
   type DoubanClient,
+  type MetaRequestCache,
   type MetaSubject,
+  RequestLimiter,
 } from "@umt/adapters";
-import { pgCacheFor } from "@umt/db";
+import { cachedCall, pgCacheFor } from "@umt/db";
 import { normalizeTitle, toSimplified } from "@umt/domain";
+import { storageSettings } from "../../../shared-cache.ts";
 import type { SourcesRuntime } from "../runtime.ts";
 import { DOUBAN_SUBJECT_REGION, type DoubanSubject, loadDoubanSubject, saveDoubanSubject } from "./douban-subject.ts";
 
@@ -20,10 +23,46 @@ const MISS_TTL_SEC = 6 * 3600;
 export type MetaSource = "douban" | "bangumi";
 const REGION: Record<MetaSource, string> = { douban: DOUBAN_SUBJECT_REGION, bangumi: BANGUMI_SUBJECT_REGION };
 
+/** 豆瓣 / bgm 请求缓存 region(对照 MP DoubanApi.__invoke / BangumiApi 的 @cached);key 见 metaRequestKey */
+export const META_REQUEST_REGIONS = { douban: "douban.request", bangumi: "bangumi.request" } as const;
+const META_REQUEST_MAXSIZE = 5000;
+/** 每源限流(进程内):豆瓣风控严,间隔大一些;缓存命中不占额度 */
+const META_LIMITERS = { douban: new RequestLimiter(2, 300), bangumi: new RequestLimiter(3, 100) };
+
+/**
+ * 请求缓存:搜索按 storage.cacheTtl.metaSearchDays,列表(榜单 / 热门 / 每日放送 / 趋势页)按 metaListHours,
+ * 空结果按 metaEmptyHours(不超过正常 TTL);失败(网络、403、风控页)在适配器里抛出,cachedCall 不写。
+ * 条目详情不走这里:douban.subject / bangumi.subject 条目级缓存(30 天)已经存了,避免同一份数据存两遍。
+ */
+export function metaRequestCache(ctx: SourcesRuntime["ctx"]): MetaRequestCache {
+  return {
+    async cached(req, load) {
+      const t = (await storageSettings(ctx)).cacheTtl;
+      const ttl = req.kind === "search" ? t.metaSearchDays * 86_400 : t.metaListHours * 3600;
+      return cachedCall(
+        pgCacheFor(ctx.db),
+        req.key,
+        {
+          region: META_REQUEST_REGIONS[req.source],
+          ttl,
+          emptyTtl: Math.min(ttl, t.metaEmptyHours * 3600),
+          emptyIf: req.empty,
+          maxsize: META_REQUEST_MAXSIZE,
+        },
+        load,
+      );
+    },
+  };
+}
+
 export function metaClients(rt: SourcesRuntime): { douban: DoubanClient; bangumi: BangumiClient } | null {
   const fetch = rt.metaFetch();
   if (!fetch) return null;
-  return { douban: createDoubanClient({ fetch }), bangumi: createBangumiClient({ fetch }) };
+  const cache = metaRequestCache(rt.ctx);
+  return {
+    douban: createDoubanClient({ fetch, cache, limiter: META_LIMITERS.douban }),
+    bangumi: createBangumiClient({ fetch, cache, limiter: META_LIMITERS.bangumi }),
+  };
 }
 
 export function toCached(s: MetaSubject, now: Date): DoubanSubject {

apps/server/src/sandbox/jobs.ts

@@ -1,7 +1,7 @@
 // 沙箱任务表(docs/adr/ADR-005-sandbox.md):只放「外部不可信字节 → 结构化数据」的纯函数。
 // 子进程加载完本表后撤销全部权限(文件 / 网络 / 环境变量 / 子进程),所以任务里不能有任何 I/O;
 // 输入输出必须能 JSON 化(二进制用 base64)。新增任务:在这里加一项,调用方用 runSandbox(名字, 输入)。
-import { parseTorrent } from "@umt/adapters";
+import { parseBangumiBrowser, parseTorrent } from "@umt/adapters";
 import { decodeSubtitle, detectSubtitleLanguage, subtitleEpisode, subtitleExtOf } from "@umt/domain";
 import { buildPreview } from "../modules/file-manager/preview-build.ts";
 import { PAGE_PARSERS, type PageKind } from "../modules/subtitles/online/parse.ts";
@@ -121,12 +121,18 @@ function subtitlePage(input: { kind: PageKind; html: string }) {
   return fn(input.html);
 }
 
+/** bgm.tv 动画浏览页(热门趋势,发现页)→ 条目列表 */
+function bangumiTrends(input: { html: string }) {
+  return parseBangumiBrowser(input.html);
+}
+
 export const SANDBOX_JOBS = {
   "subtitle.parse": subtitleParse,
   "torrent.parse": torrentParse,
   "file.preview": buildPreview,
   "captcha.solve": captchaSolve,
   "subtitle.page": subtitlePage,
+  "bangumi.trends": bangumiTrends,
   "self.test": selfTest,
 } as const;