From 9a154bc2494bf7c2ee1b56158f3d445ccfeea38b Mon Sep 17 00:00:00 2001 From: root Date: Tue, 6 Oct 2026 18:04:43 +0800 Subject: [PATCH] =?UTF-8?q?feat(server):=20=E5=8F=91=E7=8E=B0=E9=A1=B5?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E2=80=94=E2=80=94TMDB=20/=20=E8=B1=86?= =?UTF-8?q?=E7=93=A3=20/=20Bangumi=20=E5=88=97=E8=A1=A8=E3=80=81=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E7=8A=B6=E6=80=81=E3=80=81=E7=A1=AE=E5=AE=9A=E5=8C=B9?= =?UTF-8?q?=E9=85=8D=E8=AE=B0=E5=BF=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - GET /discover/{tmdb,douban,bangumi},bgm 趋势页在沙箱任务 bangumi.trends 里解析 - 豆瓣 / bgm 每个列表 / 搜索请求走 douban.request / bangumi.request 缓存(TTL 取 storage.cacheTtl),每源限流 - 条目按 TMDB 身份标已订阅 / 媒体库已有 / 下载中;豆瓣 / bgm 条目确定匹配后记在 discover.match Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei --- apps/server/src/modules/search/discover.ts | 254 ++++++++++++++-- apps/server/src/modules/search/routes.ts | 24 +- .../sources-identity/service/meta-subjects.ts | 43 ++- apps/server/src/sandbox/jobs.ts | 8 +- apps/server/test/search/discover.test.ts | 274 ++++++++++++++++++ 5 files changed, 576 insertions(+), 27 deletions(-) create mode 100644 apps/server/test/search/discover.test.ts diff --git a/apps/server/src/modules/search/discover.ts b/apps/server/src/modules/search/discover.ts index a46f0d0a..6e72634c 100644 --- a/apps/server/src/modules/search/discover.ts +++ b/apps/server/src/modules/search/discover.ts @@ -1,17 +1,52 @@ -import type { MetadataCandidate } from "@umt/adapters"; -import { type DiscoverMatchRequest, DiscoverMatchResult } from "@umt/contracts"; +import { + type MetadataCandidate, + type MetaSubject, + type parseBangumiBrowser, + TmdbError, + type TmdbListItem, + type TmdbListPage, + tmdbChart, + tmdbDiscover, + tmdbSearchPage, + tmdbTrending, +} from "@umt/adapters"; +import { + DISCOVER_PAGE_SIZE, + type DiscoverBangumiQuery, + type DiscoverCalendar, + type DiscoverDoubanQuery, + type DiscoverItem, + type DiscoverLocal, + type DiscoverMatchCandidate, + type DiscoverMatchRequest, + DiscoverMatchResult, + type DiscoverPage, + type DiscoverTmdbQuery, + type InventoryEntry, +} from "@umt/contracts"; import { pgCacheFor } from "@umt/db"; import { type IdentityCandidate, parseRelease, resolveIdentity } from "@umt/domain"; import { ApiException } from "../../http/errors.ts"; +import { runSandbox } from "../../sandbox/client.ts"; import type { SourcesRuntime } from "../sources-identity/runtime.ts"; import { subjectRecall } from "../sources-identity/service/id-escalation.ts"; import { metaClients, toCached } from "../sources-identity/service/meta-subjects.ts"; import { tmdbToCandidate } from "../sources-identity/service/tmdb-candidate.ts"; +import { localEntities } from "./media.ts"; -// 发现(对照 MP 推荐页的 Bangumi 每日放送、豆瓣热门):列表来自 bgm.tv / 豆瓣,点开时按名称 + 年份匹配 TMDB 作品。 +// 发现(对照 MP 推荐 / 探索,docs/porting/moviepilot-discover.md): +// - TMDB:流行趋势、榜单、按条件筛选、搜索,走 TMDB 请求缓存(列表类 6 小时,失败不缓存); +// - 豆瓣 / bgm:榜单、标签、筛选、搜索、每日放送、热门趋势,走 douban.request / bangumi.request 请求缓存(meta-subjects.ts); +// - bgm 热门趋势没有 API,网页在沙箱里解析(ADR-005); +// - 每条结果按 TMDB 身份标本地状态(已订阅 / 媒体库已有 / 下载中);豆瓣 / bgm 条目匹配过 TMDB 的才知道(discover.match)。 +// 本地状态每次现查,不进缓存。 -const REGION = "discover.list"; -const TTL_SEC = 3600; +/** 豆瓣 / bgm 条目 → TMDB 作品的确定匹配(点开时写入,列表据此标本地状态) */ +const MATCH_REGION = "discover.match"; +const MATCH_TTL_SEC = 30 * 86_400; +const MATCH_MAX = 20_000; + +export type InventoryLookup = (ids: string[]) => Promise>; function clients(rt: SourcesRuntime) { const c = metaClients(rt); @@ -19,29 +54,201 @@ function clients(rt: SourcesRuntime) { return c; } -async function cached(rt: SourcesRuntime, key: string, load: () => Promise): Promise { - const cache = pgCacheFor(rt.ctx.db); - const hit = await cache.get(REGION, key).catch(() => null); - if (hit) return hit; - let v: T; +async function upstream(what: string, fn: () => Promise): Promise { try { - v = await load(); + return await fn(); } catch (e) { - throw new ApiException("upstream_unavailable", (e as Error).message); + if (e instanceof ApiException) throw e; + if (e instanceof TmdbError && e.kind === "auth") + throw new ApiException("precondition_failed", "TMDB 拒绝了 API Key。到设置的元数据分区检查"); + throw new ApiException( + "upstream_unavailable", + `${what}查询失败：${e instanceof TmdbError ? e.kind : (e as Error).message}`, + ); } - await cache.set(REGION, key, v, { ttl: TTL_SEC, maxsize: 50 }).catch(() => {}); - return v; } -export function bangumiCalendar(rt: SourcesRuntime) { - return cached(rt, "bangumi:calendar", async () => ({ days: await clients(rt).bangumi.calendar() })); +const fromMeta = (s: MetaSubject): DiscoverItem => ({ + source: s.source, + id: s.id, + title: s.title, + aliases: s.aliases, + year: s.year, + kind: s.kind, + cover: s.cover, + rating: s.rating, + date: s.date, + episodes: s.episodes, + url: s.url, + genres: s.genres ?? [], + posterPath: null, + tmdb: null, + local: null, +}); + +const fromTmdb = (t: TmdbListItem): DiscoverItem => ({ + source: "tmdb", + id: `${t.type}:${t.id}`, + title: t.title, + aliases: t.originalTitle && t.originalTitle !== t.title ? [t.originalTitle] : [], + year: t.year, + kind: t.type, + cover: null, + rating: t.voteAverage, + date: t.date, + episodes: null, + url: `https://www.themoviedb.org/${t.type}/${t.id}`, + genres: t.genres, + posterPath: t.posterPath, + tmdb: { type: t.type, id: t.id }, + local: null, +}); + +// ── 本地状态 ── + +/** 豆瓣 / bgm 条目补上记住的 TMDB 匹配,再按 TMDB 身份查本地作品与库存 */ +export async function annotate(rt: SourcesRuntime, items: DiscoverItem[], inventory: InventoryLookup) { + const cache = pgCacheFor(rt.ctx.db); + await Promise.all( + items + .filter((it) => !it.tmdb && it.source !== "tmdb") + .map(async (it) => { + const hit = await cache.get(MATCH_REGION, `${it.source}:${it.id}`).catch(() => null); + if (hit) it.tmdb = { type: hit.type, id: hit.id }; + }), + ); + const refs = items.flatMap((it) => (it.tmdb ? [{ entityType: it.tmdb.type, externalId: it.tmdb.id }] : [])); + if (!refs.length) return items; + const local = await localEntities(rt.ctx, refs); + const ids = [...new Set(local.values())]; + const inv = ids.length ? await inventory(ids) : new Map(); + for (const it of items) { + const entityId = it.tmdb ? local.get(`${it.tmdb.type}:${it.tmdb.id}`) : undefined; + if (!entityId) continue; + const e = inv.get(entityId); + it.local = { + entityId, + subscribed: e?.subscribed ?? false, + versionCount: e?.versions.length ?? 0, + downloadingCount: e?.downloading.length ?? 0, + } satisfies DiscoverLocal; + } + return items; } -export function doubanHot(rt: SourcesRuntime, kind: "movie" | "tv") { - return cached(rt, `douban:hot:${kind}`, async () => ({ items: await clients(rt).douban.hot(kind, 30) })); +// ── TMDB ── + +const tmdbPage = (p: TmdbListPage, page: number): DiscoverPage => ({ + items: p.items.map(fromTmdb), + page, + hasMore: page < Math.min(p.totalPages, 500), +}); + +export async function tmdbList(rt: SourcesRuntime, q: DiscoverTmdbQuery): Promise { + const client = await rt.tmdb(); + if (!client) throw new ApiException("precondition_failed", "未配置 TMDB。到设置的元数据分区填写 API Key"); + const type = q.type === "all" ? "movie" : q.type; + const page = q.page; + const p = await upstream("TMDB ", () => { + switch (q.list) { + case "trending": + return tmdbTrending(client, { mediaType: q.type, window: q.window, page }); + case "search": + return tmdbSearchPage(client, { type: q.type, query: q.q ?? "", page }); + case "discover": + return tmdbDiscover(client, { + type, + ...(q.sort ? { sort: q.sort } : {}), + ...(q.genre ? { genre: q.genre } : {}), + ...(q.language ? { language: q.language } : {}), + ...(q.region ? { region: q.region } : {}), + ...(q.year ? { year: q.year } : {}), + ...(q.minRating ? { minRating: q.minRating } : {}), + page, + today: rt.now().toISOString().slice(0, 10), + }); + default: + return tmdbChart(client, { type, chart: q.list, page }); + } + }); + return tmdbPage(p, page); } -const view = (m: MetadataCandidate) => ({ +// ── 豆瓣 ── + +export async function doubanList(rt: SourcesRuntime, q: DiscoverDoubanQuery): Promise { + const { douban } = clients(rt); + const size = DISCOVER_PAGE_SIZE; + const start = (q.page - 1) * size; + return upstream("豆瓣", async () => { + if (q.list === "search") { + // 联想接口不分页 + const items = q.page === 1 ? await douban.suggest(q.q ?? "") : []; + return { items: items.map(fromMeta), page: q.page, hasMore: false }; + } + if (q.list === "tag") { + const items = await douban.tagSubjects(q.type, q.tag ?? "热门", start, size); + return { items: items.map(fromMeta), page: q.page, hasMore: items.length >= size }; + } + const r = + q.list === "explore" + ? await douban.explore( + q.type, + { sort: q.sort ?? "U", tags: [q.genre, q.region, q.year].filter((x): x is string => !!x) }, + start, + size, + ) + : await douban.collection(q.list, start, size); + const hasMore = r.total !== null ? start + size < r.total : r.items.length >= size; + return { items: r.items.map(fromMeta), page: q.page, hasMore }; + }); +} + +// ── bgm ── + +/** 趋势页 HTML 解析(外部字节走沙箱,ADR-005);测试可替换 */ +export let parseTrendsPage = (html: string) => + runSandbox("bangumi.trends", { html }, { timeoutMs: 15_000 }) as Promise>; +export function setTrendsParser(fn: typeof parseTrendsPage) { + parseTrendsPage = fn; +} + +export async function bangumiList(rt: SourcesRuntime, q: DiscoverBangumiQuery): Promise { + const { bangumi } = clients(rt); + const size = DISCOVER_PAGE_SIZE; + const offset = (q.page - 1) * size; + return upstream("Bangumi ", async () => { + if (q.list === "trends") { + const r = await parseTrendsPage(await bangumi.trendsHtml(q.page)); + return { items: r.items.map(fromMeta), page: q.page, hasMore: r.hasMore }; + } + const r = + q.list === "search" + ? await bangumi.searchPage(q.q ?? "", size, offset) + : await bangumi.browse({ + sort: q.list === "rank" ? "rank" : "date", + ...(q.year ? { year: q.year } : {}), + limit: size, + offset, + }); + return { items: r.items.map(fromMeta), page: q.page, hasMore: offset + size < r.total }; + }); +} + +export async function bangumiCalendar(rt: SourcesRuntime, inventory: InventoryLookup): Promise { + const days = await upstream("Bangumi ", () => clients(rt).bangumi.calendar()); + const out = days.map((d) => ({ ...d, items: d.items.map(fromMeta) })); + await annotate( + rt, + out.flatMap((d) => d.items), + inventory, + ); + return { days: out }; +} + +// ── 匹配 TMDB ── + +const view = (m: MetadataCandidate): DiscoverMatchCandidate => ({ type: m.entityType as "movie" | "tv", id: m.externalId, title: m.title, @@ -50,6 +257,10 @@ const view = (m: MetadataCandidate) => ({ }); export async function matchTmdb(rt: SourcesRuntime, req: DiscoverMatchRequest): Promise { + const cache = pgCacheFor(rt.ctx.db); + const memoKey = `${req.source}:${req.id}`; + const memo = await cache.get(MATCH_REGION, memoKey).catch(() => null); + if (memo) return { match: memo, candidates: [memo] }; const subject = toCached({ ...req, cover: null, rating: null, date: null, episodes: null, url: "" }, rt.now()); const base = parseRelease(req.title); if (req.kind === "tv") base.mediaTypeHint = "tv"; @@ -65,5 +276,8 @@ export async function matchTmdb(rt: SourcesRuntime, req: DiscoverMatchRequest): .flatMap((r) => byCand.get(r.candidate) ?? []) .slice(0, 6); const best = res.status === "auto" && res.best ? byCand.get(res.best.candidate) : undefined; - return DiscoverMatchResult.parse({ match: best ? view(best) : null, candidates: ranked.map(view) }); + const match = best ? view(best) : null; + // 只记确定的匹配;不确定 / 失败不记,下次重算 + if (match) await cache.set(MATCH_REGION, memoKey, match, { ttl: MATCH_TTL_SEC, maxsize: MATCH_MAX }).catch(() => {}); + return DiscoverMatchResult.parse({ match, candidates: ranked.map(view) }); } diff --git a/apps/server/src/modules/search/routes.ts b/apps/server/src/modules/search/routes.ts index 14df939d..5c6325ca 100644 --- a/apps/server/src/modules/search/routes.ts +++ b/apps/server/src/modules/search/routes.ts @@ -11,8 +11,10 @@ import { type ApiScope, apiResponse, DEFAULT_SETTINGS, - type DiscoverHotQuery, + type DiscoverBangumiQuery, + type DiscoverDoubanQuery, type DiscoverMatchRequest, + type DiscoverTmdbQuery, type EndpointDef, type InventoryEntry, type InventoryResponse, @@ -40,7 +42,7 @@ import { metadataLanguages, metadataTextOptions } from "../sources-identity/serv import { upsertSiteRelease } from "../sources-identity/service/snapshot.ts"; import type { ModuleApp, ServerContext } from "../types.ts"; import { matchActions } from "./actions.ts"; -import { bangumiCalendar, doubanHot, matchTmdb } from "./discover.ts"; +import { annotate, bangumiCalendar, bangumiList, doubanList, matchTmdb, tmdbList } from "./discover.ts"; import { InternalDispatcher } from "./dispatch.ts"; import { localEntities, raceAbort, searchLocalMedia, TmdbSearch, withInventory } from "./media.ts"; import { decorate, facets, quickFilter, ReleaseFanout, sortReleases } from "./releases.ts"; @@ -383,8 +385,22 @@ export function registerSearchRoutes(app: ModuleApp, ctx: ServerContext, d: Sear }); }); - route(app, Api.search.discoverCalendar, () => bangumiCalendar(d.runtime())); - route(app, Api.search.discoverHot, ({ query }) => doubanHot(d.runtime(), (query as DiscoverHotQuery).kind)); + route(app, Api.search.discoverCalendar, ({ c }) => bangumiCalendar(d.runtime(), inventory(c))); + const withLocal = + (c: Context) => + async

[1] }>(p: P): Promise

=> { + await annotate(d.runtime(), p.items, inventory(c)); + return p; + }; + route(app, Api.search.discoverTmdb, async ({ query, c }) => + withLocal(c)(await tmdbList(d.runtime(), query as DiscoverTmdbQuery)), + ); + route(app, Api.search.discoverDouban, async ({ query, c }) => + withLocal(c)(await doubanList(d.runtime(), query as DiscoverDoubanQuery)), + ); + route(app, Api.search.discoverBangumi, async ({ query, c }) => + withLocal(c)(await bangumiList(d.runtime(), query as DiscoverBangumiQuery)), + ); route(app, Api.search.discoverMatch, ({ body }) => matchTmdb(d.runtime(), body as DiscoverMatchRequest)); route(app, Api.search.tmdbDetail, async ({ params, query, c }) => { diff --git a/apps/server/src/modules/sources-identity/service/meta-subjects.ts b/apps/server/src/modules/sources-identity/service/meta-subjects.ts index cb19e591..efe038b1 100644 --- a/apps/server/src/modules/sources-identity/service/meta-subjects.ts +++ b/apps/server/src/modules/sources-identity/service/meta-subjects.ts @@ -3,10 +3,13 @@ import { createBangumiClient, createDoubanClient, type DoubanClient, + type MetaRequestCache, type MetaSubject, + RequestLimiter, } from "@umt/adapters"; -import { pgCacheFor } from "@umt/db"; +import { cachedCall, pgCacheFor } from "@umt/db"; import { normalizeTitle, toSimplified } from "@umt/domain"; +import { storageSettings } from "../../../shared-cache.ts"; import type { SourcesRuntime } from "../runtime.ts"; import { DOUBAN_SUBJECT_REGION, type DoubanSubject, loadDoubanSubject, saveDoubanSubject } from "./douban-subject.ts"; @@ -20,10 +23,46 @@ const MISS_TTL_SEC = 6 * 3600; export type MetaSource = "douban" | "bangumi"; const REGION: Record = { douban: DOUBAN_SUBJECT_REGION, bangumi: BANGUMI_SUBJECT_REGION }; +/** 豆瓣 / bgm 请求缓存 region(对照 MP DoubanApi.__invoke / BangumiApi 的 @cached);key 见 metaRequestKey */ +export const META_REQUEST_REGIONS = { douban: "douban.request", bangumi: "bangumi.request" } as const; +const META_REQUEST_MAXSIZE = 5000; +/** 每源限流(进程内):豆瓣风控严,间隔大一些;缓存命中不占额度 */ +const META_LIMITERS = { douban: new RequestLimiter(2, 300), bangumi: new RequestLimiter(3, 100) }; + +/** + * 请求缓存:搜索按 storage.cacheTtl.metaSearchDays,列表(榜单 / 热门 / 每日放送 / 趋势页)按 metaListHours, + * 空结果按 metaEmptyHours(不超过正常 TTL);失败(网络、403、风控页)在适配器里抛出,cachedCall 不写。 + * 条目详情不走这里:douban.subject / bangumi.subject 条目级缓存(30 天)已经存了,避免同一份数据存两遍。 + */ +export function metaRequestCache(ctx: SourcesRuntime["ctx"]): MetaRequestCache { + return { + async cached(req, load) { + const t = (await storageSettings(ctx)).cacheTtl; + const ttl = req.kind === "search" ? t.metaSearchDays * 86_400 : t.metaListHours * 3600; + return cachedCall( + pgCacheFor(ctx.db), + req.key, + { + region: META_REQUEST_REGIONS[req.source], + ttl, + emptyTtl: Math.min(ttl, t.metaEmptyHours * 3600), + emptyIf: req.empty, + maxsize: META_REQUEST_MAXSIZE, + }, + load, + ); + }, + }; +} + export function metaClients(rt: SourcesRuntime): { douban: DoubanClient; bangumi: BangumiClient } | null { const fetch = rt.metaFetch(); if (!fetch) return null; - return { douban: createDoubanClient({ fetch }), bangumi: createBangumiClient({ fetch }) }; + const cache = metaRequestCache(rt.ctx); + return { + douban: createDoubanClient({ fetch, cache, limiter: META_LIMITERS.douban }), + bangumi: createBangumiClient({ fetch, cache, limiter: META_LIMITERS.bangumi }), + }; } export function toCached(s: MetaSubject, now: Date): DoubanSubject { diff --git a/apps/server/src/sandbox/jobs.ts b/apps/server/src/sandbox/jobs.ts index cd8cb369..102e8c7a 100644 --- a/apps/server/src/sandbox/jobs.ts +++ b/apps/server/src/sandbox/jobs.ts @@ -1,7 +1,7 @@ // 沙箱任务表(docs/adr/ADR-005-sandbox.md):只放「外部不可信字节 → 结构化数据」的纯函数。 // 子进程加载完本表后撤销全部权限(文件 / 网络 / 环境变量 / 子进程),所以任务里不能有任何 I/O; // 输入输出必须能 JSON 化(二进制用 base64)。新增任务:在这里加一项,调用方用 runSandbox(名字, 输入)。 -import { parseTorrent } from "@umt/adapters"; +import { parseBangumiBrowser, parseTorrent } from "@umt/adapters"; import { decodeSubtitle, detectSubtitleLanguage, subtitleEpisode, subtitleExtOf } from "@umt/domain"; import { buildPreview } from "../modules/file-manager/preview-build.ts"; import { PAGE_PARSERS, type PageKind } from "../modules/subtitles/online/parse.ts"; @@ -121,12 +121,18 @@ function subtitlePage(input: { kind: PageKind; html: string }) { return fn(input.html); } +/** bgm.tv 动画浏览页(热门趋势,发现页)→ 条目列表 */ +function bangumiTrends(input: { html: string }) { + return parseBangumiBrowser(input.html); +} + export const SANDBOX_JOBS = { "subtitle.parse": subtitleParse, "torrent.parse": torrentParse, "file.preview": buildPreview, "captcha.solve": captchaSolve, "subtitle.page": subtitlePage, + "bangumi.trends": bangumiTrends, "self.test": selfTest, } as const; diff --git a/apps/server/test/search/discover.test.ts b/apps/server/test/search/discover.test.ts new file mode 100644 index 00000000..45859a60 --- /dev/null +++ b/apps/server/test/search/discover.test.ts @@ -0,0 +1,274 @@ +import { createTmdbClient, parseBangumiBrowser } from "@umt/adapters"; +import { type DB, type Kysely, pgCacheFor } from "@umt/db"; +import { createMockTmdb, type MockTmdb } from "@umt/testkit"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { createOutboxPort } from "../../src/context.ts"; +import { createApp } from "../../src/http/app.ts"; +import { EventBus } from "../../src/http/sse.ts"; +import { createAcquisitionOrganizeModule } from "../../src/modules/acquisition-organize/index.ts"; +import { createCapabilityRegistry } from "../../src/modules/agent/capability-registry.ts"; +import { setTrendsParser } from "../../src/modules/search/discover.ts"; +import { createSearchModule } from "../../src/modules/search/index.ts"; +import { createSourcesIdentityModule } from "../../src/modules/sources-identity/index.ts"; +import { META_REQUEST_REGIONS } from "../../src/modules/sources-identity/service/meta-subjects.ts"; +import type { ServerContext } from "../../src/modules/types.ts"; +import { SANDBOX_JOBS } from "../../src/sandbox/jobs.ts"; +import { DEV_KEY, memoryLogger } from "../helpers.ts"; +import { fakeEngine, fakeSecrets, plainSettings } from "../sources-identity/harness.ts"; +import { createTestSchema, pgAvailable, type TestSchema, uniqueSchema } from "../support/pg.ts"; + +// 发现页接口:TMDB 列表分页、筛选参数校验、豆瓣 / bgm 请求缓存(命中、空结果短 TTL、失败不缓存)、本地状态标注、 +// 豆瓣 / bgm 条目确定匹配后记住并用于标注。外站全部是 fixture(mock TMDB + 假 fetch)。 + +const up = await pgAvailable(); + +// biome-ignore lint/suspicious/noExplicitAny: 测试直接读 JSON +type Loose = any; + +const DOUBAN_ITEM = (id: string, title: string, year = "2023") => ({ + id, + title, + type: "movie", + year, + rating: { value: 8.1 }, + pic: { large: `https://img.doubanio.com/${id}.jpg` }, + card_subtitle: `${year} / 美国 / 剧情 传记 / 导演 / 主演`, +}); + +const BGM_HTML = `

`; + +describe.skipIf(!up)("发现页接口", () => { + let ts: TestSchema; + let db: Kysely; + let tmdb: MockTmdb; + let app: ReturnType; + const meta: string[] = []; + /** 下一次豆瓣榜单请求的应答 */ + let doubanReply: () => Response = () => new Response("{}"); + + const req = async (path: string, body?: unknown) => { + const res = await app.request(`/api/v1${path}`, { + method: body === undefined ? "GET" : "POST", + headers: { + authorization: `Bearer ${DEV_KEY}`, + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); + return { status: res.status, body: (await res.json()) as Loose }; + }; + const json = (b: unknown, status = 200) => + new Response(JSON.stringify(b), { status, headers: { "content-type": "application/json" } }); + + beforeAll(async () => { + tmdb = await createMockTmdb(); + ts = await createTestSchema(uniqueSchema("discover"), { max: 8 }); + db = ts.db; + const { logger } = memoryLogger(); + const ctx = { + role: "all", + db, + config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY } as never, + logger, + events: new EventBus(5), + engine: fakeEngine(), + outbox: createOutboxPort(), + secrets: fakeSecrets(() => db), + settings: plainSettings(() => db), + capabilities: createCapabilityRegistry({ logger }), + } as ServerContext; + const metaFetch: typeof fetch = async (input, init) => { + const u = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url); + meta.push(`${init?.method ?? "GET"} ${u.host}${u.pathname}${u.search}`); + if (u.pathname.startsWith("/rexxar/api/v2/subject_collection/")) return doubanReply(); + if (u.pathname === "/j/subject_suggest") + return json([{ id: "26100958", title: "奥本海默", year: "2023", episode: "", type: "movie" }]); + if (u.pathname === "/j/search_subjects") return json({ subjects: [] }); + if (u.host === "bgm.tv" && u.pathname === "/anime/browser") return new Response(BGM_HTML); + if (u.host === "api.bgm.tv" && u.pathname === "/v0/subjects") + return json({ data: [{ id: 1, name: "a", name_cn: "甲" }], total: 45 }); + throw new Error(`测试不允许的请求 ${u.href}`); + }; + const sources = createSourcesIdentityModule({ + tmdb: () => createTmdbClient({ apiKey: "test-tmdb-key", baseUrl: `${tmdb.url}/3`, retries: 0 }), + metaFetch, + }); + const search = createSearchModule({ runtime: (c) => sources.runtime(c) }); + app = createApp({ + config: { NODE_ENV: "development", DEV_ADMIN_KEY: DEV_KEY }, + logger, + bus: new EventBus(5), + role: "all", + version: "0.0.0-test", + modules: [sources, createAcquisitionOrganizeModule(), search], + ctx, + }); + // 趋势页解析缺省走沙箱子进程;这里在进程内跑同一个解析函数(沙箱任务登记见下面的用例) + setTrendsParser(async (html) => parseBangumiBrowser(html)); + + // 本地作品:奥本海默(TMDB movie 872585)已订阅 + const e = await db + .insertInto("media_entity") + .values({ kind: "movie", title: "奥本海默", year: 2023 }) + .returning("id") + .executeTakeFirstOrThrow(); + await db + .insertInto("external_identity") + .values({ + media_entity_id: e.id, + provider: "tmdb", + entity_type: "movie", + external_id: "872585", + status: "accepted", + }) + .execute(); + await db + .insertInto("subscription") + .values({ media_entity_id: e.id, media_kind: "movie", status: "active" }) + .execute(); + }, 120_000); + + afterAll(async () => { + await ts?.drop(); + await tmdb?.close(); + }); + + beforeEach(() => { + meta.length = 0; + }); + + it("TMDB 流行趋势:分页、TMDB 引用、本地状态", async () => { + const r = await req("/discover/tmdb?list=trending&type=all&window=week"); + expect(r.status).toBe(200); + const d = r.body.data; + expect(d.page).toBe(1); + expect(d.hasMore).toBe(false); + expect(d.items.length).toBeGreaterThan(3); + const opp = d.items.find((i: Loose) => i.tmdb?.id === "872585"); + expect(opp).toMatchObject({ + source: "tmdb", + id: "movie:872585", + kind: "movie", + posterPath: "/oppenheimer.jpg", + genres: ["剧情", "历史"], + local: { subscribed: true, versionCount: 0, downloadingCount: 0 }, + }); + expect(d.items.filter((i: Loose) => i.tmdb?.id !== "872585").every((i: Loose) => i.local === null)).toBe(true); + expect(tmdb.callsTo("/3/trending/all/week")).toHaveLength(1); + }); + + it("TMDB 榜单与筛选:参数传到 TMDB", async () => { + const r = await req("/discover/tmdb?list=discover&type=movie&genre=878&sort=rating&year=2020s&minRating=6"); + expect(r.status).toBe(200); + const call = tmdb.callsTo("/3/discover/movie").at(-1)!; + expect(call.query).toMatchObject({ + with_genres: "878", + sort_by: "vote_average.desc", + "primary_release_date.gte": "2020-01-01", + "primary_release_date.lte": "2029-12-31", + "vote_average.gte": "6", + }); + const now = await req("/discover/tmdb?list=now_playing&type=movie&page=1"); + expect(now.body.data.items.every((i: Loose) => i.kind === "movie")).toBe(true); + }); + + it("筛选参数校验:不配的类型 / 列表 / 风格 / 关键词 / 页码 → 422", async () => { + for (const q of [ + "list=now_playing&type=tv", + "list=popular&type=all", + "list=popular&type=movie&genre=878", + "list=discover&type=tv&genre=28", + "list=discover&type=movie&year=1888", + "list=discover&type=movie&minRating=11", + "list=search&type=all", + "list=trending&q=x", + "list=trending&page=0", + "list=trending&page=501", + ]) { + const r = await req(`/discover/tmdb?${q}`); + expect(r.status, q).toBe(422); + } + for (const q of ["list=nope", "list=tag&tag=不存在", "list=explore&genre=不存在", "list=movie_top250&sort=S"]) { + expect((await req(`/discover/douban?${q}`)).status, q).toBe(422); + } + expect((await req("/discover/bangumi?list=trends&year=2024")).status).toBe(422); + expect((await req("/discover/bangumi?list=search")).status).toBe(422); + }); + + it("豆瓣榜单:请求缓存命中;空结果短 TTL;失败不缓存", async () => { + const cache = pgCacheFor(db); + // 失败(403)不缓存 + doubanReply = () => new Response("forbidden", { status: 403 }); + expect((await req("/discover/douban?list=movie_showing")).status).toBe(502); + expect(await cache.count(META_REQUEST_REGIONS.douban)).toBe(0); + // 风控页(不是 JSON)不缓存 + doubanReply = () => new Response("验证", { headers: { "content-type": "text/html" } }); + expect((await req("/discover/douban?list=movie_showing")).status).toBe(502); + expect(await cache.count(META_REQUEST_REGIONS.douban)).toBe(0); + + doubanReply = () => + json({ subject_collection_items: [DOUBAN_ITEM("26100958", "奥本海默"), DOUBAN_ITEM("2", "乙")], total: 45 }); + const a = await req("/discover/douban?list=movie_showing&page=2"); + expect(a.status).toBe(200); + expect(a.body.data).toMatchObject({ page: 2, hasMore: true }); + expect(a.body.data.items[0]).toMatchObject({ source: "douban", id: "26100958", genres: ["剧情", "传记"] }); + const b = await req("/discover/douban?list=movie_showing&page=2"); + expect(b.body.data.items).toHaveLength(2); + expect(meta.filter((m) => m.includes("movie_showing"))).toHaveLength(3); + expect(meta.at(-1)).toContain("start=20&count=20"); + + // 空结果:写入但用空结果 TTL(缺省 1 小时),正常 6 小时 + doubanReply = () => json({ subject_collection_items: [], total: 0 }); + await req("/discover/douban?list=tv_korean"); + const rows = await cache.items(META_REQUEST_REGIONS.douban); + const ttlOf = (part: string) => { + const row = rows.find((x) => x.key.includes(part))!; + return Math.round((new Date(row.expiresAt!).getTime() - Date.now()) / 60_000); + }; + expect(ttlOf("tv_korean")).toBeLessThanOrEqual(60); + expect(ttlOf("tv_korean")).toBeGreaterThan(55); + expect(ttlOf("movie_showing")).toBeGreaterThan(5 * 60); + }); + + it("搜索与 bgm 浏览 / 趋势", async () => { + const s = await req("/discover/douban?list=search&q=奥本海默"); + expect(s.body.data).toMatchObject({ hasMore: false, items: [{ id: "26100958", kind: "movie", year: 2023 }] }); + const t = await req("/discover/bangumi?list=trends"); + expect(t.body.data.items[0]).toMatchObject({ source: "bangumi", id: "400602", episodes: 28 }); + const rank = await req("/discover/bangumi?list=rank&year=2024&page=2"); + expect(rank.body.data).toMatchObject({ page: 2, hasMore: true }); + expect(meta.find((m) => m.includes("/v0/subjects"))).toContain("offset=20"); + expect(SANDBOX_JOBS["bangumi.trends"]({ html: BGM_HTML }).items[0]!.id).toBe("400602"); + }); + + it("豆瓣条目确定匹配后记住,列表据此标本地状态", async () => { + const m = await req("/discover/tmdb-match", { + source: "douban", + id: "26100958", + title: "奥本海默", + aliases: ["Oppenheimer"], + year: 2023, + kind: "movie", + }); + expect(m.status).toBe(200); + expect(m.body.data.match).toMatchObject({ type: "movie", id: "872585" }); + const searches = tmdb.callsTo("/3/search").length; + const again = await req("/discover/tmdb-match", { + source: "douban", + id: "26100958", + title: "奥本海默", + aliases: [], + year: 2023, + kind: "movie", + }); + expect(again.body.data.match.id).toBe("872585"); + expect(tmdb.callsTo("/3/search").length).toBe(searches); + + const s = await req("/discover/douban?list=search&q=奥本海默"); + expect(s.body.data.items[0]).toMatchObject({ + tmdb: { type: "movie", id: "872585" }, + local: { subscribed: true }, + }); + }); +});