feat(adapters): TMDB 列表与筛选、豆瓣榜单 / 标签 / 筛选 / 搜索、bgm 浏览与趋势,豆瓣 / bgm 请求缓存端口

- TMDB:流行趋势(日 / 周)、电影 / 剧集榜单、discover 参数映射、分页搜索;列表类请求 TTL 6 小时
- 豆瓣 / bgm:列表与搜索请求经 MetaRequestCache 与限流,失败(403 / 风控页)不缓存
- contracts:发现页条目、分页、查询校验与筛选字典;storage.cacheTtl 追加 meta* 字段
- testkit:mock TMDB 加列表接口

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-10-06 18:04:43 +08:00
1 parent 27514f4985
commit e46eb0e3c3
13 files changed
+1344 -91

No files matched your search

+302 -24
View File
@@ -2,8 +2,12 @@
* 辅助信息源(对照 MP modules/bangumi、modules/douban):只读公开接口,结果只用来"拿名称 + 年份去匹配 TMDB"与发现页展示,
* 身份仍以 TMDB 为准。
* - bgm.tv:官方 API(api.bgm.tv),条目 / 搜索 / 每日放送,不需要登录;要求带可识别的 User-Agent
* - 豆瓣:移动版 rexxar 接口(m.douban.com,需要 Referer),条目详情与实时热门榜;搜索要登录,不用
* - 豆瓣:移动版 rexxar 接口(m.douban.com,需要 Referer):条目详情、榜单(subject_collection)、筛选(movie|tv/recommend);
* 桌面版 JSON(movie.douban.com/j/…):标签列表(search_subjects)、搜索联想(subject_suggest)。都不需要登录,HTTP 直连。
* 发现页(2026-10-06):列表 / 搜索请求经 MetaRequestCache(服务端 PG 请求缓存,对照 MP DoubanApi.__invoke / Bangumi 的 @cached)
* 与限流器;条目详情不走请求缓存(服务端已有条目级缓存 douban.subject / bangumi.subject,避免同一份数据存两遍)。
*/
import { load } from "cheerio";
export interface MetaSubject {
source: "bangumi" | "douban";
@@ -21,13 +25,36 @@ export interface MetaSubject {
date: string | null;
episodes: number | null;
url: string;
/** 风格 / 类型(豆瓣 card_subtitle 第三段;没有为空) */
genres?: string[];
}
/** 请求类别:决定缓存 TTL(搜索 / 列表);条目详情不走请求缓存 */
export type MetaRequestKind = "search" | "list";
/**
* 请求缓存端口(服务端用 PG kv_cache 实现,region douban.request / bangumi.request):
* 命中直接返回;否则执行 load,成功的结果按类别 TTL 写入,empty(v) 为真时用空结果短 TTL;load 抛错(网络失败、403、风控页)不写。
*/
export interface MetaRequestCache {
cached<T>(
req: { source: "bangumi" | "douban"; key: string; kind: MetaRequestKind; empty: (v: T) => boolean },
load: () => Promise<T>,
): Promise<T>;
}
export interface MetaClientOptions {
fetch?: typeof globalThis.fetch;
/** 缺省 15s */
timeoutMs?: number;
/** API 地址(bgm api.bgm.tv / 豆瓣 rexxar) */
baseUrl?: string;
/** 网页地址(bgm.tv / movie.douban.com) */
webBaseUrl?: string;
/** 请求缓存;不给就不缓存 */
cache?: MetaRequestCache;
/** 限流(每源最小间隔);只包住真正发出的网络请求,缓存命中不占额度 */
limiter?: { run<T>(fn: () => Promise<T>): Promise<T> };
}
export class MetaSourceError extends Error {
@@ -57,28 +84,70 @@ const uniq = (xs: (string | null | undefined)[], not: string) => [
...new Set(xs.filter((x): x is string => !!x && x !== not && x.length <= 120)),
];
async function fetchOk(
source: "bangumi" | "douban",
url: string,
o: MetaClientOptions,
init: RequestInit,
): Promise<Response> {
const f = o.fetch ?? globalThis.fetch;
const run = () => f(url, { ...init, signal: AbortSignal.timeout(o.timeoutMs ?? 15_000) });
let res: Response;
try {
res = await (o.limiter ? o.limiter.run(run) : run());
} catch (e) {
throw new MetaSourceError(source, "http", `请求 ${new URL(url).host} 失败:${(e as Error).message}`);
}
if (res.status === 404) {
await res.body?.cancel();
throw new MetaSourceError(source, "not_found", "条目不存在");
}
if (!res.ok) {
await res.body?.cancel();
throw new MetaSourceError(source, "http", `${new URL(url).host} 返回 HTTP ${res.status}`);
}
return res;
}
async function getJson(
source: "bangumi" | "douban",
url: string,
o: MetaClientOptions,
init: RequestInit = {},
): Promise<unknown> {
const f = o.fetch ?? globalThis.fetch;
let res: Response;
try {
res = await f(url, { ...init, signal: AbortSignal.timeout(o.timeoutMs ?? 15_000) });
} catch (e) {
throw new MetaSourceError(source, "http", `请求 ${new URL(url).host} 失败:${(e as Error).message}`);
}
if (res.status === 404) throw new MetaSourceError(source, "not_found", "条目不存在");
if (!res.ok) throw new MetaSourceError(source, "http", `${new URL(url).host} 返回 HTTP ${res.status}`);
const res = await fetchOk(source, url, o, init);
try {
return await res.json();
} catch {
// 风控 / 验证页是 HTML:按解析失败抛出,不缓存
throw new MetaSourceError(source, "parse", `${new URL(url).host} 返回的不是 JSON`);
}
}
/** 请求缓存键:方法 + 参数排序后的地址 + 请求体 */
export function metaRequestKey(method: string, url: string, body?: string | null): string {
const u = new URL(url);
const qs = new URLSearchParams([...u.searchParams].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
const q = qs.toString();
return `${method.toUpperCase()} ${u.origin}${u.pathname}${q ? `?${q}` : ""}${body ? ` ${body}` : ""}`;
}
function viaCache<T>(
o: MetaClientOptions,
source: "bangumi" | "douban",
kind: MetaRequestKind,
key: string,
empty: (v: T) => boolean,
load: () => Promise<T>,
): Promise<T> {
return o.cache ? o.cache.cached({ source, key, kind, empty }, load) : load();
}
const emptyRows = (field: string) => (v: unknown) => {
const rows = isObj(v) ? v[field] : v;
return Array.isArray(rows) && rows.length === 0;
};
// ── bgm.tv ──
export const BANGUMI_API = "https://api.bgm.tv";
@@ -114,9 +183,71 @@ export function bangumiSubject(r: Raw): MetaSubject | null {
};
}
export const BANGUMI_WEB = "https://bgm.tv";
/** bgm.tv 网页日期「2023年9月29日」/「2023-09-29」→ YYYY-MM-DD */
function bgmDate(text: string): string | null {
const m = /((?:19|20)\d{2})[年-](\d{1,2})[月-](\d{1,2})/.exec(text);
return m ? `${m[1]}-${m[2]!.padStart(2, "0")}-${m[3]!.padStart(2, "0")}` : null;
}
/**
* bgm.tv 动画浏览页(/anime/browser?sort=trends,服务端渲染 HTML)→ 条目列表。外部 HTML 不可信:
* 服务端经沙箱任务 bangumi.trends 调用(ADR-005),这里只是纯函数。
*/
export function parseBangumiBrowser(html: string, web = BANGUMI_WEB): { items: MetaSubject[]; hasMore: boolean } {
const $ = load(html);
const items: MetaSubject[] = [];
$("#browserItemList > li.item").each((_, el) => {
const li = $(el);
const id =
/(\d+)/.exec(li.attr("id") ?? "")?.[1] ?? /\/subject\/(\d+)/.exec(li.find("h3 a").attr("href") ?? "")?.[1];
const title = li.find("h3 a.l").first().text().trim();
if (!id || !title) return;
const original = li.find("h3 small.grey").first().text().trim();
const info = li.find("p.info").first().text().replace(/\s+/g, " ").trim();
const date = bgmDate(info);
const eps = numOf(/(\d+)\s*话/.exec(info)?.[1]);
const rating = numOf(li.find("p.rateInfo small.fade").first().text().trim());
let cover = str(li.find("img.cover").first().attr("src"));
if (cover?.startsWith("//")) cover = `https:${cover}`;
items.push({
source: "bangumi",
id,
title,
aliases: original && original !== title ? [original] : [],
year: yearOf(date ?? info),
kind: /剧场|电影/.test(info) ? "movie" : "tv",
cover: https(cover),
rating: rating || null,
date,
episodes: eps || null,
url: `${web}/subject/${id}`,
});
});
const hasMore = $("#multipage a.p")
.toArray()
.some((a) => $(a).text().includes("››") || $(a).text().includes("\u203a\u203a"));
return { items, hasMore };
}
export function createBangumiClient(o: MetaClientOptions = {}) {
const base = (o.baseUrl ?? BANGUMI_API).replace(/\/+$/, "");
const web = (o.webBaseUrl ?? BANGUMI_WEB).replace(/\/+$/, "");
const headers = { "User-Agent": BGM_UA, Accept: "application/json" };
const searchPage = async (keyword: string, limit = 10, offset = 0) => {
const url = `${base}/v0/search/subjects?limit=${limit}&offset=${offset}`;
const body = JSON.stringify({ keyword, filter: { type: [2] } });
const r = await viaCache(o, "bangumi", "search", metaRequestKey("POST", url, body), emptyRows("data"), () =>
getJson("bangumi", url, o, {
method: "POST",
headers: { ...headers, "Content-Type": "application/json" },
body,
}),
);
const rows = isObj(r) && Array.isArray(r.data) ? (r.data as Raw[]) : [];
return { items: rows.flatMap((x) => bangumiSubject(x) ?? []), total: isObj(r) ? (numOf(r.total) ?? 0) : 0 };
};
return {
async subject(id: string): Promise<MetaSubject> {
if (!/^\d+$/.test(id)) throw new MetaSourceError("bangumi", "not_found", "bgm 条目 ID 非法");
@@ -127,17 +258,16 @@ export function createBangumiClient(o: MetaClientOptions = {}) {
},
/** 动画条目搜索(type 2) */
async search(keyword: string, limit = 10): Promise<MetaSubject[]> {
const r = await getJson("bangumi", `${base}/v0/search/subjects?limit=${limit}`, o, {
method: "POST",
headers: { ...headers, "Content-Type": "application/json" },
body: JSON.stringify({ keyword, filter: { type: [2] } }),
});
const rows = isObj(r) && Array.isArray(r.data) ? (r.data as Raw[]) : [];
return rows.flatMap((x) => bangumiSubject(x) ?? []);
return (await searchPage(keyword, limit)).items;
},
/** 动画条目搜索,带分页与总数(发现页) */
searchPage,
/** 每日放送:周一到周日(weekday 1-7) */
async calendar(): Promise<{ weekday: number; label: string; items: MetaSubject[] }[]> {
const r = await getJson("bangumi", `${base}/calendar`, o, { headers });
const url = `${base}/calendar`;
const r = await viaCache(o, "bangumi", "list", metaRequestKey("GET", url), emptyRows(""), () =>
getJson("bangumi", url, o, { headers }),
);
if (!Array.isArray(r)) throw new MetaSourceError("bangumi", "parse", "bgm 每日放送格式不对");
return (r as Raw[]).map((d) => {
const wd = isObj(d.weekday) ? d.weekday : {};
@@ -148,6 +278,34 @@ export function createBangumiClient(o: MetaClientOptions = {}) {
};
});
},
/** 动画浏览(对照 MP bangumi discover:v0/subjects?type=2&sort=rank|date&year=) */
async browse(q: { sort: "rank" | "date"; year?: string; limit: number; offset: number }) {
const qs = new URLSearchParams({ type: "2", sort: q.sort, limit: String(q.limit), offset: String(q.offset) });
if (q.year) qs.set("year", q.year);
const url = `${base}/v0/subjects?${qs}`;
const r = await viaCache(o, "bangumi", "list", metaRequestKey("GET", url), emptyRows("data"), () =>
getJson("bangumi", url, o, { headers }),
);
if (!isObj(r) || !Array.isArray(r.data)) throw new MetaSourceError("bangumi", "parse", "bgm 浏览结果格式不对");
return { items: (r.data as Raw[]).flatMap((x) => bangumiSubject(x) ?? []), total: numOf(r.total) ?? 0 };
},
/** 热门趋势网页(无 API):返回原始 HTML,由调用方在沙箱里用 parseBangumiBrowser 解析 */
async trendsHtml(page = 1): Promise<string> {
const url = `${web}/anime/browser?sort=trends&page=${page}`;
return viaCache(
o,
"bangumi",
"list",
metaRequestKey("GET", url),
() => false,
async () => {
const res = await fetchOk("bangumi", url, o, { headers: { "User-Agent": BGM_UA, Accept: "text/html" } });
const text = await res.text();
if (!text.includes("browserItemList")) throw new MetaSourceError("bangumi", "parse", "bgm 浏览页格式不对");
return text;
},
);
},
};
}
export type BangumiClient = ReturnType<typeof createBangumiClient>;
@@ -172,6 +330,11 @@ export function doubanSubject(r: Raw): MetaSubject | null {
const pub = Array.isArray(r.pubdate) ? str(r.pubdate[0]) : null;
const date = /\d{4}-\d{2}-\d{2}/.exec(pub ?? "")?.[0] ?? null;
const type = str(r.type) ?? str(r.subtype);
// card_subtitle:「2024 / 美国 / 剧情 科幻 / 导演 / 主演」
const card = (str(r.card_subtitle) ?? "").split(" / ");
const genres =
card.length >= 3 && /^\d{4}$/.test(card[0]!.trim()) ? card[2]!.trim().split(/\s+/).filter(Boolean) : [];
const cover = str(pic.large) ?? str(pic.normal) ?? str(r.cover_url);
return {
source: "douban",
id,
@@ -179,11 +342,61 @@ export function doubanSubject(r: Raw): MetaSubject | null {
aliases: uniq([str(r.original_title), ...(Array.isArray(r.aka) ? r.aka.map(str) : [])], title).slice(0, 10),
year: yearOf(r.year) ?? yearOf(date),
kind: type === "tv" || r.is_tv === true ? "tv" : type === "movie" ? "movie" : null,
cover: str(pic.large) ?? str(pic.normal),
cover: placeholderCover(cover) ? null : cover,
rating: rating || null,
date,
episodes: numOf(r.episodes_count) || null,
url: `https://movie.douban.com/subject/${id}/`,
genres: genres.slice(0, 4),
};
}
/** 豆瓣占位海报(对照 MP _project_discover 的 invalid_posters) */
const placeholderCover = (u: string | null) =>
!!u && ["movie_large.jpg", "tv_normal.png", "tv_normal.jpg", "tv_large.jpg"].some((m) => u.includes(m));
/** 桌面版 search_subjects 条目({id, title, rate, cover, url, episodes_info}) → MetaSubject */
export function doubanTagSubject(r: Raw, kind: "movie" | "tv"): MetaSubject | null {
const id = str(r.id);
const title = str(r.title);
if (!id || !/^\d+$/.test(id) || !title) return null;
const cover = str(r.cover);
return {
source: "douban",
id,
title,
aliases: [],
year: null,
kind,
cover: placeholderCover(cover) ? null : cover,
rating: numOf(r.rate) || null,
date: null,
episodes: null,
url: `https://movie.douban.com/subject/${id}/`,
genres: [],
};
}
/** 搜索联想 subject_suggest 条目({id, title, sub_title, img, year, episode, type}) → MetaSubject;人物等返回 null */
export function doubanSuggestSubject(r: Raw): MetaSubject | null {
if (str(r.type) !== "movie") return null;
const id = str(r.id);
const title = str(r.title);
if (!id || !/^\d+$/.test(id) || !title) return null;
const sub = str(r.sub_title);
return {
source: "douban",
id,
title,
aliases: sub && sub !== title ? [sub] : [],
year: yearOf(r.year),
kind: str(r.episode) ? "tv" : "movie",
cover: str(r.img),
rating: null,
date: null,
episodes: numOf(r.episode) || null,
url: `https://movie.douban.com/subject/${id}/`,
genres: [],
};
}
@@ -192,10 +405,28 @@ export const DOUBAN_HOT = {
tv: "tv_real_time_hotest",
} as const;
export const DOUBAN_WEB = "https://movie.douban.com";
const DOUBAN_WEB_HEADERS = {
Referer: "https://movie.douban.com/",
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
Accept: "application/json",
};
export function createDoubanClient(o: MetaClientOptions = {}) {
const base = (o.baseUrl ?? DOUBAN_API).replace(/\/+$/, "");
const web = (o.webBaseUrl ?? DOUBAN_WEB).replace(/\/+$/, "");
const get = (path: string) => getJson("douban", `${base}${path}`, o, { headers: DOUBAN_HEADERS });
return {
const cachedGet = (kind: MetaRequestKind, url: string, field: string, headers: Record<string, string>) =>
viaCache(o, "douban", kind, metaRequestKey("GET", url), emptyRows(field), () =>
getJson("douban", url, o, { headers }),
);
const listOf = (r: unknown, field: string, what: string) => {
const rows = isObj(r) && Array.isArray(r[field]) ? (r[field] as Raw[]) : null;
if (!rows) throw new MetaSourceError("douban", "parse", `豆瓣${what}格式不对`);
return { rows, total: isObj(r) ? numOf(r.total) : null };
};
const client = {
/** 条目详情:类型未知时先按电影查,不存在再按剧集查 */
async subject(id: string, kind?: "movie" | "tv" | null): Promise<MetaSubject> {
if (!/^\d+$/.test(id)) throw new MetaSourceError("douban", "not_found", "豆瓣条目 ID 非法");
@@ -213,13 +444,60 @@ export function createDoubanClient(o: MetaClientOptions = {}) {
}
throw last instanceof MetaSourceError ? last : new MetaSourceError("douban", "not_found", "豆瓣条目不存在");
},
/** 榜单(rexxar subject_collection):实时热门、正在热映、Top250、各类剧集榜… */
async collection(name: string, start = 0, count = 20): Promise<{ items: MetaSubject[]; total: number | null }> {
if (!/^[a-z0-9_]+$/.test(name)) throw new MetaSourceError("douban", "not_found", "豆瓣榜单名非法");
const url = `${base}/subject_collection/${name}/items?start=${start}&count=${count}`;
const { rows, total } = listOf(
await cachedGet("list", url, "subject_collection_items", DOUBAN_HEADERS),
"subject_collection_items",
"榜单",
);
return { items: rows.flatMap((x) => doubanSubject(x) ?? []), total };
},
/** 实时热门榜(电影 / 剧集) */
async hot(kind: "movie" | "tv", count = 30): Promise<MetaSubject[]> {
const r = await get(`/subject_collection/${DOUBAN_HOT[kind]}/items?start=0&count=${count}`);
const rows = isObj(r) && Array.isArray(r.subject_collection_items) ? (r.subject_collection_items as Raw[]) : null;
if (!rows) throw new MetaSourceError("douban", "parse", "豆瓣榜单格式不对");
return rows.flatMap((x) => doubanSubject(x) ?? []);
return (await client.collection(DOUBAN_HOT[kind], 0, count)).items;
},
/** 标签列表(桌面版 j/search_subjects,tag 如 热门 / 最新 / 豆瓣高分 / 美剧) */
async tagSubjects(kind: "movie" | "tv", tag: string, start = 0, count = 20): Promise<MetaSubject[]> {
const qs = new URLSearchParams({
type: kind,
tag,
sort: "recommend",
page_limit: String(count),
page_start: String(start),
});
const url = `${web}/j/search_subjects?${qs}`;
const { rows } = listOf(await cachedGet("list", url, "subjects", DOUBAN_WEB_HEADERS), "subjects", "标签列表");
return rows.flatMap((x) => doubanTagSubject(x, kind) ?? []);
},
/** 筛选(rexxar movie|tv/recommend,对照 MP douban_discover:sort U/R/T/S,tags 为 风格,地区,年代) */
async explore(
kind: "movie" | "tv",
q: { sort: string; tags: string[] },
start = 0,
count = 20,
): Promise<{ items: MetaSubject[]; total: number | null }> {
const qs = new URLSearchParams({
refresh: "0",
start: String(start),
count: String(count),
sort: q.sort,
tags: q.tags.join(","),
});
const url = `${base}/${kind}/recommend?${qs}`;
const { rows, total } = listOf(await cachedGet("list", url, "items", DOUBAN_HEADERS), "items", "筛选结果");
return { items: rows.flatMap((x) => doubanSubject(x) ?? []), total };
},
/** 搜索联想(桌面版 j/subject_suggest,不分页,最多约 10 条) */
async suggest(q: string): Promise<MetaSubject[]> {
const url = `${web}/j/subject_suggest?${new URLSearchParams({ q: q.trim() })}`;
const r = await cachedGet("search", url, "", DOUBAN_WEB_HEADERS);
if (!Array.isArray(r)) throw new MetaSourceError("douban", "parse", "豆瓣搜索结果格式不对");
return (r as Raw[]).flatMap((x) => doubanSuggestSubject(x) ?? []);
},
};
return client;
}
export type DoubanClient = ReturnType<typeof createDoubanClient>;
@@ -0,0 +1,228 @@
import { createMockFetch } from "@umt/testkit";
import { describe, expect, it } from "vitest";
import {
createBangumiClient,
createDoubanClient,
doubanSubject,
doubanSuggestSubject,
doubanTagSubject,
type MetaRequestCache,
metaRequestKey,
parseBangumiBrowser,
} from "./index.ts";
// 发现页的豆瓣 / bgm 列表接口:字段映射、请求缓存端口(键、类别、空结果、失败不缓存)、限流只包网络请求。
// 字段形状取自 2026-10-06 协调者实测的返回(删减),不打真实站点。
const DOUBAN_ITEM = {
id: "35267208",
title: "流浪地球2",
type: "movie",
year: "2023",
rating: { value: 8.3 },
pic: { large: "https://img1.doubanio.com/view/photo/m_ratio_poster/public/p2885955777.jpg" },
card_subtitle: "2023 / 中国大陆 / 科幻 冒险 灾难 / 郭帆 / 吴京 刘德华",
};
const BGM_HTML = `<html><body>
<ul id="browserItemList" class="browserFull">
<li id="item_400602" class="item odd clearit">
<a href="/subject/400602" class="subjectCover cover ll"><span class="image"><img src="//lain.bgm.tv/r/400/pic/cover/c/13/c5/400602_ZI8Y9.jpg" class="cover" /></span></a>
<div class="inner">
<h3><span class="ico_subject_type subject_type_2 ll"></span><a href="/subject/400602" class="l">葬送的芙莉莲</a> <small class="grey">葬送のフリーレン</small></h3>
<p class="info tip">
28话 / 2023年9月29日 / 斎藤圭一郎 / 山田鐘人
</p>
<p class="rateInfo"><span class="starstop-s"><span class="starlight stars9"></span></span> <small class="fade">8.9</small> <span class="tip_j">(25627人评分)</span></p>
</div>
</li>
<li id="item_9" class="item even clearit">
<div class="inner"><h3><a href="/subject/9" class="l">剧场版 某作品</a></h3><p class="info tip">剧场版 / 2024-07-01</p></div>
</li>
</ul>
<div id="multipage"><div class="page_inner"><strong class="p_cur">1</strong><a href="?sort=trends&page=2" class="p">2</a><a href="?sort=trends&page=2" class="p">&rsaquo;&rsaquo;</a></div></div>
</body></html>`;
function memoryCache() {
const store = new Map<string, unknown>();
const writes: { key: string; kind: string; empty: boolean }[] = [];
const cache: MetaRequestCache = {
async cached(req, load) {
if (store.has(req.key)) return store.get(req.key) as never;
const v = await load();
writes.push({ key: req.key, kind: req.kind, empty: req.empty(v) });
store.set(req.key, v);
return v;
},
};
return { cache, writes };
}
describe("豆瓣列表映射", () => {
it("rexxar 条目:card_subtitle 第三段是风格;占位海报置空", () => {
expect(doubanSubject(DOUBAN_ITEM)).toMatchObject({
id: "35267208",
year: 2023,
kind: "movie",
rating: 8.3,
genres: ["科幻", "冒险", "灾难"],
});
expect(
doubanSubject({
...DOUBAN_ITEM,
pic: { large: "https://img3.doubanio.com/f/movie/30c6263b/pics/movie/movie_large.jpg" },
})?.cover,
).toBeNull();
});
it("标签列表与搜索联想", () => {
expect(
doubanTagSubject({ id: "1", title: "甲", rate: "7.9", cover: "https://x/p.jpg", episodes_info: "" }, "tv"),
).toMatchObject({ id: "1", kind: "tv", rating: 7.9, year: null });
expect(doubanTagSubject({ id: "2", title: "乙", rate: "" }, "movie")?.rating).toBeNull();
expect(
doubanSuggestSubject({
id: "3",
title: "三体",
sub_title: "Three-Body",
year: "2023",
episode: "30",
type: "movie",
}),
).toMatchObject({ kind: "tv", year: 2023, aliases: ["Three-Body"], episodes: 30 });
expect(doubanSuggestSubject({ id: "4", title: "某演员", type: "celebrity" })).toBeNull();
});
it("榜单 / 标签 / 筛选 / 搜索的请求地址与头", async () => {
const m = createMockFetch({
"GET /rexxar/api/v2/subject_collection/movie_top250/items": {
body: { subject_collection_items: [DOUBAN_ITEM], total: 250 },
},
"GET /j/search_subjects": { body: { subjects: [{ id: "1", title: "甲", rate: "8.0", cover: "c" }] } },
"GET /rexxar/api/v2/tv/recommend": { body: { items: [{ ...DOUBAN_ITEM, type: "tv" }], total: 1 } },
"GET /j/subject_suggest": { body: [{ id: "3", title: "三体", year: "2023", episode: "30", type: "movie" }] },
});
const d = createDoubanClient({ fetch: m.fetch });
expect(await d.collection("movie_top250", 20, 20)).toMatchObject({ total: 250, items: [{ id: "35267208" }] });
expect((await d.tagSubjects("movie", "豆瓣高分", 40, 20))[0]).toMatchObject({ id: "1", kind: "movie" });
expect((await d.explore("tv", { sort: "S", tags: ["悬疑", "美国"] }, 0, 20)).items[0]!.kind).toBe("tv");
expect((await d.suggest(" 三体 "))[0]!.title).toBe("三体");
const [col, tag, exp, sug] = m.requests;
expect(col!.url.search).toBe("?start=20&count=20");
expect(col!.headers.referer).toBe("https://m.douban.com/");
expect(tag!.url.host).toBe("movie.douban.com");
expect(Object.fromEntries(tag!.url.searchParams)).toEqual({
type: "movie",
tag: "豆瓣高分",
sort: "recommend",
page_limit: "20",
page_start: "40",
});
expect(Object.fromEntries(exp!.url.searchParams)).toMatchObject({ sort: "S", tags: "悬疑,美国", start: "0" });
expect(sug!.url.searchParams.get("q")).toBe("三体");
await expect(d.collection("../x")).rejects.toThrow("榜单名非法");
});
});
describe("bgm 列表", () => {
it("浏览页 HTML:标题 / 原名 / 日期 / 话数 / 评分 / 封面,有下一页", () => {
const r = parseBangumiBrowser(BGM_HTML);
expect(r.hasMore).toBe(true);
expect(r.items).toEqual([
expect.objectContaining({
source: "bangumi",
id: "400602",
title: "葬送的芙莉莲",
aliases: ["葬送のフリーレン"],
year: 2023,
date: "2023-09-29",
episodes: 28,
rating: 8.9,
kind: "tv",
cover: "https://lain.bgm.tv/r/400/pic/cover/c/13/c5/400602_ZI8Y9.jpg",
url: "https://bgm.tv/subject/400602",
}),
expect.objectContaining({ id: "9", kind: "movie", date: "2024-07-01", cover: null, rating: null }),
]);
expect(parseBangumiBrowser("<ul id='browserItemList'></ul>")).toEqual({ items: [], hasMore: false });
});
it("浏览 / 搜索 / 趋势页的请求", async () => {
const m = createMockFetch({
"GET https://api.bgm.tv/v0/subjects": {
body: { data: [{ id: 1, name: "a", name_cn: "甲", date: "2024-01-01" }], total: 100, limit: 20, offset: 20 },
},
"POST https://api.bgm.tv/v0/search/subjects": { body: { data: [{ id: 2, name: "b" }], total: 1 } },
"GET https://bgm.tv/anime/browser": { body: BGM_HTML },
});
const b = createBangumiClient({ fetch: m.fetch });
expect(await b.browse({ sort: "rank", year: "2024", limit: 20, offset: 20 })).toMatchObject({ total: 100 });
expect(await b.searchPage("芙莉莲", 20, 40)).toMatchObject({ total: 1, items: [{ id: "2" }] });
expect(await b.trendsHtml(2)).toContain("browserItemList");
expect(Object.fromEntries(m.requests[0]!.url.searchParams)).toEqual({
type: "2",
sort: "rank",
year: "2024",
limit: "20",
offset: "20",
});
expect(m.requests[1]!.url.search).toBe("?limit=20&offset=40");
expect(JSON.parse(m.requests[1]!.body!)).toEqual({ keyword: "芙莉莲", filter: { type: [2] } });
expect(m.requests[2]!.url.search).toBe("?sort=trends&page=2");
expect(m.requests[2]!.headers["user-agent"]).toContain("uhd-media-tool");
});
});
describe("请求缓存与限流", () => {
it("键含方法 / 排序后的参数 / 请求体;按类别写入,空结果标记;命中不再请求、不占限流", async () => {
expect(metaRequestKey("get", "https://a.b/x?z=1&a=2")).toBe("GET https://a.b/x?a=2&z=1");
expect(metaRequestKey("POST", "https://a.b/s", '{"k":1}')).toBe('POST https://a.b/s {"k":1}');
const m = createMockFetch({
"GET /rexxar/api/v2/subject_collection/tv_hot/items": { body: { subject_collection_items: [], total: 0 } },
"GET /j/subject_suggest": { body: [{ id: "3", title: "三体", type: "movie" }] },
"POST https://api.bgm.tv/v0/search/subjects": { body: { data: [{ id: 2, name: "b" }], total: 1 } },
});
const { cache, writes } = memoryCache();
let limited = 0;
const limiter = {
run<T>(fn: () => Promise<T>) {
limited++;
return fn();
},
};
const d = createDoubanClient({ fetch: m.fetch, cache, limiter });
const b = createBangumiClient({ fetch: m.fetch, cache, limiter });
for (let i = 0; i < 2; i++) {
await d.collection("tv_hot");
await d.suggest("三体");
await b.search("芙莉莲", 5);
}
expect(m.requests).toHaveLength(3);
expect(limited).toBe(3);
expect(writes.map((w) => [w.kind, w.empty])).toEqual([
["list", true],
["search", false],
["search", false],
]);
expect(writes[0]!.key).toBe(
"GET https://m.douban.com/rexxar/api/v2/subject_collection/tv_hot/items?count=20&start=0",
);
});
it("失败(403 / 风控页不是 JSON)抛错不写缓存;条目详情不走请求缓存", async () => {
const m = createMockFetch({
"GET /rexxar/api/v2/subject_collection/tv_hot/items": { status: 403, body: "forbidden" },
"GET /j/subject_suggest": { body: "<html>请登录</html>" },
"GET /rexxar/api/v2/movie/35267208": { body: DOUBAN_ITEM },
"GET https://bgm.tv/anime/browser": { body: "<html>验证</html>" },
});
const { cache, writes } = memoryCache();
const d = createDoubanClient({ fetch: m.fetch, cache });
const b = createBangumiClient({ fetch: m.fetch, cache });
await expect(d.collection("tv_hot")).rejects.toMatchObject({ kind: "http" });
await expect(d.suggest("x")).rejects.toMatchObject({ kind: "parse" });
await expect(b.trendsHtml()).rejects.toMatchObject({ kind: "parse" });
expect((await d.subject("35267208")).title).toBe("流浪地球2");
expect(writes).toEqual([]);
});
});
+14 -3
View File
@@ -101,8 +101,10 @@ export interface TmdbConfig {
/**
* 默认 max 1024、ttl 24h;空结果(搜索 0 条)用 emptyTtlMs,默认 30min(对照 MP EMPTY_RESULT_CACHE_TTL);
* searchTtlMs:/search/、/find/ 请求的 TTL(缺省同 ttlMs),其余(详情 / 季)用 ttlMs。
* listTtlMs:流行趋势 / 热门 / 正在上映等列表与 /discover/ 的 TTL(缺省 6 小时且不超过 ttlMs;列表会变,不能按详情缓存 30 天),
* 这类请求的空结果 TTL 也不超过它。
*/
cache?: { max?: number; ttlMs?: number; searchTtlMs?: number; emptyTtlMs?: number };
cache?: { max?: number; ttlMs?: number; searchTtlMs?: number; emptyTtlMs?: number; listTtlMs?: number };
/**
* 跨进程共享的请求缓存(服务端传 PG 缓存);给了就不再用进程内 LRU。
* 语义对照 MP @cached(skip_none、业务失败不缓存、results 为空按 emptyTtlMs)。
@@ -134,6 +136,11 @@ export function tmdbRequestKey(path: string, params: Record<string, string | und
return `${path}?${requestQuery(params)}`;
}
/** 列表类请求(发现页):流行趋势、按条件筛选、电影 / 剧集榜单 */
export const isTmdbListPath = (path: string) =>
/^\/(trending|discover)\//.test(path) ||
/^\/(movie|tv)\/(popular|now_playing|upcoming|top_rated|on_the_air|airing_today)$/.test(path);
/** 空结果(搜索 0 条)用短 TTL(对照 MP EMPTY_RESULT_CACHE_TTL) */
const isEmptyResult = (v: unknown) =>
Array.isArray((v as Json | null)?.results) && ((v as Json).results as unknown[]).length === 0;
@@ -176,9 +183,13 @@ export class TmdbClient {
private async get<T>(path: string, params: Record<string, string | undefined>): Promise<T> {
const qs = requestQuery(params);
const key = `${path}?${qs}`;
const emptyTtl = this.cfg.cache?.emptyTtlMs ?? 30 * 60_000;
let emptyTtl = this.cfg.cache?.emptyTtlMs ?? 30 * 60_000;
const baseTtl = this.cfg.cache?.ttlMs ?? 24 * 3600_000;
const ttl = /^\/(search|find)\//.test(path) ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl;
let ttl = /^\/(search|find)\//.test(path) ? (this.cfg.cache?.searchTtlMs ?? baseTtl) : baseTtl;
if (isTmdbListPath(path)) {
ttl = this.cfg.cache?.listTtlMs ?? Math.min(baseTtl, 6 * 3600_000);
emptyTtl = Math.min(emptyTtl, ttl);
}
if (this.cfg.sharedCache) {
const shared = this.cfg.sharedCache;
const hit = await shared.get(key);
+1
View File
@@ -2,6 +2,7 @@ export { RequestLimiter, TtlLruCache } from "./cache.ts";
export * from "./client.ts";
export * from "./details.ts";
export * from "./episode-groups.ts";
export * from "./lists.ts";
export * from "./match.ts";
export * from "./recall.ts";
export * from "./translations.ts";
+159
View File
@@ -0,0 +1,159 @@
import { createMockTmdb, type MockTmdb } from "@umt/testkit";
import { afterEach, describe, expect, it } from "vitest";
import {
createTmdbClient,
isTmdbListPath,
type TmdbConfig,
tmdbChart,
tmdbDiscover,
tmdbDiscoverParams,
tmdbListItem,
tmdbSearchPage,
tmdbTrending,
} from "./index.ts";
// 发现页的 TMDB 列表接口:路径与参数映射(对照 MP tmdb_discover)、条目投影、列表类缓存 TTL。
let mock: MockTmdb | null = null;
afterEach(async () => {
await mock?.close();
mock = null;
});
async function setup(extra: Partial<TmdbConfig> = {}) {
mock = await createMockTmdb();
const tmdb = createTmdbClient({ baseUrl: `${mock.url}/3`, apiKey: "test-tmdb-key", retries: 0, ...extra });
return { mock, tmdb };
}
describe("TMDB 列表条目", () => {
it("trending 条目按 media_type 定类型,人物跳过,genre_ids 转中文名", () => {
expect(
tmdbListItem({
id: 1,
media_type: "tv",
name: "熊家餐馆",
original_name: "The Bear",
first_air_date: "2022-06-23",
genre_ids: [35, 18, 999],
vote_average: 8.2,
poster_path: "/b.jpg",
}),
).toMatchObject({
type: "tv",
id: "1",
title: "熊家餐馆",
originalTitle: "The Bear",
year: 2022,
genres: ["喜剧", "剧情"],
voteAverage: 8.2,
posterPath: "/b.jpg",
});
expect(tmdbListItem({ id: 2, media_type: "person", name: "X" })).toBeNull();
expect(tmdbListItem({ id: 3, title: "无评分", vote_average: 0 }, "movie")?.voteAverage).toBeNull();
});
});
describe("TMDB discover 参数映射", () => {
it("电影:风格 / 语言 / 地区 / 单年 / 评分;排序映射", () => {
expect(
tmdbDiscoverParams(
{ type: "movie", genre: "878", language: "en", region: "US", year: "2024", minRating: 7, sort: "rating" },
"zh-CN",
),
).toEqual({
language: "zh-CN",
include_adult: "false",
page: "1",
sort_by: "vote_average.desc",
with_genres: "878",
with_original_language: "en",
with_origin_country: "US",
"vote_average.gte": "7",
"vote_count.gte": "200",
primary_release_year: "2024",
});
});
it("剧集:年代展开成首播日期区间;按最新排序不列未播出的", () => {
const p = tmdbDiscoverParams({ type: "tv", year: "2010s", sort: "date", page: 3 }, "zh-CN");
expect(p).toMatchObject({
sort_by: "first_air_date.desc",
"first_air_date.gte": "2010-01-01",
"first_air_date.lte": "2019-12-31",
page: "3",
"vote_count.gte": "10",
});
const q = tmdbDiscoverParams({ type: "movie", sort: "date", today: "2026-10-06" }, "zh-CN");
expect(q).toMatchObject({ sort_by: "primary_release_date.desc", "primary_release_date.lte": "2026-10-06" });
expect(q.primary_release_year).toBeUndefined();
});
});
describe("TMDB 列表请求", () => {
it("流行趋势 / 榜单 / 筛选 / 搜索走对应路径与参数", async () => {
const { tmdb, mock } = await setup();
const tr = await tmdbTrending(tmdb, { mediaType: "all", window: "day", page: 1 });
expect(tr.items.length).toBeGreaterThan(3);
expect(new Set(tr.items.map((i) => i.type))).toEqual(new Set(["movie", "tv"]));
expect(tr.items[0]!.popularity).toBeGreaterThanOrEqual(tr.items[1]!.popularity!);
const pop = await tmdbChart(tmdb, { type: "tv", chart: "on_the_air" });
expect(pop.items.every((i) => i.type === "tv")).toBe(true);
const d = await tmdbDiscover(tmdb, { type: "movie", genre: "878" });
expect(d.items.map((i) => i.id).sort()).toEqual(["693134", "842675"]);
expect(d.items[0]!.genres).toContain("科幻");
const s = await tmdbSearchPage(tmdb, { type: "all", query: "奥本海默" });
expect(s.items.map((i) => `${i.type}:${i.id}`)).toEqual(["movie:872585"]);
const paths = mock.calls.map((r) => r.path);
expect(paths).toEqual(["/3/trending/all/day", "/3/tv/on_the_air", "/3/discover/movie", "/3/search/multi"]);
expect(mock.calls[2]!.query).toMatchObject({
with_genres: "878",
sort_by: "popularity.desc",
language: "zh-CN",
});
});
it("列表类请求缓存 6 小时(不按详情 30 天),空结果不超过它;失败不缓存", async () => {
const writes: { key: string; ttlMs: number }[] = [];
const store = new Map<string, unknown>();
const { tmdb, mock } = await setup({
cache: { ttlMs: 30 * 86_400_000, searchTtlMs: 7 * 86_400_000, emptyTtlMs: 86_400_000 },
sharedCache: {
get: async (k) => store.get(k) ?? null,
set: async (key, v, ttlMs) => {
writes.push({ key, ttlMs });
store.set(key, v);
},
},
});
await tmdbTrending(tmdb, { mediaType: "movie", window: "week" });
await tmdbTrending(tmdb, { mediaType: "movie", window: "week" });
await tmdbDiscover(tmdb, { type: "movie", genre: "10402" });
expect(mock.callsTo("/3/trending")).toHaveLength(1);
expect(writes.map((w) => [w.key.split("?")[0], w.ttlMs])).toEqual([
["/trending/movie/week", 6 * 3_600_000],
["/discover/movie", 6 * 3_600_000],
]);
// 鉴权失败:抛错,不写缓存
const bad = createTmdbClient({
baseUrl: `${mock.url}/3`,
apiKey: "wrong",
retries: 0,
sharedCache: { get: async () => null, set: async (key, _v, ttlMs) => void writes.push({ key, ttlMs }) },
});
await expect(tmdbChart(bad, { type: "movie", chart: "popular" })).rejects.toMatchObject({ kind: "auth" });
expect(writes).toHaveLength(2);
});
it("isTmdbListPath 只认列表路径", () => {
expect(isTmdbListPath("/trending/all/week")).toBe(true);
expect(isTmdbListPath("/discover/tv")).toBe(true);
expect(isTmdbListPath("/movie/now_playing")).toBe(true);
expect(isTmdbListPath("/movie/872585")).toBe(false);
expect(isTmdbListPath("/search/movie")).toBe(false);
});
});
+172
View File
@@ -0,0 +1,172 @@
/**
* TMDB 列表类接口(发现页)。
* 迁移来源:MoviePilot app/modules/themoviedb/tmdbapi.py discover_movies / discover_tvs / discover_trending(trending.all_week)、
* app/chain/recommend.py tmdb_movies / tmdb_tvs / tmdb_trending,与前端 views/discover/TheMovieDbView.vue 的筛选参数。
* 走 TmdbClient.getJson:同一套请求缓存(列表类 TTL 见 client.ts isTmdbListPath)、限流、重试;失败抛 TmdbError,不缓存。
* 调整:MP 只给 trending/all/week;这里日 / 周 × 全部 / 电影 / 剧集都给;电影 / 剧集榜单(热门、正在上映…)直接用 TMDB 的榜单接口。
*/
import { type TmdbDiscoverSort, tmdbGenres } from "@umt/contracts";
import type { MediaEntityType, TmdbClient } from "./client.ts";
export interface TmdbListItem {
type: MediaEntityType;
id: string;
title: string;
originalTitle: string | null;
originalLanguage: string | null;
year: number | null;
date: string | null;
overview: string | null;
posterPath: string | null;
backdropPath: string | null;
voteAverage: number | null;
voteCount: number | null;
popularity: number | null;
/** genre_ids 按字典转中文名 */
genres: string[];
}
export interface TmdbListPage {
page: number;
totalPages: number;
totalResults: number;
items: TmdbListItem[];
}
export type TmdbChart = "popular" | "now_playing" | "upcoming" | "top_rated" | "on_the_air" | "airing_today";
export interface TmdbDiscoverFilter {
type: MediaEntityType;
sort?: TmdbDiscoverSort;
/** genre id */
genre?: string;
/** with_original_language */
language?: string;
/** with_origin_country(ISO 3166-1) */
region?: string;
/** 单年 2024 或年代 2010s */
year?: string;
/** vote_average.gte */
minRating?: number;
page?: number;
/** 按日期排序时不列还没上映的(YYYY-MM-DD) */
today?: string;
}
type Json = Record<string, unknown>;
const s = (v: unknown): string | null => (typeof v === "string" && v !== "" ? v : null);
const n = (v: unknown): number | null => (typeof v === "number" && Number.isFinite(v) ? v : null);
/** 一条列表结果 → TmdbListItem;type 缺省取 media_type(trending / multi),人物等非影视条目返回 null */
export function tmdbListItem(raw: Json, type?: MediaEntityType | null): TmdbListItem | null {
const t = type ?? (raw.media_type === "movie" || raw.media_type === "tv" ? raw.media_type : null);
if (!t || raw.id === undefined || raw.id === null) return null;
const title = s(t === "movie" ? raw.title : raw.name) ?? s(t === "movie" ? raw.original_title : raw.original_name);
if (!title) return null;
const date = s(t === "movie" ? raw.release_date : raw.first_air_date);
const dict = tmdbGenres(t);
return {
type: t,
id: String(raw.id),
title,
originalTitle: s(t === "movie" ? raw.original_title : raw.original_name),
originalLanguage: s(raw.original_language),
year: date && /^\d{4}/.test(date) ? Number(date.slice(0, 4)) : null,
date,
overview: s(raw.overview),
posterPath: s(raw.poster_path),
backdropPath: s(raw.backdrop_path),
voteAverage: n(raw.vote_average) || null,
voteCount: n(raw.vote_count),
popularity: n(raw.popularity),
genres: (Array.isArray(raw.genre_ids) ? raw.genre_ids : []).flatMap((g) => dict[String(g)] ?? []),
};
}
function toPage(j: Json, type?: MediaEntityType | null): TmdbListPage {
const results = (Array.isArray(j.results) ? j.results : []) as Json[];
return {
page: n(j.page) ?? 1,
totalPages: n(j.total_pages) ?? 1,
totalResults: n(j.total_results) ?? results.length,
items: results.flatMap((r) => tmdbListItem(r, type) ?? []),
};
}
/** 流行趋势:/trending/{all|movie|tv}/{day|week} */
export async function tmdbTrending(
client: TmdbClient,
o: { mediaType: "all" | MediaEntityType; window: "day" | "week"; page?: number },
): Promise<TmdbListPage> {
const j = await client.getJson<Json>(`/trending/${o.mediaType}/${o.window}`, {
language: client.language,
page: String(o.page ?? 1),
});
return toPage(j, o.mediaType === "all" ? null : o.mediaType);
}
/** 电影 / 剧集榜单:/movie/{popular|now_playing|upcoming|top_rated}、/tv/{popular|on_the_air|airing_today|top_rated} */
export async function tmdbChart(
client: TmdbClient,
o: { type: MediaEntityType; chart: TmdbChart; page?: number },
): Promise<TmdbListPage> {
const j = await client.getJson<Json>(`/${o.type}/${o.chart}`, {
language: client.language,
page: String(o.page ?? 1),
});
return toPage(j, o.type);
}
/** 筛选条件 → /discover/{movie|tv} 参数(对照 MP tmdb_discover 的参数表;纯函数,测试用) */
export function tmdbDiscoverParams(f: TmdbDiscoverFilter, language: string): Record<string, string | undefined> {
const movie = f.type === "movie";
const dateKey = movie ? "primary_release_date" : "first_air_date";
const sort = f.sort ?? "popularity";
const p: Record<string, string | undefined> = {
language,
include_adult: "false",
page: String(f.page ?? 1),
sort_by: {
popularity: "popularity.desc",
date: `${dateKey}.desc`,
rating: "vote_average.desc",
votes: "vote_count.desc",
}[sort],
with_genres: f.genre,
with_original_language: f.language,
with_origin_country: f.region,
"vote_average.gte": f.minRating ? String(f.minRating) : undefined,
// 评分排序时要求足够的评分人数,否则全是一两票的 10 分(MP 缺省 vote_count 10)
"vote_count.gte": sort === "rating" ? "200" : f.minRating ? "50" : "10",
};
if (f.year) {
const decade = /^(\d{4})s$/.exec(f.year);
if (decade) {
const y = Number(decade[1]);
p[`${dateKey}.gte`] = `${y}-01-01`;
p[`${dateKey}.lte`] = `${y + 9}-12-31`;
} else p[movie ? "primary_release_year" : "first_air_date_year"] = f.year;
}
if (sort === "date" && f.today && !p[`${dateKey}.lte`]) p[`${dateKey}.lte`] = f.today;
return p;
}
export async function tmdbDiscover(client: TmdbClient, f: TmdbDiscoverFilter): Promise<TmdbListPage> {
const j = await client.getJson<Json>(`/discover/${f.type}`, tmdbDiscoverParams(f, client.language));
return toPage(j, f.type);
}
/** 关键词搜索(分页):全部走 /search/multi(过滤人物),电影 / 剧集走 /search/{type} */
export async function tmdbSearchPage(
client: TmdbClient,
o: { type: "all" | MediaEntityType; query: string; page?: number },
): Promise<TmdbListPage> {
const path = o.type === "all" ? "/search/multi" : `/search/${o.type}`;
const j = await client.getJson<Json>(path, {
query: o.query.trim(),
language: client.language,
include_adult: "false",
page: String(o.page ?? 1),
});
return toPage(j, o.type === "all" ? null : o.type);
}