feat(adapters): TMDB 列表与筛选、豆瓣榜单 / 标签 / 筛选 / 搜索、bgm 浏览与趋势，豆瓣 / bgm 请求缓存端口
- TMDB:流行趋势(日 / 周)、电影 / 剧集榜单、discover 参数映射、分页搜索;列表类请求 TTL 6 小时 - 豆瓣 / bgm:列表与搜索请求经 MetaRequestCache 与限流,失败(403 / 风控页)不缓存 - contracts:发现页条目、分页、查询校验与筛选字典;storage.cacheTtl 追加 meta* 字段 - testkit:mock TMDB 加列表接口 Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
27514f4985
commit
e46eb0e3c3
13 files changed
+1344
-91
No files matched your search
@@ -2,8 +2,12 @@
|
||||
* 辅助信息源(对照 MP modules/bangumi、modules/douban):只读公开接口,结果只用来"拿名称 + 年份去匹配 TMDB"与发现页展示,
|
||||
* 身份仍以 TMDB 为准。
|
||||
* - bgm.tv:官方 API(api.bgm.tv),条目 / 搜索 / 每日放送,不需要登录;要求带可识别的 User-Agent
|
||||
* - 豆瓣:移动版 rexxar 接口(m.douban.com,需要 Referer),条目详情与实时热门榜;搜索要登录,不用
|
||||
* - 豆瓣:移动版 rexxar 接口(m.douban.com,需要 Referer):条目详情、榜单(subject_collection)、筛选(movie|tv/recommend);
|
||||
* 桌面版 JSON(movie.douban.com/j/…):标签列表(search_subjects)、搜索联想(subject_suggest)。都不需要登录,HTTP 直连。
|
||||
* 发现页(2026-10-06):列表 / 搜索请求经 MetaRequestCache(服务端 PG 请求缓存,对照 MP DoubanApi.__invoke / Bangumi 的 @cached)
|
||||
* 与限流器;条目详情不走请求缓存(服务端已有条目级缓存 douban.subject / bangumi.subject,避免同一份数据存两遍)。
|
||||
*/
|
||||
import { load } from "cheerio";
|
||||
|
||||
export interface MetaSubject {
|
||||
source: "bangumi" | "douban";
|
||||
@@ -21,13 +25,36 @@ export interface MetaSubject {
|
||||
date: string | null;
|
||||
episodes: number | null;
|
||||
url: string;
|
||||
/** 风格 / 类型(豆瓣 card_subtitle 第三段;没有为空) */
|
||||
genres?: string[];
|
||||
}
|
||||
|
||||
/** 请求类别:决定缓存 TTL(搜索 / 列表);条目详情不走请求缓存 */
|
||||
export type MetaRequestKind = "search" | "list";
|
||||
|
||||
/**
|
||||
* 请求缓存端口(服务端用 PG kv_cache 实现,region douban.request / bangumi.request):
|
||||
* 命中直接返回;否则执行 load,成功的结果按类别 TTL 写入,empty(v) 为真时用空结果短 TTL;load 抛错(网络失败、403、风控页)不写。
|
||||
*/
|
||||
export interface MetaRequestCache {
|
||||
cached<T>(
|
||||
req: { source: "bangumi" | "douban"; key: string; kind: MetaRequestKind; empty: (v: T) => boolean },
|
||||
load: () => Promise<T>,
|
||||
): Promise<T>;
|
||||
}
|
||||
|
||||
export interface MetaClientOptions {
|
||||
fetch?: typeof globalThis.fetch;
|
||||
/** 缺省 15s */
|
||||
timeoutMs?: number;
|
||||
/** API 地址(bgm api.bgm.tv / 豆瓣 rexxar) */
|
||||
baseUrl?: string;
|
||||
/** 网页地址(bgm.tv / movie.douban.com) */
|
||||
webBaseUrl?: string;
|
||||
/** 请求缓存;不给就不缓存 */
|
||||
cache?: MetaRequestCache;
|
||||
/** 限流(每源最小间隔);只包住真正发出的网络请求,缓存命中不占额度 */
|
||||
limiter?: { run<T>(fn: () => Promise<T>): Promise<T> };
|
||||
}
|
||||
|
||||
export class MetaSourceError extends Error {
|
||||
@@ -57,28 +84,70 @@ const uniq = (xs: (string | null | undefined)[], not: string) => [
|
||||
...new Set(xs.filter((x): x is string => !!x && x !== not && x.length <= 120)),
|
||||
];
|
||||
|
||||
async function fetchOk(
|
||||
source: "bangumi" | "douban",
|
||||
url: string,
|
||||
o: MetaClientOptions,
|
||||
init: RequestInit,
|
||||
): Promise<Response> {
|
||||
const f = o.fetch ?? globalThis.fetch;
|
||||
const run = () => f(url, { ...init, signal: AbortSignal.timeout(o.timeoutMs ?? 15_000) });
|
||||
let res: Response;
|
||||
try {
|
||||
res = await (o.limiter ? o.limiter.run(run) : run());
|
||||
} catch (e) {
|
||||
throw new MetaSourceError(source, "http", `请求 ${new URL(url).host} 失败:${(e as Error).message}`);
|
||||
}
|
||||
if (res.status === 404) {
|
||||
await res.body?.cancel();
|
||||
throw new MetaSourceError(source, "not_found", "条目不存在");
|
||||
}
|
||||
if (!res.ok) {
|
||||
await res.body?.cancel();
|
||||
throw new MetaSourceError(source, "http", `${new URL(url).host} 返回 HTTP ${res.status}`);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
async function getJson(
|
||||
source: "bangumi" | "douban",
|
||||
url: string,
|
||||
o: MetaClientOptions,
|
||||
init: RequestInit = {},
|
||||
): Promise<unknown> {
|
||||
const f = o.fetch ?? globalThis.fetch;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await f(url, { ...init, signal: AbortSignal.timeout(o.timeoutMs ?? 15_000) });
|
||||
} catch (e) {
|
||||
throw new MetaSourceError(source, "http", `请求 ${new URL(url).host} 失败:${(e as Error).message}`);
|
||||
}
|
||||
if (res.status === 404) throw new MetaSourceError(source, "not_found", "条目不存在");
|
||||
if (!res.ok) throw new MetaSourceError(source, "http", `${new URL(url).host} 返回 HTTP ${res.status}`);
|
||||
const res = await fetchOk(source, url, o, init);
|
||||
try {
|
||||
return await res.json();
|
||||
} catch {
|
||||
// 风控 / 验证页是 HTML:按解析失败抛出,不缓存
|
||||
throw new MetaSourceError(source, "parse", `${new URL(url).host} 返回的不是 JSON`);
|
||||
}
|
||||
}
|
||||
|
||||
/** 请求缓存键:方法 + 参数排序后的地址 + 请求体 */
|
||||
export function metaRequestKey(method: string, url: string, body?: string | null): string {
|
||||
const u = new URL(url);
|
||||
const qs = new URLSearchParams([...u.searchParams].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
|
||||
const q = qs.toString();
|
||||
return `${method.toUpperCase()} ${u.origin}${u.pathname}${q ? `?${q}` : ""}${body ? ` ${body}` : ""}`;
|
||||
}
|
||||
|
||||
function viaCache<T>(
|
||||
o: MetaClientOptions,
|
||||
source: "bangumi" | "douban",
|
||||
kind: MetaRequestKind,
|
||||
key: string,
|
||||
empty: (v: T) => boolean,
|
||||
load: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
return o.cache ? o.cache.cached({ source, key, kind, empty }, load) : load();
|
||||
}
|
||||
|
||||
const emptyRows = (field: string) => (v: unknown) => {
|
||||
const rows = isObj(v) ? v[field] : v;
|
||||
return Array.isArray(rows) && rows.length === 0;
|
||||
};
|
||||
|
||||
// ── bgm.tv ──
|
||||
|
||||
export const BANGUMI_API = "https://api.bgm.tv";
|
||||
@@ -114,9 +183,71 @@ export function bangumiSubject(r: Raw): MetaSubject | null {
|
||||
};
|
||||
}
|
||||
|
||||
export const BANGUMI_WEB = "https://bgm.tv";
|
||||
|
||||
/** bgm.tv 网页日期「2023年9月29日」/「2023-09-29」→ YYYY-MM-DD */
|
||||
function bgmDate(text: string): string | null {
|
||||
const m = /((?:19|20)\d{2})[年-](\d{1,2})[月-](\d{1,2})/.exec(text);
|
||||
return m ? `${m[1]}-${m[2]!.padStart(2, "0")}-${m[3]!.padStart(2, "0")}` : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* bgm.tv 动画浏览页(/anime/browser?sort=trends,服务端渲染 HTML)→ 条目列表。外部 HTML 不可信:
|
||||
* 服务端经沙箱任务 bangumi.trends 调用(ADR-005),这里只是纯函数。
|
||||
*/
|
||||
export function parseBangumiBrowser(html: string, web = BANGUMI_WEB): { items: MetaSubject[]; hasMore: boolean } {
|
||||
const $ = load(html);
|
||||
const items: MetaSubject[] = [];
|
||||
$("#browserItemList > li.item").each((_, el) => {
|
||||
const li = $(el);
|
||||
const id =
|
||||
/(\d+)/.exec(li.attr("id") ?? "")?.[1] ?? /\/subject\/(\d+)/.exec(li.find("h3 a").attr("href") ?? "")?.[1];
|
||||
const title = li.find("h3 a.l").first().text().trim();
|
||||
if (!id || !title) return;
|
||||
const original = li.find("h3 small.grey").first().text().trim();
|
||||
const info = li.find("p.info").first().text().replace(/\s+/g, " ").trim();
|
||||
const date = bgmDate(info);
|
||||
const eps = numOf(/(\d+)\s*话/.exec(info)?.[1]);
|
||||
const rating = numOf(li.find("p.rateInfo small.fade").first().text().trim());
|
||||
let cover = str(li.find("img.cover").first().attr("src"));
|
||||
if (cover?.startsWith("//")) cover = `https:${cover}`;
|
||||
items.push({
|
||||
source: "bangumi",
|
||||
id,
|
||||
title,
|
||||
aliases: original && original !== title ? [original] : [],
|
||||
year: yearOf(date ?? info),
|
||||
kind: /剧场|电影/.test(info) ? "movie" : "tv",
|
||||
cover: https(cover),
|
||||
rating: rating || null,
|
||||
date,
|
||||
episodes: eps || null,
|
||||
url: `${web}/subject/${id}`,
|
||||
});
|
||||
});
|
||||
const hasMore = $("#multipage a.p")
|
||||
.toArray()
|
||||
.some((a) => $(a).text().includes("››") || $(a).text().includes("\u203a\u203a"));
|
||||
return { items, hasMore };
|
||||
}
|
||||
|
||||
export function createBangumiClient(o: MetaClientOptions = {}) {
|
||||
const base = (o.baseUrl ?? BANGUMI_API).replace(/\/+$/, "");
|
||||
const web = (o.webBaseUrl ?? BANGUMI_WEB).replace(/\/+$/, "");
|
||||
const headers = { "User-Agent": BGM_UA, Accept: "application/json" };
|
||||
const searchPage = async (keyword: string, limit = 10, offset = 0) => {
|
||||
const url = `${base}/v0/search/subjects?limit=${limit}&offset=${offset}`;
|
||||
const body = JSON.stringify({ keyword, filter: { type: [2] } });
|
||||
const r = await viaCache(o, "bangumi", "search", metaRequestKey("POST", url, body), emptyRows("data"), () =>
|
||||
getJson("bangumi", url, o, {
|
||||
method: "POST",
|
||||
headers: { ...headers, "Content-Type": "application/json" },
|
||||
body,
|
||||
}),
|
||||
);
|
||||
const rows = isObj(r) && Array.isArray(r.data) ? (r.data as Raw[]) : [];
|
||||
return { items: rows.flatMap((x) => bangumiSubject(x) ?? []), total: isObj(r) ? (numOf(r.total) ?? 0) : 0 };
|
||||
};
|
||||
return {
|
||||
async subject(id: string): Promise<MetaSubject> {
|
||||
if (!/^\d+$/.test(id)) throw new MetaSourceError("bangumi", "not_found", "bgm 条目 ID 非法");
|
||||
@@ -127,17 +258,16 @@ export function createBangumiClient(o: MetaClientOptions = {}) {
|
||||
},
|
||||
/** 动画条目搜索(type 2) */
|
||||
async search(keyword: string, limit = 10): Promise<MetaSubject[]> {
|
||||
const r = await getJson("bangumi", `${base}/v0/search/subjects?limit=${limit}`, o, {
|
||||
method: "POST",
|
||||
headers: { ...headers, "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ keyword, filter: { type: [2] } }),
|
||||
});
|
||||
const rows = isObj(r) && Array.isArray(r.data) ? (r.data as Raw[]) : [];
|
||||
return rows.flatMap((x) => bangumiSubject(x) ?? []);
|
||||
return (await searchPage(keyword, limit)).items;
|
||||
},
|
||||
/** 动画条目搜索,带分页与总数(发现页) */
|
||||
searchPage,
|
||||
/** 每日放送:周一到周日(weekday 1-7) */
|
||||
async calendar(): Promise<{ weekday: number; label: string; items: MetaSubject[] }[]> {
|
||||
const r = await getJson("bangumi", `${base}/calendar`, o, { headers });
|
||||
const url = `${base}/calendar`;
|
||||
const r = await viaCache(o, "bangumi", "list", metaRequestKey("GET", url), emptyRows(""), () =>
|
||||
getJson("bangumi", url, o, { headers }),
|
||||
);
|
||||
if (!Array.isArray(r)) throw new MetaSourceError("bangumi", "parse", "bgm 每日放送格式不对");
|
||||
return (r as Raw[]).map((d) => {
|
||||
const wd = isObj(d.weekday) ? d.weekday : {};
|
||||
@@ -148,6 +278,34 @@ export function createBangumiClient(o: MetaClientOptions = {}) {
|
||||
};
|
||||
});
|
||||
},
|
||||
/** 动画浏览(对照 MP bangumi discover:v0/subjects?type=2&sort=rank|date&year=) */
|
||||
async browse(q: { sort: "rank" | "date"; year?: string; limit: number; offset: number }) {
|
||||
const qs = new URLSearchParams({ type: "2", sort: q.sort, limit: String(q.limit), offset: String(q.offset) });
|
||||
if (q.year) qs.set("year", q.year);
|
||||
const url = `${base}/v0/subjects?${qs}`;
|
||||
const r = await viaCache(o, "bangumi", "list", metaRequestKey("GET", url), emptyRows("data"), () =>
|
||||
getJson("bangumi", url, o, { headers }),
|
||||
);
|
||||
if (!isObj(r) || !Array.isArray(r.data)) throw new MetaSourceError("bangumi", "parse", "bgm 浏览结果格式不对");
|
||||
return { items: (r.data as Raw[]).flatMap((x) => bangumiSubject(x) ?? []), total: numOf(r.total) ?? 0 };
|
||||
},
|
||||
/** 热门趋势网页(无 API):返回原始 HTML,由调用方在沙箱里用 parseBangumiBrowser 解析 */
|
||||
async trendsHtml(page = 1): Promise<string> {
|
||||
const url = `${web}/anime/browser?sort=trends&page=${page}`;
|
||||
return viaCache(
|
||||
o,
|
||||
"bangumi",
|
||||
"list",
|
||||
metaRequestKey("GET", url),
|
||||
() => false,
|
||||
async () => {
|
||||
const res = await fetchOk("bangumi", url, o, { headers: { "User-Agent": BGM_UA, Accept: "text/html" } });
|
||||
const text = await res.text();
|
||||
if (!text.includes("browserItemList")) throw new MetaSourceError("bangumi", "parse", "bgm 浏览页格式不对");
|
||||
return text;
|
||||
},
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
export type BangumiClient = ReturnType<typeof createBangumiClient>;
|
||||
@@ -172,6 +330,11 @@ export function doubanSubject(r: Raw): MetaSubject | null {
|
||||
const pub = Array.isArray(r.pubdate) ? str(r.pubdate[0]) : null;
|
||||
const date = /\d{4}-\d{2}-\d{2}/.exec(pub ?? "")?.[0] ?? null;
|
||||
const type = str(r.type) ?? str(r.subtype);
|
||||
// card_subtitle:「2024 / 美国 / 剧情 科幻 / 导演 / 主演」
|
||||
const card = (str(r.card_subtitle) ?? "").split(" / ");
|
||||
const genres =
|
||||
card.length >= 3 && /^\d{4}$/.test(card[0]!.trim()) ? card[2]!.trim().split(/\s+/).filter(Boolean) : [];
|
||||
const cover = str(pic.large) ?? str(pic.normal) ?? str(r.cover_url);
|
||||
return {
|
||||
source: "douban",
|
||||
id,
|
||||
@@ -179,11 +342,61 @@ export function doubanSubject(r: Raw): MetaSubject | null {
|
||||
aliases: uniq([str(r.original_title), ...(Array.isArray(r.aka) ? r.aka.map(str) : [])], title).slice(0, 10),
|
||||
year: yearOf(r.year) ?? yearOf(date),
|
||||
kind: type === "tv" || r.is_tv === true ? "tv" : type === "movie" ? "movie" : null,
|
||||
cover: str(pic.large) ?? str(pic.normal),
|
||||
cover: placeholderCover(cover) ? null : cover,
|
||||
rating: rating || null,
|
||||
date,
|
||||
episodes: numOf(r.episodes_count) || null,
|
||||
url: `https://movie.douban.com/subject/${id}/`,
|
||||
genres: genres.slice(0, 4),
|
||||
};
|
||||
}
|
||||
|
||||
/** 豆瓣占位海报(对照 MP _project_discover 的 invalid_posters) */
|
||||
const placeholderCover = (u: string | null) =>
|
||||
!!u && ["movie_large.jpg", "tv_normal.png", "tv_normal.jpg", "tv_large.jpg"].some((m) => u.includes(m));
|
||||
|
||||
/** 桌面版 search_subjects 条目({id, title, rate, cover, url, episodes_info}) → MetaSubject */
|
||||
export function doubanTagSubject(r: Raw, kind: "movie" | "tv"): MetaSubject | null {
|
||||
const id = str(r.id);
|
||||
const title = str(r.title);
|
||||
if (!id || !/^\d+$/.test(id) || !title) return null;
|
||||
const cover = str(r.cover);
|
||||
return {
|
||||
source: "douban",
|
||||
id,
|
||||
title,
|
||||
aliases: [],
|
||||
year: null,
|
||||
kind,
|
||||
cover: placeholderCover(cover) ? null : cover,
|
||||
rating: numOf(r.rate) || null,
|
||||
date: null,
|
||||
episodes: null,
|
||||
url: `https://movie.douban.com/subject/${id}/`,
|
||||
genres: [],
|
||||
};
|
||||
}
|
||||
|
||||
/** 搜索联想 subject_suggest 条目({id, title, sub_title, img, year, episode, type}) → MetaSubject;人物等返回 null */
|
||||
export function doubanSuggestSubject(r: Raw): MetaSubject | null {
|
||||
if (str(r.type) !== "movie") return null;
|
||||
const id = str(r.id);
|
||||
const title = str(r.title);
|
||||
if (!id || !/^\d+$/.test(id) || !title) return null;
|
||||
const sub = str(r.sub_title);
|
||||
return {
|
||||
source: "douban",
|
||||
id,
|
||||
title,
|
||||
aliases: sub && sub !== title ? [sub] : [],
|
||||
year: yearOf(r.year),
|
||||
kind: str(r.episode) ? "tv" : "movie",
|
||||
cover: str(r.img),
|
||||
rating: null,
|
||||
date: null,
|
||||
episodes: numOf(r.episode) || null,
|
||||
url: `https://movie.douban.com/subject/${id}/`,
|
||||
genres: [],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -192,10 +405,28 @@ export const DOUBAN_HOT = {
|
||||
tv: "tv_real_time_hotest",
|
||||
} as const;
|
||||
|
||||
export const DOUBAN_WEB = "https://movie.douban.com";
|
||||
const DOUBAN_WEB_HEADERS = {
|
||||
Referer: "https://movie.douban.com/",
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
|
||||
Accept: "application/json",
|
||||
};
|
||||
|
||||
export function createDoubanClient(o: MetaClientOptions = {}) {
|
||||
const base = (o.baseUrl ?? DOUBAN_API).replace(/\/+$/, "");
|
||||
const web = (o.webBaseUrl ?? DOUBAN_WEB).replace(/\/+$/, "");
|
||||
const get = (path: string) => getJson("douban", `${base}${path}`, o, { headers: DOUBAN_HEADERS });
|
||||
return {
|
||||
const cachedGet = (kind: MetaRequestKind, url: string, field: string, headers: Record<string, string>) =>
|
||||
viaCache(o, "douban", kind, metaRequestKey("GET", url), emptyRows(field), () =>
|
||||
getJson("douban", url, o, { headers }),
|
||||
);
|
||||
const listOf = (r: unknown, field: string, what: string) => {
|
||||
const rows = isObj(r) && Array.isArray(r[field]) ? (r[field] as Raw[]) : null;
|
||||
if (!rows) throw new MetaSourceError("douban", "parse", `豆瓣${what}格式不对`);
|
||||
return { rows, total: isObj(r) ? numOf(r.total) : null };
|
||||
};
|
||||
const client = {
|
||||
/** 条目详情:类型未知时先按电影查,不存在再按剧集查 */
|
||||
async subject(id: string, kind?: "movie" | "tv" | null): Promise<MetaSubject> {
|
||||
if (!/^\d+$/.test(id)) throw new MetaSourceError("douban", "not_found", "豆瓣条目 ID 非法");
|
||||
@@ -213,13 +444,60 @@ export function createDoubanClient(o: MetaClientOptions = {}) {
|
||||
}
|
||||
throw last instanceof MetaSourceError ? last : new MetaSourceError("douban", "not_found", "豆瓣条目不存在");
|
||||
},
|
||||
/** 榜单(rexxar subject_collection):实时热门、正在热映、Top250、各类剧集榜… */
|
||||
async collection(name: string, start = 0, count = 20): Promise<{ items: MetaSubject[]; total: number | null }> {
|
||||
if (!/^[a-z0-9_]+$/.test(name)) throw new MetaSourceError("douban", "not_found", "豆瓣榜单名非法");
|
||||
const url = `${base}/subject_collection/${name}/items?start=${start}&count=${count}`;
|
||||
const { rows, total } = listOf(
|
||||
await cachedGet("list", url, "subject_collection_items", DOUBAN_HEADERS),
|
||||
"subject_collection_items",
|
||||
"榜单",
|
||||
);
|
||||
return { items: rows.flatMap((x) => doubanSubject(x) ?? []), total };
|
||||
},
|
||||
/** 实时热门榜(电影 / 剧集) */
|
||||
async hot(kind: "movie" | "tv", count = 30): Promise<MetaSubject[]> {
|
||||
const r = await get(`/subject_collection/${DOUBAN_HOT[kind]}/items?start=0&count=${count}`);
|
||||
const rows = isObj(r) && Array.isArray(r.subject_collection_items) ? (r.subject_collection_items as Raw[]) : null;
|
||||
if (!rows) throw new MetaSourceError("douban", "parse", "豆瓣榜单格式不对");
|
||||
return rows.flatMap((x) => doubanSubject(x) ?? []);
|
||||
return (await client.collection(DOUBAN_HOT[kind], 0, count)).items;
|
||||
},
|
||||
/** 标签列表(桌面版 j/search_subjects,tag 如 热门 / 最新 / 豆瓣高分 / 美剧) */
|
||||
async tagSubjects(kind: "movie" | "tv", tag: string, start = 0, count = 20): Promise<MetaSubject[]> {
|
||||
const qs = new URLSearchParams({
|
||||
type: kind,
|
||||
tag,
|
||||
sort: "recommend",
|
||||
page_limit: String(count),
|
||||
page_start: String(start),
|
||||
});
|
||||
const url = `${web}/j/search_subjects?${qs}`;
|
||||
const { rows } = listOf(await cachedGet("list", url, "subjects", DOUBAN_WEB_HEADERS), "subjects", "标签列表");
|
||||
return rows.flatMap((x) => doubanTagSubject(x, kind) ?? []);
|
||||
},
|
||||
/** 筛选(rexxar movie|tv/recommend,对照 MP douban_discover:sort U/R/T/S,tags 为 风格,地区,年代) */
|
||||
async explore(
|
||||
kind: "movie" | "tv",
|
||||
q: { sort: string; tags: string[] },
|
||||
start = 0,
|
||||
count = 20,
|
||||
): Promise<{ items: MetaSubject[]; total: number | null }> {
|
||||
const qs = new URLSearchParams({
|
||||
refresh: "0",
|
||||
start: String(start),
|
||||
count: String(count),
|
||||
sort: q.sort,
|
||||
tags: q.tags.join(","),
|
||||
});
|
||||
const url = `${base}/${kind}/recommend?${qs}`;
|
||||
const { rows, total } = listOf(await cachedGet("list", url, "items", DOUBAN_HEADERS), "items", "筛选结果");
|
||||
return { items: rows.flatMap((x) => doubanSubject(x) ?? []), total };
|
||||
},
|
||||
/** 搜索联想(桌面版 j/subject_suggest,不分页,最多约 10 条) */
|
||||
async suggest(q: string): Promise<MetaSubject[]> {
|
||||
const url = `${web}/j/subject_suggest?${new URLSearchParams({ q: q.trim() })}`;
|
||||
const r = await cachedGet("search", url, "", DOUBAN_WEB_HEADERS);
|
||||
if (!Array.isArray(r)) throw new MetaSourceError("douban", "parse", "豆瓣搜索结果格式不对");
|
||||
return (r as Raw[]).flatMap((x) => doubanSuggestSubject(x) ?? []);
|
||||
},
|
||||
};
|
||||
return client;
|
||||
}
|
||||
export type DoubanClient = ReturnType<typeof createDoubanClient>;
|
||||
@@ -0,0 +1,228 @@
|
||||
import { createMockFetch } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
createBangumiClient,
|
||||
createDoubanClient,
|
||||
doubanSubject,
|
||||
doubanSuggestSubject,
|
||||
doubanTagSubject,
|
||||
type MetaRequestCache,
|
||||
metaRequestKey,
|
||||
parseBangumiBrowser,
|
||||
} from "./index.ts";
|
||||
|
||||
// 发现页的豆瓣 / bgm 列表接口:字段映射、请求缓存端口(键、类别、空结果、失败不缓存)、限流只包网络请求。
|
||||
// 字段形状取自 2026-10-06 协调者实测的返回(删减),不打真实站点。
|
||||
|
||||
const DOUBAN_ITEM = {
|
||||
id: "35267208",
|
||||
title: "流浪地球2",
|
||||
type: "movie",
|
||||
year: "2023",
|
||||
rating: { value: 8.3 },
|
||||
pic: { large: "https://img1.doubanio.com/view/photo/m_ratio_poster/public/p2885955777.jpg" },
|
||||
card_subtitle: "2023 / 中国大陆 / 科幻 冒险 灾难 / 郭帆 / 吴京 刘德华",
|
||||
};
|
||||
|
||||
const BGM_HTML = `<html><body>
|
||||
<ul id="browserItemList" class="browserFull">
|
||||
<li id="item_400602" class="item odd clearit">
|
||||
<a href="/subject/400602" class="subjectCover cover ll"><span class="image"><img src="//lain.bgm.tv/r/400/pic/cover/c/13/c5/400602_ZI8Y9.jpg" class="cover" /></span></a>
|
||||
<div class="inner">
|
||||
<h3><span class="ico_subject_type subject_type_2 ll"></span><a href="/subject/400602" class="l">葬送的芙莉莲</a> <small class="grey">葬送のフリーレン</small></h3>
|
||||
<p class="info tip">
|
||||
28话 / 2023年9月29日 / 斎藤圭一郎 / 山田鐘人
|
||||
</p>
|
||||
<p class="rateInfo"><span class="starstop-s"><span class="starlight stars9"></span></span> <small class="fade">8.9</small> <span class="tip_j">(25627人评分)</span></p>
|
||||
</div>
|
||||
</li>
|
||||
<li id="item_9" class="item even clearit">
|
||||
<div class="inner"><h3><a href="/subject/9" class="l">剧场版 某作品</a></h3><p class="info tip">剧场版 / 2024-07-01</p></div>
|
||||
</li>
|
||||
</ul>
|
||||
<div id="multipage"><div class="page_inner"><strong class="p_cur">1</strong><a href="?sort=trends&page=2" class="p">2</a><a href="?sort=trends&page=2" class="p">››</a></div></div>
|
||||
</body></html>`;
|
||||
|
||||
function memoryCache() {
|
||||
const store = new Map<string, unknown>();
|
||||
const writes: { key: string; kind: string; empty: boolean }[] = [];
|
||||
const cache: MetaRequestCache = {
|
||||
async cached(req, load) {
|
||||
if (store.has(req.key)) return store.get(req.key) as never;
|
||||
const v = await load();
|
||||
writes.push({ key: req.key, kind: req.kind, empty: req.empty(v) });
|
||||
store.set(req.key, v);
|
||||
return v;
|
||||
},
|
||||
};
|
||||
return { cache, writes };
|
||||
}
|
||||
|
||||
describe("豆瓣列表映射", () => {
|
||||
it("rexxar 条目:card_subtitle 第三段是风格;占位海报置空", () => {
|
||||
expect(doubanSubject(DOUBAN_ITEM)).toMatchObject({
|
||||
id: "35267208",
|
||||
year: 2023,
|
||||
kind: "movie",
|
||||
rating: 8.3,
|
||||
genres: ["科幻", "冒险", "灾难"],
|
||||
});
|
||||
expect(
|
||||
doubanSubject({
|
||||
...DOUBAN_ITEM,
|
||||
pic: { large: "https://img3.doubanio.com/f/movie/30c6263b/pics/movie/movie_large.jpg" },
|
||||
})?.cover,
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("标签列表与搜索联想", () => {
|
||||
expect(
|
||||
doubanTagSubject({ id: "1", title: "甲", rate: "7.9", cover: "https://x/p.jpg", episodes_info: "" }, "tv"),
|
||||
).toMatchObject({ id: "1", kind: "tv", rating: 7.9, year: null });
|
||||
expect(doubanTagSubject({ id: "2", title: "乙", rate: "" }, "movie")?.rating).toBeNull();
|
||||
expect(
|
||||
doubanSuggestSubject({
|
||||
id: "3",
|
||||
title: "三体",
|
||||
sub_title: "Three-Body",
|
||||
year: "2023",
|
||||
episode: "30",
|
||||
type: "movie",
|
||||
}),
|
||||
).toMatchObject({ kind: "tv", year: 2023, aliases: ["Three-Body"], episodes: 30 });
|
||||
expect(doubanSuggestSubject({ id: "4", title: "某演员", type: "celebrity" })).toBeNull();
|
||||
});
|
||||
|
||||
it("榜单 / 标签 / 筛选 / 搜索的请求地址与头", async () => {
|
||||
const m = createMockFetch({
|
||||
"GET /rexxar/api/v2/subject_collection/movie_top250/items": {
|
||||
body: { subject_collection_items: [DOUBAN_ITEM], total: 250 },
|
||||
},
|
||||
"GET /j/search_subjects": { body: { subjects: [{ id: "1", title: "甲", rate: "8.0", cover: "c" }] } },
|
||||
"GET /rexxar/api/v2/tv/recommend": { body: { items: [{ ...DOUBAN_ITEM, type: "tv" }], total: 1 } },
|
||||
"GET /j/subject_suggest": { body: [{ id: "3", title: "三体", year: "2023", episode: "30", type: "movie" }] },
|
||||
});
|
||||
const d = createDoubanClient({ fetch: m.fetch });
|
||||
expect(await d.collection("movie_top250", 20, 20)).toMatchObject({ total: 250, items: [{ id: "35267208" }] });
|
||||
expect((await d.tagSubjects("movie", "豆瓣高分", 40, 20))[0]).toMatchObject({ id: "1", kind: "movie" });
|
||||
expect((await d.explore("tv", { sort: "S", tags: ["悬疑", "美国"] }, 0, 20)).items[0]!.kind).toBe("tv");
|
||||
expect((await d.suggest(" 三体 "))[0]!.title).toBe("三体");
|
||||
const [col, tag, exp, sug] = m.requests;
|
||||
expect(col!.url.search).toBe("?start=20&count=20");
|
||||
expect(col!.headers.referer).toBe("https://m.douban.com/");
|
||||
expect(tag!.url.host).toBe("movie.douban.com");
|
||||
expect(Object.fromEntries(tag!.url.searchParams)).toEqual({
|
||||
type: "movie",
|
||||
tag: "豆瓣高分",
|
||||
sort: "recommend",
|
||||
page_limit: "20",
|
||||
page_start: "40",
|
||||
});
|
||||
expect(Object.fromEntries(exp!.url.searchParams)).toMatchObject({ sort: "S", tags: "悬疑,美国", start: "0" });
|
||||
expect(sug!.url.searchParams.get("q")).toBe("三体");
|
||||
await expect(d.collection("../x")).rejects.toThrow("榜单名非法");
|
||||
});
|
||||
});
|
||||
|
||||
describe("bgm 列表", () => {
|
||||
it("浏览页 HTML:标题 / 原名 / 日期 / 话数 / 评分 / 封面,有下一页", () => {
|
||||
const r = parseBangumiBrowser(BGM_HTML);
|
||||
expect(r.hasMore).toBe(true);
|
||||
expect(r.items).toEqual([
|
||||
expect.objectContaining({
|
||||
source: "bangumi",
|
||||
id: "400602",
|
||||
title: "葬送的芙莉莲",
|
||||
aliases: ["葬送のフリーレン"],
|
||||
year: 2023,
|
||||
date: "2023-09-29",
|
||||
episodes: 28,
|
||||
rating: 8.9,
|
||||
kind: "tv",
|
||||
cover: "https://lain.bgm.tv/r/400/pic/cover/c/13/c5/400602_ZI8Y9.jpg",
|
||||
url: "https://bgm.tv/subject/400602",
|
||||
}),
|
||||
expect.objectContaining({ id: "9", kind: "movie", date: "2024-07-01", cover: null, rating: null }),
|
||||
]);
|
||||
expect(parseBangumiBrowser("<ul id='browserItemList'></ul>")).toEqual({ items: [], hasMore: false });
|
||||
});
|
||||
|
||||
it("浏览 / 搜索 / 趋势页的请求", async () => {
|
||||
const m = createMockFetch({
|
||||
"GET https://api.bgm.tv/v0/subjects": {
|
||||
body: { data: [{ id: 1, name: "a", name_cn: "甲", date: "2024-01-01" }], total: 100, limit: 20, offset: 20 },
|
||||
},
|
||||
"POST https://api.bgm.tv/v0/search/subjects": { body: { data: [{ id: 2, name: "b" }], total: 1 } },
|
||||
"GET https://bgm.tv/anime/browser": { body: BGM_HTML },
|
||||
});
|
||||
const b = createBangumiClient({ fetch: m.fetch });
|
||||
expect(await b.browse({ sort: "rank", year: "2024", limit: 20, offset: 20 })).toMatchObject({ total: 100 });
|
||||
expect(await b.searchPage("芙莉莲", 20, 40)).toMatchObject({ total: 1, items: [{ id: "2" }] });
|
||||
expect(await b.trendsHtml(2)).toContain("browserItemList");
|
||||
expect(Object.fromEntries(m.requests[0]!.url.searchParams)).toEqual({
|
||||
type: "2",
|
||||
sort: "rank",
|
||||
year: "2024",
|
||||
limit: "20",
|
||||
offset: "20",
|
||||
});
|
||||
expect(m.requests[1]!.url.search).toBe("?limit=20&offset=40");
|
||||
expect(JSON.parse(m.requests[1]!.body!)).toEqual({ keyword: "芙莉莲", filter: { type: [2] } });
|
||||
expect(m.requests[2]!.url.search).toBe("?sort=trends&page=2");
|
||||
expect(m.requests[2]!.headers["user-agent"]).toContain("uhd-media-tool");
|
||||
});
|
||||
});
|
||||
|
||||
describe("请求缓存与限流", () => {
|
||||
it("键含方法 / 排序后的参数 / 请求体;按类别写入,空结果标记;命中不再请求、不占限流", async () => {
|
||||
expect(metaRequestKey("get", "https://a.b/x?z=1&a=2")).toBe("GET https://a.b/x?a=2&z=1");
|
||||
expect(metaRequestKey("POST", "https://a.b/s", '{"k":1}')).toBe('POST https://a.b/s {"k":1}');
|
||||
const m = createMockFetch({
|
||||
"GET /rexxar/api/v2/subject_collection/tv_hot/items": { body: { subject_collection_items: [], total: 0 } },
|
||||
"GET /j/subject_suggest": { body: [{ id: "3", title: "三体", type: "movie" }] },
|
||||
"POST https://api.bgm.tv/v0/search/subjects": { body: { data: [{ id: 2, name: "b" }], total: 1 } },
|
||||
});
|
||||
const { cache, writes } = memoryCache();
|
||||
let limited = 0;
|
||||
const limiter = {
|
||||
run<T>(fn: () => Promise<T>) {
|
||||
limited++;
|
||||
return fn();
|
||||
},
|
||||
};
|
||||
const d = createDoubanClient({ fetch: m.fetch, cache, limiter });
|
||||
const b = createBangumiClient({ fetch: m.fetch, cache, limiter });
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await d.collection("tv_hot");
|
||||
await d.suggest("三体");
|
||||
await b.search("芙莉莲", 5);
|
||||
}
|
||||
expect(m.requests).toHaveLength(3);
|
||||
expect(limited).toBe(3);
|
||||
expect(writes.map((w) => [w.kind, w.empty])).toEqual([
|
||||
["list", true],
|
||||
["search", false],
|
||||
["search", false],
|
||||
]);
|
||||
expect(writes[0]!.key).toBe(
|
||||
"GET https://m.douban.com/rexxar/api/v2/subject_collection/tv_hot/items?count=20&start=0",
|
||||
);
|
||||
});
|
||||
|
||||
it("失败(403 / 风控页不是 JSON)抛错不写缓存;条目详情不走请求缓存", async () => {
|
||||
const m = createMockFetch({
|
||||
"GET /rexxar/api/v2/subject_collection/tv_hot/items": { status: 403, body: "forbidden" },
|
||||
"GET /j/subject_suggest": { body: "<html>请登录</html>" },
|
||||
"GET /rexxar/api/v2/movie/35267208": { body: DOUBAN_ITEM },
|
||||
"GET https://bgm.tv/anime/browser": { body: "<html>验证</html>" },
|
||||
});
|
||||
const { cache, writes } = memoryCache();
|
||||
const d = createDoubanClient({ fetch: m.fetch, cache });
|
||||
const b = createBangumiClient({ fetch: m.fetch, cache });
|
||||
await expect(d.collection("tv_hot")).rejects.toMatchObject({ kind: "http" });
|
||||
await expect(d.suggest("x")).rejects.toMatchObject({ kind: "parse" });
|
||||
await expect(b.trendsHtml()).rejects.toMatchObject({ kind: "parse" });
|
||||
expect((await d.subject("35267208")).title).toBe("流浪地球2");
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user