feat(sites): structure release detail pages and rework the detail drawer (DETAIL-1)
Adapters (pure functions + tests): - detail-description: strip HTML/BBCode residue, drop site template blocks and ad/signature lines (configurable rules), parse PT info blocks (◎ / 【】 / 『』 / 「键:」) into infoBlock and body paragraphs - mediainfo: MediaInfo and BDInfo (quick summary + full report) into a summary of container/duration/bitrate/size, video (HDR/DV), audio and subtitle tracks - file-tree: file list into a flattened directory tree with sizes and main/sample/subtitle/extra roles - formatDetailPage combines them; the server computes it when reading the cached detail page, so existing caches benefit without a refetch Contracts: SiteReleasePage gains infoBlock, paragraphs, media, fileTree. Web: drawer now reads top-down: meta row (promo/H&R, size, peers, upload time, IMDb/TMDB/豆瓣 tags) → 资料 → 简介 (collapsible) → 技术规格 (raw folded) → 截图 grid with keyboard lightbox → 文件 tree; inventory overlay moved into the fixed footer next to 下载. Fixtures: 6 desensitized samples from the local cache (site.detail and RSS descriptions) plus synthetic multi-track MediaInfo, BDInfo, bracket blocks and directory file lists. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
c846e91cc4
commit
9be9813021
16 files changed
+1992
-175
No files matched your search
@@ -0,0 +1,283 @@
|
||||
/**
|
||||
* 详情页简介清洗与结构化(DETAIL-1):纯函数,输入是抓取时得到的简介文本(可能残留 HTML / BBCode),
|
||||
* 输出「信息块」键值对(◎片名 / 【年代】/ 『导演』/ 片名: 等)与正文段落。
|
||||
* 清洗规则可配置:站点模板段、广告签名按正则整段或整行去掉,调用方可在默认规则上追加。
|
||||
*/
|
||||
|
||||
export interface DetailInfoItem {
|
||||
key: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
export interface StructuredDescription {
|
||||
infoBlock: DetailInfoItem[];
|
||||
paragraphs: string[];
|
||||
}
|
||||
|
||||
export interface DescriptionRules {
|
||||
/** 整段去掉(对整篇文本匹配,可跨行):站点模板段、更新提示等 */
|
||||
dropBlocks: RegExp[];
|
||||
/** 整行去掉:广告、群号、转载声明、感谢签名、分隔线 */
|
||||
dropLines: RegExp[];
|
||||
}
|
||||
|
||||
export const DEFAULT_DESCRIPTION_RULES: DescriptionRules = {
|
||||
dropBlocks: [
|
||||
// 憨憨豆瓣信息块末尾:「注意：该信息最后更新于…点击 这里 更新信息」
|
||||
/注意[:：]\s*该信息最后更新于[\s\S]{0,80}?更新信息/g,
|
||||
],
|
||||
dropLines: [
|
||||
/^[-=_*~—─━·.]{5,}$/,
|
||||
/^(?:QQ|qq|微信|TG|Telegram|电报|Discord)\s*群?(?:号)?\s*[:：]/,
|
||||
/(?:禁止|严禁|请勿|不得).{0,12}转载|转载.{0,8}(?:请注明|须注明)/,
|
||||
/^(?:感谢|鸣谢|Thanks?\s+to)\s*.{0,20}(?:分享|发布|制作|压制|上传)/i,
|
||||
/^(?:本资源|本种子?|资源)?由\s*\S{1,24}\s*(?:小组|组)?\s*(?:制作|压制|发布|分享)\S{0,4}$/,
|
||||
/^(?:点击|更多).{0,10}(?:下载|资源).{0,20}(?:访问|请到|见)/,
|
||||
],
|
||||
};
|
||||
|
||||
/** 常见信息块键(去掉空白后比较);冒号式 / 【】/『』 只认这些,避免把正文里的「【第一版】」当键 */
|
||||
const KNOWN_KEYS = [
|
||||
"译名",
|
||||
"片名",
|
||||
"原名",
|
||||
"又名",
|
||||
"别名",
|
||||
"中文名",
|
||||
"英文名",
|
||||
"年代",
|
||||
"年份",
|
||||
"产地",
|
||||
"国家",
|
||||
"地区",
|
||||
"国家地区",
|
||||
"制片国家/地区",
|
||||
"类别",
|
||||
"类型",
|
||||
"语言",
|
||||
"对白语言",
|
||||
"字幕",
|
||||
"上映日期",
|
||||
"首播",
|
||||
"首播日期",
|
||||
"IMDb评分",
|
||||
"IMDB评分",
|
||||
"IMDb链接",
|
||||
"IMDB链接",
|
||||
"豆瓣评分",
|
||||
"豆瓣链接",
|
||||
"TMDB",
|
||||
"TMDb链接",
|
||||
"评分",
|
||||
"票数",
|
||||
"集数",
|
||||
"季数",
|
||||
"片长",
|
||||
"单集片长",
|
||||
"视频尺寸",
|
||||
"文件格式",
|
||||
"文件大小",
|
||||
"导演",
|
||||
"编剧",
|
||||
"主演",
|
||||
"演员",
|
||||
"配音",
|
||||
"制片人",
|
||||
"标签",
|
||||
"标语",
|
||||
"简介",
|
||||
"剧情简介",
|
||||
"内容简介",
|
||||
"剧情概要",
|
||||
"剧情介绍",
|
||||
"获奖情况",
|
||||
];
|
||||
const KNOWN = new Set(KNOWN_KEYS.map((k) => k.toLowerCase()));
|
||||
/** 值就是正文的键:内容进段落 */
|
||||
const DESC_KEYS = new Set(["简介", "剧情简介", "内容简介", "剧情概要", "剧情介绍"]);
|
||||
/** 人名列表:续行合并,修复「Kai Xu周也」这类元素拼接 */
|
||||
const PEOPLE_KEYS = new Set(["导演", "编剧", "主演", "演员", "配音", "制片人"]);
|
||||
|
||||
const WS = "[\\s\\u3000]";
|
||||
const KEY_ALTERNATION = [...KNOWN_KEYS]
|
||||
.sort((a, b) => b.length - a.length)
|
||||
.map((k) => [...k].map((c) => c.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")).join(`${WS}*`))
|
||||
.join("|");
|
||||
/** ◎ 标记:认已知键,否则取到第一个空白 / 冒号为止的短词 */
|
||||
const MARK_KNOWN = new RegExp(`^◎${WS}*(${KEY_ALTERNATION})${WS}*[:：]?${WS}*(.*)$`, "i");
|
||||
const MARK_ANY = /^◎[\s ]*([^\s :：]{1,12})[\s ]*[:：]?[\s ]*(.*)$/;
|
||||
const BRACKET = new RegExp(`^(?:【|『|\\[)${WS}*(${KEY_ALTERNATION})${WS}*(?:】|』|\\])${WS}*[:：]?${WS}*(.*)$`, "i");
|
||||
const COLON = new RegExp(`^(${KEY_ALTERNATION})${WS}*[:：]${WS}*(.*)$`, "i");
|
||||
|
||||
const ENTITIES: Record<string, string> = {
|
||||
nbsp: " ",
|
||||
amp: "&",
|
||||
lt: "<",
|
||||
gt: ">",
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
ensp: " ",
|
||||
emsp: " ",
|
||||
};
|
||||
|
||||
function decodeEntities(s: string): string {
|
||||
return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (all, e: string) => {
|
||||
if (e[0] === "#") {
|
||||
const code = e[1] === "x" || e[1] === "X" ? Number.parseInt(e.slice(2), 16) : Number.parseInt(e.slice(1), 10);
|
||||
return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : all;
|
||||
}
|
||||
return ENTITIES[e.toLowerCase()] ?? all;
|
||||
});
|
||||
}
|
||||
|
||||
/** 去 HTML / BBCode 残留:图片整段去掉,链接留文字,块级标签换行,实体解码 */
|
||||
export function stripMarkup(text: string): string {
|
||||
return decodeEntities(
|
||||
text
|
||||
.replace(/\r/g, "")
|
||||
.replace(/<(script|style)\b[\s\S]*?<\/\1>/gi, "")
|
||||
// 站点截断在标签中间:「…<br…」
|
||||
.replace(/<[a-z/][^>\n]*…?$/i, "…")
|
||||
.replace(/<img\b[^>]*>/gi, "")
|
||||
.replace(/<br\s*\/?>[ \t]*\n?/gi, "\n")
|
||||
.replace(/<\/(?:p|div|li|tr|h[1-6]|blockquote|fieldset|pre)>[ \t]*\n?/gi, "\n")
|
||||
.replace(/<[a-z/!][^>]*>/gi, "")
|
||||
.replace(/\[img(?:=[^\]]*)?\][^[]*\[\/img\]|\[img=[^\]]+\]/gi, "")
|
||||
.replace(/\[url=[^\]]*\]([\s\S]*?)\[\/url\]/gi, "$1")
|
||||
.replace(
|
||||
/\[\/?(?:b|i|u|s|color|size|font|align|center|left|right|url|quote|code|spoiler|hide|box|list|\*|pre|table|tr|td)(?:=[^\]]*)?\]/gi,
|
||||
"",
|
||||
),
|
||||
).replace(/ /g, " ");
|
||||
}
|
||||
|
||||
/** 清洗:去标记、按规则删段删行、行尾空白、连续空行压成一行 */
|
||||
export function cleanDescription(text: string, rules: DescriptionRules = DEFAULT_DESCRIPTION_RULES): string {
|
||||
let t = stripMarkup(text);
|
||||
for (const re of rules.dropBlocks) t = t.replace(re, "");
|
||||
return t
|
||||
.split("\n")
|
||||
.map((l) => l.replace(/[\s ]+$/, ""))
|
||||
.filter((l) => {
|
||||
const s = l.trim().replace(/^ +/, "");
|
||||
return !s || !rules.dropLines.some((re) => re.test(s));
|
||||
})
|
||||
.join("\n")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
interface KeyHit {
|
||||
key: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
const normKey = (k: string) => k.replace(/[\s ]+/g, "");
|
||||
|
||||
function matchKey(line: string): KeyHit | null {
|
||||
const m = MARK_KNOWN.exec(line) ?? MARK_ANY.exec(line) ?? BRACKET.exec(line) ?? COLON.exec(line);
|
||||
if (!m) return null;
|
||||
const key = normKey(m[1]!);
|
||||
if (!key) return null;
|
||||
// 冒号式:键后面要么没东西(值在下一行),要么是冒号分隔的值;「片名」开头的正文句子不当键
|
||||
if (!line.startsWith("◎") && !KNOWN.has(key.toLowerCase())) return null;
|
||||
return { key, value: (m[2] ?? "").trim() };
|
||||
}
|
||||
|
||||
const isSeparator = (l: string) => /^[-=_*~—─━·.]{5,}$/.test(l);
|
||||
const SENTENCE_END = /[。！？!?.…」』"”)）]$/;
|
||||
|
||||
/** 相邻行合并成段:英文硬换行(下一行小写开头且上一行没有句末标点)接成一行,其余每行一段 */
|
||||
function toParagraphs(lines: string[]): string[] {
|
||||
const out: string[] = [];
|
||||
let prevJoinable = false;
|
||||
for (const raw of lines) {
|
||||
const l = raw.replace(/^[\s ]+/, "").trim();
|
||||
if (!l) {
|
||||
prevJoinable = false;
|
||||
continue;
|
||||
}
|
||||
if (prevJoinable && /^[a-z]/.test(l) && out.length) out[out.length - 1] += ` ${l}`;
|
||||
else out.push(l);
|
||||
prevJoinable = /[A-Za-z,]$/.test(l) && !SENTENCE_END.test(l);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function tidyValue(key: string, lines: string[]): string {
|
||||
const parts = lines.map((l) => l.replace(/[\s ]+/g, " ").trim()).filter(Boolean);
|
||||
if (PEOPLE_KEYS.has(key)) {
|
||||
return parts
|
||||
.flatMap((p) => p.replace(/([a-z])(?=[一-鿿])/g, "$1\u0000").split("\u0000"))
|
||||
.map((p) => p.trim())
|
||||
.filter(Boolean)
|
||||
.join(" / ");
|
||||
}
|
||||
return parts.every((p) => p.length <= 40) ? parts.join(" / ") : parts.join("\n");
|
||||
}
|
||||
|
||||
const URL_ONLY = /^(?:https?:\/\/\S+|www\.\S+)$/i;
|
||||
const isLinkKey = (k: string) => /(?:链接|网址|link)$/i.test(k) || /^tmdb$/i.test(k);
|
||||
|
||||
/**
|
||||
* 信息块 + 段落。信息块的键:◎ 开头任意短键;【】『』[] 包裹或「键:」开头只认常见键。
|
||||
* 键后的值可在同一行,也可在下一个非空行(憨憨豆瓣块);续行(主演多行)并入上一键,空行结束。
|
||||
* 简介类键的内容进段落;信息块之前的零碎短行(海报卡片的片名 / 年份 / 评分)在有信息块时丢弃。
|
||||
*/
|
||||
export function structureDescription(
|
||||
text: string | null | undefined,
|
||||
rules: DescriptionRules = DEFAULT_DESCRIPTION_RULES,
|
||||
): StructuredDescription {
|
||||
if (!text?.trim()) return { infoBlock: [], paragraphs: [] };
|
||||
const lines = cleanDescription(text, rules).split("\n");
|
||||
const info: { key: string; lines: string[] }[] = [];
|
||||
const lead: string[] = [];
|
||||
const body: string[] = [];
|
||||
let cur: { key: string; lines: string[] } | "desc" | null = null;
|
||||
const pushBody = (l: string) => (info.length ? body : lead).push(l);
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (isSeparator(trimmed)) {
|
||||
if (cur && cur !== "desc" && cur.lines.length) cur = null;
|
||||
if (cur === "desc") body.push("");
|
||||
continue;
|
||||
}
|
||||
const hit = trimmed ? matchKey(trimmed) : null;
|
||||
if (hit) {
|
||||
if (DESC_KEYS.has(hit.key)) {
|
||||
cur = "desc";
|
||||
body.push("");
|
||||
if (hit.value) body.push(hit.value);
|
||||
} else {
|
||||
const entry = { key: hit.key, lines: hit.value ? [hit.value] : [] };
|
||||
info.push(entry);
|
||||
cur = entry;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!trimmed) {
|
||||
if (cur === "desc") body.push("");
|
||||
else if (cur?.lines.length) cur = null;
|
||||
else if (!cur) pushBody("");
|
||||
continue;
|
||||
}
|
||||
if (cur && cur !== "desc") cur.lines.push(line);
|
||||
else pushBody(line);
|
||||
}
|
||||
|
||||
const seen = new Set<string>();
|
||||
const infoBlock: DetailInfoItem[] = [];
|
||||
for (const e of info) {
|
||||
const value = tidyValue(e.key, e.lines);
|
||||
if (!value || seen.has(e.key)) continue;
|
||||
if (isLinkKey(e.key) && value.split(/\s+/).every((v) => URL_ONLY.test(v) || !v)) continue;
|
||||
seen.add(e.key);
|
||||
infoBlock.push({ key: e.key, value });
|
||||
}
|
||||
// 信息块前面只有零碎短行(卡片摘要)就丢掉;有像样的句子则保留为正文
|
||||
const leadParas = toParagraphs(lead);
|
||||
const keepLead = !infoBlock.length || leadParas.some((p) => p.length > 24 || /[。！？!?]$/.test(p));
|
||||
const paragraphs = [...(keepLead ? leadParas : []), ...toParagraphs(body)];
|
||||
return { infoBlock, paragraphs };
|
||||
}
|
||||
Reference in new issue
Block a user