/** * 详情页简介清洗与结构化(DETAIL-1):纯函数,输入是抓取时得到的简介文本(可能残留 HTML / BBCode), * 输出「信息块」键值对(◎片名 / 【年代】/ 『导演』/ 片名: 等)与正文段落。 * 清洗规则可配置:站点模板段、广告签名按正则整段或整行去掉,调用方可在默认规则上追加。 */ export interface DetailInfoItem { key: string; value: string; } export interface StructuredDescription { infoBlock: DetailInfoItem[]; paragraphs: string[]; } export interface DescriptionRules { /** 整段去掉(对整篇文本匹配,可跨行):站点模板段、更新提示等 */ dropBlocks: RegExp[]; /** 整行去掉:广告、群号、转载声明、感谢签名、分隔线 */ dropLines: RegExp[]; } export const DEFAULT_DESCRIPTION_RULES: DescriptionRules = { dropBlocks: [ // 憨憨豆瓣信息块末尾:「注意:该信息最后更新于…点击 这里 更新信息」 /注意[::]\s*该信息最后更新于[\s\S]{0,80}?更新信息/g, ], dropLines: [ /^[-=_*~—─━·.]{5,}$/, /^(?:QQ|qq|微信|TG|Telegram|电报|Discord)\s*群?(?:号)?\s*[::]/, /(?:禁止|严禁|请勿|不得).{0,12}转载|转载.{0,8}(?:请注明|须注明)/, /^(?:感谢|鸣谢|Thanks?\s+to)\s*.{0,20}(?:分享|发布|制作|压制|上传)/i, /^(?:本资源|本种子?|资源)?由\s*\S{1,24}\s*(?:小组|组)?\s*(?:制作|压制|发布|分享)\S{0,4}$/, /^(?:点击|更多).{0,10}(?:下载|资源).{0,20}(?:访问|请到|见)/, ], }; /** 常见信息块键(去掉空白后比较);冒号式 / 【】/『』 只认这些,避免把正文里的「【第一版】」当键 */ const KNOWN_KEYS = [ "译名", "片名", "原名", "又名", "别名", "中文名", "英文名", "年代", "年份", "产地", "国家", "地区", "国家地区", "制片国家/地区", "类别", "类型", "语言", "对白语言", "字幕", "上映日期", "首播", "首播日期", "IMDb评分", "IMDB评分", "IMDb链接", "IMDB链接", "豆瓣评分", "豆瓣链接", "TMDB", "TMDb链接", "评分", "票数", "集数", "季数", "片长", "单集片长", "视频尺寸", "文件格式", "文件大小", "导演", "编剧", "主演", "演员", "配音", "制片人", "标签", "标语", "简介", "剧情简介", "内容简介", "剧情概要", "剧情介绍", "获奖情况", ]; const KNOWN = new Set(KNOWN_KEYS.map((k) => k.toLowerCase())); /** 值就是正文的键:内容进段落 */ const DESC_KEYS = new Set(["简介", "剧情简介", "内容简介", "剧情概要", "剧情介绍"]); /** 人名列表:续行合并,修复「Kai Xu周也」这类元素拼接 */ const PEOPLE_KEYS = new Set(["导演", "编剧", "主演", "演员", "配音", "制片人"]); const WS = "[\\s\\u3000]"; const KEY_ALTERNATION = [...KNOWN_KEYS] .sort((a, b) => b.length - a.length) .map((k) => [...k].map((c) => c.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")).join(`${WS}*`)) .join("|"); /** ◎ 标记:认已知键,否则取到第一个空白 / 冒号为止的短词 */ const MARK_KNOWN = new RegExp(`^◎${WS}*(${KEY_ALTERNATION})${WS}*[::]?${WS}*(.*)$`, "i"); const MARK_ANY = /^◎[\s ]*([^\s ::]{1,12})[\s ]*[::]?[\s ]*(.*)$/; const BRACKET = new RegExp(`^(?:【|『|\\[)${WS}*(${KEY_ALTERNATION})${WS}*(?:】|』|\\])${WS}*[::]?${WS}*(.*)$`, "i"); const COLON = new RegExp(`^(${KEY_ALTERNATION})${WS}*[::]${WS}*(.*)$`, "i"); const ENTITIES: Record = { nbsp: " ", amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", ensp: " ", emsp: " ", }; function decodeEntities(s: string): string { return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (all, e: string) => { if (e[0] === "#") { const code = e[1] === "x" || e[1] === "X" ? Number.parseInt(e.slice(2), 16) : Number.parseInt(e.slice(1), 10); return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : all; } return ENTITIES[e.toLowerCase()] ?? all; }); } /** 去 HTML / BBCode 残留:图片整段去掉,链接留文字,块级标签换行,实体解码 */ export function stripMarkup(text: string): string { return decodeEntities( text .replace(/\r/g, "") .replace(/<(script|style)\b[\s\S]*?<\/\1>/gi, "") // 站点截断在标签中间:「…\n]*…?$/i, "…") .replace(/]*>/gi, "") .replace(/[ \t]*\n?/gi, "\n") .replace(/<\/(?:p|div|li|tr|h[1-6]|blockquote|fieldset|pre)>[ \t]*\n?/gi, "\n") .replace(/<[a-z/!][^>]*>/gi, "") .replace(/\[img(?:=[^\]]*)?\][^[]*\[\/img\]|\[img=[^\]]+\]/gi, "") .replace(/\[url=[^\]]*\]([\s\S]*?)\[\/url\]/gi, "$1") .replace( /\[\/?(?:b|i|u|s|color|size|font|align|center|left|right|url|quote|code|spoiler|hide|box|list|\*|pre|table|tr|td)(?:=[^\]]*)?\]/gi, "", ), ).replace(/ /g, " "); } /** 清洗:去标记、按规则删段删行、行尾空白、连续空行压成一行 */ export function cleanDescription(text: string, rules: DescriptionRules = DEFAULT_DESCRIPTION_RULES): string { let t = stripMarkup(text); for (const re of rules.dropBlocks) t = t.replace(re, ""); return t .split("\n") .map((l) => l.replace(/[\s ]+$/, "")) .filter((l) => { const s = l.trim().replace(/^ +/, ""); return !s || !rules.dropLines.some((re) => re.test(s)); }) .join("\n") .replace(/\n{3,}/g, "\n\n") .trim(); } interface KeyHit { key: string; value: string; } const normKey = (k: string) => k.replace(/[\s ]+/g, ""); function matchKey(line: string): KeyHit | null { const m = MARK_KNOWN.exec(line) ?? MARK_ANY.exec(line) ?? BRACKET.exec(line) ?? COLON.exec(line); if (!m) return null; const key = normKey(m[1]!); if (!key) return null; // 冒号式:键后面要么没东西(值在下一行),要么是冒号分隔的值;「片名」开头的正文句子不当键 if (!line.startsWith("◎") && !KNOWN.has(key.toLowerCase())) return null; return { key, value: (m[2] ?? "").trim() }; } const isSeparator = (l: string) => /^[-=_*~—─━·.]{5,}$/.test(l); const SENTENCE_END = /[。!?!?.…」』"”))]$/; /** 相邻行合并成段:英文硬换行(下一行小写开头且上一行没有句末标点)接成一行,其余每行一段 */ function toParagraphs(lines: string[]): string[] { const out: string[] = []; let prevJoinable = false; for (const raw of lines) { const l = raw.replace(/^[\s ]+/, "").trim(); if (!l) { prevJoinable = false; continue; } if (prevJoinable && /^[a-z]/.test(l) && out.length) out[out.length - 1] += ` ${l}`; else out.push(l); prevJoinable = /[A-Za-z,]$/.test(l) && !SENTENCE_END.test(l); } return out; } function tidyValue(key: string, lines: string[]): string { const parts = lines.map((l) => l.replace(/[\s ]+/g, " ").trim()).filter(Boolean); if (PEOPLE_KEYS.has(key)) { return parts .flatMap((p) => p.replace(/([a-z])(?=[一-鿿])/g, "$1\u0000").split("\u0000")) .map((p) => p.trim()) .filter(Boolean) .join(" / "); } return parts.every((p) => p.length <= 40) ? parts.join(" / ") : parts.join("\n"); } const URL_ONLY = /^(?:https?:\/\/\S+|www\.\S+)$/i; const isLinkKey = (k: string) => /(?:链接|网址|link)$/i.test(k) || /^tmdb$/i.test(k); /** * 信息块 + 段落。信息块的键:◎ 开头任意短键;【】『』[] 包裹或「键:」开头只认常见键。 * 键后的值可在同一行,也可在下一个非空行(憨憨豆瓣块);续行(主演多行)并入上一键,空行结束。 * 简介类键的内容进段落;信息块之前的零碎短行(海报卡片的片名 / 年份 / 评分)在有信息块时丢弃。 */ export function structureDescription( text: string | null | undefined, rules: DescriptionRules = DEFAULT_DESCRIPTION_RULES, ): StructuredDescription { if (!text?.trim()) return { infoBlock: [], paragraphs: [] }; const lines = cleanDescription(text, rules).split("\n"); const info: { key: string; lines: string[] }[] = []; const lead: string[] = []; const body: string[] = []; let cur: { key: string; lines: string[] } | "desc" | null = null; const pushBody = (l: string) => (info.length ? body : lead).push(l); for (const line of lines) { const trimmed = line.trim(); if (isSeparator(trimmed)) { if (cur && cur !== "desc" && cur.lines.length) cur = null; if (cur === "desc") body.push(""); continue; } const hit = trimmed ? matchKey(trimmed) : null; if (hit) { if (DESC_KEYS.has(hit.key)) { cur = "desc"; body.push(""); if (hit.value) body.push(hit.value); } else { const entry = { key: hit.key, lines: hit.value ? [hit.value] : [] }; info.push(entry); cur = entry; } continue; } if (!trimmed) { if (cur === "desc") body.push(""); else if (cur?.lines.length) cur = null; else if (!cur) pushBody(""); continue; } if (cur && cur !== "desc") cur.lines.push(line); else pushBody(line); } const seen = new Set(); const infoBlock: DetailInfoItem[] = []; for (const e of info) { const value = tidyValue(e.key, e.lines); if (!value || seen.has(e.key)) continue; if (isLinkKey(e.key) && value.split(/\s+/).every((v) => URL_ONLY.test(v) || !v)) continue; seen.add(e.key); infoBlock.push({ key: e.key, value }); } // 信息块前面只有零碎短行(卡片摘要)就丢掉;有像样的句子则保留为正文 const leadParas = toParagraphs(lead); const keepLead = !infoBlock.length || leadParas.some((p) => p.length > 24 || /[。!?!?]$/.test(p)); const paragraphs = [...(keepLead ? leadParas : []), ...toParagraphs(body)]; return { infoBlock, paragraphs }; }