import { createHash } from "node:crypto"; import { XMLParser } from "fast-xml-parser"; import { redactUrl } from "./redact.ts"; import { FeedError, type FeedItem, type ParsedFeed } from "./types.ts"; export interface ParseFeedOptions { /** 条目上限,超出截断并置 truncated(默认 1000,同 MP) */ maxItems?: number; /** 文本长度上限(字符,默认 50 MiB,同 MP) */ maxChars?: number; /** 抓取时间,默认 now */ fetchedAt?: Date; } /** MP RssHelper.MAX_RSS_ITEMS */ const DEFAULT_MAX_ITEMS = 1000; /** MP RssHelper.MAX_RSS_SIZE */ const DEFAULT_MAX_CHARS = 50 * 1024 * 1024; /** * 预检:拒绝内部 DTD 子集与任何 ENTITY 声明(XXE / billion laughs)。 * 仅外部 DOCTYPE 声明(如 RSS 0.91 的 PUBLIC 标识)无子集时放行,但解析器同样不会去取。 */ export function assertNoDtdEntities(xml: string): void { if (/[]*(\[)?/i.exec(xml); if (doctype?.[1]) throw new FeedError("forbidden_dtd", "XML 含内部 DTD 子集,已拒绝"); } const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@_", removeNSPrefix: true, parseTagValue: false, parseAttributeValue: false, trimValues: true, processEntities: false, htmlEntities: false, cdataPropName: "__cdata", ignoreDeclaration: true, ignorePiTags: true, isArray: (name) => ["item", "entry", "category", "link", "enclosure"].includes(name), }); const XML_ENTITIES: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" }; /** 只解码 XML 预定义实体与数字字符引用;不做任何自定义实体展开。 */ export function decodeXmlEntities(s: string): string { return s.replace(/&(#x[0-9a-f]+|#[0-9]+|[a-z]+);/gi, (m, body: string) => { if (body[0] === "#") { const cp = body[1] === "x" || body[1] === "X" ? Number.parseInt(body.slice(2), 16) : Number(body.slice(1)); return Number.isFinite(cp) && cp > 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : m; } return XML_ENTITIES[body.toLowerCase()] ?? m; }); } type XNode = unknown; function textOf(node: XNode): string { if (node == null) return ""; if (typeof node === "string") return decodeXmlEntities(node); if (typeof node === "number" || typeof node === "boolean") return String(node); if (Array.isArray(node)) return node.map(textOf).join(""); if (typeof node === "object") { const o = node as Record; const cdata = o.__cdata; const text = o["#text"]; return `${text != null ? textOf(text) : ""}${cdata != null ? (Array.isArray(cdata) ? cdata.join("") : String(cdata)) : ""}`.trim(); } return ""; } function attr(node: XNode, name: string): string | undefined { if (node && typeof node === "object" && !Array.isArray(node)) { const v = (node as Record)[`@_${name}`]; return v == null ? undefined : decodeXmlEntities(String(v)); } return undefined; } function asArray(v: XNode): XNode[] { if (v == null) return []; return Array.isArray(v) ? v : [v]; } function toIso(s: string | undefined): string | undefined { if (!s) return undefined; const t = Date.parse(s); return Number.isNaN(t) ? undefined : new Date(t).toISOString(); } function looksLikeUrl(s: string): boolean { return /^[a-z][a-z0-9+.-]*:/i.test(s); } function sha256(s: string): string { return createHash("sha256").update(s).digest("hex"); } interface RawFields { guid: string; title: string; link: string; enclosure: string; enclosureType?: string; size: number; published?: string; categories: string[]; description?: string; nickname?: string; } function buildItem(raw: RawFields, fetchedAt: string): FeedItem { // 摘要基于脱敏后的值(避免 passkey 轮换误判为更新) const enclosureRedacted = raw.enclosure ? redactUrl(raw.enclosure) : undefined; const linkRedacted = raw.link ? redactUrl(raw.link) : null; const guidBase = raw.guid || linkRedacted || enclosureRedacted || ""; const guid = guidBase ? looksLikeUrl(guidBase) ? redactUrl(guidBase) : guidBase : `digest:${sha256(`${raw.title}\u0000${raw.published ?? ""}`).slice(0, 32)}`; const rawDigest = sha256( JSON.stringify([ guid, raw.title, linkRedacted, enclosureRedacted ?? null, raw.size, raw.published ?? null, [...raw.categories].sort(), raw.description ?? null, ]), ); const item: FeedItem = { guid, title: raw.title, link: linkRedacted, size: raw.size, categories: raw.categories, rawDigest, fetchedAt, }; if (enclosureRedacted) { item.enclosureUrl = enclosureRedacted; item.enclosureHasSecret = enclosureRedacted !== raw.enclosure; } if (raw.enclosureType) item.enclosureType = raw.enclosureType; const publishedAt = toIso(raw.published); if (publishedAt) item.publishedAt = publishedAt; if (raw.description) item.description = raw.description; if (raw.nickname) item.nickname = raw.nickname; return item; } /** * 单条 item/entry → 字段。迁移自 MP `RssHelper._parse_impl` 的条目循环: * title(空则跳过)、description|summary、link(文本或 href)、enclosure@url(缺失回退 link,两者皆无跳过)、 * enclosure@length(纯数字才取)、pubDate|published|updated、creator(nickname)。 * guid/categories 为本项目扩展(去重与整理视图需要)。 */ function itemFields(it: Record): RawFields | null { const title = textOf(asArray(it.title)[0]); if (!title) return null; const description = textOf(asArray(it.description)[0]) || textOf(asArray(it.summary)[0]); const firstLink = asArray(it.link)[0]; const link = textOf(firstLink) || attr(firstLink, "href") || ""; const encNode = asArray(it.enclosure)[0]; let enclosure = attr(encNode, "url") ?? ""; if (!enclosure && !link) return null; if (!enclosure && link) enclosure = link; const lengthAttr = attr(encNode, "length") ?? "0"; const size = /^\d+$/.test(lengthAttr) ? Number(lengthAttr) : 0; const published = textOf(asArray(it.pubDate)[0]) || textOf(asArray(it.published)[0]) || textOf(asArray(it.updated)[0]); const nickname = textOf(asArray(it.creator)[0]); const raw: RawFields = { guid: textOf(asArray(it.guid)[0]) || textOf(asArray(it.id)[0]), title, link, enclosure, size, categories: asArray(it.category) .map((c) => attr(c, "term") ?? textOf(c)) .filter((c) => c.length > 0), }; const encType = attr(encNode, "type"); if (encType) raw.enclosureType = encType; if (published) raw.published = published; if (description) raw.description = description; if (nickname) raw.nickname = nickname; return raw; } /** MP xpath `.//item | .//entry`:在整棵树中按文档顺序收集条目节点(深度受限)。 */ function collectItems(node: XNode, out: Array>, depth = 0): void { if (depth > 12 || node == null || typeof node !== "object") return; if (Array.isArray(node)) { for (const n of node) collectItems(n, out, depth + 1); return; } for (const [k, v] of Object.entries(node as Record)) { if (k.startsWith("@_") || k === "#text" || k === "__cdata") continue; if (k === "item" || k === "entry") { for (const it of asArray(v)) if (it && typeof it === "object") out.push(it as Record); } else { collectItems(v, out, depth + 1); } } } /** MP `_rss_expired_msg`:站点返回的 RSS 过期提示(MP 返回 None,本项目映射为 FeedError expired)。 */ export const RSS_EXPIRED_MESSAGES = [ "RSS 链接已过期, 您需要获得一个新的!", "RSS Link has expired, You need to get a new one!", "RSS Link has expired, You need to get new!", ]; /** * 安全解析 RSS 2.0 / RSS 1.0(RDF) / Atom。 * 迁移自 MP `app/application/rss.py RssHelper._parse_impl`(MAX_RSS_SIZE 50MB、MAX_RSS_ITEMS 1000、 * `resolve_entities=False`、非 `<` 开头拒绝、过期提示识别);本项目额外拒绝 DTD 实体声明并返回脱敏字段。 */ export function parseFeed(xml: string, opts: ParseFeedOptions = {}): ParsedFeed { const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS; const maxItems = opts.maxItems ?? DEFAULT_MAX_ITEMS; if (xml.length > maxChars) throw new FeedError("too_large", `Feed 文本 ${xml.length} 字符超过上限 ${maxChars}`); const trimmed = xml.trim(); if (!trimmed) throw new FeedError("not_feed", "RSS 内容为空"); if (RSS_EXPIRED_MESSAGES.includes(trimmed)) throw new FeedError("expired", "RSS 链接已过期,需要重新获取"); if (!trimmed.startsWith("<")) throw new FeedError("not_feed", "RSS 内容不是有效的 XML 格式"); assertNoDtdEntities(xml); let doc: Record; try { doc = parser.parse(xml) as Record; } catch (e) { throw new FeedError("parse_error", `XML 解析失败:${(e as Error).message.slice(0, 200)}`); } const fetchedAt = (opts.fetchedAt ?? new Date()).toISOString(); const rss = doc.rss as Record | undefined; const rdf = doc.RDF as Record | undefined; const atom = doc.feed as Record | undefined; let format: ParsedFeed["format"]; let title: string | null; if (rss && typeof rss === "object") { format = "rss"; title = textOf((rss.channel as Record | undefined)?.title) || null; } else if (rdf && typeof rdf === "object") { format = "rdf"; title = textOf((rdf.channel as Record | undefined)?.title) || null; } else if (atom && typeof atom === "object") { format = "atom"; title = textOf(atom.title) || null; } else { throw new FeedError("not_feed", "内容不是 RSS/Atom Feed"); } const nodes: Array> = []; collectItems(doc, nodes); const truncated = nodes.length > maxItems; const items: FeedItem[] = []; for (const n of nodes.slice(0, maxItems)) { try { const raw = itemFields(n); if (raw) items.push(buildItem(raw, fetchedAt)); } catch { // MP:单条解析失败跳过 } } return { format, title, items, truncated }; }