feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b)
- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified, backoff + Retry-After, circuit breaker with manual reset, passkey redaction, overlap-window GUID+digest dedupe - sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder, field DSL, filters, Jinja templates via nunjucks, login-page detection); audiences definition hand-written as JSON; raw category id preserved; shared per-site token bucket - sites/mteam: port of MP MTorrentSpider (params, result mapping, promo factors, genDlToken flow) without embedding the API key in enclosures; business code/message checks - testkit: mock RSS server, mock NexusPHP site, mock M-Team API - docs: W1-3b migration mapping appended to porting ledger Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
rootandClaude Fable 5.1 committed
1 parent
ef2e504a94
commit
a6d049db81
47 files changed
+5414
No files matched your search
@@ -0,0 +1,271 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { XMLParser } from "fast-xml-parser";
|
||||
import { redactUrl } from "./redact";
|
||||
import { FeedError, type FeedItem, type ParsedFeed } from "./types";
|
||||
|
||||
export interface ParseFeedOptions {
|
||||
/** 条目上限,超出截断并置 truncated(默认 1000,同 MP) */
|
||||
maxItems?: number;
|
||||
/** 文本长度上限(字符,默认 50 MiB,同 MP) */
|
||||
maxChars?: number;
|
||||
/** 抓取时间,默认 now */
|
||||
fetchedAt?: Date;
|
||||
}
|
||||
|
||||
/** MP RssHelper.MAX_RSS_ITEMS */
|
||||
const DEFAULT_MAX_ITEMS = 1000;
|
||||
/** MP RssHelper.MAX_RSS_SIZE */
|
||||
const DEFAULT_MAX_CHARS = 50 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* 预检:拒绝内部 DTD 子集与任何 ENTITY 声明(XXE / billion laughs)。
|
||||
* 仅外部 DOCTYPE 声明(如 RSS 0.91 的 PUBLIC 标识)无子集时放行,但解析器同样不会去取。
|
||||
*/
|
||||
export function assertNoDtdEntities(xml: string): void {
|
||||
if (/<!ENTITY/i.test(xml)) throw new FeedError("forbidden_dtd", "XML 含 ENTITY 声明,已拒绝");
|
||||
const doctype = /<!DOCTYPE[^>[]*(\[)?/i.exec(xml);
|
||||
if (doctype?.[1]) throw new FeedError("forbidden_dtd", "XML 含内部 DTD 子集,已拒绝");
|
||||
}
|
||||
|
||||
const parser = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: "@_",
|
||||
removeNSPrefix: true,
|
||||
parseTagValue: false,
|
||||
parseAttributeValue: false,
|
||||
trimValues: true,
|
||||
processEntities: false,
|
||||
htmlEntities: false,
|
||||
cdataPropName: "__cdata",
|
||||
ignoreDeclaration: true,
|
||||
ignorePiTags: true,
|
||||
isArray: (name) => ["item", "entry", "category", "link", "enclosure"].includes(name),
|
||||
});
|
||||
|
||||
const XML_ENTITIES: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
|
||||
|
||||
/** 只解码 XML 预定义实体与数字字符引用;不做任何自定义实体展开。 */
|
||||
export function decodeXmlEntities(s: string): string {
|
||||
return s.replace(/&(#x[0-9a-f]+|#[0-9]+|[a-z]+);/gi, (m, body: string) => {
|
||||
if (body[0] === "#") {
|
||||
const cp = body[1] === "x" || body[1] === "X" ? Number.parseInt(body.slice(2), 16) : Number(body.slice(1));
|
||||
return Number.isFinite(cp) && cp > 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : m;
|
||||
}
|
||||
return XML_ENTITIES[body.toLowerCase()] ?? m;
|
||||
});
|
||||
}
|
||||
|
||||
type XNode = unknown;
|
||||
|
||||
function textOf(node: XNode): string {
|
||||
if (node == null) return "";
|
||||
if (typeof node === "string") return decodeXmlEntities(node);
|
||||
if (typeof node === "number" || typeof node === "boolean") return String(node);
|
||||
if (Array.isArray(node)) return node.map(textOf).join("");
|
||||
if (typeof node === "object") {
|
||||
const o = node as Record<string, unknown>;
|
||||
const cdata = o.__cdata;
|
||||
const text = o["#text"];
|
||||
return `${text != null ? textOf(text) : ""}${cdata != null ? (Array.isArray(cdata) ? cdata.join("") : String(cdata)) : ""}`.trim();
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function attr(node: XNode, name: string): string | undefined {
|
||||
if (node && typeof node === "object" && !Array.isArray(node)) {
|
||||
const v = (node as Record<string, unknown>)[`@_${name}`];
|
||||
return v == null ? undefined : decodeXmlEntities(String(v));
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function asArray(v: XNode): XNode[] {
|
||||
if (v == null) return [];
|
||||
return Array.isArray(v) ? v : [v];
|
||||
}
|
||||
|
||||
function toIso(s: string | undefined): string | undefined {
|
||||
if (!s) return undefined;
|
||||
const t = Date.parse(s);
|
||||
return Number.isNaN(t) ? undefined : new Date(t).toISOString();
|
||||
}
|
||||
|
||||
function looksLikeUrl(s: string): boolean {
|
||||
return /^[a-z][a-z0-9+.-]*:/i.test(s);
|
||||
}
|
||||
|
||||
function sha256(s: string): string {
|
||||
return createHash("sha256").update(s).digest("hex");
|
||||
}
|
||||
|
||||
interface RawFields {
|
||||
guid: string;
|
||||
title: string;
|
||||
link: string;
|
||||
enclosure: string;
|
||||
enclosureType?: string;
|
||||
size: number;
|
||||
published?: string;
|
||||
categories: string[];
|
||||
description?: string;
|
||||
nickname?: string;
|
||||
}
|
||||
|
||||
function buildItem(raw: RawFields, fetchedAt: string): FeedItem {
|
||||
// 摘要基于脱敏后的值(避免 passkey 轮换误判为更新)
|
||||
const enclosureRedacted = raw.enclosure ? redactUrl(raw.enclosure) : undefined;
|
||||
const linkRedacted = raw.link ? redactUrl(raw.link) : null;
|
||||
const guidBase = raw.guid || linkRedacted || enclosureRedacted || "";
|
||||
const guid = guidBase
|
||||
? looksLikeUrl(guidBase)
|
||||
? redactUrl(guidBase)
|
||||
: guidBase
|
||||
: `digest:${sha256(`${raw.title}\u0000${raw.published ?? ""}`).slice(0, 32)}`;
|
||||
const rawDigest = sha256(
|
||||
JSON.stringify([
|
||||
guid,
|
||||
raw.title,
|
||||
linkRedacted,
|
||||
enclosureRedacted ?? null,
|
||||
raw.size,
|
||||
raw.published ?? null,
|
||||
[...raw.categories].sort(),
|
||||
raw.description ?? null,
|
||||
]),
|
||||
);
|
||||
const item: FeedItem = {
|
||||
guid,
|
||||
title: raw.title,
|
||||
link: linkRedacted,
|
||||
size: raw.size,
|
||||
categories: raw.categories,
|
||||
rawDigest,
|
||||
fetchedAt,
|
||||
};
|
||||
if (enclosureRedacted) {
|
||||
item.enclosureUrl = enclosureRedacted;
|
||||
item.enclosureHasSecret = enclosureRedacted !== raw.enclosure;
|
||||
}
|
||||
if (raw.enclosureType) item.enclosureType = raw.enclosureType;
|
||||
const publishedAt = toIso(raw.published);
|
||||
if (publishedAt) item.publishedAt = publishedAt;
|
||||
if (raw.description) item.description = raw.description;
|
||||
if (raw.nickname) item.nickname = raw.nickname;
|
||||
return item;
|
||||
}
|
||||
|
||||
/**
|
||||
* 单条 item/entry → 字段。迁移自 MP `RssHelper._parse_impl` 的条目循环:
|
||||
* title(空则跳过)、description|summary、link(文本或 href)、enclosure@url(缺失回退 link,两者皆无跳过)、
|
||||
* enclosure@length(纯数字才取)、pubDate|published|updated、creator(nickname)。
|
||||
* guid/categories 为本项目扩展(去重与整理视图需要)。
|
||||
*/
|
||||
function itemFields(it: Record<string, XNode>): RawFields | null {
|
||||
const title = textOf(asArray(it.title)[0]);
|
||||
if (!title) return null;
|
||||
const description = textOf(asArray(it.description)[0]) || textOf(asArray(it.summary)[0]);
|
||||
const firstLink = asArray(it.link)[0];
|
||||
const link = textOf(firstLink) || attr(firstLink, "href") || "";
|
||||
const encNode = asArray(it.enclosure)[0];
|
||||
let enclosure = attr(encNode, "url") ?? "";
|
||||
if (!enclosure && !link) return null;
|
||||
if (!enclosure && link) enclosure = link;
|
||||
const lengthAttr = attr(encNode, "length") ?? "0";
|
||||
const size = /^\d+$/.test(lengthAttr) ? Number(lengthAttr) : 0;
|
||||
const published =
|
||||
textOf(asArray(it.pubDate)[0]) || textOf(asArray(it.published)[0]) || textOf(asArray(it.updated)[0]);
|
||||
const nickname = textOf(asArray(it.creator)[0]);
|
||||
const raw: RawFields = {
|
||||
guid: textOf(asArray(it.guid)[0]) || textOf(asArray(it.id)[0]),
|
||||
title,
|
||||
link,
|
||||
enclosure,
|
||||
size,
|
||||
categories: asArray(it.category)
|
||||
.map((c) => attr(c, "term") ?? textOf(c))
|
||||
.filter((c) => c.length > 0),
|
||||
};
|
||||
const encType = attr(encNode, "type");
|
||||
if (encType) raw.enclosureType = encType;
|
||||
if (published) raw.published = published;
|
||||
if (description) raw.description = description;
|
||||
if (nickname) raw.nickname = nickname;
|
||||
return raw;
|
||||
}
|
||||
|
||||
/** MP xpath `.//item | .//entry`:在整棵树中按文档顺序收集条目节点(深度受限)。 */
|
||||
function collectItems(node: XNode, out: Array<Record<string, XNode>>, depth = 0): void {
|
||||
if (depth > 12 || node == null || typeof node !== "object") return;
|
||||
if (Array.isArray(node)) {
|
||||
for (const n of node) collectItems(n, out, depth + 1);
|
||||
return;
|
||||
}
|
||||
for (const [k, v] of Object.entries(node as Record<string, XNode>)) {
|
||||
if (k.startsWith("@_") || k === "#text" || k === "__cdata") continue;
|
||||
if (k === "item" || k === "entry") {
|
||||
for (const it of asArray(v)) if (it && typeof it === "object") out.push(it as Record<string, XNode>);
|
||||
} else {
|
||||
collectItems(v, out, depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** MP `_rss_expired_msg`:站点返回的 RSS 过期提示(MP 返回 None,本项目映射为 FeedError expired)。 */
|
||||
export const RSS_EXPIRED_MESSAGES = [
|
||||
"RSS 链接已过期, 您需要获得一个新的!",
|
||||
"RSS Link has expired, You need to get a new one!",
|
||||
"RSS Link has expired, You need to get new!",
|
||||
];
|
||||
|
||||
/**
|
||||
* 安全解析 RSS 2.0 / RSS 1.0(RDF) / Atom。
|
||||
* 迁移自 MP `app/application/rss.py RssHelper._parse_impl`(MAX_RSS_SIZE 50MB、MAX_RSS_ITEMS 1000、
|
||||
* `resolve_entities=False`、非 `<` 开头拒绝、过期提示识别);本项目额外拒绝 DTD 实体声明并返回脱敏字段。
|
||||
*/
|
||||
export function parseFeed(xml: string, opts: ParseFeedOptions = {}): ParsedFeed {
|
||||
const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS;
|
||||
const maxItems = opts.maxItems ?? DEFAULT_MAX_ITEMS;
|
||||
if (xml.length > maxChars) throw new FeedError("too_large", `Feed 文本 ${xml.length} 字符超过上限 ${maxChars}`);
|
||||
const trimmed = xml.trim();
|
||||
if (!trimmed) throw new FeedError("not_feed", "RSS 内容为空");
|
||||
if (RSS_EXPIRED_MESSAGES.includes(trimmed)) throw new FeedError("expired", "RSS 链接已过期,需要重新获取");
|
||||
if (!trimmed.startsWith("<")) throw new FeedError("not_feed", "RSS 内容不是有效的 XML 格式");
|
||||
assertNoDtdEntities(xml);
|
||||
let doc: Record<string, XNode>;
|
||||
try {
|
||||
doc = parser.parse(xml) as Record<string, XNode>;
|
||||
} catch (e) {
|
||||
throw new FeedError("parse_error", `XML 解析失败:${(e as Error).message.slice(0, 200)}`);
|
||||
}
|
||||
const fetchedAt = (opts.fetchedAt ?? new Date()).toISOString();
|
||||
const rss = doc.rss as Record<string, XNode> | undefined;
|
||||
const rdf = doc.RDF as Record<string, XNode> | undefined;
|
||||
const atom = doc.feed as Record<string, XNode> | undefined;
|
||||
let format: ParsedFeed["format"];
|
||||
let title: string | null;
|
||||
if (rss && typeof rss === "object") {
|
||||
format = "rss";
|
||||
title = textOf((rss.channel as Record<string, XNode> | undefined)?.title) || null;
|
||||
} else if (rdf && typeof rdf === "object") {
|
||||
format = "rdf";
|
||||
title = textOf((rdf.channel as Record<string, XNode> | undefined)?.title) || null;
|
||||
} else if (atom && typeof atom === "object") {
|
||||
format = "atom";
|
||||
title = textOf(atom.title) || null;
|
||||
} else {
|
||||
throw new FeedError("not_feed", "内容不是 RSS/Atom Feed");
|
||||
}
|
||||
const nodes: Array<Record<string, XNode>> = [];
|
||||
collectItems(doc, nodes);
|
||||
const truncated = nodes.length > maxItems;
|
||||
const items: FeedItem[] = [];
|
||||
for (const n of nodes.slice(0, maxItems)) {
|
||||
try {
|
||||
const raw = itemFields(n);
|
||||
if (raw) items.push(buildItem(raw, fetchedAt));
|
||||
} catch {
|
||||
// MP:单条解析失败跳过
|
||||
}
|
||||
}
|
||||
return { format, title, items, truncated };
|
||||
}
|
||||
Reference in new issue
Block a user