rootandClaude Fable 5.1
a6d049db81
- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified, backoff + Retry-After, circuit breaker with manual reset, passkey redaction, overlap-window GUID+digest dedupe - sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder, field DSL, filters, Jinja templates via nunjucks, login-page detection); audiences definition hand-written as JSON; raw category id preserved; shared per-site token bucket - sites/mteam: port of MP MTorrentSpider (params, result mapping, promo factors, genDlToken flow) without embedding the API key in enclosures; business code/message checks - testkit: mock RSS server, mock NexusPHP site, mock M-Team API - docs: W1-3b migration mapping appended to porting ledger Co-Authored-By: Claude Fable 5.1 <[email protected]>
61 lines
2.4 KiB
TypeScript
61 lines
2.4 KiB
TypeScript
/**
|
|
* 站点 RSS 地址获取。迁移自 MP e6820a8 `app/application/rss.py`:
|
|
* `RssHelper.rss_link_conf`(仅 default 与 audiences.me 两项,v0.1 范围)、`_get_site_domain`、`get_rss_link`。
|
|
* MP 用 lxml xpath `//a[@class='faqlink']/@href` 取最后一个;这里用等价 CSS `a.faqlink` + href。
|
|
* 请求本身由调用方(站点适配器)带 Cookie 发送并走站点限流;本模块只构造请求与解析结果。
|
|
*/
|
|
import { load } from "cheerio";
|
|
|
|
export interface RssLinkConf {
|
|
/** 等价于 MP xpath 的 CSS 选择器与属性 */
|
|
selector: string;
|
|
attribute: string;
|
|
url: string;
|
|
params: Record<string, string | number>;
|
|
}
|
|
|
|
export const RSS_LINK_CONF: Record<string, RssLinkConf> = {
|
|
default: {
|
|
selector: "a.faqlink",
|
|
attribute: "href",
|
|
url: "getrss.php",
|
|
params: { inclbookmarked: 0, itemsmalldescr: 1, showrows: 50, search_mode: 1 },
|
|
},
|
|
"audiences.me": {
|
|
selector: "a.faqlink",
|
|
attribute: "href",
|
|
url: "getrss.php",
|
|
params: { inclbookmarked: 0, itemsmalldescr: 1, showrows: 50, search_mode: 1, torrent_type: 1, exp: 180 },
|
|
},
|
|
};
|
|
|
|
/** MP `_get_site_domain`:匹配配置域名,否则回退最后两级域名。 */
|
|
export function rssSiteDomain(url: string): string {
|
|
const host = new URL(url).hostname.toLowerCase();
|
|
for (const d of Object.keys(RSS_LINK_CONF)) {
|
|
if (d !== "default" && (host === d || host.endsWith(`.${d}`))) return d;
|
|
}
|
|
const parts = host.split(".");
|
|
return parts.length >= 2 ? parts.slice(-2).join(".") : host;
|
|
}
|
|
|
|
/** 构造 getrss.php 的 POST 请求(表单参数) */
|
|
export function buildRssLinkRequest(siteUrl: string): { url: string; form: Record<string, string> } {
|
|
const conf = RSS_LINK_CONF[rssSiteDomain(siteUrl)] ?? RSS_LINK_CONF.default!;
|
|
return {
|
|
url: new URL(conf.url, siteUrl.endsWith("/") ? siteUrl : `${siteUrl}/`).toString(),
|
|
form: Object.fromEntries(Object.entries(conf.params).map(([k, v]) => [k, String(v)])),
|
|
};
|
|
}
|
|
|
|
/** 从 getrss.php 返回页提取 RSS 地址(MP 取匹配结果的最后一个)。返回值含 passkey,调用方须当作凭据保存。 */
|
|
export function extractRssLink(siteUrl: string, html: string): string | null {
|
|
const conf = RSS_LINK_CONF[rssSiteDomain(siteUrl)] ?? RSS_LINK_CONF.default!;
|
|
const $ = load(html);
|
|
const links = $(conf.selector)
|
|
.map((_, el) => $(el).attr(conf.attribute))
|
|
.get()
|
|
.filter((x): x is string => Boolean(x));
|
|
return links.at(-1) ?? null;
|
|
}
|