feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b)

- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe
  XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified,
  backoff + Retry-After, circuit breaker with manual reset, passkey redaction,
  overlap-window GUID+digest dedupe
- sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder,
  field DSL, filters, Jinja templates via nunjucks, login-page detection);
  audiences definition hand-written as JSON; raw category id preserved;
  shared per-site token bucket
- sites/mteam: port of MP MTorrentSpider (params, result mapping, promo
  factors, genDlToken flow) without embedding the API key in enclosures;
  business code/message checks
- testkit: mock RSS server, mock NexusPHP site, mock M-Team API
- docs: W1-3b migration mapping appended to porting ledger

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
rootandClaude Fable 5.1 committed 2026-09-27 18:10:34 +08:00
1 parent ef2e504a94
commit a6d049db81
47 files changed
+5414

No files matched your search

+104
View File
@@ -0,0 +1,104 @@
/**
* URL/文本脱敏。含 passkey 的 URL 不得进入日志、错误信息、模型上下文(REQ §4.1)。
* 规则:query 中凭据类参数值替换为 [REDACTED],URL userinfo 一并抹除。
*/
export const REDACTED = "[REDACTED]";
/** 视为凭据的 query 参数名(小写比较)。 */
export const SENSITIVE_QUERY_KEYS: ReadonlySet<string> = new Set([
"passkey",
"token",
"authkey",
"auth",
"torrent_pass",
"rsskey",
"apikey",
"api_key",
"access_token",
"secret",
"sign",
"downhash",
"credential",
"key",
]);
function isSensitiveKey(key: string, extra?: Iterable<string>): boolean {
const k = key.toLowerCase();
if (SENSITIVE_QUERY_KEYS.has(k)) return true;
if (extra) for (const e of extra) if (e.toLowerCase() === k) return true;
return false;
}
/**
* 把 URL 中的凭据参数替换为 [REDACTED]。输入不是合法绝对 URL 时按原文做正则兜底,不抛错。
* 输出是确定性的(同一 URL 永远得到同一结果),可直接用作去重键。
*/
export function redactUrl(input: string, extraKeys?: Iterable<string>): string {
let url: URL;
try {
url = new URL(input);
} catch {
return redactText(input, extraKeys);
}
if (url.username || url.password) {
url.username = "";
url.password = "";
}
// 手工重建 query,避免 URLSearchParams 把 [REDACTED] 转义成 %5BREDACTED%5D
if (url.search) {
const parts = url.search
.slice(1)
.split("&")
.map((pair) => {
const eq = pair.indexOf("=");
const rawKey = eq === -1 ? pair : pair.slice(0, eq);
let key = rawKey;
try {
key = decodeURIComponent(rawKey.replace(/\+/g, " "));
} catch {
// 保留原样
}
return eq !== -1 && isSensitiveKey(key, extraKeys) ? `${rawKey}=${REDACTED}` : pair;
});
const out = `${url.origin}${url.pathname}?${parts.join("&")}${url.hash}`;
return url.protocol === "magnet:" ? `magnet:?${parts.join("&")}` : out;
}
return url.toString();
}
const QUERY_PARAM_RE = /([?&;])([A-Za-z0-9_.-]+)=([^&#\s"'<>]*)/g;
const USERINFO_RE = /\b([a-z][a-z0-9+.-]*:\/\/)[^/\s@:]+(?::[^/\s@]*)?@/gi;
/** 对任意文本(错误消息、日志行)中出现的 URL 凭据参数与 userinfo 做脱敏。 */
export function redactText(text: string, extraKeys?: Iterable<string>): string {
return text
.replace(USERINFO_RE, "$1")
.replace(QUERY_PARAM_RE, (m, sep: string, key: string) =>
isSensitiveKey(key, extraKeys) ? `${sep}${key}=${REDACTED}` : m,
);
}
/** 从 URL 中取出被脱敏的凭据参数(调用方自行交给 secret store;本模块不持久化)。 */
export function extractSecretParams(input: string, extraKeys?: Iterable<string>): Record<string, string> {
const out: Record<string, string> = {};
try {
const url = new URL(input);
for (const [k, v] of url.searchParams) if (isSensitiveKey(k, extraKeys)) out[k] = v;
} catch {
// 非 URL 返回空
}
return out;
}
/**
* 把脱敏 URL 中的 [REDACTED] 还原为真实凭据,只在即将发出请求时于内存中调用,结果不得记录。
* 找不到对应 secret 的参数保持 [REDACTED],调用方应视为不可用。
*/
export function unredactUrl(redacted: string, secrets: Record<string, string>): string {
const lower = new Map(Object.entries(secrets).map(([k, v]) => [k.toLowerCase(), v]));
return redacted.replace(/([?&;])([A-Za-z0-9_.-]+)=\[REDACTED\]/g, (m, sep: string, key: string) => {
const v = lower.get(key.toLowerCase());
return v === undefined ? m : `${sep}${key}=${encodeURIComponent(v)}`;
});
}