feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b)
- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified, backoff + Retry-After, circuit breaker with manual reset, passkey redaction, overlap-window GUID+digest dedupe - sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder, field DSL, filters, Jinja templates via nunjucks, login-page detection); audiences definition hand-written as JSON; raw category id preserved; shared per-site token bucket - sites/mteam: port of MP MTorrentSpider (params, result mapping, promo factors, genDlToken flow) without embedding the API key in enclosures; business code/message checks - testkit: mock RSS server, mock NexusPHP site, mock M-Team API - docs: W1-3b migration mapping appended to porting ledger Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
1 parent
ef2e504a94
commit
a6d049db81
47 files changed
+5414
No files matched your search
@@ -0,0 +1,104 @@
|
||||
/**
|
||||
* URL/文本脱敏。含 passkey 的 URL 不得进入日志、错误信息、模型上下文(REQ §4.1)。
|
||||
* 规则:query 中凭据类参数值替换为 [REDACTED],URL userinfo 一并抹除。
|
||||
*/
|
||||
|
||||
export const REDACTED = "[REDACTED]";
|
||||
|
||||
/** 视为凭据的 query 参数名(小写比较)。 */
|
||||
export const SENSITIVE_QUERY_KEYS: ReadonlySet<string> = new Set([
|
||||
"passkey",
|
||||
"token",
|
||||
"authkey",
|
||||
"auth",
|
||||
"torrent_pass",
|
||||
"rsskey",
|
||||
"apikey",
|
||||
"api_key",
|
||||
"access_token",
|
||||
"secret",
|
||||
"sign",
|
||||
"downhash",
|
||||
"credential",
|
||||
"key",
|
||||
]);
|
||||
|
||||
function isSensitiveKey(key: string, extra?: Iterable<string>): boolean {
|
||||
const k = key.toLowerCase();
|
||||
if (SENSITIVE_QUERY_KEYS.has(k)) return true;
|
||||
if (extra) for (const e of extra) if (e.toLowerCase() === k) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 把 URL 中的凭据参数替换为 [REDACTED]。输入不是合法绝对 URL 时按原文做正则兜底,不抛错。
|
||||
* 输出是确定性的(同一 URL 永远得到同一结果),可直接用作去重键。
|
||||
*/
|
||||
export function redactUrl(input: string, extraKeys?: Iterable<string>): string {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(input);
|
||||
} catch {
|
||||
return redactText(input, extraKeys);
|
||||
}
|
||||
if (url.username || url.password) {
|
||||
url.username = "";
|
||||
url.password = "";
|
||||
}
|
||||
// 手工重建 query,避免 URLSearchParams 把 [REDACTED] 转义成 %5BREDACTED%5D
|
||||
if (url.search) {
|
||||
const parts = url.search
|
||||
.slice(1)
|
||||
.split("&")
|
||||
.map((pair) => {
|
||||
const eq = pair.indexOf("=");
|
||||
const rawKey = eq === -1 ? pair : pair.slice(0, eq);
|
||||
let key = rawKey;
|
||||
try {
|
||||
key = decodeURIComponent(rawKey.replace(/\+/g, " "));
|
||||
} catch {
|
||||
// 保留原样
|
||||
}
|
||||
return eq !== -1 && isSensitiveKey(key, extraKeys) ? `${rawKey}=${REDACTED}` : pair;
|
||||
});
|
||||
const out = `${url.origin}${url.pathname}?${parts.join("&")}${url.hash}`;
|
||||
return url.protocol === "magnet:" ? `magnet:?${parts.join("&")}` : out;
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
const QUERY_PARAM_RE = /([?&;])([A-Za-z0-9_.-]+)=([^&#\s"'<>]*)/g;
|
||||
const USERINFO_RE = /\b([a-z][a-z0-9+.-]*:\/\/)[^/\s@:]+(?::[^/\s@]*)?@/gi;
|
||||
|
||||
/** 对任意文本(错误消息、日志行)中出现的 URL 凭据参数与 userinfo 做脱敏。 */
|
||||
export function redactText(text: string, extraKeys?: Iterable<string>): string {
|
||||
return text
|
||||
.replace(USERINFO_RE, "$1")
|
||||
.replace(QUERY_PARAM_RE, (m, sep: string, key: string) =>
|
||||
isSensitiveKey(key, extraKeys) ? `${sep}${key}=${REDACTED}` : m,
|
||||
);
|
||||
}
|
||||
|
||||
/** 从 URL 中取出被脱敏的凭据参数(调用方自行交给 secret store;本模块不持久化)。 */
|
||||
export function extractSecretParams(input: string, extraKeys?: Iterable<string>): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
try {
|
||||
const url = new URL(input);
|
||||
for (const [k, v] of url.searchParams) if (isSensitiveKey(k, extraKeys)) out[k] = v;
|
||||
} catch {
|
||||
// 非 URL 返回空
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* 把脱敏 URL 中的 [REDACTED] 还原为真实凭据,只在即将发出请求时于内存中调用,结果不得记录。
|
||||
* 找不到对应 secret 的参数保持 [REDACTED],调用方应视为不可用。
|
||||
*/
|
||||
export function unredactUrl(redacted: string, secrets: Record<string, string>): string {
|
||||
const lower = new Map(Object.entries(secrets).map(([k, v]) => [k.toLowerCase(), v]));
|
||||
return redacted.replace(/([?&;])([A-Za-z0-9_.-]+)=\[REDACTED\]/g, (m, sep: string, key: string) => {
|
||||
const v = lower.get(key.toLowerCase());
|
||||
return v === undefined ? m : `${sep}${key}=${encodeURIComponent(v)}`;
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user