feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b)
- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified, backoff + Retry-After, circuit breaker with manual reset, passkey redaction, overlap-window GUID+digest dedupe - sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder, field DSL, filters, Jinja templates via nunjucks, login-page detection); audiences definition hand-written as JSON; raw category id preserved; shared per-site token bucket - sites/mteam: port of MP MTorrentSpider (params, result mapping, promo factors, genDlToken flow) without embedding the API key in enclosures; business code/message checks - testkit: mock RSS server, mock NexusPHP site, mock M-Team API - docs: W1-3b migration mapping appended to porting ledger Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
rootandClaude Fable 5.1 committed
1 parent
ef2e504a94
commit
a6d049db81
47 files changed
+5414
No files matched your search
@@ -0,0 +1,18 @@
|
||||
# testkit/rss
|
||||
|
||||
mock RSS/Atom 服务器(127.0.0.1 随机端口,node:http,无外部依赖)。
|
||||
|
||||
```ts
|
||||
const feed = await createMockRssServer({ items, passkey: "pk", format: "rss" | "atom" });
|
||||
feed.feedUrl; // 带 ?passkey= 的完整 URL
|
||||
feed.addItem(item); // 新条目置前
|
||||
feed.updateItem(guid, {title}); // 同 GUID 内容变化
|
||||
feed.pinItem(guid); feed.shuffle(seed); // 置顶 / 乱序
|
||||
feed.failNext(2, { status: 503, retryAfter: "120" });
|
||||
feed.setRawBody(maliciousXml); // DTD/XXE/超大响应样本;null 恢复
|
||||
feed.setConditional(false); // 关闭 ETag/Last-Modified
|
||||
feed.requests; feed.notModifiedCount;
|
||||
await feed.close();
|
||||
```
|
||||
|
||||
ETag 随内容与版本变化;If-None-Match 命中或 If-Modified-Since 不早于最后修改时间返回 304。passkey 错误返回 403。
|
||||
@@ -0,0 +1,9 @@
|
||||
export {
|
||||
createMockRssServer,
|
||||
type MockFailure,
|
||||
type MockFeedItem,
|
||||
type MockRssOptions,
|
||||
type MockRssServer,
|
||||
renderAtom,
|
||||
renderRss,
|
||||
} from "./mock-rss";
|
||||
@@ -0,0 +1,206 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { type StartedServer, startHttpServer } from "./server";
|
||||
|
||||
export interface MockFeedItem {
|
||||
guid: string;
|
||||
title: string;
|
||||
link?: string;
|
||||
enclosureUrl?: string;
|
||||
size?: number;
|
||||
pubDate?: Date;
|
||||
categories?: string[];
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export interface MockFailure {
|
||||
status: number;
|
||||
retryAfter?: string;
|
||||
body?: string;
|
||||
}
|
||||
|
||||
export interface MockRssOptions {
|
||||
format?: "rss" | "atom";
|
||||
items?: MockFeedItem[];
|
||||
/** 访问需带的 passkey(query ?passkey=);缺失或错误返回 403 */
|
||||
passkey?: string;
|
||||
/** Feed 路径,默认 /rss.xml */
|
||||
path?: string;
|
||||
title?: string;
|
||||
}
|
||||
|
||||
export interface MockRssServer extends StartedServer {
|
||||
/** 带 passkey(若配置)的完整 Feed URL */
|
||||
feedUrl: string;
|
||||
items: MockFeedItem[];
|
||||
setItems(items: MockFeedItem[]): void;
|
||||
/** 新条目放到最前(RSS 惯例新→旧) */
|
||||
addItem(item: MockFeedItem): void;
|
||||
/** 同 GUID 修改内容(标题修正/促销等) */
|
||||
updateItem(guid: string, patch: Partial<MockFeedItem>): void;
|
||||
/** 置顶:把条目挪到最前 */
|
||||
pinItem(guid: string): void;
|
||||
/** 打乱顺序(确定性:按 seed 洗牌) */
|
||||
shuffle(seed?: number): void;
|
||||
/** 接下来 n 次请求返回失败(如 503 + Retry-After) */
|
||||
failNext(n: number, failure: MockFailure): void;
|
||||
/** 直接返回原始 XML(用于 DTD/XXE/超大响应等恶意样本);传 null 恢复正常 */
|
||||
setRawBody(xml: string | null): void;
|
||||
/** 是否支持 ETag/Last-Modified 条件请求(默认 true) */
|
||||
setConditional(enabled: boolean): void;
|
||||
/** 统计:返回 304 的次数 */
|
||||
readonly notModifiedCount: number;
|
||||
}
|
||||
|
||||
function esc(s: string): string {
|
||||
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
||||
}
|
||||
|
||||
export function renderRss(title: string, items: MockFeedItem[]): string {
|
||||
const body = items
|
||||
.map((it) => {
|
||||
const parts = [
|
||||
`<title>${esc(it.title)}</title>`,
|
||||
`<guid isPermaLink="false">${esc(it.guid)}</guid>`,
|
||||
it.link ? `<link>${esc(it.link)}</link>` : "",
|
||||
it.enclosureUrl
|
||||
? `<enclosure url="${esc(it.enclosureUrl)}" length="${it.size ?? 0}" type="application/x-bittorrent"/>`
|
||||
: "",
|
||||
it.pubDate ? `<pubDate>${it.pubDate.toUTCString()}</pubDate>` : "",
|
||||
...(it.categories ?? []).map((c) => `<category>${esc(c)}</category>`),
|
||||
it.description ? `<description><![CDATA[${it.description}]]></description>` : "",
|
||||
];
|
||||
return `<item>${parts.join("")}</item>`;
|
||||
})
|
||||
.join("\n");
|
||||
return `<?xml version="1.0" encoding="UTF-8"?>\n<rss version="2.0"><channel><title>${esc(title)}</title>\n${body}\n</channel></rss>`;
|
||||
}
|
||||
|
||||
export function renderAtom(title: string, items: MockFeedItem[]): string {
|
||||
const body = items
|
||||
.map((it) => {
|
||||
const parts = [
|
||||
`<id>${esc(it.guid)}</id>`,
|
||||
`<title>${esc(it.title)}</title>`,
|
||||
it.link ? `<link rel="alternate" href="${esc(it.link)}"/>` : "",
|
||||
it.enclosureUrl ? `<link rel="enclosure" href="${esc(it.enclosureUrl)}" length="${it.size ?? 0}"/>` : "",
|
||||
it.pubDate ? `<published>${it.pubDate.toISOString()}</published>` : "",
|
||||
...(it.categories ?? []).map((c) => `<category term="${esc(c)}"/>`),
|
||||
it.description ? `<summary>${esc(it.description)}</summary>` : "",
|
||||
];
|
||||
return `<entry>${parts.join("")}</entry>`;
|
||||
})
|
||||
.join("\n");
|
||||
return `<?xml version="1.0" encoding="utf-8"?>\n<feed xmlns="http://www.w3.org/2005/Atom"><title>${esc(title)}</title>\n${body}\n</feed>`;
|
||||
}
|
||||
|
||||
/** 可编程 mock RSS/Atom 服务器:ETag/Last-Modified/304、乱序/置顶/更新、5xx + Retry-After。 */
|
||||
export async function createMockRssServer(opts: MockRssOptions = {}): Promise<MockRssServer> {
|
||||
const path = opts.path ?? "/rss.xml";
|
||||
const title = opts.title ?? "Mock Feed";
|
||||
let items = [...(opts.items ?? [])];
|
||||
let version = 1;
|
||||
let lastModified = new Date(Date.UTC(2026, 8, 1));
|
||||
const failures: MockFailure[] = [];
|
||||
let raw: string | null = null;
|
||||
let conditional = true;
|
||||
let notModified = 0;
|
||||
|
||||
const bump = () => {
|
||||
version++;
|
||||
lastModified = new Date(lastModified.getTime() + 60_000);
|
||||
};
|
||||
|
||||
const server = await startHttpServer((req, res) => {
|
||||
const url = new URL(req.url ?? "/", "http://mock");
|
||||
if (url.pathname !== path) {
|
||||
res.writeHead(404).end("not found");
|
||||
return;
|
||||
}
|
||||
if (opts.passkey && url.searchParams.get("passkey") !== opts.passkey) {
|
||||
res.writeHead(403, { "content-type": "text/plain" }).end("invalid passkey");
|
||||
return;
|
||||
}
|
||||
const failure = failures.shift();
|
||||
if (failure) {
|
||||
const headers: Record<string, string> = { "content-type": "text/plain" };
|
||||
if (failure.retryAfter !== undefined) headers["retry-after"] = failure.retryAfter;
|
||||
res.writeHead(failure.status, headers).end(failure.body ?? `mock failure ${failure.status}`);
|
||||
return;
|
||||
}
|
||||
const body = raw ?? (opts.format === "atom" ? renderAtom(title, items) : renderRss(title, items));
|
||||
const etag = `"${createHash("sha1").update(body).digest("hex").slice(0, 16)}-${version}"`;
|
||||
const lm = lastModified.toUTCString();
|
||||
if (conditional) {
|
||||
const inm = req.headers["if-none-match"];
|
||||
const ims = req.headers["if-modified-since"];
|
||||
if ((inm && inm === etag) || (!inm && ims && Date.parse(ims) >= lastModified.getTime())) {
|
||||
notModified++;
|
||||
res.writeHead(304, { etag, "last-modified": lm }).end();
|
||||
return;
|
||||
}
|
||||
}
|
||||
const headers: Record<string, string> = {
|
||||
"content-type":
|
||||
opts.format === "atom" ? "application/atom+xml; charset=utf-8" : "application/rss+xml; charset=utf-8",
|
||||
};
|
||||
if (conditional) {
|
||||
headers.etag = etag;
|
||||
headers["last-modified"] = lm;
|
||||
}
|
||||
res.writeHead(200, headers).end(body);
|
||||
});
|
||||
|
||||
const feedUrl = `${server.baseUrl}${path}${opts.passkey ? `?passkey=${encodeURIComponent(opts.passkey)}` : ""}`;
|
||||
return {
|
||||
...server,
|
||||
feedUrl,
|
||||
get items() {
|
||||
return items;
|
||||
},
|
||||
setItems(next) {
|
||||
items = [...next];
|
||||
bump();
|
||||
},
|
||||
addItem(item) {
|
||||
items = [item, ...items];
|
||||
bump();
|
||||
},
|
||||
updateItem(guid, patch) {
|
||||
items = items.map((it) => (it.guid === guid ? { ...it, ...patch, guid } : it));
|
||||
bump();
|
||||
},
|
||||
pinItem(guid) {
|
||||
const it = items.find((i) => i.guid === guid);
|
||||
if (!it) return;
|
||||
items = [it, ...items.filter((i) => i.guid !== guid)];
|
||||
bump();
|
||||
},
|
||||
shuffle(seed = 42) {
|
||||
let s = seed;
|
||||
const rnd = () => {
|
||||
s = (s * 1103515245 + 12345) % 2147483648;
|
||||
return s / 2147483648;
|
||||
};
|
||||
const arr = [...items];
|
||||
for (let i = arr.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(rnd() * (i + 1));
|
||||
[arr[i], arr[j]] = [arr[j]!, arr[i]!];
|
||||
}
|
||||
items = arr;
|
||||
bump();
|
||||
},
|
||||
failNext(n, failure) {
|
||||
for (let i = 0; i < n; i++) failures.push(failure);
|
||||
},
|
||||
setRawBody(xml) {
|
||||
raw = xml;
|
||||
bump();
|
||||
},
|
||||
setConditional(enabled) {
|
||||
conditional = enabled;
|
||||
},
|
||||
get notModifiedCount() {
|
||||
return notModified;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
export interface RecordedRequest {
|
||||
method: string;
|
||||
path: string;
|
||||
headers: Record<string, string | string[] | undefined>;
|
||||
body: string;
|
||||
}
|
||||
|
||||
export interface StartedServer {
|
||||
/** 形如 http://127.0.0.1:PORT(不带尾斜杠) */
|
||||
baseUrl: string;
|
||||
requests: RecordedRequest[];
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export type Handler = (req: IncomingMessage, res: ServerResponse, body: string) => void | Promise<void>;
|
||||
|
||||
/** 在 127.0.0.1 随机端口起 HTTP 服务;记录所有请求供断言。 */
|
||||
export async function startHttpServer(handler: Handler): Promise<StartedServer> {
|
||||
const requests: RecordedRequest[] = [];
|
||||
const server = createServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (c: Buffer) => chunks.push(c));
|
||||
req.on("end", () => {
|
||||
const body = Buffer.concat(chunks).toString("utf8");
|
||||
requests.push({ method: req.method ?? "GET", path: req.url ?? "/", headers: { ...req.headers }, body });
|
||||
Promise.resolve(handler(req, res, body)).catch((e: unknown) => {
|
||||
if (!res.headersSent) res.writeHead(500, { "content-type": "text/plain" });
|
||||
res.end(`mock handler error: ${(e as Error).message}`);
|
||||
});
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const { port } = server.address() as AddressInfo;
|
||||
return {
|
||||
baseUrl: `http://127.0.0.1:${port}`,
|
||||
requests,
|
||||
close: () =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
server.closeAllConnections();
|
||||
server.close((err) => (err ? reject(err) : resolve()));
|
||||
}),
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user