feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b)

- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe
  XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified,
  backoff + Retry-After, circuit breaker with manual reset, passkey redaction,
  overlap-window GUID+digest dedupe
- sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder,
  field DSL, filters, Jinja templates via nunjucks, login-page detection);
  audiences definition hand-written as JSON; raw category id preserved;
  shared per-site token bucket
- sites/mteam: port of MP MTorrentSpider (params, result mapping, promo
  factors, genDlToken flow) without embedding the API key in enclosures;
  business code/message checks
- testkit: mock RSS server, mock NexusPHP site, mock M-Team API
- docs: W1-3b migration mapping appended to porting ledger

Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
rootandClaude Fable 5.1 committed 2026-09-27 18:10:34 +08:00
1 parent ef2e504a94
commit a6d049db81
47 files changed
+5414

No files matched your search

+68
View File
@@ -0,0 +1,68 @@
import type { FeedCursor, FeedItem } from "./types";
export function emptyFeedCursor(): FeedCursor {
return { lastGuidSet: [], digests: {}, lastFetchedAt: null };
}
export interface ApplyFeedResult {
/** 窗口内首次出现的 GUID */
newItems: FeedItem[];
/** 同 GUID 摘要变化(标题修正、促销、体积、置顶变化等),视为更新 */
updatedItems: FeedItem[];
unchangedCount: number;
/**
* 疑似缺口:本次条目与上次窗口没有任何重叠,且上次窗口非空。
* 说明两次抓取之间可能有条目被挤出 Feed,需要调度有预算的列表/搜索补漏(REQ §4.3)。
*/
possibleGap: boolean;
cursor: FeedCursor;
}
export interface ApplyFeedOptions {
/** 游标保留的 GUID 数上限,默认 max(500, 本次条目数 * 3) */
windowSize?: number;
}
/**
* 重叠窗口去重:不依赖条目顺序或递增 ID,乱序/置顶/重复出现都按 GUID+摘要判定。
* 游标只是快速过滤;持久去重仍以数据库 (feed, guid) 唯一约束为准。
*/
export function applyFeedItems(cursor: FeedCursor, items: FeedItem[], opts: ApplyFeedOptions = {}): ApplyFeedResult {
const newItems: FeedItem[] = [];
const updatedItems: FeedItem[] = [];
let unchangedCount = 0;
let overlap = false;
const digests: Record<string, string> = { ...cursor.digests };
const seenThisRound = new Set<string>();
for (const item of items) {
if (seenThisRound.has(item.guid)) continue; // 同一次抓取内重复 GUID 只取首条
seenThisRound.add(item.guid);
const prev = digests[item.guid];
if (prev === undefined) {
newItems.push(item);
} else {
overlap = true;
if (prev !== item.rawDigest) updatedItems.push(item);
else unchangedCount++;
}
digests[item.guid] = item.rawDigest;
}
const windowSize = opts.windowSize ?? Math.max(500, items.length * 3);
const order = [...seenThisRound, ...cursor.lastGuidSet.filter((g) => !seenThisRound.has(g))].slice(0, windowSize);
const kept: Record<string, string> = {};
for (const g of order) {
const d = digests[g];
if (d !== undefined) kept[g] = d;
}
const lastFetchedAt = items.reduce<string | null>(
(acc, it) => (acc === null || it.fetchedAt > acc ? it.fetchedAt : acc),
cursor.lastFetchedAt,
);
return {
newItems,
updatedItems,
unchangedCount,
possibleGap: cursor.lastGuidSet.length > 0 && items.length > 0 && !overlap,
cursor: { lastGuidSet: order, digests: kept, lastFetchedAt },
};
}