feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b)
- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified, backoff + Retry-After, circuit breaker with manual reset, passkey redaction, overlap-window GUID+digest dedupe - sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder, field DSL, filters, Jinja templates via nunjucks, login-page detection); audiences definition hand-written as JSON; raw category id preserved; shared per-site token bucket - sites/mteam: port of MP MTorrentSpider (params, result mapping, promo factors, genDlToken flow) without embedding the API key in enclosures; business code/message checks - testkit: mock RSS server, mock NexusPHP site, mock M-Team API - docs: W1-3b migration mapping appended to porting ledger Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
1 parent
ef2e504a94
commit
a6d049db81
47 files changed
+5414
No files matched your search
@@ -0,0 +1,68 @@
|
||||
import type { FeedCursor, FeedItem } from "./types";
|
||||
|
||||
export function emptyFeedCursor(): FeedCursor {
|
||||
return { lastGuidSet: [], digests: {}, lastFetchedAt: null };
|
||||
}
|
||||
|
||||
export interface ApplyFeedResult {
|
||||
/** 窗口内首次出现的 GUID */
|
||||
newItems: FeedItem[];
|
||||
/** 同 GUID 摘要变化(标题修正、促销、体积、置顶变化等),视为更新 */
|
||||
updatedItems: FeedItem[];
|
||||
unchangedCount: number;
|
||||
/**
|
||||
* 疑似缺口:本次条目与上次窗口没有任何重叠,且上次窗口非空。
|
||||
* 说明两次抓取之间可能有条目被挤出 Feed,需要调度有预算的列表/搜索补漏(REQ §4.3)。
|
||||
*/
|
||||
possibleGap: boolean;
|
||||
cursor: FeedCursor;
|
||||
}
|
||||
|
||||
export interface ApplyFeedOptions {
|
||||
/** 游标保留的 GUID 数上限,默认 max(500, 本次条目数 * 3) */
|
||||
windowSize?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 重叠窗口去重:不依赖条目顺序或递增 ID,乱序/置顶/重复出现都按 GUID+摘要判定。
|
||||
* 游标只是快速过滤;持久去重仍以数据库 (feed, guid) 唯一约束为准。
|
||||
*/
|
||||
export function applyFeedItems(cursor: FeedCursor, items: FeedItem[], opts: ApplyFeedOptions = {}): ApplyFeedResult {
|
||||
const newItems: FeedItem[] = [];
|
||||
const updatedItems: FeedItem[] = [];
|
||||
let unchangedCount = 0;
|
||||
let overlap = false;
|
||||
const digests: Record<string, string> = { ...cursor.digests };
|
||||
const seenThisRound = new Set<string>();
|
||||
for (const item of items) {
|
||||
if (seenThisRound.has(item.guid)) continue; // 同一次抓取内重复 GUID 只取首条
|
||||
seenThisRound.add(item.guid);
|
||||
const prev = digests[item.guid];
|
||||
if (prev === undefined) {
|
||||
newItems.push(item);
|
||||
} else {
|
||||
overlap = true;
|
||||
if (prev !== item.rawDigest) updatedItems.push(item);
|
||||
else unchangedCount++;
|
||||
}
|
||||
digests[item.guid] = item.rawDigest;
|
||||
}
|
||||
const windowSize = opts.windowSize ?? Math.max(500, items.length * 3);
|
||||
const order = [...seenThisRound, ...cursor.lastGuidSet.filter((g) => !seenThisRound.has(g))].slice(0, windowSize);
|
||||
const kept: Record<string, string> = {};
|
||||
for (const g of order) {
|
||||
const d = digests[g];
|
||||
if (d !== undefined) kept[g] = d;
|
||||
}
|
||||
const lastFetchedAt = items.reduce<string | null>(
|
||||
(acc, it) => (acc === null || it.fetchedAt > acc ? it.fetchedAt : acc),
|
||||
cursor.lastFetchedAt,
|
||||
);
|
||||
return {
|
||||
newItems,
|
||||
updatedItems,
|
||||
unchangedCount,
|
||||
possibleGap: cursor.lastGuidSet.length > 0 && items.length > 0 && !overlap,
|
||||
cursor: { lastGuidSet: order, digests: kept, lastFetchedAt },
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user