From a6d049db812fa89eaf283a05871ba787872efef4 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 27 Sep 2026 18:10:34 +0800 Subject: [PATCH] feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b) - rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified, backoff + Retry-After, circuit breaker with manual reset, passkey redaction, overlap-window GUID+digest dedupe - sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder, field DSL, filters, Jinja templates via nunjucks, login-page detection); audiences definition hand-written as JSON; raw category id preserved; shared per-site token bucket - sites/mteam: port of MP MTorrentSpider (params, result mapping, promo factors, genDlToken flow) without embedding the API key in enclosures; business code/message checks - testkit: mock RSS server, mock NexusPHP site, mock M-Team API - docs: W1-3b migration mapping appended to porting ledger Co-Authored-By: Claude Fable 5.1 --- .../moviepilot-downloader-and-sites.md | 54 ++ packages/adapters/package.json | 6 + packages/adapters/src/index.ts | 2 + packages/adapters/src/rss/README.md | 36 + packages/adapters/src/rss/cursor.ts | 68 ++ packages/adapters/src/rss/fetch.ts | 229 ++++++ packages/adapters/src/rss/index.ts | 29 + packages/adapters/src/rss/parse.ts | 271 +++++++ packages/adapters/src/rss/redact.ts | 104 +++ packages/adapters/src/rss/rss-link.ts | 60 ++ packages/adapters/src/rss/rss.test.ts | 296 +++++++ packages/adapters/src/rss/types.ts | 92 +++ packages/adapters/src/sites/README.md | 60 ++ .../src/sites/definitions/audiences.json | 110 +++ .../adapters/src/sites/definitions/index.ts | 16 + packages/adapters/src/sites/deps.ts | 31 + packages/adapters/src/sites/http.ts | 92 +++ packages/adapters/src/sites/index.ts | 61 ++ packages/adapters/src/sites/mteam/adapter.ts | 211 +++++ packages/adapters/src/sites/mteam/client.ts | 109 +++ .../adapters/src/sites/mteam/definition.ts | 70 ++ .../adapters/src/sites/mteam/mteam.test.ts | 196 +++++ packages/adapters/src/sites/mteam/spider.ts | 151 ++++ .../adapters/src/sites/rate-limit.test.ts | 50 ++ packages/adapters/src/sites/rate-limit.ts | 95 +++ packages/adapters/src/sites/registry.ts | 35 + packages/adapters/src/sites/spider/adapter.ts | 249 ++++++ packages/adapters/src/sites/spider/detail.ts | 110 +++ packages/adapters/src/sites/spider/map.ts | 82 ++ packages/adapters/src/sites/spider/schema.ts | 126 +++ .../adapters/src/sites/spider/spider.test.ts | 308 +++++++ packages/adapters/src/sites/spider/spider.ts | 755 ++++++++++++++++++ .../adapters/src/sites/spider/template.ts | 17 + packages/adapters/src/sites/types.ts | 189 +++++ packages/adapters/src/sites/util.ts | 100 +++ packages/testkit/src/index.ts | 2 + packages/testkit/src/rss/README.md | 18 + packages/testkit/src/rss/index.ts | 9 + packages/testkit/src/rss/mock-rss.ts | 206 +++++ packages/testkit/src/rss/server.ts | 46 ++ packages/testkit/src/sites/README.md | 30 + packages/testkit/src/sites/index.ts | 10 + packages/testkit/src/sites/mock-mteam.ts | 152 ++++ packages/testkit/src/sites/mock-nexusphp.ts | 220 +++++ packages/testkit/src/sites/mteam-fixtures.ts | 102 +++ .../testkit/src/sites/nexusphp-fixtures.ts | 102 +++ packages/testkit/src/sites/torrent.ts | 47 ++ 47 files changed, 5414 insertions(+) create mode 100644 packages/adapters/src/rss/README.md create mode 100644 packages/adapters/src/rss/cursor.ts create mode 100644 packages/adapters/src/rss/fetch.ts create mode 100644 packages/adapters/src/rss/index.ts create mode 100644 packages/adapters/src/rss/parse.ts create mode 100644 packages/adapters/src/rss/redact.ts create mode 100644 packages/adapters/src/rss/rss-link.ts create mode 100644 packages/adapters/src/rss/rss.test.ts create mode 100644 packages/adapters/src/rss/types.ts create mode 100644 packages/adapters/src/sites/README.md create mode 100644 packages/adapters/src/sites/definitions/audiences.json create mode 100644 packages/adapters/src/sites/definitions/index.ts create mode 100644 packages/adapters/src/sites/deps.ts create mode 100644 packages/adapters/src/sites/http.ts create mode 100644 packages/adapters/src/sites/index.ts create mode 100644 packages/adapters/src/sites/mteam/adapter.ts create mode 100644 packages/adapters/src/sites/mteam/client.ts create mode 100644 packages/adapters/src/sites/mteam/definition.ts create mode 100644 packages/adapters/src/sites/mteam/mteam.test.ts create mode 100644 packages/adapters/src/sites/mteam/spider.ts create mode 100644 packages/adapters/src/sites/rate-limit.test.ts create mode 100644 packages/adapters/src/sites/rate-limit.ts create mode 100644 packages/adapters/src/sites/registry.ts create mode 100644 packages/adapters/src/sites/spider/adapter.ts create mode 100644 packages/adapters/src/sites/spider/detail.ts create mode 100644 packages/adapters/src/sites/spider/map.ts create mode 100644 packages/adapters/src/sites/spider/schema.ts create mode 100644 packages/adapters/src/sites/spider/spider.test.ts create mode 100644 packages/adapters/src/sites/spider/spider.ts create mode 100644 packages/adapters/src/sites/spider/template.ts create mode 100644 packages/adapters/src/sites/types.ts create mode 100644 packages/adapters/src/sites/util.ts create mode 100644 packages/testkit/src/rss/README.md create mode 100644 packages/testkit/src/rss/index.ts create mode 100644 packages/testkit/src/rss/mock-rss.ts create mode 100644 packages/testkit/src/rss/server.ts create mode 100644 packages/testkit/src/sites/README.md create mode 100644 packages/testkit/src/sites/index.ts create mode 100644 packages/testkit/src/sites/mock-mteam.ts create mode 100644 packages/testkit/src/sites/mock-nexusphp.ts create mode 100644 packages/testkit/src/sites/mteam-fixtures.ts create mode 100644 packages/testkit/src/sites/nexusphp-fixtures.ts create mode 100644 packages/testkit/src/sites/torrent.ts diff --git a/docs/porting/moviepilot-downloader-and-sites.md b/docs/porting/moviepilot-downloader-and-sites.md index d370b810..048953db 100644 --- a/docs/porting/moviepilot-downloader-and-sites.md +++ b/docs/porting/moviepilot-downloader-and-sites.md @@ -608,3 +608,57 @@ MC 注册表 `MC:src/movieclaw_tracker/registry.py:172` `_load_site_yaml` 读取 ### 文件系统(`packages/adapters/src/fs/`) 对照 `docs/porting/moviepilot-transfer.md` §2.6:`_write_atomically`/`_partial_path`/`_cleanup_stale_partials` → `copyWithVerify`/`partialPathFor`/`sweepStalePartials`(新增 fsync、校验、no-clobber);`LocalStorage.move` → `moveSafe`(EXDEV 不降级)+ `crossDeviceMove`(三步可重入)。`statSafe`/`scanIncremental`/`ffprobe`/`diskUsage` 为本项目新增。 + +## 10. W1-3b 实际迁移对照(RSS / 站点) + +按用户指示"照抄 MoviePilot v3 实现",以 MP 提交 `e6820a8` 为蓝本迁移为 TS。只有安全、数据保真与去重相关的偏离,逐条列在"调整"列。MovieClaw 仍只参考字段结构(data-ref)。 + +### 10.1 通用站点 spider(`packages/adapters/src/sites/spider/`) + +| 上游函数 | TS 函数 | 保留 / 调整 | +| --- | --- | --- | +| `spider/__init__.py select_media_categories` | `selectMediaCategories` | 保留 | +| `resolve_category_media_type` | `resolveCategoryMediaType` | 保留(投影值仍写 `category`) | +| `SiteSpider.__init__` 读取的 indexer 键 | `IndexerDefinition`(`schema.ts`) | 保留全部键;新增 `umt` 扩展(版本、时区、限流、登录标志、详情页、分类名) | +| `SiteSpider.__get_search_url` / `__format_search_word` | `SiteSpider.getSearchUrl` / `formatSearchWord` | 保留(默认参数、search_area 仅 IMDb 保留、batch、typed path、category field/param/catN、browse.start、result_media_type) | +| `UrlUtils.combine_url` | `combineUrl` | 保留(doseq) | +| `__can_search` | `canSearch` | 保留 | +| `get_info` + `__get_title … __get_category` | `getInfo` + 同名私有方法 | 保留字段顺序与缺省值(size/seeders/peers/grabs 缺失为 0,相对时间不写 pubdate,case 顺序首个命中);**调整**:系数 selector 模式浮点(SA4);`__get_category` 额外输出 `category_id` 原值 | +| `_safe_query / __remove / __attribute_or_text / __index / __filter_text` | 同名方法 | 保留(re_search/split(含负下标)/replace/dateparse/strip/appendleft/querystring,单个过滤器失败跳过) | +| `__is_login_or_permission_page` | `SiteSpider.isLoginOrPermissionPage` | 保留判定;**调整**:adapter 抛 `auth_expired` 而不是返回空列表 | +| `parse`(PyQuery 分支) | `SiteSpider.parse` | 保留;htmlparser2 HTML 模式对齐 lxml(不插 tbody);`:has("x")` 去引号;**未移植** Rust 加速与字幕分支 | +| jinja2 模板 | `template.ts`(nunjucks) | 保留 Jinja 语法 | +| `foundation/size.parse_size` | `parseSizeMp` | 保留(round) | +| `IndexerModule.__search_check` / `__clear_search_text` | `search` 前检查 / `clearSearchText` | 保留(英文站不搜中文、去标点) | +| `SitesHelper.check(domain)`(闭源) | `createSiteRateLimiter` | 以令牌桶实现(SA6) | +| `submission.download_torrent` | `downloadTarget` | 保留(Cookie GET、磁力原样);**调整**:不跟随 3xx、校验 bencode 并计算 infohash、登录页识别 | +| — | `detail.ts parseDetailPage` | 新增(MP 通用 spider 无详情页) | +| MP 站点定义(加密 `user.sites.v3.bin`) | `definitions/audiences.json` | 按 MP schema 手写,待真实页面核验 | + +### 10.2 M-Team(`packages/adapters/src/sites/mteam/`) + +| 上游函数 | TS 函数 | 保留 / 调整 | +| --- | --- | --- | +| `mtorrent.py _movie_category/_tv_category/_music_category/_labels` | `MOVIE_CATEGORY/TV_CATEGORY/MUSIC_CATEGORY/LABELS` | 保留 | +| `MTorrentSpider.__get_params` | `mteamSearchParams` | 保留(`tt` 前缀改写 IMDb URL、pageNumber=page+1、pageSize 100、visible 1、无 mode);**调整**:显式 `categoryIds` 覆盖 mtype 分类组 | +| `__parse_result` | `parseMteamResult` | 保留字段映射、promotionRule 覆盖下载系数、mallSingleFree ONGOING → 0、freedate 取值顺序;**调整**:不生成含 apikey 的 base64 enclosure(MT3);输出 `category_id` | +| `__find_imdbid / __get_downloadvolumefactor / __get_uploadvolumefactor` | `findImdbid / getDownloadVolumeFactor / getUploadVolumeFactor` | 保留 | +| `search` / `__process_response` | `mteamPost` + adapter `list/search` | 保留 header(x-api-key、UA、JSON);**调整**:检查业务 code/message 区分凭据/限流/不存在(MT2) | +| `__get_download_url` + `submission._resolve_indirect_download_url` | adapter `downloadTarget` | 保留(POST genDlToken,id 走 query,取 `data`,不带 Cookie GET);令牌不入库不入日志 | +| `parser/mtorrent.py`(api/member/profile) | `healthCheck` | 只用作登录有效性 | +| — | `detail`(`/api/torrent/detail`) | 新增,字段结构参考 MovieClaw(data-ref) | + +### 10.3 RSS(`packages/adapters/src/rss/`) + +| 上游函数 | TS 函数 | 保留 / 调整 | +| --- | --- | --- | +| `RssHelper.MAX_RSS_SIZE / MAX_RSS_ITEMS` | fetcher `maxBytes` / parser `maxItems` 默认值 | 保留 50MB / 1000;大小按解压后字节流式限制 | +| `RssHelper._parse_impl` 条目循环 | `parseFeed` / `itemFields` | 保留字段语义(见 rss/README);**调整**:fast-xml-parser 关闭实体处理并预检拒绝 ENTITY/内部 DTD;新增 guid/categories/rawDigest | +| `_rss_expired_msg` | `RSS_EXPIRED_MESSAGES` → `FeedError('expired')` | 保留识别,返回值改为错误类型 | +| `RssHelper.parse`(GET,非 200 失败) | `createRssFetcher().fetchFeed` | 保留;新增 ETag/Last-Modified、退避 + Retry-After、熔断 + 手动恢复、passkey 脱敏 | +| `rss_link_conf` / `_get_site_domain` / `get_rss_link` | `RSS_LINK_CONF` / `rssSiteDomain` / `buildRssLinkRequest` + `extractRssLink` | 只迁移 default 与 audiences.me;xpath 改等价 CSS | +| — | `applyFeedItems` / `FeedCursor` | 新增重叠窗口去重(REQ §4.2/§4.3) | + +### 10.4 许可提示 + +本节模块是对 MP(GPL-3.0)实现的逐函数 TS 移植,性质上比 `behavior-ref` 更接近衍生作品。项目目前为 internal-only 使用;若将来对外分发,需按 `license-and-provenance.md` §2 重新评估(GPL-3.0 同许可与源码提供义务)。 diff --git a/packages/adapters/package.json b/packages/adapters/package.json index 5685ebb4..7f5ef6c1 100644 --- a/packages/adapters/package.json +++ b/packages/adapters/package.json @@ -10,7 +10,13 @@ "typecheck": "tsc -p tsconfig.json", "test": "vitest run" }, + "dependencies": { + "cheerio": "^1.2.0", + "fast-xml-parser": "^5.11.1", + "nunjucks": "^3.2.4" + }, "devDependencies": { + "@types/nunjucks": "^3.2.6", "@umt/testkit": "workspace:*" } } diff --git a/packages/adapters/src/index.ts b/packages/adapters/src/index.ts index 97e0c2f9..44102f12 100644 --- a/packages/adapters/src/index.ts +++ b/packages/adapters/src/index.ts @@ -3,4 +3,6 @@ export const PACKAGE_NAME = "@umt/adapters"; export * from "./emby"; export * from "./fs"; export * from "./qbittorrent"; +export * from "./rss"; +export * from "./sites"; export * from "./tmdb"; diff --git a/packages/adapters/src/rss/README.md b/packages/adapters/src/rss/README.md new file mode 100644 index 00000000..68e6bc64 --- /dev/null +++ b/packages/adapters/src/rss/README.md @@ -0,0 +1,36 @@ +# adapters/rss + +RSS 2.0 / RSS 1.0(RDF) / Atom 抓取与解析,按 MoviePilot e6820a8 `app/application/rss.py`(`RssHelper._parse_impl` / `get_rss_link`)迁移,再叠加任务书的安全、条件请求、退避、熔断、脱敏与去重。**仅对 `@umt/testkit` 的 mock RSS 服务器验证,未连接真实站点 Feed。** + +## 能力 + +| 能力 | 入口 | 说明 | +| --- | --- | --- | +| 安全解析 | `parseFeed(xml, {maxItems, maxChars})` | fast-xml-parser 关闭实体处理;预检拒绝 ` = { ...cursor.digests }; + const seenThisRound = new Set(); + for (const item of items) { + if (seenThisRound.has(item.guid)) continue; // 同一次抓取内重复 GUID 只取首条 + seenThisRound.add(item.guid); + const prev = digests[item.guid]; + if (prev === undefined) { + newItems.push(item); + } else { + overlap = true; + if (prev !== item.rawDigest) updatedItems.push(item); + else unchangedCount++; + } + digests[item.guid] = item.rawDigest; + } + const windowSize = opts.windowSize ?? Math.max(500, items.length * 3); + const order = [...seenThisRound, ...cursor.lastGuidSet.filter((g) => !seenThisRound.has(g))].slice(0, windowSize); + const kept: Record = {}; + for (const g of order) { + const d = digests[g]; + if (d !== undefined) kept[g] = d; + } + const lastFetchedAt = items.reduce( + (acc, it) => (acc === null || it.fetchedAt > acc ? it.fetchedAt : acc), + cursor.lastFetchedAt, + ); + return { + newItems, + updatedItems, + unchangedCount, + possibleGap: cursor.lastGuidSet.length > 0 && items.length > 0 && !overlap, + cursor: { lastGuidSet: order, digests: kept, lastFetchedAt }, + }; +} diff --git a/packages/adapters/src/rss/fetch.ts b/packages/adapters/src/rss/fetch.ts new file mode 100644 index 00000000..ec47c5ba --- /dev/null +++ b/packages/adapters/src/rss/fetch.ts @@ -0,0 +1,229 @@ +/** + * RSS 抓取。迁移自 MP `app/application/rss.py RssHelper.parse/_parse_impl`(GET + UA + 自定义头,非 200 视为失败, + * 大小上限后再解析);条件请求、退避、Retry-After、熔断、脱敏为任务书新增要求(MP 无)。 + */ +import { parseFeed } from "./parse"; +import { redactText, redactUrl } from "./redact"; +import { FeedError, type FeedFetchResult, type FeedFetchState } from "./types"; + +export interface RssFetcherOptions { + fetch?: typeof globalThis.fetch; + now?: () => Date; + /** 单次请求超时,默认 30s(MP `timeout or 30`) */ + timeoutMs?: number; + /** 解压后响应体上限(字节),默认 50 MiB(MP MAX_RSS_SIZE);同时约束 gzip/br 解压量 */ + maxBytes?: number; + /** 条目上限,默认 1000(MP MAX_RSS_ITEMS) */ + maxItems?: number; + userAgent?: string; + /** 指数退避:base * 2^(n-1),上限 max。默认 60s / 6h */ + backoff?: { baseMs: number; maxMs: number }; + /** 连续失败达到该次数进入熔断(open),默认 5 */ + circuitThreshold?: number; + /** 熔断冷却后允许一次半开探测;null(默认)= 只能手动 resetFeedCircuit 恢复 */ + circuitCooldownMs?: number | null; +} + +export interface FetchFeedOptions { + signal?: AbortSignal; + /** 手动刷新:忽略退避窗口(不忽略熔断) */ + ignoreBackoff?: boolean; +} + +export function initialFeedState(): FeedFetchState { + return { consecutiveFailures: 0, circuit: "closed" }; +} + +/** 手动恢复:关闭熔断并清空退避,保留 ETag/Last-Modified。 */ +export function resetFeedCircuit(state: FeedFetchState): FeedFetchState { + const { nextAttemptAt: _n, circuitOpenedAt: _c, ...rest } = state; + return { ...rest, consecutiveFailures: 0, circuit: "closed" }; +} + +export function computeBackoffMs(failures: number, baseMs: number, maxMs: number): number { + if (failures <= 0) return 0; + return Math.min(maxMs, baseMs * 2 ** Math.min(failures - 1, 30)); +} + +/** Retry-After:秒数或 HTTP-date;返回毫秒,无效返回 undefined。 */ +export function parseRetryAfter(value: string | null | undefined, now: Date): number | undefined { + if (!value) return undefined; + const v = value.trim(); + if (/^\d+$/.test(v)) return Number(v) * 1000; + const t = Date.parse(v); + if (Number.isNaN(t)) return undefined; + return Math.max(0, t - now.getTime()); +} + +async function readCapped(res: Response, maxBytes: number): Promise { + const len = Number(res.headers.get("content-length") ?? ""); + // content-length 是压缩后长度,只做早拒;真正的上限在流式读取里按解压后字节执行 + if (Number.isFinite(len) && len > maxBytes) { + await res.body?.cancel().catch(() => {}); + throw new FeedError("too_large", `响应声明长度 ${len} 超过上限 ${maxBytes}`); + } + if (!res.body) return new Uint8Array(); + const reader = res.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > maxBytes) { + await reader.cancel().catch(() => {}); + throw new FeedError("too_large", `响应体(解压后)超过上限 ${maxBytes} 字节`); + } + chunks.push(value); + } + const out = new Uint8Array(total); + let off = 0; + for (const c of chunks) { + out.set(c, off); + off += c.byteLength; + } + return out; +} + +function decodeBody(bytes: Uint8Array, contentType: string | null): string { + const head = new TextDecoder("latin1").decode(bytes.subarray(0, 256)); + const label = + /charset=([\w-]+)/i.exec(contentType ?? "")?.[1] ?? + /<\?xml[^>]*encoding=["']([\w-]+)["']/i.exec(head)?.[1] ?? + "utf-8"; + try { + return new TextDecoder(label).decode(bytes); + } catch { + return new TextDecoder("utf-8").decode(bytes); + } +} + +export interface RssFetcher { + fetchFeed(url: string, state?: FeedFetchState, opts?: FetchFeedOptions): Promise; +} + +export function createRssFetcher(options: RssFetcherOptions = {}): RssFetcher { + const doFetch = options.fetch ?? globalThis.fetch; + const now = options.now ?? (() => new Date()); + const timeoutMs = options.timeoutMs ?? 30_000; + const maxBytes = options.maxBytes ?? 50 * 1024 * 1024; + const backoff = options.backoff ?? { baseMs: 60_000, maxMs: 6 * 3600_000 }; + const threshold = options.circuitThreshold ?? 5; + const cooldown = options.circuitCooldownMs ?? null; + + function fail(state: FeedFetchState, error: FeedError): FeedFetchResult { + const at = now(); + const failures = state.consecutiveFailures + 1; + const wait = Math.max(computeBackoffMs(failures, backoff.baseMs, backoff.maxMs), error.retryAfterMs ?? 0); + const next: FeedFetchState = { + ...state, + consecutiveFailures: failures, + nextAttemptAt: new Date(at.getTime() + wait).toISOString(), + lastError: { + kind: error.kind, + message: error.message, + at: at.toISOString(), + ...(error.httpStatus !== undefined ? { httpStatus: error.httpStatus } : {}), + }, + }; + if (failures >= threshold) { + next.circuit = "open"; + next.circuitOpenedAt = at.toISOString(); + } + return { kind: "error", error, state: next }; + } + + async function fetchFeed( + url: string, + state: FeedFetchState = initialFeedState(), + opts: FetchFeedOptions = {}, + ): Promise { + const at = now(); + if (state.circuit === "open") { + const openedAt = state.circuitOpenedAt ? Date.parse(state.circuitOpenedAt) : 0; + const halfOpen = cooldown !== null && at.getTime() - openedAt >= cooldown; + if (!halfOpen) { + return { + kind: "skipped", + reason: "circuit_open", + ...(cooldown !== null ? { retryAt: new Date(openedAt + cooldown).toISOString() } : {}), + state, + }; + } + } else if (!opts.ignoreBackoff && state.nextAttemptAt && Date.parse(state.nextAttemptAt) > at.getTime()) { + return { kind: "skipped", reason: "backoff", retryAt: state.nextAttemptAt, state }; + } + + const safeUrl = redactUrl(url); + const headers: Record = { + Accept: "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.8, */*;q=0.1", + }; + if (options.userAgent) headers["User-Agent"] = options.userAgent; + if (state.etag) headers["If-None-Match"] = state.etag; + if (state.lastModified) headers["If-Modified-Since"] = state.lastModified; + + const timeout = AbortSignal.timeout(timeoutMs); + const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout; + let res: Response; + try { + res = await doFetch(url, { headers, signal, redirect: "follow" }); + } catch (e) { + if (opts.signal?.aborted) throw e; + const err = e as Error; + const kind = timeout.aborted ? "timeout" : "network"; + return fail(state, new FeedError(kind, redactText(`请求 ${safeUrl} 失败:${err.message}`))); + } + + if (res.status === 304) { + await res.body?.cancel().catch(() => {}); + const next: FeedFetchState = { + ...state, + consecutiveFailures: 0, + circuit: "closed", + lastSuccessAt: at.toISOString(), + }; + delete next.nextAttemptAt; + delete next.circuitOpenedAt; + return { kind: "not_modified", state: next }; + } + if (!res.ok) { + await res.body?.cancel().catch(() => {}); + const retryAfterMs = parseRetryAfter(res.headers.get("retry-after"), at); + const kind = + res.status === 429 || (res.status === 503 && retryAfterMs !== undefined) ? "rate_limited" : "http_error"; + return fail( + state, + new FeedError(kind, `请求 ${safeUrl} 返回 HTTP ${res.status}`, { + httpStatus: res.status, + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), + }), + ); + } + + try { + const bytes = await readCapped(res, maxBytes); + const text = decodeBody(bytes, res.headers.get("content-type")); + const feed = parseFeed(text, { maxItems: options.maxItems ?? 1000, maxChars: maxBytes, fetchedAt: at }); + const next: FeedFetchState = { + ...state, + consecutiveFailures: 0, + circuit: "closed", + lastSuccessAt: at.toISOString(), + }; + delete next.nextAttemptAt; + delete next.circuitOpenedAt; + const etag = res.headers.get("etag"); + const lastModified = res.headers.get("last-modified"); + if (etag) next.etag = etag; + if (lastModified) next.lastModified = lastModified; + return { kind: "ok", feed, state: next, httpStatus: res.status }; + } catch (e) { + if (opts.signal?.aborted) throw e; + if (e instanceof FeedError) return fail(state, new FeedError(e.kind, redactText(e.message))); + const kind = timeout.aborted ? "timeout" : "network"; + return fail(state, new FeedError(kind, redactText(`读取 ${safeUrl} 失败:${(e as Error).message}`))); + } + } + + return { fetchFeed }; +} diff --git a/packages/adapters/src/rss/index.ts b/packages/adapters/src/rss/index.ts new file mode 100644 index 00000000..b4e12d55 --- /dev/null +++ b/packages/adapters/src/rss/index.ts @@ -0,0 +1,29 @@ +export { type ApplyFeedOptions, type ApplyFeedResult, applyFeedItems, emptyFeedCursor } from "./cursor"; +export { + computeBackoffMs, + createRssFetcher, + type FetchFeedOptions, + initialFeedState, + parseRetryAfter, + type RssFetcher, + type RssFetcherOptions, + resetFeedCircuit, +} from "./fetch"; +export { + assertNoDtdEntities, + decodeXmlEntities, + type ParseFeedOptions, + parseFeed, + RSS_EXPIRED_MESSAGES, +} from "./parse"; +export { extractSecretParams, REDACTED, redactText, redactUrl, SENSITIVE_QUERY_KEYS, unredactUrl } from "./redact"; +export { buildRssLinkRequest, extractRssLink, RSS_LINK_CONF, type RssLinkConf, rssSiteDomain } from "./rss-link"; +export { + type FeedCursor, + FeedError, + type FeedErrorKind, + type FeedFetchResult, + type FeedFetchState, + type FeedItem, + type ParsedFeed, +} from "./types"; diff --git a/packages/adapters/src/rss/parse.ts b/packages/adapters/src/rss/parse.ts new file mode 100644 index 00000000..ed59df87 --- /dev/null +++ b/packages/adapters/src/rss/parse.ts @@ -0,0 +1,271 @@ +import { createHash } from "node:crypto"; +import { XMLParser } from "fast-xml-parser"; +import { redactUrl } from "./redact"; +import { FeedError, type FeedItem, type ParsedFeed } from "./types"; + +export interface ParseFeedOptions { + /** 条目上限,超出截断并置 truncated(默认 1000,同 MP) */ + maxItems?: number; + /** 文本长度上限(字符,默认 50 MiB,同 MP) */ + maxChars?: number; + /** 抓取时间,默认 now */ + fetchedAt?: Date; +} + +/** MP RssHelper.MAX_RSS_ITEMS */ +const DEFAULT_MAX_ITEMS = 1000; +/** MP RssHelper.MAX_RSS_SIZE */ +const DEFAULT_MAX_CHARS = 50 * 1024 * 1024; + +/** + * 预检:拒绝内部 DTD 子集与任何 ENTITY 声明(XXE / billion laughs)。 + * 仅外部 DOCTYPE 声明(如 RSS 0.91 的 PUBLIC 标识)无子集时放行,但解析器同样不会去取。 + */ +export function assertNoDtdEntities(xml: string): void { + if (/[]*(\[)?/i.exec(xml); + if (doctype?.[1]) throw new FeedError("forbidden_dtd", "XML 含内部 DTD 子集,已拒绝"); +} + +const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "@_", + removeNSPrefix: true, + parseTagValue: false, + parseAttributeValue: false, + trimValues: true, + processEntities: false, + htmlEntities: false, + cdataPropName: "__cdata", + ignoreDeclaration: true, + ignorePiTags: true, + isArray: (name) => ["item", "entry", "category", "link", "enclosure"].includes(name), +}); + +const XML_ENTITIES: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" }; + +/** 只解码 XML 预定义实体与数字字符引用;不做任何自定义实体展开。 */ +export function decodeXmlEntities(s: string): string { + return s.replace(/&(#x[0-9a-f]+|#[0-9]+|[a-z]+);/gi, (m, body: string) => { + if (body[0] === "#") { + const cp = body[1] === "x" || body[1] === "X" ? Number.parseInt(body.slice(2), 16) : Number(body.slice(1)); + return Number.isFinite(cp) && cp > 0 && cp <= 0x10ffff ? String.fromCodePoint(cp) : m; + } + return XML_ENTITIES[body.toLowerCase()] ?? m; + }); +} + +type XNode = unknown; + +function textOf(node: XNode): string { + if (node == null) return ""; + if (typeof node === "string") return decodeXmlEntities(node); + if (typeof node === "number" || typeof node === "boolean") return String(node); + if (Array.isArray(node)) return node.map(textOf).join(""); + if (typeof node === "object") { + const o = node as Record; + const cdata = o.__cdata; + const text = o["#text"]; + return `${text != null ? textOf(text) : ""}${cdata != null ? (Array.isArray(cdata) ? cdata.join("") : String(cdata)) : ""}`.trim(); + } + return ""; +} + +function attr(node: XNode, name: string): string | undefined { + if (node && typeof node === "object" && !Array.isArray(node)) { + const v = (node as Record)[`@_${name}`]; + return v == null ? undefined : decodeXmlEntities(String(v)); + } + return undefined; +} + +function asArray(v: XNode): XNode[] { + if (v == null) return []; + return Array.isArray(v) ? v : [v]; +} + +function toIso(s: string | undefined): string | undefined { + if (!s) return undefined; + const t = Date.parse(s); + return Number.isNaN(t) ? undefined : new Date(t).toISOString(); +} + +function looksLikeUrl(s: string): boolean { + return /^[a-z][a-z0-9+.-]*:/i.test(s); +} + +function sha256(s: string): string { + return createHash("sha256").update(s).digest("hex"); +} + +interface RawFields { + guid: string; + title: string; + link: string; + enclosure: string; + enclosureType?: string; + size: number; + published?: string; + categories: string[]; + description?: string; + nickname?: string; +} + +function buildItem(raw: RawFields, fetchedAt: string): FeedItem { + // 摘要基于脱敏后的值(避免 passkey 轮换误判为更新) + const enclosureRedacted = raw.enclosure ? redactUrl(raw.enclosure) : undefined; + const linkRedacted = raw.link ? redactUrl(raw.link) : null; + const guidBase = raw.guid || linkRedacted || enclosureRedacted || ""; + const guid = guidBase + ? looksLikeUrl(guidBase) + ? redactUrl(guidBase) + : guidBase + : `digest:${sha256(`${raw.title}\u0000${raw.published ?? ""}`).slice(0, 32)}`; + const rawDigest = sha256( + JSON.stringify([ + guid, + raw.title, + linkRedacted, + enclosureRedacted ?? null, + raw.size, + raw.published ?? null, + [...raw.categories].sort(), + raw.description ?? null, + ]), + ); + const item: FeedItem = { + guid, + title: raw.title, + link: linkRedacted, + size: raw.size, + categories: raw.categories, + rawDigest, + fetchedAt, + }; + if (enclosureRedacted) { + item.enclosureUrl = enclosureRedacted; + item.enclosureHasSecret = enclosureRedacted !== raw.enclosure; + } + if (raw.enclosureType) item.enclosureType = raw.enclosureType; + const publishedAt = toIso(raw.published); + if (publishedAt) item.publishedAt = publishedAt; + if (raw.description) item.description = raw.description; + if (raw.nickname) item.nickname = raw.nickname; + return item; +} + +/** + * 单条 item/entry → 字段。迁移自 MP `RssHelper._parse_impl` 的条目循环: + * title(空则跳过)、description|summary、link(文本或 href)、enclosure@url(缺失回退 link,两者皆无跳过)、 + * enclosure@length(纯数字才取)、pubDate|published|updated、creator(nickname)。 + * guid/categories 为本项目扩展(去重与整理视图需要)。 + */ +function itemFields(it: Record): RawFields | null { + const title = textOf(asArray(it.title)[0]); + if (!title) return null; + const description = textOf(asArray(it.description)[0]) || textOf(asArray(it.summary)[0]); + const firstLink = asArray(it.link)[0]; + const link = textOf(firstLink) || attr(firstLink, "href") || ""; + const encNode = asArray(it.enclosure)[0]; + let enclosure = attr(encNode, "url") ?? ""; + if (!enclosure && !link) return null; + if (!enclosure && link) enclosure = link; + const lengthAttr = attr(encNode, "length") ?? "0"; + const size = /^\d+$/.test(lengthAttr) ? Number(lengthAttr) : 0; + const published = + textOf(asArray(it.pubDate)[0]) || textOf(asArray(it.published)[0]) || textOf(asArray(it.updated)[0]); + const nickname = textOf(asArray(it.creator)[0]); + const raw: RawFields = { + guid: textOf(asArray(it.guid)[0]) || textOf(asArray(it.id)[0]), + title, + link, + enclosure, + size, + categories: asArray(it.category) + .map((c) => attr(c, "term") ?? textOf(c)) + .filter((c) => c.length > 0), + }; + const encType = attr(encNode, "type"); + if (encType) raw.enclosureType = encType; + if (published) raw.published = published; + if (description) raw.description = description; + if (nickname) raw.nickname = nickname; + return raw; +} + +/** MP xpath `.//item | .//entry`:在整棵树中按文档顺序收集条目节点(深度受限)。 */ +function collectItems(node: XNode, out: Array>, depth = 0): void { + if (depth > 12 || node == null || typeof node !== "object") return; + if (Array.isArray(node)) { + for (const n of node) collectItems(n, out, depth + 1); + return; + } + for (const [k, v] of Object.entries(node as Record)) { + if (k.startsWith("@_") || k === "#text" || k === "__cdata") continue; + if (k === "item" || k === "entry") { + for (const it of asArray(v)) if (it && typeof it === "object") out.push(it as Record); + } else { + collectItems(v, out, depth + 1); + } + } +} + +/** MP `_rss_expired_msg`:站点返回的 RSS 过期提示(MP 返回 None,本项目映射为 FeedError expired)。 */ +export const RSS_EXPIRED_MESSAGES = [ + "RSS 链接已过期, 您需要获得一个新的!", + "RSS Link has expired, You need to get a new one!", + "RSS Link has expired, You need to get new!", +]; + +/** + * 安全解析 RSS 2.0 / RSS 1.0(RDF) / Atom。 + * 迁移自 MP `app/application/rss.py RssHelper._parse_impl`(MAX_RSS_SIZE 50MB、MAX_RSS_ITEMS 1000、 + * `resolve_entities=False`、非 `<` 开头拒绝、过期提示识别);本项目额外拒绝 DTD 实体声明并返回脱敏字段。 + */ +export function parseFeed(xml: string, opts: ParseFeedOptions = {}): ParsedFeed { + const maxChars = opts.maxChars ?? DEFAULT_MAX_CHARS; + const maxItems = opts.maxItems ?? DEFAULT_MAX_ITEMS; + if (xml.length > maxChars) throw new FeedError("too_large", `Feed 文本 ${xml.length} 字符超过上限 ${maxChars}`); + const trimmed = xml.trim(); + if (!trimmed) throw new FeedError("not_feed", "RSS 内容为空"); + if (RSS_EXPIRED_MESSAGES.includes(trimmed)) throw new FeedError("expired", "RSS 链接已过期,需要重新获取"); + if (!trimmed.startsWith("<")) throw new FeedError("not_feed", "RSS 内容不是有效的 XML 格式"); + assertNoDtdEntities(xml); + let doc: Record; + try { + doc = parser.parse(xml) as Record; + } catch (e) { + throw new FeedError("parse_error", `XML 解析失败:${(e as Error).message.slice(0, 200)}`); + } + const fetchedAt = (opts.fetchedAt ?? new Date()).toISOString(); + const rss = doc.rss as Record | undefined; + const rdf = doc.RDF as Record | undefined; + const atom = doc.feed as Record | undefined; + let format: ParsedFeed["format"]; + let title: string | null; + if (rss && typeof rss === "object") { + format = "rss"; + title = textOf((rss.channel as Record | undefined)?.title) || null; + } else if (rdf && typeof rdf === "object") { + format = "rdf"; + title = textOf((rdf.channel as Record | undefined)?.title) || null; + } else if (atom && typeof atom === "object") { + format = "atom"; + title = textOf(atom.title) || null; + } else { + throw new FeedError("not_feed", "内容不是 RSS/Atom Feed"); + } + const nodes: Array> = []; + collectItems(doc, nodes); + const truncated = nodes.length > maxItems; + const items: FeedItem[] = []; + for (const n of nodes.slice(0, maxItems)) { + try { + const raw = itemFields(n); + if (raw) items.push(buildItem(raw, fetchedAt)); + } catch { + // MP:单条解析失败跳过 + } + } + return { format, title, items, truncated }; +} diff --git a/packages/adapters/src/rss/redact.ts b/packages/adapters/src/rss/redact.ts new file mode 100644 index 00000000..1b331fb4 --- /dev/null +++ b/packages/adapters/src/rss/redact.ts @@ -0,0 +1,104 @@ +/** + * URL/文本脱敏。含 passkey 的 URL 不得进入日志、错误信息、模型上下文(REQ §4.1)。 + * 规则:query 中凭据类参数值替换为 [REDACTED],URL userinfo 一并抹除。 + */ + +export const REDACTED = "[REDACTED]"; + +/** 视为凭据的 query 参数名(小写比较)。 */ +export const SENSITIVE_QUERY_KEYS: ReadonlySet = new Set([ + "passkey", + "token", + "authkey", + "auth", + "torrent_pass", + "rsskey", + "apikey", + "api_key", + "access_token", + "secret", + "sign", + "downhash", + "credential", + "key", +]); + +function isSensitiveKey(key: string, extra?: Iterable): boolean { + const k = key.toLowerCase(); + if (SENSITIVE_QUERY_KEYS.has(k)) return true; + if (extra) for (const e of extra) if (e.toLowerCase() === k) return true; + return false; +} + +/** + * 把 URL 中的凭据参数替换为 [REDACTED]。输入不是合法绝对 URL 时按原文做正则兜底,不抛错。 + * 输出是确定性的(同一 URL 永远得到同一结果),可直接用作去重键。 + */ +export function redactUrl(input: string, extraKeys?: Iterable): string { + let url: URL; + try { + url = new URL(input); + } catch { + return redactText(input, extraKeys); + } + if (url.username || url.password) { + url.username = ""; + url.password = ""; + } + // 手工重建 query,避免 URLSearchParams 把 [REDACTED] 转义成 %5BREDACTED%5D + if (url.search) { + const parts = url.search + .slice(1) + .split("&") + .map((pair) => { + const eq = pair.indexOf("="); + const rawKey = eq === -1 ? pair : pair.slice(0, eq); + let key = rawKey; + try { + key = decodeURIComponent(rawKey.replace(/\+/g, " ")); + } catch { + // 保留原样 + } + return eq !== -1 && isSensitiveKey(key, extraKeys) ? `${rawKey}=${REDACTED}` : pair; + }); + const out = `${url.origin}${url.pathname}?${parts.join("&")}${url.hash}`; + return url.protocol === "magnet:" ? `magnet:?${parts.join("&")}` : out; + } + return url.toString(); +} + +const QUERY_PARAM_RE = /([?&;])([A-Za-z0-9_.-]+)=([^&#\s"'<>]*)/g; +const USERINFO_RE = /\b([a-z][a-z0-9+.-]*:\/\/)[^/\s@:]+(?::[^/\s@]*)?@/gi; + +/** 对任意文本(错误消息、日志行)中出现的 URL 凭据参数与 userinfo 做脱敏。 */ +export function redactText(text: string, extraKeys?: Iterable): string { + return text + .replace(USERINFO_RE, "$1") + .replace(QUERY_PARAM_RE, (m, sep: string, key: string) => + isSensitiveKey(key, extraKeys) ? `${sep}${key}=${REDACTED}` : m, + ); +} + +/** 从 URL 中取出被脱敏的凭据参数(调用方自行交给 secret store;本模块不持久化)。 */ +export function extractSecretParams(input: string, extraKeys?: Iterable): Record { + const out: Record = {}; + try { + const url = new URL(input); + for (const [k, v] of url.searchParams) if (isSensitiveKey(k, extraKeys)) out[k] = v; + } catch { + // 非 URL 返回空 + } + return out; +} + +/** + * 把脱敏 URL 中的 [REDACTED] 还原为真实凭据,只在即将发出请求时于内存中调用,结果不得记录。 + * 找不到对应 secret 的参数保持 [REDACTED],调用方应视为不可用。 + */ +export function unredactUrl(redacted: string, secrets: Record): string { + const lower = new Map(Object.entries(secrets).map(([k, v]) => [k.toLowerCase(), v])); + return redacted.replace(/([?&;])([A-Za-z0-9_.-]+)=\[REDACTED\]/g, (m, sep: string, key: string) => { + const v = lower.get(key.toLowerCase()); + return v === undefined ? m : `${sep}${key}=${encodeURIComponent(v)}`; + }); +} diff --git a/packages/adapters/src/rss/rss-link.ts b/packages/adapters/src/rss/rss-link.ts new file mode 100644 index 00000000..5fd158a4 --- /dev/null +++ b/packages/adapters/src/rss/rss-link.ts @@ -0,0 +1,60 @@ +/** + * 站点 RSS 地址获取。迁移自 MP e6820a8 `app/application/rss.py`: + * `RssHelper.rss_link_conf`(仅 default 与 audiences.me 两项,v0.1 范围)、`_get_site_domain`、`get_rss_link`。 + * MP 用 lxml xpath `//a[@class='faqlink']/@href` 取最后一个;这里用等价 CSS `a.faqlink` + href。 + * 请求本身由调用方(站点适配器)带 Cookie 发送并走站点限流;本模块只构造请求与解析结果。 + */ +import { load } from "cheerio"; + +export interface RssLinkConf { + /** 等价于 MP xpath 的 CSS 选择器与属性 */ + selector: string; + attribute: string; + url: string; + params: Record; +} + +export const RSS_LINK_CONF: Record = { + default: { + selector: "a.faqlink", + attribute: "href", + url: "getrss.php", + params: { inclbookmarked: 0, itemsmalldescr: 1, showrows: 50, search_mode: 1 }, + }, + "audiences.me": { + selector: "a.faqlink", + attribute: "href", + url: "getrss.php", + params: { inclbookmarked: 0, itemsmalldescr: 1, showrows: 50, search_mode: 1, torrent_type: 1, exp: 180 }, + }, +}; + +/** MP `_get_site_domain`:匹配配置域名,否则回退最后两级域名。 */ +export function rssSiteDomain(url: string): string { + const host = new URL(url).hostname.toLowerCase(); + for (const d of Object.keys(RSS_LINK_CONF)) { + if (d !== "default" && (host === d || host.endsWith(`.${d}`))) return d; + } + const parts = host.split("."); + return parts.length >= 2 ? parts.slice(-2).join(".") : host; +} + +/** 构造 getrss.php 的 POST 请求(表单参数) */ +export function buildRssLinkRequest(siteUrl: string): { url: string; form: Record } { + const conf = RSS_LINK_CONF[rssSiteDomain(siteUrl)] ?? RSS_LINK_CONF.default!; + return { + url: new URL(conf.url, siteUrl.endsWith("/") ? siteUrl : `${siteUrl}/`).toString(), + form: Object.fromEntries(Object.entries(conf.params).map(([k, v]) => [k, String(v)])), + }; +} + +/** 从 getrss.php 返回页提取 RSS 地址(MP 取匹配结果的最后一个)。返回值含 passkey,调用方须当作凭据保存。 */ +export function extractRssLink(siteUrl: string, html: string): string | null { + const conf = RSS_LINK_CONF[rssSiteDomain(siteUrl)] ?? RSS_LINK_CONF.default!; + const $ = load(html); + const links = $(conf.selector) + .map((_, el) => $(el).attr(conf.attribute)) + .get() + .filter((x): x is string => Boolean(x)); + return links.at(-1) ?? null; +} diff --git a/packages/adapters/src/rss/rss.test.ts b/packages/adapters/src/rss/rss.test.ts new file mode 100644 index 00000000..3fdd0619 --- /dev/null +++ b/packages/adapters/src/rss/rss.test.ts @@ -0,0 +1,296 @@ +import { createMockRssServer, type MockFeedItem, type MockRssServer } from "@umt/testkit"; +import { afterEach, describe, expect, it } from "vitest"; +import { + applyFeedItems, + buildRssLinkRequest, + computeBackoffMs, + createRssFetcher, + emptyFeedCursor, + extractRssLink, + FeedError, + type FeedFetchResult, + initialFeedState, + parseFeed, + parseRetryAfter, + redactText, + redactUrl, + resetFeedCircuit, + unredactUrl, +} from "./index"; + +const PASSKEY = "abcdef0123456789secret"; + +function sample(n: number, host = "https://pt.example"): MockFeedItem[] { + return Array.from({ length: n }, (_, i) => ({ + guid: `${host}/details.php?id=${1000 + i}`, + title: `Show.S01E${String(i + 1).padStart(2, "0")}.2160p.WEB-DL.H265-GRP`, + link: `${host}/details.php?id=${1000 + i}`, + enclosureUrl: `${host}/download.php?id=${1000 + i}&passkey=${PASSKEY}`, + size: 1_000_000 * (i + 1), + pubDate: new Date(Date.UTC(2026, 8, 20, 12, i)), + categories: ["402"], + })); +} + +function okFeed(r: FeedFetchResult) { + if (r.kind !== "ok") throw new Error(`expected ok, got ${r.kind}`); + return r.feed; +} + +let server: MockRssServer | undefined; +afterEach(async () => { + await server?.close(); + server = undefined; +}); + +describe("parseFeed 安全解析", () => { + it("解析 RSS 2.0 字段并脱敏 enclosure 中的 passkey", () => { + const xml = `T & F + Dune & Part.Two.2024.2160phttps://pt.example/details.php?id=1 + https://pt.example/details.php?id=1 + + Sat, 26 Sep 2026 13:14:03 GMT401电影 + `; + const feed = parseFeed(xml, { fetchedAt: new Date("2026-09-27T00:00:00Z") }); + expect(feed.format).toBe("rss"); + expect(feed.title).toBe("T & F"); + const item = feed.items[0]!; + expect(item.title).toBe("Dune & Part.Two.2024.2160p"); + expect(item.guid).toBe("https://pt.example/details.php?id=1"); + expect(item.enclosureUrl).toBe("https://pt.example/download.php?id=1&passkey=[REDACTED]"); + expect(item.enclosureHasSecret).toBe(true); + expect(item.size).toBe(84202833838); + expect(item.publishedAt).toBe("2026-09-26T13:14:03.000Z"); + expect(item.categories).toEqual(["401", "电影"]); + expect(item.rawDigest).toMatch(/^[0-9a-f]{64}$/); + expect(item.fetchedAt).toBe("2026-09-27T00:00:00.000Z"); + expect(JSON.stringify(feed)).not.toContain(PASSKEY); + }); + + it("解析 Atom(MP 语义:enclosure 缺失回退首个 link)", () => { + const xml = `A + urn:x:1X + 2026-09-01T00:00:00Z + `; + const feed = parseFeed(xml); + expect(feed.format).toBe("atom"); + expect(feed.items[0]).toMatchObject({ + guid: "urn:x:1", + link: "https://a/1", + enclosureUrl: "https://a/1", + size: 0, + categories: ["tv"], + }); + }); + + it("拒绝外部实体(XXE)", () => { + const xml = `]> + &xxe;`; + expect(() => parseFeed(xml)).toThrowError(FeedError); + try { + parseFeed(xml); + } catch (e) { + expect((e as FeedError).kind).toBe("forbidden_dtd"); + } + }); + + it("拒绝实体膨胀(billion laughs)与内部 DTD 子集", () => { + const lol = `]>&lol2;`; + expect(() => parseFeed(lol)).toThrow(/ENTITY/); + const subset = ` ]>`; + expect(() => parseFeed(subset)).toThrow(/DTD/); + }); + + it("条目数超过上限截断,非 Feed 报 not_feed", () => { + const xml = `${Array.from({ length: 20 }, (_, i) => `g${i}t${i}https://x/${i}`).join("")}`; + const feed = parseFeed(xml, { maxItems: 5 }); + expect(feed.items).toHaveLength(5); + expect(feed.truncated).toBe(true); + expect(() => parseFeed("hi")).toThrow(/不是 RSS/); + }); +}); + +describe("MP RssHelper 条目语义", () => { + it("无标题跳过、无 enclosure 且无 link 跳过、creator → nickname、过期提示识别", () => { + const xml = ` + https://x/1 + no link + okhttps://x/3豆友Sat, 26 Sep 2026 13:14:03 GMT + `; + const feed = parseFeed(xml); + expect(feed.items).toHaveLength(1); + expect(feed.items[0]).toMatchObject({ title: "ok", enclosureUrl: "https://x/3", nickname: "豆友", size: 0 }); + expect(() => parseFeed("RSS Link has expired, You need to get a new one!")).toThrow(/过期/); + expect(() => parseFeed("not xml")).toThrow(/不是有效的 XML/); + }); +}); + +describe("站点 RSS 地址获取(MP get_rss_link)", () => { + it("观众使用专用参数,默认站点回退 default;取最后一个 faqlink", () => { + const req = buildRssLinkRequest("https://audiences.me/"); + expect(req.url).toBe("https://audiences.me/getrss.php"); + expect(req.form).toMatchObject({ torrent_type: "1", exp: "180", showrows: "50" }); + expect(buildRssLinkRequest("https://pt.example.org").form.torrent_type).toBeUndefined(); + const html = `12`; + expect(extractRssLink("https://audiences.me/", html)).toContain("torrentrss.php"); + }); +}); + +describe("redact", () => { + it("redactUrl/redactText/unredactUrl", () => { + const u = `https://user:pw@pt.example/rss.php?rows=50&passkey=${PASSKEY}&linktype=dl`; + const r = redactUrl(u); + expect(r).toBe("https://pt.example/rss.php?rows=50&passkey=[REDACTED]&linktype=dl"); + expect(redactText(`fetch failed for ${u}: ECONNRESET`)).not.toContain(PASSKEY); + expect(redactText(`fetch failed for ${u}`)).not.toContain("pw@"); + expect(unredactUrl(r, { passkey: PASSKEY })).toBe( + `https://pt.example/rss.php?rows=50&passkey=${PASSKEY}&linktype=dl`, + ); + expect(redactUrl("https://api.m-team.cc/api/rss/dlv2?sign=abc&t=1&tid=9")).toContain("sign=[REDACTED]"); + }); +}); + +describe("createRssFetcher + mock RSS", () => { + it("ETag/Last-Modified 条件请求命中 304", async () => { + server = await createMockRssServer({ items: sample(3), passkey: PASSKEY }); + const fetcher = createRssFetcher(); + const r1 = await fetcher.fetchFeed(server.feedUrl); + expect(okFeed(r1).items).toHaveLength(3); + expect(r1.state.etag).toBeTruthy(); + const r2 = await fetcher.fetchFeed(server.feedUrl, r1.state); + expect(r2.kind).toBe("not_modified"); + expect(server.notModifiedCount).toBe(1); + expect(server.requests[1]!.headers["if-none-match"]).toBe(r1.state.etag); + server.addItem(sample(1, "https://other.example")[0]!); + const r3 = await fetcher.fetchFeed(server.feedUrl, r2.state); + expect(okFeed(r3).items).toHaveLength(4); + }); + + it("5xx + Retry-After:退避窗口取 max(指数退避, Retry-After),窗口内跳过", async () => { + server = await createMockRssServer({ items: sample(2) }); + let now = new Date("2026-09-27T00:00:00Z"); + const fetcher = createRssFetcher({ now: () => now, backoff: { baseMs: 10_000, maxMs: 3600_000 } }); + server.failNext(1, { status: 503, retryAfter: "120" }); + const r1 = await fetcher.fetchFeed(server.feedUrl); + expect(r1.kind).toBe("error"); + if (r1.kind !== "error") return; + expect(r1.error.kind).toBe("rate_limited"); + expect(r1.error.retryAfterMs).toBe(120_000); + expect(r1.state.nextAttemptAt).toBe("2026-09-27T00:02:00.000Z"); + const r2 = await fetcher.fetchFeed(server.feedUrl, r1.state); + expect(r2).toMatchObject({ kind: "skipped", reason: "backoff" }); + expect(server.requests).toHaveLength(1); + now = new Date("2026-09-27T00:02:01Z"); + const r3 = await fetcher.fetchFeed(server.feedUrl, r2.state); + expect(r3.kind).toBe("ok"); + expect(r3.state.consecutiveFailures).toBe(0); + expect(r3.state.nextAttemptAt).toBeUndefined(); + }); + + it("computeBackoffMs 指数增长封顶,parseRetryAfter 支持 HTTP-date", () => { + expect([1, 2, 3, 4].map((n) => computeBackoffMs(n, 1000, 5000))).toEqual([1000, 2000, 4000, 5000]); + const now = new Date("2026-09-27T00:00:00Z"); + expect(parseRetryAfter("Sun, 27 Sep 2026 00:00:30 GMT", now)).toBe(30_000); + expect(parseRetryAfter("garbage", now)).toBeUndefined(); + }); + + it("连续失败 N 次进入熔断,手动恢复后可继续抓取", async () => { + server = await createMockRssServer({ items: sample(1) }); + const fetcher = createRssFetcher({ circuitThreshold: 3, backoff: { baseMs: 0, maxMs: 0 } }); + server.failNext(3, { status: 500 }); + let state = initialFeedState(); + for (let i = 0; i < 3; i++) state = (await fetcher.fetchFeed(server.feedUrl, state)).state; + expect(state.circuit).toBe("open"); + expect(state.consecutiveFailures).toBe(3); + const skipped = await fetcher.fetchFeed(server.feedUrl, state, { ignoreBackoff: true }); + expect(skipped).toMatchObject({ kind: "skipped", reason: "circuit_open" }); + expect(server.requests).toHaveLength(3); + const ok = await fetcher.fetchFeed(server.feedUrl, resetFeedCircuit(state)); + expect(ok.kind).toBe("ok"); + expect(ok.state.circuit).toBe("closed"); + }); + + it("熔断冷却后半开探测", async () => { + server = await createMockRssServer({ items: sample(1) }); + let now = new Date("2026-09-27T00:00:00Z"); + const fetcher = createRssFetcher({ now: () => now, circuitThreshold: 1, circuitCooldownMs: 60_000 }); + server.failNext(1, { status: 502 }); + const r1 = await fetcher.fetchFeed(server.feedUrl); + expect(r1.state.circuit).toBe("open"); + expect((await fetcher.fetchFeed(server.feedUrl, r1.state)).kind).toBe("skipped"); + now = new Date("2026-09-27T00:01:00Z"); + expect((await fetcher.fetchFeed(server.feedUrl, r1.state)).kind).toBe("ok"); + }); + + it("错误信息与状态中不含 passkey(HTTP 错误与网络错误)", async () => { + server = await createMockRssServer({ items: sample(1), passkey: "the-right-one" }); + const fetcher = createRssFetcher(); + const wrong = `${server.baseUrl}/rss.xml?passkey=${PASSKEY}`; + const r = await fetcher.fetchFeed(wrong); + expect(r.kind).toBe("error"); + expect(JSON.stringify(r)).not.toContain(PASSKEY); + if (r.kind === "error") expect(r.error.message).toContain("[REDACTED]"); + const dead = `http://127.0.0.1:1/rss.xml?passkey=${PASSKEY}`; + const r2 = await fetcher.fetchFeed(dead); + expect(r2.kind).toBe("error"); + expect(JSON.stringify(r2)).not.toContain(PASSKEY); + }); + + it("响应体超过上限拒绝;恶意 DTD 响应计为失败", async () => { + server = await createMockRssServer({ items: sample(1) }); + server.setRawBody(`${"x".repeat(10_000)}`); + const fetcher = createRssFetcher({ maxBytes: 1024 }); + const r = await fetcher.fetchFeed(server.feedUrl); + expect(r.kind === "error" && r.error.kind).toBe("too_large"); + server.setRawBody(`]>`); + const r2 = await createRssFetcher().fetchFeed(server.feedUrl); + expect(r2.kind === "error" && r2.error.kind).toBe("forbidden_dtd"); + expect(r2.state.consecutiveFailures).toBe(1); + }); +}); + +describe("重叠窗口去重", () => { + it("乱序/置顶不产生新条目,同 GUID 内容变化视为更新", async () => { + server = await createMockRssServer({ items: sample(5) }); + server.setConditional(false); + const fetcher = createRssFetcher(); + const a = applyFeedItems(emptyFeedCursor(), okFeed(await fetcher.fetchFeed(server.feedUrl)).items); + expect(a.newItems).toHaveLength(5); + server.shuffle(7); + server.pinItem("https://pt.example/details.php?id=1003"); + const b = applyFeedItems(a.cursor, okFeed(await fetcher.fetchFeed(server.feedUrl)).items); + expect(b.newItems).toHaveLength(0); + expect(b.updatedItems).toHaveLength(0); + expect(b.unchangedCount).toBe(5); + server.updateItem("https://pt.example/details.php?id=1001", { title: "Show.S01E02.REPACK.2160p.WEB-DL.H265-GRP" }); + server.addItem({ + guid: "https://pt.example/details.php?id=2000", + title: "New.Movie.2026.1080p", + link: "https://pt.example/details.php?id=2000", + }); + const c = applyFeedItems(b.cursor, okFeed(await fetcher.fetchFeed(server.feedUrl)).items); + expect(c.newItems.map((i) => i.guid)).toEqual(["https://pt.example/details.php?id=2000"]); + expect(c.updatedItems.map((i) => i.title)).toEqual(["Show.S01E02.REPACK.2160p.WEB-DL.H265-GRP"]); + expect(c.possibleGap).toBe(false); + expect(c.cursor.lastGuidSet).toHaveLength(6); + }); + + it("与上次窗口零重叠标记疑似缺口;窗口按上限裁剪", () => { + const mk = (g: string) => ({ + guid: g, + title: g, + link: null, + size: 0, + categories: [], + rawDigest: g, + fetchedAt: "2026-09-27T00:00:00.000Z", + }); + const a = applyFeedItems(emptyFeedCursor(), [mk("a"), mk("b"), mk("c")], { windowSize: 2 }); + expect(a.possibleGap).toBe(false); + expect(a.cursor.lastGuidSet).toEqual(["a", "b"]); + expect(Object.keys(a.cursor.digests)).toEqual(["a", "b"]); + const b = applyFeedItems(a.cursor, [mk("x"), mk("y")]); + expect(b.possibleGap).toBe(true); + expect(b.cursor.lastFetchedAt).toBe("2026-09-27T00:00:00.000Z"); + }); +}); diff --git a/packages/adapters/src/rss/types.ts b/packages/adapters/src/rss/types.ts new file mode 100644 index 00000000..2408ad20 --- /dev/null +++ b/packages/adapters/src/rss/types.ts @@ -0,0 +1,92 @@ +/** + * RSS/Atom 适配器自有类型。FeedItem 对应 MP `RssHelper._parse_impl` 返回 dict: + * title→title、enclosure→enclosureUrl、size→size、description→description、link→link、pubdate→publishedAt、nickname→nickname; + * guid/categories/rawDigest/fetchedAt 为本项目扩展(去重/增量)。 + */ +/** RSS/Atom 适配器自有类型(不依赖 @umt/contracts;W2 负责映射到 observation/feed_source)。 */ + +export interface FeedItem { + /** Feed 内稳定标识:guid → link → enclosure → 标题+时间摘要;URL 形态已脱敏 */ + guid: string; + title: string; + /** 详情页链接(已脱敏) */ + link: string | null; + /** MP `enclosure`:下载链接,缺失时回退 link(已脱敏,凭据参数为 [REDACTED];需要时用 unredactUrl 在内存中还原) */ + enclosureUrl?: string; + /** 原 enclosure 是否含凭据参数 */ + enclosureHasSecret?: boolean; + enclosureType?: string; + /** MP `size`:enclosure@length,缺失为 0 */ + size: number; + /** MP `nickname`(dc:creator) */ + nickname?: string; + /** 来源发布时间(UTC ISO),≠ 发现时间 */ + publishedAt?: string; + categories: string[]; + description?: string; + /** 原始条目规范化后的 sha256(同 GUID 摘要变化 = 条目更新) */ + rawDigest: string; + /** 本次抓取时间(UTC ISO) */ + fetchedAt: string; +} + +export interface ParsedFeed { + format: "rss" | "atom" | "rdf"; + title: string | null; + items: FeedItem[]; + /** 条目数超过上限被截断 */ + truncated: boolean; +} + +export type FeedErrorKind = + | "forbidden_dtd" + | "expired" + | "too_large" + | "not_feed" + | "parse_error" + | "http_error" + | "rate_limited" + | "network" + | "timeout"; + +/** 所有 message 在构造时已经脱敏。 */ +export class FeedError extends Error { + readonly kind: FeedErrorKind; + readonly httpStatus?: number; + readonly retryAfterMs?: number; + constructor(kind: FeedErrorKind, message: string, opts: { httpStatus?: number; retryAfterMs?: number } = {}) { + super(message); + this.name = "FeedError"; + this.kind = kind; + if (opts.httpStatus !== undefined) this.httpStatus = opts.httpStatus; + if (opts.retryAfterMs !== undefined) this.retryAfterMs = opts.retryAfterMs; + } +} + +/** 单个 Feed 的抓取状态;纯数据,由调用方持久化(feed_source 列或 JSON)。 */ +export interface FeedFetchState { + etag?: string; + lastModified?: string; + consecutiveFailures: number; + /** 退避截止(UTC ISO),之前的调度直接 skipped */ + nextAttemptAt?: string; + circuit: "closed" | "open"; + circuitOpenedAt?: string; + lastSuccessAt?: string; + lastError?: { kind: FeedErrorKind; message: string; at: string; httpStatus?: number }; +} + +export type FeedFetchResult = + | { kind: "ok"; feed: ParsedFeed; state: FeedFetchState; httpStatus: number } + | { kind: "not_modified"; state: FeedFetchState } + | { kind: "skipped"; reason: "circuit_open" | "backoff"; retryAt?: string; state: FeedFetchState } + | { kind: "error"; error: FeedError; state: FeedFetchState }; + +/** 重叠窗口游标:不依赖递增 ID,保存最近见过的 GUID 与内容摘要。 */ +export interface FeedCursor { + /** 最近见过的 GUID(新→旧),长度受窗口上限约束 */ + lastGuidSet: string[]; + /** guid → rawDigest,用于识别同 GUID 内容变化 */ + digests: Record; + lastFetchedAt: string | null; +} diff --git a/packages/adapters/src/sites/README.md b/packages/adapters/src/sites/README.md new file mode 100644 index 00000000..e9ed448d --- /dev/null +++ b/packages/adapters/src/sites/README.md @@ -0,0 +1,60 @@ +# adapters/sites + +PT 站点适配器,按 MoviePilot v3(提交 e6820a8)迁移:通用配置驱动 spider(观众等 NexusPHP 站)与馒头 M-Team API。**仅对 `@umt/testkit` 的 mock 站点验证,未连接真实站点;选择器与 API 字段均待真实账号核验。** + +## 模块与上游对照 + +| 本项目 | MoviePilot 上游 | 说明 | +| --- | --- | --- | +| `spider/schema.ts` `IndexerDefinition` | `SiteSpider.__init__` 读取的 indexer dict 键 + `app/application/site/sites.pyi` | 站点定义 schema(domain / search.paths / params / batch / browse / category / torrents.list / torrents.fields DSL);`umt` 为本项目扩展 | +| `spider/spider.ts` `SiteSpider` | `app/modules/indexer/spider/__init__.py` `SiteSpider` | `getSearchUrl`(=`__get_search_url`)、`parse`/`getInfo`(=`parse`/`get_info` 全部字段)、`filterText`/`safeQuery`、`isLoginOrPermissionPage`、`selectMediaCategories`、`resolveCategoryMediaType`、`parseSizeMp`(=`foundation/size.parse_size`) | +| `spider/template.ts` | jinja2.Template | nunjucks(Jinja 兼容)渲染 title/description/date 模板 | +| `spider/adapter.ts` | `IndexerModule.search_torrents/refresh_torrents/__search_check/__clear_search_text`、`submission.download_torrent` | 请求、Cookie 注入、登录失效识别、下载校验 | +| `spider/detail.ts` | —(MP 通用 spider 无详情页) | 本项目扩展,选择器在定义 `umt.detail` | +| `definitions/audiences.json` | MP 站点定义在加密 `user.sites.v3.bin`,不可读 | 按 MP schema 手写;字段结构参考 MovieClaw `audiences.yaml`(data-ref,未复制);加站只需新增 JSON 并在 `registry.ts` 登记 | +| `mteam/spider.ts` | `app/modules/indexer/spider/mtorrent.py` `MTorrentSpider` | `mteamSearchParams`(=`__get_params`)、`parseMteamResult`(=`__parse_result`)、系数表、标签表、分类组 | +| `mteam/adapter.ts` | `MTorrentSpider.search`、`__get_download_url` + `submission._resolve_indirect_download_url` | genDlToken(id 走 query)→ 临时直链 → 不带 Cookie GET | +| `mteam/client.ts` | `MTorrentSpider.__process_response` | 另加业务 code/message 检查(MP 只看 HTTP 200) | +| `rate-limit.ts` | `SitesHelper.check(domain)`(闭源流控) | 站点级令牌桶,list/search/detail/download 共享 | + +## 与 MP 的偏离(只限安全、数据保真与去重) + +1. **原始分类**:spider/馒头结果保留 MP 的 `category`(电影/电视剧/音乐/未知 投影),另加 `category_id` 原值;`SiteRelease.category` 用原值 + 站点分类名(REQ §4.5,观众 403 综艺、馒头 404/405/407 不压成 tv)。 +2. **凭据**:MP 候选携带 `site_cookie`,馒头 enclosure 为含 `x-api-key` 的 base64 JSON;本项目凭据只经 `SecretResolver` 在请求时注入,候选、raw、错误消息均不含 cookie/apikey/passkey/一次性直链(迁移账本 SA5/MT3)。 +3. **登录失效**:MP 返回空列表并置 `is_error`;本项目抛 `SiteError('auth_expired')`,不把登录页当成 0 条正常结果(REQ §4.7.1)。3xx 不自动跟随,跳 login 视为失效。 +4. **系数 selector 模式**用浮点(MP `int()` 对 0.5 抛异常丢整条,SA4)。 +5. **H&R 三态**:定义里没有 `hr` 字段时 MP 缺省 False;本项目映射为 `'unknown'`(REQ §8.3 保守处理)。spider 层仍与 MP 一致:未配置不输出 `hit_and_run`。 +6. **HTML 解析**用 htmlparser2 HTML 模式(与 lxml 一样不插入 ``);`:has("x")` 引号写法自动去引号(cheerio-select 不接受)。未移植 MP 的 Rust 加速与字幕搜索分支。 +7. **馒头**:显式 `categoryIds` 覆盖 mtype 分类组;`freeOnly` 为本地过滤;业务 message 判定 `auth_invalid / rate_limited / not_found`。 + +## 接口 + +`SiteAdapter{ id, kind, displayName, definitionVersion, capabilities, list(page, filters), search(query, filters), detail(id), downloadTarget(id), healthCheck() }`,页码对外 1 起(MP 内部 0 起)。`filters = { mtype?: 'movie'|'tv'|'music', categoryIds?, freeOnly?, page? }`。 + +`SiteRelease{ siteId, siteReleaseId, title, subtitle?, category{id,name}, sizeBytes, seeders, leechers, snatched, promotion{type, downloadFactor, uploadFactor, expiresAt}, hr, uploadedAt, detailUrl, downloadUrl?, infoHash?, imdbId?, doubanId?, tags[], raw(完整 MP 字段), observedAt }`。促销类型由 MP 系数组合推出(0/1 free、0/2 2xfree、0.5/1 50%、0.5/2 2x50%、0.3 30%、1/2 2x、1/1 none),系数字段未配置为 `unknown`;`expiresAt` 为 MP `freedate` 按 `umt.timezone` 转 UTC。 + +错误 `SiteError{kind}`:`auth_missing / auth_expired / auth_invalid`(`isAuthError`,停止重试)、`rate_limited`(`retryAfterMs`,已顺延限流器)、`parse_error`、`network`、`site_maintenance`、`not_found`、`unsupported`。 + +## 观众定义待真实页面核验 + +行选择器、`td:nth-child(4..8)` 列号、`span.torrent-subtitle-line`、`?cat=` 分类、`img.pro_*` 促销 case 表、`freedate` 取促销图标 title、`search_area=4` 是否支持 IMDb、浏览 0 起分页、H&R 选择器(未配置 → unknown)、`span.tags` 标签、详情页 `h1#top`/`td.rowhead`/`#kdescr`。 + +## 馒头字段待真实账号核验 + +1. 搜索体只发 MP 的 `{keyword, categories, pageNumber, pageSize, visible}`(无 `mode`);浏览时 `keyword: null` 是否被接受。 +2. 数值字段为字符串还是数字(两者兼容);`totalPages` 字段名。 +3. `status.discount` 全集(MP 表:FREE/PERCENT_50/PERCENT_70/_2X_FREE/_2X_PERCENT_50/_2X,其余按 1)、`discountEndTime` / `createdDate` 时区(按 +08:00;MP `format_timestamp` 对非数字字符串原样返回)。 +4. `promotionRule{discount,endTime}` 与 `mallSingleFree{status,endDate}` 是否出现在 search 结果中。 +5. `labelsNew` / `labels`(MP 表 "0"–"7")。 +6. `/api/torrent/detail`(表单 id,MovieClaw 字段结构)是否可用、是否含 `descr`。 +7. `genDlToken`(id 走 query,同 MP)直链有效期、次数与频率;返回空 `data` 的原因。 +8. 错误 message 文案(停用/封禁/key 无效/频繁/不存在)与 HTTP 状态码。 +9. `categoryNames` 为自行编制的粗分类名;默认网页域 `tp.m-team.cc` 需按账号实际域名配置。 +10. H&R:API 未提供,恒为 `unknown`。 + +## M2 预留:站点用户数据解析器(首版未实现) + +- `app/modules/indexer/parser/nexus_php.py`(NexusPHP 通用:上传/下载/分享率/做种) +- `app/modules/indexer/parser/nexus_audiences.py`(观众新版 userbar;未读私信首页不能带 page 参数) +- `app/modules/indexer/parser/nexus_hhanclub.py`(HHClub) +- `app/modules/indexer/parser/mtorrent.py`(M-Team:`api/member/profile`、`api/tracker/myPeerStatus` 等) diff --git a/packages/adapters/src/sites/definitions/audiences.json b/packages/adapters/src/sites/definitions/audiences.json new file mode 100644 index 00000000..68b25216 --- /dev/null +++ b/packages/adapters/src/sites/definitions/audiences.json @@ -0,0 +1,110 @@ +{ + "_provenance": "按 MoviePilot SiteSpider indexer schema 手写;字段结构参考 MovieClaw audiences.yaml(data-ref,未复制);选择器值按 NexusPHP 公开页面惯例编写,仅对 testkit mock 验证,待真实页面核验。见 ../README.md", + "id": "audiences", + "name": "观众", + "domain": "https://audiences.me/", + "encoding": "UTF-8", + "public": false, + "proxy": false, + "language": "zh", + "result_num": 100, + "timeout": 30, + "search": { + "paths": [{ "path": "torrents.php", "type": "all" }], + "params": { "search": "{keyword}", "search_area": 4 } + }, + "batch": { "delimiter": " ", "space_replace": "_" }, + "browse": { "path": "torrents.php?page={page}", "start": 0 }, + "category": { + "movie": [{ "id": "401", "cat": "Movies", "desc": "电影" }], + "tv": [{ "id": "402", "cat": "TV Series", "desc": "剧集" }, { "id": "403", "cat": "TV Shows", "desc": "综艺" }], + "music": [] + }, + "torrents": { + "list": { + "selector": "table.torrents > tbody > tr:has(table.torrentname), table.torrents > tr:has(table.torrentname)" + }, + "fields": { + "title_default": { "selector": "a[href*=\"details.php?id=\"]" }, + "title_optional": { "selector": "a[title][href*=\"details.php?id=\"]", "attribute": "title" }, + "title": { + "text": "{% if fields['title_optional'] %}{{ fields['title_optional'] }}{% else %}{{ fields['title_default'] }}{% endif %}" + }, + "details": { "selector": "a[href*=\"details.php?id=\"]", "attribute": "href" }, + "download": { "selector": "a[href*=\"download.php?id=\"]", "attribute": "href" }, + "imdbid": { + "selector": "a[href*=\"imdb.com/title/tt\"]", + "attribute": "href", + "filters": [{ "name": "re_search", "args": ["tt\\d+", 0] }] + }, + "date_elapsed": { "selector": "td:nth-child(4) > span" }, + "date_added": { "selector": "td:nth-child(4) > span", "attribute": "title" }, + "date": { + "text": "{% if fields['date_elapsed'] or fields['date_added'] %}{{ fields['date_added'] if fields['date_added'] else fields['date_elapsed'] }}{% else %}now{% endif %}", + "filters": [{ "name": "dateparse", "args": "%Y-%m-%d %H:%M:%S" }] + }, + "size": { "selector": "td:nth-child(5)" }, + "seeders": { "selector": "td:nth-child(6)" }, + "leechers": { "selector": "td:nth-child(7)" }, + "grabs": { "selector": "td:nth-child(8)" }, + "downloadvolumefactor": { + "case": { + "img.pro_free": 0, + "img.pro_free2up": 0, + "img.pro_50pctdown": 0.5, + "img.pro_50pctdown2up": 0.5, + "img.pro_30pctdown": 0.3, + "*": 1 + } + }, + "uploadvolumefactor": { + "case": { + "img.pro_50pctdown2up": 2, + "img.pro_free2up": 2, + "img.pro_2up": 2, + "*": 1 + } + }, + "freedate": { + "selector": "img[class^=\"pro_\"]", + "attribute": "title", + "filters": [{ "name": "re_search", "args": ["\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}", 0] }] + }, + "description": { "selector": "span.torrent-subtitle-line" }, + "labels": { "selector": "span.tags" }, + "category": { + "selector": "a[href*=\"?cat=\"]", + "attribute": "href", + "filters": [{ "name": "querystring", "args": "cat" }] + } + } + }, + "umt": { + "version": "audiences@0.2.0-unverified", + "kind": "nexusphp", + "displayName": "观众", + "timezone": "+08:00", + "rateLimit": { "minIntervalMs": 3000, "burst": 1 }, + "capabilities": { "list": true, "search": true, "detail": true, "download": true, "rss": true }, + "categoryNames": { "401": "电影", "402": "剧集", "403": "综艺", "406": "纪录片" }, + "loginSuccess": "a[href*=\"logout\"]", + "emptyMarkers": ["没有种子", "没有结果", "Nothing found"], + "pagination": "a[href*=\"page=\"]", + "detailPath": "details.php", + "downloadPath": "download.php", + "detail": { + "path": "details.php", + "title": "h1#top", + "rowLabel": "td.rowhead", + "rowValue": "td.rowfollow", + "labels": { + "subtitle": ["副标题", "Small Description"], + "basicInfo": ["基本信息", "Basic Info"], + "peers": ["同伴", "Peers"] + }, + "description": "#kdescr", + "headerScope": "h1#top", + "notFoundMarkers": ["没有该ID的种子", "No torrent with ID"] + } + } +} diff --git a/packages/adapters/src/sites/definitions/index.ts b/packages/adapters/src/sites/definitions/index.ts new file mode 100644 index 00000000..4c629331 --- /dev/null +++ b/packages/adapters/src/sites/definitions/index.ts @@ -0,0 +1,16 @@ +import type { IndexerDefinition } from "../spider/schema"; +import audiencesJson from "./audiences.json"; + +/** 站点定义最小校验(随产品发布的定义文件;加站只需新增 JSON 并在此登记)。 */ +export function loadIndexerDefinition(json: unknown): IndexerDefinition { + const d = json as Partial; + if (!d || typeof d !== "object") throw new Error("站点定义必须是对象"); + for (const k of ["id", "name", "domain"] as const) { + if (typeof d[k] !== "string" || !d[k]) throw new Error(`站点定义缺少 ${k}`); + } + if (!d.torrents?.list?.selector || !d.torrents.fields) throw new Error(`站点定义 ${d.id} 缺少 torrents.list/fields`); + if (!d.umt?.version || !d.umt.timezone || !d.umt.rateLimit) throw new Error(`站点定义 ${d.id} 缺少 umt 扩展`); + return d as IndexerDefinition; +} + +export const audiencesDefinition: IndexerDefinition = loadIndexerDefinition(audiencesJson); diff --git a/packages/adapters/src/sites/deps.ts b/packages/adapters/src/sites/deps.ts new file mode 100644 index 00000000..86f4fda0 --- /dev/null +++ b/packages/adapters/src/sites/deps.ts @@ -0,0 +1,31 @@ +import type { SiteHttpContext } from "./http"; +import { createSiteRateLimiter, type SiteRateLimiter } from "./rate-limit"; +import type { SecretResolver } from "./types"; + +export interface SiteAdapterDeps { + secrets: SecretResolver; + fetch?: typeof globalThis.fetch; + /** 同站共享的限流器;不传则按站点定义新建(列表/搜索/详情/下载共享) */ + limiter?: SiteRateLimiter; + now?: () => Date; + timeoutMs?: number; + userAgent?: string; + /** 覆盖站点基址(测试指向 mock) */ + baseUrl?: string; +} + +export function buildHttpContext( + siteId: string, + deps: SiteAdapterDeps, + limiterDefaults: { minIntervalMs: number; burst: number }, +): SiteHttpContext { + const ctx: SiteHttpContext = { + siteId, + fetch: deps.fetch ?? globalThis.fetch, + limiter: deps.limiter ?? createSiteRateLimiter(limiterDefaults), + timeoutMs: deps.timeoutMs ?? 30_000, + now: deps.now ?? (() => new Date()), + }; + if (deps.userAgent) ctx.userAgent = deps.userAgent; + return ctx; +} diff --git a/packages/adapters/src/sites/http.ts b/packages/adapters/src/sites/http.ts new file mode 100644 index 00000000..8072e5b5 --- /dev/null +++ b/packages/adapters/src/sites/http.ts @@ -0,0 +1,92 @@ +import { redactText } from "../rss/redact"; +import type { SiteRateLimiter } from "./rate-limit"; +import { SiteError } from "./types"; + +export interface SiteHttpContext { + siteId: string; + fetch: typeof globalThis.fetch; + limiter: SiteRateLimiter; + timeoutMs: number; + userAgent?: string; + now: () => Date; +} + +function retryAfterMs(res: Response, now: Date): number | undefined { + const v = res.headers.get("retry-after"); + if (!v) return undefined; + if (/^\d+$/.test(v.trim())) return Number(v.trim()) * 1000; + const t = Date.parse(v); + return Number.isNaN(t) ? undefined : Math.max(0, t - now.getTime()); +} + +/** + * 站点请求:先取站点令牌,再发请求;429/503 统一映射并顺延限流器。 + * 返回的 Response 由调用方消费。错误消息只含脱敏后的 URL。 + */ +export async function siteFetch( + ctx: SiteHttpContext, + url: string, + init: RequestInit & { signal?: AbortSignal } = {}, +): Promise { + await ctx.limiter.acquire(init.signal ?? undefined); + const timeout = AbortSignal.timeout(ctx.timeoutMs); + const signal = init.signal ? AbortSignal.any([init.signal, timeout]) : timeout; + const headers = new Headers(init.headers); + if (ctx.userAgent && !headers.has("user-agent")) headers.set("user-agent", ctx.userAgent); + const safeUrl = redactText(url); + let res: Response; + try { + res = await ctx.fetch(url, { ...init, headers, signal }); + } catch (e) { + if (init.signal?.aborted) throw e; + const reason = timeout.aborted ? "超时" : (e as Error).message; + throw new SiteError(ctx.siteId, "network", redactText(`请求 ${safeUrl} 失败:${reason}`)); + } + if (res.status === 429 || res.status === 503) { + const ra = retryAfterMs(res, ctx.now()); + await res.body?.cancel().catch(() => {}); + if (ra !== undefined) ctx.limiter.penalize(ra); + throw new SiteError( + ctx.siteId, + res.status === 429 ? "rate_limited" : "site_maintenance", + `请求 ${safeUrl} 返回 HTTP ${res.status}`, + { httpStatus: res.status, ...(ra !== undefined ? { retryAfterMs: ra } : {}) }, + ); + } + if (res.status >= 500) { + await res.body?.cancel().catch(() => {}); + throw new SiteError(ctx.siteId, "site_maintenance", `请求 ${safeUrl} 返回 HTTP ${res.status}`, { + httpStatus: res.status, + }); + } + return res; +} + +/** 读取响应体并限制大小(默认 16 MiB,足够容纳 .torrent 与列表页)。 */ +export async function readBody(ctx: SiteHttpContext, res: Response, maxBytes = 16 * 1024 * 1024): Promise { + if (!res.body) return new Uint8Array(); + const reader = res.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > maxBytes) { + await reader.cancel().catch(() => {}); + throw new SiteError(ctx.siteId, "parse_error", `响应超过 ${maxBytes} 字节上限`); + } + chunks.push(value); + } + const out = new Uint8Array(total); + let off = 0; + for (const c of chunks) { + out.set(c, off); + off += c.byteLength; + } + return out; +} + +export async function readText(ctx: SiteHttpContext, res: Response): Promise { + return new TextDecoder("utf-8").decode(await readBody(ctx, res)); +} diff --git a/packages/adapters/src/sites/index.ts b/packages/adapters/src/sites/index.ts new file mode 100644 index 00000000..e04b66a7 --- /dev/null +++ b/packages/adapters/src/sites/index.ts @@ -0,0 +1,61 @@ +export { audiencesDefinition, loadIndexerDefinition } from "./definitions"; +export type { SiteAdapterDeps } from "./deps"; +export { createMteamAdapter, mteamTorrentToRelease } from "./mteam/adapter"; +export { type MteamApiPath, mteamPost } from "./mteam/client"; +export { type MteamDefinition, mteamApiBase, mteamDefinition } from "./mteam/definition"; +export { + findImdbid, + getDownloadVolumeFactor, + getUploadVolumeFactor, + type MteamTorrentJson, + mteamSearchParams, + parseMteamResult, +} from "./mteam/spider"; +export { createSiteRateLimiter, type RateLimiterOptions, type SiteRateLimiter } from "./rate-limit"; +export { createSiteAdapter, getSiteAdapter, SITE_DEFINITIONS, type SiteDefinition } from "./registry"; +export { clearSearchText, createSpiderSiteAdapter } from "./spider/adapter"; +export { parseDetailPage } from "./spider/detail"; +export { promotionFromFactors, spiderTorrentToRelease } from "./spider/map"; +export type { + IndexerDefinition, + SpiderCategoryConfig, + SpiderCategoryItem, + SpiderFieldSelector, + SpiderFields, + SpiderFilter, + UmtSiteExtension, +} from "./spider/schema"; +export { + combineUrl, + type MediaKind, + type MpMediaType, + parseSizeMp, + resolveCategoryMediaType, + SiteSpider, + type SiteSpiderQuery, + type SpiderTorrent, + selectMediaCategories, +} from "./spider/spider"; +export { + type DownloadTarget, + type HitAndRun, + type PromotionType, + type SecretResolver, + type SiteAdapter, + type SiteCallOptions, + type SiteCapabilities, + type SiteCategory, + type SiteCredentialKind, + SiteError, + type SiteErrorKind, + type SiteHealth, + type SiteHealthStatus, + type SiteKind, + type SiteListFilters, + type SitePage, + type SitePromotion, + type SiteRelease, + type SiteReleaseDetail, + staticSecretResolver, +} from "./types"; +export { parseCount, parseSiteTime } from "./util"; diff --git a/packages/adapters/src/sites/mteam/adapter.ts b/packages/adapters/src/sites/mteam/adapter.ts new file mode 100644 index 00000000..218f3d91 --- /dev/null +++ b/packages/adapters/src/sites/mteam/adapter.ts @@ -0,0 +1,211 @@ +/** + * 馒头(M-Team)API Key 适配器。 + * + * 对应 MoviePilot e6820a8: + * - `MTorrentSpider.search`(POST /api/torrent/search,x-api-key + UA + JSON)→ `list` / `search` + * - `MTorrentSpider.__process_response` → `mteamPost`(另加业务 code/message 检查,迁移账本 MT2) + * - `MTorrentSpider.__get_download_url` + `submission.py _resolve_indirect_download_url` + `download_torrent` + * (POST genDlToken?id= → data 为临时直链 → 不带 cookie GET)→ `downloadTarget` + * - `app/modules/indexer/parser/mtorrent.py`(api/member/profile)→ `healthCheck` + * 详情 `/api/torrent/detail` 为本项目扩展(MP spider 无详情调用,字段结构参考 MovieClaw mteam.py,data-ref)。 + */ +import { redactUrl } from "../../rss/redact"; +import { buildHttpContext, type SiteAdapterDeps } from "../deps"; +import { readBody, siteFetch } from "../http"; +import { healthFromError } from "../spider/adapter"; +import { promotionFromFactors } from "../spider/map"; +import type { SpiderTorrent } from "../spider/spider"; +import { + type DownloadTarget, + type SiteAdapter, + SiteError, + type SiteHealth, + type SiteListFilters, + type SitePage, + type SiteRelease, + type SiteReleaseDetail, +} from "../types"; +import { extractDoubanId, inspectTorrent, parseSiteTime } from "../util"; +import { mteamPost } from "./client"; +import { type MteamDefinition, mteamApiBase } from "./definition"; +import { type MteamTorrentJson, mteamSearchParams, parseMteamResult } from "./spider"; + +interface MteamSearchData { + data?: MteamTorrentJson[] | null; + totalPages?: string | number | null; + total?: string | number | null; +} + +export function mteamTorrentToRelease( + t: SpiderTorrent, + src: MteamTorrentJson, + def: MteamDefinition, + observedAt: string, +): SiteRelease { + const tz = def.umt.timezone; + const id = String(src.id); + const release: SiteRelease = { + siteId: def.id, + siteReleaseId: id, + title: (t.title ?? "").trim(), + category: { + id: t.category_id ?? null, + name: t.category_id ? (def.umt.categoryNames[t.category_id] ?? null) : null, + }, + sizeBytes: t.size ?? null, + seeders: t.seeders ?? null, + leechers: t.peers ?? null, + snatched: t.grabs ?? null, + promotion: promotionFromFactors(t.downloadvolumefactor, t.uploadvolumefactor, parseSiteTime(t.freedate, tz)), + // API 搜索结果无 H&R 字段(MP 未映射)→ 三态 unknown + hr: "unknown", + uploadedAt: parseSiteTime(t.pubdate, tz), + detailUrl: t.page_url ?? `${def.domain.replace(/\/$/, "")}/detail/${id}`, + tags: t.labels ?? [], + raw: { ...t, status: src.status ?? null, labelsRaw: src.labels ?? null }, + observedAt, + }; + if (t.description) release.subtitle = String(t.description).trim(); + if (t.imdbid) release.imdbId = t.imdbid; + const douban = + extractDoubanId(src.douban ?? null) ?? + (src.douban && /^\d+$/.test(String(src.douban)) ? String(src.douban) : undefined); + if (douban) release.doubanId = douban; + return release; +} + +export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps): SiteAdapter { + const def: MteamDefinition = deps.baseUrl ? { ...input, umt: { ...input.umt, apiBaseUrl: deps.baseUrl } } : input; + const http = buildHttpContext(def.id, { timeoutMs: def.timeout * 1000, ...deps }, def.umt.rateLimit); + const api = mteamApiBase(def); + + async function apiKey(): Promise { + const k = await deps.secrets.resolve({ siteId: def.id, kind: "apiKey" }); + // MP:无 apikey 直接返回错误 + if (!k) throw new SiteError(def.id, "auth_missing", "未配置 M-Team API Key"); + return k; + } + + async function doSearch( + page: number, + filters: SiteListFilters, + keyword: string | null, + signal?: AbortSignal, + ): Promise { + const params = mteamSearchParams(keyword, filters.mtype, Math.max(1, page) - 1); + // 本项目扩展:显式分类 ID 优先于 mtype 分类组 + if (filters.categoryIds?.length) params.categories = filters.categoryIds; + const data = await mteamPost( + http, + api, + await apiKey(), + "/api/torrent/search", + { json: params }, + signal, + ); + const observedAt = http.now().toISOString(); + const src = data?.data ?? []; + const rows = parseMteamResult(src, def.domain); + let items = rows.map((t, i) => mteamTorrentToRelease(t, src[i]!, def, observedAt)); + if (filters.freeOnly) items = items.filter((r) => r.promotion.downloadFactor === 0); + const tp = Number(data?.totalPages); + return { + items, + page: Math.max(1, page), + totalPages: Number.isFinite(tp) && tp >= 0 ? tp : null, + totalPagesEstimated: false, + fetchedAt: observedAt, + }; + } + + function checkId(id: string): string { + // 兼容详情页 URL(MP get_subtitle_links 从 page_url 取末段) + const m = /(\d+)\/?$/.exec(id.trim()); + if (!m) throw new SiteError(def.id, "not_found", "资源 ID 非法"); + return m[1]!; + } + + return { + id: def.id, + kind: "mteam", + displayName: def.umt.displayName, + capabilities: def.umt.capabilities, + definitionVersion: def.umt.version, + + list(page, filters = {}, opts = {}) { + return doSearch(page, filters, null, opts.signal); + }, + + search(query, filters = {}, opts = {}) { + return doSearch(filters.page ?? 1, filters, query.trim() || null, opts.signal); + }, + + async detail(siteReleaseId, opts = {}): Promise { + const id = checkId(siteReleaseId); + const data = await mteamPost( + http, + api, + await apiKey(), + "/api/torrent/detail", + { form: { id } }, + opts.signal, + ); + if (!data || data.id == null) throw new SiteError(def.id, "not_found", `详情 ${id}:无数据`); + const [row] = parseMteamResult([data], def.domain); + const r: SiteReleaseDetail = mteamTorrentToRelease(row!, data, def, http.now().toISOString()); + if (data.descr) r.descriptionText = String(data.descr).slice(0, 20_000); + return r; + }, + + async downloadTarget(siteReleaseId, opts = {}): Promise { + const id = checkId(siteReleaseId); + // MP __get_download_url: method=post, params={id}(query), header 含 x-api-key, result="data" + const url = await mteamPost( + http, + api, + await apiKey(), + "/api/torrent/genDlToken", + { query: { id } }, + opts.signal, + ); + if (!url || typeof url !== "string") { + throw new SiteError(def.id, "rate_limited", `genDlToken ${id}:未返回下载链接(可能未认证或达到下载频率限制)`); + } + // MP:涉及解析地址的不使用 Cookie 下载;直链含一次性签名,只在内存中使用 + const res = await siteFetch(http, url, { method: "GET", ...(opts.signal ? { signal: opts.signal } : {}) }); + if (!res.ok) { + await res.body?.cancel().catch(() => {}); + throw new SiteError( + def.id, + res.status === 404 ? "not_found" : "network", + `下载 ${redactUrl(url)}:HTTP ${res.status}`, + { + httpStatus: res.status, + }, + ); + } + const bytes = await readBody(http, res); + const info = inspectTorrent(bytes); + if (!info) throw new SiteError(def.id, "parse_error", `下载 ${id}:返回内容不是 .torrent`); + return { + kind: "torrent", + bytes, + infoHash: info.infoHash, + name: info.name, + fileName: null, + verifiedAt: http.now().toISOString(), + }; + }, + + async healthCheck(opts = {}): Promise { + const started = Date.now(); + const checkedAt = http.now().toISOString(); + try { + await mteamPost(http, api, await apiKey(), "/api/member/profile", { form: {} }, opts.signal); + return { siteId: def.id, status: "ok", ok: true, checkedAt, latencyMs: Date.now() - started }; + } catch (e) { + return healthFromError(def.id, e, checkedAt); + } + }, + }; +} diff --git a/packages/adapters/src/sites/mteam/client.ts b/packages/adapters/src/sites/mteam/client.ts new file mode 100644 index 00000000..3fd03895 --- /dev/null +++ b/packages/adapters/src/sites/mteam/client.ts @@ -0,0 +1,109 @@ +/** + * M-Team API 调用。对应 MP `MTorrentSpider.search/__process_response`(MP 只看 HTTP 200); + * 业务 code/message 检查与凭据/解析错误分层参考 MovieClaw `_post_api`(behavior-ref,迁移账本 MT2)。 + */ +import { redactText } from "../../rss/redact"; +import { readText, type SiteHttpContext, siteFetch } from "../http"; +import { SiteError } from "../types"; + +export type MteamApiPath = + | "/api/torrent/search" + | "/api/torrent/detail" + | "/api/torrent/genDlToken" + | "/api/member/profile" + | "/api/tracker/myPeerStatus"; + +const SUCCESS_MESSAGES = new Set(["SUCCESS", "OK", "0", ""]); +const AUTH_KEYWORDS = [ + "停用", + "封禁", + "禁用", + "banned", + "disabled", + "無效", + "无效", + "invalid", + "unauthorized", + "未授權", + "未授权", + "api key", + "apikey", + "key無效", + "key无效", +]; +const NOT_FOUND_KEYWORDS = ["不存在", "not found", "not exist"]; +const RATE_KEYWORDS = ["頻繁", "频繁", "too many", "rate limit"]; + +/** + * M-Team API 调用:x-api-key 头 + 业务 code/message 检查(MT2)。 + * 凭据类错误 → auth_invalid(停止重试);非 JSON → site_maintenance/parse_error。 + */ +export async function mteamPost( + http: SiteHttpContext, + apiBase: string, + apiKey: string, + path: MteamApiPath, + body: { json?: unknown; form?: Record; query?: Record }, + signal?: AbortSignal, +): Promise { + const headers: Record = { "x-api-key": apiKey, accept: "application/json, text/plain, */*" }; + let payload: string | undefined; + if (body.json !== undefined) { + headers["content-type"] = "application/json"; + payload = JSON.stringify(body.json); + } else if (body.form) { + headers["content-type"] = "application/x-www-form-urlencoded"; + payload = new URLSearchParams(body.form).toString(); + } + const qs = body.query ? `?${new URLSearchParams(body.query).toString()}` : ""; + const url = `${apiBase.replace(/\/$/, "")}${path}${qs}`; + const res = await siteFetch(http, url, { + method: "POST", + headers, + ...(payload !== undefined ? { body: payload } : {}), + ...(signal ? { signal } : {}), + }); + if (res.status === 401 || res.status === 403) { + await res.body?.cancel().catch(() => {}); + throw new SiteError(http.siteId, "auth_invalid", `${path}:HTTP ${res.status},API Key 无效或无权限`, { + httpStatus: res.status, + }); + } + if (!res.ok) { + await res.body?.cancel().catch(() => {}); + throw new SiteError(http.siteId, res.status === 404 ? "not_found" : "network", `${path}:HTTP ${res.status}`, { + httpStatus: res.status, + }); + } + const text = await readText(http, res); + let json: { code?: unknown; message?: unknown; data?: unknown }; + try { + json = JSON.parse(text) as typeof json; + } catch { + const maintenance = /維護|维护|maintenance/i.test(text); + throw new SiteError( + http.siteId, + maintenance ? "site_maintenance" : "parse_error", + `${path}:响应不是 JSON${maintenance ? "(站点维护)" : ""}`, + { httpStatus: res.status }, + ); + } + const message = json.message == null ? "" : String(json.message); + const code = json.code == null ? "0" : String(json.code); + if (code !== "0" || !SUCCESS_MESSAGES.has(message.toUpperCase())) { + const lower = message.toLowerCase(); + const kind = RATE_KEYWORDS.some((k) => lower.includes(k)) + ? "rate_limited" + : NOT_FOUND_KEYWORDS.some((k) => lower.includes(k)) + ? "not_found" + : AUTH_KEYWORDS.some((k) => lower.includes(k.toLowerCase())) + ? "auth_invalid" + : "parse_error"; + throw new SiteError( + http.siteId, + kind, + redactText(`${path}:业务错误 code=${code} message=${message.slice(0, 200)}`), + ); + } + return json.data as T; +} diff --git a/packages/adapters/src/sites/mteam/definition.ts b/packages/adapters/src/sites/mteam/definition.ts new file mode 100644 index 00000000..94cfb87b --- /dev/null +++ b/packages/adapters/src/sites/mteam/definition.ts @@ -0,0 +1,70 @@ +import type { SiteCapabilities } from "../types"; + +/** + * 馒头站点定义:键名沿用 MP indexer dict(id/name/domain/parser/timeout), + * `umt` 为本项目扩展。API Key 不在定义里,经 SecretResolver 注入(MP 放在 indexer['apikey'])。 + */ +export interface MteamDefinition { + id: string; + name: string; + /** 网页域(MP `_url`,详情页 = `${domain}detail/${id}`) */ + domain: string; + parser: "mTorrent"; + timeout: number; + umt: { + version: string; + kind: "mteam"; + displayName: string; + /** API 基址;缺省为 `https://api.<根域>`(MP `_searchurl % _domain`) */ + apiBaseUrl?: string; + timezone: string; + rateLimit: { minIntervalMs: number; burst: number }; + capabilities: SiteCapabilities; + /** 原始分类 ID → 名称(自行编制,待核验) */ + categoryNames: Record; + }; +} + +export const mteamDefinition: MteamDefinition = { + id: "mteam", + name: "馒头", + domain: "https://tp.m-team.cc/", + parser: "mTorrent", + timeout: 15, + umt: { + version: "mteam@0.2.0-unverified", + kind: "mteam", + displayName: "馒头", + timezone: "+08:00", + rateLimit: { minIntervalMs: 1000, burst: 1 }, + capabilities: { list: true, search: true, detail: true, download: true, rss: false }, + categoryNames: { + "401": "电影", + "419": "电影", + "420": "电影", + "421": "电影", + "439": "电影", + "402": "剧集", + "403": "综艺", + "435": "剧集", + "438": "剧集", + "404": "纪录", + "405": "动画", + "407": "体育", + "406": "音乐", + "408": "音乐", + "434": "音乐", + }, + }, +}; + +/** MP `site_rules.extract_domain` 的简化:取主机名最后两级。 */ +export function rootDomain(url: string): string { + const host = new URL(url).hostname; + const parts = host.split("."); + return parts.length >= 2 ? parts.slice(-2).join(".") : host; +} + +export function mteamApiBase(def: MteamDefinition): string { + return (def.umt.apiBaseUrl ?? `https://api.${rootDomain(def.domain)}`).replace(/\/$/, ""); +} diff --git a/packages/adapters/src/sites/mteam/mteam.test.ts b/packages/adapters/src/sites/mteam/mteam.test.ts new file mode 100644 index 00000000..a410dac4 --- /dev/null +++ b/packages/adapters/src/sites/mteam/mteam.test.ts @@ -0,0 +1,196 @@ +import { createMockMteamApi, type MockMteamApi } from "@umt/testkit"; +import { afterEach, describe, expect, it } from "vitest"; +import { + createMteamAdapter, + createSiteRateLimiter, + getDownloadVolumeFactor, + mteamDefinition, + type SiteAdapter, + SiteError, + type SiteRateLimiter, + staticSecretResolver, +} from "../index"; + +let api: MockMteamApi | undefined; +afterEach(async () => { + await api?.close(); + api = undefined; +}); + +function adapterFor(m: MockMteamApi, key: string | null = m.apiKey, limiter?: SiteRateLimiter): SiteAdapter { + return createMteamAdapter(mteamDefinition, { + baseUrl: m.baseUrl, + secrets: staticSecretResolver(key ? { mteam: { apiKey: key } } : {}), + limiter: limiter ?? createSiteRateLimiter({ minIntervalMs: 0 }), + }); +} + +async function expectSiteError(p: Promise, kind: string): Promise { + try { + await p; + } catch (e) { + expect(e).toBeInstanceOf(SiteError); + expect((e as SiteError).kind).toBe(kind); + return e as SiteError; + } + throw new Error(`expected SiteError ${kind}`); +} + +describe("M-Team API 适配器(mock)", () => { + it("列表:x-api-key 头 + 搜索体,映射 MV1 字段,结果不含 API Key", async () => { + api = await createMockMteamApi(); + const page = await adapterFor(api).list(1); + const req = api.requests[0]!; + expect(req.path).toBe("/api/torrent/search"); + expect(req.headers["x-api-key"]).toBe(api.apiKey); + expect(api.lastSearchBody).toEqual({ keyword: null, categories: [], pageNumber: 1, pageSize: 100, visible: 1 }); + expect(page).toMatchObject({ page: 1, totalPages: 1, totalPagesEstimated: false }); + const shogun = page.items.find((i) => i.siteReleaseId === "912233")!; + expect(shogun).toMatchObject({ + siteId: "mteam", + title: "Shogun.2024.S01.2160p.DSNP.WEB-DL.DDP5.1.Atmos.DV.HDR.H.265-FLUX", + subtitle: "幕府将军 第一季 全10集 杜比视界", + category: { id: "402", name: "剧集" }, + raw: expect.objectContaining({ category: "电视剧", category_id: "402" }), + sizeBytes: 61234567890, + seeders: 532, + leechers: 4, + snatched: 2011, + promotion: { type: "50%", downloadFactor: 0.5, uploadFactor: 1, expiresAt: "2026-09-28T00:15:00.000Z" }, + hr: "unknown", + uploadedAt: "2026-09-20T00:15:00.000Z", + detailUrl: "https://tp.m-team.cc/detail/912233", + imdbId: "tt2788316", + tags: ["中字", "DV"], + }); + expect(shogun.downloadUrl).toBeUndefined(); + expect(JSON.stringify(page)).not.toContain(api.apiKey); + }); + + it("促销优先级:商城单种免费 > 全站促销 > 基础折扣;2x 免费;标签位图", async () => { + api = await createMockMteamApi(); + const { items } = await adapterFor(api).list(1); + const by = Object.fromEntries(items.map((i) => [i.siteReleaseId, i])); + expect(by["912234"]!.promotion).toEqual({ + type: "free", + downloadFactor: 0, + uploadFactor: 1, + expiresAt: "2026-09-27T12:00:00.000Z", + }); + expect(by["912234"]!.tags).toEqual(["国配", "中字"]); + expect(by["912235"]!.promotion).toMatchObject({ + type: "free", + downloadFactor: 0, + expiresAt: "2026-10-07T15:59:59.000Z", + }); + expect(by["912236"]!.promotion).toMatchObject({ type: "2xfree", downloadFactor: 0, uploadFactor: 2 }); + expect(by["912237"]!.promotion).toEqual({ type: "none", downloadFactor: 1, uploadFactor: 1, expiresAt: null }); + // MP __get_downloadvolumefactor:未知折扣代码按 1 处理 + expect(getDownloadVolumeFactor("SOMETHING_NEW")).toBe(1); + }); + + it("分类原始 ID 保留(404 纪录 / 405 动画 / 407 体育)", async () => { + api = await createMockMteamApi(); + const { items } = await adapterFor(api).list(1); + const cats = Object.fromEntries(items.map((i) => [i.siteReleaseId, i.category])); + expect(cats["912235"]).toEqual({ id: "405", name: "动画" }); + expect(cats["912236"]).toEqual({ id: "404", name: "纪录" }); + expect(cats["912237"]).toEqual({ id: "407", name: "体育" }); + }); + + it("搜索:IMDb 关键词改写为完整 URL;分类与免费筛选下推", async () => { + api = await createMockMteamApi(); + const a = adapterFor(api); + const r = await a.search("tt15398776"); + expect(api.lastSearchBody!.keyword).toBe("https://www.imdb.com/title/tt15398776"); + expect(r.items.map((i) => i.siteReleaseId)).toEqual(["912234"]); + const free = await a.search("", { freeOnly: true, categoryIds: ["404", "405"] }); + expect(api.lastSearchBody).toMatchObject({ keyword: null, categories: ["404", "405"] }); + const tv = await a.search("Shogun", { mtype: "tv" }); + expect(api.lastSearchBody!.categories).toEqual(["403", "402", "435", "438", "404", "405"]); + expect(tv.items.map((i) => i.siteReleaseId)).toEqual(["912233"]); + // 912235 基础 NORMAL 但全站促销 FREE → 下载系数 0 + expect(free.items.map((i) => i.siteReleaseId)).toEqual(["912235", "912236"]); + const kw = await a.search("芙莉莲"); + expect(kw.items.map((i) => i.siteReleaseId)).toEqual(["912235"]); + }); + + it("详情:表单 id,带简介;不存在 → not_found", async () => { + api = await createMockMteamApi(); + const a = adapterFor(api); + const d = await a.detail("912233"); + expect(api.requests[0]!.body).toBe("id=912233"); + expect(d.descriptionText).toContain("幕府将军"); + expect(d.imdbId).toBe("tt2788316"); + const byUrl = await a.detail("https://tp.m-team.cc/detail/912233"); + expect(byUrl.siteReleaseId).toBe("912233"); + await expectSiteError(a.detail("1"), "not_found"); + }); + + it("下载令牌:现场 genDlToken → 一次性直链 → .torrent,infohash 校验", async () => { + api = await createMockMteamApi(); + const a = adapterFor(api); + const t = await a.downloadTarget("912233"); + expect(api.issuedTokens).toBe(1); + expect(api.requests.map((r) => r.path.split("?")[0])).toEqual(["/api/torrent/genDlToken", "/api/rss/dlv2"]); + // MP:genDlToken 的 id 走 query 参数;直链下载不带 x-api-key + expect(api.requests[0]!.path).toBe("/api/torrent/genDlToken?id=912233"); + expect(api.requests[1]!.headers["x-api-key"]).toBeUndefined(); + expect(t.kind).toBe("torrent"); + if (t.kind === "torrent") expect(t.infoHash).toBe(api.infoHashOf("912233")); + // 每次下载都重新签发令牌 + await a.downloadTarget("912233"); + expect(api.issuedTokens).toBe(2); + api.setFailMode("no_dl_token"); + await expectSiteError(a.downloadTarget("912233"), "rate_limited"); + }); + + it("Key 失效/停用 → auth_invalid(停止重试),健康检查报告;缺 Key → auth_missing", async () => { + api = await createMockMteamApi(); + const wrong = await expectSiteError(adapterFor(api, "wrong-key").list(1), "auth_invalid"); + expect(wrong.isAuthError).toBe(true); + api.setFailMode("banned_key"); + const banned = await expectSiteError(adapterFor(api).search("x"), "auth_invalid"); + expect(banned.message).not.toContain(api.apiKey); + expect(await adapterFor(api).healthCheck()).toMatchObject({ ok: false, status: "auth_invalid" }); + api.setFailMode(null); + expect((await adapterFor(api).healthCheck()).status).toBe("ok"); + const before = api.requests.length; + await expectSiteError(adapterFor(api, null).list(1), "auth_missing"); + expect(api.requests.length).toBe(before); + }); + + it("非 JSON 维护页 → site_maintenance;业务限流/HTTP 429 → rate_limited 并顺延限流器", async () => { + api = await createMockMteamApi(); + const limiter = createSiteRateLimiter({ minIntervalMs: 0 }); + const a = adapterFor(api, api.apiKey, limiter); + api.setFailMode("non_json"); + await expectSiteError(a.list(1), "site_maintenance"); + api.setFailMode("rate_limit"); + await expectSiteError(a.list(1), "rate_limited"); + api.setFailMode("http_429"); + const e = await expectSiteError(a.list(1), "rate_limited"); + expect(e.retryAfterMs).toBe(5000); + expect(limiter.tryAcquire()).toBe(false); + }); + + it("站点级限流 1s:列表/搜索/详情/下载共享", async () => { + api = await createMockMteamApi(); + let clock = 0; + const sleeps: number[] = []; + const limiter = createSiteRateLimiter({ + minIntervalMs: mteamDefinition.umt.rateLimit.minIntervalMs, + now: () => clock, + sleep: async (ms) => { + sleeps.push(ms); + clock += ms; + }, + }); + const a = adapterFor(api, api.apiKey, limiter); + await a.list(1); + await a.search("Shogun"); + await a.detail("912233"); + await a.downloadTarget("912233"); // genDlToken + GET 直链 = 2 次 + expect(sleeps).toEqual([1000, 1000, 1000, 1000]); + }); +}); diff --git a/packages/adapters/src/sites/mteam/spider.ts b/packages/adapters/src/sites/mteam/spider.ts new file mode 100644 index 00000000..f9b3980c --- /dev/null +++ b/packages/adapters/src/sites/mteam/spider.ts @@ -0,0 +1,151 @@ +/** + * M-Team API 解析。 + * + * 迁移自 MoviePilot e6820a8 `app/modules/indexer/spider/mtorrent.py` `MTorrentSpider`: + * - `_movie_category/_tv_category/_music_category/_labels` → 同名常量 + * - `__get_params` → `mteamSearchParams` + * - `__parse_result` → `parseMteamResult` + * - `__find_imdbid / __get_downloadvolumefactor / __get_uploadvolumefactor` → 同名函数 + * - `_pageurl` → `page_url` + * + * 与 MP 的差异: + * 1. 不生成 `[base64(json 含 x-api-key)]genDlToken` 形式的 enclosure(凭据不得进入候选对象,迁移账本 MT3), + * 下载时由 adapter 现场调用 genDlToken(行为同 MP `_resolve_indirect_download_url`)。 + * 2. 额外输出 `category_id`(原始分类 ID),`category` 仍是 MP 的媒体类型投影。 + * 3. 额外保留 `status` 原文到 raw,便于核验。 + */ +import type { MediaKind, MpMediaType, SpiderTorrent } from "../spider/spider"; + +export const MOVIE_CATEGORY = ["401", "419", "420", "421", "439", "405", "404"]; +export const TV_CATEGORY = ["403", "402", "435", "438", "404", "405"]; +export const MUSIC_CATEGORY = ["434", "406"]; + +export const LABELS: Record = { + "0": "", + "1": "DIY", + "2": "国配", + "3": "DIY 国配", + "4": "中字", + "5": "DIY 中字", + "6": "国配 中字", + "7": "DIY 国配 中字", +}; + +export const PAGE_SIZE = 100; + +/** M-Team 种子 JSON(字段名按公开 API,待真实账号核验)。 */ +export interface MteamTorrentJson { + id: string | number; + name?: string | null; + smallDescr?: string | null; + category?: string | number | null; + size?: string | number | null; + createdDate?: string | null; + imdb?: string | null; + douban?: string | null; + labels?: string | number | null; + labelsNew?: string[] | null; + descr?: string | null; + status?: { + seeders?: string | number | null; + leechers?: string | number | null; + timesCompleted?: string | number | null; + discount?: string | null; + discountEndTime?: string | null; + promotionRule?: { discount?: string | null; endTime?: string | null } | null; + mallSingleFree?: { status?: string | null; endDate?: string | null } | null; + } | null; + [k: string]: unknown; +} + +/** MP `__get_params`(page 为 0 起) */ +export function mteamSearchParams(keyword: string | null | undefined, mtype: MediaKind | undefined, page = 0) { + let categories: string[]; + if (!mtype) categories = []; + else if (mtype === "tv") categories = TV_CATEGORY; + else if (mtype === "music") categories = MUSIC_CATEGORY; + else categories = MOVIE_CATEGORY; + // mtorrent 搜索 imdb 需要输入完整 imdb 链接(MP 注释引 wiki.m-team.cc/zh-tw/imdbtosearch) + const kw = keyword?.startsWith("tt") ? `https://www.imdb.com/title/${keyword}` : (keyword ?? null); + return { keyword: kw, categories, pageNumber: Number(page) + 1, pageSize: PAGE_SIZE, visible: 1 }; +} + +/** MP `__find_imdbid` */ +export function findImdbid(imdb: string | null | undefined): string { + return imdb ? (/tt\d+/.exec(imdb)?.[0] ?? "") : ""; +} + +/** MP `__get_downloadvolumefactor` */ +export function getDownloadVolumeFactor(discount: string | null | undefined): number { + const map: Record = { FREE: 0, PERCENT_50: 0.5, PERCENT_70: 0.3, _2X_FREE: 0, _2X_PERCENT_50: 0.5 }; + return discount ? (map[discount] ?? 1) : 1; +} + +/** MP `__get_uploadvolumefactor` */ +export function getUploadVolumeFactor(discount: string | null | undefined): number { + const map: Record = { _2X: 2.0, _2X_FREE: 2.0, _2X_PERCENT_50: 2.0 }; + return discount ? (map[discount] ?? 1) : 1; +} + +/** MP `time_tools.format_timestamp`:非数字字符串原样返回,Unix 秒转本地时间(此处按站点时区交给调用方)。 */ +function formatTimestamp(v: string | null | undefined): string | undefined { + if (v == null) return undefined; + return String(v); +} + +function toInt(v: unknown): number { + const n = Number.parseInt(String(v ?? "0") || "0", 10); + return Number.isFinite(n) ? n : 0; +} + +/** MP `__parse_result` */ +export function parseMteamResult(results: MteamTorrentJson[] | null | undefined, webUrl: string): SpiderTorrent[] { + const out: SpiderTorrent[] = []; + if (!results) return out; + const url = webUrl.endsWith("/") ? webUrl : `${webUrl}/`; + for (const result of results) { + const categoryValue = result.category == null ? undefined : String(result.category); + let category: MpMediaType; + if (categoryValue && MUSIC_CATEGORY.includes(categoryValue)) category = "音乐"; + else if (categoryValue && TV_CATEGORY.includes(categoryValue) && !MOVIE_CATEGORY.includes(categoryValue)) + category = "电视剧"; + else if (categoryValue && MOVIE_CATEGORY.includes(categoryValue)) category = "电影"; + else category = "未知"; + let labels: string[] = []; + if (result.labelsNew && result.labelsNew.length > 0) labels = result.labelsNew.map(String); + else { + const v = LABELS[String(result.labels || "0")] ?? ""; + if (v) labels = v.split(/\s+/); + } + const status = result.status ?? {}; + const torrent: SpiderTorrent = { + title: result.name ?? undefined, + description: result.smallDescr ?? undefined, + size: toInt(result.size), + seeders: toInt(status.seeders), + peers: toInt(status.leechers), + grabs: toInt(status.timesCompleted), + downloadvolumefactor: getDownloadVolumeFactor(status.discount), + uploadvolumefactor: getUploadVolumeFactor(status.discount), + page_url: `${url}detail/${result.id}`, + imdbid: findImdbid(result.imdb), + labels, + category, + }; + const pubdate = formatTimestamp(result.createdDate); + if (pubdate) torrent.pubdate = pubdate; + if (categoryValue) torrent.category_id = categoryValue; + if (status.discountEndTime) torrent.freedate = formatTimestamp(status.discountEndTime); + // 全站促销规则(当前馒头只有下载促销) + if (status.promotionRule) { + torrent.downloadvolumefactor = getDownloadVolumeFactor(status.promotionRule.discount ?? "NORMAL"); + if (status.promotionRule.endTime) torrent.freedate = formatTimestamp(status.promotionRule.endTime); + } + if (status.mallSingleFree && status.mallSingleFree.status === "ONGOING") { + torrent.downloadvolumefactor = getDownloadVolumeFactor("FREE"); + if (status.mallSingleFree.endDate) torrent.freedate = formatTimestamp(status.mallSingleFree.endDate); + } + out.push(torrent); + } + return out; +} diff --git a/packages/adapters/src/sites/rate-limit.test.ts b/packages/adapters/src/sites/rate-limit.test.ts new file mode 100644 index 00000000..5f5ea1ba --- /dev/null +++ b/packages/adapters/src/sites/rate-limit.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { createSiteRateLimiter } from "./rate-limit"; +import { parseSizeMp as parseSize } from "./spider/spider"; +import { parseCount, parseSiteTime } from "./util"; + +describe("站点令牌桶", () => { + it("burst 内不等待,之后按间隔补充;penalize 顺延;并发按顺序排队", async () => { + let clock = 0; + const sleeps: number[] = []; + const l = createSiteRateLimiter({ + minIntervalMs: 1000, + burst: 2, + now: () => clock, + sleep: async (ms) => { + sleeps.push(ms); + clock += ms; + }, + }); + await Promise.all([l.acquire(), l.acquire(), l.acquire()]); + expect(sleeps).toEqual([1000]); + l.penalize(5000); + expect(l.tryAcquire()).toBe(false); + await l.acquire(); + expect(sleeps.at(-1)).toBe(5000); + }); + + it("取消等待", async () => { + const l = createSiteRateLimiter({ minIntervalMs: 60_000 }); + await l.acquire(); + const ac = new AbortController(); + const p = l.acquire(ac.signal); + ac.abort(new Error("cancelled")); + await expect(p).rejects.toThrow("cancelled"); + }); +}); + +describe("解析辅助", () => { + it("parseSize/parseCount/parseSiteTime", () => { + expect(parseSize("78.42 GB")).toBe(84202833838); + expect(parseSize("850 KB")).toBe(870400); + expect(parseSize("n/a")).toBe(0); + expect(parseSize("1,234.56 MiB")).toBe(1294529987); + expect(parseSize("2.1 TB")).toBe(2308974418330); + expect(parseCount("1,204")).toBe(1204); + expect(parseCount("532 / 3")).toBe(532); + expect(parseCount("--")).toBeNull(); + expect(parseSiteTime("2026-09-26 21:14:03", "+08:00")).toBe("2026-09-26T13:14:03.000Z"); + expect(parseSiteTime("3小时前", "+08:00")).toBeNull(); + }); +}); diff --git a/packages/adapters/src/sites/rate-limit.ts b/packages/adapters/src/sites/rate-limit.ts new file mode 100644 index 00000000..2ae5f81b --- /dev/null +++ b/packages/adapters/src/sites/rate-limit.ts @@ -0,0 +1,95 @@ +/** + * 站点级令牌桶:同一站点的列表/搜索/详情/下载(以及将来的浏览器操作)共享一个实例。 + * 容量 burst,每 minIntervalMs 补 1 个令牌;429/Retry-After 通过 penalize 整体顺延。 + */ + +export interface SiteRateLimiter { + acquire(signal?: AbortSignal): Promise; + /** 不等待:有令牌返回 true 并消耗 */ + tryAcquire(): boolean; + /** 站点要求暂停(Retry-After 等),在 ms 内不发放令牌 */ + penalize(ms: number): void; + readonly stats: { granted: number; waitedMs: number }; +} + +export interface RateLimiterOptions { + minIntervalMs: number; + burst?: number; + now?: () => number; + sleep?: (ms: number, signal?: AbortSignal) => Promise; +} + +function defaultSleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason); + return; + } + const t = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(t); + reject(signal?.reason); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +export function createSiteRateLimiter(opts: RateLimiterOptions): SiteRateLimiter { + const interval = Math.max(0, opts.minIntervalMs); + const burst = Math.max(1, opts.burst ?? 1); + const now = opts.now ?? Date.now; + const sleep = opts.sleep ?? defaultSleep; + let tokens = burst; + let last = now(); + let blockedUntil = 0; + let queue: Promise = Promise.resolve(); + const stats = { granted: 0, waitedMs: 0 }; + + const refill = () => { + const t = now(); + if (interval === 0) tokens = burst; + else tokens = Math.min(burst, tokens + (t - last) / interval); + last = t; + }; + + const waitMs = (): number => { + refill(); + const t = now(); + if (t < blockedUntil) return blockedUntil - t; + if (tokens >= 1) return 0; + return Math.ceil((1 - tokens) * interval); + }; + + return { + stats, + tryAcquire() { + if (waitMs() > 0) return false; + tokens -= 1; + stats.granted++; + return true; + }, + penalize(ms) { + blockedUntil = Math.max(blockedUntil, now() + ms); + tokens = Math.min(tokens, 0); + }, + acquire(signal) { + // 串行排队,保证先到先得、不会并发抢同一个令牌 + const run = queue.then(async () => { + for (;;) { + signal?.throwIfAborted(); + const w = waitMs(); + if (w <= 0) break; + stats.waitedMs += w; + await sleep(w, signal); + } + tokens -= 1; + stats.granted++; + }); + queue = run.catch(() => {}); + return run; + }, + }; +} diff --git a/packages/adapters/src/sites/registry.ts b/packages/adapters/src/sites/registry.ts new file mode 100644 index 00000000..0caa4a8b --- /dev/null +++ b/packages/adapters/src/sites/registry.ts @@ -0,0 +1,35 @@ +/** + * 随产品发布的站点注册表(迁移账本 SA1)。 + * 对应 MP `SitesHelper.get_indexers()/get_indexer(domain)`(闭源,sites.pyi)与 + * `IndexerModule` 按 `indexer.parser` 分派专用解析器(`mTorrent` → MTorrentSpider,其余 → SiteSpider)。 + * 加站:新增 definitions/.json 并在此登记。 + */ + +import { audiencesDefinition } from "./definitions"; +import type { SiteAdapterDeps } from "./deps"; +import { createMteamAdapter } from "./mteam/adapter"; +import { type MteamDefinition, mteamDefinition } from "./mteam/definition"; +import { createSpiderSiteAdapter } from "./spider/adapter"; +import type { IndexerDefinition } from "./spider/schema"; +import type { SiteAdapter } from "./types"; + +export type SiteDefinition = IndexerDefinition | MteamDefinition; + +export const SITE_DEFINITIONS: Record = { + audiences: audiencesDefinition, + mteam: mteamDefinition, +}; + +function isMteam(def: SiteDefinition): def is MteamDefinition { + return (def as MteamDefinition).parser === "mTorrent"; +} + +export function createSiteAdapter(def: SiteDefinition, deps: SiteAdapterDeps): SiteAdapter { + return isMteam(def) ? createMteamAdapter(def, deps) : createSpiderSiteAdapter(def, deps); +} + +export function getSiteAdapter(siteId: string, deps: SiteAdapterDeps): SiteAdapter { + const def = SITE_DEFINITIONS[siteId]; + if (!def) throw new Error(`未知站点:${siteId}`); + return createSiteAdapter(def, deps); +} diff --git a/packages/adapters/src/sites/spider/adapter.ts b/packages/adapters/src/sites/spider/adapter.ts new file mode 100644 index 00000000..360cef24 --- /dev/null +++ b/packages/adapters/src/sites/spider/adapter.ts @@ -0,0 +1,249 @@ +/** + * 通用 spider 站点适配器(NexusPHP 等 HTML 站)。 + * + * 对应 MoviePilot e6820a8: + * - `app/modules/indexer/__init__.py IndexerModule.search_torrents / refresh_torrents` → `search` / `list` + * (`__search_check` 的英文站不搜中文 → search 前检查;`__clear_search_text` → `clearSearchText`; + * `SitesHelper.check(domain)` 流控 → 站点令牌桶) + * - `SiteSpider.get_torrents`(RequestUtils GET + cookie + ua)→ `fetchPage` + * - `app/chain/download/submission.py download_torrent`(带 cookie GET enclosure,磁力原样返回)→ `downloadTarget` + * + * 本项目调整:登录页不再"返回空列表",而是抛 `auth_expired`(REQ §4.7.1 不展示空列表冒充无内容); + * 3xx 不自动跟随(避免把 cookie 带到站外,并把跳转 login 识别为失效);凭证经 SecretResolver 注入。 + */ +import { redactText } from "../../rss/redact"; +import { buildHttpContext, type SiteAdapterDeps } from "../deps"; +import { readBody, readText, siteFetch } from "../http"; +import { + type DownloadTarget, + type SiteAdapter, + type SiteCallOptions, + SiteError, + type SiteHealth, + type SiteListFilters, + type SitePage, + type SiteReleaseDetail, +} from "../types"; +import { inspectTorrent } from "../util"; +import { parseDetailPage } from "./detail"; +import { spiderTorrentToRelease } from "./map"; +import type { IndexerDefinition } from "./schema"; +import { loadHtml, normalizeSelector, SiteSpider } from "./spider"; + +/** MP `text_tools.remove_punctuation(text, replacement=" ", allow_space=True)` */ +export function clearSearchText(text: string): string { + return text + .replace(/[​-‍]/g, "") + .replace(/[、.。,，·:：;；!！?？'’"“”()（）[\]【】「」\-—―+|\\_/&#～~]/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function containsChinese(s: string): boolean { + return /[一-鿿]/.test(s); +} + +export function healthFromError(siteId: string, e: unknown, checkedAt: string): SiteHealth { + if (e instanceof SiteError) { + const status: SiteHealth["status"] = + e.kind === "network" + ? "unreachable" + : e.kind === "not_found" || e.kind === "unsupported" + ? "parse_error" + : e.kind; + return { siteId, status, ok: false, checkedAt, latencyMs: null, message: e.message }; + } + return { siteId, status: "unreachable", ok: false, checkedAt, latencyMs: null, message: redactText(String(e)) }; +} + +export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdapterDeps): SiteAdapter { + const def: IndexerDefinition = deps.baseUrl ? { ...input, domain: deps.baseUrl } : input; + const ext = def.umt; + const http = buildHttpContext(def.id, deps, ext.rateLimit); + const base = def.domain.endsWith("/") ? def.domain : `${def.domain}/`; + + async function cookie(): Promise { + const c = await deps.secrets.resolve({ siteId: def.id, kind: "cookie" }); + if (!c) throw new SiteError(def.id, "auth_missing", "未配置站点 Cookie"); + return c; + } + + async function fetchPage(url: string, what: string, opts: SiteCallOptions): Promise { + const res = await siteFetch(http, url, { + method: "GET", + headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base }, + redirect: "manual", + ...(opts.signal ? { signal: opts.signal } : {}), + }); + if (res.status >= 300 && res.status < 400) { + const loc = res.headers.get("location") ?? ""; + await res.body?.cancel().catch(() => {}); + if (/login/i.test(loc)) throw new SiteError(def.id, "auth_expired", `${what}:跳转登录页,Cookie 已失效`); + throw new SiteError(def.id, "parse_error", redactText(`${what}:意外重定向到 ${loc}`), { httpStatus: res.status }); + } + if (res.status === 401 || res.status === 403) { + await res.body?.cancel().catch(() => {}); + throw new SiteError(def.id, "auth_invalid", `${what}:HTTP ${res.status}`, { httpStatus: res.status }); + } + if (!res.ok) { + await res.body?.cancel().catch(() => {}); + throw new SiteError(def.id, res.status === 404 ? "not_found" : "network", `${what}:HTTP ${res.status}`, { + httpStatus: res.status, + }); + } + return readText(http, res); + } + + async function run( + page: number, + filters: SiteListFilters, + keyword: string | undefined, + opts: SiteCallOptions, + ): Promise { + const what = keyword ? "搜索" : `列表第 ${page} 页`; + const spider = new SiteSpider(def, { + ...(keyword ? { keyword } : {}), + ...(filters.mtype ? { mtype: filters.mtype } : {}), + ...(filters.categoryIds?.length ? { cat: filters.categoryIds.join(",") } : {}), + page: Math.max(1, page) - 1, // 对外 1 起,MP 内部 0 起 + }); + if (!spider.canSearch()) throw new SiteError(def.id, "unsupported", `${what}:站点定义不支持该媒体类型`); + const html = await fetchPage(spider.getSearchUrl(), what, opts); + const parsed = spider.parse(html); + if (parsed.loginPage) throw new SiteError(def.id, "auth_expired", `${what}:返回登录或权限提示页`); + if (parsed.isError || !parsed.$) throw new SiteError(def.id, "parse_error", `${what}:响应为空`); + const $ = parsed.$; + if (parsed.torrents.length === 0) { + const text = $("body").text(); + const hasList = + $(normalizeSelector(spider.def.torrents.list.selector)).length > 0 || $("table.torrents").length > 0; + const empty = (ext.emptyMarkers ?? []).some((m) => text.includes(m)); + if (!hasList && !empty) throw new SiteError(def.id, "parse_error", `${what}:页面结构无法识别(可能改版)`); + if (ext.loginSuccess && $(ext.loginSuccess).length === 0 && !empty) { + throw new SiteError(def.id, "auth_expired", `${what}:未找到登录标志`); + } + } + const observedAt = http.now().toISOString(); + let items = parsed.torrents + .map((t) => spiderTorrentToRelease(t, def, observedAt)) + .filter((r): r is NonNullable => r !== null); + if (filters.freeOnly) items = items.filter((r) => r.promotion.downloadFactor === 0); + if (!keyword && filters.categoryIds?.length) { + // MP 浏览模式不带分类参数;本地按原始分类过滤 + const set = new Set(filters.categoryIds); + items = items.filter((r) => r.category.id !== null && set.has(r.category.id)); + } + let maxRaw: number | null = null; + if (ext.pagination) { + $(ext.pagination).each((_, el) => { + const p = Number(/[?&]page=(\d+)/.exec($(el).attr("href") ?? "")?.[1]); + if (Number.isInteger(p) && (maxRaw === null || p > maxRaw)) maxRaw = p; + }); + } + const start = keyword ? 0 : Number(def.browse?.start ?? 0); + return { + items, + page: Math.max(1, page), + totalPages: maxRaw === null ? null : Math.max(page, (maxRaw as number) - start + 1), + totalPagesEstimated: true, + fetchedAt: observedAt, + }; + } + + return { + id: def.id, + kind: ext.kind, + displayName: ext.displayName ?? def.name, + capabilities: ext.capabilities, + definitionVersion: ext.version, + + list(page, filters = {}, opts = {}) { + return run(page, filters, undefined, opts); + }, + + async search(query, filters = {}, opts = {}) { + const kw = /^tt\d+$/.test(query.trim()) ? query.trim() : clearSearchText(query); + if (kw && def.language === "en" && containsChinese(kw)) { + throw new SiteError(def.id, "unsupported", "该站点不支持中文搜索"); + } + return run(filters.page ?? 1, filters, kw, opts); + }, + + async detail(siteReleaseId, opts = {}): Promise { + if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法"); + if (!ext.detail) throw new SiteError(def.id, "unsupported", "站点定义未配置详情页解析"); + const url = new URL(`${ext.detail.path}?id=${siteReleaseId}&hit=1`, base).toString(); + const html = await fetchPage(url, `详情 ${siteReleaseId}`, opts); + const r = parseDetailPage(html, def, siteReleaseId, http.now().toISOString()); + if (r.kind === "login") throw new SiteError(def.id, "auth_expired", `详情 ${siteReleaseId}:返回登录页`); + if (r.kind === "not_found") throw new SiteError(def.id, "not_found", `详情 ${siteReleaseId}:资源不存在`); + if (!r.detail) throw new SiteError(def.id, "parse_error", `详情 ${siteReleaseId}:页面结构无法识别`); + return r.detail; + }, + + async downloadTarget(siteReleaseId, opts = {}): Promise { + if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法"); + const what = `下载 ${siteReleaseId}`; + const url = new URL(`${ext.downloadPath}?id=${siteReleaseId}`, base).toString(); + const res = await siteFetch(http, url, { + method: "GET", + headers: { cookie: await cookie(), referer: base }, + redirect: "manual", + ...(opts.signal ? { signal: opts.signal } : {}), + }); + const verifiedAt = http.now().toISOString(); + if (res.status >= 300 && res.status < 400) { + const loc = res.headers.get("location") ?? ""; + await res.body?.cancel().catch(() => {}); + // MP download_torrent:磁力链原样返回 + if (loc.startsWith("magnet:")) return { kind: "magnet", uri: loc, verifiedAt }; + if (/login/i.test(loc)) throw new SiteError(def.id, "auth_expired", `${what}:跳转登录页,Cookie 已失效`); + throw new SiteError(def.id, "parse_error", `${what}:意外重定向`, { httpStatus: res.status }); + } + if (!res.ok) { + await res.body?.cancel().catch(() => {}); + throw new SiteError(def.id, res.status === 404 ? "not_found" : "network", `${what}:HTTP ${res.status}`, { + httpStatus: res.status, + }); + } + const bytes = await readBody(http, res); + const info = inspectTorrent(bytes); + if (!info) { + // 站点在 cookie 失效时常返回 200 + 登录页 HTML + const $ = loadHtml(new TextDecoder().decode(bytes.subarray(0, 200_000))); + if (SiteSpider.isLoginOrPermissionPage($)) throw new SiteError(def.id, "auth_expired", `${what}:返回登录页`); + const text = $("body").text(); + if (ext.detail?.notFoundMarkers.some((m) => text.includes(m))) { + throw new SiteError(def.id, "not_found", `${what}:资源不存在`); + } + throw new SiteError(def.id, "parse_error", `${what}:返回内容不是 .torrent`); + } + const disposition = res.headers.get("content-disposition") ?? ""; + const fileName = + /filename\*=UTF-8''([^;]+)/i.exec(disposition)?.[1] ?? /filename="?([^";]+)"?/i.exec(disposition)?.[1] ?? null; + return { + kind: "torrent", + bytes, + infoHash: info.infoHash, + name: info.name, + fileName: fileName ? decodeURIComponent(fileName) : null, + verifiedAt, + }; + }, + + async healthCheck(opts = {}): Promise { + const started = Date.now(); + const checkedAt = http.now().toISOString(); + try { + const html = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts); + const $ = loadHtml(html); + if (SiteSpider.isLoginOrPermissionPage($) || (ext.loginSuccess && $(ext.loginSuccess).length === 0)) { + throw new SiteError(def.id, "auth_expired", "健康检查:未登录"); + } + return { siteId: def.id, status: "ok", ok: true, checkedAt, latencyMs: Date.now() - started }; + } catch (e) { + return healthFromError(def.id, e, checkedAt); + } + }, + }; +} diff --git a/packages/adapters/src/sites/spider/detail.ts b/packages/adapters/src/sites/spider/detail.ts new file mode 100644 index 00000000..940fa3fa --- /dev/null +++ b/packages/adapters/src/sites/spider/detail.ts @@ -0,0 +1,110 @@ +/** + * 详情页解析 —— 本项目扩展。MP 通用 SiteSpider 不解析详情页(MP 只在列表/搜索结果上工作), + * 这里按 NexusPHP details.php 公开结构(h1#top + td.rowhead/td.rowfollow)实现,选择器在站点定义 `umt.detail` 中配置。 + * 促销/H&R 复用站点定义中 downloadvolumefactor / uploadvolumefactor / freedate / hr 字段(与列表同语义)。 + */ +import type { CheerioAPI } from "cheerio"; +import type { SiteReleaseDetail } from "../types"; +import { extractDoubanId, extractImdbId, parseCount, parseSiteTime } from "../util"; +import { categoryName, promotionFromFactors } from "./map"; +import type { IndexerDefinition } from "./schema"; +import { loadHtml, parseSizeMp, SiteSpider } from "./spider"; + +export type DetailPageKind = "ok" | "login" | "not_found" | "unknown"; + +export function parseDetailPage( + html: string, + def: IndexerDefinition, + id: string, + observedAt: string, +): { kind: DetailPageKind; detail: SiteReleaseDetail | null } { + const conf = def.umt.detail; + if (!conf) return { kind: "unknown", detail: null }; + const $ = loadHtml(html); + if (SiteSpider.isLoginOrPermissionPage($)) return { kind: "login", detail: null }; + const bodyHead = $("body").text().replace(/\s+/g, " ").slice(0, 2000); + if (conf.notFoundMarkers.some((m) => bodyHead.includes(m))) return { kind: "not_found", detail: null }; + const h1 = $(conf.title).first(); + if (h1.length === 0) return { kind: "unknown", detail: null }; + + const row = (labels: string[]) => { + let found: ReturnType | null = null; + $(conf.rowLabel).each((_, el) => { + const label = $(el).text().replace(/\s+/g, ""); + if (!found && labels.some((l) => label.startsWith(l.replace(/\s+/g, "")))) { + found = $(el).nextAll(conf.rowValue).first(); + } + }); + return found as ReturnType | null; + }; + + const title = h1 + .contents() + .filter((_, n) => n.type === "text") + .text() + .replace(/\s+/g, " ") + .trim(); + const subtitle = row(conf.labels.subtitle)?.text().trim(); + const basicText = row(conf.labels.basicInfo)?.text().replace(/\s+/g, " ") ?? ""; + const sizeText = /(?:大小|Size)\s*[:：]\s*([\d.,]+\s*[KMGTP]?i?B)/i.exec(basicText)?.[1] ?? null; + const catText = /(?:类型|Type)\s*[:：]\s*(\S+)/i.exec(basicText)?.[1] ?? null; + const catId = catText + ? (Object.entries(def.umt.categoryNames ?? {}).find(([, name]) => name === catText)?.[0] ?? null) + : null; + const peersText = row(conf.labels.peers)?.text().replace(/\s+/g, " ") ?? ""; + const seeders = /(\d[\d,]*)\s*个?做种者|(\d[\d,]*)\s*seeder/i.exec(peersText); + const leechers = /(\d[\d,]*)\s*个?下载者|(\d[\d,]*)\s*leecher/i.exec(peersText); + const descEl = $(conf.description).first(); + const descText = descEl.text().trim(); + const descHtml = descEl.html() ?? ""; + + // 促销/HR:只用这四个字段在详情页头部取值 + const fields = def.torrents.fields; + const promoDef: IndexerDefinition = { + ...def, + torrents: { + list: def.torrents.list, + fields: Object.fromEntries( + (["downloadvolumefactor", "uploadvolumefactor", "freedate", "hr"] as const) + .filter((k) => fields[k]) + .map((k) => [k, fields[k]]), + ), + }, + browse: undefined, + }; + const promo = new SiteSpider(promoDef).infoFrom($, $(conf.headerScope).first()); + const timeTitle = $("span[title]") + .filter((_, el) => /\d{4}-\d{2}-\d{2} \d{2}:\d{2}/.test($(el).attr("title") ?? "")) + .first() + .attr("title"); + + const detail: SiteReleaseDetail = { + siteId: def.id, + siteReleaseId: id, + title, + category: { id: catId, name: catText ?? categoryName(def, catId ?? undefined) }, + sizeBytes: sizeText ? parseSizeMp(sizeText) : null, + seeders: seeders ? parseCount(seeders[1] ?? seeders[2]) : null, + leechers: leechers ? parseCount(leechers[1] ?? leechers[2]) : null, + snatched: null, + promotion: promotionFromFactors( + promo.downloadvolumefactor, + promo.uploadvolumefactor, + promo.freedate ? parseSiteTime(promo.freedate, def.umt.timezone) : null, + ), + hr: promo.hit_and_run === undefined ? "unknown" : promo.hit_and_run, + uploadedAt: parseSiteTime(timeTitle, def.umt.timezone), + detailUrl: new URL(`${def.umt.detailPath}?id=${id}`, def.domain).toString(), + downloadUrl: new URL(`${def.umt.downloadPath}?id=${id}`, def.domain).toString(), + tags: [], + raw: { basicInfoText: basicText.trim().slice(0, 500), peersText: peersText.trim(), ...promo }, + observedAt, + }; + if (subtitle) detail.subtitle = subtitle; + if (descText) detail.descriptionText = descText.slice(0, 20_000); + const imdb = extractImdbId(descHtml) ?? extractImdbId(descText); + if (imdb) detail.imdbId = imdb; + const douban = extractDoubanId(descHtml); + if (douban) detail.doubanId = douban; + return { kind: "ok", detail }; +} diff --git a/packages/adapters/src/sites/spider/map.ts b/packages/adapters/src/sites/spider/map.ts new file mode 100644 index 00000000..6303f0cb --- /dev/null +++ b/packages/adapters/src/sites/spider/map.ts @@ -0,0 +1,82 @@ +import { redactUrl } from "../../rss/redact"; +import type { PromotionType, SitePromotion, SiteRelease } from "../types"; +import { parseSiteTime } from "../util"; +import type { IndexerDefinition } from "./schema"; +import type { SpiderTorrent } from "./spider"; + +/** + * MP 系数 → 统一促销类型。MP 只有 downloadvolumefactor/uploadvolumefactor/freedate 三个字段, + * 类型名由系数组合推出;字段未配置(undefined)时为 unknown。 + */ +export function promotionFromFactors( + dl: number | undefined, + ul: number | undefined, + freedate: string | null, +): SitePromotion { + if (dl === undefined && ul === undefined) + return { type: "unknown", downloadFactor: null, uploadFactor: null, expiresAt: null }; + const d = dl ?? 1; + const u = ul ?? 1; + let type: PromotionType; + if (d === 0) type = u >= 2 ? "2xfree" : "free"; + else if (d === 0.5) type = u >= 2 ? "2x50%" : "50%"; + else if (d === 0.3) type = "30%"; + else if (d === 1 && u >= 2) type = "2x"; + else if (d === 1 && u === 1) type = "none"; + else type = "unknown"; + return { type, downloadFactor: dl ?? null, uploadFactor: ul ?? null, expiresAt: type === "none" ? null : freedate }; +} + +/** 从详情链接取站点资源 ID(NexusPHP `details.php?id=`)。 */ +export function releaseIdFromUrl(url: string | undefined): string | null { + if (!url) return null; + return /[?&]id=(\d+)/.exec(url)?.[1] ?? null; +} + +export function categoryName(def: IndexerDefinition, id: string | undefined): string | null { + if (!id) return null; + const named = def.umt.categoryNames?.[id]; + if (named) return named; + for (const k of ["movie", "tv", "music"] as const) { + const hit = (def.category?.[k] ?? []).find((c) => String(c.id) === id); + if (hit?.desc) return hit.desc; + } + return null; +} + +/** SpiderTorrent(MP TorrentInfo 字段)→ SiteRelease。raw 保存完整 MP 字段(enclosure 脱敏)。 */ +export function spiderTorrentToRelease( + t: SpiderTorrent, + def: IndexerDefinition, + observedAt: string, +): SiteRelease | null { + const id = releaseIdFromUrl(t.page_url) ?? releaseIdFromUrl(t.enclosure); + if (!id || !t.title) return null; + const tz = def.umt.timezone; + const freedate = t.freedate ? parseSiteTime(t.freedate, tz) : null; + const raw: Record = { ...t }; + if (t.enclosure) raw.enclosure = redactUrl(t.enclosure); + const release: SiteRelease = { + siteId: def.id, + siteReleaseId: id, + title: t.title, + category: { id: t.category_id ?? null, name: categoryName(def, t.category_id) }, + sizeBytes: t.size ?? null, + seeders: t.seeders ?? null, + leechers: t.peers ?? null, + snatched: t.grabs ?? null, + promotion: promotionFromFactors(t.downloadvolumefactor, t.uploadvolumefactor, freedate), + hr: t.hit_and_run === undefined ? "unknown" : t.hit_and_run, + uploadedAt: parseSiteTime(t.pubdate, tz), + detailUrl: t.page_url ?? new URL(`${def.umt.detailPath}?id=${id}`, def.domain).toString(), + downloadUrl: t.enclosure + ? redactUrl(t.enclosure) + : new URL(`${def.umt.downloadPath}?id=${id}`, def.domain).toString(), + tags: t.labels ?? [], + raw, + observedAt, + }; + if (t.description) release.subtitle = t.description; + if (t.imdbid) release.imdbId = t.imdbid; + return release; +} diff --git a/packages/adapters/src/sites/spider/schema.ts b/packages/adapters/src/sites/spider/schema.ts new file mode 100644 index 00000000..3f261701 --- /dev/null +++ b/packages/adapters/src/sites/spider/schema.ts @@ -0,0 +1,126 @@ +/** + * 站点索引定义(indexer)schema。 + * + * 来源:MoviePilot e6820a8 `app/modules/indexer/spider/__init__.py` `SiteSpider.__init__`/`__get_search_url`/`get_info` + * 读取的全部键,以及 `app/application/site/sites.pyi` `SitesHelper.get_indexer(domain) -> dict`。 + * MP 的真实站点定义在加密资源 user.sites.v3.bin 中不可读,本项目的站点定义按同一 schema 手写 + * (见 ../definitions/*.json)。`umt` 键是本项目扩展(限流、时区、登录判定、详情页、分类名),MP 无对应。 + */ + +/** 字段过滤器(MP `SiteSpider.__filter_text`)。 */ +export type SpiderFilter = + | { name: "re_search"; args: [string, number] } + | { name: "split"; args: [string, number] } + | { name: "replace"; args: [string, string] } + | { name: "dateparse"; args: string } + | { name: "strip"; args?: unknown } + | { name: "appendleft"; args: string } + | { name: "querystring"; args: string }; + +/** 字段选择器(MP `_safe_query` / `__attribute_or_text` / `__index` / `__remove` 读取的键)。 */ +export interface SpiderFieldSelector { + selector?: string; + /** description 专用:selectors 等价 selector */ + selectors?: string; + attribute?: string; + index?: number; + contents?: number; + /** 逗号+空格分隔的待移除子元素选择器 */ + remove?: string; + filters?: SpiderFilter[]; + /** Jinja 模板(title/description/date) */ + text?: string; + /** 下载/上传系数:按声明顺序首个命中的选择器 → 系数 */ + case?: Record; +} + +export interface SpiderCategoryItem { + id: string | number; + cat?: string; + desc?: string; + /** 分类参数名(如 HHClub 的 cat[]) */ + param?: string; + /** 参数值(缺省用 id) */ + value?: string | number; + /** 附加开关 */ + params?: Record; +} + +export interface SpiderCategoryConfig { + movie?: SpiderCategoryItem[]; + tv?: SpiderCategoryItem[]; + music?: SpiderCategoryItem[]; + field?: string; + delimiter?: string; + param?: string; +} + +export interface SpiderListConfig { + selector: string; +} + +export interface SpiderFields { + [name: string]: SpiderFieldSelector | undefined; +} + +export interface IndexerDefinition { + id: string; + name: string; + domain: string; + encoding?: string; + public?: boolean; + proxy?: boolean; + language?: string; + /** 专用解析器(mTorrent 等);缺省走通用 SiteSpider */ + parser?: string; + result_num?: number; + timeout?: number; + /** 站点只提供某一媒体类型时填写(电影/电视剧/音乐) */ + media_type?: string; + search?: { + paths: Array<{ path: string; type?: "all" | "movie" | "tv" | "music" }>; + params?: Record; + imdbid_format?: string; + result_media_type?: "requested"; + }; + batch?: { delimiter?: string; space_replace?: string }; + browse?: { path: string; start?: number; list?: SpiderListConfig; fields?: SpiderFields }; + category?: SpiderCategoryConfig; + torrents: { list: SpiderListConfig; fields: SpiderFields }; + /** 本项目扩展(MP 无对应) */ + umt: UmtSiteExtension; +} + +export interface UmtSiteExtension { + /** 定义版本,入 site_snapshot */ + version: string; + kind: "nexusphp"; + displayName?: string; + /** 站点时间时区偏移,如 "+08:00";pubdate/freedate 按此转 UTC */ + timezone: string; + rateLimit: { minIntervalMs: number; burst: number }; + capabilities: { list: boolean; search: boolean; detail: boolean; download: boolean; rss: boolean }; + /** 原始分类 ID → 站点分类名(列表只有 ID 时补名称;不做投影) */ + categoryNames?: Record; + /** 登录成功标志;缺失且命中登录页判定时为 auth_expired */ + loginSuccess?: string; + /** 列表为空时用于区分"确实无结果"与"结构变化"的提示文本 */ + emptyMarkers?: string[]; + /** 分页链接选择器(估算总页数) */ + pagination?: string; + /** 详情页解析(MP 通用 spider 不含详情页,本项目扩展) */ + detail?: { + path: string; + title: string; + rowLabel: string; + rowValue: string; + labels: { subtitle: string[]; basicInfo: string[]; peers: string[] }; + description: string; + /** 促销/HR 标记所在范围 */ + headerScope: string; + notFoundMarkers: string[]; + }; + /** 下载路径(按资源 ID 拼接) */ + downloadPath: string; + detailPath: string; +} diff --git a/packages/adapters/src/sites/spider/spider.test.ts b/packages/adapters/src/sites/spider/spider.test.ts new file mode 100644 index 00000000..f157f3d0 --- /dev/null +++ b/packages/adapters/src/sites/spider/spider.test.ts @@ -0,0 +1,308 @@ +import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit"; +import { afterEach, describe, expect, it } from "vitest"; +import { + audiencesDefinition, + createSiteRateLimiter, + createSpiderSiteAdapter, + type IndexerDefinition, + type SiteAdapter, + SiteError, + SiteSpider, + staticSecretResolver, +} from "../index"; + +let site: MockNexusPhpSite | undefined; +afterEach(async () => { + await site?.close(); + site = undefined; +}); + +function adapterFor(s: MockNexusPhpSite, def = audiencesDefinition, cookie: string | null = s.cookie): SiteAdapter { + return createSpiderSiteAdapter(def, { + baseUrl: `${s.baseUrl}/`, + secrets: staticSecretResolver(cookie ? { [def.id]: { cookie } } : {}), + limiter: createSiteRateLimiter({ minIntervalMs: 0 }), + }); +} + +async function expectSiteError(p: Promise, kind: string): Promise { + try { + await p; + } catch (e) { + expect(e).toBeInstanceOf(SiteError); + expect((e as SiteError).kind).toBe(kind); + return e as SiteError; + } + throw new Error(`expected SiteError ${kind}`); +} + +describe("SiteSpider(MP spider 迁移)", () => { + const def: IndexerDefinition = { + ...audiencesDefinition, + domain: "https://example.com/", + category: { + movie: [{ id: "401" }], + tv: [{ id: "402" }, { id: "403" }], + }, + }; + + it("__get_search_url:默认参数、search_area 仅 IMDb 保留、mtype 选分类组、cat 白名单", () => { + const u1 = new URL(new SiteSpider(def, { keyword: "Dune", page: 2 }).getSearchUrl()); + expect(u1.pathname).toBe("/torrents.php"); + expect(Object.fromEntries(u1.searchParams)).toEqual({ + search_mode: "0", + search_area: "0", + page: "2", + notnewword: "1", + search: "Dune", + cat401: "1", + cat402: "1", + cat403: "1", + }); + const u2 = new URL(new SiteSpider(def, { keyword: "tt15239678" }).getSearchUrl()); + expect(u2.searchParams.get("search_area")).toBe("4"); + const u3 = new URL(new SiteSpider(def, { keyword: "x", mtype: "tv", cat: "403" }).getSearchUrl()); + expect([...u3.searchParams.keys()].filter((k) => k.startsWith("cat"))).toEqual(["cat403"]); + const u4 = new URL(new SiteSpider(def, { keyword: ["A B", "C"] }).getSearchUrl()); + expect(u4.searchParams.get("search")).toBe("A_B C"); + expect(u4.searchParams.get("search_mode")).toBe("1"); + // 浏览:browse.path + start + expect(new SiteSpider(def, { page: 3 }).getSearchUrl()).toBe("https://example.com/torrents.php?page=3"); + }); + + it("分类参数 param(cat[] 重复)与 field 拼接两种写法", () => { + const withParam: IndexerDefinition = { ...def, category: { param: "cat[]", tv: [{ id: "402" }, { id: "403" }] } }; + const u = new URL(new SiteSpider(withParam, { keyword: "x" }).getSearchUrl()); + expect(u.searchParams.getAll("cat[]")).toEqual(["402", "403"]); + const withField: IndexerDefinition = { ...def, category: { field: "cat", delimiter: ",", tv: [{ id: "402" }] } }; + expect(new URL(new SiteSpider(withField, { keyword: "x" }).getSearchUrl()).searchParams.get("cat")).toBe(",402"); + }); + + it("get_info 字段 DSL:模板标题、remove、dateparse、case 系数、querystring、hr 配置后两态", () => { + const html = `t + +
h
+ Full.Title… + 副标题 [置顶]x中字DIY +
01天1.5 GB1,2345/79
+ No.Title.Attr +
03 小时--x
`; + const d: IndexerDefinition = { + ...def, + torrents: { + list: { selector: 'table.torrents > tr:has("table.torrentname")' }, + fields: { + ...def.torrents.fields, + description: { selector: "font.subtitle", remove: "span,a" }, + labels: { selector: "span.label" }, + hr: { selector: "img.hitandrun" }, + downloadvolumefactor: { case: { "img.free": 0, "*": 1 } }, + }, + }, + }; + const { torrents } = new SiteSpider(d).parse(html); + expect(torrents[0]).toMatchObject({ + title: "Full.Title.2025.2160p", + description: "副标题", + page_url: "https://example.com/details.php?id=100", + size: 1610612736, + seeders: 1234, + peers: 5, + grabs: 9, + downloadvolumefactor: 0, + uploadvolumefactor: 1, + pubdate: "2025-05-01 12:13:14", + labels: ["中字", "DIY"], + hit_and_run: false, + category: "电视剧", + category_id: "402", + }); + expect(torrents[1]).toMatchObject({ + title: "No.Title.Attr", + hit_and_run: true, + size: 0, + seeders: 0, + category: "电影", + }); + expect(torrents[1]!.pubdate).toBeUndefined(); // 相对时间不写入 pubdate(MP __is_invalid_pubdate_text) + }); + + it("登录/权限页判定(MP __is_login_or_permission_page)", () => { + const r = new SiteSpider(def).parse("观众 :: 登录"); + expect(r).toMatchObject({ isError: true, loginPage: true, torrents: [] }); + const r2 = new SiteSpider(def).parse("

该页面必须在登录后才能访问

"); + expect(r2.loginPage).toBe(true); + }); +}); + +describe("观众适配器(spider + mock NexusPHP)", () => { + it("列表:page=1 发送 page=0;免费+截止、时间、原始分类、副标题;H&R 未配置为 unknown", async () => { + site = await createMockNexusPhpSite(); + const page = await adapterFor(site).list(1); + const req = site.requests[0]!; + expect(req.path).toBe("/torrents.php?page=0"); + expect(req.headers.cookie).toBe(site.cookie); + expect(page.items).toHaveLength(5); + expect(page.totalPages).toBe(1); + const dune = page.items[0]!; + expect(dune).toMatchObject({ + siteId: "audiences", + siteReleaseId: "812345", + title: "Dune.Part.Two.2024.2160p.UHD.BluRay.REMUX.DV.HDR.HEVC.TrueHD.Atmos.7.1-FraMeSToR", + subtitle: "沙丘2 / 沙丘：第二部 | 杜比视界 国英双语 中字", + category: { id: "401", name: "电影" }, + sizeBytes: 84202833838, + seeders: 1204, + leechers: 17, + snatched: 3388, + promotion: { type: "free", downloadFactor: 0, uploadFactor: 1, expiresAt: "2026-10-01T04:00:00.000Z" }, + hr: "unknown", + uploadedAt: "2026-09-26T13:14:03.000Z", + }); + expect(dune.raw).toMatchObject({ category: "电影", category_id: "401", pubdate: "2026-09-26 21:14:03" }); + expect(dune.detailUrl).toBe(`${site.baseUrl}/details.php?id=812345&hit=1`); + expect(JSON.stringify(page)).not.toContain("mock-session-secret"); + }); + + it("列表:综艺保留 403、50%+2x、2x、截断标题、TB/KB", async () => { + site = await createMockNexusPhpSite(); + const { items } = await adapterFor(site).list(1); + const byId = Object.fromEntries(items.map((i) => [i.siteReleaseId, i])); + expect(byId["812346"]!.category).toEqual({ id: "403", name: "综艺" }); + expect(byId["812346"]!.raw.category).toBe("电视剧"); + expect(byId["812346"]!.promotion.type).toBe("none"); + expect(byId["812347"]!.promotion).toMatchObject({ type: "2x50%", downloadFactor: 0.5, uploadFactor: 2 }); + expect(byId["812347"]!.promotion.expiresAt).toBe("2026-09-29T00:30:00.000Z"); + expect(byId["812347"]!.sizeBytes).toBe(1294529987); + expect(byId["812348"]!.title).toBe("The.Last.of.Us.S02.2160p.MAX.WEB-DL.DDP5.1.Atmos.D…"); + expect(byId["812348"]!.sizeBytes).toBe(2308974418330); + expect(byId["812349"]!.promotion).toMatchObject({ + type: "2x", + downloadFactor: 1, + uploadFactor: 2, + expiresAt: null, + }); + expect(byId["812349"]!.category).toEqual({ id: "406", name: "纪录片" }); + }); + + it("配置 hr 字段后 H&R 为 true/false", async () => { + site = await createMockNexusPhpSite(); + const def: IndexerDefinition = { + ...audiencesDefinition, + torrents: { + ...audiencesDefinition.torrents, + fields: { ...audiencesDefinition.torrents.fields, hr: { selector: "img.hitandrun" } }, + }, + }; + const { items } = await adapterFor(site, def).list(1); + expect(items.find((i) => i.siteReleaseId === "812346")!.hr).toBe(true); + expect(items.find((i) => i.siteReleaseId === "812345")!.hr).toBe(false); + }); + + it("搜索:关键词清洗与分类下推;无结果返回空页", async () => { + site = await createMockNexusPhpSite(); + const a = adapterFor(site); + const r = await a.search("奔跑吧!", { categoryIds: ["403"] }); + const u = new URL(site.requests[0]!.path, site.baseUrl); + expect(u.searchParams.get("search")).toBe("奔跑吧"); + expect(u.searchParams.get("cat403")).toBe("1"); + expect(u.searchParams.get("cat401")).toBeNull(); + expect(r.items.map((i) => i.siteReleaseId)).toEqual(["812346"]); + expect((await a.search("不存在的片名")).items).toEqual([]); + }); + + it("分页:pageSize=2 时 page=2 发送 page=1,总页数由分页链接估计", async () => { + site = await createMockNexusPhpSite({ pageSize: 2 }); + const p2 = await adapterFor(site).list(2); + expect(site.requests[0]!.path).toBe("/torrents.php?page=1"); + expect(p2).toMatchObject({ totalPages: 3, totalPagesEstimated: true }); + expect(p2.items.map((i) => i.siteReleaseId)).toEqual(["812347", "812348"]); + }); + + it("详情:标题/副标题/体积/同伴/促销/IMDb/豆瓣;不存在 → not_found", async () => { + site = await createMockNexusPhpSite(); + const a = adapterFor(site); + const d = await a.detail("812345"); + expect(d).toMatchObject({ + title: "Dune.Part.Two.2024.2160p.UHD.BluRay.REMUX.DV.HDR.HEVC.TrueHD.Atmos.7.1-FraMeSToR", + subtitle: "沙丘2 / 沙丘：第二部 | 杜比视界 国英双语 中字", + sizeBytes: 84202833838, + seeders: 1204, + leechers: 17, + imdbId: "tt15239678", + doubanId: "35575567", + uploadedAt: "2026-09-26T13:14:03.000Z", + category: { id: "401", name: "电影" }, + promotion: { type: "free", expiresAt: "2026-10-01T04:00:00.000Z" }, + }); + await expectSiteError(a.detail("999999"), "not_found"); + }); + + it("下载目标:带 Cookie 取 .torrent 并校验 infohash", async () => { + site = await createMockNexusPhpSite(); + const t = await adapterFor(site).downloadTarget("812345"); + expect(t.kind).toBe("torrent"); + if (t.kind !== "torrent") return; + expect(t.infoHash).toBe(site.infoHashOf("812345")); + expect(t.name).toContain("Dune.Part.Two"); + expect(t.fileName).toContain(".torrent"); + }); + + it("登录失效识别:列表/详情/下载抛 auth_expired,健康检查报告", async () => { + site = await createMockNexusPhpSite(); + const a = adapterFor(site); + expect((await a.healthCheck()).status).toBe("ok"); + site.setLoggedIn(false); + const err = await expectSiteError(a.list(1), "auth_expired"); + expect(err.isAuthError).toBe(true); + await expectSiteError(a.detail("812345"), "auth_expired"); + await expectSiteError(a.downloadTarget("812345"), "auth_expired"); + expect(await a.healthCheck()).toMatchObject({ ok: false, status: "auth_expired" }); + }); + + it("登录失效(302 login.php)与缺少 Cookie", async () => { + site = await createMockNexusPhpSite({ loginMode: "redirect" }); + await expectSiteError( + adapterFor(site, audiencesDefinition, "c_secure_uid=1; c_secure_pass=wrong").list(1), + "auth_expired", + ); + const before = site.requests.length; + await expectSiteError(adapterFor(site, audiencesDefinition, null).list(1), "auth_missing"); + expect(site.requests.length).toBe(before); + }); + + it("页面改版 → parse_error;429 + Retry-After → rate_limited", async () => { + site = await createMockNexusPhpSite(); + const a = adapterFor(site); + site.setBrokenLayout(true); + await expectSiteError(a.list(1), "parse_error"); + site.setBrokenLayout(false); + site.failNext(1, 429, "7"); + expect((await expectSiteError(a.list(1), "rate_limited")).retryAfterMs).toBe(7000); + }); + + it("站点级限流:列表/搜索/详情/下载共享同一令牌桶", async () => { + site = await createMockNexusPhpSite(); + let clock = 0; + const sleeps: number[] = []; + const limiter = createSiteRateLimiter({ + minIntervalMs: 3000, + now: () => clock, + sleep: async (ms) => { + sleeps.push(ms); + clock += ms; + }, + }); + const a = createSpiderSiteAdapter(audiencesDefinition, { + baseUrl: `${site.baseUrl}/`, + secrets: staticSecretResolver({ audiences: { cookie: site.cookie } }), + limiter, + }); + await a.list(1); + await a.search("Dune"); + await a.detail("812345"); + await a.downloadTarget("812345"); + expect(sleeps).toEqual([3000, 3000, 3000]); + expect(limiter.stats.granted).toBe(4); + }); +}); diff --git a/packages/adapters/src/sites/spider/spider.ts b/packages/adapters/src/sites/spider/spider.ts new file mode 100644 index 00000000..e9eb08ed --- /dev/null +++ b/packages/adapters/src/sites/spider/spider.ts @@ -0,0 +1,755 @@ +/** + * 通用站点爬虫(配置驱动)。 + * + * 迁移自 MoviePilot e6820a8 `app/modules/indexer/spider/__init__.py`: + * - `select_media_categories` / `resolve_category_media_type` → 同名函数 + * - `SiteSpider.__get_search_url` / `__format_search_word` → `SiteSpider.getSearchUrl` / `formatSearchWord` + * - `SiteSpider.get_info` 及 `__get_title/__get_description/__get_detail/__get_download/__get_grabs/__get_leechers/ + * __get_seeders/__get_size/__get_imdbid/__get_downloadvolumefactor/__get_uploadvolumefactor/__get_pubdate/__get_date/ + * __get_date_from_cell/__get_date_elapsed/__get_free_date/__get_labels/__get_hit_and_run/__get_category` + * → `SiteSpider.getInfo` 与同名私有方法 + * - `_safe_query / __filter_text / __remove / __attribute_or_text / __index` → 同名方法 + * - `__is_login_or_permission_page` → `isLoginOrPermissionPage` + * - `parse` → `SiteSpider.parse`(不含 Rust 加速分支与字幕分支) + * - `app/foundation/size.py parse_size` → `parseSizeMp` + * + * 与 MP 的差异(均在 README 与迁移账本登记): + * 1. 额外输出 `category_id`(原始分类 ID),`category` 仍是 MP 的媒体类型投影(REQ §4.5 原始分类不得丢失)。 + * 2. 系数 `selector` 模式用浮点而非 int()(MP 对 0.5 抛异常导致整条丢失,迁移账本 SA4)。 + * 3. 不发请求:HTTP/Cookie/UA 由 adapter 注入,本类只负责 URL 构造与 HTML 解析(便于限流与脱敏统一)。 + */ +import { type CheerioAPI, load } from "cheerio"; +import type { IndexerDefinition, SpiderFieldSelector, SpiderFields, SpiderFilter, SpiderListConfig } from "./schema"; +import { renderTemplate } from "./template"; + +type Sel = ReturnType; + +/** MP `MediaType` 取值 */ +export type MpMediaType = "电影" | "电视剧" | "音乐" | "未知"; +export type MediaKind = "movie" | "tv" | "music"; + +const MEDIA_VALUE: Record = { movie: "电影", tv: "电视剧", music: "音乐" }; + +/** + * 以 htmlparser2(HTML 模式)加载:与 MP 的 lxml.html 一致不自动插入 , + * 使 `table.torrents > tr` 这类 MP 风格选择器保持原语义。 + */ +export function loadHtml(html: string): CheerioAPI { + return load(html, { xml: { xmlMode: false, decodeEntities: true } }); +} + +/** + * PyQuery(cssselect)接受 `:has("sel")` 引号写法,cheerio-select 不接受;统一去引号以兼容 MP 风格定义。 + */ +export function normalizeSelector(sel: string): string { + return sel.replace(/:has\((["'])(.*?)\1\)/g, ":has($2)"); +} + +/** 解析结果:键名与 MP `TorrentInfo` 字段一致,`category_id` 为本项目扩展。 */ +export interface SpiderTorrent { + title?: string; + description?: string; + page_url?: string; + enclosure?: string; + grabs?: number; + peers?: number; + seeders?: number; + size?: number; + imdbid?: string; + downloadvolumefactor?: number; + uploadvolumefactor?: number; + pubdate?: string; + date_elapsed?: string; + freedate?: string; + labels?: string[]; + hit_and_run?: boolean; + category?: MpMediaType; + category_id?: string; +} + +/** MP `select_media_categories` */ +export function selectMediaCategories(category: IndexerDefinition["category"], mtype?: MediaKind) { + if (!category) return []; + if (mtype === "tv") return category.tv ?? []; + if (mtype === "movie") return category.movie ?? []; + if (mtype === "music") return category.music ?? []; + return [...(category.movie ?? []), ...(category.tv ?? []), ...(category.music ?? [])]; +} + +/** MP `resolve_category_media_type`:同一 ID 只落在一个媒体类型里才投影,否则未知。 */ +export function resolveCategoryMediaType(value: unknown, category: IndexerDefinition["category"]): MpMediaType { + if (value == null || !category) return "未知"; + const id = String(value); + const matches = (["movie", "tv", "music"] as const).filter((k) => + (category[k] ?? []).some((item) => item.id != null && String(item.id) === id), + ); + return matches.length === 1 ? MEDIA_VALUE[matches[0]!] : "未知"; +} + +/** MP `size_tools.parse_size`:二进制单位,round。 */ +export function parseSizeMp(text: string | number | null | undefined): number { + if (!text) return 0; + const s = String(text); + if (/^\d+$/.test(s)) return Number(s); + const normalized = s.replace(/,/g, "").replace(/ /g, "").toUpperCase(); + const sizeText = normalized.replace(/[KMGTPI]*B?/gi, ""); + const size = Number(sizeText); + if (sizeText === "" || !Number.isFinite(size)) return 0; + let v = size; + if (normalized.includes("PB") || normalized.includes("PIB")) v *= 1024 ** 5; + else if (normalized.includes("TB") || normalized.includes("TIB")) v *= 1024 ** 4; + else if (normalized.includes("GB") || normalized.includes("GIB")) v *= 1024 ** 3; + else if (normalized.includes("MB") || normalized.includes("MIB")) v *= 1024 ** 2; + else if (normalized.includes("KB") || normalized.includes("KIB")) v *= 1024; + return Math.round(v); +} + +/** strptime 结果(MP dateparse 过滤器返回 datetime) */ +class ParsedDateTime { + constructor(readonly text: string) {} +} + +const STRPTIME_TOKENS: Record = { + "%Y": "(\\d{4})", + "%m": "(\\d{1,2})", + "%d": "(\\d{1,2})", + "%H": "(\\d{1,2})", + "%M": "(\\d{1,2})", + "%S": "(\\d{1,2})", +}; + +/** Python strptime 的最小子集(%Y %m %d %H %M %S),失败抛错(与 MP 一致由调用方吞掉)。 */ +function strptime(text: string, fmt: string): ParsedDateTime { + const order: string[] = []; + const re = fmt.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/%[YmdHMS]/g, (t) => { + order.push(t); + return STRPTIME_TOKENS[t]!; + }); + const m = new RegExp(`^${re}$`).exec(text); + if (!m) throw new Error(`time data '${text}' does not match format '${fmt}'`); + const v: Record = { "%Y": 1900, "%m": 1, "%d": 1, "%H": 0, "%M": 0, "%S": 0 }; + order.forEach((t, i) => { + v[t] = Number(m[i + 1]); + }); + const p = (n: number) => String(n).padStart(2, "0"); + return new ParsedDateTime(`${v["%Y"]}-${p(v["%m"]!)}-${p(v["%d"]!)} ${p(v["%H"]!)}:${p(v["%M"]!)}:${p(v["%S"]!)}`); +} + +const STD_TIME = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/; + +/** MP `time_tools.normalize_datetime` 的保守子集:只接受可确定解析的绝对时间。 */ +function normalizeDatetime(value: string): string { + const m = /^(\d{4})[-/年.](\d{1,2})[-/月.](\d{1,2})日?(?:[ T](\d{1,2}):(\d{1,2})(?::(\d{1,2}))?)?$/.exec( + value.trim(), + ); + if (!m) return value; + const p = (n: string | undefined) => String(Number(n ?? 0)).padStart(2, "0"); + return `${m[1]}-${p(m[2])}-${p(m[3])} ${p(m[4])}:${p(m[5])}:${p(m[6])}`; +} + +function isRelativePubdateText(pubdate: string | null | undefined): boolean { + if (!pubdate) return false; + const text = String(pubdate).trim().toLowerCase(); + if (/\d{4}[-/年]\d{1,2}/.test(text)) return false; + if (text.includes("ago")) return true; + return /\d+\s*(秒|分钟|分|小时|天|周|月|年)/.test(text); +} + +function isInvalidPubdateText(pubdate: string | null | undefined): boolean { + if (!pubdate) return true; + const text = String(pubdate).trim(); + if (text.toLowerCase() === "now" || text === "0") return true; + if (isRelativePubdateText(text)) return true; + return !STD_TIME.test(text); +} + +function pyIndex(arr: T[], i: number): T | undefined { + return i < 0 ? arr[arr.length + i] : arr[i]; +} + +/** Python `parse_qs(urlparse(url).query).get(key)[0]` */ +function querystringValue(text: string, key: string): string { + const q = text.includes("?") ? text.slice(text.indexOf("?") + 1) : ""; + const v = new URLSearchParams(q.split("#")[0]).getAll(key).filter((x) => x !== ""); + return v[0] ?? ""; +} + +/** PyQuery `.text()` 近似:`
` 视为换行,行内空白折叠。 */ +function pqText(el: Sel): string { + const c = el.clone(); + c.find("br").replaceWith("\n"); + return c + .text() + .split("\n") + .map((l) => l.replace(/[ \t\r\f\v\u00a0]+/g, " ").trim()) + .filter((l) => l !== "") + .join("\n"); +} + +export interface SiteSpiderQuery { + keyword?: string | string[]; + mtype?: MediaKind; + /** 逗号分隔的分类 ID 白名单(MP `cat`) */ + cat?: string; + /** 站点内部页码(MP 0 起) */ + page?: number; +} + +export class SiteSpider { + readonly def: IndexerDefinition; + readonly keyword: string | string[] | undefined; + readonly mtype: MediaKind | undefined; + readonly cat: string | undefined; + readonly page: number; + readonly domain: string; + readonly resultNum: number; + readonly siteMediaType: MediaKind | undefined; + private list: SpiderListConfig; + private fields: SpiderFields; + private requestedResultMediaType: MediaKind | undefined; + private $!: CheerioAPI; + private info: SpiderTorrent = {}; + + constructor(def: IndexerDefinition, q: SiteSpiderQuery = {}) { + this.def = def; + this.keyword = q.keyword; + this.mtype = q.mtype; + this.cat = q.cat; + this.page = q.page ?? 0; + this.siteMediaType = fromAgent(def.media_type); + this.list = def.torrents.list; + this.fields = def.torrents.fields ?? {}; + if (!q.keyword && def.browse) { + this.list = def.browse.list ?? this.list; + this.fields = def.browse.fields ?? this.fields; + } + this.domain = def.domain.endsWith("/") ? def.domain : `${def.domain}/`; + this.resultNum = Number(def.result_num ?? 100); + } + + /** MP `__can_search` */ + canSearch(): boolean { + if (this.siteMediaType && this.mtype && this.siteMediaType !== this.mtype) return false; + return Boolean(this.def.search && this.domain); + } + + /** MP `__format_search_word` */ + private formatSearchWord(word: string): string { + if (word && /^tt\d+$/.test(word) && this.def.search?.imdbid_format) { + return this.def.search.imdbid_format + .replace(/\{keyword\}/g, word) + .replace(/\{imdbid\}/g, word) + .replace(/\{imdbid_num\}/g, word.slice(2)); + } + return word; + } + + /** MP `__get_search_url` */ + getSearchUrl(): string { + const search = this.def.search ?? { paths: [] }; + const paths = search.paths ?? []; + let torrentspath = ""; + let typedPathSelected = false; + let categoryFilterSelected = false; + if (paths.length === 1) { + torrentspath = paths[0]!.path ?? ""; + } else { + let fallback = ""; + const expected = this.mtype; + for (const p of paths) { + if (p.type === "all" && !fallback) fallback = p.path ?? ""; + if ((expected && p.type === expected) || (!expected && p.type === "all")) { + torrentspath = p.path ?? ""; + typedPathSelected = Boolean(expected && p.type === expected); + break; + } + } + if (!torrentspath) torrentspath = fallback; + } + + let searchurl: string; + if (this.keyword && (typeof this.keyword === "string" || this.keyword.length > 0)) { + let searchWord: string; + let searchMode: string; + if (Array.isArray(this.keyword)) { + if (this.def.batch) { + const delimiter = this.def.batch.delimiter || " "; + const spaceReplace = this.def.batch.space_replace || " "; + searchWord = this.keyword.map((k) => String(k).replace(/ /g, spaceReplace)).join(delimiter); + } else { + searchWord = this.keyword.join(" "); + } + searchMode = "1"; + } else { + searchWord = this.keyword; + searchMode = "0"; + } + const isImdb = typeof this.keyword === "string" && /^tt\d+$/.test(this.keyword); + searchWord = this.formatSearchWord(searchWord); + const indexerParams: Record = { ...(search.params ?? {}) }; + if (Object.keys(indexerParams).length > 0) { + if (indexerParams.search_area && !isImdb) delete indexerParams.search_area; + const params: Record> = { + search_mode: searchMode, + search_area: 0, + page: this.page || 0, + notnewword: 1, + }; + for (const [k, v] of Object.entries(indexerParams)) params[k] = String(v).replace(/\{keyword\}/g, searchWord); + const category = this.def.category; + if (category) { + const cats = selectMediaCategories(category, this.mtype); + const allowed = this.cat ? new Set(this.cat.split(",")) : null; + for (const c of cats) { + if (allowed && !allowed.has(String(c.id))) continue; + if (category.field) { + categoryFilterSelected = true; + const cur = String(params[category.field] ?? ""); + params[category.field] = cur + (category.delimiter ?? " ") + String(c.id); + } else { + const categoryParam = c.param || category.param; + if (categoryParam) { + categoryFilterSelected = true; + const cid = c.value ?? c.id; + const cur = params[categoryParam]; + if (cur === undefined) params[categoryParam] = cid; + else if (Array.isArray(cur)) cur.push(cid); + else params[categoryParam] = [cur, cid]; + } else { + categoryFilterSelected = true; + params[`cat${c.id}`] = 1; + } + } + if (c.params) Object.assign(params, c.params); + } + } + searchurl = combineUrl(this.domain, torrentspath, params); + } else { + searchurl = + this.domain + + torrentspath + .replace(/\{keyword\}/g, encodeURIComponent(searchWord)) + .replace(/\{page\}/g, String(this.page || 0)); + } + } else { + let page = this.page || 0; + if (this.def.browse && !typedPathSelected) { + torrentspath = this.def.browse.path; + if (this.def.browse.start) page = Number(this.def.browse.start) + Number(this.page || 0); + } else if (this.page && !torrentspath.includes("{page}")) { + torrentspath += `${torrentspath.includes("?") ? "&" : "?"}page=${this.page}`; + } + searchurl = this.domain + torrentspath.replace(/\{page\}/g, String(page)).replace(/\{keyword\}/g, ""); + } + + this.requestedResultMediaType = + search.result_media_type === "requested" && this.mtype && (typedPathSelected || categoryFilterSelected) + ? this.mtype + : undefined; + return searchurl; + } + + /** MP `__is_login_or_permission_page` */ + static isLoginOrPermissionPage($: CheerioAPI): boolean { + const title = $("title").first().text().trim(); + const pageText = $.root().text().split(/\s+/).filter(Boolean).join(" ").slice(0, 1000); + if (title === "登录" || title.includes(":: 登录")) return true; + return ["未登录", "登录 / 注册", "必须在登录后才能访问", "你需要启用cookies才能登录"].some((m) => + pageText.includes(m), + ); + } + + /** MP `parse`:登录/权限页返回 isError=true 与空列表。 */ + parse(html: string): { isError: boolean; loginPage: boolean; torrents: SpiderTorrent[]; $: CheerioAPI | null } { + if (!html) return { isError: true, loginPage: false, torrents: [], $: null }; + const $ = loadHtml(html); + if (SiteSpider.isLoginOrPermissionPage($)) return { isError: true, loginPage: true, torrents: [], $ }; + this.$ = $; + const out: SpiderTorrent[] = []; + const rows = $(normalizeSelector(this.list.selector ?? "")); + rows.each((i, el) => { + if (i >= this.resultNum) return false; + const info = this.getInfo($(el)); + if (Object.keys(info).length > 0) out.push(this.applyRequestedMediaType(info)); + return undefined; + }); + return { isError: false, loginPage: false, torrents: out, $ }; + } + + /** 对任意片段(如详情页头部)按字段配置取值 */ + infoFrom($: CheerioAPI, el: Sel): SpiderTorrent { + this.$ = $; + return this.getInfo(el); + } + + private applyRequestedMediaType(t: SpiderTorrent): SpiderTorrent { + if (this.requestedResultMediaType) t.category = MEDIA_VALUE[this.requestedResultMediaType]; + return t; + } + + /** MP `get_info`:任一字段异常则整条返回 {}。 */ + getInfo(row: Sel): SpiderTorrent { + this.info = {}; + try { + this.getTitle(row); + this.getDescription(row); + this.getDetail(row); + this.getDownload(row); + this.getGrabs(row); + this.getLeechers(row); + this.getSeeders(row); + this.getSize(row); + this.getImdbid(row); + this.getVolumeFactor(row, "downloadvolumefactor"); + this.getVolumeFactor(row, "uploadvolumefactor"); + this.getPubdate(row); + this.getDateElapsed(row); + this.getFreeDate(row); + this.getLabels(row); + this.getHitAndRun(row); + this.getCategory(row); + return { ...this.info }; + } catch { + return {}; + } + } + + // ---- 字段 ---- + + private getTitle(row: Sel) { + if (!("title" in this.fields)) return; + const selector = this.fields.title ?? {}; + let title: string | null | undefined; + if (selector.selector) { + title = this.safeQuery(row, selector); + } else if (selector.text) { + const render: Record = {}; + if ("title_default" in this.fields) render.title_default = this.safeQuery(row, this.fields.title_default); + if ("title_optional" in this.fields) render.title_optional = this.safeQuery(row, this.fields.title_optional); + title = renderTemplate(selector.text, render); + } + const v = this.filterText(title, selector.filters); + if (v != null) this.info.title = String(v); + } + + private getDescription(row: Sel) { + if (!("description" in this.fields)) return; + const selector = this.fields.description ?? {}; + let desc: string | null | undefined; + if (selector.selector || selector.selectors) { + const s: SpiderFieldSelector = { ...selector }; + if (selector.selectors && !selector.selector) s.selector = selector.selectors; + desc = this.safeQuery(row, s); + } else if (selector.text) { + const render: Record = {}; + for (const k of ["tags", "subject", "description_free_forever", "description_normal"]) { + if (k in this.fields) render[k] = this.safeQuery(row, this.fields[k]); + } + desc = renderTemplate(selector.text, render); + } + const v = this.filterText(desc, selector.filters); + if (v != null) this.info.description = String(v); + } + + private normalizeLink(link: string): string { + if (link.startsWith("http")) return link; + if (link.startsWith("//")) return `${this.domain.split(":")[0]}:${link}`; + if (link.startsWith("/")) return this.domain + link.slice(1); + return this.domain + link; + } + + private getDetail(row: Sel) { + if (!("details" in this.fields)) return; + const selector = this.fields.details ?? {}; + const link = this.filterText(this.safeQuery(row, selector), selector.filters); + if (link) this.info.page_url = this.normalizeLink(String(link)); + } + + private getDownload(row: Sel) { + if (!("download" in this.fields)) return; + const selector = this.fields.download ?? {}; + const raw = this.filterText(this.safeQuery(row, selector), selector.filters); + if (!raw) return; + const link = String(raw); + if (!link.startsWith("http") && !link.startsWith("magnet")) { + const [scheme, host] = splitNetloc(this.domain); + if (host && link.includes(host)) { + this.info.enclosure = link.startsWith("/") ? `${scheme}:${link}` : `${scheme}://${link}`; + } else { + this.info.enclosure = link.startsWith("/") ? `${this.domain}${link.slice(1)}` : `${this.domain}${link}`; + } + } else { + this.info.enclosure = link; + } + } + + private countField(row: Sel, name: string, key: "grabs" | "peers" | "seeders", allowZeroText: boolean) { + if (!(name in this.fields)) return; + const selector = this.fields[name] ?? {}; + const item = this.safeQuery(row, selector); + const present = allowZeroText ? item != null && item !== "" : Boolean(item); + if (present) { + let v: unknown = String(item).split("/")[0]!.replace(/,/g, ""); + v = this.filterText(v as string, selector.filters); + this.info[key] = typeof v === "string" && /^\d+$/.test(v) ? Number(v) : 0; + } else { + this.info[key] = 0; + } + } + + private getGrabs(row: Sel) { + this.countField(row, "grabs", "grabs", true); + } + private getLeechers(row: Sel) { + this.countField(row, "leechers", "peers", false); + } + private getSeeders(row: Sel) { + this.countField(row, "seeders", "seeders", false); + } + + private getSize(row: Sel) { + if (!("size" in this.fields)) return; + const selector = this.fields.size ?? {}; + const item = this.safeQuery(row, selector); + if (item != null && item !== "") { + const v = this.filterText(item.replace(/\n/g, "").trim(), selector.filters); + this.info.size = parseSizeMp(v == null ? "" : String(v)); + } else { + this.info.size = 0; + } + } + + private getImdbid(row: Sel) { + if (!("imdbid" in this.fields)) return; + const selector = this.fields.imdbid ?? {}; + const v = this.filterText(this.safeQuery(row, selector), selector.filters); + if (v != null) this.info.imdbid = String(v); + } + + private getVolumeFactor(row: Sel, key: "downloadvolumefactor" | "uploadvolumefactor") { + const selector = this.fields[key]; + if (!selector || Object.keys(selector).length === 0) return; + this.info[key] = 1; + if (selector.case) { + for (const [css, value] of Object.entries(selector.case)) { + if (row.find(normalizeSelector(css)).length > 0) { + this.info[key] = value; + break; + } + } + } else if (selector.selector) { + const item = this.safeQuery(row, selector); + const m = item ? /(\d+\.?\d*)/.exec(item) : null; + // 偏离 MP:MP 用 int() 转换,"0.5" 抛异常导致整条丢失(迁移账本 SA4) + if (m) this.info[key] = Number.parseFloat(m[1]!); + } + } + + private getPubdate(row: Sel) { + if (!("date_added" in this.fields) && !("date" in this.fields)) return; + let selector: SpiderFieldSelector = this.fields.date_added ?? {}; + let pubdate: string | null | undefined = this.safeQuery(row, selector); + if (!pubdate) { + selector = this.fields.date ?? {}; + pubdate = this.getDate(row, selector); + } + if (pubdate) pubdate = pubdate.replace(/\n/g, " ").trim(); + const filtered = this.filterText(pubdate, selector.filters); + let value: string | undefined; + if (filtered instanceof ParsedDateTime) value = filtered.text; + else if (filtered) value = STD_TIME.test(String(filtered)) ? String(filtered) : normalizeDatetime(String(filtered)); + if (value && !isInvalidPubdateText(value)) this.info.pubdate = value; + else delete this.info.pubdate; + } + + private getDate(row: Sel, selector: SpiderFieldSelector): string | null { + if (!selector || Object.keys(selector).length === 0) return null; + if (selector.selector) return this.safeQuery(row, selector) ?? null; + if (!selector.text) return null; + const render: Record = {}; + for (const name of ["date_elapsed", "date_added"]) { + const fs = this.fields[name] ?? {}; + let v = this.safeQuery(row, fs) ?? null; + if (!v) v = this.getDateFromCell(row, fs); + render[name] = v; + } + if (!Object.values(render).some(Boolean)) return null; + const s = renderTemplate(selector.text, render); + if (s === "now" || isRelativePubdateText(s)) return null; + return s; + } + + private getDateFromCell(row: Sel, selector: SpiderFieldSelector): string | null { + const sel = selector.selector?.trim(); + if (!sel?.includes("> span")) return null; + return this.safeQuery(row, { selector: sel.split("> span")[0]!.trim() }) ?? null; + } + + private getDateElapsed(row: Sel) { + if (!("date_elapsed" in this.fields)) return; + const selector = this.fields.date_elapsed ?? {}; + const v = this.filterText(this.safeQuery(row, selector), selector.filters); + if (v != null) this.info.date_elapsed = String(v); + } + + private getFreeDate(row: Sel) { + if (!("freedate" in this.fields)) return; + const selector = this.fields.freedate ?? {}; + const v = this.filterText(this.safeQuery(row, selector), selector.filters); + if (v != null) this.info.freedate = v instanceof ParsedDateTime ? v.text : String(v); + } + + private getLabels(row: Sel) { + if (!("labels" in this.fields)) return; + const selector = this.fields.labels ?? {}; + if (!selector.selector) { + this.info.labels = []; + return; + } + const q = row.find(normalizeSelector(selector.selector)).clone(); + this.remove(q, selector); + this.info.labels = this.attributeOrText(q, selector).filter((x): x is string => Boolean(x)); + } + + private getHitAndRun(row: Sel) { + if (!("hr" in this.fields)) return; + const selector = this.fields.hr ?? {}; + this.info.hit_and_run = selector.selector ? row.find(normalizeSelector(selector.selector)).length > 0 : false; + } + + private getCategory(row: Sel) { + if (this.requestedResultMediaType) { + this.info.category = MEDIA_VALUE[this.requestedResultMediaType]; + return; + } + if (!("category" in this.fields)) { + if (this.siteMediaType) this.info.category = MEDIA_VALUE[this.siteMediaType]; + return; + } + const selector = this.fields.category ?? {}; + const value = this.filterText(this.safeQuery(row, selector), selector.filters); + if (value != null && value !== "") this.info.category_id = String(value); + let resolved = resolveCategoryMediaType(value, this.def.category); + if (resolved === "未知" && this.siteMediaType) resolved = MEDIA_VALUE[this.siteMediaType]; + this.info.category = resolved; + } + + // ---- 通用查询 ---- + + /** MP `_safe_query` */ + safeQuery(row: Sel, config: SpiderFieldSelector | undefined): string | null | undefined { + if (!config?.selector) return null; + let q = row.find(normalizeSelector(config.selector)); + if (config.remove) q = q.clone(); + this.remove(q, config); + return this.index(this.attributeOrText(q, config), config); + } + + private remove(q: Sel, config: SpiderFieldSelector) { + if (!config.remove) return; + for (const v of config.remove.split(", ")) q.find(normalizeSelector(v)).remove(); + } + + private attributeOrText(q: Sel, config: SpiderFieldSelector): Array { + if (q.length === 0) return []; + const out: Array = []; + q.each((_, el) => { + const item = this.$(el); + out.push(config.attribute ? item.attr(config.attribute) : pqText(item)); + }); + return out; + } + + private index(items: Array, config: SpiderFieldSelector): string | undefined { + if (items.length === 0) return undefined; + if (config.contents !== undefined && items.length > Number(config.contents)) { + return pyIndex((items[0] ?? "").split("\n"), Number(config.contents)); + } + if (config.index !== undefined && items.length > Number(config.index)) return pyIndex(items, Number(config.index)); + return items[0]; + } + + /** MP `__filter_text`;单个过滤器失败只跳过该过滤器。 */ + filterText( + text: string | null | undefined, + filters: SpiderFilter[] | undefined, + ): string | ParsedDateTime | null | undefined { + if (!text || !filters || !Array.isArray(filters)) return text; + let t: string | ParsedDateTime = String(text); + for (const f of filters) { + if (!t) break; + try { + if (t instanceof ParsedDateTime) break; + const s: string = t; + switch (f.name) { + case "re_search": { + if (!Array.isArray(f.args)) break; + const m = new RegExp(f.args[0]).exec(s); + if (m) t = m[Number(f.args[f.args.length - 1])] ?? ""; + break; + } + case "split": + if (Array.isArray(f.args)) { + const parts = s.split(f.args[0]); + const v = pyIndex(parts, Number(f.args[f.args.length - 1])); + if (v === undefined) throw new Error("list index out of range"); + t = v; + } + break; + case "replace": + if (Array.isArray(f.args)) t = s.split(String(f.args[0])).join(String(f.args[f.args.length - 1])); + break; + case "dateparse": + if (typeof f.args === "string") t = strptime(s.replace(/\n/g, " ").trim(), f.args); + break; + case "strip": + t = s.trim(); + break; + case "appendleft": + t = `${f.args}${s}`; + break; + case "querystring": + t = querystringValue(s, String(f.args)); + break; + } + } catch { + // MP 仅 debug 日志,继续下一个过滤器 + } + } + return typeof t === "string" ? t.trim() : t; + } +} + +/** MP `MediaType.from_agent` 的子集 */ +function fromAgent(v: string | undefined): MediaKind | undefined { + if (!v) return undefined; + const s = v.toLowerCase(); + if (s === "movie" || v === "电影") return "movie"; + if (s === "tv" || v === "电视剧") return "tv"; + if (s === "music" || v === "音乐") return "music"; + return undefined; +} + +/** MP `url.split_netloc` */ +function splitNetloc(url: string): [string, string] { + if (!url) return ["", ""]; + if (!url.startsWith("http")) return ["http", url]; + const u = new URL(url); + return [u.protocol.replace(":", ""), u.host]; +} + +/** MP `UrlUtils.combine_url`:urljoin + 合并已有 query + urlencode(doseq=True)。 */ +export function combineUrl( + host: string, + path: string, + query: Record>, +): string { + const base = host.endsWith("/") ? host : `${host}/`; + const url = new URL(path || "/", base); + for (const [k, v] of Object.entries(query)) { + url.searchParams.delete(k); + if (Array.isArray(v)) for (const x of v) url.searchParams.append(k, String(x)); + else url.searchParams.set(k, String(v)); + } + return url.toString(); +} diff --git a/packages/adapters/src/sites/spider/template.ts b/packages/adapters/src/sites/spider/template.ts new file mode 100644 index 00000000..ff066c59 --- /dev/null +++ b/packages/adapters/src/sites/spider/template.ts @@ -0,0 +1,17 @@ +import nunjucks from "nunjucks"; + +/** + * Jinja 模板渲染(MP `SiteSpider.__build_field_templates` 用 jinja2.Template)。 + * nunjucks 兼容 Jinja 语法(`{% if %}`、`fields['x']`、`a if b else c`);模板只来自随产品发布的站点定义。 + */ +const env = new nunjucks.Environment(null, { autoescape: false, throwOnUndefined: false }); +const cache = new Map(); + +export function renderTemplate(text: string, fields: Record): string { + let tpl = cache.get(text); + if (!tpl) { + tpl = new nunjucks.Template(text, env); + cache.set(text, tpl); + } + return tpl.render({ fields }); +} diff --git a/packages/adapters/src/sites/types.ts b/packages/adapters/src/sites/types.ts new file mode 100644 index 00000000..1cea0954 --- /dev/null +++ b/packages/adapters/src/sites/types.ts @@ -0,0 +1,189 @@ +/** 站点适配器自有类型(不依赖 @umt/contracts;W2 负责映射到 source_release / site_snapshot)。 */ + +export type SiteKind = "nexusphp" | "mteam"; + +export interface SiteCapabilities { + list: boolean; + search: boolean; + detail: boolean; + download: boolean; + rss: boolean; +} + +/** 站点原始分类:ID 原样保存(观众 403 综艺、馒头 404/405/407 等),不投影成 movie/tv。 */ +export interface SiteCategory { + id: string | null; + name: string | null; +} + +export type PromotionType = "none" | "free" | "2xfree" | "50%" | "2x50%" | "30%" | "2x" | "unknown"; + +export interface SitePromotion { + type: PromotionType; + /** 下载计量系数(0 = 免费);无法判定为 null */ + downloadFactor: number | null; + uploadFactor: number | null; + /** 促销截止(UTC ISO);未知或永久为 null */ + expiresAt: string | null; +} + +/** H&R 三态:true/false/'unknown';unknown 在清理/Move 判定中按"有义务"保守处理。 */ +export type HitAndRun = boolean | "unknown"; + +export interface SiteRelease { + siteId: string; + siteReleaseId: string; + title: string; + subtitle?: string; + category: SiteCategory; + /** 未知为 null(未知 ≠ 0) */ + sizeBytes: number | null; + seeders: number | null; + leechers: number | null; + snatched: number | null; + promotion: SitePromotion; + hr: HitAndRun; + /** 站点发布时间(UTC ISO),≠ 发现时间 */ + uploadedAt: string | null; + /** 不含任何凭据的详情页 URL */ + detailUrl: string; + /** 不含凭据的下载入口(可能仍需 cookie);M-Team 为空,下载时现场 genDlToken */ + downloadUrl?: string; + infoHash?: string; + imdbId?: string; + doubanId?: string; + tags: string[]; + /** 原始字段摘录(可 JSON 化,入 raw jsonb) */ + raw: Record; + /** 本次观察时间(UTC ISO) */ + observedAt: string; +} + +export interface SiteReleaseDetail extends SiteRelease { + descriptionText?: string; +} + +export interface SiteListFilters { + /** 媒体类型(MP mtype):选择站点定义中的分类组与专用搜索路径 */ + mtype?: "movie" | "tv" | "music"; + /** 站点原始分类 ID */ + categoryIds?: string[]; + /** 只要免费(站点支持时下推到请求,否则本地过滤) */ + freeOnly?: boolean; +} + +export interface SitePage { + items: SiteRelease[]; + /** 对外页码,1 起 */ + page: number; + totalPages: number | null; + /** 总页数是否由分页链接推断(估计值) */ + totalPagesEstimated: boolean; + fetchedAt: string; +} + +export type DownloadTarget = + | { kind: "magnet"; uri: string; verifiedAt: string } + /** URL 形态:url 可能含一次性令牌,只可在内存中使用,不得入库入日志;redactedUrl 可记录 */ + | { kind: "url"; url: string; redactedUrl: string; verifiedAt: string } + | { + kind: "torrent"; + bytes: Uint8Array; + infoHash: string | null; + name: string | null; + fileName: string | null; + verifiedAt: string; + }; + +export type SiteHealthStatus = + | "ok" + | "auth_missing" + | "auth_expired" + | "auth_invalid" + | "rate_limited" + | "unreachable" + | "site_maintenance" + | "parse_error"; + +export interface SiteHealth { + siteId: string; + status: SiteHealthStatus; + ok: boolean; + checkedAt: string; + latencyMs: number | null; + message?: string; +} + +export interface SiteCallOptions { + signal?: AbortSignal; +} + +export interface SiteAdapter { + readonly id: string; + readonly kind: SiteKind; + readonly displayName: string; + readonly capabilities: SiteCapabilities; + /** 站点定义版本,入 site_snapshot 便于审计解析器变化 */ + readonly definitionVersion: string; + list(page: number, filters?: SiteListFilters, opts?: SiteCallOptions): Promise; + search(query: string, filters?: SiteListFilters & { page?: number }, opts?: SiteCallOptions): Promise; + detail(siteReleaseId: string, opts?: SiteCallOptions): Promise; + /** 下载前现场取目标:会复核登录态/令牌,校验返回确为 .torrent */ + downloadTarget(siteReleaseId: string, opts?: SiteCallOptions): Promise; + /** 不抛错;把失败归类为状态 */ + healthCheck(opts?: SiteCallOptions): Promise; +} + +export type SiteErrorKind = + | "auth_missing" + | "auth_expired" + | "auth_invalid" + | "rate_limited" + | "parse_error" + | "network" + | "site_maintenance" + | "not_found" + | "unsupported"; + +/** 所有 message 已脱敏,不含 cookie/apikey/passkey/一次性下载令牌。 */ +export class SiteError extends Error { + readonly kind: SiteErrorKind; + readonly siteId: string; + readonly httpStatus?: number; + readonly retryAfterMs?: number; + constructor( + siteId: string, + kind: SiteErrorKind, + message: string, + opts: { httpStatus?: number; retryAfterMs?: number } = {}, + ) { + super(message); + this.name = "SiteError"; + this.siteId = siteId; + this.kind = kind; + if (opts.httpStatus !== undefined) this.httpStatus = opts.httpStatus; + if (opts.retryAfterMs !== undefined) this.retryAfterMs = opts.retryAfterMs; + } + /** 凭据类错误:应立即置账号验证失败,不再重试 */ + get isAuthError(): boolean { + return this.kind === "auth_missing" || this.kind === "auth_expired" || this.kind === "auth_invalid"; + } +} + +export type SiteCredentialKind = "cookie" | "apiKey" | "passkey"; + +/** 凭证注入:调用方从 secret store 解出明文字符串交给适配器;适配器不持久化、不打印。 */ +export interface SecretResolver { + resolve(req: { siteId: string; kind: SiteCredentialKind }): Promise; +} + +/** 测试/脚本用:固定凭证表。 */ +export function staticSecretResolver( + table: Record>>, +): SecretResolver { + return { + async resolve({ siteId, kind }) { + return table[siteId]?.[kind] ?? null; + }, + }; +} diff --git a/packages/adapters/src/sites/util.ts b/packages/adapters/src/sites/util.ts new file mode 100644 index 00000000..80c71a15 --- /dev/null +++ b/packages/adapters/src/sites/util.ts @@ -0,0 +1,100 @@ +import { createHash } from "node:crypto"; + +/** "1,204" / "532 / 3" → 1204 / 532;非数字返回 null(未知 ≠ 0)。 */ +export function parseCount(text: string | null | undefined): number | null { + if (text == null) return null; + const head = String(text).split("/")[0]!.replace(/[,\s]/g, ""); + return /^\d+$/.test(head) ? Number(head) : null; +} + +/** 站点本地时间 "YYYY-MM-DD HH:mm:ss" + 站点时区偏移("+08:00")→ UTC ISO。 */ +export function parseSiteTime(text: string | null | undefined, tzOffset: string): string | null { + if (!text) return null; + const m = /(\d{4})-(\d{2})-(\d{2})[ T](\d{2}):(\d{2})(?::(\d{2}))?/.exec(text); + if (!m) return null; + const iso = `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6] ?? "00"}${tzOffset}`; + const t = Date.parse(iso); + return Number.isNaN(t) ? null : new Date(t).toISOString(); +} + +export function extractImdbId(text: string | null | undefined): string | undefined { + return text ? (/\b(tt\d{6,10})\b/.exec(text)?.[1] ?? undefined) : undefined; +} + +export function extractDoubanId(text: string | null | undefined): string | undefined { + return text ? (/douban\.com\/subject\/(\d+)/.exec(text)?.[1] ?? undefined) : undefined; +} + +// ---- .torrent 校验与 infohash(最小 bencode 扫描,只为定位 info 字典字节范围) ---- + +function skipValue(buf: Uint8Array, pos: number, depth = 0): number { + if (depth > 64) throw new Error("bencode 嵌套过深"); + const c = buf[pos]; + if (c === undefined) throw new Error("bencode 意外结束"); + if (c === 0x69 /* i */) { + const end = buf.indexOf(0x65, pos); + if (end < 0) throw new Error("bencode 整数未结束"); + return end + 1; + } + if (c === 0x6c /* l */ || c === 0x64 /* d */) { + let p = pos + 1; + while (buf[p] !== 0x65) { + if (p >= buf.length) throw new Error("bencode 容器未结束"); + p = skipValue(buf, p, depth + 1); + } + return p + 1; + } + if (c >= 0x30 && c <= 0x39) { + const colon = buf.indexOf(0x3a, pos); + if (colon < 0) throw new Error("bencode 字符串缺少冒号"); + const len = Number(new TextDecoder().decode(buf.subarray(pos, colon))); + if (!Number.isInteger(len) || len < 0 || colon + 1 + len > buf.length) throw new Error("bencode 字符串越界"); + return colon + 1 + len; + } + throw new Error("bencode 非法前缀"); +} + +function readString(buf: Uint8Array, pos: number): { value: Uint8Array; next: number } { + const colon = buf.indexOf(0x3a, pos); + const len = Number(new TextDecoder().decode(buf.subarray(pos, colon))); + return { value: buf.subarray(colon + 1, colon + 1 + len), next: colon + 1 + len }; +} + +export interface TorrentInspection { + infoHash: string; + name: string | null; +} + +/** 校验字节是 .torrent(顶层字典且含 info),返回 v1 infohash(info 字典原始字节的 sha1)。非 torrent 返回 null。 */ +export function inspectTorrent(bytes: Uint8Array): TorrentInspection | null { + try { + if (bytes[0] !== 0x64) return null; + let p = 1; + while (bytes[p] !== 0x65) { + const key = readString(bytes, p); + const keyText = new TextDecoder().decode(key.value); + const end = skipValue(bytes, key.next); + if (keyText === "info") { + const info = bytes.subarray(key.next, end); + const infoHash = createHash("sha1").update(info).digest("hex"); + let name: string | null = null; + // info 字典内找 name + let q = 1; + while (info[q] !== 0x65 && q < info.length) { + const k = readString(info, q); + const vEnd = skipValue(info, k.next); + if (new TextDecoder().decode(k.value) === "name" && info[k.next]! >= 0x30 && info[k.next]! <= 0x39) { + name = new TextDecoder().decode(readString(info, k.next).value); + } + q = vEnd; + } + return { infoHash, name }; + } + p = end; + if (p >= bytes.length) return null; + } + return null; + } catch { + return null; + } +} diff --git a/packages/testkit/src/index.ts b/packages/testkit/src/index.ts index 8c2eea57..b3a6d674 100644 --- a/packages/testkit/src/index.ts +++ b/packages/testkit/src/index.ts @@ -3,4 +3,6 @@ export const PACKAGE_NAME = "@umt/testkit"; export * from "./emby"; export * from "./fs"; export * from "./qbittorrent"; +export * from "./rss"; +export * from "./sites"; export * from "./tmdb"; diff --git a/packages/testkit/src/rss/README.md b/packages/testkit/src/rss/README.md new file mode 100644 index 00000000..8e1db6d1 --- /dev/null +++ b/packages/testkit/src/rss/README.md @@ -0,0 +1,18 @@ +# testkit/rss + +mock RSS/Atom 服务器(127.0.0.1 随机端口,node:http,无外部依赖)。 + +```ts +const feed = await createMockRssServer({ items, passkey: "pk", format: "rss" | "atom" }); +feed.feedUrl; // 带 ?passkey= 的完整 URL +feed.addItem(item); // 新条目置前 +feed.updateItem(guid, {title}); // 同 GUID 内容变化 +feed.pinItem(guid); feed.shuffle(seed); // 置顶 / 乱序 +feed.failNext(2, { status: 503, retryAfter: "120" }); +feed.setRawBody(maliciousXml); // DTD/XXE/超大响应样本;null 恢复 +feed.setConditional(false); // 关闭 ETag/Last-Modified +feed.requests; feed.notModifiedCount; +await feed.close(); +``` + +ETag 随内容与版本变化;If-None-Match 命中或 If-Modified-Since 不早于最后修改时间返回 304。passkey 错误返回 403。 diff --git a/packages/testkit/src/rss/index.ts b/packages/testkit/src/rss/index.ts new file mode 100644 index 00000000..975cfef8 --- /dev/null +++ b/packages/testkit/src/rss/index.ts @@ -0,0 +1,9 @@ +export { + createMockRssServer, + type MockFailure, + type MockFeedItem, + type MockRssOptions, + type MockRssServer, + renderAtom, + renderRss, +} from "./mock-rss"; diff --git a/packages/testkit/src/rss/mock-rss.ts b/packages/testkit/src/rss/mock-rss.ts new file mode 100644 index 00000000..a427b79b --- /dev/null +++ b/packages/testkit/src/rss/mock-rss.ts @@ -0,0 +1,206 @@ +import { createHash } from "node:crypto"; +import { type StartedServer, startHttpServer } from "./server"; + +export interface MockFeedItem { + guid: string; + title: string; + link?: string; + enclosureUrl?: string; + size?: number; + pubDate?: Date; + categories?: string[]; + description?: string; +} + +export interface MockFailure { + status: number; + retryAfter?: string; + body?: string; +} + +export interface MockRssOptions { + format?: "rss" | "atom"; + items?: MockFeedItem[]; + /** 访问需带的 passkey(query ?passkey=);缺失或错误返回 403 */ + passkey?: string; + /** Feed 路径,默认 /rss.xml */ + path?: string; + title?: string; +} + +export interface MockRssServer extends StartedServer { + /** 带 passkey(若配置)的完整 Feed URL */ + feedUrl: string; + items: MockFeedItem[]; + setItems(items: MockFeedItem[]): void; + /** 新条目放到最前(RSS 惯例新→旧) */ + addItem(item: MockFeedItem): void; + /** 同 GUID 修改内容(标题修正/促销等) */ + updateItem(guid: string, patch: Partial): void; + /** 置顶:把条目挪到最前 */ + pinItem(guid: string): void; + /** 打乱顺序(确定性:按 seed 洗牌) */ + shuffle(seed?: number): void; + /** 接下来 n 次请求返回失败(如 503 + Retry-After) */ + failNext(n: number, failure: MockFailure): void; + /** 直接返回原始 XML(用于 DTD/XXE/超大响应等恶意样本);传 null 恢复正常 */ + setRawBody(xml: string | null): void; + /** 是否支持 ETag/Last-Modified 条件请求(默认 true) */ + setConditional(enabled: boolean): void; + /** 统计:返回 304 的次数 */ + readonly notModifiedCount: number; +} + +function esc(s: string): string { + return s.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); +} + +export function renderRss(title: string, items: MockFeedItem[]): string { + const body = items + .map((it) => { + const parts = [ + `${esc(it.title)}`, + `${esc(it.guid)}`, + it.link ? `${esc(it.link)}` : "", + it.enclosureUrl + ? `` + : "", + it.pubDate ? `${it.pubDate.toUTCString()}` : "", + ...(it.categories ?? []).map((c) => `${esc(c)}`), + it.description ? `` : "", + ]; + return `${parts.join("")}`; + }) + .join("\n"); + return `\n${esc(title)}\n${body}\n`; +} + +export function renderAtom(title: string, items: MockFeedItem[]): string { + const body = items + .map((it) => { + const parts = [ + `${esc(it.guid)}`, + `${esc(it.title)}`, + it.link ? `` : "", + it.enclosureUrl ? `` : "", + it.pubDate ? `${it.pubDate.toISOString()}` : "", + ...(it.categories ?? []).map((c) => ``), + it.description ? `${esc(it.description)}` : "", + ]; + return `${parts.join("")}`; + }) + .join("\n"); + return `\n${esc(title)}\n${body}\n`; +} + +/** 可编程 mock RSS/Atom 服务器:ETag/Last-Modified/304、乱序/置顶/更新、5xx + Retry-After。 */ +export async function createMockRssServer(opts: MockRssOptions = {}): Promise { + const path = opts.path ?? "/rss.xml"; + const title = opts.title ?? "Mock Feed"; + let items = [...(opts.items ?? [])]; + let version = 1; + let lastModified = new Date(Date.UTC(2026, 8, 1)); + const failures: MockFailure[] = []; + let raw: string | null = null; + let conditional = true; + let notModified = 0; + + const bump = () => { + version++; + lastModified = new Date(lastModified.getTime() + 60_000); + }; + + const server = await startHttpServer((req, res) => { + const url = new URL(req.url ?? "/", "http://mock"); + if (url.pathname !== path) { + res.writeHead(404).end("not found"); + return; + } + if (opts.passkey && url.searchParams.get("passkey") !== opts.passkey) { + res.writeHead(403, { "content-type": "text/plain" }).end("invalid passkey"); + return; + } + const failure = failures.shift(); + if (failure) { + const headers: Record = { "content-type": "text/plain" }; + if (failure.retryAfter !== undefined) headers["retry-after"] = failure.retryAfter; + res.writeHead(failure.status, headers).end(failure.body ?? `mock failure ${failure.status}`); + return; + } + const body = raw ?? (opts.format === "atom" ? renderAtom(title, items) : renderRss(title, items)); + const etag = `"${createHash("sha1").update(body).digest("hex").slice(0, 16)}-${version}"`; + const lm = lastModified.toUTCString(); + if (conditional) { + const inm = req.headers["if-none-match"]; + const ims = req.headers["if-modified-since"]; + if ((inm && inm === etag) || (!inm && ims && Date.parse(ims) >= lastModified.getTime())) { + notModified++; + res.writeHead(304, { etag, "last-modified": lm }).end(); + return; + } + } + const headers: Record = { + "content-type": + opts.format === "atom" ? "application/atom+xml; charset=utf-8" : "application/rss+xml; charset=utf-8", + }; + if (conditional) { + headers.etag = etag; + headers["last-modified"] = lm; + } + res.writeHead(200, headers).end(body); + }); + + const feedUrl = `${server.baseUrl}${path}${opts.passkey ? `?passkey=${encodeURIComponent(opts.passkey)}` : ""}`; + return { + ...server, + feedUrl, + get items() { + return items; + }, + setItems(next) { + items = [...next]; + bump(); + }, + addItem(item) { + items = [item, ...items]; + bump(); + }, + updateItem(guid, patch) { + items = items.map((it) => (it.guid === guid ? { ...it, ...patch, guid } : it)); + bump(); + }, + pinItem(guid) { + const it = items.find((i) => i.guid === guid); + if (!it) return; + items = [it, ...items.filter((i) => i.guid !== guid)]; + bump(); + }, + shuffle(seed = 42) { + let s = seed; + const rnd = () => { + s = (s * 1103515245 + 12345) % 2147483648; + return s / 2147483648; + }; + const arr = [...items]; + for (let i = arr.length - 1; i > 0; i--) { + const j = Math.floor(rnd() * (i + 1)); + [arr[i], arr[j]] = [arr[j]!, arr[i]!]; + } + items = arr; + bump(); + }, + failNext(n, failure) { + for (let i = 0; i < n; i++) failures.push(failure); + }, + setRawBody(xml) { + raw = xml; + bump(); + }, + setConditional(enabled) { + conditional = enabled; + }, + get notModifiedCount() { + return notModified; + }, + }; +} diff --git a/packages/testkit/src/rss/server.ts b/packages/testkit/src/rss/server.ts new file mode 100644 index 00000000..b76cf14e --- /dev/null +++ b/packages/testkit/src/rss/server.ts @@ -0,0 +1,46 @@ +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; + +export interface RecordedRequest { + method: string; + path: string; + headers: Record; + body: string; +} + +export interface StartedServer { + /** 形如 http://127.0.0.1:PORT(不带尾斜杠) */ + baseUrl: string; + requests: RecordedRequest[]; + close(): Promise; +} + +export type Handler = (req: IncomingMessage, res: ServerResponse, body: string) => void | Promise; + +/** 在 127.0.0.1 随机端口起 HTTP 服务;记录所有请求供断言。 */ +export async function startHttpServer(handler: Handler): Promise { + const requests: RecordedRequest[] = []; + const server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const body = Buffer.concat(chunks).toString("utf8"); + requests.push({ method: req.method ?? "GET", path: req.url ?? "/", headers: { ...req.headers }, body }); + Promise.resolve(handler(req, res, body)).catch((e: unknown) => { + if (!res.headersSent) res.writeHead(500, { "content-type": "text/plain" }); + res.end(`mock handler error: ${(e as Error).message}`); + }); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + return { + baseUrl: `http://127.0.0.1:${port}`, + requests, + close: () => + new Promise((resolve, reject) => { + server.closeAllConnections(); + server.close((err) => (err ? reject(err) : resolve())); + }), + }; +} diff --git a/packages/testkit/src/sites/README.md b/packages/testkit/src/sites/README.md new file mode 100644 index 00000000..ab186f98 --- /dev/null +++ b/packages/testkit/src/sites/README.md @@ -0,0 +1,30 @@ +# testkit/sites + +mock PT 站点(配合 adapters/sites 的 MoviePilot 迁移实现)。HTML/JSON fixture 为自行编写,**不代表已核对真实站点**,只用于验证适配器的选择器配置与错误分类。 + +## mock NexusPHP(观众风格) + +```ts +const site = await createMockNexusPhpSite({ pageSize: 50, pageBase: 0, loginMode: "page" | "redirect" }); +site.baseUrl; site.cookie; // 请求 Cookie 需包含 site.cookie +site.setLoggedIn(false); // 全站返回登录页(或 302 login.php) +site.failNext(1, 429, "7"); // 限流 / 5xx +site.setBrokenLayout(true); // 改版模拟 +site.infoHashOf("812345"); // 下载 .torrent 的 infohash +``` + +路由:`index.php`、`torrents.php`(search / catN / spstate=2 / page)、`details.php?id=`(不存在返回"没有该ID的种子")、`download.php?id=`(合法 .torrent,或 `magnet` 字段时 302 磁力)、`login.php`。 +默认 fixture `AUDIENCES_FIXTURE_RELEASES`:免费+截止(812345)、综艺 403 + H&R(812346)、50%+2x(812347)、截断标题 + TB(812348)、纪录片 2x + KB(812349)。 + +## mock M-Team API + +```ts +const api = await createMockMteamApi({ apiKey, torrents, failMode }); +api.setFailMode("banned_key" | "rate_limit" | "http_429" | "non_json" | "no_dl_token" | null); +api.lastSearchBody; api.issuedTokens; api.infoHashOf("912233"); +``` + +路由:`POST /api/torrent/search`、`/api/torrent/detail`、`/api/torrent/genDlToken`(id 可走 query 或表单,签发一次性 `/api/rss/dlv2?sign=&t=&tid=`)、`/api/member/profile`;`x-api-key` 错误返回 `{code:"1", message:"key無效"}`。 +默认 fixture `MTEAM_FIXTURE_TORRENTS`:50% 限时(912233)、商城单种免费(912234)、全站促销(912235,动画 405)、2x 免费(912236,纪录 404)、体育 407(912237)。 + +`buildFakeTorrent({name, length})` 生成结构合法的 .torrent 与其 infohash。 diff --git a/packages/testkit/src/sites/index.ts b/packages/testkit/src/sites/index.ts new file mode 100644 index 00000000..45368b34 --- /dev/null +++ b/packages/testkit/src/sites/index.ts @@ -0,0 +1,10 @@ +export { + createMockMteamApi, + type MockMteamApi, + type MockMteamFailMode, + type MockMteamOptions, +} from "./mock-mteam"; +export { createMockNexusPhpSite, type MockNexusPhpOptions, type MockNexusPhpSite } from "./mock-nexusphp"; +export { type MockMteamTorrent, MTEAM_FIXTURE_TORRENTS } from "./mteam-fixtures"; +export { AUDIENCES_FIXTURE_RELEASES, type MockNexusPromo, type MockNexusRelease } from "./nexusphp-fixtures"; +export { buildFakeTorrent, type FakeTorrent } from "./torrent"; diff --git a/packages/testkit/src/sites/mock-mteam.ts b/packages/testkit/src/sites/mock-mteam.ts new file mode 100644 index 00000000..1edd90af --- /dev/null +++ b/packages/testkit/src/sites/mock-mteam.ts @@ -0,0 +1,152 @@ +import { randomBytes } from "node:crypto"; +import type { ServerResponse } from "node:http"; +import { type StartedServer, startHttpServer } from "../rss/server"; +import { type MockMteamTorrent, MTEAM_FIXTURE_TORRENTS } from "./mteam-fixtures"; +import { buildFakeTorrent } from "./torrent"; + +export type MockMteamFailMode = "banned_key" | "rate_limit" | "http_429" | "non_json" | "no_dl_token"; + +export interface MockMteamOptions { + apiKey?: string; + torrents?: MockMteamTorrent[]; + failMode?: MockMteamFailMode | null; +} + +export interface MockMteamApi extends StartedServer { + apiKey: string; + torrents: MockMteamTorrent[]; + setFailMode(mode: MockMteamFailMode | null): void; + /** 已签发的下载令牌(sign)数 */ + readonly issuedTokens: number; + infoHashOf(id: string): string; + /** 最近一次 search 请求体 */ + readonly lastSearchBody: Record | null; +} + +function json(res: ServerResponse, body: unknown, status = 200) { + res.writeHead(status, { "content-type": "application/json; charset=utf-8" }).end(JSON.stringify(body)); +} + +function formOrJson(body: string, contentType: string | undefined): Record { + if (!body) return {}; + if (contentType?.includes("application/json")) return JSON.parse(body) as Record; + return Object.fromEntries(new URLSearchParams(body)); +} + +/** mock M-Team API:search/detail/genDlToken/dlv2/member.profile;x-api-key 校验与业务错误模拟。 */ +export async function createMockMteamApi(opts: MockMteamOptions = {}): Promise { + const apiKey = opts.apiKey ?? "mock-mteam-api-key-0123456789"; + const torrents = [...(opts.torrents ?? MTEAM_FIXTURE_TORRENTS)]; + let failMode: MockMteamFailMode | null = opts.failMode ?? null; + const tokens = new Map(); // sign → torrent id(一次性) + let issued = 0; + let lastSearchBody: Record | null = null; + const files = new Map(torrents.map((t) => [t.id, buildFakeTorrent({ name: t.name, length: 8 * 1024 * 1024 })])); + + const server = await startHttpServer((req, res, raw) => { + const url = new URL(req.url ?? "/", "http://mock"); + if (url.pathname === "/api/rss/dlv2" && req.method === "GET") { + const sign = url.searchParams.get("sign") ?? ""; + const id = tokens.get(sign); + if (!id || id !== url.searchParams.get("tid")) return json(res, { code: "1", message: "下載鏈接無效或已過期" }); + tokens.delete(sign); + res.writeHead(200, { "content-type": "application/x-bittorrent" }).end(Buffer.from(files.get(id)!.bytes)); + return; + } + if (req.method !== "POST") return json(res, { code: "1", message: "method not allowed" }, 405); + if (failMode === "non_json") { + res + .writeHead(200, { "content-type": "text/html" }) + .end("系統維護中 / Under maintenance"); + return; + } + if (failMode === "http_429") { + res.writeHead(429, { "retry-after": "5" }).end("too many"); + return; + } + if (req.headers["x-api-key"] !== apiKey) return json(res, { code: "1", message: "key無效" }); + if (failMode === "banned_key") return json(res, { code: "1", message: "帳號已停用" }); + if (failMode === "rate_limit") return json(res, { code: "1", message: "請求過於頻繁" }); + const body = formOrJson(raw, req.headers["content-type"]); + switch (url.pathname) { + case "/api/torrent/search": { + lastSearchBody = body; + const kw = String(body.keyword ?? "").toLowerCase(); + const cats = (body.categories as string[] | undefined) ?? []; + const pageNumber = Number(body.pageNumber ?? 1); + const pageSize = Number(body.pageSize ?? 100); + let list = torrents.filter((t) => { + if (cats.length && !cats.map(String).includes(t.category)) return false; + if (body.discount === "FREE" && !["FREE", "_2X_FREE"].includes(t.status.discount)) return false; + if (!kw) return true; + if (kw.startsWith("https://www.imdb.com/title/")) return (t.imdb ?? "").toLowerCase().startsWith(kw); + return t.name.toLowerCase().includes(kw) || (t.smallDescr ?? "").toLowerCase().includes(kw); + }); + const total = list.length; + list = list.slice((pageNumber - 1) * pageSize, pageNumber * pageSize); + return json(res, { + code: "0", + message: "SUCCESS", + data: { + pageNumber: String(pageNumber), + pageSize: String(pageSize), + total: String(total), + totalPages: String(Math.max(1, Math.ceil(total / pageSize))), + data: list, + }, + }); + } + case "/api/torrent/detail": { + const t = torrents.find((x) => x.id === String(body.id ?? url.searchParams.get("id"))); + if (!t) return json(res, { code: "1", message: "種子不存在" }); + return json(res, { code: "0", message: "SUCCESS", data: t }); + } + case "/api/torrent/genDlToken": { + const id = String(body.id ?? url.searchParams.get("id")); + if (!torrents.some((x) => x.id === id)) return json(res, { code: "1", message: "種子不存在" }); + if (failMode === "no_dl_token") return json(res, { code: "0", message: "SUCCESS", data: null }); + const sign = randomBytes(12).toString("hex"); + tokens.set(sign, id); + issued++; + const host = req.headers.host ?? "127.0.0.1"; + return json(res, { + code: "0", + message: "SUCCESS", + data: `http://${host}/api/rss/dlv2?sign=${sign}&t=${Date.now()}&tid=${id}`, + }); + } + case "/api/member/profile": + return json(res, { + code: "0", + message: "SUCCESS", + data: { + id: "10001", + username: "mockuser", + memberCount: { uploaded: "1000", downloaded: "10", shareRate: "100" }, + }, + }); + default: + return json(res, { code: "1", message: "not found" }, 404); + } + }); + + return { + ...server, + apiKey, + torrents, + setFailMode(mode) { + failMode = mode; + }, + get issuedTokens() { + return issued; + }, + get lastSearchBody() { + return lastSearchBody; + }, + infoHashOf(id) { + const f = files.get(id); + if (!f) throw new Error(`no torrent ${id}`); + return f.infoHash; + }, + }; +} diff --git a/packages/testkit/src/sites/mock-nexusphp.ts b/packages/testkit/src/sites/mock-nexusphp.ts new file mode 100644 index 00000000..68864d17 --- /dev/null +++ b/packages/testkit/src/sites/mock-nexusphp.ts @@ -0,0 +1,220 @@ +import type { IncomingMessage, ServerResponse } from "node:http"; +import { type StartedServer, startHttpServer } from "../rss/server"; +import { AUDIENCES_FIXTURE_RELEASES, type MockNexusRelease } from "./nexusphp-fixtures"; +import { buildFakeTorrent } from "./torrent"; + +export interface MockNexusPhpOptions { + /** 登录态 Cookie;请求 Cookie 头需包含它 */ + cookie?: string; + releases?: MockNexusRelease[]; + /** 每页条数,默认 50 */ + pageSize?: number; + /** 站点页码起点(观众 0 起),默认 0 */ + pageBase?: 0 | 1; + /** 登录失效时的表现:200 登录页(默认)或 302 跳转 login.php */ + loginMode?: "page" | "redirect"; + siteName?: string; +} + +export interface MockNexusPhpSite extends StartedServer { + cookie: string; + releases: MockNexusRelease[]; + /** 让站点认为会话失效(所有页面返回登录页) */ + setLoggedIn(v: boolean): void; + /** 接下来 n 次请求返回指定状态(如 429 + Retry-After / 502) */ + failNext(n: number, status: number, retryAfter?: string): void; + /** 改版模拟:列表页返回无法识别的结构 */ + setBrokenLayout(v: boolean): void; + /** 各 .torrent 的 infohash,供断言 */ + infoHashOf(id: string): string; +} + +const PROMO_ALT: Record = { + free: "Free", + free2up: "2X Free", + "50pctdown": "50%", + "50pctdown2up": "2X 50%", + "30pctdown": "30%", + "2up": "2X", +}; + +function esc(s: string): string { + return s.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); +} + +function shell(siteName: string, title: string, body: string, loggedIn: boolean): string { + const userbar = loggedIn + ? `
mockuser + 12.5 TB + 3.2 TB + 退出
` + : ""; + return `${esc(siteName)} :: ${esc(title)} +${userbar}
${body}
`; +} + +function loginPage(siteName: string): string { + return `${esc(siteName)} :: 登录 +

你需要启用cookies才能登录

+
`; +} + +function promoImg(r: MockNexusRelease): string { + if (!r.promo) return ""; + const title = r.promoUntil ?? ""; + return ` ${PROMO_ALT[r.promo]}`; +} + +function row(r: MockNexusRelease): string { + const titleLink = r.truncatedText + ? `${esc(r.truncatedText)}` + : `${esc(r.title)}`; + const hr = r.hr ? ` H&R` : ""; + const sub = r.subtitle ? `
${esc(r.subtitle)}` : ""; + return ` +${esc(r.catName)} + + + +
${titleLink}${promoImg(r)}${hr}${sub}download
+0 +${esc(r.relativeTime ?? r.uploadedAt)} +${esc(r.sizeText)} +${r.seeders} +${r.leechers} +${r.snatched} +${esc(r.uploader ?? "anonymous")} +`; +} + +function listBody(rows: MockNexusRelease[], pageCount: number, pageBase: number, query: URLSearchParams): string { + const header = `类型标题评论存活时间大小种子下载完成发布者`; + const links: string[] = []; + for (let i = 0; i < pageCount; i++) { + const q = new URLSearchParams(query); + q.set("page", String(i + pageBase)); + links.push(`${i + 1}`); + } + if (rows.length === 0) return `

没有种子。请用准确的关键字重试。

`; + return `

${links.join(" | ")}

+${header}${rows.map(row).join("\n")}
`; +} + +function detailBody(r: MockNexusRelease): string { + const hr = r.hr ? ` H&R` : ""; + return `

${esc(r.title)}${promoImg(r)}${hr}

+ + + + + + +
下载[Mock].${esc(r.title)}.torrent
副标题${esc(r.subtitle ?? "")}
基本信息大小：${esc(r.sizeText)}   类型: ${esc(r.catName)}  发布于 ${esc(r.relativeTime ?? r.uploadedAt)}
同伴${r.seeders.replace(/,/g, "")}个做种者 | ${r.leechers.replace(/,/g, "")}个下载者
简介
${esc(r.description ?? "暂无简介")}
`; +} + +function hasCookie(req: IncomingMessage, cookie: string): boolean { + const header = req.headers.cookie ?? ""; + return cookie.split(";").every((part) => header.includes(part.trim())); +} + +/** mock NexusPHP 站点:列表/搜索/详情/下载/首页,支持登录失效、限流与改版模拟。 */ +export async function createMockNexusPhpSite(opts: MockNexusPhpOptions = {}): Promise { + const cookie = opts.cookie ?? "c_secure_uid=10001; c_secure_pass=mock-session-secret"; + const releases = [...(opts.releases ?? AUDIENCES_FIXTURE_RELEASES)]; + const pageSize = opts.pageSize ?? 50; + const pageBase = opts.pageBase ?? 0; + const siteName = opts.siteName ?? "观众"; + let loggedIn = true; + let broken = false; + const failures: Array<{ status: number; retryAfter?: string }> = []; + const torrents = new Map(releases.map((r) => [r.id, buildFakeTorrent({ name: r.title, length: 4 * 1024 * 1024 })])); + + const send = (res: ServerResponse, status: number, html: string) => { + res.writeHead(status, { "content-type": "text/html; charset=utf-8" }).end(html); + }; + + const server = await startHttpServer((req, res) => { + const url = new URL(req.url ?? "/", "http://mock"); + const failure = failures.shift(); + if (failure) { + res.writeHead(failure.status, failure.retryAfter ? { "retry-after": failure.retryAfter } : {}).end("busy"); + return; + } + if (url.pathname === "/login.php") return send(res, 200, loginPage(siteName)); + if (!loggedIn || !hasCookie(req, cookie)) { + if (opts.loginMode === "redirect") { + res.writeHead(302, { location: `login.php?returnto=${encodeURIComponent(url.pathname.slice(1))}` }).end(); + return; + } + return send(res, 200, loginPage(siteName)); + } + const q = url.searchParams; + switch (url.pathname) { + case "/index.php": + case "/": + return send(res, 200, shell(siteName, "首页", "

最新公告

", true)); + case "/torrents.php": { + if (broken) return send(res, 200, shell(siteName, "种子", `
改版了
`, true)); + const kw = (q.get("search") ?? "").toLowerCase().trim(); + const cats = [...q.keys()].filter((k) => /^cat\d+$/.test(k)).map((k) => k.slice(3)); + let rows = releases.filter((r) => { + if (cats.length && !cats.includes(r.catId)) return false; + if (q.get("spstate") === "2" && r.promo !== "free" && r.promo !== "free2up") return false; + if (!kw) return true; + return kw + .split(/\s+/) + .every((w) => r.title.toLowerCase().includes(w) || (r.subtitle ?? "").toLowerCase().includes(w)); + }); + const pageCount = Math.max(1, Math.ceil(rows.length / pageSize)); + const idx = Number(q.get("page") ?? pageBase) - pageBase; + rows = rows.slice(idx * pageSize, idx * pageSize + pageSize); + const base = new URLSearchParams(q); + base.delete("page"); + return send(res, 200, shell(siteName, "种子", listBody(rows, pageCount, pageBase, base), true)); + } + case "/details.php": { + const r = releases.find((x) => x.id === q.get("id")); + if (!r) return send(res, 200, shell(siteName, "错误", "

错误!

没有该ID的种子

", true)); + return send(res, 200, shell(siteName, `种子详情 "${r.title}"`, detailBody(r), true)); + } + case "/download.php": { + const r = releases.find((x) => x.id === q.get("id")); + if (!r) return send(res, 200, shell(siteName, "错误", "

没有该ID的种子

", true)); + if (r.magnet) { + res.writeHead(302, { location: r.magnet }).end(); + return; + } + const t = torrents.get(r.id)!; + res + .writeHead(200, { + "content-type": "application/x-bittorrent", + "content-disposition": `attachment; filename*=UTF-8''${encodeURIComponent(`[Mock].${r.title}.torrent`)}`, + }) + .end(Buffer.from(t.bytes)); + return; + } + default: + res.writeHead(404).end("not found"); + } + }); + + return { + ...server, + cookie, + releases, + setLoggedIn(v) { + loggedIn = v; + }, + failNext(n, status, retryAfter) { + for (let i = 0; i < n; i++) failures.push(retryAfter === undefined ? { status } : { status, retryAfter }); + }, + setBrokenLayout(v) { + broken = v; + }, + infoHashOf(id) { + const t = torrents.get(id); + if (!t) throw new Error(`no torrent ${id}`); + return t.infoHash; + }, + }; +} diff --git a/packages/testkit/src/sites/mteam-fixtures.ts b/packages/testkit/src/sites/mteam-fixtures.ts new file mode 100644 index 00000000..b8b3f653 --- /dev/null +++ b/packages/testkit/src/sites/mteam-fixtures.ts @@ -0,0 +1,102 @@ +/** + * mock M-Team API fixture。字段名按公开 API 与迁移账本 §7.2 整理,待真实账号核验。 + * 数值字段在真实 API 中多为字符串,这里保持同样形态。 + */ + +export interface MockMteamTorrent { + id: string; + name: string; + smallDescr?: string; + category: string; + size: string; + createdDate: string; + imdb?: string; + douban?: string; + labels?: string; + labelsNew?: string[]; + descr?: string; + status: { + seeders: string; + leechers: string; + timesCompleted: string; + discount: string; + discountEndTime?: string | null; + promotionRule?: { discount: string; endTime: string } | null; + mallSingleFree?: { status: string; endDate: string } | null; + }; +} + +export const MTEAM_FIXTURE_TORRENTS: MockMteamTorrent[] = [ + { + id: "912233", + name: "Shogun.2024.S01.2160p.DSNP.WEB-DL.DDP5.1.Atmos.DV.HDR.H.265-FLUX", + smallDescr: "幕府将军 第一季 全10集 杜比视界", + category: "402", + size: "61234567890", + createdDate: "2026-09-20 08:15:00", + imdb: "https://www.imdb.com/title/tt2788316/", + labelsNew: ["中字", "DV"], + descr: "[img]poster[/img] 幕府将军", + status: { + seeders: "532", + leechers: "4", + timesCompleted: "2011", + discount: "PERCENT_50", + discountEndTime: "2026-09-28 08:15:00", + }, + }, + { + id: "912234", + name: "Oppenheimer.2023.2160p.UHD.BluRay.REMUX.DV.HDR.HEVC.DTS-HD.MA.5.1-FGT", + smallDescr: "奥本海默", + category: "439", + size: "91234567890", + createdDate: "2026-09-21 10:00:00", + imdb: "https://www.imdb.com/title/tt15398776/", + labels: "6", + status: { + seeders: "120", + leechers: "3", + timesCompleted: "800", + discount: "PERCENT_50", + mallSingleFree: { status: "ONGOING", endDate: "2026-09-27 20:00:00" }, + }, + }, + { + id: "912235", + name: "Frieren.S01.2023.1080p.BluRay.x265.10bit.FLAC-VCB", + smallDescr: "葬送的芙莉莲 第一季", + category: "405", + size: "45000000000", + createdDate: "2026-09-22 12:00:00", + status: { + seeders: "300", + leechers: "10", + timesCompleted: "1500", + discount: "NORMAL", + promotionRule: { discount: "FREE", endTime: "2026-10-07 23:59:59" }, + }, + }, + { + id: "912236", + name: "Planet.Earth.III.2023.2160p.iP.WEB-DL.DDP5.1.HDR.H.265-GRP", + category: "404", + size: "30000000000", + createdDate: "2026-09-23 09:00:00", + status: { + seeders: "50", + leechers: "1", + timesCompleted: "90", + discount: "_2X_FREE", + discountEndTime: "2026-09-30 00:00:00", + }, + }, + { + id: "912237", + name: "Paris.2024.Olympics.Opening.Ceremony.2160p.WEB-DL.H265-GRP", + category: "407", + size: "12000000000", + createdDate: "2026-09-24 09:00:00", + status: { seeders: "10", leechers: "0", timesCompleted: "22", discount: "NORMAL" }, + }, +]; diff --git a/packages/testkit/src/sites/nexusphp-fixtures.ts b/packages/testkit/src/sites/nexusphp-fixtures.ts new file mode 100644 index 00000000..908928ee --- /dev/null +++ b/packages/testkit/src/sites/nexusphp-fixtures.ts @@ -0,0 +1,102 @@ +/** + * mock NexusPHP(观众风格)资源 fixture。HTML 结构按 NexusPHP 公开页面惯例自行编写, + * 类名(torrent-subtitle-line / pro_* / hitandrun)用于验证选择器配置,不代表已核对真实站点。 + */ + +export type MockNexusPromo = "free" | "free2up" | "50pctdown" | "50pctdown2up" | "30pctdown" | "2up"; + +export interface MockNexusRelease { + id: string; + title: string; + /** 不给 title 属性,只显示截断文字 */ + truncatedText?: string; + subtitle?: string; + catId: string; + catName: string; + sizeText: string; + seeders: string; + leechers: string; + snatched: string; + /** 站点本地时间 YYYY-MM-DD HH:mm:ss */ + uploadedAt: string; + relativeTime?: string; + promo?: MockNexusPromo; + promoUntil?: string; + hr?: boolean; + uploader?: string; + description?: string; + /** 下载返回磁力而不是 .torrent */ + magnet?: string; +} + +export const AUDIENCES_FIXTURE_RELEASES: MockNexusRelease[] = [ + { + id: "812345", + title: "Dune.Part.Two.2024.2160p.UHD.BluRay.REMUX.DV.HDR.HEVC.TrueHD.Atmos.7.1-FraMeSToR", + subtitle: "沙丘2 / 沙丘：第二部 | 杜比视界 国英双语 中字", + catId: "401", + catName: "电影", + sizeText: "78.42 GB", + seeders: "1,204", + leechers: "17", + snatched: "3,388", + uploadedAt: "2026-09-26 21:14:03", + relativeTime: "3小时前", + promo: "free", + promoUntil: "2026-10-01 12:00:00", + uploader: "uploader1", + description: "https://www.imdb.com/title/tt15239678/ https://movie.douban.com/subject/35575567/", + }, + { + id: "812346", + title: "Keep.Running.S12E01.20240426.2160p.WEB-DL.H265.AAC-LelveTV", + subtitle: "奔跑吧 第十二季 第1期", + catId: "403", + catName: "综艺", + sizeText: "5.12 GB", + seeders: "88", + leechers: "2", + snatched: "410", + uploadedAt: "2026-09-26 20:00:00", + hr: true, + }, + { + id: "812347", + title: "The.Bear.S03.2160p.DSNP.WEB-DL.DDP5.1.Atmos.DV.H.265-FLUX", + subtitle: "熊家餐馆 第三季", + catId: "402", + catName: "剧集", + sizeText: "1,234.56 MiB", + seeders: "35", + leechers: "0", + snatched: "120", + uploadedAt: "2026-09-25 08:30:00", + promo: "50pctdown2up", + promoUntil: "2026-09-29 08:30:00", + }, + { + id: "812348", + title: "The.Last.of.Us.S02.2160p.MAX.WEB-DL.DDP5.1.Atmos.DV.HDR.H.265-FLUX", + truncatedText: "The.Last.of.Us.S02.2160p.MAX.WEB-DL.DDP5.1.Atmos.D…", + catId: "402", + catName: "剧集", + sizeText: "2.1 TB", + seeders: "12", + leechers: "1", + snatched: "44", + uploadedAt: "2026-09-24 10:00:00", + }, + { + id: "812349", + title: "Planet.Earth.III.2023.2160p.BluRay.REMUX.HEVC.DTS-HD.MA.5.1-GRP", + subtitle: "地球脉动 第三季", + catId: "406", + catName: "纪录片", + sizeText: "850 KB", + seeders: "5", + leechers: "0", + snatched: "9", + uploadedAt: "2026-09-23 10:00:00", + promo: "2up", + }, +]; diff --git a/packages/testkit/src/sites/torrent.ts b/packages/testkit/src/sites/torrent.ts new file mode 100644 index 00000000..ee5b1e58 --- /dev/null +++ b/packages/testkit/src/sites/torrent.ts @@ -0,0 +1,47 @@ +import { createHash } from "node:crypto"; + +type BValue = number | string | Uint8Array | BValue[] | { [k: string]: BValue }; + +function concat(parts: Uint8Array[]): Uint8Array { + const total = parts.reduce((n, p) => n + p.byteLength, 0); + const out = new Uint8Array(total); + let off = 0; + for (const p of parts) { + out.set(p, off); + off += p.byteLength; + } + return out; +} + +const enc = new TextEncoder(); + +export function bencode(v: BValue): Uint8Array { + if (typeof v === "number") return enc.encode(`i${Math.trunc(v)}e`); + if (typeof v === "string") { + const b = enc.encode(v); + return concat([enc.encode(`${b.byteLength}:`), b]); + } + if (v instanceof Uint8Array) return concat([enc.encode(`${v.byteLength}:`), v]); + if (Array.isArray(v)) return concat([enc.encode("l"), ...v.map(bencode), enc.encode("e")]); + const keys = Object.keys(v).sort(); + return concat([enc.encode("d"), ...keys.flatMap((k) => [bencode(k), bencode(v[k]!)]), enc.encode("e")]); +} + +export interface FakeTorrent { + bytes: Uint8Array; + /** v1 infohash(小写 hex) */ + infoHash: string; +} + +/** 生成结构合法的单文件 .torrent(pieces 为伪数据,只用于协议/解析测试)。 */ +export function buildFakeTorrent(opts: { name: string; length?: number; announce?: string }): FakeTorrent { + const length = opts.length ?? 1024 * 1024; + const pieceLength = 256 * 1024; + const pieceCount = Math.max(1, Math.ceil(length / pieceLength)); + const pieces = new Uint8Array(20 * pieceCount); + for (let i = 0; i < pieceCount; i++) pieces.set(createHash("sha1").update(`${opts.name}:${i}`).digest(), i * 20); + const info = { length, name: opts.name, "piece length": pieceLength, pieces, private: 1 }; + const infoBytes = bencode(info); + const bytes = bencode({ announce: opts.announce ?? "https://tracker.invalid/announce", info }); + return { bytes, infoHash: createHash("sha1").update(infoBytes).digest("hex") }; +}