From c38e7131f10939a8eb68c6519e8fceffc27b51d8 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 6 Nov 2026 00:05:12 +0800 Subject: [PATCH] feat(sites): sync MoviePilot v3.2.4 spiders and M-Team subtitle APIs --- packages/adapters/src/sites/mteam/adapter.ts | 19 +- packages/adapters/src/sites/mteam/client.ts | 10 +- .../adapters/src/sites/mteam/definition.ts | 18 +- packages/adapters/src/sites/mteam/spider.ts | 13 +- .../src/sites/mteam/subtitles.test.ts | 178 ++++++++++++++++++ .../adapters/src/sites/mteam/subtitles.ts | 154 +++++++++++++++ packages/adapters/src/sites/spider/adapter.ts | 20 +- .../src/sites/spider/pagination-rss.test.ts | 90 +++++++++ packages/adapters/src/sites/spider/spider.ts | 34 +++- .../adapters/src/sites/spiders/adapter.ts | 14 +- packages/adapters/src/sites/spiders/haidan.ts | 2 +- .../src/sites/spiders/hddolby.test.ts | 12 +- .../adapters/src/sites/spiders/hddolby.ts | 43 ++++- packages/adapters/src/sites/spiders/index.ts | 5 +- packages/adapters/src/sites/spiders/map.ts | 2 +- .../adapters/src/sites/spiders/milkie.test.ts | 67 +++++++ packages/adapters/src/sites/spiders/milkie.ts | 80 ++++++++ .../adapters/src/sites/spiders/mtorrent.ts | 13 +- packages/adapters/src/sites/spiders/rousi.ts | 2 +- .../adapters/src/sites/spiders/sunnypt.ts | 2 +- .../adapters/src/sites/spiders/tnode.test.ts | 23 +++ packages/adapters/src/sites/spiders/tnode.ts | 8 +- .../src/sites/spiders/torrentleech.ts | 2 +- packages/adapters/src/sites/spiders/types.ts | 8 +- .../adapters/src/sites/spiders/yema.test.ts | 5 + packages/adapters/src/sites/spiders/yema.ts | 7 +- 26 files changed, 775 insertions(+), 56 deletions(-) create mode 100644 packages/adapters/src/sites/mteam/subtitles.test.ts create mode 100644 packages/adapters/src/sites/mteam/subtitles.ts create mode 100644 packages/adapters/src/sites/spider/pagination-rss.test.ts create mode 100644 packages/adapters/src/sites/spiders/milkie.test.ts create mode 100644 packages/adapters/src/sites/spiders/milkie.ts diff --git a/packages/adapters/src/sites/mteam/adapter.ts b/packages/adapters/src/sites/mteam/adapter.ts index c608216e..80161b13 100644 --- a/packages/adapters/src/sites/mteam/adapter.ts +++ b/packages/adapters/src/sites/mteam/adapter.ts @@ -1,7 +1,8 @@ /** + * @mp-source app/modules/indexer/spider/mtorrent.py @ 3b156a5 * 馒头(M-Team)API Key 适配器。 * - * 对应 MoviePilot e6820a8: + * 对应 MoviePilot v3.2.4: * - `MTorrentSpider.search`(POST /api/torrent/search,x-api-key + UA + JSON)→ `list` / `search` * - `MTorrentSpider.__process_response` → `mteamPost`(另加业务 code/message 检查,迁移账本 MT2) * - `MTorrentSpider.__get_download_url` + `submission.py _resolve_indirect_download_url` + `download_torrent` @@ -32,6 +33,7 @@ import { extractDoubanId, parseSiteTime } from "../util.ts"; import { mteamPost } from "./client.ts"; import { type MteamDefinition, mteamApiBase } from "./definition.ts"; import { type MteamTorrentJson, mteamSearchParams, parseMteamResult } from "./spider.ts"; +import { createMteamSubtitleClient } from "./subtitles.ts"; interface MteamSearchData { data?: MteamTorrentJson[] | null; @@ -78,7 +80,9 @@ export function mteamTorrentToRelease( } export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps): SiteAdapter { - const def: MteamDefinition = deps.baseUrl ? { ...input, umt: { ...input.umt, apiBaseUrl: deps.baseUrl } } : input; + const def: MteamDefinition = deps.baseUrl + ? { ...input, api_domain: "", api_host: "", umt: { ...input.umt, apiBaseUrl: deps.baseUrl } } + : input; const http = buildHttpContext(def.id, { timeoutMs: def.timeout * 1000, ...deps }, def.umt.rateLimit); const api = mteamApiBase(def); @@ -95,9 +99,7 @@ export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps keyword: string | null, signal?: AbortSignal, ): Promise { - const params = mteamSearchParams(keyword, filters.mtype, Math.max(1, page) - 1); - // 本项目扩展:显式分类 ID 优先于 mtype 分类组 - if (filters.categoryIds?.length) params.categories = filters.categoryIds; + const params = mteamSearchParams(keyword, filters.mtype, Math.max(1, page) - 1, filters.categoryIds?.join(",")); const data = await mteamPost( http, api, @@ -106,16 +108,18 @@ export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps { json: params }, signal, ); + if (!data || !Array.isArray(data.data)) + throw new SiteError(def.id, "parse_error", "M-Team 搜索响应 data 不是资源列表"); const observedAt = http.now().toISOString(); const src = data?.data ?? []; const rows = parseMteamResult(src, def.domain); let items = rows.map((t, i) => mteamTorrentToRelease(t, src[i]!, def, observedAt)); if (filters.freeOnly) items = items.filter((r) => r.promotion.downloadFactor === 0); - const tp = Number(data?.totalPages); + const tp = data.totalPages == null ? NaN : Number(data.totalPages); return { items, page: Math.max(1, page), - totalPages: Number.isFinite(tp) && tp >= 0 ? tp : null, + totalPages: Number.isInteger(tp) && tp >= 0 ? tp : null, totalPagesEstimated: false, fetchedAt: observedAt, }; @@ -134,6 +138,7 @@ export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps displayName: def.umt.displayName, capabilities: def.umt.capabilities, definitionVersion: def.umt.version, + ...createMteamSubtitleClient({ http, apiBase: api, apiKey, webUrl: def.domain, timezone: def.umt.timezone }), list(page, filters = {}, opts = {}) { return doSearch(page, filters, null, opts.signal); diff --git a/packages/adapters/src/sites/mteam/client.ts b/packages/adapters/src/sites/mteam/client.ts index 8cf57fad..bc030f59 100644 --- a/packages/adapters/src/sites/mteam/client.ts +++ b/packages/adapters/src/sites/mteam/client.ts @@ -1,4 +1,5 @@ /** + * @mp-source app/modules/indexer/spider/mtorrent.py @ 3b156a5 * M-Team API 调用。对应 MP `MTorrentSpider.search/__process_response`(MP 只看 HTTP 200); * 业务 code/message 检查与凭据/解析错误分层参考 MovieClaw `_post_api`(behavior-ref,迁移账本 MT2)。 */ @@ -11,7 +12,10 @@ export type MteamApiPath = | "/api/torrent/detail" | "/api/torrent/genDlToken" | "/api/member/profile" - | "/api/tracker/myPeerStatus"; + | "/api/tracker/myPeerStatus" + | "/api/subtitle/search" + | "/api/subtitle/list" + | "/api/subtitle/genlink"; const SUCCESS_MESSAGES = new Set(["SUCCESS", "OK", "0", ""]); const AUTH_KEYWORDS = [ @@ -88,8 +92,10 @@ export async function mteamPost( { httpStatus: res.status }, ); } + if (!json || typeof json !== "object" || Array.isArray(json)) + throw new SiteError(http.siteId, "parse_error", `${path}:响应结构无效`); const message = json.message == null ? "" : String(json.message); - const code = json.code == null ? "0" : String(json.code); + const code = json.code == null ? "" : String(json.code); if (code !== "0" || !SUCCESS_MESSAGES.has(message.toUpperCase())) { const lower = message.toLowerCase(); const kind = RATE_KEYWORDS.some((k) => lower.includes(k)) diff --git a/packages/adapters/src/sites/mteam/definition.ts b/packages/adapters/src/sites/mteam/definition.ts index 8b814357..d9ef6a3d 100644 --- a/packages/adapters/src/sites/mteam/definition.ts +++ b/packages/adapters/src/sites/mteam/definition.ts @@ -1,3 +1,6 @@ +/** + * @mp-source app/modules/indexer/spider/mtorrent.py @ 3b156a5 + */ import mteamJson from "../definitions/mteam.json" with { type: "json" }; import { parseMteamDefinition } from "../definitions/schema.ts"; import type { DefinitionProvenance } from "../spider/schema.ts"; @@ -17,6 +20,8 @@ export interface MteamDefinition { /** 网页域(MP `_url`,详情页 = `${domain}detail/${id}`) */ domain: string; parser: "mTorrent"; + api_domain?: string; + api_host?: string; timeout: number; umt: { version: string; @@ -47,6 +52,15 @@ export function rootDomain(url: string): string { return parts.length >= 2 ? parts.slice(-2).join(".") : host; } -export function mteamApiBase(def: MteamDefinition): string { - return (def.umt.apiBaseUrl ?? `https://api.${rootDomain(def.domain)}`).replace(/\/$/, ""); +export function mteamApiBase( + def: Pick & { umt: { apiBaseUrl?: string } }, +): string { + const configured = (def.api_domain || def.api_host)?.trim(); + if (configured) { + const host = new URL(configured.includes("://") ? configured : `https://${configured}`).host; + return `https://${host}`; + } + if (def.umt.apiBaseUrl) return def.umt.apiBaseUrl.replace(/\/+$/, ""); + const domain = new URL(def.domain); + return domain.hostname.startsWith("api.") ? `https://${domain.host}` : `https://api.${rootDomain(def.domain)}`; } diff --git a/packages/adapters/src/sites/mteam/spider.ts b/packages/adapters/src/sites/mteam/spider.ts index 9f0f7d30..77a72491 100644 --- a/packages/adapters/src/sites/mteam/spider.ts +++ b/packages/adapters/src/sites/mteam/spider.ts @@ -1,7 +1,8 @@ /** + * @mp-source app/modules/indexer/spider/mtorrent.py @ 3b156a5 * M-Team API 解析。 * - * 迁移自 MoviePilot e6820a8 `app/modules/indexer/spider/mtorrent.py` `MTorrentSpider`: + * 迁移自 MoviePilot v3.2.4 `app/modules/indexer/spider/mtorrent.py` `MTorrentSpider`: * - `_movie_category/_tv_category/_music_category/_labels` → 同名常量 * - `__get_params` → `mteamSearchParams` * - `__parse_result` → `parseMteamResult` @@ -60,9 +61,15 @@ export interface MteamTorrentJson { } /** MP `__get_params`(page 为 0 起) */ -export function mteamSearchParams(keyword: string | null | undefined, mtype: MediaKind | undefined, page = 0) { +export function mteamSearchParams( + keyword: string | null | undefined, + mtype: MediaKind | undefined, + page = 0, + cat?: string, +) { let categories: string[]; - if (!mtype) categories = []; + if (cat) categories = cat.split(",").map((id) => id.trim()); + else if (!mtype) categories = []; else if (mtype === "tv") categories = TV_CATEGORY; else if (mtype === "music") categories = MUSIC_CATEGORY; else categories = MOVIE_CATEGORY; diff --git a/packages/adapters/src/sites/mteam/subtitles.test.ts b/packages/adapters/src/sites/mteam/subtitles.test.ts new file mode 100644 index 00000000..7fe9a336 --- /dev/null +++ b/packages/adapters/src/sites/mteam/subtitles.test.ts @@ -0,0 +1,178 @@ +import { createMockFetch } from "@umt/testkit"; +import { describe, expect, it } from "vitest"; +import { createSiteRateLimiter } from "../rate-limit.ts"; +import { mtorrentSpider } from "../spiders/mtorrent.ts"; +import { adapterWith, testDef } from "../spiders/testing.ts"; +import { staticSecretResolver } from "../types.ts"; +import { createMteamAdapter } from "./adapter.ts"; +import { mteamApiBase, mteamDefinition } from "./definition.ts"; +import { mteamSearchParams } from "./spider.ts"; + +function adapter(fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise) { + return createMteamAdapter( + { ...mteamDefinition, api_host: "https://subtitles.example/api" }, + { + fetch, + secrets: staticSecretResolver({ mteam: { apiKey: "SECRET-API" } }), + limiter: createSiteRateLimiter({ minIntervalMs: 0 }), + }, + ); +} + +describe("M-Team v3.2.4 API", () => { + it("专用分派同样透传分类和分页元数据，按原始页数判断尾页", async () => { + const mock = createMockFetch({ + "/api/torrent/search": { + body: { + code: 0, + data: { + pageNumber: 2, + totalPages: 4, + data: [{ id: "8001", name: "Demo S01E01", category: "402", size: "1000", status: {} }], + }, + }, + }, + }); + const adapter = adapterWith(testDef("mTorrent", { domain: "https://api.m-team.io/" }), mtorrentSpider, mock.fetch); + const page = await adapter.search("Demo", { mtype: "movie", categoryIds: ["402"], page: 2 }); + expect(JSON.parse(mock.requests[0]!.body!)).toMatchObject({ categories: ["402"], pageNumber: 2 }); + expect(page).toMatchObject({ page: 2, totalPages: 4, totalPagesEstimated: false }); + expect(page.items).toHaveLength(1); + }); + + it("种子与字幕请求共用同一个限流器，下载两阶段也计入频率", async () => { + const mock = createMockFetch({ + "/api/torrent/search": { body: { code: 0, data: { data: [], totalPages: 0 } } }, + "/api/subtitle/search": { body: { code: 0, data: [] } }, + "/api/subtitle/genlink": { body: { code: 0, data: "signed-once" } }, + "/api/subtitle/dlV2": { + body: "1\n00:00:01,000 --> 00:00:02,000\nText", + headers: { "content-type": "application/octet-stream" }, + }, + }); + let clock = 0; + const waits: number[] = []; + const client = createMteamAdapter(mteamDefinition, { + fetch: mock.fetch, + secrets: staticSecretResolver({ mteam: { apiKey: "SECRET-API" } }), + limiter: createSiteRateLimiter({ + minIntervalMs: 250, + now: () => clock, + sleep: async (ms) => { + waits.push(ms); + clock += ms; + }, + }), + }); + await client.list(1); + await client.searchSubtitles!("Demo"); + await client.downloadSubtitle!("2001"); + expect(waits).toEqual([250, 250, 250]); + }); + it("API主机覆盖兼容两种字段；已是api子域时保留主机和端口", () => { + expect(mteamApiBase({ ...mteamDefinition, api_domain: "https://other.example:9443/api" })).toBe( + "https://other.example:9443", + ); + expect(mteamApiBase({ ...mteamDefinition, api_host: "other.example" })).toBe("https://other.example"); + expect(mteamApiBase({ domain: "https://api.m-team.io:9443/", umt: {} })).toBe("https://api.m-team.io:9443"); + expect(mteamSearchParams("tt1234", "tv", 2, "401, 419")).toMatchObject({ + categories: ["401", "419"], + pageNumber: 3, + }); + }); + + it("搜索字幕返回稳定元数据和页数，不生成下载凭据", async () => { + const mock = createMockFetch({ + "/api/subtitle/search": { + body: { + code: 0, + data: { + totalPages: 3, + data: [ + { + id: "sub-100", + name: "Arrival 简中", + filename: "arrival.zh.srt", + lang: "简中", + size: "12 KB", + hits: 7, + author: "Uploader", + torrent: { id: "8001" }, + createdDate: "2026-11-03T01:02:03Z", + }, + ], + }, + }, + }, + }); + const result = await adapter(mock.fetch).searchSubtitles!("Arrival", { page: 2 }); + expect(result).toMatchObject({ + page: 2, + totalPages: 3, + items: [ + { + id: "sub-100", + title: "Arrival 简中", + sizeBytes: 12 * 1024, + language: "简中", + uploadedAt: "2026-11-03T01:02:03.000Z", + torrentId: "8001", + detailUrl: "https://tp.m-team.cc/detail/8001", + }, + ], + }); + expect(mock.requests).toHaveLength(1); + expect(mock.requests[0]!.headers["x-api-key"]).toBe("SECRET-API"); + expect(mock.requests[0]!.url.origin).toBe("https://subtitles.example"); + expect(JSON.parse(mock.requests[0]!.body!)).toEqual({ keyword: "Arrival", pageNumber: 2, pageSize: 20 }); + expect(JSON.stringify(result)).not.toContain("SECRET"); + }); + + it.each(["items", "records", "list"])("兼容%s分页包裹，空数组为成功空页", async (key) => { + const mock = createMockFetch({ "/api/subtitle/search": { body: { code: "0", data: { [key]: [] } } } }); + expect((await adapter(mock.fetch).searchSubtitles!("Unknown")).items).toEqual([]); + }); + + it("按种子列字幕保留关联；下载时换取并正确编码credential，不带API Key下载", async () => { + let credentials = 0; + const mock = createMockFetch({ + "/api/subtitle/list": { body: { code: 0, data: [{ id: "sub-200", fileName: "demo.srt", size: "30" }] } }, + "/api/subtitle/genlink": () => ({ body: { code: 0, data: `token+/${++credentials}` } }), + "/api/subtitle/dlV2": { + body: "1\n00:00:01,000 --> 00:00:02,000\nHello", + headers: { + "content-disposition": "attachment; filename*=UTF-8''%E5%AD%97%E5%B9%95.srt", + "content-type": "application/octet-stream", + }, + }, + }); + const client = adapter(mock.fetch); + expect((await client.listSubtitles!("8002"))[0]).toMatchObject({ + id: "sub-200", + torrentId: "8002", + detailUrl: "https://tp.m-team.cc/detail/8002", + }); + expect(mock.requests[0]!.body).toBe("id=8002"); + const first = await client.downloadSubtitle!("sub-200"); + await client.downloadSubtitle!("sub-200"); + expect(credentials).toBe(2); + expect(first.fileName).toBe("字幕.srt"); + expect(new TextDecoder().decode(first.bytes)).toContain("Hello"); + const downloads = mock.requests.filter((request) => request.url.pathname === "/api/subtitle/dlV2"); + expect(downloads[0]!.url.searchParams.get("credential")).toBe("token+/1"); + expect(downloads[0]!.headers["x-api-key"]).toBeUndefined(); + expect(downloads[0]!.headers.cookie).toBeUndefined(); + }); + + it("API业务失败、错误JSON结构和伪装HTML下载不会冒充成功", async () => { + const invalid = createMockFetch({ "/api/subtitle/search": { body: { code: 1, message: "invalid API key" } } }); + await expect(adapter(invalid.fetch).searchSubtitles!("x")).rejects.toMatchObject({ kind: "auth_invalid" }); + const malformed = createMockFetch({ "/api/torrent/search": { body: { code: 0, data: {} } } }); + await expect(adapter(malformed.fetch).list(1)).rejects.toMatchObject({ kind: "parse_error" }); + const html = createMockFetch({ + "/api/subtitle/genlink": { body: { code: 0, data: "one-use" } }, + "/api/subtitle/dlV2": { body: "Expired", headers: { "content-type": "text/html" } }, + }); + await expect(adapter(html.fetch).downloadSubtitle!("sub-200")).rejects.toMatchObject({ kind: "parse_error" }); + }); +}); diff --git a/packages/adapters/src/sites/mteam/subtitles.ts b/packages/adapters/src/sites/mteam/subtitles.ts new file mode 100644 index 00000000..d723440c --- /dev/null +++ b/packages/adapters/src/sites/mteam/subtitles.ts @@ -0,0 +1,154 @@ +/** + * SPDX-License-Identifier: GPL-3.0-only + * @mp-source app/modules/indexer/spider/mtorrent.py @ 3b156a5 + * 对照 v3.2.4 字幕 API；搜索仅返回稳定 ID，下载时现场换取 credential，不缓存签名链接。 + */ +import { readBody, type SiteHttpContext, siteFetch } from "../http.ts"; +import { parseSizeMp } from "../spider/spider.ts"; +import { isoWithZoneToUtc, isPlainObject } from "../spiders/common.ts"; +import { type SiteAdapter, SiteError, type SiteSubtitle } from "../types.ts"; +import { parseSiteTime } from "../util.ts"; +import { mteamPost } from "./client.ts"; + +function string(value: unknown): string | null { + return value == null || value === "" ? null : String(value); +} + +function rowsOf(data: unknown): unknown[] { + if (Array.isArray(data)) return data; + if (isPlainObject(data)) { + for (const key of ["data", "items", "records", "list"]) if (Array.isArray(data[key])) return data[key]; + } + return []; +} + +export function parseMteamSubtitles(data: unknown, webUrl: string, timezone: string): SiteSubtitle[] { + return rowsOf(data).flatMap((row): SiteSubtitle[] => { + if (!isPlainObject(row) || row.id == null) return []; + const id = String(row.id); + const torrentId = string(isPlainObject(row.torrent) ? row.torrent.id : row.torrent); + const created = row.createdDate ?? row.lastModifiedDate; + let uploadedAt: string | null = null; + if (created != null && /^\d+(?:\.\d+)?$/.test(String(created))) { + const date = new Date(Number(created) * 1000); + if (Number.isFinite(date.getTime())) uploadedAt = date.toISOString(); + } else if (typeof created === "string") { + uploadedAt = isoWithZoneToUtc(created) ?? parseSiteTime(created, timezone); + } + const downloads = row.hits == null ? NaN : Number(row.hits); + return [ + { + id, + title: string(row.name) ?? string(row.filename) ?? string(row.fileName) ?? id, + fileName: string(row.filename ?? row.fileName), + language: string(row.lang ?? row.language), + sizeBytes: row.size == null ? null : parseSizeMp(String(row.size)), + uploadedAt, + downloads: Number.isFinite(downloads) ? downloads : null, + uploader: string(row.author), + torrentId, + detailUrl: torrentId + ? new URL(`detail/${encodeURIComponent(torrentId)}`, `${webUrl.replace(/\/+$/, "")}/`).href + : null, + }, + ]; + }); +} + +function fileName(headers: Headers): string | null { + const header = headers.get("content-disposition") ?? ""; + const encoded = /filename\*=UTF-8''([^;]+)/i.exec(header)?.[1]; + if (encoded) { + try { + return decodeURIComponent(encoded); + } catch { + /* 回退普通文件名。 */ + } + } + return /filename="([^"]+)"/i.exec(header)?.[1] ?? /filename=([^;\s]+)/i.exec(header)?.[1] ?? null; +} + +export function createMteamSubtitleClient(options: { + http: SiteHttpContext; + apiBase: string; + apiKey(): Promise; + webUrl: string; + timezone: string; +}): Pick { + const { http, apiBase, webUrl, timezone } = options; + return { + async searchSubtitles(query, opts = {}) { + const page = Math.max(1, opts.page ?? 1); + const data = await mteamPost( + http, + apiBase, + await options.apiKey(), + "/api/subtitle/search", + { + json: { keyword: query, pageNumber: page, pageSize: 20 }, + }, + opts.signal, + ); + const totalPages = isPlainObject(data) && data.totalPages != null ? Number(data.totalPages) : NaN; + return { + items: parseMteamSubtitles(data, webUrl, timezone), + page, + totalPages: Number.isInteger(totalPages) && totalPages >= 0 ? totalPages : null, + }; + }, + async listSubtitles(torrentId, opts = {}) { + const data = await mteamPost( + http, + apiBase, + await options.apiKey(), + "/api/subtitle/list", + { + form: { id: torrentId }, + }, + opts.signal, + ); + return parseMteamSubtitles(data, webUrl, timezone).map((row) => ({ + ...row, + torrentId: row.torrentId ?? torrentId, + detailUrl: + row.detailUrl ?? new URL(`detail/${encodeURIComponent(torrentId)}`, `${webUrl.replace(/\/+$/, "")}/`).href, + })); + }, + async downloadSubtitle(subtitleId, opts = {}) { + const credential = await mteamPost( + http, + apiBase, + await options.apiKey(), + "/api/subtitle/genlink", + { + form: { id: subtitleId }, + }, + opts.signal, + ); + if (typeof credential !== "string" || !credential) + throw new SiteError(http.siteId, "parse_error", "M-Team 字幕接口未返回下载凭据"); + const url = new URL(`${apiBase.replace(/\/+$/, "")}/api/subtitle/dlV2`); + url.searchParams.set("credential", credential); + const response = await siteFetch(http, url.href, { + redirect: "manual", + ...(opts.signal ? { signal: opts.signal } : {}), + }); + if (!response.ok) { + await response.body?.cancel().catch(() => {}); + throw new SiteError( + http.siteId, + response.status === 401 || response.status === 403 ? "auth_invalid" : "network", + `M-Team 字幕下载：HTTP ${response.status}`, + { httpStatus: response.status }, + ); + } + const bytes = await readBody(http, response, 30 * 1024 * 1024); + if ( + /text\/html/i.test(response.headers.get("content-type") ?? "") || + /^\s*(?: { const c = await deps.secrets.resolve({ siteId: def.id, kind: "cookie" }); + if (!c && def.public) return ""; if (!c) throw new SiteError(def.id, "auth_missing", "未配置站点 Cookie"); return c; } @@ -153,9 +155,10 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap /** 页面认不出来时的错误:像人机验证就报 challenge,否则 parse_error;都附页面样本 */ function unrecognized(page: FetchedPage, what: string, kind: SiteRequestKind, parseMessage: string): SiteError { const rule = challengeRuleOfUnrecognizedPage(page.html); + const title = redactText(loadHtml(page.html)("title").first().text().replace(/\s+/g, " ").trim()).slice(0, 120); const message = rule ? `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText("unhandled")}` - : `${parseMessage}（页面样本见站点页技术信息）`; + : `${parseMessage}（HTTP ${page.status}${title ? `，页面标题：${title}` : ""}；页面样本见站点页技术信息）`; const sample = buildFailureSample({ kind: rule ? "challenge" : "parse_error", requestKind: kind, @@ -186,7 +189,11 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap try { res = await siteFetch(http, url, { method: "GET", - headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base }, + headers: { + cookie: await cookie(), + accept: "text/html,application/xhtml+xml,application/rss+xml,application/xml", + referer: base, + }, redirect: "manual", ...(opts.signal ? { signal: opts.signal } : {}), }); @@ -260,7 +267,8 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap if (parsed.torrents.length === 0) { const text = $("body").text(); const hasList = - $(normalizeSelector(spider.def.torrents.list.selector)).length > 0 || $("table.torrents").length > 0; + $(normalizeSelector(spider.def.torrents.list.selector)).length > 0 || + $("table.torrents, rss > channel, feed").length > 0; const empty = (ext.emptyMarkers ?? []).some((m) => text.includes(m)); // 认不出列表:先看是不是人机验证页(搜索页单独加盾、间歇触发),不是才报改版 if (!hasList && !empty) throw unrecognized(page$, what, kind, `${what}:页面结构无法识别(可能改版)`); @@ -289,8 +297,8 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap return { items, page: Math.max(1, page), - totalPages: maxRaw === null ? null : Math.max(page, (maxRaw as number) - start + 1), - totalPagesEstimated: true, + totalPages: parsed.totalPages ?? (maxRaw === null ? null : Math.max(page, (maxRaw as number) - start + 1)), + totalPagesEstimated: parsed.totalPages === undefined, fetchedAt: observedAt, }; } diff --git a/packages/adapters/src/sites/spider/pagination-rss.test.ts b/packages/adapters/src/sites/spider/pagination-rss.test.ts new file mode 100644 index 00000000..7d059542 --- /dev/null +++ b/packages/adapters/src/sites/spider/pagination-rss.test.ts @@ -0,0 +1,90 @@ +import { createMockFetch } from "@umt/testkit"; +import { describe, expect, it } from "vitest"; +import { createSiteRateLimiter } from "../rate-limit.ts"; +import { staticSecretResolver } from "../types.ts"; +import { createSpiderSiteAdapter } from "./adapter.ts"; +import type { IndexerDefinition } from "./schema.ts"; +import { SiteSpider } from "./spider.ts"; + +const def: IndexerDefinition = { + id: "public-index", + name: "公开索引", + domain: "https://index.example/", + public: true, + result_num: 2, + search: { paths: [{ path: "search?q={keyword}" }] }, + torrents: { + list: { selector: "item" }, + fields: { + title: { selector: "title" }, + download: { selector: "enclosure", attribute: "url" }, + size: { selector: "enclosure", attribute: "length" }, + }, + }, + umt: { + version: "test", + kind: "nexusphp", + timezone: "+08:00", + downloadPath: "download.php", + detailPath: "details.php", + rateLimit: { minIntervalMs: 0, burst: 10 }, + capabilities: { list: true, search: true, detail: false, download: true, rss: false }, + }, +}; +const xml = `\ufeff ${[1, 2, 3, 4, 5].map((id) => `Title ${id} & special`).join("")}`; + +describe("通用索引XML与本地分页", () => { + it("已解码XML按HTML大小写规则提取字段；无页码搜索切出互不重叠的窗口", () => { + const first = new SiteSpider(def, { keyword: "special", page: 0 }).parse(xml); + const second = new SiteSpider(def, { keyword: "special", page: 1 }).parse(xml); + const last = new SiteSpider(def, { keyword: "special", page: 2 }).parse(xml); + const empty = new SiteSpider(def, { keyword: "special", page: 3 }).parse(xml); + expect(first.torrents.map((row) => row.title)).toEqual(["Title 1 & special", "Title 2 & special"]); + expect(second.torrents.map((row) => row.title)).toEqual(["Title 3 & special", "Title 4 & special"]); + expect(last.torrents).toHaveLength(1); + expect(last.torrents[0]).toMatchObject({ title: "Title 5 & special", size: 2000 }); + expect(empty).toMatchObject({ torrents: [], isError: false, totalPages: 3 }); + }); + + it.each([ + { paths: [{ path: "search?q={keyword}&page={page}" }] }, + { paths: [{ path: "search" }], params: { q: "{keyword}" } }, + ])("服务端已收到page时不重复切片 %#", (search) => { + const spider = new SiteSpider({ ...def, search }, { keyword: "special", page: 2 }); + expect(new URL(spider.getSearchUrl()).searchParams.get("page")).toBe("2"); + expect(spider.parse(xml).torrents[0]!.title).toBe("Title 1 & special"); + }); + + it("纯媒体类型站的缺失分类兜底，不覆盖已知分类", () => { + const result = new SiteSpider({ ...def, media_type: "music" }).parse(xml); + expect(result.torrents.every((row) => row.category === "音乐")).toBe(true); + }); + + it("公开RSS不要求Cookie，空channel和分页结束均为成功空页", async () => { + let body = xml; + const mock = createMockFetch({ "/search": () => ({ body, headers: { "content-type": "application/rss+xml" } }) }); + const adapter = createSpiderSiteAdapter(def, { + fetch: mock.fetch, + secrets: staticSecretResolver({}), + limiter: createSiteRateLimiter({ minIntervalMs: 0 }), + }); + const page = await adapter.search("special", { page: 2 }); + expect(page).toMatchObject({ page: 2, totalPages: 3, totalPagesEstimated: false }); + expect(page.items.map((row) => row.siteReleaseId)).toEqual(["3", "4"]); + expect((await adapter.search("special", { page: 4 })).items).toEqual([]); + body = ""; + expect((await adapter.search("special")).items).toEqual([]); + }); + + it("无法识别的200页面错误包含安全标题", async () => { + const mock = createMockFetch({ + "/search": { body: "Search temporarily unavailableLater" }, + }); + const adapter = createSpiderSiteAdapter(def, { + fetch: mock.fetch, + secrets: staticSecretResolver({}), + limiter: createSiteRateLimiter({ minIntervalMs: 0 }), + }); + await expect(adapter.search("special")).rejects.toThrow("HTTP 200，页面标题：Search temporarily unavailable"); + }); +}); diff --git a/packages/adapters/src/sites/spider/spider.ts b/packages/adapters/src/sites/spider/spider.ts index ece36689..7d365c81 100644 --- a/packages/adapters/src/sites/spider/spider.ts +++ b/packages/adapters/src/sites/spider/spider.ts @@ -1,7 +1,8 @@ /** * 通用站点爬虫(配置驱动)。 * - * 迁移自 MoviePilot e6820a8 `app/modules/indexer/spider/__init__.py`: + * 迁移自 MoviePilot v3.2.4 `app/modules/indexer/spider/__init__.py`: + * @mp-source app/modules/indexer/spider/__init__.py @ 3b156a5 * - `select_media_categories` / `resolve_category_media_type` → 同名函数 * - `SiteSpider.__get_search_url` / `__format_search_word` → `SiteSpider.getSearchUrl` / `formatSearchWord` * - `SiteSpider.get_info` 及 `__get_title/__get_description/__get_detail/__get_download/__get_grabs/__get_leechers/ @@ -46,7 +47,8 @@ const MEDIA_VALUE: Record = { movie: "电影", tv: "电 * 使 `table.torrents > tr` 这类 MP 风格选择器保持原语义。 */ export function loadHtml(html: string): CheerioAPI { - return load(html, { xml: { xmlMode: false, decodeEntities: true } }); + const decoded = html.replace(/^[\s\ufeff]*<\?xml\b[^>]*\?>/i, ""); + return load(decoded, { xml: { xmlMode: false, decodeEntities: true, recognizeSelfClosing: true } }); } /** @@ -245,6 +247,8 @@ export class SiteSpider { private browseCfg: IndexerDefinition["browse"]; private categoryCfg: SpiderCategoryConfig | undefined; private requestedResultMediaType: MediaKind | undefined; + private resultOffset = 0; + private paginateLocally = false; private $!: CheerioAPI; private info: SpiderTorrent = {}; @@ -300,6 +304,8 @@ export class SiteSpider { /** MP `__get_search_url` */ getSearchUrl(): string { + this.resultOffset = 0; + this.paginateLocally = false; const search = this.searchCfg ?? { paths: [] }; const paths = search.paths ?? []; let torrentspath = ""; @@ -379,6 +385,8 @@ export class SiteSpider { } searchurl = combineUrl(this.domain, torrentspath, params); } else { + this.paginateLocally = !torrentspath.includes("{page}"); + this.resultOffset = this.paginateLocally ? Math.max(0, this.page) * this.resultNum : 0; searchurl = this.domain + torrentspath @@ -438,20 +446,33 @@ export class SiteSpider { } /** MP `parse`:登录/权限页返回 isError=true 与空列表。 */ - parse(html: string): { isError: boolean; loginPage: boolean; torrents: SpiderTorrent[]; $: CheerioAPI | null } { + parse(html: string): { + isError: boolean; + loginPage: boolean; + torrents: SpiderTorrent[]; + $: CheerioAPI | null; + totalPages?: number; + } { if (!html) return { isError: true, loginPage: false, torrents: [], $: null }; + // 单独调用解析器时也与请求入口使用相同的分页窗口和媒体类型投影。 + this.getSearchUrl(); const $ = loadHtml(html); if (SiteSpider.isLoginOrPermissionPage($)) return { isError: true, loginPage: true, torrents: [], $ }; this.$ = $; const out: SpiderTorrent[] = []; const rows = $(normalizeSelector(this.list.selector ?? "")); - rows.each((i, el) => { - if (i >= this.resultNum) return false; + rows.slice(this.resultOffset, this.resultOffset + this.resultNum).each((_, el) => { const info = this.searchType === "subtitles" ? this.getSubtitleInfo($(el)) : this.getInfo($(el)); if (Object.keys(info).length > 0) out.push(this.applyRequestedMediaType(info)); return undefined; }); - return { isError: false, loginPage: false, torrents: out, $ }; + return { + isError: false, + loginPage: false, + torrents: out, + $, + ...(this.paginateLocally ? { totalPages: Math.ceil(rows.length / this.resultNum) } : {}), + }; } /** 对任意片段(如详情页头部)按字段配置取值 */ @@ -462,6 +483,7 @@ export class SiteSpider { private applyRequestedMediaType(t: SpiderTorrent): SpiderTorrent { if (this.requestedResultMediaType) t.category = MEDIA_VALUE[this.requestedResultMediaType]; + else if (this.siteMediaType && (!t.category || t.category === "未知")) t.category = MEDIA_VALUE[this.siteMediaType]; return t; } diff --git a/packages/adapters/src/sites/spiders/adapter.ts b/packages/adapters/src/sites/spiders/adapter.ts index 8bbb0d1d..341405de 100644 --- a/packages/adapters/src/sites/spiders/adapter.ts +++ b/packages/adapters/src/sites/spiders/adapter.ts @@ -1,7 +1,7 @@ /** * SPDX-License-Identifier: GPL-3.0-only * 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。 - * @mp-source app/modules/indexer/__init__.py @ e6820a8 + * @mp-source app/modules/indexer/__init__.py @ 3b156a5 * * 专用 spider 站点适配器:对应 MP `IndexerModule.search_torrents/refresh_torrents` 按 `indexer.parser` * 分派到 `SPIDER_PARSER_CLASSES` 并以 `_SPECIALIZED_SEARCH_ARGUMENTS` 传参; @@ -33,6 +33,9 @@ export function createSpecializedSiteAdapter( const def: SpecializedSiteDefinition = deps.baseUrl ? { ...input, + api_url: "", + api_domain: "", + api_host: "", domain: deps.baseUrl.endsWith("/") ? deps.baseUrl : `${deps.baseUrl}/`, umt: { ...input.umt, apiBaseUrl: deps.baseUrl.replace(/\/+$/, "") }, } @@ -78,7 +81,12 @@ export function createSpecializedSiteAdapter( const q: Parameters[1] = { keyword, page: p - 1 }; if (filters.mtype) q.mtype = filters.mtype; if (filters.categoryIds?.length) q.cat = filters.categoryIds.join(","); - const rows = await spider.search(ctx(signal), q); + let totalPages: number | null = null; + const context = ctx(signal); + context.reportPage = (info) => { + totalPages = info.totalPages; + }; + const rows = await spider.search(context, q); let items = rows.flatMap((t) => { const r = specializedTorrentToRelease(t, def, fetchedAt); if (r && t.enclosure) enclosures.set(r.siteReleaseId, t.enclosure); @@ -89,7 +97,7 @@ export function createSpecializedSiteAdapter( items = items.filter((r) => r.category.id === null || allow.has(r.category.id)); } if (filters.freeOnly) items = items.filter((r) => r.promotion.downloadFactor === 0); - return { items, page: p, totalPages: null, totalPagesEstimated: false, fetchedAt } satisfies SitePage; + return { items, page: p, totalPages, totalPagesEstimated: false, fetchedAt } satisfies SitePage; } return { diff --git a/packages/adapters/src/sites/spiders/haidan.ts b/packages/adapters/src/sites/spiders/haidan.ts index 4b9d08e0..e25613e4 100644 --- a/packages/adapters/src/sites/spiders/haidan.ts +++ b/packages/adapters/src/sites/spiders/haidan.ts @@ -1,7 +1,7 @@ /** * SPDX-License-Identifier: GPL-3.0-only * 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。 - * @mp-source app/modules/indexer/spider/haidan.py @ e6820a8 + * @mp-source app/modules/indexer/spider/haidan.py @ 3b156a5 * * 海胆 haidan.video(parser "Haidan",Cookie + `torrents.php?isapi=1` JSON)。函数对照 `HaiDanSpider`: * - `_movie_category/_tv_category/_music_category/_dl_state/_up_state` → 同名常量 diff --git a/packages/adapters/src/sites/spiders/hddolby.test.ts b/packages/adapters/src/sites/spiders/hddolby.test.ts index 1d3c853e..286afc1f 100644 --- a/packages/adapters/src/sites/spiders/hddolby.test.ts +++ b/packages/adapters/src/sites/spiders/hddolby.test.ts @@ -7,7 +7,7 @@ const def = testDef("HDDolby", { domain: "https://www.hddolby.com/" }); describe("HddolbySpider(MP hddolby.py)", () => { it("__get_params 与 API 地址:page_number 为 0 起、无类型时 movie+tv 去重升序", () => { - expect(hddolbyApiBase(def)).toBe("https://api.hddolby.com"); + expect(hddolbyApiBase(def)).toBe("https://api.hddolby.com/api/v1"); expect(hddolbySearchParams("Lost")).toEqual({ keyword: "Lost", page_number: 0, @@ -16,6 +16,16 @@ describe("HddolbySpider(MP hddolby.py)", () => { visible: 1, }); expect(hddolbySearchParams(null, "music", 2).categories).toEqual([406, 408]); + expect(hddolbySearchParams("movie", "tv", 0, "401, 405").categories).toEqual([401, 405]); + }); + + it("显式API版本路径与分类透传，网页域不决定API路径", async () => { + const configured = { ...def, api_url: "https://gateway.example/open/v2/" }; + const mock = createMockFetch({ + "POST https://gateway.example/open/v2/torrent/search": { body: HDDOLBY_SEARCH_OK }, + }); + await adapterWith(configured, hddolbySpider, mock.fetch).search("Lost", { mtype: "movie", categoryIds: ["402"] }); + expect(JSON.parse(mock.requests[0]!.body!).categories).toEqual([402]); }); it("列表解析:促销类型表、标签表、分类优先级(405 归电视剧)、downhash 脱敏", async () => { diff --git a/packages/adapters/src/sites/spiders/hddolby.ts b/packages/adapters/src/sites/spiders/hddolby.ts index 603dd0a8..0c8e6bb1 100644 --- a/packages/adapters/src/sites/spiders/hddolby.ts +++ b/packages/adapters/src/sites/spiders/hddolby.ts @@ -1,7 +1,7 @@ /** * SPDX-License-Identifier: GPL-3.0-only * 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。 - * @mp-source app/modules/indexer/spider/hddolby.py @ e6820a8 + * @mp-source app/modules/indexer/spider/hddolby.py @ 3b156a5 * * HDDolby(parser "HDDolby",`https://api.<域>/api/v1/torrent/search`,x-api-key + Cookie)。函数对照 `HddolbySpider`: * - `_movie_category/_tv_category/_music_category/_labels` → 同名常量 @@ -13,6 +13,7 @@ * 差异:API 返回的 `hr` 字段 MP 未映射,这里同样不映射(H&R = unknown,待核验语义)。 */ import type { MpMediaType } from "../spider/spider.ts"; +import { SiteError } from "../types.ts"; import { extractDomain } from "../user-parsers/common.ts"; import { apiRequest, businessError, checkReleaseId, downloadTorrentFile, isPlainObject, siteBase } from "./common.ts"; import type { SpecializedSiteDefinition, SpecializedSpider, SpecializedTorrent, SpiderSearchQuery } from "./types.ts"; @@ -60,13 +61,21 @@ export function hddolbyUploadVolumeFactor(discount: unknown): number { } export function hddolbyApiBase(def: SpecializedSiteDefinition): string { - return (def.umt.apiBaseUrl ?? `https://api.${extractDomain(def.domain)}`).replace(/\/+$/, ""); + if (def.api_url) return def.api_url.replace(/\/+$/, ""); + const base = (def.umt.apiBaseUrl ?? `https://api.${extractDomain(def.domain)}`).replace(/\/+$/, ""); + return new URL(base).pathname === "/" ? `${base}/api/v1` : base; } /** MP `__get_params` */ -export function hddolbySearchParams(keyword: string | null | undefined, mtype?: SpiderSearchQuery["mtype"], page = 0) { +export function hddolbySearchParams( + keyword: string | null | undefined, + mtype?: SpiderSearchQuery["mtype"], + page = 0, + cat?: string, +) { let categories: number[]; - if (mtype === "tv") categories = HDDOLBY_TV_CATEGORY; + if (cat) categories = cat.split(",").map((id) => Number(id.trim())); + else if (mtype === "tv") categories = HDDOLBY_TV_CATEGORY; else if (mtype === "movie") categories = HDDOLBY_MOVIE_CATEGORY; else if (mtype === "music") categories = HDDOLBY_MUSIC_CATEGORY; // list(set(movie + tv)):CPython 小整数集合的迭代顺序即升序 @@ -149,11 +158,27 @@ export const hddolbySpider: SpecializedSpider = { Referer: siteBase(ctx.def), }; if (cookie) headers.cookie = cookie; - const data = await apiRequest(ctx, `${hddolbyApiBase(ctx.def)}/api/v1/torrent/search`, { - method: "POST", - headers, - json: hddolbySearchParams(q.keyword, q.mtype, q.page ?? 0), - }); + const api = hddolbyApiBase(ctx.def); + let data: unknown; + try { + data = await apiRequest(ctx, `${api}/torrent/search`, { + method: "POST", + headers, + json: hddolbySearchParams(q.keyword, q.mtype, q.page ?? 0, q.cat), + }); + } catch (error) { + if (error instanceof SiteError && error.kind === "network") { + throw new SiteError( + ctx.def.id, + "network", + `HDDolby API 主机 ${new URL(api).hostname} 无法连接，检查 API 地址及网络代理`, + { + ...(error.httpStatus === undefined ? {} : { httpStatus: error.httpStatus }), + }, + ); + } + throw error; + } if (!isPlainObject(data)) throw businessError(ctx, "搜索", "响应结构无效"); if (data.error) { const err = data.error; diff --git a/packages/adapters/src/sites/spiders/index.ts b/packages/adapters/src/sites/spiders/index.ts index 761cbaed..31934708 100644 --- a/packages/adapters/src/sites/spiders/index.ts +++ b/packages/adapters/src/sites/spiders/index.ts @@ -1,7 +1,7 @@ /** * SPDX-License-Identifier: GPL-3.0-only * 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。 - * @mp-source app/modules/indexer/__init__.py @ e6820a8 + * @mp-source app/modules/indexer/__init__.py @ 3b156a5 * * 专用 spider 汇总:`SPECIALIZED_SPIDERS` = MP `SPIDER_PARSER_CLASSES`。 */ @@ -10,6 +10,7 @@ import type { SiteAdapter } from "../types.ts"; import { createSpecializedSiteAdapter } from "./adapter.ts"; import { haidanSpider } from "./haidan.ts"; import { hddolbySpider } from "./hddolby.ts"; +import { milkieSpider } from "./milkie.ts"; import { mtorrentSpider } from "./mtorrent.ts"; import { anibtSpider, animeGardenSpider, bangumiMoeSpider, mikanSpider, nyaaSpider } from "./public.ts"; import { rousiSpider } from "./rousi.ts"; @@ -23,6 +24,7 @@ export const SPECIALIZED_SPIDERS: Record = TNodeSpider: tnodeSpider, TorrentLeech: torrentleechSpider, mTorrent: mtorrentSpider, + Milkie: milkieSpider, Yema: yemaSpider, Haidan: haidanSpider, HDDolby: hddolbySpider, @@ -60,6 +62,7 @@ export { export * from "./haidan.ts"; export * from "./hddolby.ts"; export { releaseIdFromPageUrl, specializedTorrentToRelease } from "./map.ts"; +export * from "./milkie.ts"; export * from "./mtorrent.ts"; export * from "./rousi.ts"; export * from "./sunnypt.ts"; diff --git a/packages/adapters/src/sites/spiders/map.ts b/packages/adapters/src/sites/spiders/map.ts index 378fb8c1..d9bcb84a 100644 --- a/packages/adapters/src/sites/spiders/map.ts +++ b/packages/adapters/src/sites/spiders/map.ts @@ -1,7 +1,7 @@ /** * SPDX-License-Identifier: GPL-3.0-only * 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。 - * @mp-source app/modules/indexer/__init__.py @ e6820a8 + * @mp-source app/modules/indexer/__init__.py @ 3b156a5 * * 专用 spider 输出(MP TorrentInfo 字段)→ 统一 `SiteRelease`。 * 促销类型由 MP 系数组合推出(复用 spider/map.ts `promotionFromFactors`);H&R 未提供时为 unknown; diff --git a/packages/adapters/src/sites/spiders/milkie.test.ts b/packages/adapters/src/sites/spiders/milkie.test.ts new file mode 100644 index 00000000..67975d65 --- /dev/null +++ b/packages/adapters/src/sites/spiders/milkie.test.ts @@ -0,0 +1,67 @@ +import { createMockFetch } from "@umt/testkit"; +import { describe, expect, it } from "vitest"; +import { milkieSearchParams, milkieSpider } from "./milkie.ts"; +import { adapterWith, expectSiteError, testDef, withTorrent } from "./testing.ts"; + +const def = testDef("Milkie", { domain: "https://milkie.example/" }); +const id = "abc123-48ef-9001"; + +describe("Milkie API", () => { + it("浏览与关键词搜索使用不同分类策略、API页码0起", () => { + expect(milkieSearchParams({ mtype: "tv", page: 2 })).toEqual({ pi: 2, ps: 100, oby: "created_at", odir: "desc" }); + expect(milkieSearchParams({ keyword: "Arrival", mtype: "movie" })).toMatchObject({ + query: "Arrival", + categories: "1", + pi: 0, + }); + }); + + it("保留UUID资源ID、分类和原时区；搜索结果不携带API Key或签名链接", async () => { + const mock = createMockFetch({ + "/api/v1/torrents": { + body: { + torrents: [ + { + id, + releaseName: "Arrival.2016.1080p", + category: 1, + createdAt: "2026-11-03T09:30:00+02:00", + size: 5000, + seeders: 12, + leechers: 1, + downloaded: 80, + externals: { imdb: "tt2543164" }, + }, + ], + }, + }, + }); + const page = await adapterWith(def, milkieSpider, mock.fetch).search("Arrival", { mtype: "movie", page: 3 }); + expect(mock.requests[0]!.headers["x-milkie-auth"]).toBe("APIKEYSECRET"); + expect(mock.requests[0]!.url.searchParams.get("pi")).toBe("2"); + expect(page.items[0]).toMatchObject({ + siteReleaseId: id, + title: "Arrival.2016.1080p", + category: { id: "1" }, + uploadedAt: "2026-11-03T07:30:00.000Z", + promotion: { downloadFactor: 0 }, + imdbId: "tt2543164", + }); + expect(JSON.stringify(page)).not.toContain("APIKEYSECRET"); + expect(page.items[0]!.raw.enclosure).toBeUndefined(); + }); + + it("下载时编码当前API Key；缺凭据、无效凭据与响应结构各自报错", async () => { + const mock = createMockFetch({ "/api/v1/torrents": { body: {} } }); + const torrent = withTorrent(mock.fetch, (url) => url.pathname.endsWith("/torrent")); + const adapter = adapterWith(def, milkieSpider, torrent.fetch, { apiKey: "a+b/&secret" }); + await adapter.downloadTarget(id); + expect(torrent.hits[0]!.url.searchParams.get("key")).toBe("a+b/&secret"); + expect(torrent.hits[0]!.url.pathname).toBe(`/api/v1/torrents/${id}/torrent`); + expect(torrent.hits[0]!.headers.get("x-milkie-auth")).toBeNull(); + await expectSiteError(adapter.list(1), "parse_error"); + await expectSiteError(adapterWith(def, milkieSpider, mock.fetch, {}).list(1), "auth_missing"); + const denied = createMockFetch({ "/api/v1/torrents": { status: 401 } }); + await expectSiteError(adapterWith(def, milkieSpider, denied.fetch).list(1), "auth_invalid"); + }); +}); diff --git a/packages/adapters/src/sites/spiders/milkie.ts b/packages/adapters/src/sites/spiders/milkie.ts new file mode 100644 index 00000000..edce1527 --- /dev/null +++ b/packages/adapters/src/sites/spiders/milkie.ts @@ -0,0 +1,80 @@ +/** + * SPDX-License-Identifier: GPL-3.0-only + * @mp-source app/modules/indexer/spider/milkie.py @ 3b156a5 + * 对照 MoviePilot v3.2.4 的公开 Milkie API 行为实现。搜索结果不保存含 API Key 的下载地址。 + */ +import { SiteError } from "../types.ts"; +import { apiRequest, businessError, downloadTorrentFile, isoWithZoneToUtc, isPlainObject, siteBase } from "./common.ts"; +import type { SpecializedSiteDefinition, SpecializedSpider, SpecializedTorrent, SpiderSearchQuery } from "./types.ts"; + +export function milkieSearchParams(query: SpiderSearchQuery): Record { + const params: Record = { pi: query.page ?? 0, ps: 100, oby: "created_at", odir: "desc" }; + if (query.keyword) { + params.query = query.keyword; + const category = query.mtype ? { movie: "1", tv: "2", music: "3" }[query.mtype] : undefined; + if (category) params.categories = category; + } + return params; +} + +/** Milkie 资源 ID 可能为 UUID；不使用其它站点的数字 ID 提取规则。 */ +function milkieId(siteId: string, value: unknown): string { + const id = String(value ?? ""); + if (!/^[a-zA-Z0-9_-]+$/.test(id)) throw new SiteError(siteId, "not_found", "Milkie 资源 ID 非法"); + return id; +} + +export function parseMilkieResult(rows: unknown[], def: SpecializedSiteDefinition): SpecializedTorrent[] { + return rows.flatMap((row): SpecializedTorrent[] => { + if (!isPlainObject(row) || row.id == null || !row.releaseName) return []; + const id = milkieId(def.id, row.id); + const category = { "1": "电影", "2": "电视剧", "3": "音乐" } as const; + const result: SpecializedTorrent = { + title: String(row.releaseName), + description: id, + site_release_id: id, + page_url: `${siteBase(def)}browse/${encodeURIComponent(id)}`, + category: category[String(row.category) as keyof typeof category] ?? "未知", + downloadvolumefactor: 0, + uploadvolumefactor: 1, + }; + if (row.category != null) result.category_id = String(row.category); + for (const [source, target] of [ + ["size", "size"], + ["seeders", "seeders"], + ["leechers", "peers"], + ["downloaded", "grabs"], + ] as const) { + const value = row[source] == null ? NaN : Number(row[source]); + if (Number.isFinite(value)) result[target] = value; + } + const utc = isoWithZoneToUtc(row.createdAt); + if (utc) result.pubdate_utc = utc; + else if (typeof row.createdAt === "string") result.pubdate = row.createdAt; + if (isPlainObject(row.externals) && typeof row.externals.imdb === "string") result.imdbid = row.externals.imdb; + return [result]; + }); +} + +export const milkieSpider: SpecializedSpider = { + parser: "Milkie", + credential: "apiKey", + pageSize: () => 100, + async search(ctx, query) { + const key = (await ctx.secret("apiKey", true))!; + const payload = await apiRequest(ctx, `${siteBase(ctx.def)}api/v1/torrents`, { + headers: { "x-milkie-auth": key }, + query: milkieSearchParams(query), + }); + if (!isPlainObject(payload) || !Array.isArray(payload.torrents)) + throw businessError(ctx, "搜索", "Milkie 响应不含 torrents 列表"); + return parseMilkieResult(payload.torrents, ctx.def); + }, + async resolveDownload(ctx, siteReleaseId) { + const key = (await ctx.secret("apiKey", true))!; + const id = milkieId(ctx.def.id, siteReleaseId); + const url = new URL(`api/v1/torrents/${encodeURIComponent(id)}/torrent`, siteBase(ctx.def)); + url.searchParams.set("key", key); + return downloadTorrentFile(ctx, url.href); + }, +}; diff --git a/packages/adapters/src/sites/spiders/mtorrent.ts b/packages/adapters/src/sites/spiders/mtorrent.ts index 9d3594eb..c5e4996e 100644 --- a/packages/adapters/src/sites/spiders/mtorrent.ts +++ b/packages/adapters/src/sites/spiders/mtorrent.ts @@ -1,21 +1,21 @@ /** * SPDX-License-Identifier: GPL-3.0-only * 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。 - * @mp-source app/modules/indexer/spider/mtorrent.py @ e6820a8 + * @mp-source app/modules/indexer/spider/mtorrent.py @ 3b156a5 * * `mTorrent` 键的专用 spider 形态,复用 W1-3b 已移植的 mteam/spider.ts(`__get_params`/`__parse_result`) * 与 mteam/client.ts(`__process_response` + 业务码检查),供 `SPECIALIZED_SPIDERS` 统一分派。 * 完整适配(详情、商城免费、healthCheck=profile)仍以 `createMteamAdapter` 为准。 */ import { mteamPost } from "../mteam/client.ts"; -import { rootDomain } from "../mteam/definition.ts"; +import { mteamApiBase } from "../mteam/definition.ts"; import { type MteamTorrentJson, mteamSearchParams, parseMteamResult } from "../mteam/spider.ts"; import { SiteError } from "../types.ts"; import { checkReleaseId, downloadTorrentFile, siteBase } from "./common.ts"; import type { SpecializedSiteDefinition, SpecializedSpider, SpecializedTorrent } from "./types.ts"; export function mtorrentApiBase(def: SpecializedSiteDefinition): string { - return (def.umt.apiBaseUrl ?? `https://api.${rootDomain(def.domain)}`).replace(/\/+$/, ""); + return mteamApiBase(def); } export const mtorrentSpider: SpecializedSpider = { @@ -24,8 +24,8 @@ export const mtorrentSpider: SpecializedSpider = { pageSize: () => 100, async search(ctx, q) { const apikey = (await ctx.secret("apiKey", true))!; - const params = mteamSearchParams(q.keyword ?? null, q.mtype, q.page ?? 0); - const data = await mteamPost<{ data?: MteamTorrentJson[] | null }>( + const params = mteamSearchParams(q.keyword ?? null, q.mtype, q.page ?? 0, q.cat); + const data = await mteamPost<{ data?: MteamTorrentJson[] | null; totalPages?: number }>( ctx.http, mtorrentApiBase(ctx.def), apikey, @@ -33,6 +33,9 @@ export const mtorrentSpider: SpecializedSpider = { { json: params }, ctx.signal, ); + if (!data || !Array.isArray(data.data)) + throw new SiteError(ctx.def.id, "parse_error", "M-Team 搜索响应 data 不是资源列表"); + if (Number.isInteger(data.totalPages) && data.totalPages! >= 0) ctx.reportPage?.({ totalPages: data.totalPages! }); const src = data?.data ?? []; return parseMteamResult(src, siteBase(ctx.def)).map( (t, i): SpecializedTorrent => ({ ...t, site_release_id: String(src[i]!.id) }), diff --git a/packages/adapters/src/sites/spiders/rousi.ts b/packages/adapters/src/sites/spiders/rousi.ts index b1c3b72c..23e3b22f 100644 --- a/packages/adapters/src/sites/spiders/rousi.ts +++ b/packages/adapters/src/sites/spiders/rousi.ts @@ -1,7 +1,7 @@ /** * SPDX-License-Identifier: GPL-3.0-only * 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。 - * @mp-source app/modules/indexer/spider/rousi.py @ e6820a8 + * @mp-source app/modules/indexer/spider/rousi.py @ 3b156a5 * * Rousi.pro PeerGo 兼容 API(parser "RousiPro",Bearer 个人 API Key,需 torrent:read + torrent:download)。 * 函数对照 `RousiSpider`: diff --git a/packages/adapters/src/sites/spiders/sunnypt.ts b/packages/adapters/src/sites/spiders/sunnypt.ts index c235e6d0..2649d83c 100644 --- a/packages/adapters/src/sites/spiders/sunnypt.ts +++ b/packages/adapters/src/sites/spiders/sunnypt.ts @@ -1,7 +1,7 @@ /** * SPDX-License-Identifier: GPL-3.0-only * 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。 - * @mp-source app/modules/indexer/spider/sunnypt.py @ e6820a8 + * @mp-source app/modules/indexer/spider/sunnypt.py @ 3b156a5 * * SunnyPT MoviePilot API(parser "SunnyPT",X-API-Key)。函数对照 `SunnyPTSpider`: * - `_size/_category_cache_ttl` → `SUNNYPT_PAGE_SIZE/SUNNYPT_CATEGORY_CACHE_TTL_MS` diff --git a/packages/adapters/src/sites/spiders/tnode.test.ts b/packages/adapters/src/sites/spiders/tnode.test.ts index a457b10c..1d4de06c 100644 --- a/packages/adapters/src/sites/spiders/tnode.test.ts +++ b/packages/adapters/src/sites/spiders/tnode.test.ts @@ -1,11 +1,34 @@ import { createMockFetch, TNODE_HOME_HTML, TNODE_HOME_NO_TOKEN, TNODE_SEARCH_OK } from "@umt/testkit"; import { describe, expect, it } from "vitest"; +import { createSiteRateLimiter } from "../rate-limit.ts"; +import { createSpecializedSiteAdapter } from "./adapter.ts"; import { adapterWith, expectSiteError, testDef, withTorrent } from "./testing.ts"; import { parseTnodeToken, tnodeSearchParams, tnodeSpider } from "./tnode.ts"; const def = testDef("TNodeSpider", { domain: "https://tnode.example/" }); describe("TNodeSpider(MP tnode.py)", () => { + it("凭据变更后不复用旧CSRF令牌", async () => { + let cookie = "session=first"; + const mock = createMockFetch({ + "GET /": { body: TNODE_HOME_HTML }, + "POST /api/torrent/advancedSearch": { body: TNODE_SEARCH_OK }, + }); + const adapter = createSpecializedSiteAdapter(def, tnodeSpider, { + fetch: mock.fetch, + secrets: { + async resolve() { + return cookie; + }, + }, + limiter: createSiteRateLimiter({ minIntervalMs: 0 }), + }); + await adapter.list(1); + await adapter.list(1); + cookie = "session=second"; + await adapter.list(1); + expect(mock.requests.filter((request) => request.url.pathname === "/")).toHaveLength(2); + }); it("__get_params / token 解析", () => { expect(tnodeSearchParams("tt0923811", 1)).toMatchObject({ page: 2, diff --git a/packages/adapters/src/sites/spiders/tnode.ts b/packages/adapters/src/sites/spiders/tnode.ts index 23b62ad3..e57b20cd 100644 --- a/packages/adapters/src/sites/spiders/tnode.ts +++ b/packages/adapters/src/sites/spiders/tnode.ts @@ -1,7 +1,7 @@ /** * SPDX-License-Identifier: GPL-3.0-only * 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。 - * @mp-source app/modules/indexer/spider/tnode.py @ e6820a8 + * @mp-source app/modules/indexer/spider/tnode.py @ 3b156a5 * * TNode(parser "TNodeSpider",Cookie + 首页 CSRF token + `api/torrent/advancedSearch`)。函数对照 `TNodeSpider`: * - `get_search_page_size` → `pageSize`(100) @@ -11,6 +11,7 @@ * - `_downloadurl/_pageurl` → `resolveDownload`(带 Cookie 下载)/ page_url * 差异:取不到 token 时 MP 返回 `(True, [])`;这里抛 auth_expired(首页取不到 token 基本意味着 Cookie 失效)。 */ +import { createHash } from "node:crypto"; import { SiteError } from "../types.ts"; import { apiRequest, @@ -33,7 +34,10 @@ export function parseTnodeToken(html: string): string | null { } async function tnodeToken(ctx: SpiderContext, cookie: string): Promise { - const key = `tnode:token:${ctx.def.domain}`; + const identity = createHash("sha256") + .update(JSON.stringify([ctx.def.domain, cookie, ctx.userAgent ?? ""])) + .digest("hex"); + const key = `tnode:token:${identity}`; const cached = cacheGet(ctx, key); if (cached) return cached; const token = parseTnodeToken(await textRequest(ctx, siteBase(ctx.def), { cookie })); diff --git a/packages/adapters/src/sites/spiders/torrentleech.ts b/packages/adapters/src/sites/spiders/torrentleech.ts index 3147238d..edfc3395 100644 --- a/packages/adapters/src/sites/spiders/torrentleech.ts +++ b/packages/adapters/src/sites/spiders/torrentleech.ts @@ -1,7 +1,7 @@ /** * SPDX-License-Identifier: GPL-3.0-only * 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。 - * @mp-source app/modules/indexer/spider/torrentleech.py @ e6820a8 + * @mp-source app/modules/indexer/spider/torrentleech.py @ 3b156a5 * * TorrentLeech JSON 搜索接口(parser "TorrentLeech",Cookie)。函数对照 `TorrentLeech`: * - `get_search_page_size` → `pageSize`(关键词搜索 null,浏览 100) diff --git a/packages/adapters/src/sites/spiders/types.ts b/packages/adapters/src/sites/spiders/types.ts index 3e5ae9e1..4fc819a9 100644 --- a/packages/adapters/src/sites/spiders/types.ts +++ b/packages/adapters/src/sites/spiders/types.ts @@ -1,7 +1,7 @@ /** * SPDX-License-Identifier: GPL-3.0-only * 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。 - * @mp-source app/modules/indexer/__init__.py @ e6820a8 + * @mp-source app/modules/indexer/__init__.py @ 3b156a5 * * 专用 spider(API 型站点)共用类型。 * - `SpecializedParser` = MP `SPIDER_PARSER_CLASSES` 的键;`SPECIALIZED_SEARCH_ARGUMENTS` 的参数集合 → `SpiderSearchQuery` @@ -21,6 +21,7 @@ export type SpecializedParser = | "TorrentLeech" | "Yema" | "mTorrent" + | "Milkie" // 本项目扩展:公开 BT 站(public.ts),MP 无对应 | "Nyaa" | "Mikan" @@ -45,6 +46,9 @@ export interface SpecializedSiteDefinition { category?: SpiderCategoryConfig; /** 独立 API 地址(MP indexer['api_url'],SunnyPT) */ api_url?: string; + /** M-Team 公开 API 主机覆盖，兼容上游的两种字段名。 */ + api_domain?: string; + api_host?: string; /** 本项目扩展(MP 无对应) */ umt: { version: string; @@ -93,6 +97,8 @@ export interface SpiderContext { cache: Map; userAgent?: string; signal?: AbortSignal; + /** 上游明确给出页数时，避免通过过滤后的资源数猜测是否还有下一页。 */ + reportPage?(page: { totalPages: number }): void; } export interface SpecializedSpider { diff --git a/packages/adapters/src/sites/spiders/yema.test.ts b/packages/adapters/src/sites/spiders/yema.test.ts index e2610935..619b0836 100644 --- a/packages/adapters/src/sites/spiders/yema.test.ts +++ b/packages/adapters/src/sites/spiders/yema.test.ts @@ -13,6 +13,11 @@ const def = testDef("Yema", { domain: "https://www.yemapt.org/" }); const URL_SEARCH = "POST /openApi/torrent/fetchOpenTorrentList.json"; describe("YemaSpider(MP yema.py)", () => { + it("显式分类优先于媒体类型，逐分类查询", async () => { + const mock = createMockFetch({ [URL_SEARCH]: { body: YEMA_SEARCH_TV_A } }); + await adapterWith(def, yemaSpider, mock.fetch).search("series", { mtype: "movie", categoryIds: ["5", "13"] }); + expect(mock.requests.map((request) => JSON.parse(request.body!).categoryId)).toEqual([5, 13]); + }); it("列表解析:促销枚举、标签表(未知 ID 丢弃)、H&R、非整数 ID 跳过", async () => { const m = createMockFetch({ [URL_SEARCH]: { body: YEMA_SEARCH_MOVIE } }); const page = await adapterWith(def, yemaSpider, m.fetch).search("past", { mtype: "movie" }); diff --git a/packages/adapters/src/sites/spiders/yema.ts b/packages/adapters/src/sites/spiders/yema.ts index db9471ea..0f751a07 100644 --- a/packages/adapters/src/sites/spiders/yema.ts +++ b/packages/adapters/src/sites/spiders/yema.ts @@ -1,7 +1,7 @@ /** * SPDX-License-Identifier: GPL-3.0-only * 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。 - * @mp-source app/modules/indexer/spider/yema.py @ e6820a8 + * @mp-source app/modules/indexer/spider/yema.py @ 3b156a5 * * YemaPT 开放 API(parser "Yema",Authorization: AuthKey)。函数对照 `YemaSpider`: * - `_size/_movie_category/_tv_category/_music_category/_labels` → `YEMA_*` @@ -76,7 +76,8 @@ export function yemaBuildParams( } /** MP `_search_category_ids`:分类参数不支持数组,逐个查询 */ -export function yemaSearchCategoryIds(mtype?: SpiderSearchQuery["mtype"]): Array { +export function yemaSearchCategoryIds(mtype?: SpiderSearchQuery["mtype"], cat?: string): Array { + if (cat) return cat.split(",").map((id) => Number(id.trim())); if (mtype === "movie") return [...YEMA_MOVIE_CATEGORY]; if (mtype === "tv") return [...YEMA_TV_CATEGORY]; if (mtype === "music") return [...YEMA_MUSIC_CATEGORY]; @@ -189,7 +190,7 @@ export const yemaSpider: SpecializedSpider = { const url = `${yemaSiteUrl(ctx.def)}/openApi/torrent/fetchOpenTorrentList.json`; const groups: SpecializedTorrent[][] = []; const errors: unknown[] = []; - for (const categoryId of yemaSearchCategoryIds(q.mtype)) { + for (const categoryId of yemaSearchCategoryIds(q.mtype, q.cat)) { try { const payload = await apiRequest(ctx, url, { method: "POST",