feat(sites): sync MoviePilot v3.2.4 spiders and M-Team subtitle APIs

This commit is contained in:
root committed 2026-11-06 00:06:12 +08:00
1 parent 3b9376c75a
commit c38e7131f1
26 files changed
+775 -56

No files matched your search

+12 -7
View File
@@ -1,7 +1,8 @@
/**
* @mp-source app/modules/indexer/spider/mtorrent.py @ 3b156a5
* 馒头(M-Team)API Key 适配器。
*
* 对应 MoviePilot e6820a8:
* 对应 MoviePilot v3.2.4:
* - `MTorrentSpider.search`(POST /api/torrent/search,x-api-key + UA + JSON)→ `list` / `search`
* - `MTorrentSpider.__process_response` → `mteamPost`(另加业务 code/message 检查,迁移账本 MT2)
* - `MTorrentSpider.__get_download_url` + `submission.py _resolve_indirect_download_url` + `download_torrent`
@@ -32,6 +33,7 @@ import { extractDoubanId, parseSiteTime } from "../util.ts";
import { mteamPost } from "./client.ts";
import { type MteamDefinition, mteamApiBase } from "./definition.ts";
import { type MteamTorrentJson, mteamSearchParams, parseMteamResult } from "./spider.ts";
import { createMteamSubtitleClient } from "./subtitles.ts";
interface MteamSearchData {
data?: MteamTorrentJson[] | null;
@@ -78,7 +80,9 @@ export function mteamTorrentToRelease(
}
export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps): SiteAdapter {
const def: MteamDefinition = deps.baseUrl ? { ...input, umt: { ...input.umt, apiBaseUrl: deps.baseUrl } } : input;
const def: MteamDefinition = deps.baseUrl
? { ...input, api_domain: "", api_host: "", umt: { ...input.umt, apiBaseUrl: deps.baseUrl } }
: input;
const http = buildHttpContext(def.id, { timeoutMs: def.timeout * 1000, ...deps }, def.umt.rateLimit);
const api = mteamApiBase(def);
@@ -95,9 +99,7 @@ export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps
keyword: string | null,
signal?: AbortSignal,
): Promise<SitePage> {
const params = mteamSearchParams(keyword, filters.mtype, Math.max(1, page) - 1);
// 本项目扩展:显式分类 ID 优先于 mtype 分类组
if (filters.categoryIds?.length) params.categories = filters.categoryIds;
const params = mteamSearchParams(keyword, filters.mtype, Math.max(1, page) - 1, filters.categoryIds?.join(","));
const data = await mteamPost<MteamSearchData>(
http,
api,
@@ -106,16 +108,18 @@ export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps
{ json: params },
signal,
);
if (!data || !Array.isArray(data.data))
throw new SiteError(def.id, "parse_error", "M-Team 搜索响应 data 不是资源列表");
const observedAt = http.now().toISOString();
const src = data?.data ?? [];
const rows = parseMteamResult(src, def.domain);
let items = rows.map((t, i) => mteamTorrentToRelease(t, src[i]!, def, observedAt));
if (filters.freeOnly) items = items.filter((r) => r.promotion.downloadFactor === 0);
const tp = Number(data?.totalPages);
const tp = data.totalPages == null ? NaN : Number(data.totalPages);
return {
items,
page: Math.max(1, page),
totalPages: Number.isFinite(tp) && tp >= 0 ? tp : null,
totalPages: Number.isInteger(tp) && tp >= 0 ? tp : null,
totalPagesEstimated: false,
fetchedAt: observedAt,
};
@@ -134,6 +138,7 @@ export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps
displayName: def.umt.displayName,
capabilities: def.umt.capabilities,
definitionVersion: def.umt.version,
...createMteamSubtitleClient({ http, apiBase: api, apiKey, webUrl: def.domain, timezone: def.umt.timezone }),
list(page, filters = {}, opts = {}) {
return doSearch(page, filters, null, opts.signal);
+8 -2
View File
@@ -1,4 +1,5 @@
/**
* @mp-source app/modules/indexer/spider/mtorrent.py @ 3b156a5
* M-Team API 调用。对应 MP `MTorrentSpider.search/__process_response`(MP 只看 HTTP 200);
* 业务 code/message 检查与凭据/解析错误分层参考 MovieClaw `_post_api`(behavior-ref,迁移账本 MT2)。
*/
@@ -11,7 +12,10 @@ export type MteamApiPath =
| "/api/torrent/detail"
| "/api/torrent/genDlToken"
| "/api/member/profile"
| "/api/tracker/myPeerStatus";
| "/api/tracker/myPeerStatus"
| "/api/subtitle/search"
| "/api/subtitle/list"
| "/api/subtitle/genlink";
const SUCCESS_MESSAGES = new Set(["SUCCESS", "OK", "0", ""]);
const AUTH_KEYWORDS = [
@@ -88,8 +92,10 @@ export async function mteamPost<T>(
{ httpStatus: res.status },
);
}
if (!json || typeof json !== "object" || Array.isArray(json))
throw new SiteError(http.siteId, "parse_error", `${path}:响应结构无效`);
const message = json.message == null ? "" : String(json.message);
const code = json.code == null ? "0" : String(json.code);
const code = json.code == null ? "" : String(json.code);
if (code !== "0" || !SUCCESS_MESSAGES.has(message.toUpperCase())) {
const lower = message.toLowerCase();
const kind = RATE_KEYWORDS.some((k) => lower.includes(k))
@@ -1,3 +1,6 @@
/**
* @mp-source app/modules/indexer/spider/mtorrent.py @ 3b156a5
*/
import mteamJson from "../definitions/mteam.json" with { type: "json" };
import { parseMteamDefinition } from "../definitions/schema.ts";
import type { DefinitionProvenance } from "../spider/schema.ts";
@@ -17,6 +20,8 @@ export interface MteamDefinition {
/** 网页域(MP `_url`,详情页 = `${domain}detail/${id}`) */
domain: string;
parser: "mTorrent";
api_domain?: string;
api_host?: string;
timeout: number;
umt: {
version: string;
@@ -47,6 +52,15 @@ export function rootDomain(url: string): string {
return parts.length >= 2 ? parts.slice(-2).join(".") : host;
}
export function mteamApiBase(def: MteamDefinition): string {
return (def.umt.apiBaseUrl ?? `https://api.${rootDomain(def.domain)}`).replace(/\/$/, "");
export function mteamApiBase(
def: Pick<MteamDefinition, "domain" | "api_domain" | "api_host"> & { umt: { apiBaseUrl?: string } },
): string {
const configured = (def.api_domain || def.api_host)?.trim();
if (configured) {
const host = new URL(configured.includes("://") ? configured : `https://${configured}`).host;
return `https://${host}`;
}
if (def.umt.apiBaseUrl) return def.umt.apiBaseUrl.replace(/\/+$/, "");
const domain = new URL(def.domain);
return domain.hostname.startsWith("api.") ? `https://${domain.host}` : `https://api.${rootDomain(def.domain)}`;
}
+10 -3
View File
@@ -1,7 +1,8 @@
/**
* @mp-source app/modules/indexer/spider/mtorrent.py @ 3b156a5
* M-Team API 解析。
*
* 迁移自 MoviePilot e6820a8 `app/modules/indexer/spider/mtorrent.py` `MTorrentSpider`:
* 迁移自 MoviePilot v3.2.4 `app/modules/indexer/spider/mtorrent.py` `MTorrentSpider`:
* - `_movie_category/_tv_category/_music_category/_labels` → 同名常量
* - `__get_params` → `mteamSearchParams`
* - `__parse_result` → `parseMteamResult`
@@ -60,9 +61,15 @@ export interface MteamTorrentJson {
}
/** MP `__get_params`(page 为 0 起) */
export function mteamSearchParams(keyword: string | null | undefined, mtype: MediaKind | undefined, page = 0) {
export function mteamSearchParams(
keyword: string | null | undefined,
mtype: MediaKind | undefined,
page = 0,
cat?: string,
) {
let categories: string[];
if (!mtype) categories = [];
if (cat) categories = cat.split(",").map((id) => id.trim());
else if (!mtype) categories = [];
else if (mtype === "tv") categories = TV_CATEGORY;
else if (mtype === "music") categories = MUSIC_CATEGORY;
else categories = MOVIE_CATEGORY;
@@ -0,0 +1,178 @@
import { createMockFetch } from "@umt/testkit";
import { describe, expect, it } from "vitest";
import { createSiteRateLimiter } from "../rate-limit.ts";
import { mtorrentSpider } from "../spiders/mtorrent.ts";
import { adapterWith, testDef } from "../spiders/testing.ts";
import { staticSecretResolver } from "../types.ts";
import { createMteamAdapter } from "./adapter.ts";
import { mteamApiBase, mteamDefinition } from "./definition.ts";
import { mteamSearchParams } from "./spider.ts";
function adapter(fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>) {
return createMteamAdapter(
{ ...mteamDefinition, api_host: "https://subtitles.example/api" },
{
fetch,
secrets: staticSecretResolver({ mteam: { apiKey: "SECRET-API" } }),
limiter: createSiteRateLimiter({ minIntervalMs: 0 }),
},
);
}
describe("M-Team v3.2.4 API", () => {
it("专用分派同样透传分类和分页元数据,按原始页数判断尾页", async () => {
const mock = createMockFetch({
"/api/torrent/search": {
body: {
code: 0,
data: {
pageNumber: 2,
totalPages: 4,
data: [{ id: "8001", name: "Demo S01E01", category: "402", size: "1000", status: {} }],
},
},
},
});
const adapter = adapterWith(testDef("mTorrent", { domain: "https://api.m-team.io/" }), mtorrentSpider, mock.fetch);
const page = await adapter.search("Demo", { mtype: "movie", categoryIds: ["402"], page: 2 });
expect(JSON.parse(mock.requests[0]!.body!)).toMatchObject({ categories: ["402"], pageNumber: 2 });
expect(page).toMatchObject({ page: 2, totalPages: 4, totalPagesEstimated: false });
expect(page.items).toHaveLength(1);
});
it("种子与字幕请求共用同一个限流器,下载两阶段也计入频率", async () => {
const mock = createMockFetch({
"/api/torrent/search": { body: { code: 0, data: { data: [], totalPages: 0 } } },
"/api/subtitle/search": { body: { code: 0, data: [] } },
"/api/subtitle/genlink": { body: { code: 0, data: "signed-once" } },
"/api/subtitle/dlV2": {
body: "1\n00:00:01,000 --> 00:00:02,000\nText",
headers: { "content-type": "application/octet-stream" },
},
});
let clock = 0;
const waits: number[] = [];
const client = createMteamAdapter(mteamDefinition, {
fetch: mock.fetch,
secrets: staticSecretResolver({ mteam: { apiKey: "SECRET-API" } }),
limiter: createSiteRateLimiter({
minIntervalMs: 250,
now: () => clock,
sleep: async (ms) => {
waits.push(ms);
clock += ms;
},
}),
});
await client.list(1);
await client.searchSubtitles!("Demo");
await client.downloadSubtitle!("2001");
expect(waits).toEqual([250, 250, 250]);
});
it("API主机覆盖兼容两种字段;已是api子域时保留主机和端口", () => {
expect(mteamApiBase({ ...mteamDefinition, api_domain: "https://other.example:9443/api" })).toBe(
"https://other.example:9443",
);
expect(mteamApiBase({ ...mteamDefinition, api_host: "other.example" })).toBe("https://other.example");
expect(mteamApiBase({ domain: "https://api.m-team.io:9443/", umt: {} })).toBe("https://api.m-team.io:9443");
expect(mteamSearchParams("tt1234", "tv", 2, "401, 419")).toMatchObject({
categories: ["401", "419"],
pageNumber: 3,
});
});
it("搜索字幕返回稳定元数据和页数,不生成下载凭据", async () => {
const mock = createMockFetch({
"/api/subtitle/search": {
body: {
code: 0,
data: {
totalPages: 3,
data: [
{
id: "sub-100",
name: "Arrival 简中",
filename: "arrival.zh.srt",
lang: "简中",
size: "12 KB",
hits: 7,
author: "Uploader",
torrent: { id: "8001" },
createdDate: "2026-11-03T01:02:03Z",
},
],
},
},
},
});
const result = await adapter(mock.fetch).searchSubtitles!("Arrival", { page: 2 });
expect(result).toMatchObject({
page: 2,
totalPages: 3,
items: [
{
id: "sub-100",
title: "Arrival 简中",
sizeBytes: 12 * 1024,
language: "简中",
uploadedAt: "2026-11-03T01:02:03.000Z",
torrentId: "8001",
detailUrl: "https://tp.m-team.cc/detail/8001",
},
],
});
expect(mock.requests).toHaveLength(1);
expect(mock.requests[0]!.headers["x-api-key"]).toBe("SECRET-API");
expect(mock.requests[0]!.url.origin).toBe("https://subtitles.example");
expect(JSON.parse(mock.requests[0]!.body!)).toEqual({ keyword: "Arrival", pageNumber: 2, pageSize: 20 });
expect(JSON.stringify(result)).not.toContain("SECRET");
});
it.each(["items", "records", "list"])("兼容%s分页包裹,空数组为成功空页", async (key) => {
const mock = createMockFetch({ "/api/subtitle/search": { body: { code: "0", data: { [key]: [] } } } });
expect((await adapter(mock.fetch).searchSubtitles!("Unknown")).items).toEqual([]);
});
it("按种子列字幕保留关联;下载时换取并正确编码credential,不带API Key下载", async () => {
let credentials = 0;
const mock = createMockFetch({
"/api/subtitle/list": { body: { code: 0, data: [{ id: "sub-200", fileName: "demo.srt", size: "30" }] } },
"/api/subtitle/genlink": () => ({ body: { code: 0, data: `token+/${++credentials}` } }),
"/api/subtitle/dlV2": {
body: "1\n00:00:01,000 --> 00:00:02,000\nHello",
headers: {
"content-disposition": "attachment; filename*=UTF-8''%E5%AD%97%E5%B9%95.srt",
"content-type": "application/octet-stream",
},
},
});
const client = adapter(mock.fetch);
expect((await client.listSubtitles!("8002"))[0]).toMatchObject({
id: "sub-200",
torrentId: "8002",
detailUrl: "https://tp.m-team.cc/detail/8002",
});
expect(mock.requests[0]!.body).toBe("id=8002");
const first = await client.downloadSubtitle!("sub-200");
await client.downloadSubtitle!("sub-200");
expect(credentials).toBe(2);
expect(first.fileName).toBe("字幕.srt");
expect(new TextDecoder().decode(first.bytes)).toContain("Hello");
const downloads = mock.requests.filter((request) => request.url.pathname === "/api/subtitle/dlV2");
expect(downloads[0]!.url.searchParams.get("credential")).toBe("token+/1");
expect(downloads[0]!.headers["x-api-key"]).toBeUndefined();
expect(downloads[0]!.headers.cookie).toBeUndefined();
});
it("API业务失败、错误JSON结构和伪装HTML下载不会冒充成功", async () => {
const invalid = createMockFetch({ "/api/subtitle/search": { body: { code: 1, message: "invalid API key" } } });
await expect(adapter(invalid.fetch).searchSubtitles!("x")).rejects.toMatchObject({ kind: "auth_invalid" });
const malformed = createMockFetch({ "/api/torrent/search": { body: { code: 0, data: {} } } });
await expect(adapter(malformed.fetch).list(1)).rejects.toMatchObject({ kind: "parse_error" });
const html = createMockFetch({
"/api/subtitle/genlink": { body: { code: 0, data: "one-use" } },
"/api/subtitle/dlV2": { body: "<html><title>Expired</title></html>", headers: { "content-type": "text/html" } },
});
await expect(adapter(html.fetch).downloadSubtitle!("sub-200")).rejects.toMatchObject({ kind: "parse_error" });
});
});
@@ -0,0 +1,154 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* @mp-source app/modules/indexer/spider/mtorrent.py @ 3b156a5
* 对照 v3.2.4 字幕 API;搜索仅返回稳定 ID,下载时现场换取 credential,不缓存签名链接。
*/
import { readBody, type SiteHttpContext, siteFetch } from "../http.ts";
import { parseSizeMp } from "../spider/spider.ts";
import { isoWithZoneToUtc, isPlainObject } from "../spiders/common.ts";
import { type SiteAdapter, SiteError, type SiteSubtitle } from "../types.ts";
import { parseSiteTime } from "../util.ts";
import { mteamPost } from "./client.ts";
function string(value: unknown): string | null {
return value == null || value === "" ? null : String(value);
}
function rowsOf(data: unknown): unknown[] {
if (Array.isArray(data)) return data;
if (isPlainObject(data)) {
for (const key of ["data", "items", "records", "list"]) if (Array.isArray(data[key])) return data[key];
}
return [];
}
export function parseMteamSubtitles(data: unknown, webUrl: string, timezone: string): SiteSubtitle[] {
return rowsOf(data).flatMap((row): SiteSubtitle[] => {
if (!isPlainObject(row) || row.id == null) return [];
const id = String(row.id);
const torrentId = string(isPlainObject(row.torrent) ? row.torrent.id : row.torrent);
const created = row.createdDate ?? row.lastModifiedDate;
let uploadedAt: string | null = null;
if (created != null && /^\d+(?:\.\d+)?$/.test(String(created))) {
const date = new Date(Number(created) * 1000);
if (Number.isFinite(date.getTime())) uploadedAt = date.toISOString();
} else if (typeof created === "string") {
uploadedAt = isoWithZoneToUtc(created) ?? parseSiteTime(created, timezone);
}
const downloads = row.hits == null ? NaN : Number(row.hits);
return [
{
id,
title: string(row.name) ?? string(row.filename) ?? string(row.fileName) ?? id,
fileName: string(row.filename ?? row.fileName),
language: string(row.lang ?? row.language),
sizeBytes: row.size == null ? null : parseSizeMp(String(row.size)),
uploadedAt,
downloads: Number.isFinite(downloads) ? downloads : null,
uploader: string(row.author),
torrentId,
detailUrl: torrentId
? new URL(`detail/${encodeURIComponent(torrentId)}`, `${webUrl.replace(/\/+$/, "")}/`).href
: null,
},
];
});
}
function fileName(headers: Headers): string | null {
const header = headers.get("content-disposition") ?? "";
const encoded = /filename\*=UTF-8''([^;]+)/i.exec(header)?.[1];
if (encoded) {
try {
return decodeURIComponent(encoded);
} catch {
/* 回退普通文件名。 */
}
}
return /filename="([^"]+)"/i.exec(header)?.[1] ?? /filename=([^;\s]+)/i.exec(header)?.[1] ?? null;
}
export function createMteamSubtitleClient(options: {
http: SiteHttpContext;
apiBase: string;
apiKey(): Promise<string>;
webUrl: string;
timezone: string;
}): Pick<SiteAdapter, "searchSubtitles" | "listSubtitles" | "downloadSubtitle"> {
const { http, apiBase, webUrl, timezone } = options;
return {
async searchSubtitles(query, opts = {}) {
const page = Math.max(1, opts.page ?? 1);
const data = await mteamPost<unknown>(
http,
apiBase,
await options.apiKey(),
"/api/subtitle/search",
{
json: { keyword: query, pageNumber: page, pageSize: 20 },
},
opts.signal,
);
const totalPages = isPlainObject(data) && data.totalPages != null ? Number(data.totalPages) : NaN;
return {
items: parseMteamSubtitles(data, webUrl, timezone),
page,
totalPages: Number.isInteger(totalPages) && totalPages >= 0 ? totalPages : null,
};
},
async listSubtitles(torrentId, opts = {}) {
const data = await mteamPost<unknown>(
http,
apiBase,
await options.apiKey(),
"/api/subtitle/list",
{
form: { id: torrentId },
},
opts.signal,
);
return parseMteamSubtitles(data, webUrl, timezone).map((row) => ({
...row,
torrentId: row.torrentId ?? torrentId,
detailUrl:
row.detailUrl ?? new URL(`detail/${encodeURIComponent(torrentId)}`, `${webUrl.replace(/\/+$/, "")}/`).href,
}));
},
async downloadSubtitle(subtitleId, opts = {}) {
const credential = await mteamPost<unknown>(
http,
apiBase,
await options.apiKey(),
"/api/subtitle/genlink",
{
form: { id: subtitleId },
},
opts.signal,
);
if (typeof credential !== "string" || !credential)
throw new SiteError(http.siteId, "parse_error", "M-Team 字幕接口未返回下载凭据");
const url = new URL(`${apiBase.replace(/\/+$/, "")}/api/subtitle/dlV2`);
url.searchParams.set("credential", credential);
const response = await siteFetch(http, url.href, {
redirect: "manual",
...(opts.signal ? { signal: opts.signal } : {}),
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new SiteError(
http.siteId,
response.status === 401 || response.status === 403 ? "auth_invalid" : "network",
`M-Team 字幕下载:HTTP ${response.status}`,
{ httpStatus: response.status },
);
}
const bytes = await readBody(http, response, 30 * 1024 * 1024);
if (
/text\/html/i.test(response.headers.get("content-type") ?? "") ||
/^\s*(?:<!doctype\s+html|<html\b)/i.test(new TextDecoder().decode(bytes.subarray(0, 256)))
)
throw new SiteError(http.siteId, "parse_error", "M-Team 字幕下载返回网页,重新搜索后重试");
return { bytes, fileName: fileName(response.headers) };
},
};
}
+14 -6
View File
@@ -1,7 +1,8 @@
/**
* @mp-source app/modules/indexer/__init__.py @ 3b156a5
* 通用 spider 站点适配器(NexusPHP 等 HTML 站)。
*
* 对应 MoviePilot e6820a8:
* 对应 MoviePilot v3.2.4:
* - `app/modules/indexer/__init__.py IndexerModule.search_torrents / refresh_torrents` → `search` / `list`
* (`__search_check` 的英文站不搜中文 → search 前检查;`__clear_search_text` → `clearSearchText`;
* `SitesHelper.check(domain)` 流控 → 站点令牌桶)
@@ -134,6 +135,7 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
async function cookie(): Promise<string> {
const c = await deps.secrets.resolve({ siteId: def.id, kind: "cookie" });
if (!c && def.public) return "";
if (!c) throw new SiteError(def.id, "auth_missing", "未配置站点 Cookie");
return c;
}
@@ -153,9 +155,10 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
/** 页面认不出来时的错误:像人机验证就报 challenge,否则 parse_error;都附页面样本 */
function unrecognized(page: FetchedPage, what: string, kind: SiteRequestKind, parseMessage: string): SiteError {
const rule = challengeRuleOfUnrecognizedPage(page.html);
const title = redactText(loadHtml(page.html)("title").first().text().replace(/\s+/g, " ").trim()).slice(0, 120);
const message = rule
? `${what}:${PAGE_LABEL[kind]}需要人机验证(Cloudflare 等),${challengeOutcomeText("unhandled")}`
: `${parseMessage}(页面样本见站点页技术信息)`;
: `${parseMessage}(HTTP ${page.status}${title ? `,页面标题:${title}` : ""};页面样本见站点页技术信息)`;
const sample = buildFailureSample({
kind: rule ? "challenge" : "parse_error",
requestKind: kind,
@@ -186,7 +189,11 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
try {
res = await siteFetch(http, url, {
method: "GET",
headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base },
headers: {
cookie: await cookie(),
accept: "text/html,application/xhtml+xml,application/rss+xml,application/xml",
referer: base,
},
redirect: "manual",
...(opts.signal ? { signal: opts.signal } : {}),
});
@@ -260,7 +267,8 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
if (parsed.torrents.length === 0) {
const text = $("body").text();
const hasList =
$(normalizeSelector(spider.def.torrents.list.selector)).length > 0 || $("table.torrents").length > 0;
$(normalizeSelector(spider.def.torrents.list.selector)).length > 0 ||
$("table.torrents, rss > channel, feed").length > 0;
const empty = (ext.emptyMarkers ?? []).some((m) => text.includes(m));
// 认不出列表:先看是不是人机验证页(搜索页单独加盾、间歇触发),不是才报改版
if (!hasList && !empty) throw unrecognized(page$, what, kind, `${what}:页面结构无法识别(可能改版)`);
@@ -289,8 +297,8 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
return {
items,
page: Math.max(1, page),
totalPages: maxRaw === null ? null : Math.max(page, (maxRaw as number) - start + 1),
totalPagesEstimated: true,
totalPages: parsed.totalPages ?? (maxRaw === null ? null : Math.max(page, (maxRaw as number) - start + 1)),
totalPagesEstimated: parsed.totalPages === undefined,
fetchedAt: observedAt,
};
}
@@ -0,0 +1,90 @@
import { createMockFetch } from "@umt/testkit";
import { describe, expect, it } from "vitest";
import { createSiteRateLimiter } from "../rate-limit.ts";
import { staticSecretResolver } from "../types.ts";
import { createSpiderSiteAdapter } from "./adapter.ts";
import type { IndexerDefinition } from "./schema.ts";
import { SiteSpider } from "./spider.ts";
const def: IndexerDefinition = {
id: "public-index",
name: "公开索引",
domain: "https://index.example/",
public: true,
result_num: 2,
search: { paths: [{ path: "search?q={keyword}" }] },
torrents: {
list: { selector: "item" },
fields: {
title: { selector: "title" },
download: { selector: "enclosure", attribute: "url" },
size: { selector: "enclosure", attribute: "length" },
},
},
umt: {
version: "test",
kind: "nexusphp",
timezone: "+08:00",
downloadPath: "download.php",
detailPath: "details.php",
rateLimit: { minIntervalMs: 0, burst: 10 },
capabilities: { list: true, search: true, detail: false, download: true, rss: false },
},
};
const xml = `\ufeff <?xml version='1.0' encoding='GBK'?><rss><channel>${[1, 2, 3, 4, 5].map((id) => `<item><title>Title ${id} &amp; special</title><enclosure url="https://index.example/download.php?id=${id}" length="2000" /></item>`).join("")}</channel></rss>`;
describe("通用索引XML与本地分页", () => {
it("已解码XML按HTML大小写规则提取字段;无页码搜索切出互不重叠的窗口", () => {
const first = new SiteSpider(def, { keyword: "special", page: 0 }).parse(xml);
const second = new SiteSpider(def, { keyword: "special", page: 1 }).parse(xml);
const last = new SiteSpider(def, { keyword: "special", page: 2 }).parse(xml);
const empty = new SiteSpider(def, { keyword: "special", page: 3 }).parse(xml);
expect(first.torrents.map((row) => row.title)).toEqual(["Title 1 & special", "Title 2 & special"]);
expect(second.torrents.map((row) => row.title)).toEqual(["Title 3 & special", "Title 4 & special"]);
expect(last.torrents).toHaveLength(1);
expect(last.torrents[0]).toMatchObject({ title: "Title 5 & special", size: 2000 });
expect(empty).toMatchObject({ torrents: [], isError: false, totalPages: 3 });
});
it.each([
{ paths: [{ path: "search?q={keyword}&page={page}" }] },
{ paths: [{ path: "search" }], params: { q: "{keyword}" } },
])("服务端已收到page时不重复切片 %#", (search) => {
const spider = new SiteSpider({ ...def, search }, { keyword: "special", page: 2 });
expect(new URL(spider.getSearchUrl()).searchParams.get("page")).toBe("2");
expect(spider.parse(xml).torrents[0]!.title).toBe("Title 1 & special");
});
it("纯媒体类型站的缺失分类兜底,不覆盖已知分类", () => {
const result = new SiteSpider({ ...def, media_type: "music" }).parse(xml);
expect(result.torrents.every((row) => row.category === "音乐")).toBe(true);
});
it("公开RSS不要求Cookie,空channel和分页结束均为成功空页", async () => {
let body = xml;
const mock = createMockFetch({ "/search": () => ({ body, headers: { "content-type": "application/rss+xml" } }) });
const adapter = createSpiderSiteAdapter(def, {
fetch: mock.fetch,
secrets: staticSecretResolver({}),
limiter: createSiteRateLimiter({ minIntervalMs: 0 }),
});
const page = await adapter.search("special", { page: 2 });
expect(page).toMatchObject({ page: 2, totalPages: 3, totalPagesEstimated: false });
expect(page.items.map((row) => row.siteReleaseId)).toEqual(["3", "4"]);
expect((await adapter.search("special", { page: 4 })).items).toEqual([]);
body = "<?xml version='1.0'?><rss><channel /></rss>";
expect((await adapter.search("special")).items).toEqual([]);
});
it("无法识别的200页面错误包含安全标题", async () => {
const mock = createMockFetch({
"/search": { body: "<html><head><title>Search temporarily unavailable</title></head><body>Later</body></html>" },
});
const adapter = createSpiderSiteAdapter(def, {
fetch: mock.fetch,
secrets: staticSecretResolver({}),
limiter: createSiteRateLimiter({ minIntervalMs: 0 }),
});
await expect(adapter.search("special")).rejects.toThrow("HTTP 200,页面标题:Search temporarily unavailable");
});
});
+28 -6
View File
@@ -1,7 +1,8 @@
/**
* 通用站点爬虫(配置驱动)。
*
* 迁移自 MoviePilot e6820a8 `app/modules/indexer/spider/__init__.py`:
* 迁移自 MoviePilot v3.2.4 `app/modules/indexer/spider/__init__.py`:
* @mp-source app/modules/indexer/spider/__init__.py @ 3b156a5
* - `select_media_categories` / `resolve_category_media_type` → 同名函数
* - `SiteSpider.__get_search_url` / `__format_search_word` → `SiteSpider.getSearchUrl` / `formatSearchWord`
* - `SiteSpider.get_info` 及 `__get_title/__get_description/__get_detail/__get_download/__get_grabs/__get_leechers/
@@ -46,7 +47,8 @@ const MEDIA_VALUE: Record<MediaKind, MpMediaType> = { movie: "电影", tv: "电
* 使 `table.torrents > tr` 这类 MP 风格选择器保持原语义。
*/
export function loadHtml(html: string): CheerioAPI {
return load(html, { xml: { xmlMode: false, decodeEntities: true } });
const decoded = html.replace(/^[\s\ufeff]*<\?xml\b[^>]*\?>/i, "");
return load(decoded, { xml: { xmlMode: false, decodeEntities: true, recognizeSelfClosing: true } });
}
/**
@@ -245,6 +247,8 @@ export class SiteSpider {
private browseCfg: IndexerDefinition["browse"];
private categoryCfg: SpiderCategoryConfig | undefined;
private requestedResultMediaType: MediaKind | undefined;
private resultOffset = 0;
private paginateLocally = false;
private $!: CheerioAPI;
private info: SpiderTorrent = {};
@@ -300,6 +304,8 @@ export class SiteSpider {
/** MP `__get_search_url` */
getSearchUrl(): string {
this.resultOffset = 0;
this.paginateLocally = false;
const search = this.searchCfg ?? { paths: [] };
const paths = search.paths ?? [];
let torrentspath = "";
@@ -379,6 +385,8 @@ export class SiteSpider {
}
searchurl = combineUrl(this.domain, torrentspath, params);
} else {
this.paginateLocally = !torrentspath.includes("{page}");
this.resultOffset = this.paginateLocally ? Math.max(0, this.page) * this.resultNum : 0;
searchurl =
this.domain +
torrentspath
@@ -438,20 +446,33 @@ export class SiteSpider {
}
/** MP `parse`:登录/权限页返回 isError=true 与空列表。 */
parse(html: string): { isError: boolean; loginPage: boolean; torrents: SpiderTorrent[]; $: CheerioAPI | null } {
parse(html: string): {
isError: boolean;
loginPage: boolean;
torrents: SpiderTorrent[];
$: CheerioAPI | null;
totalPages?: number;
} {
if (!html) return { isError: true, loginPage: false, torrents: [], $: null };
// 单独调用解析器时也与请求入口使用相同的分页窗口和媒体类型投影。
this.getSearchUrl();
const $ = loadHtml(html);
if (SiteSpider.isLoginOrPermissionPage($)) return { isError: true, loginPage: true, torrents: [], $ };
this.$ = $;
const out: SpiderTorrent[] = [];
const rows = $(normalizeSelector(this.list.selector ?? ""));
rows.each((i, el) => {
if (i >= this.resultNum) return false;
rows.slice(this.resultOffset, this.resultOffset + this.resultNum).each((_, el) => {
const info = this.searchType === "subtitles" ? this.getSubtitleInfo($(el)) : this.getInfo($(el));
if (Object.keys(info).length > 0) out.push(this.applyRequestedMediaType(info));
return undefined;
});
return { isError: false, loginPage: false, torrents: out, $ };
return {
isError: false,
loginPage: false,
torrents: out,
$,
...(this.paginateLocally ? { totalPages: Math.ceil(rows.length / this.resultNum) } : {}),
};
}
/** 对任意片段(如详情页头部)按字段配置取值 */
@@ -462,6 +483,7 @@ export class SiteSpider {
private applyRequestedMediaType(t: SpiderTorrent): SpiderTorrent {
if (this.requestedResultMediaType) t.category = MEDIA_VALUE[this.requestedResultMediaType];
else if (this.siteMediaType && (!t.category || t.category === "未知")) t.category = MEDIA_VALUE[this.siteMediaType];
return t;
}
+11 -3
View File
@@ -1,7 +1,7 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/__init__.py @ e6820a8
* @mp-source app/modules/indexer/__init__.py @ 3b156a5
*
* 专用 spider 站点适配器:对应 MP `IndexerModule.search_torrents/refresh_torrents` 按 `indexer.parser`
* 分派到 `SPIDER_PARSER_CLASSES` 并以 `_SPECIALIZED_SEARCH_ARGUMENTS` 传参;
@@ -33,6 +33,9 @@ export function createSpecializedSiteAdapter(
const def: SpecializedSiteDefinition = deps.baseUrl
? {
...input,
api_url: "",
api_domain: "",
api_host: "",
domain: deps.baseUrl.endsWith("/") ? deps.baseUrl : `${deps.baseUrl}/`,
umt: { ...input.umt, apiBaseUrl: deps.baseUrl.replace(/\/+$/, "") },
}
@@ -78,7 +81,12 @@ export function createSpecializedSiteAdapter(
const q: Parameters<SpecializedSpider["search"]>[1] = { keyword, page: p - 1 };
if (filters.mtype) q.mtype = filters.mtype;
if (filters.categoryIds?.length) q.cat = filters.categoryIds.join(",");
const rows = await spider.search(ctx(signal), q);
let totalPages: number | null = null;
const context = ctx(signal);
context.reportPage = (info) => {
totalPages = info.totalPages;
};
const rows = await spider.search(context, q);
let items = rows.flatMap((t) => {
const r = specializedTorrentToRelease(t, def, fetchedAt);
if (r && t.enclosure) enclosures.set(r.siteReleaseId, t.enclosure);
@@ -89,7 +97,7 @@ export function createSpecializedSiteAdapter(
items = items.filter((r) => r.category.id === null || allow.has(r.category.id));
}
if (filters.freeOnly) items = items.filter((r) => r.promotion.downloadFactor === 0);
return { items, page: p, totalPages: null, totalPagesEstimated: false, fetchedAt } satisfies SitePage;
return { items, page: p, totalPages, totalPagesEstimated: false, fetchedAt } satisfies SitePage;
}
return {
@@ -1,7 +1,7 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/spider/haidan.py @ e6820a8
* @mp-source app/modules/indexer/spider/haidan.py @ 3b156a5
*
* 海胆 haidan.video(parser "Haidan",Cookie + `torrents.php?isapi=1` JSON)。函数对照 `HaiDanSpider`:
* - `_movie_category/_tv_category/_music_category/_dl_state/_up_state` → 同名常量
@@ -7,7 +7,7 @@ const def = testDef("HDDolby", { domain: "https://www.hddolby.com/" });
describe("HddolbySpider(MP hddolby.py)", () => {
it("__get_params 与 API 地址:page_number 为 0 起、无类型时 movie+tv 去重升序", () => {
expect(hddolbyApiBase(def)).toBe("https://api.hddolby.com");
expect(hddolbyApiBase(def)).toBe("https://api.hddolby.com/api/v1");
expect(hddolbySearchParams("Lost")).toEqual({
keyword: "Lost",
page_number: 0,
@@ -16,6 +16,16 @@ describe("HddolbySpider(MP hddolby.py)", () => {
visible: 1,
});
expect(hddolbySearchParams(null, "music", 2).categories).toEqual([406, 408]);
expect(hddolbySearchParams("movie", "tv", 0, "401, 405").categories).toEqual([401, 405]);
});
it("显式API版本路径与分类透传,网页域不决定API路径", async () => {
const configured = { ...def, api_url: "https://gateway.example/open/v2/" };
const mock = createMockFetch({
"POST https://gateway.example/open/v2/torrent/search": { body: HDDOLBY_SEARCH_OK },
});
await adapterWith(configured, hddolbySpider, mock.fetch).search("Lost", { mtype: "movie", categoryIds: ["402"] });
expect(JSON.parse(mock.requests[0]!.body!).categories).toEqual([402]);
});
it("列表解析:促销类型表、标签表、分类优先级(405 归电视剧)、downhash 脱敏", async () => {
+34 -9
View File
@@ -1,7 +1,7 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/spider/hddolby.py @ e6820a8
* @mp-source app/modules/indexer/spider/hddolby.py @ 3b156a5
*
* HDDolby(parser "HDDolby",`https://api.<域>/api/v1/torrent/search`,x-api-key + Cookie)。函数对照 `HddolbySpider`:
* - `_movie_category/_tv_category/_music_category/_labels` → 同名常量
@@ -13,6 +13,7 @@
* 差异:API 返回的 `hr` 字段 MP 未映射,这里同样不映射(H&R = unknown,待核验语义)。
*/
import type { MpMediaType } from "../spider/spider.ts";
import { SiteError } from "../types.ts";
import { extractDomain } from "../user-parsers/common.ts";
import { apiRequest, businessError, checkReleaseId, downloadTorrentFile, isPlainObject, siteBase } from "./common.ts";
import type { SpecializedSiteDefinition, SpecializedSpider, SpecializedTorrent, SpiderSearchQuery } from "./types.ts";
@@ -60,13 +61,21 @@ export function hddolbyUploadVolumeFactor(discount: unknown): number {
}
export function hddolbyApiBase(def: SpecializedSiteDefinition): string {
return (def.umt.apiBaseUrl ?? `https://api.${extractDomain(def.domain)}`).replace(/\/+$/, "");
if (def.api_url) return def.api_url.replace(/\/+$/, "");
const base = (def.umt.apiBaseUrl ?? `https://api.${extractDomain(def.domain)}`).replace(/\/+$/, "");
return new URL(base).pathname === "/" ? `${base}/api/v1` : base;
}
/** MP `__get_params` */
export function hddolbySearchParams(keyword: string | null | undefined, mtype?: SpiderSearchQuery["mtype"], page = 0) {
export function hddolbySearchParams(
keyword: string | null | undefined,
mtype?: SpiderSearchQuery["mtype"],
page = 0,
cat?: string,
) {
let categories: number[];
if (mtype === "tv") categories = HDDOLBY_TV_CATEGORY;
if (cat) categories = cat.split(",").map((id) => Number(id.trim()));
else if (mtype === "tv") categories = HDDOLBY_TV_CATEGORY;
else if (mtype === "movie") categories = HDDOLBY_MOVIE_CATEGORY;
else if (mtype === "music") categories = HDDOLBY_MUSIC_CATEGORY;
// list(set(movie + tv)):CPython 小整数集合的迭代顺序即升序
@@ -149,11 +158,27 @@ export const hddolbySpider: SpecializedSpider = {
Referer: siteBase(ctx.def),
};
if (cookie) headers.cookie = cookie;
const data = await apiRequest(ctx, `${hddolbyApiBase(ctx.def)}/api/v1/torrent/search`, {
method: "POST",
headers,
json: hddolbySearchParams(q.keyword, q.mtype, q.page ?? 0),
});
const api = hddolbyApiBase(ctx.def);
let data: unknown;
try {
data = await apiRequest(ctx, `${api}/torrent/search`, {
method: "POST",
headers,
json: hddolbySearchParams(q.keyword, q.mtype, q.page ?? 0, q.cat),
});
} catch (error) {
if (error instanceof SiteError && error.kind === "network") {
throw new SiteError(
ctx.def.id,
"network",
`HDDolby API 主机 ${new URL(api).hostname} 无法连接,检查 API 地址及网络代理`,
{
...(error.httpStatus === undefined ? {} : { httpStatus: error.httpStatus }),
},
);
}
throw error;
}
if (!isPlainObject(data)) throw businessError(ctx, "搜索", "响应结构无效");
if (data.error) {
const err = data.error;
+4 -1
View File
@@ -1,7 +1,7 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/__init__.py @ e6820a8
* @mp-source app/modules/indexer/__init__.py @ 3b156a5
*
* 专用 spider 汇总:`SPECIALIZED_SPIDERS` = MP `SPIDER_PARSER_CLASSES`。
*/
@@ -10,6 +10,7 @@ import type { SiteAdapter } from "../types.ts";
import { createSpecializedSiteAdapter } from "./adapter.ts";
import { haidanSpider } from "./haidan.ts";
import { hddolbySpider } from "./hddolby.ts";
import { milkieSpider } from "./milkie.ts";
import { mtorrentSpider } from "./mtorrent.ts";
import { anibtSpider, animeGardenSpider, bangumiMoeSpider, mikanSpider, nyaaSpider } from "./public.ts";
import { rousiSpider } from "./rousi.ts";
@@ -23,6 +24,7 @@ export const SPECIALIZED_SPIDERS: Record<SpecializedParser, SpecializedSpider> =
TNodeSpider: tnodeSpider,
TorrentLeech: torrentleechSpider,
mTorrent: mtorrentSpider,
Milkie: milkieSpider,
Yema: yemaSpider,
Haidan: haidanSpider,
HDDolby: hddolbySpider,
@@ -60,6 +62,7 @@ export {
export * from "./haidan.ts";
export * from "./hddolby.ts";
export { releaseIdFromPageUrl, specializedTorrentToRelease } from "./map.ts";
export * from "./milkie.ts";
export * from "./mtorrent.ts";
export * from "./rousi.ts";
export * from "./sunnypt.ts";
+1 -1
View File
@@ -1,7 +1,7 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/__init__.py @ e6820a8
* @mp-source app/modules/indexer/__init__.py @ 3b156a5
*
* 专用 spider 输出(MP TorrentInfo 字段)→ 统一 `SiteRelease`。
* 促销类型由 MP 系数组合推出(复用 spider/map.ts `promotionFromFactors`);H&R 未提供时为 unknown;
@@ -0,0 +1,67 @@
import { createMockFetch } from "@umt/testkit";
import { describe, expect, it } from "vitest";
import { milkieSearchParams, milkieSpider } from "./milkie.ts";
import { adapterWith, expectSiteError, testDef, withTorrent } from "./testing.ts";
const def = testDef("Milkie", { domain: "https://milkie.example/" });
const id = "abc123-48ef-9001";
describe("Milkie API", () => {
it("浏览与关键词搜索使用不同分类策略、API页码0起", () => {
expect(milkieSearchParams({ mtype: "tv", page: 2 })).toEqual({ pi: 2, ps: 100, oby: "created_at", odir: "desc" });
expect(milkieSearchParams({ keyword: "Arrival", mtype: "movie" })).toMatchObject({
query: "Arrival",
categories: "1",
pi: 0,
});
});
it("保留UUID资源ID、分类和原时区;搜索结果不携带API Key或签名链接", async () => {
const mock = createMockFetch({
"/api/v1/torrents": {
body: {
torrents: [
{
id,
releaseName: "Arrival.2016.1080p",
category: 1,
createdAt: "2026-11-03T09:30:00+02:00",
size: 5000,
seeders: 12,
leechers: 1,
downloaded: 80,
externals: { imdb: "tt2543164" },
},
],
},
},
});
const page = await adapterWith(def, milkieSpider, mock.fetch).search("Arrival", { mtype: "movie", page: 3 });
expect(mock.requests[0]!.headers["x-milkie-auth"]).toBe("APIKEYSECRET");
expect(mock.requests[0]!.url.searchParams.get("pi")).toBe("2");
expect(page.items[0]).toMatchObject({
siteReleaseId: id,
title: "Arrival.2016.1080p",
category: { id: "1" },
uploadedAt: "2026-11-03T07:30:00.000Z",
promotion: { downloadFactor: 0 },
imdbId: "tt2543164",
});
expect(JSON.stringify(page)).not.toContain("APIKEYSECRET");
expect(page.items[0]!.raw.enclosure).toBeUndefined();
});
it("下载时编码当前API Key;缺凭据、无效凭据与响应结构各自报错", async () => {
const mock = createMockFetch({ "/api/v1/torrents": { body: {} } });
const torrent = withTorrent(mock.fetch, (url) => url.pathname.endsWith("/torrent"));
const adapter = adapterWith(def, milkieSpider, torrent.fetch, { apiKey: "a+b/&secret" });
await adapter.downloadTarget(id);
expect(torrent.hits[0]!.url.searchParams.get("key")).toBe("a+b/&secret");
expect(torrent.hits[0]!.url.pathname).toBe(`/api/v1/torrents/${id}/torrent`);
expect(torrent.hits[0]!.headers.get("x-milkie-auth")).toBeNull();
await expectSiteError(adapter.list(1), "parse_error");
await expectSiteError(adapterWith(def, milkieSpider, mock.fetch, {}).list(1), "auth_missing");
const denied = createMockFetch({ "/api/v1/torrents": { status: 401 } });
await expectSiteError(adapterWith(def, milkieSpider, denied.fetch).list(1), "auth_invalid");
});
});
@@ -0,0 +1,80 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* @mp-source app/modules/indexer/spider/milkie.py @ 3b156a5
* 对照 MoviePilot v3.2.4 的公开 Milkie API 行为实现。搜索结果不保存含 API Key 的下载地址。
*/
import { SiteError } from "../types.ts";
import { apiRequest, businessError, downloadTorrentFile, isoWithZoneToUtc, isPlainObject, siteBase } from "./common.ts";
import type { SpecializedSiteDefinition, SpecializedSpider, SpecializedTorrent, SpiderSearchQuery } from "./types.ts";
export function milkieSearchParams(query: SpiderSearchQuery): Record<string, string | number> {
const params: Record<string, string | number> = { pi: query.page ?? 0, ps: 100, oby: "created_at", odir: "desc" };
if (query.keyword) {
params.query = query.keyword;
const category = query.mtype ? { movie: "1", tv: "2", music: "3" }[query.mtype] : undefined;
if (category) params.categories = category;
}
return params;
}
/** Milkie 资源 ID 可能为 UUID;不使用其它站点的数字 ID 提取规则。 */
function milkieId(siteId: string, value: unknown): string {
const id = String(value ?? "");
if (!/^[a-zA-Z0-9_-]+$/.test(id)) throw new SiteError(siteId, "not_found", "Milkie 资源 ID 非法");
return id;
}
export function parseMilkieResult(rows: unknown[], def: SpecializedSiteDefinition): SpecializedTorrent[] {
return rows.flatMap((row): SpecializedTorrent[] => {
if (!isPlainObject(row) || row.id == null || !row.releaseName) return [];
const id = milkieId(def.id, row.id);
const category = { "1": "电影", "2": "电视剧", "3": "音乐" } as const;
const result: SpecializedTorrent = {
title: String(row.releaseName),
description: id,
site_release_id: id,
page_url: `${siteBase(def)}browse/${encodeURIComponent(id)}`,
category: category[String(row.category) as keyof typeof category] ?? "未知",
downloadvolumefactor: 0,
uploadvolumefactor: 1,
};
if (row.category != null) result.category_id = String(row.category);
for (const [source, target] of [
["size", "size"],
["seeders", "seeders"],
["leechers", "peers"],
["downloaded", "grabs"],
] as const) {
const value = row[source] == null ? NaN : Number(row[source]);
if (Number.isFinite(value)) result[target] = value;
}
const utc = isoWithZoneToUtc(row.createdAt);
if (utc) result.pubdate_utc = utc;
else if (typeof row.createdAt === "string") result.pubdate = row.createdAt;
if (isPlainObject(row.externals) && typeof row.externals.imdb === "string") result.imdbid = row.externals.imdb;
return [result];
});
}
export const milkieSpider: SpecializedSpider = {
parser: "Milkie",
credential: "apiKey",
pageSize: () => 100,
async search(ctx, query) {
const key = (await ctx.secret("apiKey", true))!;
const payload = await apiRequest(ctx, `${siteBase(ctx.def)}api/v1/torrents`, {
headers: { "x-milkie-auth": key },
query: milkieSearchParams(query),
});
if (!isPlainObject(payload) || !Array.isArray(payload.torrents))
throw businessError(ctx, "搜索", "Milkie 响应不含 torrents 列表");
return parseMilkieResult(payload.torrents, ctx.def);
},
async resolveDownload(ctx, siteReleaseId) {
const key = (await ctx.secret("apiKey", true))!;
const id = milkieId(ctx.def.id, siteReleaseId);
const url = new URL(`api/v1/torrents/${encodeURIComponent(id)}/torrent`, siteBase(ctx.def));
url.searchParams.set("key", key);
return downloadTorrentFile(ctx, url.href);
},
};
@@ -1,21 +1,21 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/spider/mtorrent.py @ e6820a8
* @mp-source app/modules/indexer/spider/mtorrent.py @ 3b156a5
*
* `mTorrent` 键的专用 spider 形态,复用 W1-3b 已移植的 mteam/spider.ts(`__get_params`/`__parse_result`)
* 与 mteam/client.ts(`__process_response` + 业务码检查),供 `SPECIALIZED_SPIDERS` 统一分派。
* 完整适配(详情、商城免费、healthCheck=profile)仍以 `createMteamAdapter` 为准。
*/
import { mteamPost } from "../mteam/client.ts";
import { rootDomain } from "../mteam/definition.ts";
import { mteamApiBase } from "../mteam/definition.ts";
import { type MteamTorrentJson, mteamSearchParams, parseMteamResult } from "../mteam/spider.ts";
import { SiteError } from "../types.ts";
import { checkReleaseId, downloadTorrentFile, siteBase } from "./common.ts";
import type { SpecializedSiteDefinition, SpecializedSpider, SpecializedTorrent } from "./types.ts";
export function mtorrentApiBase(def: SpecializedSiteDefinition): string {
return (def.umt.apiBaseUrl ?? `https://api.${rootDomain(def.domain)}`).replace(/\/+$/, "");
return mteamApiBase(def);
}
export const mtorrentSpider: SpecializedSpider = {
@@ -24,8 +24,8 @@ export const mtorrentSpider: SpecializedSpider = {
pageSize: () => 100,
async search(ctx, q) {
const apikey = (await ctx.secret("apiKey", true))!;
const params = mteamSearchParams(q.keyword ?? null, q.mtype, q.page ?? 0);
const data = await mteamPost<{ data?: MteamTorrentJson[] | null }>(
const params = mteamSearchParams(q.keyword ?? null, q.mtype, q.page ?? 0, q.cat);
const data = await mteamPost<{ data?: MteamTorrentJson[] | null; totalPages?: number }>(
ctx.http,
mtorrentApiBase(ctx.def),
apikey,
@@ -33,6 +33,9 @@ export const mtorrentSpider: SpecializedSpider = {
{ json: params },
ctx.signal,
);
if (!data || !Array.isArray(data.data))
throw new SiteError(ctx.def.id, "parse_error", "M-Team 搜索响应 data 不是资源列表");
if (Number.isInteger(data.totalPages) && data.totalPages! >= 0) ctx.reportPage?.({ totalPages: data.totalPages! });
const src = data?.data ?? [];
return parseMteamResult(src, siteBase(ctx.def)).map(
(t, i): SpecializedTorrent => ({ ...t, site_release_id: String(src[i]!.id) }),
+1 -1
View File
@@ -1,7 +1,7 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/spider/rousi.py @ e6820a8
* @mp-source app/modules/indexer/spider/rousi.py @ 3b156a5
*
* Rousi.pro PeerGo 兼容 API(parser "RousiPro",Bearer 个人 API Key,需 torrent:read + torrent:download)。
* 函数对照 `RousiSpider`:
@@ -1,7 +1,7 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/spider/sunnypt.py @ e6820a8
* @mp-source app/modules/indexer/spider/sunnypt.py @ 3b156a5
*
* SunnyPT MoviePilot API(parser "SunnyPT",X-API-Key)。函数对照 `SunnyPTSpider`:
* - `_size/_category_cache_ttl` → `SUNNYPT_PAGE_SIZE/SUNNYPT_CATEGORY_CACHE_TTL_MS`
@@ -1,11 +1,34 @@
import { createMockFetch, TNODE_HOME_HTML, TNODE_HOME_NO_TOKEN, TNODE_SEARCH_OK } from "@umt/testkit";
import { describe, expect, it } from "vitest";
import { createSiteRateLimiter } from "../rate-limit.ts";
import { createSpecializedSiteAdapter } from "./adapter.ts";
import { adapterWith, expectSiteError, testDef, withTorrent } from "./testing.ts";
import { parseTnodeToken, tnodeSearchParams, tnodeSpider } from "./tnode.ts";
const def = testDef("TNodeSpider", { domain: "https://tnode.example/" });
describe("TNodeSpider(MP tnode.py)", () => {
it("凭据变更后不复用旧CSRF令牌", async () => {
let cookie = "session=first";
const mock = createMockFetch({
"GET /": { body: TNODE_HOME_HTML },
"POST /api/torrent/advancedSearch": { body: TNODE_SEARCH_OK },
});
const adapter = createSpecializedSiteAdapter(def, tnodeSpider, {
fetch: mock.fetch,
secrets: {
async resolve() {
return cookie;
},
},
limiter: createSiteRateLimiter({ minIntervalMs: 0 }),
});
await adapter.list(1);
await adapter.list(1);
cookie = "session=second";
await adapter.list(1);
expect(mock.requests.filter((request) => request.url.pathname === "/")).toHaveLength(2);
});
it("__get_params / token 解析", () => {
expect(tnodeSearchParams("tt0923811", 1)).toMatchObject({
page: 2,
+6 -2
View File
@@ -1,7 +1,7 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/spider/tnode.py @ e6820a8
* @mp-source app/modules/indexer/spider/tnode.py @ 3b156a5
*
* TNode(parser "TNodeSpider",Cookie + 首页 CSRF token + `api/torrent/advancedSearch`)。函数对照 `TNodeSpider`:
* - `get_search_page_size` → `pageSize`(100)
@@ -11,6 +11,7 @@
* - `_downloadurl/_pageurl` → `resolveDownload`(带 Cookie 下载)/ page_url
* 差异:取不到 token 时 MP 返回 `(True, [])`;这里抛 auth_expired(首页取不到 token 基本意味着 Cookie 失效)。
*/
import { createHash } from "node:crypto";
import { SiteError } from "../types.ts";
import {
apiRequest,
@@ -33,7 +34,10 @@ export function parseTnodeToken(html: string): string | null {
}
async function tnodeToken(ctx: SpiderContext, cookie: string): Promise<string> {
const key = `tnode:token:${ctx.def.domain}`;
const identity = createHash("sha256")
.update(JSON.stringify([ctx.def.domain, cookie, ctx.userAgent ?? ""]))
.digest("hex");
const key = `tnode:token:${identity}`;
const cached = cacheGet<string>(ctx, key);
if (cached) return cached;
const token = parseTnodeToken(await textRequest(ctx, siteBase(ctx.def), { cookie }));
@@ -1,7 +1,7 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/spider/torrentleech.py @ e6820a8
* @mp-source app/modules/indexer/spider/torrentleech.py @ 3b156a5
*
* TorrentLeech JSON 搜索接口(parser "TorrentLeech",Cookie)。函数对照 `TorrentLeech`:
* - `get_search_page_size` → `pageSize`(关键词搜索 null,浏览 100)
+7 -1
View File
@@ -1,7 +1,7 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/__init__.py @ e6820a8
* @mp-source app/modules/indexer/__init__.py @ 3b156a5
*
* 专用 spider(API 型站点)共用类型。
* - `SpecializedParser` = MP `SPIDER_PARSER_CLASSES` 的键;`SPECIALIZED_SEARCH_ARGUMENTS` 的参数集合 → `SpiderSearchQuery`
@@ -21,6 +21,7 @@ export type SpecializedParser =
| "TorrentLeech"
| "Yema"
| "mTorrent"
| "Milkie"
// 本项目扩展:公开 BT 站(public.ts),MP 无对应
| "Nyaa"
| "Mikan"
@@ -45,6 +46,9 @@ export interface SpecializedSiteDefinition {
category?: SpiderCategoryConfig;
/** 独立 API 地址(MP indexer['api_url'],SunnyPT) */
api_url?: string;
/** M-Team 公开 API 主机覆盖,兼容上游的两种字段名。 */
api_domain?: string;
api_host?: string;
/** 本项目扩展(MP 无对应) */
umt: {
version: string;
@@ -93,6 +97,8 @@ export interface SpiderContext {
cache: Map<string, { value: unknown; expiresAt: number }>;
userAgent?: string;
signal?: AbortSignal;
/** 上游明确给出页数时,避免通过过滤后的资源数猜测是否还有下一页。 */
reportPage?(page: { totalPages: number }): void;
}
export interface SpecializedSpider {
@@ -13,6 +13,11 @@ const def = testDef("Yema", { domain: "https://www.yemapt.org/" });
const URL_SEARCH = "POST /openApi/torrent/fetchOpenTorrentList.json";
describe("YemaSpider(MP yema.py)", () => {
it("显式分类优先于媒体类型,逐分类查询", async () => {
const mock = createMockFetch({ [URL_SEARCH]: { body: YEMA_SEARCH_TV_A } });
await adapterWith(def, yemaSpider, mock.fetch).search("series", { mtype: "movie", categoryIds: ["5", "13"] });
expect(mock.requests.map((request) => JSON.parse(request.body!).categoryId)).toEqual([5, 13]);
});
it("列表解析:促销枚举、标签表(未知 ID 丢弃)、H&R、非整数 ID 跳过", async () => {
const m = createMockFetch({ [URL_SEARCH]: { body: YEMA_SEARCH_MOVIE } });
const page = await adapterWith(def, yemaSpider, m.fetch).search("past", { mtype: "movie" });
+4 -3
View File
@@ -1,7 +1,7 @@
/**
* SPDX-License-Identifier: GPL-3.0-only
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
* @mp-source app/modules/indexer/spider/yema.py @ e6820a8
* @mp-source app/modules/indexer/spider/yema.py @ 3b156a5
*
* YemaPT 开放 API(parser "Yema",Authorization: AuthKey)。函数对照 `YemaSpider`:
* - `_size/_movie_category/_tv_category/_music_category/_labels` → `YEMA_*`
@@ -76,7 +76,8 @@ export function yemaBuildParams(
}
/** MP `_search_category_ids`:分类参数不支持数组,逐个查询 */
export function yemaSearchCategoryIds(mtype?: SpiderSearchQuery["mtype"]): Array<number | null> {
export function yemaSearchCategoryIds(mtype?: SpiderSearchQuery["mtype"], cat?: string): Array<number | null> {
if (cat) return cat.split(",").map((id) => Number(id.trim()));
if (mtype === "movie") return [...YEMA_MOVIE_CATEGORY];
if (mtype === "tv") return [...YEMA_TV_CATEGORY];
if (mtype === "music") return [...YEMA_MUSIC_CATEGORY];
@@ -189,7 +190,7 @@ export const yemaSpider: SpecializedSpider = {
const url = `${yemaSiteUrl(ctx.def)}/openApi/torrent/fetchOpenTorrentList.json`;
const groups: SpecializedTorrent[][] = [];
const errors: unknown[] = [];
for (const categoryId of yemaSearchCategoryIds(q.mtype)) {
for (const categoryId of yemaSearchCategoryIds(q.mtype, q.cat)) {
try {
const payload = await apiRequest(ctx, url, {
method: "POST",