feat(adapters): port MoviePilot open site framework (7 spiders, 20 user parsers) + MovieClaw v0.27.0 site definitions + mp-sync tools (W1-3c)
- sites/spiders: Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema (+mTorrent wrapper), specialized SiteAdapter, indirect download resolved at download time - sites/user-parsers: SiteParserBase + all 20 MP SiteSchema parsers (lxml XPath via xpath+xmldom), refreshSiteUserData with schema fallback - sites/definitions: zod SiteDefinition schemas, 23 MovieClaw v0.27.0 (MIT) definitions via scripts/mp-sync/movieclaw-import.mjs, registry getSiteDefinition/listSiteDefinitions, NOTICE - scripts/mp-sync: ledger.json (upstream sha256), check.mjs, build-ledger.mjs; docs/porting/mp-sync-ledger.md - license-and-provenance: add copied-mit kind, re-anchor MovieClaw rows to v0.27.0 Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
cf85481d95
commit
d9f2af2f3c
166 files changed
+16577
-186
No files matched your search
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/__init__.py @ e6820a8
|
||||
*
|
||||
* 专用 spider 站点适配器:对应 MP `IndexerModule.search_torrents/refresh_torrents` 按 `indexer.parser`
|
||||
* 分派到 `SPIDER_PARSER_CLASSES` 并以 `_SPECIALIZED_SEARCH_ARGUMENTS` 传参;
|
||||
* 下载对应 `submission.download_torrent`(两段式换票由 spider.resolveDownload 现场完成)。
|
||||
*
|
||||
* 与 MP 的差异:MP `search` 失败返回 `(True, [])`;本项目抛 SiteError(凭据/限流/解析分类),
|
||||
* 健康检查即一次空关键词列表。list 对外页码 1 起;`pageSize=null` 的站(Haidan、TorrentLeech 关键词)只有第 1 页。
|
||||
*/
|
||||
import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
|
||||
import { healthFromError } from "../spider/adapter.ts";
|
||||
import {
|
||||
type DownloadTarget,
|
||||
type SiteAdapter,
|
||||
type SiteCredentialKind,
|
||||
SiteError,
|
||||
type SiteHealth,
|
||||
type SiteListFilters,
|
||||
type SitePage,
|
||||
} from "../types.ts";
|
||||
import { specializedTorrentToRelease } from "./map.ts";
|
||||
import type { SpecializedSiteDefinition, SpecializedSpider, SpiderContext } from "./types.ts";
|
||||
|
||||
export function createSpecializedSiteAdapter(
|
||||
input: SpecializedSiteDefinition,
|
||||
spider: SpecializedSpider,
|
||||
deps: SiteAdapterDeps,
|
||||
): SiteAdapter {
|
||||
const def: SpecializedSiteDefinition = deps.baseUrl
|
||||
? {
|
||||
...input,
|
||||
domain: deps.baseUrl.endsWith("/") ? deps.baseUrl : `${deps.baseUrl}/`,
|
||||
umt: { ...input.umt, apiBaseUrl: deps.baseUrl.replace(/\/+$/, "") },
|
||||
}
|
||||
: input;
|
||||
const http = buildHttpContext(
|
||||
def.id,
|
||||
{ ...(def.timeout ? { timeoutMs: def.timeout * 1000 } : {}), ...deps },
|
||||
def.umt.rateLimit,
|
||||
);
|
||||
const cache = new Map<string, { value: unknown; expiresAt: number }>();
|
||||
/** 最近搜索结果的直链(仅内存,供 downloadTarget 复用;含 passkey 类参数,不外泄) */
|
||||
const enclosures = new Map<string, string>();
|
||||
|
||||
function ctx(signal?: AbortSignal): SpiderContext {
|
||||
const c: SpiderContext = {
|
||||
def,
|
||||
http,
|
||||
cache,
|
||||
async secret(kind: SiteCredentialKind, required = false) {
|
||||
const v = await deps.secrets.resolve({ siteId: def.id, kind });
|
||||
if (!v && required) {
|
||||
throw new SiteError(
|
||||
def.id,
|
||||
"auth_missing",
|
||||
`${def.name} 未配置${kind === "cookie" ? " Cookie" : " API Key"}`,
|
||||
);
|
||||
}
|
||||
return v;
|
||||
},
|
||||
};
|
||||
if (deps.userAgent) c.userAgent = deps.userAgent;
|
||||
if (signal) c.signal = signal;
|
||||
return c;
|
||||
}
|
||||
|
||||
async function doSearch(page: number, filters: SiteListFilters, keyword: string | null, signal?: AbortSignal) {
|
||||
const p = Math.max(1, page);
|
||||
const fetchedAt = http.now().toISOString();
|
||||
const size = spider.pageSize(keyword);
|
||||
if (size === null && p > 1) {
|
||||
return { items: [], page: p, totalPages: 1, totalPagesEstimated: true, fetchedAt } satisfies SitePage;
|
||||
}
|
||||
const q: Parameters<SpecializedSpider["search"]>[1] = { keyword, page: p - 1 };
|
||||
if (filters.mtype) q.mtype = filters.mtype;
|
||||
if (filters.categoryIds?.length) q.cat = filters.categoryIds.join(",");
|
||||
const rows = await spider.search(ctx(signal), q);
|
||||
let items = rows.flatMap((t) => {
|
||||
const r = specializedTorrentToRelease(t, def, fetchedAt);
|
||||
if (r && t.enclosure) enclosures.set(r.siteReleaseId, t.enclosure);
|
||||
return r ? [r] : [];
|
||||
});
|
||||
if (filters.categoryIds?.length) {
|
||||
const allow = new Set(filters.categoryIds.map(String));
|
||||
items = items.filter((r) => r.category.id === null || allow.has(r.category.id));
|
||||
}
|
||||
if (filters.freeOnly) items = items.filter((r) => r.promotion.downloadFactor === 0);
|
||||
return { items, page: p, totalPages: null, totalPagesEstimated: false, fetchedAt } satisfies SitePage;
|
||||
}
|
||||
|
||||
return {
|
||||
id: def.id,
|
||||
kind: "api",
|
||||
displayName: def.umt.displayName,
|
||||
capabilities: def.umt.capabilities,
|
||||
definitionVersion: def.umt.version,
|
||||
list(page, filters = {}, opts = {}) {
|
||||
return doSearch(page, filters, null, opts.signal);
|
||||
},
|
||||
search(query, filters = {}, opts = {}) {
|
||||
return doSearch(filters.page ?? 1, filters, query.trim() || null, opts.signal);
|
||||
},
|
||||
async detail() {
|
||||
throw new SiteError(def.id, "unsupported", `${def.name}:专用 API spider 不提供详情接口`);
|
||||
},
|
||||
async downloadTarget(siteReleaseId, opts = {}): Promise<DownloadTarget> {
|
||||
return spider.resolveDownload(ctx(opts.signal), siteReleaseId, enclosures.get(String(siteReleaseId)));
|
||||
},
|
||||
async healthCheck(opts = {}): Promise<SiteHealth> {
|
||||
const started = Date.now();
|
||||
const checkedAt = http.now().toISOString();
|
||||
try {
|
||||
await spider.search(ctx(opts.signal), { keyword: null, page: 0 });
|
||||
return { siteId: def.id, status: "ok", ok: true, checkedAt, latencyMs: Date.now() - started };
|
||||
} catch (e) {
|
||||
return healthFromError(def.id, e, checkedAt);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/chain/download/submission.py @ e6820a8
|
||||
* @mp-source app/foundation/temporal.py @ e6820a8
|
||||
*
|
||||
* 专用 spider 共用 IO 与工具:
|
||||
* - `submission._resolve_indirect_download_url` → `resolveIndirectDownloadUrl`(配置对象在内存中构造,不再 base64 编进 enclosure)
|
||||
* - `submission._normalize_indirect_download_url` → `normalizeIndirectDownloadUrl`
|
||||
* - `submission.download_torrent`(带/不带 cookie 下载并校验)→ `downloadTorrentFile`
|
||||
* - `temporal.format_timestamp` → `formatTimestampTz`;`urllib.parse.quote` → `pyQuote`
|
||||
* - 各 spider `__process_response` 的错误分支 → `apiRequest`(HTTP 401/403 → auth_invalid)+ `businessError`
|
||||
*/
|
||||
import { redactText, redactUrl } from "../../rss/redact.ts";
|
||||
import { readBody, readText, siteFetch } from "../http.ts";
|
||||
import { type DownloadTarget, SiteError, type SiteErrorKind } from "../types.ts";
|
||||
import { inspectTorrent } from "../util.ts";
|
||||
import type { SpiderContext } from "./types.ts";
|
||||
|
||||
/** 下载链接中需要脱敏的额外参数名(HDDolby downhash 等效 passkey) */
|
||||
export const EXTRA_SECRET_KEYS = ["downhash", "token", "credential", "sign", "key"];
|
||||
|
||||
const RATE_KEYWORDS = ["頻繁", "频繁", "too many", "rate limit", "稍后"];
|
||||
const NOT_FOUND_KEYWORDS = ["不存在", "not found", "not exist"];
|
||||
const AUTH_KEYWORDS = [
|
||||
"停用",
|
||||
"封禁",
|
||||
"禁用",
|
||||
"banned",
|
||||
"disabled",
|
||||
"無效",
|
||||
"无效",
|
||||
"invalid",
|
||||
"unauthorized",
|
||||
"未授權",
|
||||
"未授权",
|
||||
"未登录",
|
||||
"登录",
|
||||
"api key",
|
||||
"apikey",
|
||||
"token",
|
||||
"permission",
|
||||
"权限",
|
||||
"forbidden",
|
||||
];
|
||||
|
||||
/** 业务错误文案 → SiteError 类别(默认 parse_error) */
|
||||
export function classifyMessage(message: string | null | undefined): SiteErrorKind {
|
||||
const lower = String(message ?? "").toLowerCase();
|
||||
if (RATE_KEYWORDS.some((k) => lower.includes(k))) return "rate_limited";
|
||||
if (NOT_FOUND_KEYWORDS.some((k) => lower.includes(k))) return "not_found";
|
||||
if (AUTH_KEYWORDS.some((k) => lower.includes(k.toLowerCase()))) return "auth_invalid";
|
||||
return "parse_error";
|
||||
}
|
||||
|
||||
export function businessError(ctx: SpiderContext, what: string, message: unknown): SiteError {
|
||||
const msg = message == null || message === "" ? "未知错误" : String(message).slice(0, 200);
|
||||
return new SiteError(ctx.def.id, classifyMessage(msg), redactText(`${ctx.def.name} ${what}失败:${msg}`));
|
||||
}
|
||||
|
||||
export interface ApiRequestInit {
|
||||
method?: "GET" | "POST";
|
||||
headers?: Record<string, string>;
|
||||
query?: Record<string, string | number | null | undefined>;
|
||||
json?: unknown;
|
||||
/** 原样 query 串(已编码,不含 ?) */
|
||||
rawQuery?: string;
|
||||
}
|
||||
|
||||
export function withQuery(url: string, init: Pick<ApiRequestInit, "query" | "rawQuery">): string {
|
||||
const parts: string[] = [];
|
||||
if (init.rawQuery) parts.push(init.rawQuery);
|
||||
if (init.query) {
|
||||
const qs = new URLSearchParams();
|
||||
for (const [k, v] of Object.entries(init.query)) if (v !== undefined && v !== null) qs.append(k, String(v));
|
||||
const s = qs.toString();
|
||||
if (s) parts.push(s);
|
||||
}
|
||||
if (!parts.length) return url;
|
||||
return `${url}${url.includes("?") ? "&" : "?"}${parts.join("&")}`;
|
||||
}
|
||||
|
||||
/** 不跟随跳转(请求头可能带 cookie/key);跳 login 视为登录失效 */
|
||||
function checkRedirect(ctx: SpiderContext, res: Response, safe: string): void {
|
||||
if (res.status < 300 || res.status >= 400) return;
|
||||
void res.body?.cancel().catch(() => {});
|
||||
const loc = res.headers.get("location") ?? "";
|
||||
if (/login/i.test(loc))
|
||||
throw new SiteError(ctx.def.id, "auth_expired", `${ctx.def.name} 请求 ${safe}:跳转登录页,登录已失效`);
|
||||
throw new SiteError(ctx.def.id, "network", `${ctx.def.name} 请求 ${safe}:意外跳转 HTTP ${res.status}`, {
|
||||
httpStatus: res.status,
|
||||
});
|
||||
}
|
||||
|
||||
/** 发请求并解析 JSON;HTTP 401/403 → auth_invalid,404 → not_found,其它非 2xx → network,非 JSON → parse_error/site_maintenance */
|
||||
export async function apiRequest(ctx: SpiderContext, url: string, init: ApiRequestInit = {}): Promise<unknown> {
|
||||
const headers: Record<string, string> = { ...(init.headers ?? {}) };
|
||||
if (ctx.userAgent && !Object.keys(headers).some((k) => k.toLowerCase() === "user-agent"))
|
||||
headers["User-Agent"] = ctx.userAgent;
|
||||
let body: string | undefined;
|
||||
if (init.json !== undefined) {
|
||||
body = JSON.stringify(init.json);
|
||||
if (!Object.keys(headers).some((k) => k.toLowerCase() === "content-type"))
|
||||
headers["Content-Type"] = "application/json";
|
||||
}
|
||||
const full = withQuery(url, init);
|
||||
const safe = redactUrl(full, EXTRA_SECRET_KEYS);
|
||||
const res = await siteFetch(ctx.http, full, {
|
||||
method: init.method ?? (body !== undefined ? "POST" : "GET"),
|
||||
headers,
|
||||
...(body !== undefined ? { body } : {}),
|
||||
redirect: "manual",
|
||||
...(ctx.signal ? { signal: ctx.signal } : {}),
|
||||
});
|
||||
checkRedirect(ctx, res, safe);
|
||||
if (res.status === 401 || res.status === 403) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(
|
||||
ctx.def.id,
|
||||
"auth_invalid",
|
||||
`${ctx.def.name} 请求 ${safe}:HTTP ${res.status},凭据无效或无权限`,
|
||||
{
|
||||
httpStatus: res.status,
|
||||
},
|
||||
);
|
||||
}
|
||||
if (!res.ok) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(
|
||||
ctx.def.id,
|
||||
res.status === 404 ? "not_found" : "network",
|
||||
`${ctx.def.name} 请求 ${safe}:HTTP ${res.status}`,
|
||||
{
|
||||
httpStatus: res.status,
|
||||
},
|
||||
);
|
||||
}
|
||||
const text = await readText(ctx.http, res);
|
||||
try {
|
||||
return JSON.parse(text) as unknown;
|
||||
} catch {
|
||||
const maintenance = /維護|维护|maintenance/i.test(text);
|
||||
throw new SiteError(
|
||||
ctx.def.id,
|
||||
maintenance ? "site_maintenance" : "parse_error",
|
||||
`${ctx.def.name} 请求 ${safe}:响应不是有效 JSON${maintenance ? "(站点维护)" : ""}`,
|
||||
{ httpStatus: res.status },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** 发请求取文本(TNode 首页取 CSRF token) */
|
||||
export async function textRequest(ctx: SpiderContext, url: string, headers: Record<string, string>): Promise<string> {
|
||||
const h = { ...headers };
|
||||
if (ctx.userAgent) h["User-Agent"] = ctx.userAgent;
|
||||
const res = await siteFetch(ctx.http, url, {
|
||||
method: "GET",
|
||||
headers: h,
|
||||
redirect: "manual",
|
||||
...(ctx.signal ? { signal: ctx.signal } : {}),
|
||||
});
|
||||
checkRedirect(ctx, res, redactUrl(url));
|
||||
if (res.status !== 200) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(
|
||||
ctx.def.id,
|
||||
res.status === 401 || res.status === 403 ? "auth_invalid" : "network",
|
||||
`${ctx.def.name} 请求 ${redactUrl(url)}:HTTP ${res.status}`,
|
||||
{ httpStatus: res.status },
|
||||
);
|
||||
}
|
||||
return readText(ctx.http, res);
|
||||
}
|
||||
|
||||
export function isPlainObject(v: unknown): v is Record<string, unknown> {
|
||||
return v != null && typeof v === "object" && !Array.isArray(v);
|
||||
}
|
||||
|
||||
// ---- 时间 ----
|
||||
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
|
||||
function offsetMinutes(tz: string): number {
|
||||
const m = /^([+-])(\d{2}):?(\d{2})$/.exec(tz.trim());
|
||||
if (!m) return 0;
|
||||
return (m[1] === "-" ? -1 : 1) * (Number(m[2]) * 60 + Number(m[3]));
|
||||
}
|
||||
|
||||
/**
|
||||
* MP `format_timestamp`:非纯数字字符串原样返回;Unix 秒 → 本地墙钟 "YYYY-MM-DD HH:MM:SS"。
|
||||
* MP 用进程本地时区,这里用站点时区(`umt.timezone`),随后 map 再按同一时区转 UTC,结果等价。
|
||||
*/
|
||||
export function formatTimestampTz(timestamp: unknown, tz: string): string | undefined {
|
||||
if (timestamp == null || timestamp === "") return undefined;
|
||||
if (typeof timestamp === "string" && !/^\d+$/.test(timestamp)) return timestamp;
|
||||
const n = Number(timestamp);
|
||||
if (!Number.isFinite(n)) return String(timestamp);
|
||||
const d = new Date(Math.trunc(n) * 1000 + offsetMinutes(tz) * 60_000);
|
||||
return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
|
||||
}
|
||||
|
||||
/** 带显式时区(Z / ±HH:MM)的 ISO 时间 → UTC ISO;无时区返回 undefined */
|
||||
export function isoWithZoneToUtc(v: unknown): string | undefined {
|
||||
if (typeof v !== "string") return undefined;
|
||||
if (!/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?\s*(Z|[+-]\d{2}:?\d{2})$/i.test(v.trim())) return undefined;
|
||||
const t = Date.parse(v.trim().replace(" ", "T"));
|
||||
return Number.isNaN(t) ? undefined : new Date(t).toISOString();
|
||||
}
|
||||
|
||||
/** Python `urllib.parse.quote`(safe='/') */
|
||||
export function pyQuote(s: string): string {
|
||||
return encodeURIComponent(s)
|
||||
.replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)
|
||||
.replace(/%2F/g, "/");
|
||||
}
|
||||
|
||||
/** Python `int(x or 0)`;非法值 0(MP 会抛异常丢整页,这里保守取 0) */
|
||||
export function toInt(v: unknown): number {
|
||||
if (typeof v === "number") return Number.isFinite(v) ? Math.trunc(v) : 0;
|
||||
const n = Number.parseInt(String(v ?? "0") || "0", 10);
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
}
|
||||
|
||||
/** Python `float(x)`;非法值回退 */
|
||||
export function toFloat(v: unknown, fallback = 1): number {
|
||||
const n = typeof v === "number" ? v : Number(v);
|
||||
return Number.isFinite(n) ? n : fallback;
|
||||
}
|
||||
|
||||
// ---- 缓存 ----
|
||||
|
||||
export function cacheGet<T>(ctx: SpiderContext, key: string): T | undefined {
|
||||
const hit = ctx.cache.get(key);
|
||||
if (!hit) return undefined;
|
||||
if (hit.expiresAt <= ctx.http.now().getTime()) {
|
||||
ctx.cache.delete(key);
|
||||
return undefined;
|
||||
}
|
||||
return hit.value as T;
|
||||
}
|
||||
|
||||
export function cacheSet(ctx: SpiderContext, key: string, value: unknown, ttlMs: number): void {
|
||||
ctx.cache.set(key, { value, expiresAt: ctx.http.now().getTime() + ttlMs });
|
||||
}
|
||||
|
||||
// ---- 下载 ----
|
||||
|
||||
/** MP 两段式下载的请求配置(原本 base64 编进 enclosure;这里只在内存中构造) */
|
||||
export interface IndirectRequestConfig {
|
||||
method: "get" | "post";
|
||||
header: Record<string, string>;
|
||||
/** requests params → query string */
|
||||
params?: Record<string, string | number>;
|
||||
/** 成功标志键(Yema: "success") */
|
||||
success?: string;
|
||||
/** 结果字段路径,点分(如 "data.download_url") */
|
||||
result?: string;
|
||||
result_base_url?: string;
|
||||
result_path?: string;
|
||||
result_query_param?: string;
|
||||
}
|
||||
|
||||
/** MP `_normalize_indirect_download_url`:把换票结果约束到可信 API 来源 */
|
||||
export function normalizeIndirectDownloadUrl(url: string, baseUrl?: string): string {
|
||||
if (!url || !baseUrl) return url;
|
||||
let base: URL;
|
||||
try {
|
||||
base = new URL(baseUrl);
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
let u: URL;
|
||||
try {
|
||||
u = new URL(url);
|
||||
} catch {
|
||||
return new URL(url, `${baseUrl.replace(/\/+$/, "")}/`).toString();
|
||||
}
|
||||
u.protocol = base.protocol;
|
||||
u.host = base.host;
|
||||
return u.toString();
|
||||
}
|
||||
|
||||
/** MP `_resolve_indirect_download_url`:换票 → 临时直链(失败抛 SiteError,MP 返回 None) */
|
||||
export async function resolveIndirectDownloadUrl(
|
||||
ctx: SpiderContext,
|
||||
requestUrl: string,
|
||||
cfg: IndirectRequestConfig,
|
||||
): Promise<string> {
|
||||
const data = await apiRequest(ctx, requestUrl, {
|
||||
method: cfg.method === "get" ? "GET" : "POST",
|
||||
headers: cfg.header,
|
||||
...(cfg.params ? { query: cfg.params } : {}),
|
||||
});
|
||||
const fail = (why: string): never => {
|
||||
const msg = isPlainObject(data) ? (data.message ?? data.msg ?? data.errorMessage) : undefined;
|
||||
throw new SiteError(
|
||||
ctx.def.id,
|
||||
msg ? classifyMessage(String(msg)) : "parse_error",
|
||||
redactText(`${ctx.def.name} 换取下载地址失败:${msg ? String(msg).slice(0, 200) : why}`),
|
||||
);
|
||||
};
|
||||
if (!cfg.result) return typeof data === "string" ? data : fail("响应不是字符串");
|
||||
if (cfg.success && !(isPlainObject(data) && data[cfg.success])) fail("success 标志为假");
|
||||
let cur: unknown = data;
|
||||
for (const key of cfg.result.split(".")) {
|
||||
if (!isPlainObject(cur)) fail(`缺少字段 ${cfg.result}`);
|
||||
cur = (cur as Record<string, unknown>)[key];
|
||||
if (!cur) fail(`字段 ${cfg.result} 为空`);
|
||||
}
|
||||
if (cfg.result_path && cfg.result_query_param) {
|
||||
const u = new URL(cfg.result_path.replace(/^\/+/, ""), `${String(cfg.result_base_url).replace(/\/+$/, "")}/`);
|
||||
return `${u.toString()}?${new URLSearchParams({ [cfg.result_query_param]: String(cur) }).toString()}`;
|
||||
}
|
||||
if (typeof cur !== "string") return fail("下载地址不是字符串");
|
||||
return normalizeIndirectDownloadUrl(cur, cfg.result_base_url);
|
||||
}
|
||||
|
||||
/**
|
||||
* MP `download_torrent`:GET 种子并校验。cookie 为空表示"涉及解析地址的不使用 Cookie 下载"。
|
||||
* 带 cookie 时不跟随跳转(避免 cookie 外泄;跳 login 视为失效;跳磁力原样返回)。
|
||||
*/
|
||||
export async function downloadTorrentFile(
|
||||
ctx: SpiderContext,
|
||||
url: string,
|
||||
opts: { cookie?: string | null; headers?: Record<string, string> } = {},
|
||||
): Promise<DownloadTarget> {
|
||||
const headers: Record<string, string> = { ...(opts.headers ?? {}) };
|
||||
if (ctx.userAgent) headers["User-Agent"] = ctx.userAgent;
|
||||
if (opts.cookie) headers.cookie = opts.cookie;
|
||||
const safe = redactUrl(url, EXTRA_SECRET_KEYS);
|
||||
const res = await siteFetch(ctx.http, url, {
|
||||
method: "GET",
|
||||
headers,
|
||||
...(opts.cookie ? { redirect: "manual" as const } : {}),
|
||||
...(ctx.signal ? { signal: ctx.signal } : {}),
|
||||
});
|
||||
const verifiedAt = ctx.http.now().toISOString();
|
||||
if (res.status >= 300 && res.status < 400) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
const loc = res.headers.get("location") ?? "";
|
||||
if (loc.startsWith("magnet:")) return { kind: "magnet", uri: loc, verifiedAt };
|
||||
if (/login/i.test(loc)) throw new SiteError(ctx.def.id, "auth_expired", `下载 ${safe}:跳转登录页,Cookie 已失效`);
|
||||
throw new SiteError(ctx.def.id, "network", `下载 ${safe}:意外跳转 HTTP ${res.status}`, { httpStatus: res.status });
|
||||
}
|
||||
if (res.status === 401 || res.status === 403) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(ctx.def.id, "auth_invalid", `下载 ${safe}:HTTP ${res.status}`, { httpStatus: res.status });
|
||||
}
|
||||
if (!res.ok) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(ctx.def.id, res.status === 404 ? "not_found" : "network", `下载 ${safe}:HTTP ${res.status}`, {
|
||||
httpStatus: res.status,
|
||||
});
|
||||
}
|
||||
const bytes = await readBody(ctx.http, res);
|
||||
const info = inspectTorrent(bytes);
|
||||
if (!info) throw new SiteError(ctx.def.id, "parse_error", `下载 ${safe}:返回内容不是 .torrent`);
|
||||
return { kind: "torrent", bytes, infoHash: info.infoHash, name: info.name, fileName: null, verifiedAt };
|
||||
}
|
||||
|
||||
/** 站点前端地址(以 / 结尾) */
|
||||
export function siteBase(def: { domain: string }): string {
|
||||
return def.domain.endsWith("/") ? def.domain : `${def.domain}/`;
|
||||
}
|
||||
|
||||
/** 提取资源 ID:纯数字或 URL 末段数字 */
|
||||
export function checkReleaseId(siteId: string, id: string): string {
|
||||
const m = /(\d+)\/?$/.exec(String(id).trim());
|
||||
if (!m) throw new SiteError(siteId, "not_found", "资源 ID 非法");
|
||||
return m[1]!;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { createMockFetch, HAIDAN_SEARCH_ERROR, HAIDAN_SEARCH_OK } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { haidanSearchQuery, haidanSpider } from "./haidan.ts";
|
||||
import { adapterWith, expectSiteError, testDef, withTorrent } from "./testing.ts";
|
||||
|
||||
const def = testDef("Haidan", { domain: "https://www.haidan.video/" });
|
||||
|
||||
describe("HaiDanSpider(MP haidan.py)", () => {
|
||||
it("__get_params:isapi/search_area/空关键词/分类逗号拼接", () => {
|
||||
expect(haidanSearchQuery(null)).toBe("isapi=1&search_area=0&search=&search_mode=0&cat=");
|
||||
expect(haidanSearchQuery("tt1160419", "tv")).toBe(
|
||||
"isapi=1&search_area=4&search=tt1160419&search_mode=0&cat=402%2C403%2C404%2C405",
|
||||
);
|
||||
});
|
||||
|
||||
it("列表解析:字段映射、分类投影、促销系数表、passkey 脱敏", async () => {
|
||||
const m = createMockFetch({ "/torrents.php": { body: HAIDAN_SEARCH_OK } });
|
||||
const page = await adapterWith(def, haidanSpider, m.fetch).search("Dune", { mtype: "movie" });
|
||||
expect(m.requests[0]!.headers.cookie).toBe("c=COOKIESECRET");
|
||||
expect(m.requests[0]!.url.searchParams.get("cat")).toBe("401,404,405");
|
||||
const by = Object.fromEntries(page.items.map((i) => [i.siteReleaseId, i]));
|
||||
expect(by["30001"]).toMatchObject({
|
||||
title: "Dune.Part.Two.2024.2160p.UHD.BluRay.REMUX.HDR.HEVC.Atmos-HDH",
|
||||
subtitle: "沙丘2 | 4K 原盘 REMUX",
|
||||
category: { id: "401" },
|
||||
sizeBytes: 76543210987,
|
||||
seeders: 88,
|
||||
leechers: 3,
|
||||
snatched: 412,
|
||||
promotion: { type: "free", downloadFactor: 0, uploadFactor: 1 },
|
||||
hr: "unknown",
|
||||
uploadedAt: "2024-09-20T00:00:00.000Z",
|
||||
detailUrl: "https://www.haidan.video/details.php?group_id=9001&torrent_id=30001",
|
||||
raw: expect.objectContaining({ category: "电影" }),
|
||||
});
|
||||
expect(by["30002"]!.promotion).toMatchObject({ type: "2x50%", downloadFactor: 0.5, uploadFactor: 2 });
|
||||
expect(by["30002"]!.raw.category).toBe("电视剧");
|
||||
expect(by["30003"]!.raw.category).toBe("音乐");
|
||||
expect(JSON.stringify(page)).not.toContain("HDPASSKEY123");
|
||||
expect(page.totalPages).toBeNull();
|
||||
// 不翻页:第 2 页直接为空
|
||||
expect((await adapterWith(def, haidanSpider, m.fetch).list(2)).items).toEqual([]);
|
||||
});
|
||||
|
||||
it("业务错误 / 缺 Cookie / 下载沿用缓存直链带 Cookie", async () => {
|
||||
const bad = createMockFetch({ "/torrents.php": { body: HAIDAN_SEARCH_ERROR } });
|
||||
await expectSiteError(adapterWith(def, haidanSpider, bad.fetch).list(1), "auth_invalid");
|
||||
await expectSiteError(adapterWith(def, haidanSpider, bad.fetch, {}).list(1), "auth_missing");
|
||||
const m = createMockFetch({ "/torrents.php": { body: HAIDAN_SEARCH_OK } });
|
||||
const t = withTorrent(m.fetch, (u) => u.pathname === "/download.php");
|
||||
const a = adapterWith(def, haidanSpider, t.fetch);
|
||||
await a.list(1);
|
||||
const target = await a.downloadTarget("30001");
|
||||
expect(target).toMatchObject({ kind: "torrent", infoHash: t.infoHash });
|
||||
expect(t.hits[0]!.url.searchParams.get("passkey")).toBe("HDPASSKEY123");
|
||||
expect(t.hits[0]!.headers.get("cookie")).toBe("c=COOKIESECRET");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/spider/haidan.py @ e6820a8
|
||||
*
|
||||
* 海胆 haidan.video(parser "Haidan",Cookie + `torrents.php?isapi=1` JSON)。函数对照 `HaiDanSpider`:
|
||||
* - `_movie_category/_tv_category/_music_category/_dl_state/_up_state` → 同名常量
|
||||
* - `get_search_page_size` → `pageSize`(恒 null:不翻页)
|
||||
* - `__get_params`(含 `__dict_to_query`)→ `haidanSearchQuery`
|
||||
* - `__parse_result` → `parseHaidanResult`;`__process_response` → `processHaidanResponse`
|
||||
* - `search/async_search` → `haidanSpider.search`
|
||||
* - `__get_downloadvolumefactor/__get_uploadvolumefactor` → `haidanDownloadVolumeFactor/haidanUploadVolumeFactor`
|
||||
* 下载:MP 用结果里的 `url` 字段(可能含 passkey)带 Cookie 下载;缓存缺失时回退 NexusPHP `download.php?id=`(TODO(verify))。
|
||||
*/
|
||||
import type { MpMediaType } from "../spider/spider.ts";
|
||||
import { SiteError } from "../types.ts";
|
||||
import {
|
||||
apiRequest,
|
||||
businessError,
|
||||
downloadTorrentFile,
|
||||
formatTimestampTz,
|
||||
isPlainObject,
|
||||
siteBase,
|
||||
toInt,
|
||||
} from "./common.ts";
|
||||
import type { SpecializedSiteDefinition, SpecializedSpider, SpecializedTorrent, SpiderSearchQuery } from "./types.ts";
|
||||
|
||||
export const HAIDAN_MOVIE_CATEGORY = ["401", "404", "405"];
|
||||
export const HAIDAN_TV_CATEGORY = ["402", "403", "404", "405"];
|
||||
/** 音乐搜索同时覆盖高品质音频和音乐视频/演唱会分区 */
|
||||
export const HAIDAN_MUSIC_CATEGORY = ["406", "408"];
|
||||
|
||||
/** 促销状态 1-普通,2-免费,3-2X,4-2X免费,5-50%,6-2X50%,7-30% */
|
||||
export const HAIDAN_DL_STATE: Record<string, number> = { "1": 1, "2": 0, "3": 1, "4": 0, "5": 0.5, "6": 0.5, "7": 0.3 };
|
||||
export const HAIDAN_UP_STATE: Record<string, number> = { "1": 1, "2": 1, "3": 2, "4": 2, "5": 1, "6": 2, "7": 1 };
|
||||
|
||||
export function haidanDownloadVolumeFactor(discount: unknown): number {
|
||||
return discount ? (HAIDAN_DL_STATE[String(discount)] ?? 1) : 1;
|
||||
}
|
||||
|
||||
export function haidanUploadVolumeFactor(discount: unknown): number {
|
||||
return discount ? (HAIDAN_UP_STATE[String(discount)] ?? 1) : 1;
|
||||
}
|
||||
|
||||
/** MP `__get_params`:返回已编码的 query 串(列表转逗号分隔;空关键词显式传空串) */
|
||||
export function haidanSearchQuery(keyword: string | null | undefined, mtype?: SpiderSearchQuery["mtype"]): string {
|
||||
let categories: string[];
|
||||
if (!mtype) categories = [];
|
||||
else if (mtype === "tv") categories = HAIDAN_TV_CATEGORY;
|
||||
else if (mtype === "music") categories = HAIDAN_MUSIC_CATEGORY;
|
||||
else categories = HAIDAN_MOVIE_CATEGORY;
|
||||
// 0-标题 1-简介(较慢)3-发种用户名 4-IMDb
|
||||
const searchArea = keyword?.startsWith("tt") ? "4" : "0";
|
||||
return new URLSearchParams({
|
||||
isapi: "1",
|
||||
search_area: searchArea,
|
||||
search: keyword || "",
|
||||
search_mode: "0", // 0-与 1-或 2-精准
|
||||
cat: categories.join(","),
|
||||
}).toString();
|
||||
}
|
||||
|
||||
export interface HaidanItem {
|
||||
group_id?: string | number;
|
||||
category?: string | number | null;
|
||||
name?: string;
|
||||
small_descr?: string;
|
||||
url?: string;
|
||||
added?: string | number;
|
||||
size?: string | number;
|
||||
seeders?: string | number;
|
||||
leechers?: string | number;
|
||||
times_completed?: string | number;
|
||||
sp_state?: string | number;
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
/** MP `__parse_result`(data 为 {tid: item} 字典) */
|
||||
export function parseHaidanResult(
|
||||
result: { data?: Record<string, HaidanItem> | null },
|
||||
def: SpecializedSiteDefinition,
|
||||
) {
|
||||
const torrents: SpecializedTorrent[] = [];
|
||||
const base = siteBase(def);
|
||||
for (const [tid, item] of Object.entries(result.data ?? {})) {
|
||||
// 分类字段在每条种子内,接口顶层没有 category 字段
|
||||
const categoryValue = String(item.category || "");
|
||||
let category: MpMediaType;
|
||||
if (HAIDAN_MUSIC_CATEGORY.includes(categoryValue)) category = "音乐";
|
||||
else if (HAIDAN_TV_CATEGORY.includes(categoryValue) && !HAIDAN_MOVIE_CATEGORY.includes(categoryValue))
|
||||
category = "电视剧";
|
||||
else if (HAIDAN_MOVIE_CATEGORY.includes(categoryValue)) category = "电影";
|
||||
else category = "未知";
|
||||
const t: SpecializedTorrent = {
|
||||
title: item.name,
|
||||
description: item.small_descr,
|
||||
size: toInt(item.size),
|
||||
seeders: toInt(item.seeders),
|
||||
peers: toInt(item.leechers),
|
||||
grabs: toInt(item.times_completed),
|
||||
downloadvolumefactor: haidanDownloadVolumeFactor(item.sp_state),
|
||||
uploadvolumefactor: haidanUploadVolumeFactor(item.sp_state),
|
||||
page_url: `${base}details.php?group_id=${item.group_id}&torrent_id=${tid}`,
|
||||
labels: [],
|
||||
category,
|
||||
site_release_id: tid,
|
||||
};
|
||||
if (item.url) t.enclosure = String(item.url);
|
||||
const pubdate = formatTimestampTz(item.added, def.umt.timezone);
|
||||
if (pubdate) t.pubdate = pubdate;
|
||||
if (categoryValue) t.category_id = categoryValue;
|
||||
torrents.push(t);
|
||||
}
|
||||
return torrents;
|
||||
}
|
||||
|
||||
export const haidanSpider: SpecializedSpider = {
|
||||
parser: "Haidan",
|
||||
credential: "cookie",
|
||||
pageSize: () => null,
|
||||
async search(ctx, q) {
|
||||
const cookie = await ctx.secret("cookie", true);
|
||||
const data = await apiRequest(ctx, `${siteBase(ctx.def)}torrents.php`, {
|
||||
headers: { cookie: cookie! },
|
||||
rawQuery: haidanSearchQuery(q.keyword, q.mtype),
|
||||
});
|
||||
// MP __process_response:code != 0 → 失败
|
||||
if (!isPlainObject(data) || data.code !== 0) {
|
||||
throw businessError(ctx, "搜索", isPlainObject(data) ? data.msg : "响应结构无效");
|
||||
}
|
||||
return parseHaidanResult(data as { data?: Record<string, HaidanItem> }, ctx.def);
|
||||
},
|
||||
async resolveDownload(ctx, siteReleaseId, enclosure) {
|
||||
const cookie = await ctx.secret("cookie", true);
|
||||
const id = /(\d+)\/?$/.exec(String(siteReleaseId))?.[1];
|
||||
if (!id) throw new SiteError(ctx.def.id, "not_found", "资源 ID 非法");
|
||||
// TODO(verify):无缓存直链时回退 download.php?id=
|
||||
const url = enclosure ?? `${siteBase(ctx.def)}download.php?id=${id}`;
|
||||
return downloadTorrentFile(ctx, url, { cookie });
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,61 @@
|
||||
import { createMockFetch, HDDOLBY_SEARCH_ERROR, HDDOLBY_SEARCH_OK } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { hddolbyApiBase, hddolbySearchParams, hddolbySpider } from "./hddolby.ts";
|
||||
import { adapterWith, expectSiteError, testDef, withTorrent } from "./testing.ts";
|
||||
|
||||
const def = testDef("HDDolby", { domain: "https://www.hddolby.com/" });
|
||||
|
||||
describe("HddolbySpider(MP hddolby.py)", () => {
|
||||
it("__get_params 与 API 地址:page_number 为 0 起、无类型时 movie+tv 去重升序", () => {
|
||||
expect(hddolbyApiBase(def)).toBe("https://api.hddolby.com");
|
||||
expect(hddolbySearchParams("Lost")).toEqual({
|
||||
keyword: "Lost",
|
||||
page_number: 0,
|
||||
page_size: 100,
|
||||
categories: [401, 402, 403, 404, 405],
|
||||
visible: 1,
|
||||
});
|
||||
expect(hddolbySearchParams(null, "music", 2).categories).toEqual([406, 408]);
|
||||
});
|
||||
|
||||
it("列表解析:促销类型表、标签表、分类优先级(405 归电视剧)、downhash 脱敏", async () => {
|
||||
const m = createMockFetch({ "POST https://api.hddolby.com/api/v1/torrent/search": { body: HDDOLBY_SEARCH_OK } });
|
||||
const page = await adapterWith(def, hddolbySpider, m.fetch).list(1);
|
||||
const req = m.requests[0]!;
|
||||
expect(req.headers["x-api-key"]).toBe("APIKEYSECRET");
|
||||
expect(req.headers.cookie).toBe("c=COOKIESECRET");
|
||||
expect(JSON.parse(req.body!)).toMatchObject({ keyword: null, page_number: 0 });
|
||||
const [a, b, c] = page.items;
|
||||
expect(a).toMatchObject({
|
||||
siteReleaseId: "120202",
|
||||
title: "[DBY] Lost S06 2010 Complete 1080p Netflix WEB-DL AVC DDP5.1-DBTV",
|
||||
category: { id: "402" },
|
||||
promotion: { type: "free", downloadFactor: 0, uploadFactor: 1, expiresAt: "2026-09-30T04:00:00.000Z" },
|
||||
tags: ["官方", "中文字幕"],
|
||||
uploadedAt: "2025-02-18T11:47:56.000Z",
|
||||
detailUrl: "https://www.hddolby.com/details.php?id=120202&hit=1",
|
||||
hr: "unknown",
|
||||
});
|
||||
expect(a!.raw.category).toBe("电视剧");
|
||||
// promotion_time_type 6 → 2x(下载 1 上传 2),0000 截止无效
|
||||
expect(b!.promotion).toEqual({ type: "2x", downloadFactor: 1, uploadFactor: 2, expiresAt: null });
|
||||
expect(b!.raw.category).toBe("电影");
|
||||
expect(c!.raw.category).toBe("音乐");
|
||||
expect(c!.tags).toEqual([]);
|
||||
expect(JSON.stringify(page)).not.toContain("DOWNHASHSECRET");
|
||||
const t = withTorrent(m.fetch, (u) => u.pathname === "/download.php");
|
||||
const ad = adapterWith(def, hddolbySpider, t.fetch);
|
||||
await ad.list(1);
|
||||
await ad.downloadTarget("120202");
|
||||
expect(t.hits[0]!.url.searchParams.get("downhash")).toBe("DOWNHASHSECRET");
|
||||
});
|
||||
|
||||
it("error 字段 / HTTP 401 / 缺 API Key", async () => {
|
||||
const m = createMockFetch({ "/api/v1/torrent/search": { body: HDDOLBY_SEARCH_ERROR } });
|
||||
const e = await expectSiteError(adapterWith(def, hddolbySpider, m.fetch).list(1), "auth_invalid");
|
||||
expect(e.message).toContain("invalid api key");
|
||||
const h = createMockFetch({ "/api/v1/torrent/search": { status: 401, body: "no" } });
|
||||
await expectSiteError(adapterWith(def, hddolbySpider, h.fetch).list(1), "auth_invalid");
|
||||
await expectSiteError(adapterWith(def, hddolbySpider, m.fetch, { cookie: "c=1" }).list(1), "auth_missing");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/spider/hddolby.py @ e6820a8
|
||||
*
|
||||
* HDDolby(parser "HDDolby",`https://api.<域>/api/v1/torrent/search`,x-api-key + Cookie)。函数对照 `HddolbySpider`:
|
||||
* - `_movie_category/_tv_category/_music_category/_labels` → 同名常量
|
||||
* - `get_search_page_size` → `pageSize`(100)
|
||||
* - `__get_params` → `hddolbySearchParams`(注意 MP 的 page_number 直接传 0 起页码,原样保留)
|
||||
* - `__parse_result` → `parseHddolbyResult`;`__process_response` → `hddolbySpider.search` 内的错误分支
|
||||
* - `__get_downloadvolumefactor/__get_uploadvolumefactor` → `hddolbyDownloadVolumeFactor/hddolbyUploadVolumeFactor`
|
||||
* - `__get_download_url` → `hddolbyDownloadUrl`(含 downhash,raw/downloadUrl 中脱敏)
|
||||
* 差异:API 返回的 `hr` 字段 MP 未映射,这里同样不映射(H&R = unknown,待核验语义)。
|
||||
*/
|
||||
import type { MpMediaType } from "../spider/spider.ts";
|
||||
import { extractDomain } from "../user-parsers/common.ts";
|
||||
import { apiRequest, businessError, checkReleaseId, downloadTorrentFile, isPlainObject, siteBase } from "./common.ts";
|
||||
import type { SpecializedSiteDefinition, SpecializedSpider, SpecializedTorrent, SpiderSearchQuery } from "./types.ts";
|
||||
|
||||
export const HDDOLBY_MOVIE_CATEGORY = [401, 405];
|
||||
export const HDDOLBY_TV_CATEGORY = [402, 403, 404, 405];
|
||||
/** 音乐搜索同时覆盖高品质音频和音乐视频/演唱会分区 */
|
||||
export const HDDOLBY_MUSIC_CATEGORY = [406, 408];
|
||||
|
||||
export const HDDOLBY_LABELS: Record<string, string> = {
|
||||
gf: "官方",
|
||||
gy: "国语",
|
||||
yy: "粤语",
|
||||
ja: "日语",
|
||||
ko: "韩语",
|
||||
zz: "中文字幕",
|
||||
jz: "禁转",
|
||||
xz: "限转",
|
||||
diy: "DIY",
|
||||
sf: "首发",
|
||||
yq: "应求",
|
||||
m0: "零魔",
|
||||
yc: "原创",
|
||||
gz: "官字",
|
||||
db: "Dolby Vision",
|
||||
hdr10: "HDR10",
|
||||
hdrm: "HDR10+",
|
||||
tx: "特效",
|
||||
lz: "连载",
|
||||
wj: "完结",
|
||||
hdrv: "HDR Vivid",
|
||||
hlg: "HLG",
|
||||
hq: "高码率",
|
||||
hfr: "高帧率",
|
||||
};
|
||||
|
||||
export function hddolbyDownloadVolumeFactor(discount: unknown): number {
|
||||
const map: Record<number, number> = { 2: 0, 5: 0.5, 6: 1, 7: 0.3 };
|
||||
return discount ? (map[Number(discount)] ?? 1) : 1;
|
||||
}
|
||||
|
||||
export function hddolbyUploadVolumeFactor(discount: unknown): number {
|
||||
const map: Record<number, number> = { 3: 2, 4: 2, 6: 2 };
|
||||
return discount ? (map[Number(discount)] ?? 1) : 1;
|
||||
}
|
||||
|
||||
export function hddolbyApiBase(def: SpecializedSiteDefinition): string {
|
||||
return (def.umt.apiBaseUrl ?? `https://api.${extractDomain(def.domain)}`).replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
/** MP `__get_params` */
|
||||
export function hddolbySearchParams(keyword: string | null | undefined, mtype?: SpiderSearchQuery["mtype"], page = 0) {
|
||||
let categories: number[];
|
||||
if (mtype === "tv") categories = HDDOLBY_TV_CATEGORY;
|
||||
else if (mtype === "movie") categories = HDDOLBY_MOVIE_CATEGORY;
|
||||
else if (mtype === "music") categories = HDDOLBY_MUSIC_CATEGORY;
|
||||
// list(set(movie + tv)):CPython 小整数集合的迭代顺序即升序
|
||||
else categories = [...new Set([...HDDOLBY_MOVIE_CATEGORY, ...HDDOLBY_TV_CATEGORY])].sort((a, b) => a - b);
|
||||
return { keyword: keyword ?? null, page_number: page, page_size: 100, categories, visible: 1 };
|
||||
}
|
||||
|
||||
/** MP `__get_download_url` */
|
||||
export function hddolbyDownloadUrl(def: SpecializedSiteDefinition, id: unknown, downhash: unknown): string {
|
||||
return `${siteBase(def)}download.php?id=${id}&downhash=${downhash}`;
|
||||
}
|
||||
|
||||
export interface HddolbyItem {
|
||||
id: number | string;
|
||||
promotion_time_type?: number;
|
||||
promotion_until?: string;
|
||||
category?: number;
|
||||
name?: string;
|
||||
small_descr?: string;
|
||||
times_completed?: number;
|
||||
size?: number;
|
||||
added?: string;
|
||||
seeders?: number;
|
||||
leechers?: number;
|
||||
downhash?: string;
|
||||
tags?: string | null;
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
/** MP `__parse_result` */
|
||||
export function parseHddolbyResult(results: HddolbyItem[] | null | undefined, def: SpecializedSiteDefinition) {
|
||||
const torrents: SpecializedTorrent[] = [];
|
||||
if (!results?.length) return torrents;
|
||||
for (const result of results) {
|
||||
const cv = result.category;
|
||||
let category: MpMediaType;
|
||||
if (cv != null && HDDOLBY_MUSIC_CATEGORY.includes(cv)) category = "音乐";
|
||||
else if (cv != null && HDDOLBY_TV_CATEGORY.includes(cv)) category = "电视剧";
|
||||
else if (cv != null && HDDOLBY_MOVIE_CATEGORY.includes(cv)) category = "电影";
|
||||
else category = "未知";
|
||||
const labels: string[] = [];
|
||||
for (const labelId of String(result.tags ?? "").split(";")) {
|
||||
const l = HDDOLBY_LABELS[labelId];
|
||||
if (l !== undefined) labels.push(l);
|
||||
}
|
||||
const t: SpecializedTorrent = {
|
||||
title: result.name,
|
||||
description: result.small_descr,
|
||||
enclosure: hddolbyDownloadUrl(def, result.id, result.downhash),
|
||||
size: result.size,
|
||||
seeders: result.seeders,
|
||||
peers: result.leechers,
|
||||
grabs: result.times_completed,
|
||||
downloadvolumefactor: hddolbyDownloadVolumeFactor(result.promotion_time_type),
|
||||
uploadvolumefactor: hddolbyUploadVolumeFactor(result.promotion_time_type),
|
||||
page_url: `${siteBase(def)}details.php?id=${result.id}&hit=1`,
|
||||
labels,
|
||||
category,
|
||||
site_release_id: String(result.id),
|
||||
};
|
||||
if (result.added) t.pubdate = result.added;
|
||||
if (result.promotion_until) t.freedate = result.promotion_until;
|
||||
if (cv != null) t.category_id = String(cv);
|
||||
torrents.push(t);
|
||||
}
|
||||
return torrents;
|
||||
}
|
||||
|
||||
export const hddolbySpider: SpecializedSpider = {
|
||||
parser: "HDDolby",
|
||||
credential: "apiKey",
|
||||
pageSize: () => 100,
|
||||
async search(ctx, q) {
|
||||
const apikey = await ctx.secret("apiKey", true);
|
||||
const cookie = await ctx.secret("cookie");
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json, text/plain, */*",
|
||||
"x-api-key": apikey!,
|
||||
Referer: siteBase(ctx.def),
|
||||
};
|
||||
if (cookie) headers.cookie = cookie;
|
||||
const data = await apiRequest(ctx, `${hddolbyApiBase(ctx.def)}/api/v1/torrent/search`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
json: hddolbySearchParams(q.keyword, q.mtype, q.page ?? 0),
|
||||
});
|
||||
if (!isPlainObject(data)) throw businessError(ctx, "搜索", "响应结构无效");
|
||||
if (data.error) {
|
||||
const err = data.error;
|
||||
throw businessError(ctx, "搜索", isPlainObject(err) ? err.message : err);
|
||||
}
|
||||
return parseHddolbyResult(data.data as HddolbyItem[], ctx.def);
|
||||
},
|
||||
async resolveDownload(ctx, siteReleaseId, enclosure) {
|
||||
const cookie = await ctx.secret("cookie");
|
||||
const id = checkReleaseId(ctx.def.id, siteReleaseId);
|
||||
// TODO(verify):无 downhash 时是否仅凭 Cookie 可下载
|
||||
const url = enclosure ?? `${siteBase(ctx.def)}download.php?id=${id}`;
|
||||
return downloadTorrentFile(ctx, url, { cookie });
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/__init__.py @ e6820a8
|
||||
*
|
||||
* 专用 spider 汇总:`SPECIALIZED_SPIDERS` = MP `SPIDER_PARSER_CLASSES`。
|
||||
*/
|
||||
import type { SiteAdapterDeps } from "../deps.ts";
|
||||
import type { SiteAdapter } from "../types.ts";
|
||||
import { createSpecializedSiteAdapter } from "./adapter.ts";
|
||||
import { haidanSpider } from "./haidan.ts";
|
||||
import { hddolbySpider } from "./hddolby.ts";
|
||||
import { mtorrentSpider } from "./mtorrent.ts";
|
||||
import { rousiSpider } from "./rousi.ts";
|
||||
import { sunnyptSpider } from "./sunnypt.ts";
|
||||
import { tnodeSpider } from "./tnode.ts";
|
||||
import { torrentleechSpider } from "./torrentleech.ts";
|
||||
import type { SpecializedParser, SpecializedSiteDefinition, SpecializedSpider } from "./types.ts";
|
||||
import { yemaSpider } from "./yema.ts";
|
||||
|
||||
export const SPECIALIZED_SPIDERS: Record<SpecializedParser, SpecializedSpider> = {
|
||||
TNodeSpider: tnodeSpider,
|
||||
TorrentLeech: torrentleechSpider,
|
||||
mTorrent: mtorrentSpider,
|
||||
Yema: yemaSpider,
|
||||
Haidan: haidanSpider,
|
||||
HDDolby: hddolbySpider,
|
||||
RousiPro: rousiSpider,
|
||||
SunnyPT: sunnyptSpider,
|
||||
};
|
||||
|
||||
export function isSpecializedParser(v: unknown): v is SpecializedParser {
|
||||
return typeof v === "string" && Object.hasOwn(SPECIALIZED_SPIDERS, v);
|
||||
}
|
||||
|
||||
/** 按定义的 parser 键选择专用 spider 创建适配器 */
|
||||
export function createSpecializedAdapterFor(def: SpecializedSiteDefinition, deps: SiteAdapterDeps): SiteAdapter {
|
||||
const spider = SPECIALIZED_SPIDERS[def.parser];
|
||||
if (!spider) throw new Error(`未知专用 spider:${def.parser}`);
|
||||
return createSpecializedSiteAdapter(def, spider, deps);
|
||||
}
|
||||
|
||||
export { createSpecializedSiteAdapter } from "./adapter.ts";
|
||||
export {
|
||||
apiRequest,
|
||||
classifyMessage,
|
||||
downloadTorrentFile,
|
||||
formatTimestampTz,
|
||||
type IndirectRequestConfig,
|
||||
normalizeIndirectDownloadUrl,
|
||||
pyQuote,
|
||||
resolveIndirectDownloadUrl,
|
||||
} from "./common.ts";
|
||||
export * from "./haidan.ts";
|
||||
export * from "./hddolby.ts";
|
||||
export { releaseIdFromPageUrl, specializedTorrentToRelease } from "./map.ts";
|
||||
export * from "./mtorrent.ts";
|
||||
export * from "./rousi.ts";
|
||||
export * from "./sunnypt.ts";
|
||||
export * from "./tnode.ts";
|
||||
export * from "./torrentleech.ts";
|
||||
export type {
|
||||
SpecializedParser,
|
||||
SpecializedSiteDefinition,
|
||||
SpecializedSpider,
|
||||
SpecializedTorrent,
|
||||
SpiderContext,
|
||||
SpiderSearchQuery,
|
||||
} from "./types.ts";
|
||||
export * from "./yema.ts";
|
||||
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/__init__.py @ e6820a8
|
||||
*
|
||||
* 专用 spider 输出(MP TorrentInfo 字段)→ 统一 `SiteRelease`。
|
||||
* 促销类型由 MP 系数组合推出(复用 spider/map.ts `promotionFromFactors`);H&R 未提供时为 unknown;
|
||||
* raw 为完整 MP 字段,enclosure 脱敏(HDDolby downhash、Haidan passkey 等)。
|
||||
*/
|
||||
import { redactUrl } from "../../rss/redact.ts";
|
||||
import { promotionFromFactors } from "../spider/map.ts";
|
||||
import type { SiteRelease } from "../types.ts";
|
||||
import { parseSiteTime } from "../util.ts";
|
||||
import { EXTRA_SECRET_KEYS } from "./common.ts";
|
||||
import type { SpecializedSiteDefinition, SpecializedTorrent } from "./types.ts";
|
||||
|
||||
function num(v: unknown): number | null {
|
||||
if (v == null || v === "") return null;
|
||||
const n = typeof v === "number" ? v : Number(v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
|
||||
function factor(v: unknown): number | undefined {
|
||||
const n = num(v);
|
||||
return n === null ? undefined : n;
|
||||
}
|
||||
|
||||
/** 从详情链接取资源 ID:torrent_id= → id= → 末段数字(含 #/torrent/detail/1/ 形式) */
|
||||
export function releaseIdFromPageUrl(url: string | undefined): string | null {
|
||||
if (!url) return null;
|
||||
return (
|
||||
/[?&]torrent_id=(\d+)/.exec(url)?.[1] ??
|
||||
/[?&]id=(\d+)/.exec(url)?.[1] ??
|
||||
/\/(\d+)\/?(?:[?#].*)?$/.exec(url)?.[1] ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
export function specializedCategoryName(def: SpecializedSiteDefinition, id: string | undefined): string | null {
|
||||
if (!id) return null;
|
||||
const named = def.umt.categoryNames?.[id];
|
||||
if (named) return named;
|
||||
for (const k of ["movie", "tv", "music"] as const) {
|
||||
const hit = (def.category?.[k] ?? []).find((c) => String(c.id) === id);
|
||||
if (hit?.desc) return hit.desc;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function specializedTorrentToRelease(
|
||||
t: SpecializedTorrent,
|
||||
def: SpecializedSiteDefinition,
|
||||
observedAt: string,
|
||||
): SiteRelease | null {
|
||||
const id = t.site_release_id ?? releaseIdFromPageUrl(t.page_url);
|
||||
if (!id || !t.title) return null;
|
||||
const tz = def.umt.timezone;
|
||||
const freedate = t.freedate_utc ?? (t.freedate ? parseSiteTime(t.freedate, tz) : null);
|
||||
const raw: Record<string, unknown> = { ...t };
|
||||
if (t.enclosure) raw.enclosure = redactUrl(t.enclosure, EXTRA_SECRET_KEYS);
|
||||
const release: SiteRelease = {
|
||||
siteId: def.id,
|
||||
siteReleaseId: id,
|
||||
title: String(t.title).trim(),
|
||||
category: { id: t.category_id ?? null, name: specializedCategoryName(def, t.category_id) },
|
||||
sizeBytes: num(t.size),
|
||||
seeders: num(t.seeders),
|
||||
leechers: num(t.peers),
|
||||
snatched: num(t.grabs),
|
||||
promotion: promotionFromFactors(factor(t.downloadvolumefactor), factor(t.uploadvolumefactor), freedate),
|
||||
hr: t.hit_and_run === undefined ? "unknown" : t.hit_and_run,
|
||||
uploadedAt: t.pubdate_utc ?? parseSiteTime(t.pubdate, tz),
|
||||
detailUrl: t.page_url ?? `${def.domain}`,
|
||||
tags: t.labels ?? [],
|
||||
raw,
|
||||
observedAt,
|
||||
};
|
||||
if (t.enclosure) release.downloadUrl = redactUrl(t.enclosure, EXTRA_SECRET_KEYS);
|
||||
if (t.description) release.subtitle = String(t.description).trim();
|
||||
if (t.imdbid) release.imdbId = t.imdbid;
|
||||
return release;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/spider/mtorrent.py @ e6820a8
|
||||
*
|
||||
* `mTorrent` 键的专用 spider 形态,复用 W1-3b 已移植的 mteam/spider.ts(`__get_params`/`__parse_result`)
|
||||
* 与 mteam/client.ts(`__process_response` + 业务码检查),供 `SPECIALIZED_SPIDERS` 统一分派。
|
||||
* 完整适配(详情、商城免费、healthCheck=profile)仍以 `createMteamAdapter` 为准。
|
||||
*/
|
||||
import { mteamPost } from "../mteam/client.ts";
|
||||
import { rootDomain } from "../mteam/definition.ts";
|
||||
import { type MteamTorrentJson, mteamSearchParams, parseMteamResult } from "../mteam/spider.ts";
|
||||
import { SiteError } from "../types.ts";
|
||||
import { checkReleaseId, downloadTorrentFile, siteBase } from "./common.ts";
|
||||
import type { SpecializedSiteDefinition, SpecializedSpider, SpecializedTorrent } from "./types.ts";
|
||||
|
||||
export function mtorrentApiBase(def: SpecializedSiteDefinition): string {
|
||||
return (def.umt.apiBaseUrl ?? `https://api.${rootDomain(def.domain)}`).replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
export const mtorrentSpider: SpecializedSpider = {
|
||||
parser: "mTorrent",
|
||||
credential: "apiKey",
|
||||
pageSize: () => 100,
|
||||
async search(ctx, q) {
|
||||
const apikey = (await ctx.secret("apiKey", true))!;
|
||||
const params = mteamSearchParams(q.keyword ?? null, q.mtype, q.page ?? 0);
|
||||
const data = await mteamPost<{ data?: MteamTorrentJson[] | null }>(
|
||||
ctx.http,
|
||||
mtorrentApiBase(ctx.def),
|
||||
apikey,
|
||||
"/api/torrent/search",
|
||||
{ json: params },
|
||||
ctx.signal,
|
||||
);
|
||||
const src = data?.data ?? [];
|
||||
return parseMteamResult(src, siteBase(ctx.def)).map(
|
||||
(t, i): SpecializedTorrent => ({ ...t, site_release_id: String(src[i]!.id) }),
|
||||
);
|
||||
},
|
||||
async resolveDownload(ctx, siteReleaseId) {
|
||||
const apikey = (await ctx.secret("apiKey", true))!;
|
||||
const id = checkReleaseId(ctx.def.id, siteReleaseId);
|
||||
const url = await mteamPost<string | null>(
|
||||
ctx.http,
|
||||
mtorrentApiBase(ctx.def),
|
||||
apikey,
|
||||
"/api/torrent/genDlToken",
|
||||
{ query: { id } },
|
||||
ctx.signal,
|
||||
);
|
||||
if (!url || typeof url !== "string")
|
||||
throw new SiteError(ctx.def.id, "rate_limited", `genDlToken ${id}:未返回下载链接`);
|
||||
// 涉及解析地址的不使用 Cookie 下载
|
||||
return downloadTorrentFile(ctx, url);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,54 @@
|
||||
import { createMockFetch, ROUSI_DETAIL_OK, ROUSI_SEARCH_ERROR, ROUSI_SEARCH_OK } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { rousiCategoryNamesByIds, rousiSearchParams, rousiSpider } from "./rousi.ts";
|
||||
import { adapterWith, expectSiteError, testDef, withTorrent } from "./testing.ts";
|
||||
|
||||
const def = testDef("RousiPro", { domain: "https://rousi.pro/" });
|
||||
|
||||
describe("RousiSpider(MP rousi.py)", () => {
|
||||
it("__get_params:页码 +1、cat 映射只取第一个、否则按 mtype", () => {
|
||||
expect(rousiCategoryNamesByIds("3, 6,99")).toEqual(["documentary", "variety"]);
|
||||
expect(rousiSearchParams({ keyword: "x", cat: "6,1", page: 0 })).toEqual({
|
||||
page: 1,
|
||||
page_size: 100,
|
||||
keyword: "x",
|
||||
category: "variety",
|
||||
});
|
||||
expect(rousiSearchParams({ mtype: "tv", page: 2 })).toEqual({ page: 3, page_size: 100, category: "tv" });
|
||||
});
|
||||
|
||||
it("列表解析:Bearer、促销 is_active、带时区时间转 UTC、分类 slug", async () => {
|
||||
const m = createMockFetch({ "GET https://rousi.pro/api/v1/torrents": { body: ROUSI_SEARCH_OK } });
|
||||
const page = await adapterWith(def, rousiSpider, m.fetch).search("bear", { mtype: "tv" });
|
||||
expect(m.requests[0]!.headers.authorization).toBe("Bearer APIKEYSECRET");
|
||||
expect(m.requests[0]!.url.searchParams.get("category")).toBe("tv");
|
||||
const [a, b, c] = page.items;
|
||||
expect(a).toMatchObject({
|
||||
siteReleaseId: "501",
|
||||
category: { id: "tv" },
|
||||
sizeBytes: 50000000000,
|
||||
snatched: 77,
|
||||
promotion: { type: "2xfree", downloadFactor: 0, uploadFactor: 2, expiresAt: "2026-09-27T16:00:00.000Z" },
|
||||
uploadedAt: "2026-09-20T02:00:00.000Z",
|
||||
detailUrl: "https://rousi.pro/torrents/501",
|
||||
});
|
||||
expect(a!.raw.category).toBe("电视剧");
|
||||
// 未激活促销 → 1/1
|
||||
expect(b!.promotion).toMatchObject({ type: "none", downloadFactor: 1, uploadFactor: 1 });
|
||||
expect(b!.uploadedAt).toBe("2026-09-19T02:00:00.000Z");
|
||||
expect(c!.raw.category).toBe("未知");
|
||||
});
|
||||
|
||||
it("业务错误 / 缺 Key / 下载经详情接口换取 download_url,直链不带凭据", async () => {
|
||||
const bad = createMockFetch({ "/api/v1/torrents": { body: ROUSI_SEARCH_ERROR } });
|
||||
await expectSiteError(adapterWith(def, rousiSpider, bad.fetch).list(1), "auth_invalid");
|
||||
await expectSiteError(adapterWith(def, rousiSpider, bad.fetch, {}).list(1), "auth_missing");
|
||||
const m = createMockFetch({ "GET https://rousi.pro/api/v1/torrents/501": { body: ROUSI_DETAIL_OK } });
|
||||
const t = withTorrent(m.fetch, (u) => u.host === "dl.rousi.invalid");
|
||||
const target = await adapterWith(def, rousiSpider, t.fetch).downloadTarget("501");
|
||||
expect(target).toMatchObject({ kind: "torrent", infoHash: t.infoHash });
|
||||
expect(m.requests[0]!.headers.authorization).toBe("Bearer APIKEYSECRET");
|
||||
expect(t.hits[0]!.headers.get("authorization")).toBeNull();
|
||||
expect(t.hits[0]!.headers.get("cookie")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,175 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/spider/rousi.py @ e6820a8
|
||||
*
|
||||
* Rousi.pro PeerGo 兼容 API(parser "RousiPro",Bearer 个人 API Key,需 torrent:read + torrent:download)。
|
||||
* 函数对照 `RousiSpider`:
|
||||
* - `_movie_category/_tv_category/_music_category` → `ROUSI_*_CATEGORY`
|
||||
* - `get_search_page_size` → `pageSize`(100)
|
||||
* - `__get_params` → `rousiSearchParams`;`__get_category_names_by_ids` → `rousiCategoryNamesByIds`
|
||||
* - `__process_response` → `rousiSpider.search` 错误分支;`__parse_result` → `parseRousiResult`
|
||||
* - `__get_download_url`(GET 详情接口,result=data.download_url)→ `rousiSpider.resolveDownload`(现场换票,不生成含 key 的 enclosure)
|
||||
*/
|
||||
import type { MpMediaType } from "../spider/spider.ts";
|
||||
import { extractDomain, normalizeDatetime } from "../user-parsers/common.ts";
|
||||
import {
|
||||
apiRequest,
|
||||
businessError,
|
||||
checkReleaseId,
|
||||
downloadTorrentFile,
|
||||
isoWithZoneToUtc,
|
||||
isPlainObject,
|
||||
resolveIndirectDownloadUrl,
|
||||
toFloat,
|
||||
toInt,
|
||||
} from "./common.ts";
|
||||
import type { SpecializedSiteDefinition, SpecializedSpider, SpecializedTorrent, SpiderSearchQuery } from "./types.ts";
|
||||
|
||||
/** API 不支持多分类搜索,每次只使用一个分类 */
|
||||
export const ROUSI_MOVIE_CATEGORY = "movie";
|
||||
export const ROUSI_TV_CATEGORY = "tv";
|
||||
export const ROUSI_MUSIC_CATEGORY = "music";
|
||||
|
||||
const ROUSI_ID_TO_NAME: Record<string, string> = {
|
||||
"1": "movie",
|
||||
"2": "tv",
|
||||
"3": "documentary",
|
||||
"4": "animation",
|
||||
"5": "music",
|
||||
"6": "variety",
|
||||
};
|
||||
|
||||
/** 站点 origin(MP `https://%s` % extract_domain(domain)) */
|
||||
export function rousiBase(def: SpecializedSiteDefinition): string {
|
||||
return (def.umt.apiBaseUrl ?? `https://${extractDomain(def.domain)}`).replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
/** MP `__get_category_names_by_ids` */
|
||||
export function rousiCategoryNamesByIds(cat: string | null | undefined): string[] | null {
|
||||
if (!cat) return null;
|
||||
const ids = cat
|
||||
.split(",")
|
||||
.map((c) => c.trim())
|
||||
.filter(Boolean);
|
||||
const names = ids.filter((id) => id in ROUSI_ID_TO_NAME).map((id) => ROUSI_ID_TO_NAME[id]!);
|
||||
return names.length ? names : null;
|
||||
}
|
||||
|
||||
/** MP `__get_params` */
|
||||
export function rousiSearchParams(q: SpiderSearchQuery): Record<string, string | number> {
|
||||
const params: Record<string, string | number> = { page: Number(q.page ?? 0) + 1, page_size: 100 };
|
||||
if (q.keyword) params.keyword = q.keyword;
|
||||
if (q.cat) {
|
||||
const names = rousiCategoryNamesByIds(q.cat);
|
||||
if (names) params.category = names[0]!;
|
||||
} else if (q.mtype) {
|
||||
if (q.mtype === "movie") params.category = ROUSI_MOVIE_CATEGORY;
|
||||
else if (q.mtype === "tv") params.category = ROUSI_TV_CATEGORY;
|
||||
else if (q.mtype === "music") params.category = ROUSI_MUSIC_CATEGORY;
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
export interface RousiItem {
|
||||
id: number | string;
|
||||
title?: string;
|
||||
subtitle?: string;
|
||||
category?: string | { slug?: string; name?: string } | null;
|
||||
size?: number | string;
|
||||
seeders?: number | string;
|
||||
leechers?: number | string;
|
||||
downloads?: number | string;
|
||||
created_at?: string;
|
||||
promotion?: { is_active?: boolean; down_multiplier?: number; up_multiplier?: number; until?: string } | null;
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
/** MP `__parse_result` */
|
||||
export function parseRousiResult(results: RousiItem[] | null | undefined, def: SpecializedSiteDefinition) {
|
||||
const torrents: SpecializedTorrent[] = [];
|
||||
if (!results?.length) return torrents;
|
||||
const base = rousiBase(def);
|
||||
for (const result of results) {
|
||||
const rawCat = result.category;
|
||||
let catVal: string | null | undefined = null;
|
||||
let category: MpMediaType = "未知";
|
||||
if (rawCat && typeof rawCat === "object") catVal = rawCat.slug || rawCat.name;
|
||||
else if (typeof rawCat === "string") catVal = rawCat;
|
||||
if (catVal) {
|
||||
catVal = String(catVal).toLowerCase();
|
||||
if (catVal === ROUSI_MOVIE_CATEGORY) category = "电影";
|
||||
else if (catVal === ROUSI_TV_CATEGORY) category = "电视剧";
|
||||
else if (catVal === ROUSI_MUSIC_CATEGORY) category = "音乐";
|
||||
}
|
||||
// API 后端已处理全站促销优先级,直接使用返回的 promotion 数据
|
||||
let dl = 1.0;
|
||||
let ul = 1.0;
|
||||
let freedate: string | undefined;
|
||||
let freedateUtc: string | undefined;
|
||||
const promotion = result.promotion;
|
||||
if (promotion?.is_active) {
|
||||
dl = toFloat(promotion.down_multiplier ?? 1.0);
|
||||
ul = toFloat(promotion.up_multiplier ?? 1.0);
|
||||
if (promotion.until) {
|
||||
freedate = normalizeDatetime(promotion.until) ?? undefined;
|
||||
freedateUtc = isoWithZoneToUtc(promotion.until);
|
||||
}
|
||||
}
|
||||
const id = result.id;
|
||||
const t: SpecializedTorrent = {
|
||||
title: result.title,
|
||||
description: result.subtitle,
|
||||
size: toInt(result.size),
|
||||
seeders: toInt(result.seeders),
|
||||
peers: toInt(result.leechers),
|
||||
grabs: toInt(result.downloads),
|
||||
downloadvolumefactor: dl,
|
||||
uploadvolumefactor: ul,
|
||||
page_url: `${base}/torrents/${id}`,
|
||||
labels: [],
|
||||
category,
|
||||
site_release_id: String(id),
|
||||
};
|
||||
const pubdate = normalizeDatetime(result.created_at);
|
||||
if (pubdate) t.pubdate = pubdate;
|
||||
const pubUtc = isoWithZoneToUtc(result.created_at);
|
||||
if (pubUtc) t.pubdate_utc = pubUtc;
|
||||
if (freedate) t.freedate = freedate;
|
||||
if (freedateUtc) t.freedate_utc = freedateUtc;
|
||||
if (catVal) t.category_id = catVal;
|
||||
torrents.push(t);
|
||||
}
|
||||
return torrents;
|
||||
}
|
||||
|
||||
export const rousiSpider: SpecializedSpider = {
|
||||
parser: "RousiPro",
|
||||
credential: "apiKey",
|
||||
pageSize: () => 100,
|
||||
async search(ctx, q) {
|
||||
const apikey = await ctx.secret("apiKey", true);
|
||||
const data = await apiRequest(ctx, `${rousiBase(ctx.def)}/api/v1/torrents`, {
|
||||
method: "GET",
|
||||
headers: { Authorization: `Bearer ${apikey}`, Accept: "application/json" },
|
||||
query: rousiSearchParams(q),
|
||||
});
|
||||
if (!isPlainObject(data)) throw businessError(ctx, "搜索", "搜索响应结构无效");
|
||||
if (data.code === 0) {
|
||||
const d = isPlainObject(data.data) ? data.data : {};
|
||||
return parseRousiResult((d.torrents as RousiItem[]) ?? [], ctx.def);
|
||||
}
|
||||
throw businessError(ctx, "搜索", data.message);
|
||||
},
|
||||
async resolveDownload(ctx, siteReleaseId) {
|
||||
const apikey = await ctx.secret("apiKey", true);
|
||||
const id = checkReleaseId(ctx.def.id, siteReleaseId);
|
||||
// MP __get_download_url:method=get, cookie=False, Bearer, result=data.download_url
|
||||
const url = await resolveIndirectDownloadUrl(ctx, `${rousiBase(ctx.def)}/api/v1/torrents/${id}`, {
|
||||
method: "get",
|
||||
header: { Authorization: `Bearer ${apikey}`, Accept: "application/json" },
|
||||
result: "data.download_url",
|
||||
});
|
||||
return downloadTorrentFile(ctx, url);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,71 @@
|
||||
import {
|
||||
createMockFetch,
|
||||
SUNNYPT_CATEGORIES_OK,
|
||||
SUNNYPT_DOWNLOAD_TOKEN_OK,
|
||||
SUNNYPT_SEARCH_ERROR,
|
||||
SUNNYPT_SEARCH_OK,
|
||||
} from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { sunnyptBuildParams, sunnyptParseCategoryItems, sunnyptSpider } from "./sunnypt.ts";
|
||||
import { adapterWith, expectSiteError, testDef, withTorrent } from "./testing.ts";
|
||||
|
||||
const def = testDef("SunnyPT", {
|
||||
domain: "https://sunnypt.top/",
|
||||
api_url: "https://api.sunnypt.top/api/v1/mp",
|
||||
category: { movie: [{ id: 401 }], tv: [{ id: 402 }] },
|
||||
});
|
||||
|
||||
describe("SunnyPTSpider(MP sunnypt.py)", () => {
|
||||
it("_parse_category_items / _build_params", () => {
|
||||
const map = sunnyptParseCategoryItems(SUNNYPT_CATEGORIES_OK.data);
|
||||
expect(map).toEqual({ movie: ["401", "405"], tv: ["402", "405"], music: ["406"] });
|
||||
expect(sunnyptBuildParams({ keyword: "x", mtype: "tv", page: 1 }, map)).toEqual({
|
||||
page: 2,
|
||||
page_size: 100,
|
||||
sort: "created_at",
|
||||
order: "desc",
|
||||
keyword: "x",
|
||||
media_type: "tv",
|
||||
categories: "402,405",
|
||||
});
|
||||
});
|
||||
|
||||
it("列表解析:分类接口缓存、促销、H&R 布尔、标签", async () => {
|
||||
const m = createMockFetch({
|
||||
"GET https://api.sunnypt.top/api/v1/mp/categories": { body: SUNNYPT_CATEGORIES_OK },
|
||||
"GET https://api.sunnypt.top/api/v1/mp/torrents": { body: SUNNYPT_SEARCH_OK },
|
||||
});
|
||||
const a = adapterWith(def, sunnyptSpider, m.fetch);
|
||||
const page = await a.search("arcane", { mtype: "tv" });
|
||||
await a.search("arcane", { mtype: "tv" });
|
||||
expect(m.requests.filter((r) => r.url.pathname.endsWith("/categories"))).toHaveLength(1);
|
||||
expect(m.requests[0]!.headers["x-api-key"]).toBe("APIKEYSECRET");
|
||||
expect(m.requests[1]!.url.searchParams.get("categories")).toBe("402,405");
|
||||
const [x, y] = page.items;
|
||||
expect(x).toMatchObject({
|
||||
siteReleaseId: "7001",
|
||||
category: { id: "402" },
|
||||
promotion: { type: "free", downloadFactor: 0, uploadFactor: 1, expiresAt: "2026-09-30T15:59:59.000Z" },
|
||||
hr: true,
|
||||
tags: ["官方", "中字"],
|
||||
imdbId: "tt11126994",
|
||||
detailUrl: "https://sunnypt.top/details.php?id=7001",
|
||||
});
|
||||
expect(y).toMatchObject({ hr: false, promotion: { type: "none" }, sizeBytes: 9000000000 });
|
||||
});
|
||||
|
||||
it("业务错误 / 分类接口失败回退配置 / 下载 token 相对地址按 API 基址补全", async () => {
|
||||
const bad = createMockFetch({ "/api/v1/mp/torrents": { body: SUNNYPT_SEARCH_ERROR } });
|
||||
await expectSiteError(adapterWith(def, sunnyptSpider, bad.fetch).list(1), "auth_invalid");
|
||||
const fb = createMockFetch({
|
||||
"/api/v1/mp/categories": { status: 500, body: "x" },
|
||||
"/api/v1/mp/torrents": { body: SUNNYPT_SEARCH_OK },
|
||||
});
|
||||
await adapterWith(def, sunnyptSpider, fb.fetch).search("x", { mtype: "movie" });
|
||||
expect(fb.requests.at(-1)!.url.searchParams.get("categories")).toBe("401");
|
||||
const d = createMockFetch({ "POST /api/v1/mp/torrents/7001/download-token": { body: SUNNYPT_DOWNLOAD_TOKEN_OK } });
|
||||
const t = withTorrent(d.fetch, (u) => u.pathname.endsWith("/download"));
|
||||
await adapterWith(def, sunnyptSpider, t.fetch).downloadTarget("7001");
|
||||
expect(t.hits[0]!.url.toString()).toBe("https://api.sunnypt.top/api/v1/mp/torrents/7001/download?token=TOK");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,225 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/spider/sunnypt.py @ e6820a8
|
||||
*
|
||||
* SunnyPT MoviePilot API(parser "SunnyPT",X-API-Key)。函数对照 `SunnyPTSpider`:
|
||||
* - `_size/_category_cache_ttl` → `SUNNYPT_PAGE_SIZE/SUNNYPT_CATEGORY_CACHE_TTL_MS`
|
||||
* - `_parse_configured_categories` → `sunnyptConfiguredCategories`;`_parse_category_items` → `sunnyptParseCategoryItems`
|
||||
* - `_get_cached_categories/_set_cached_categories`(类级缓存)→ 适配器实例缓存 `ctx.cache`
|
||||
* - `_request_headers` → `sunnyptHeaders`;`_response_data` → `sunnyptResponseData`
|
||||
* - `_load_categories/_async_load_categories` → `sunnyptLoadCategories`
|
||||
* - `_media_type_value` → `mtype` 直用;`_build_params` → `sunnyptBuildParams`;`_parse_result` → `parseSunnyptResult`
|
||||
* - `_process_search_response` + `search` → `sunnyptSpider.search`
|
||||
* - `_build_download_url`(POST download-token,result=data.download_url,result_base_url=api_url)→ `resolveDownload`
|
||||
* 差异:分类接口失败时与 MP 一样回退站点配置分类,但凭据错误(HTTP 401/403)直接抛出,不再静默。
|
||||
*/
|
||||
import type { MpMediaType } from "../spider/spider.ts";
|
||||
import { SiteError } from "../types.ts";
|
||||
import { normalizeDatetime } from "../user-parsers/common.ts";
|
||||
import {
|
||||
apiRequest,
|
||||
businessError,
|
||||
cacheGet,
|
||||
cacheSet,
|
||||
checkReleaseId,
|
||||
downloadTorrentFile,
|
||||
isoWithZoneToUtc,
|
||||
isPlainObject,
|
||||
resolveIndirectDownloadUrl,
|
||||
toFloat,
|
||||
toInt,
|
||||
} from "./common.ts";
|
||||
import type {
|
||||
SpecializedSiteDefinition,
|
||||
SpecializedSpider,
|
||||
SpecializedTorrent,
|
||||
SpiderContext,
|
||||
SpiderSearchQuery,
|
||||
} from "./types.ts";
|
||||
|
||||
export const SUNNYPT_PAGE_SIZE = 100;
|
||||
export const SUNNYPT_CATEGORY_CACHE_TTL_MS = 3600 * 1000;
|
||||
|
||||
export type SunnyptCategoryMap = { movie: string[]; tv: string[]; music: string[] };
|
||||
|
||||
export function sunnyptApiUrl(def: SpecializedSiteDefinition): string {
|
||||
return String(def.umt.apiBaseUrl ?? def.api_url ?? "https://api.sunnypt.top/api/v1/mp").replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
/** MP `_parse_configured_categories` */
|
||||
export function sunnyptConfiguredCategories(config: SpecializedSiteDefinition["category"]): SunnyptCategoryMap {
|
||||
const map: SunnyptCategoryMap = { movie: [], tv: [], music: [] };
|
||||
for (const k of ["movie", "tv", "music"] as const) {
|
||||
for (const item of config?.[k] ?? []) {
|
||||
const id = typeof item === "object" ? item.id : item;
|
||||
if (id != null) map[k].push(String(id));
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/** MP `_parse_category_items` */
|
||||
export function sunnyptParseCategoryItems(items: unknown[]): SunnyptCategoryMap {
|
||||
const map: SunnyptCategoryMap = { movie: [], tv: [], music: [] };
|
||||
for (const item of items ?? []) {
|
||||
if (!isPlainObject(item) || item.id == null) continue;
|
||||
const id = String(item.id);
|
||||
for (const mt of (item.media_types as unknown[]) ?? []) {
|
||||
if (mt === "movie" || mt === "tv" || mt === "music") {
|
||||
if (!map[mt].includes(id)) map[mt].push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
function sunnyptHeaders(apikey: string): Record<string, string> {
|
||||
return { Accept: "application/json", "X-API-Key": apikey };
|
||||
}
|
||||
|
||||
/** MP `_response_data`:code 不为 "0" → 失败 */
|
||||
function sunnyptResponseData(ctx: SpiderContext, payload: unknown, operation: string): unknown {
|
||||
if (!isPlainObject(payload)) throw businessError(ctx, operation, "响应结构无效");
|
||||
if (String(payload.code) !== "0") throw businessError(ctx, operation, payload.msg || "未知错误");
|
||||
return payload.data;
|
||||
}
|
||||
|
||||
/** MP `_load_categories`:读取并缓存分类映射,失败回退站点配置 */
|
||||
export async function sunnyptLoadCategories(ctx: SpiderContext, apikey: string): Promise<SunnyptCategoryMap> {
|
||||
const api = sunnyptApiUrl(ctx.def);
|
||||
const cached = cacheGet<SunnyptCategoryMap>(ctx, `sunnypt:categories:${api}`);
|
||||
if (cached) return cached;
|
||||
const configured = sunnyptConfiguredCategories(ctx.def.category);
|
||||
let items: unknown;
|
||||
try {
|
||||
items = sunnyptResponseData(
|
||||
ctx,
|
||||
await apiRequest(ctx, `${api}/categories`, { method: "GET", headers: sunnyptHeaders(apikey) }),
|
||||
"获取分类",
|
||||
);
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError && e.isAuthError) throw e;
|
||||
return configured;
|
||||
}
|
||||
const map = Array.isArray(items) ? sunnyptParseCategoryItems(items) : null;
|
||||
if (!map || !Object.values(map).some((v) => v.length)) return configured;
|
||||
cacheSet(ctx, `sunnypt:categories:${api}`, map, SUNNYPT_CATEGORY_CACHE_TTL_MS);
|
||||
return map;
|
||||
}
|
||||
|
||||
/** MP `_build_params` */
|
||||
export function sunnyptBuildParams(q: SpiderSearchQuery, categoryMap: SunnyptCategoryMap) {
|
||||
const params: Record<string, string | number> = {
|
||||
page: Number(q.page || 0) + 1,
|
||||
page_size: SUNNYPT_PAGE_SIZE,
|
||||
sort: "created_at",
|
||||
order: "desc",
|
||||
};
|
||||
if (q.keyword) params.keyword = q.keyword;
|
||||
const apiMediaType = q.mtype ?? null;
|
||||
if (apiMediaType) params.media_type = apiMediaType;
|
||||
let categories = q.cat;
|
||||
if (!categories && apiMediaType) categories = (categoryMap[apiMediaType] ?? []).join(",");
|
||||
if (categories) params.categories = categories;
|
||||
return params;
|
||||
}
|
||||
|
||||
export interface SunnyptItem {
|
||||
id: number | string;
|
||||
title?: string;
|
||||
subtitle?: string;
|
||||
media_type?: string;
|
||||
category_id?: number | string;
|
||||
promotion?: { is_active?: boolean; down_multiplier?: number; up_multiplier?: number; until?: string } | null;
|
||||
created_at?: string;
|
||||
size?: number | string;
|
||||
seeders?: number | string;
|
||||
leechers?: number | string;
|
||||
completed?: number | string;
|
||||
details_url?: string;
|
||||
imdb_id?: string;
|
||||
tags?: string[];
|
||||
hit_and_run?: boolean | number;
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
/** MP `_parse_result` */
|
||||
export function parseSunnyptResult(results: unknown[] | null | undefined): SpecializedTorrent[] {
|
||||
const torrents: SpecializedTorrent[] = [];
|
||||
for (const r of results ?? []) {
|
||||
if (!isPlainObject(r)) continue;
|
||||
const result = r as SunnyptItem;
|
||||
let category: MpMediaType;
|
||||
if (result.media_type === "movie") category = "电影";
|
||||
else if (result.media_type === "tv") category = "电视剧";
|
||||
else if (result.media_type === "music") category = "音乐";
|
||||
else category = "未知";
|
||||
const promotion = result.promotion ?? {};
|
||||
const active = Boolean(promotion.is_active);
|
||||
const t: SpecializedTorrent = {
|
||||
title: result.title,
|
||||
description: result.subtitle,
|
||||
size: toInt(result.size),
|
||||
seeders: toInt(result.seeders),
|
||||
peers: toInt(result.leechers),
|
||||
grabs: toInt(result.completed),
|
||||
downloadvolumefactor: active ? toFloat(promotion.down_multiplier ?? 1.0) : 1.0,
|
||||
uploadvolumefactor: active ? toFloat(promotion.up_multiplier ?? 1.0) : 1.0,
|
||||
labels: result.tags ?? [],
|
||||
hit_and_run: Boolean(result.hit_and_run),
|
||||
category,
|
||||
site_release_id: String(result.id),
|
||||
};
|
||||
if (result.details_url) t.page_url = result.details_url;
|
||||
if (result.imdb_id) t.imdbid = result.imdb_id;
|
||||
if (active && promotion.until) {
|
||||
const fd = normalizeDatetime(promotion.until);
|
||||
if (fd) t.freedate = fd;
|
||||
const fu = isoWithZoneToUtc(promotion.until);
|
||||
if (fu) t.freedate_utc = fu;
|
||||
}
|
||||
const pubdate = normalizeDatetime(result.created_at);
|
||||
if (pubdate) t.pubdate = pubdate;
|
||||
const pu = isoWithZoneToUtc(result.created_at);
|
||||
if (pu) t.pubdate_utc = pu;
|
||||
if (result.category_id != null) t.category_id = String(result.category_id);
|
||||
torrents.push(t);
|
||||
}
|
||||
return torrents;
|
||||
}
|
||||
|
||||
export const sunnyptSpider: SpecializedSpider = {
|
||||
parser: "SunnyPT",
|
||||
credential: "apiKey",
|
||||
pageSize: () => SUNNYPT_PAGE_SIZE,
|
||||
async search(ctx, q) {
|
||||
const apikey = (await ctx.secret("apiKey", true))!;
|
||||
const categoryMap =
|
||||
q.mtype && !q.cat ? await sunnyptLoadCategories(ctx, apikey) : sunnyptConfiguredCategories(ctx.def.category);
|
||||
const data = sunnyptResponseData(
|
||||
ctx,
|
||||
await apiRequest(ctx, `${sunnyptApiUrl(ctx.def)}/torrents`, {
|
||||
method: "GET",
|
||||
headers: sunnyptHeaders(apikey),
|
||||
query: sunnyptBuildParams(q, categoryMap),
|
||||
}),
|
||||
"搜索",
|
||||
);
|
||||
if (!isPlainObject(data)) throw businessError(ctx, "搜索", "data 结构无效");
|
||||
return parseSunnyptResult((data.items as unknown[]) ?? []);
|
||||
},
|
||||
async resolveDownload(ctx, siteReleaseId) {
|
||||
const apikey = (await ctx.secret("apiKey", true))!;
|
||||
const id = checkReleaseId(ctx.def.id, siteReleaseId);
|
||||
const api = sunnyptApiUrl(ctx.def);
|
||||
// MP _build_download_url:POST download-token,result=data.download_url,result_base_url=api_url
|
||||
const url = await resolveIndirectDownloadUrl(ctx, `${api}/torrents/${id}/download-token`, {
|
||||
method: "post",
|
||||
header: { "X-API-Key": apikey, Accept: "application/json" },
|
||||
result: "data.download_url",
|
||||
result_base_url: api,
|
||||
});
|
||||
return downloadTorrentFile(ctx, url);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,74 @@
|
||||
/** 测试辅助(非公开 API):构造适配器、在 mock fetch 前挂二进制 .torrent 路由。 */
|
||||
import { buildFakeTorrent } from "@umt/testkit";
|
||||
import { createSiteRateLimiter } from "../rate-limit.ts";
|
||||
import { type SiteAdapter, type SiteCredentialKind, SiteError, staticSecretResolver } from "../types.ts";
|
||||
import { createSpecializedSiteAdapter } from "./adapter.ts";
|
||||
import type { SpecializedSiteDefinition, SpecializedSpider } from "./types.ts";
|
||||
|
||||
export function testDef(
|
||||
parser: SpecializedSiteDefinition["parser"],
|
||||
over: Partial<SpecializedSiteDefinition> = {},
|
||||
): SpecializedSiteDefinition {
|
||||
return {
|
||||
id: "demo",
|
||||
name: "Demo",
|
||||
domain: "https://pt.example.com/",
|
||||
parser,
|
||||
...over,
|
||||
umt: {
|
||||
version: "[email protected]",
|
||||
kind: "api",
|
||||
displayName: "Demo",
|
||||
timezone: "+08:00",
|
||||
rateLimit: { minIntervalMs: 0, burst: 10 },
|
||||
capabilities: { list: true, search: true, detail: false, download: true, rss: false },
|
||||
...(over.umt ?? {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function adapterWith(
|
||||
def: SpecializedSiteDefinition,
|
||||
spider: SpecializedSpider,
|
||||
fetchImpl: typeof globalThis.fetch,
|
||||
secrets: Partial<Record<SiteCredentialKind, string>> = { cookie: "c=COOKIESECRET", apiKey: "APIKEYSECRET" },
|
||||
): SiteAdapter {
|
||||
return createSpecializedSiteAdapter(def, spider, {
|
||||
fetch: fetchImpl,
|
||||
secrets: staticSecretResolver({ [def.id]: secrets }),
|
||||
limiter: createSiteRateLimiter({ minIntervalMs: 0 }),
|
||||
});
|
||||
}
|
||||
|
||||
/** 包一层 fetch:命中 predicate 的请求返回合法 .torrent 字节 */
|
||||
export function withTorrent(
|
||||
inner: typeof globalThis.fetch,
|
||||
predicate: (url: URL) => boolean,
|
||||
name = "demo",
|
||||
): { fetch: typeof globalThis.fetch; infoHash: string; hits: Array<{ url: URL; headers: Headers }> } {
|
||||
const t = buildFakeTorrent({ name });
|
||||
const hits: Array<{ url: URL; headers: Headers }> = [];
|
||||
const f = (async (input: string | URL | Request, init?: RequestInit) => {
|
||||
const url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url);
|
||||
if (predicate(url)) {
|
||||
hits.push({ url, headers: new Headers(init?.headers) });
|
||||
return new Response(t.bytes as Uint8Array<ArrayBuffer>, {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/x-bittorrent" },
|
||||
});
|
||||
}
|
||||
return inner(input, init);
|
||||
}) as typeof globalThis.fetch;
|
||||
return { fetch: f, infoHash: t.infoHash, hits };
|
||||
}
|
||||
|
||||
export async function expectSiteError(p: Promise<unknown>, kind: string): Promise<SiteError> {
|
||||
try {
|
||||
await p;
|
||||
} catch (e) {
|
||||
if (!(e instanceof SiteError)) throw e;
|
||||
if (e.kind !== kind) throw new Error(`expected SiteError ${kind}, got ${e.kind}: ${e.message}`);
|
||||
return e;
|
||||
}
|
||||
throw new Error(`expected SiteError ${kind}`);
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { createMockFetch, TNODE_HOME_HTML, TNODE_HOME_NO_TOKEN, TNODE_SEARCH_OK } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { adapterWith, expectSiteError, testDef, withTorrent } from "./testing.ts";
|
||||
import { parseTnodeToken, tnodeSearchParams, tnodeSpider } from "./tnode.ts";
|
||||
|
||||
const def = testDef("TNodeSpider", { domain: "https://tnode.example/" });
|
||||
|
||||
describe("TNodeSpider(MP tnode.py)", () => {
|
||||
it("__get_params / token 解析", () => {
|
||||
expect(tnodeSearchParams("tt0923811", 1)).toMatchObject({
|
||||
page: 2,
|
||||
size: 100,
|
||||
type: "imdbid",
|
||||
keyword: "tt0923811",
|
||||
});
|
||||
expect(tnodeSearchParams(null)).toMatchObject({ type: "title", keyword: "", category: [501, 502, 503, 504] });
|
||||
expect(parseTnodeToken(TNODE_HOME_HTML)).toBe("CSRF-TOKEN-XYZ");
|
||||
});
|
||||
|
||||
it("列表解析:CSRF token 缓存、系数直取、时间戳转 UTC", async () => {
|
||||
const m = createMockFetch({
|
||||
"GET /": { body: TNODE_HOME_HTML },
|
||||
"POST /api/torrent/advancedSearch": { body: TNODE_SEARCH_OK },
|
||||
});
|
||||
const a = adapterWith(def, tnodeSpider, m.fetch);
|
||||
const page = await a.search("eva");
|
||||
await a.list(1);
|
||||
expect(m.requests.filter((r) => r.url.pathname === "/")).toHaveLength(1);
|
||||
expect(m.requests[1]!.headers["x-csrf-token"]).toBe("CSRF-TOKEN-XYZ");
|
||||
const [x, y] = page.items;
|
||||
expect(x).toMatchObject({
|
||||
siteReleaseId: "3301",
|
||||
promotion: { type: "free", downloadFactor: 0, uploadFactor: 1 },
|
||||
uploadedAt: "2024-09-20T00:00:00.000Z",
|
||||
imdbId: "tt0923811",
|
||||
detailUrl: "https://tnode.example/torrent/info/3301",
|
||||
hr: "unknown",
|
||||
});
|
||||
expect(y!.promotion.type).toBe("2x50%");
|
||||
const t = withTorrent(m.fetch, (u) => u.pathname.startsWith("/api/torrent/download/"));
|
||||
await adapterWith(def, tnodeSpider, t.fetch).downloadTarget("3301");
|
||||
expect(t.hits[0]!.headers.get("cookie")).toBe("c=COOKIESECRET");
|
||||
});
|
||||
|
||||
it("取不到 token → auth_expired;缺 Cookie → auth_missing;搜索 HTTP 403 → auth_invalid", async () => {
|
||||
const m = createMockFetch({ "GET /": { body: TNODE_HOME_NO_TOKEN } });
|
||||
await expectSiteError(adapterWith(def, tnodeSpider, m.fetch).list(1), "auth_expired");
|
||||
await expectSiteError(adapterWith(def, tnodeSpider, m.fetch, {}).list(1), "auth_missing");
|
||||
const f = createMockFetch({ "GET /": { body: TNODE_HOME_HTML }, "/api/torrent/advancedSearch": { status: 403 } });
|
||||
await expectSiteError(adapterWith(def, tnodeSpider, f.fetch).list(1), "auth_invalid");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/spider/tnode.py @ e6820a8
|
||||
*
|
||||
* TNode(parser "TNodeSpider",Cookie + 首页 CSRF token + `api/torrent/advancedSearch`)。函数对照 `TNodeSpider`:
|
||||
* - `get_search_page_size` → `pageSize`(100)
|
||||
* - `__get_token/__async_get_token`(@cached 24h)→ `tnodeToken`(适配器实例缓存,TTL 24h)
|
||||
* - `__parse_token_response` → `parseTnodeToken`
|
||||
* - `__get_params` → `tnodeSearchParams`;`__parse_result` → `parseTnodeResult`;`__process_response` → `tnodeSpider.search`
|
||||
* - `_downloadurl/_pageurl` → `resolveDownload`(带 Cookie 下载)/ page_url
|
||||
* 差异:取不到 token 时 MP 返回 `(True, [])`;这里抛 auth_expired(首页取不到 token 基本意味着 Cookie 失效)。
|
||||
*/
|
||||
import { SiteError } from "../types.ts";
|
||||
import {
|
||||
apiRequest,
|
||||
cacheGet,
|
||||
cacheSet,
|
||||
checkReleaseId,
|
||||
downloadTorrentFile,
|
||||
formatTimestampTz,
|
||||
isPlainObject,
|
||||
siteBase,
|
||||
textRequest,
|
||||
} from "./common.ts";
|
||||
import type { SpecializedSiteDefinition, SpecializedSpider, SpecializedTorrent, SpiderContext } from "./types.ts";
|
||||
|
||||
export const TNODE_TOKEN_TTL_MS = 24 * 3600 * 1000;
|
||||
|
||||
/** MP `__parse_token_response` */
|
||||
export function parseTnodeToken(html: string): string | null {
|
||||
return /<meta name="x-csrf-token" content="(.+?)">/.exec(html)?.[1] ?? null;
|
||||
}
|
||||
|
||||
async function tnodeToken(ctx: SpiderContext, cookie: string): Promise<string> {
|
||||
const key = `tnode:token:${ctx.def.domain}`;
|
||||
const cached = cacheGet<string>(ctx, key);
|
||||
if (cached) return cached;
|
||||
const token = parseTnodeToken(await textRequest(ctx, siteBase(ctx.def), { cookie }));
|
||||
if (!token)
|
||||
throw new SiteError(ctx.def.id, "auth_expired", `${ctx.def.name} 未获取到 token,无法搜索(Cookie 可能失效)`);
|
||||
cacheSet(ctx, key, token, TNODE_TOKEN_TTL_MS);
|
||||
return token;
|
||||
}
|
||||
|
||||
/** MP `__get_params` */
|
||||
export function tnodeSearchParams(keyword: string | null | undefined, page = 0) {
|
||||
return {
|
||||
page: Number(page) + 1,
|
||||
size: 100,
|
||||
type: keyword?.startsWith("tt") ? "imdbid" : "title",
|
||||
keyword: keyword || "",
|
||||
sorter: "id",
|
||||
order: "desc",
|
||||
tags: [],
|
||||
category: [501, 502, 503, 504],
|
||||
medium: [],
|
||||
videoCoding: [],
|
||||
audioCoding: [],
|
||||
resolution: [],
|
||||
group: [],
|
||||
};
|
||||
}
|
||||
|
||||
export interface TnodeItem {
|
||||
id: number | string;
|
||||
title?: string;
|
||||
subtitle?: string;
|
||||
upload_time?: number | string;
|
||||
size?: number;
|
||||
seeding?: number;
|
||||
leeching?: number;
|
||||
complete?: number;
|
||||
downloadRate?: number;
|
||||
uploadRate?: number;
|
||||
imdb?: string;
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
/** MP `__parse_result` */
|
||||
export function parseTnodeResult(results: TnodeItem[] | null | undefined, def: SpecializedSiteDefinition) {
|
||||
const torrents: SpecializedTorrent[] = [];
|
||||
if (!results?.length) return torrents;
|
||||
const base = siteBase(def);
|
||||
for (const result of results) {
|
||||
const t: SpecializedTorrent = {
|
||||
title: result.title,
|
||||
description: result.subtitle,
|
||||
enclosure: `${base}api/torrent/download/${result.id}`,
|
||||
size: result.size,
|
||||
seeders: result.seeding,
|
||||
peers: result.leeching,
|
||||
grabs: result.complete,
|
||||
page_url: `${base}torrent/info/${result.id}`,
|
||||
site_release_id: String(result.id),
|
||||
};
|
||||
if (result.downloadRate != null) t.downloadvolumefactor = result.downloadRate;
|
||||
if (result.uploadRate != null) t.uploadvolumefactor = result.uploadRate;
|
||||
if (result.imdb) t.imdbid = result.imdb;
|
||||
const pubdate = formatTimestampTz(result.upload_time, def.umt.timezone);
|
||||
if (pubdate) t.pubdate = pubdate;
|
||||
torrents.push(t);
|
||||
}
|
||||
return torrents;
|
||||
}
|
||||
|
||||
export const tnodeSpider: SpecializedSpider = {
|
||||
parser: "TNodeSpider",
|
||||
credential: "cookie",
|
||||
pageSize: () => 100,
|
||||
async search(ctx, q) {
|
||||
const cookie = (await ctx.secret("cookie", true))!;
|
||||
const token = await tnodeToken(ctx, cookie);
|
||||
const data = await apiRequest(ctx, `${siteBase(ctx.def)}api/torrent/advancedSearch`, {
|
||||
method: "POST",
|
||||
headers: { "X-CSRF-TOKEN": token, "Content-Type": "application/json; charset=utf-8", cookie },
|
||||
json: tnodeSearchParams(q.keyword, q.page ?? 0),
|
||||
});
|
||||
const d = isPlainObject(data) && isPlainObject(data.data) ? data.data : {};
|
||||
return parseTnodeResult((d.torrents as TnodeItem[]) ?? [], ctx.def);
|
||||
},
|
||||
async resolveDownload(ctx, siteReleaseId, enclosure) {
|
||||
const cookie = await ctx.secret("cookie", true);
|
||||
const id = checkReleaseId(ctx.def.id, siteReleaseId);
|
||||
return downloadTorrentFile(ctx, enclosure ?? `${siteBase(ctx.def)}api/torrent/download/${id}`, { cookie });
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,64 @@
|
||||
import { createMockFetch, TORRENTLEECH_SEARCH_OK } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { adapterWith, expectSiteError, testDef, withTorrent } from "./testing.ts";
|
||||
import { torrentleechSearchUrl, torrentleechSpider } from "./torrentleech.ts";
|
||||
|
||||
const def = testDef("TorrentLeech", {
|
||||
domain: "https://www.torrentleech.org/",
|
||||
category: { movie: [{ id: 47 }, { id: 8 }], tv: [{ id: 32 }] },
|
||||
umt: {
|
||||
version: "tl@test",
|
||||
kind: "api",
|
||||
displayName: "TL",
|
||||
timezone: "+00:00",
|
||||
rateLimit: { minIntervalMs: 0, burst: 10 },
|
||||
capabilities: { list: true, search: true, detail: false, download: true, rss: false },
|
||||
},
|
||||
});
|
||||
|
||||
describe("TorrentLeech(MP torrentleech.py)", () => {
|
||||
it("__get_search_url:分类 exact 查询与 quote 编码;浏览页码 +1", async () => {
|
||||
expect(torrentleechSearchUrl(def, "The Matrix (1999)", "movie")).toBe(
|
||||
"https://www.torrentleech.org/torrents/browse/list/categories/47,8/exact/1/query/The%20Matrix%20%281999%29",
|
||||
);
|
||||
expect(torrentleechSearchUrl(def, "a/b")).toBe("https://www.torrentleech.org/torrents/browse/list/query/a/b");
|
||||
const m = createMockFetch({ "/torrents/browse/list/page/3": { body: TORRENTLEECH_SEARCH_OK } });
|
||||
await adapterWith(def, torrentleechSpider, m.fetch).list(3);
|
||||
expect(m.requests[0]!.url.pathname).toBe("/torrents/browse/list/page/3");
|
||||
});
|
||||
|
||||
it("列表解析:download_multiplier、请求分类投影、UTC 时间", async () => {
|
||||
const m = createMockFetch({
|
||||
"/torrents/browse/list/categories/47,8/exact/1/query/Matrix": { body: TORRENTLEECH_SEARCH_OK },
|
||||
});
|
||||
const page = await adapterWith(def, torrentleechSpider, m.fetch).search("Matrix", { mtype: "movie" });
|
||||
const [a, b] = page.items;
|
||||
expect(a).toMatchObject({
|
||||
siteReleaseId: "240001",
|
||||
promotion: { type: "free", downloadFactor: 0, uploadFactor: 1 },
|
||||
uploadedAt: "2026-09-20T08:00:00.000Z",
|
||||
imdbId: "tt0133093",
|
||||
category: { id: "47" },
|
||||
detailUrl: "https://www.torrentleech.org/torrent/240001",
|
||||
downloadUrl: "https://www.torrentleech.org/download/240001/The.Matrix.1999.2160p.UHD.BluRay.x265-TL.torrent",
|
||||
});
|
||||
expect(a!.raw.category).toBe("电影");
|
||||
expect(b!.raw.category).toBe("未知");
|
||||
expect(b!.promotion.type).toBe("none");
|
||||
// 关键词搜索不翻页
|
||||
expect((await adapterWith(def, torrentleechSpider, m.fetch).search("Matrix", { page: 2 })).items).toEqual([]);
|
||||
});
|
||||
|
||||
it("中文关键词返回空且不发请求;缺 Cookie;跳登录页 → auth_expired;下载带 Cookie", async () => {
|
||||
const m = createMockFetch({
|
||||
"/torrents/browse/list/page/1": { status: 302, headers: { location: "/user/account/login" } },
|
||||
});
|
||||
expect((await adapterWith(def, torrentleechSpider, m.fetch).search("黑客帝国")).items).toEqual([]);
|
||||
expect(m.requests).toHaveLength(0);
|
||||
await expectSiteError(adapterWith(def, torrentleechSpider, m.fetch, {}).list(1), "auth_missing");
|
||||
await expectSiteError(adapterWith(def, torrentleechSpider, m.fetch).list(1), "auth_expired");
|
||||
const t = withTorrent(m.fetch, (u) => u.pathname.startsWith("/download/"));
|
||||
await adapterWith(def, torrentleechSpider, t.fetch).downloadTarget("240001");
|
||||
expect(t.hits[0]!.headers.get("cookie")).toBe("c=COOKIESECRET");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/spider/torrentleech.py @ e6820a8
|
||||
*
|
||||
* TorrentLeech JSON 搜索接口(parser "TorrentLeech",Cookie)。函数对照 `TorrentLeech`:
|
||||
* - `get_search_page_size` → `pageSize`(关键词搜索 null,浏览 100)
|
||||
* - `__category_ids` → `torrentleechCategoryIds`;`__get_search_url` → `torrentleechSearchUrl`
|
||||
* - `_browseurl` → `torrentleechBrowseUrl`;`__parse_result` → `parseTorrentleechResult`
|
||||
* - `__process_response` + `search` → `torrentleechSpider.search`(含中文关键词直接返回空,MP 行为)
|
||||
* 差异:MP 只在请求了媒体类型时写 category;这里另外总是输出原始 `category_id`。
|
||||
*/
|
||||
import type { MediaKind } from "../spider/spider.ts";
|
||||
import { containsChinese } from "../user-parsers/common.ts";
|
||||
import {
|
||||
apiRequest,
|
||||
checkReleaseId,
|
||||
downloadTorrentFile,
|
||||
formatTimestampTz,
|
||||
isPlainObject,
|
||||
pyQuote,
|
||||
siteBase,
|
||||
} from "./common.ts";
|
||||
import type { SpecializedSiteDefinition, SpecializedSpider, SpecializedTorrent } from "./types.ts";
|
||||
|
||||
const MEDIA_VALUE = { movie: "电影", tv: "电视剧", music: "音乐" } as const;
|
||||
|
||||
/** MP `__category_ids` */
|
||||
export function torrentleechCategoryIds(def: SpecializedSiteDefinition, mtype?: MediaKind): string[] {
|
||||
if (!mtype) return [];
|
||||
return (def.category?.[mtype] ?? []).filter((i) => i && i.id != null).map((i) => String(i.id));
|
||||
}
|
||||
|
||||
/** MP `__get_search_url` */
|
||||
export function torrentleechSearchUrl(def: SpecializedSiteDefinition, keyword: string, mtype?: MediaKind): string {
|
||||
const domain = siteBase(def);
|
||||
const encoded = pyQuote(keyword);
|
||||
const ids = torrentleechCategoryIds(def, mtype);
|
||||
if (ids.length) return `${domain}torrents/browse/list/categories/${ids.join(",")}/exact/1/query/${encoded}`;
|
||||
return `${domain}torrents/browse/list/query/${encoded}`;
|
||||
}
|
||||
|
||||
export function torrentleechBrowseUrl(def: SpecializedSiteDefinition, page = 0): string {
|
||||
return `${siteBase(def)}torrents/browse/list/page/${Number(page) + 1}`;
|
||||
}
|
||||
|
||||
export interface TorrentleechItem {
|
||||
fid: number | string;
|
||||
filename?: string;
|
||||
name?: string;
|
||||
addedTimestamp?: string | number;
|
||||
size?: number;
|
||||
seeders?: number;
|
||||
leechers?: number;
|
||||
completed?: number;
|
||||
download_multiplier?: number;
|
||||
imdbID?: string;
|
||||
categoryID?: number | string;
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
/** MP `__parse_result` */
|
||||
export function parseTorrentleechResult(
|
||||
results: TorrentleechItem[] | null | undefined,
|
||||
def: SpecializedSiteDefinition,
|
||||
mtype?: MediaKind,
|
||||
) {
|
||||
const torrents: SpecializedTorrent[] = [];
|
||||
if (!results?.length) return torrents;
|
||||
const domain = siteBase(def);
|
||||
const requested = new Set(torrentleechCategoryIds(def, mtype));
|
||||
for (const result of results) {
|
||||
const t: SpecializedTorrent = {
|
||||
title: result.name,
|
||||
enclosure: `${domain}download/${result.fid}/${result.filename}`,
|
||||
size: result.size,
|
||||
seeders: result.seeders,
|
||||
peers: result.leechers,
|
||||
grabs: result.completed,
|
||||
uploadvolumefactor: 1,
|
||||
page_url: `${domain}torrent/${result.fid}`,
|
||||
site_release_id: String(result.fid),
|
||||
};
|
||||
if (result.download_multiplier != null) t.downloadvolumefactor = result.download_multiplier;
|
||||
if (result.imdbID) t.imdbid = result.imdbID;
|
||||
const pubdate = formatTimestampTz(result.addedTimestamp, def.umt.timezone);
|
||||
if (pubdate) t.pubdate = pubdate;
|
||||
if (requested.size && mtype) {
|
||||
t.category = requested.has(String(result.categoryID)) ? MEDIA_VALUE[mtype] : "未知";
|
||||
}
|
||||
if (result.categoryID != null) t.category_id = String(result.categoryID);
|
||||
torrents.push(t);
|
||||
}
|
||||
return torrents;
|
||||
}
|
||||
|
||||
export const torrentleechSpider: SpecializedSpider = {
|
||||
parser: "TorrentLeech",
|
||||
credential: "cookie",
|
||||
pageSize: (keyword) => (keyword ? null : 100),
|
||||
async search(ctx, q) {
|
||||
// 不支持中文(MP 返回 (True, []))
|
||||
if (q.keyword && containsChinese(q.keyword)) return [];
|
||||
const cookie = (await ctx.secret("cookie", true))!;
|
||||
const url = q.keyword
|
||||
? torrentleechSearchUrl(ctx.def, q.keyword, q.mtype)
|
||||
: torrentleechBrowseUrl(ctx.def, q.page ?? 0);
|
||||
const data = await apiRequest(ctx, url, {
|
||||
method: "GET",
|
||||
headers: { "Content-Type": "application/json; charset=utf-8", cookie },
|
||||
});
|
||||
const list = isPlainObject(data) ? (data.torrentList as TorrentleechItem[]) : [];
|
||||
return parseTorrentleechResult(list ?? [], ctx.def, q.mtype);
|
||||
},
|
||||
async resolveDownload(ctx, siteReleaseId, enclosure) {
|
||||
const cookie = await ctx.secret("cookie", true);
|
||||
const id = checkReleaseId(ctx.def.id, siteReleaseId);
|
||||
// TODO(verify):无缓存直链时文件名段是否可任意
|
||||
return downloadTorrentFile(ctx, enclosure ?? `${siteBase(ctx.def)}download/${id}/${id}.torrent`, { cookie });
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,100 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/__init__.py @ e6820a8
|
||||
*
|
||||
* 专用 spider(API 型站点)共用类型。
|
||||
* - `SpecializedParser` = MP `SPIDER_PARSER_CLASSES` 的键;`SPECIALIZED_SEARCH_ARGUMENTS` 的参数集合 → `SpiderSearchQuery`
|
||||
* - `SpecializedSiteDefinition` = 这些 spider 的 `__init__(indexer)` 读取的 indexer dict 键 + 本项目 `umt` 扩展
|
||||
*/
|
||||
import type { SiteHttpContext } from "../http.ts";
|
||||
import type { SpiderCategoryConfig } from "../spider/schema.ts";
|
||||
import type { MediaKind, SpiderTorrent } from "../spider/spider.ts";
|
||||
import type { DownloadTarget, SiteCapabilities, SiteCredentialKind } from "../types.ts";
|
||||
|
||||
export type SpecializedParser =
|
||||
| "Haidan"
|
||||
| "HDDolby"
|
||||
| "RousiPro"
|
||||
| "SunnyPT"
|
||||
| "TNodeSpider"
|
||||
| "TorrentLeech"
|
||||
| "Yema"
|
||||
| "mTorrent";
|
||||
|
||||
export interface SpecializedSiteDefinition {
|
||||
id: string;
|
||||
name: string;
|
||||
enabled?: boolean;
|
||||
/** MP 用户信息解析器 schema(SiteSchema 值) */
|
||||
schema?: string;
|
||||
provenance?: import("../spider/schema.ts").DefinitionProvenance;
|
||||
/** 站点前端地址(MP indexer['domain'],以 / 结尾) */
|
||||
domain: string;
|
||||
parser: SpecializedParser;
|
||||
timeout?: number;
|
||||
proxy?: boolean;
|
||||
public?: boolean;
|
||||
/** 分类组(TorrentLeech 按媒体类型取分类 ID;SunnyPT 分类接口失败时兜底) */
|
||||
category?: SpiderCategoryConfig;
|
||||
/** 独立 API 地址(MP indexer['api_url'],SunnyPT) */
|
||||
api_url?: string;
|
||||
/** 本项目扩展(MP 无对应) */
|
||||
umt: {
|
||||
version: string;
|
||||
kind: "api";
|
||||
displayName: string;
|
||||
/** 站点时间时区偏移(pubdate/freedate 无时区时按此转 UTC) */
|
||||
timezone: string;
|
||||
rateLimit: { minIntervalMs: number; burst: number };
|
||||
capabilities: SiteCapabilities;
|
||||
categoryNames?: Record<string, string>;
|
||||
/** 覆盖 API 基址(测试指向 mock,或站点换域) */
|
||||
apiBaseUrl?: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 专用 spider 输出:MP TorrentInfo 字段 + 本项目扩展:
|
||||
* - `site_release_id`:站点资源 ID(MP 只在 page_url/enclosure 里隐含)
|
||||
* - `pubdate_utc` / `freedate_utc`:源数据自带时区时的精确 UTC(MP normalize_datetime 会丢时区)
|
||||
*/
|
||||
export interface SpecializedTorrent extends SpiderTorrent {
|
||||
site_release_id?: string;
|
||||
pubdate_utc?: string;
|
||||
freedate_utc?: string;
|
||||
}
|
||||
|
||||
/** MP `_SPECIALIZED_SEARCH_ARGUMENTS` 的并集;page 为 MP 内部 0 起页码 */
|
||||
export interface SpiderSearchQuery {
|
||||
keyword?: string | null;
|
||||
mtype?: MediaKind;
|
||||
/** 逗号分隔的站点分类 ID(RousiPro/SunnyPT) */
|
||||
cat?: string;
|
||||
page?: number;
|
||||
}
|
||||
|
||||
export interface SpiderContext {
|
||||
def: SpecializedSiteDefinition;
|
||||
http: SiteHttpContext;
|
||||
/** 取凭据;required 时缺失抛 SiteError('auth_missing') */
|
||||
secret(kind: SiteCredentialKind, required?: boolean): Promise<string | null>;
|
||||
/** 实例内缓存(TNode CSRF token、SunnyPT 分类);值带过期时间 */
|
||||
cache: Map<string, { value: unknown; expiresAt: number }>;
|
||||
userAgent?: string;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface SpecializedSpider {
|
||||
parser: SpecializedParser;
|
||||
/** 必需凭据(MP search 开头的检查) */
|
||||
credential: SiteCredentialKind;
|
||||
/** MP `get_search_page_size`:null 表示不支持翻页 */
|
||||
pageSize(keyword?: string | null): number | null;
|
||||
search(ctx: SpiderContext, query: SpiderSearchQuery): Promise<SpecializedTorrent[]>;
|
||||
/**
|
||||
* 下载前现场取种子(MP `__get_download_url` + submission `_resolve_indirect_download_url` + `download_torrent`)。
|
||||
* `enclosure` 为最近一次搜索结果里的直链(仅内存,不落库);缺失时按站点规则构造。
|
||||
*/
|
||||
resolveDownload(ctx: SpiderContext, siteReleaseId: string, enclosure?: string): Promise<DownloadTarget>;
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import {
|
||||
createMockFetch,
|
||||
YEMA_DOWNLOAD_KEY_OK,
|
||||
YEMA_SEARCH_ERROR,
|
||||
YEMA_SEARCH_MOVIE,
|
||||
YEMA_SEARCH_TV_A,
|
||||
} from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { adapterWith, expectSiteError, testDef, withTorrent } from "./testing.ts";
|
||||
import { yemaSpider } from "./yema.ts";
|
||||
|
||||
const def = testDef("Yema", { domain: "https://www.yemapt.org/" });
|
||||
const URL_SEARCH = "POST /openApi/torrent/fetchOpenTorrentList.json";
|
||||
|
||||
describe("YemaSpider(MP yema.py)", () => {
|
||||
it("列表解析:促销枚举、标签表(未知 ID 丢弃)、H&R、非整数 ID 跳过", async () => {
|
||||
const m = createMockFetch({ [URL_SEARCH]: { body: YEMA_SEARCH_MOVIE } });
|
||||
const page = await adapterWith(def, yemaSpider, m.fetch).search("past", { mtype: "movie" });
|
||||
expect(m.requests[0]!.headers.authorization).toBe("APIKEYSECRET");
|
||||
expect(JSON.parse(m.requests[0]!.body!)).toEqual({
|
||||
pageParam: { current: 1, pageSize: 40 },
|
||||
sorter: {},
|
||||
keyword: "past",
|
||||
categoryId: 4,
|
||||
});
|
||||
expect(page.items).toHaveLength(1);
|
||||
expect(page.items[0]).toMatchObject({
|
||||
siteReleaseId: "88001",
|
||||
promotion: { type: "2xfree", downloadFactor: 0, uploadFactor: 2, expiresAt: "2026-09-28T16:00:00.000Z" },
|
||||
tags: ["官方", "中字"],
|
||||
hr: true,
|
||||
detailUrl: "https://www.yemapt.org/#/torrent/detail/88001/",
|
||||
category: { id: "4" },
|
||||
});
|
||||
});
|
||||
|
||||
it("电视剧按 6 个分类逐个查询,部分失败仍合并去重", async () => {
|
||||
let n = 0;
|
||||
const m = createMockFetch({
|
||||
[URL_SEARCH]: () => {
|
||||
n++;
|
||||
return n === 2 ? { body: { success: false, errorMessage: "系统繁忙" } } : { body: YEMA_SEARCH_TV_A };
|
||||
},
|
||||
});
|
||||
const page = await adapterWith(def, yemaSpider, m.fetch).list(1, { mtype: "tv" });
|
||||
expect(m.requests.map((r) => JSON.parse(r.body!).categoryId)).toEqual([5, 6, 13, 14, 15, 17]);
|
||||
expect(page.items.map((i) => i.siteReleaseId)).toEqual(["88002"]);
|
||||
expect(page.items[0]!.promotion).toMatchObject({ type: "50%", downloadFactor: 0.5, uploadFactor: 1.5 });
|
||||
});
|
||||
|
||||
it("全部失败才报错;缺 AuthKey;下载 key 拼成 download1?token=", async () => {
|
||||
const bad = createMockFetch({ [URL_SEARCH]: { body: YEMA_SEARCH_ERROR } });
|
||||
await expectSiteError(adapterWith(def, yemaSpider, bad.fetch).list(1), "auth_invalid");
|
||||
await expectSiteError(adapterWith(def, yemaSpider, bad.fetch, {}).list(1), "auth_missing");
|
||||
const d = createMockFetch({ "POST /openApi/torrent/generateDownloadKey.json": { body: YEMA_DOWNLOAD_KEY_OK } });
|
||||
const t = withTorrent(d.fetch, (u) => u.pathname === "/api/torrent/download1");
|
||||
await adapterWith(def, yemaSpider, t.fetch).downloadTarget("88001");
|
||||
expect(d.requests[0]!.url.searchParams.get("id")).toBe("88001");
|
||||
expect(t.hits[0]!.url.toString()).toBe("https://www.yemapt.org/api/torrent/download1?token=DLKEY-123");
|
||||
expect(t.hits[0]!.headers.get("authorization")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,226 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/spider/yema.py @ e6820a8
|
||||
*
|
||||
* YemaPT 开放 API(parser "Yema",Authorization: AuthKey)。函数对照 `YemaSpider`:
|
||||
* - `_size/_movie_category/_tv_category/_music_category/_labels` → `YEMA_*`
|
||||
* - `_request_headers` → `yemaHeaders`;`_build_params` → `yemaBuildParams`
|
||||
* - `_search_category_ids` → `yemaSearchCategoryIds`;`_merge_search_results` → `yemaMergeSearchResults`
|
||||
* - `_parse_result` → `parseYemaResult`;`_process_search_response` → `yemaProcessSearchResponse`
|
||||
* - `_prepare_search_requests/_finalize_search_responses/search` → `yemaSpider.search`(全部分类失败才算失败)
|
||||
* - `_download_factor/_upload_factor` → `yemaDownloadFactor/yemaUploadFactor`
|
||||
* - `_build_download_url`(POST generateDownloadKey.json?id=,success/result=data → api/torrent/download1?token=)→ `resolveDownload`
|
||||
*/
|
||||
import type { MpMediaType } from "../spider/spider.ts";
|
||||
import { SiteError } from "../types.ts";
|
||||
import { normalizeDatetime } from "../user-parsers/common.ts";
|
||||
import {
|
||||
apiRequest,
|
||||
businessError,
|
||||
checkReleaseId,
|
||||
downloadTorrentFile,
|
||||
isoWithZoneToUtc,
|
||||
isPlainObject,
|
||||
resolveIndirectDownloadUrl,
|
||||
} from "./common.ts";
|
||||
import type {
|
||||
SpecializedSiteDefinition,
|
||||
SpecializedSpider,
|
||||
SpecializedTorrent,
|
||||
SpiderContext,
|
||||
SpiderSearchQuery,
|
||||
} from "./types.ts";
|
||||
|
||||
/** YemaPT 开放 API 使用更大的分页容量时会返回空结果 */
|
||||
export const YEMA_PAGE_SIZE = 40;
|
||||
export const YEMA_MOVIE_CATEGORY = [4];
|
||||
export const YEMA_TV_CATEGORY = [5, 6, 13, 14, 15, 17];
|
||||
export const YEMA_MUSIC_CATEGORY = [8, 16];
|
||||
export const YEMA_LABELS: Record<string, string> = {
|
||||
"1": "禁转",
|
||||
"2": "首发",
|
||||
"3": "官方",
|
||||
"4": "自制",
|
||||
"5": "国语",
|
||||
"6": "中字",
|
||||
"7": "粤语",
|
||||
"8": "英字",
|
||||
"9": "HDR10",
|
||||
"10": "杜比视界",
|
||||
"11": "分集",
|
||||
"12": "完结",
|
||||
};
|
||||
|
||||
export function yemaSiteUrl(def: SpecializedSiteDefinition): string {
|
||||
return String(def.umt.apiBaseUrl ?? def.domain ?? "https://www.yemapt.org/").replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
function yemaHeaders(apikey: string): Record<string, string> {
|
||||
return { Authorization: apikey, "Content-Type": "application/json", Accept: "application/json, text/plain, */*" };
|
||||
}
|
||||
|
||||
/** MP `_build_params` */
|
||||
export function yemaBuildParams(
|
||||
keyword: string | null | undefined,
|
||||
page: number | undefined,
|
||||
categoryId?: number | null,
|
||||
) {
|
||||
const params: Record<string, unknown> = {
|
||||
pageParam: { current: Number(page || 0) + 1, pageSize: YEMA_PAGE_SIZE },
|
||||
sorter: {},
|
||||
};
|
||||
if (keyword) params.keyword = keyword;
|
||||
if (categoryId != null) params.categoryId = categoryId;
|
||||
return params;
|
||||
}
|
||||
|
||||
/** MP `_search_category_ids`:分类参数不支持数组,逐个查询 */
|
||||
export function yemaSearchCategoryIds(mtype?: SpiderSearchQuery["mtype"]): Array<number | null> {
|
||||
if (mtype === "movie") return [...YEMA_MOVIE_CATEGORY];
|
||||
if (mtype === "tv") return [...YEMA_TV_CATEGORY];
|
||||
if (mtype === "music") return [...YEMA_MUSIC_CATEGORY];
|
||||
return [null];
|
||||
}
|
||||
|
||||
/** MP `_merge_search_results`:按详情链接合并去重,保持首次出现顺序 */
|
||||
export function yemaMergeSearchResults(groups: SpecializedTorrent[][]): SpecializedTorrent[] {
|
||||
const merged: SpecializedTorrent[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const results of groups) {
|
||||
for (const t of results) {
|
||||
const identity = t.page_url || t.enclosure;
|
||||
if (identity && seen.has(identity)) continue;
|
||||
if (identity) seen.add(identity);
|
||||
merged.push(t);
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
export function yemaDownloadFactor(promotion: string): number {
|
||||
return ({ free: 0, half: 0.5, none: 1 } as Record<string, number>)[promotion] ?? 1;
|
||||
}
|
||||
|
||||
export function yemaUploadFactor(promotion: string): number {
|
||||
return ({ none: 1, one_half: 1.5, double_upload: 2 } as Record<string, number>)[promotion] ?? 1;
|
||||
}
|
||||
|
||||
export interface YemaItem {
|
||||
id: number;
|
||||
showName?: string;
|
||||
shortDesc?: string;
|
||||
categoryId?: number;
|
||||
tagList?: string[];
|
||||
listingTime?: string;
|
||||
fileSize?: number;
|
||||
seedNum?: number;
|
||||
leechNum?: number;
|
||||
completedNum?: number;
|
||||
downloadPromotion?: string;
|
||||
uploadPromotion?: string;
|
||||
downloadPromotionEndTime?: string;
|
||||
hrPunishEnable?: boolean;
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
/** MP `_parse_result` */
|
||||
export function parseYemaResult(results: unknown[] | null | undefined, def: SpecializedSiteDefinition) {
|
||||
const torrents: SpecializedTorrent[] = [];
|
||||
const site = yemaSiteUrl(def);
|
||||
for (const r of results ?? []) {
|
||||
if (!isPlainObject(r)) continue;
|
||||
const result = r as YemaItem;
|
||||
const cv = result.categoryId;
|
||||
let category: MpMediaType;
|
||||
if (cv != null && YEMA_TV_CATEGORY.includes(cv)) category = "电视剧";
|
||||
else if (cv != null && YEMA_MOVIE_CATEGORY.includes(cv)) category = "电影";
|
||||
else if (cv != null && YEMA_MUSIC_CATEGORY.includes(cv)) category = "音乐";
|
||||
else category = "未知";
|
||||
// MP:label_id in self._labels(键为字符串)
|
||||
const labels = (result.tagList ?? [])
|
||||
.filter((id) => typeof id === "string" && id in YEMA_LABELS)
|
||||
.map((id) => YEMA_LABELS[id]!);
|
||||
const id = result.id;
|
||||
if (typeof id !== "number" || !Number.isInteger(id)) continue;
|
||||
const t: SpecializedTorrent = {
|
||||
title: result.showName,
|
||||
description: result.shortDesc,
|
||||
size: result.fileSize,
|
||||
seeders: result.seedNum,
|
||||
peers: result.leechNum,
|
||||
grabs: result.completedNum,
|
||||
downloadvolumefactor: yemaDownloadFactor(String(result.downloadPromotion || "")),
|
||||
uploadvolumefactor: yemaUploadFactor(String(result.uploadPromotion || "")),
|
||||
page_url: `${site}/#/torrent/detail/${id}/`,
|
||||
labels,
|
||||
hit_and_run: Boolean(result.hrPunishEnable),
|
||||
category,
|
||||
site_release_id: String(id),
|
||||
};
|
||||
const pubdate = normalizeDatetime(result.listingTime);
|
||||
if (pubdate) t.pubdate = pubdate;
|
||||
const pu = isoWithZoneToUtc(result.listingTime);
|
||||
if (pu) t.pubdate_utc = pu;
|
||||
const fd = normalizeDatetime(result.downloadPromotionEndTime);
|
||||
if (fd) t.freedate = fd;
|
||||
const fu = isoWithZoneToUtc(result.downloadPromotionEndTime);
|
||||
if (fu) t.freedate_utc = fu;
|
||||
if (cv != null) t.category_id = String(cv);
|
||||
torrents.push(t);
|
||||
}
|
||||
return torrents;
|
||||
}
|
||||
|
||||
/** MP `_process_search_response` */
|
||||
export function yemaProcessSearchResponse(ctx: SpiderContext, payload: unknown): SpecializedTorrent[] {
|
||||
if (!isPlainObject(payload)) throw businessError(ctx, "搜索", "搜索响应结构无效");
|
||||
if (!payload.success) throw businessError(ctx, "搜索", payload.errorMessage || "未知错误");
|
||||
if (!Array.isArray(payload.data)) throw businessError(ctx, "搜索", "搜索响应 data 不是数组");
|
||||
return parseYemaResult(payload.data, ctx.def);
|
||||
}
|
||||
|
||||
export const yemaSpider: SpecializedSpider = {
|
||||
parser: "Yema",
|
||||
credential: "apiKey",
|
||||
pageSize: () => YEMA_PAGE_SIZE,
|
||||
async search(ctx, q) {
|
||||
const apikey = (await ctx.secret("apiKey", true))!;
|
||||
const url = `${yemaSiteUrl(ctx.def)}/openApi/torrent/fetchOpenTorrentList.json`;
|
||||
const groups: SpecializedTorrent[][] = [];
|
||||
const errors: unknown[] = [];
|
||||
for (const categoryId of yemaSearchCategoryIds(q.mtype)) {
|
||||
try {
|
||||
const payload = await apiRequest(ctx, url, {
|
||||
method: "POST",
|
||||
headers: yemaHeaders(apikey),
|
||||
json: yemaBuildParams(q.keyword, q.page, categoryId),
|
||||
});
|
||||
groups.push(yemaProcessSearchResponse(ctx, payload));
|
||||
} catch (e) {
|
||||
// 凭据错误所有分类都会失败,直接抛出
|
||||
if (e instanceof SiteError && e.isAuthError) throw e;
|
||||
errors.push(e);
|
||||
}
|
||||
}
|
||||
// MP:all(errors) 才算失败
|
||||
if (!groups.length && errors.length) throw errors[0];
|
||||
return yemaMergeSearchResults(groups);
|
||||
},
|
||||
async resolveDownload(ctx, siteReleaseId) {
|
||||
const apikey = (await ctx.secret("apiKey", true))!;
|
||||
const id = checkReleaseId(ctx.def.id, siteReleaseId);
|
||||
const site = yemaSiteUrl(ctx.def);
|
||||
const url = await resolveIndirectDownloadUrl(ctx, `${site}/openApi/torrent/generateDownloadKey.json`, {
|
||||
method: "post",
|
||||
header: { Authorization: apikey, Accept: "application/json" },
|
||||
params: { id: Number(id) },
|
||||
success: "success",
|
||||
result: "data",
|
||||
result_base_url: site,
|
||||
result_path: "api/torrent/download1",
|
||||
result_query_param: "token",
|
||||
});
|
||||
return downloadTorrentFile(ctx, url);
|
||||
},
|
||||
};
|
||||
Reference in new issue
Block a user