feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b)
- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified, backoff + Retry-After, circuit breaker with manual reset, passkey redaction, overlap-window GUID+digest dedupe - sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder, field DSL, filters, Jinja templates via nunjucks, login-page detection); audiences definition hand-written as JSON; raw category id preserved; shared per-site token bucket - sites/mteam: port of MP MTorrentSpider (params, result mapping, promo factors, genDlToken flow) without embedding the API key in enclosures; business code/message checks - testkit: mock RSS server, mock NexusPHP site, mock M-Team API - docs: W1-3b migration mapping appended to porting ledger Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
rootandClaude Fable 5.1 committed
1 parent
ef2e504a94
commit
a6d049db81
47 files changed
+5414
No files matched your search
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* 通用 spider 站点适配器(NexusPHP 等 HTML 站)。
|
||||
*
|
||||
* 对应 MoviePilot e6820a8:
|
||||
* - `app/modules/indexer/__init__.py IndexerModule.search_torrents / refresh_torrents` → `search` / `list`
|
||||
* (`__search_check` 的英文站不搜中文 → search 前检查;`__clear_search_text` → `clearSearchText`;
|
||||
* `SitesHelper.check(domain)` 流控 → 站点令牌桶)
|
||||
* - `SiteSpider.get_torrents`(RequestUtils GET + cookie + ua)→ `fetchPage`
|
||||
* - `app/chain/download/submission.py download_torrent`(带 cookie GET enclosure,磁力原样返回)→ `downloadTarget`
|
||||
*
|
||||
* 本项目调整:登录页不再"返回空列表",而是抛 `auth_expired`(REQ §4.7.1 不展示空列表冒充无内容);
|
||||
* 3xx 不自动跟随(避免把 cookie 带到站外,并把跳转 login 识别为失效);凭证经 SecretResolver 注入。
|
||||
*/
|
||||
import { redactText } from "../../rss/redact";
|
||||
import { buildHttpContext, type SiteAdapterDeps } from "../deps";
|
||||
import { readBody, readText, siteFetch } from "../http";
|
||||
import {
|
||||
type DownloadTarget,
|
||||
type SiteAdapter,
|
||||
type SiteCallOptions,
|
||||
SiteError,
|
||||
type SiteHealth,
|
||||
type SiteListFilters,
|
||||
type SitePage,
|
||||
type SiteReleaseDetail,
|
||||
} from "../types";
|
||||
import { inspectTorrent } from "../util";
|
||||
import { parseDetailPage } from "./detail";
|
||||
import { spiderTorrentToRelease } from "./map";
|
||||
import type { IndexerDefinition } from "./schema";
|
||||
import { loadHtml, normalizeSelector, SiteSpider } from "./spider";
|
||||
|
||||
/** MP `text_tools.remove_punctuation(text, replacement=" ", allow_space=True)` */
|
||||
export function clearSearchText(text: string): string {
|
||||
return text
|
||||
.replace(/[-]/g, "")
|
||||
.replace(/[、.。,，·:：;；!！?？'’"“”()（）[\]【】「」\-—―+|\\_/&#～~]/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function containsChinese(s: string): boolean {
|
||||
return /[一-鿿]/.test(s);
|
||||
}
|
||||
|
||||
export function healthFromError(siteId: string, e: unknown, checkedAt: string): SiteHealth {
|
||||
if (e instanceof SiteError) {
|
||||
const status: SiteHealth["status"] =
|
||||
e.kind === "network"
|
||||
? "unreachable"
|
||||
: e.kind === "not_found" || e.kind === "unsupported"
|
||||
? "parse_error"
|
||||
: e.kind;
|
||||
return { siteId, status, ok: false, checkedAt, latencyMs: null, message: e.message };
|
||||
}
|
||||
return { siteId, status: "unreachable", ok: false, checkedAt, latencyMs: null, message: redactText(String(e)) };
|
||||
}
|
||||
|
||||
export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdapterDeps): SiteAdapter {
|
||||
const def: IndexerDefinition = deps.baseUrl ? { ...input, domain: deps.baseUrl } : input;
|
||||
const ext = def.umt;
|
||||
const http = buildHttpContext(def.id, deps, ext.rateLimit);
|
||||
const base = def.domain.endsWith("/") ? def.domain : `${def.domain}/`;
|
||||
|
||||
async function cookie(): Promise<string> {
|
||||
const c = await deps.secrets.resolve({ siteId: def.id, kind: "cookie" });
|
||||
if (!c) throw new SiteError(def.id, "auth_missing", "未配置站点 Cookie");
|
||||
return c;
|
||||
}
|
||||
|
||||
async function fetchPage(url: string, what: string, opts: SiteCallOptions): Promise<string> {
|
||||
const res = await siteFetch(http, url, {
|
||||
method: "GET",
|
||||
headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base },
|
||||
redirect: "manual",
|
||||
...(opts.signal ? { signal: opts.signal } : {}),
|
||||
});
|
||||
if (res.status >= 300 && res.status < 400) {
|
||||
const loc = res.headers.get("location") ?? "";
|
||||
await res.body?.cancel().catch(() => {});
|
||||
if (/login/i.test(loc)) throw new SiteError(def.id, "auth_expired", `${what}:跳转登录页,Cookie 已失效`);
|
||||
throw new SiteError(def.id, "parse_error", redactText(`${what}:意外重定向到 ${loc}`), { httpStatus: res.status });
|
||||
}
|
||||
if (res.status === 401 || res.status === 403) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(def.id, "auth_invalid", `${what}:HTTP ${res.status}`, { httpStatus: res.status });
|
||||
}
|
||||
if (!res.ok) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(def.id, res.status === 404 ? "not_found" : "network", `${what}:HTTP ${res.status}`, {
|
||||
httpStatus: res.status,
|
||||
});
|
||||
}
|
||||
return readText(http, res);
|
||||
}
|
||||
|
||||
async function run(
|
||||
page: number,
|
||||
filters: SiteListFilters,
|
||||
keyword: string | undefined,
|
||||
opts: SiteCallOptions,
|
||||
): Promise<SitePage> {
|
||||
const what = keyword ? "搜索" : `列表第 ${page} 页`;
|
||||
const spider = new SiteSpider(def, {
|
||||
...(keyword ? { keyword } : {}),
|
||||
...(filters.mtype ? { mtype: filters.mtype } : {}),
|
||||
...(filters.categoryIds?.length ? { cat: filters.categoryIds.join(",") } : {}),
|
||||
page: Math.max(1, page) - 1, // 对外 1 起,MP 内部 0 起
|
||||
});
|
||||
if (!spider.canSearch()) throw new SiteError(def.id, "unsupported", `${what}:站点定义不支持该媒体类型`);
|
||||
const html = await fetchPage(spider.getSearchUrl(), what, opts);
|
||||
const parsed = spider.parse(html);
|
||||
if (parsed.loginPage) throw new SiteError(def.id, "auth_expired", `${what}:返回登录或权限提示页`);
|
||||
if (parsed.isError || !parsed.$) throw new SiteError(def.id, "parse_error", `${what}:响应为空`);
|
||||
const $ = parsed.$;
|
||||
if (parsed.torrents.length === 0) {
|
||||
const text = $("body").text();
|
||||
const hasList =
|
||||
$(normalizeSelector(spider.def.torrents.list.selector)).length > 0 || $("table.torrents").length > 0;
|
||||
const empty = (ext.emptyMarkers ?? []).some((m) => text.includes(m));
|
||||
if (!hasList && !empty) throw new SiteError(def.id, "parse_error", `${what}:页面结构无法识别(可能改版)`);
|
||||
if (ext.loginSuccess && $(ext.loginSuccess).length === 0 && !empty) {
|
||||
throw new SiteError(def.id, "auth_expired", `${what}:未找到登录标志`);
|
||||
}
|
||||
}
|
||||
const observedAt = http.now().toISOString();
|
||||
let items = parsed.torrents
|
||||
.map((t) => spiderTorrentToRelease(t, def, observedAt))
|
||||
.filter((r): r is NonNullable<typeof r> => r !== null);
|
||||
if (filters.freeOnly) items = items.filter((r) => r.promotion.downloadFactor === 0);
|
||||
if (!keyword && filters.categoryIds?.length) {
|
||||
// MP 浏览模式不带分类参数;本地按原始分类过滤
|
||||
const set = new Set(filters.categoryIds);
|
||||
items = items.filter((r) => r.category.id !== null && set.has(r.category.id));
|
||||
}
|
||||
let maxRaw: number | null = null;
|
||||
if (ext.pagination) {
|
||||
$(ext.pagination).each((_, el) => {
|
||||
const p = Number(/[?&]page=(\d+)/.exec($(el).attr("href") ?? "")?.[1]);
|
||||
if (Number.isInteger(p) && (maxRaw === null || p > maxRaw)) maxRaw = p;
|
||||
});
|
||||
}
|
||||
const start = keyword ? 0 : Number(def.browse?.start ?? 0);
|
||||
return {
|
||||
items,
|
||||
page: Math.max(1, page),
|
||||
totalPages: maxRaw === null ? null : Math.max(page, (maxRaw as number) - start + 1),
|
||||
totalPagesEstimated: true,
|
||||
fetchedAt: observedAt,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
id: def.id,
|
||||
kind: ext.kind,
|
||||
displayName: ext.displayName ?? def.name,
|
||||
capabilities: ext.capabilities,
|
||||
definitionVersion: ext.version,
|
||||
|
||||
list(page, filters = {}, opts = {}) {
|
||||
return run(page, filters, undefined, opts);
|
||||
},
|
||||
|
||||
async search(query, filters = {}, opts = {}) {
|
||||
const kw = /^tt\d+$/.test(query.trim()) ? query.trim() : clearSearchText(query);
|
||||
if (kw && def.language === "en" && containsChinese(kw)) {
|
||||
throw new SiteError(def.id, "unsupported", "该站点不支持中文搜索");
|
||||
}
|
||||
return run(filters.page ?? 1, filters, kw, opts);
|
||||
},
|
||||
|
||||
async detail(siteReleaseId, opts = {}): Promise<SiteReleaseDetail> {
|
||||
if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法");
|
||||
if (!ext.detail) throw new SiteError(def.id, "unsupported", "站点定义未配置详情页解析");
|
||||
const url = new URL(`${ext.detail.path}?id=${siteReleaseId}&hit=1`, base).toString();
|
||||
const html = await fetchPage(url, `详情 ${siteReleaseId}`, opts);
|
||||
const r = parseDetailPage(html, def, siteReleaseId, http.now().toISOString());
|
||||
if (r.kind === "login") throw new SiteError(def.id, "auth_expired", `详情 ${siteReleaseId}:返回登录页`);
|
||||
if (r.kind === "not_found") throw new SiteError(def.id, "not_found", `详情 ${siteReleaseId}:资源不存在`);
|
||||
if (!r.detail) throw new SiteError(def.id, "parse_error", `详情 ${siteReleaseId}:页面结构无法识别`);
|
||||
return r.detail;
|
||||
},
|
||||
|
||||
async downloadTarget(siteReleaseId, opts = {}): Promise<DownloadTarget> {
|
||||
if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法");
|
||||
const what = `下载 ${siteReleaseId}`;
|
||||
const url = new URL(`${ext.downloadPath}?id=${siteReleaseId}`, base).toString();
|
||||
const res = await siteFetch(http, url, {
|
||||
method: "GET",
|
||||
headers: { cookie: await cookie(), referer: base },
|
||||
redirect: "manual",
|
||||
...(opts.signal ? { signal: opts.signal } : {}),
|
||||
});
|
||||
const verifiedAt = http.now().toISOString();
|
||||
if (res.status >= 300 && res.status < 400) {
|
||||
const loc = res.headers.get("location") ?? "";
|
||||
await res.body?.cancel().catch(() => {});
|
||||
// MP download_torrent:磁力链原样返回
|
||||
if (loc.startsWith("magnet:")) return { kind: "magnet", uri: loc, verifiedAt };
|
||||
if (/login/i.test(loc)) throw new SiteError(def.id, "auth_expired", `${what}:跳转登录页,Cookie 已失效`);
|
||||
throw new SiteError(def.id, "parse_error", `${what}:意外重定向`, { httpStatus: res.status });
|
||||
}
|
||||
if (!res.ok) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new SiteError(def.id, res.status === 404 ? "not_found" : "network", `${what}:HTTP ${res.status}`, {
|
||||
httpStatus: res.status,
|
||||
});
|
||||
}
|
||||
const bytes = await readBody(http, res);
|
||||
const info = inspectTorrent(bytes);
|
||||
if (!info) {
|
||||
// 站点在 cookie 失效时常返回 200 + 登录页 HTML
|
||||
const $ = loadHtml(new TextDecoder().decode(bytes.subarray(0, 200_000)));
|
||||
if (SiteSpider.isLoginOrPermissionPage($)) throw new SiteError(def.id, "auth_expired", `${what}:返回登录页`);
|
||||
const text = $("body").text();
|
||||
if (ext.detail?.notFoundMarkers.some((m) => text.includes(m))) {
|
||||
throw new SiteError(def.id, "not_found", `${what}:资源不存在`);
|
||||
}
|
||||
throw new SiteError(def.id, "parse_error", `${what}:返回内容不是 .torrent`);
|
||||
}
|
||||
const disposition = res.headers.get("content-disposition") ?? "";
|
||||
const fileName =
|
||||
/filename\*=UTF-8''([^;]+)/i.exec(disposition)?.[1] ?? /filename="?([^";]+)"?/i.exec(disposition)?.[1] ?? null;
|
||||
return {
|
||||
kind: "torrent",
|
||||
bytes,
|
||||
infoHash: info.infoHash,
|
||||
name: info.name,
|
||||
fileName: fileName ? decodeURIComponent(fileName) : null,
|
||||
verifiedAt,
|
||||
};
|
||||
},
|
||||
|
||||
async healthCheck(opts = {}): Promise<SiteHealth> {
|
||||
const started = Date.now();
|
||||
const checkedAt = http.now().toISOString();
|
||||
try {
|
||||
const html = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts);
|
||||
const $ = loadHtml(html);
|
||||
if (SiteSpider.isLoginOrPermissionPage($) || (ext.loginSuccess && $(ext.loginSuccess).length === 0)) {
|
||||
throw new SiteError(def.id, "auth_expired", "健康检查:未登录");
|
||||
}
|
||||
return { siteId: def.id, status: "ok", ok: true, checkedAt, latencyMs: Date.now() - started };
|
||||
} catch (e) {
|
||||
return healthFromError(def.id, e, checkedAt);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
/**
|
||||
* 详情页解析 —— 本项目扩展。MP 通用 SiteSpider 不解析详情页(MP 只在列表/搜索结果上工作),
|
||||
* 这里按 NexusPHP details.php 公开结构(h1#top + td.rowhead/td.rowfollow)实现,选择器在站点定义 `umt.detail` 中配置。
|
||||
* 促销/H&R 复用站点定义中 downloadvolumefactor / uploadvolumefactor / freedate / hr 字段(与列表同语义)。
|
||||
*/
|
||||
import type { CheerioAPI } from "cheerio";
|
||||
import type { SiteReleaseDetail } from "../types";
|
||||
import { extractDoubanId, extractImdbId, parseCount, parseSiteTime } from "../util";
|
||||
import { categoryName, promotionFromFactors } from "./map";
|
||||
import type { IndexerDefinition } from "./schema";
|
||||
import { loadHtml, parseSizeMp, SiteSpider } from "./spider";
|
||||
|
||||
export type DetailPageKind = "ok" | "login" | "not_found" | "unknown";
|
||||
|
||||
export function parseDetailPage(
|
||||
html: string,
|
||||
def: IndexerDefinition,
|
||||
id: string,
|
||||
observedAt: string,
|
||||
): { kind: DetailPageKind; detail: SiteReleaseDetail | null } {
|
||||
const conf = def.umt.detail;
|
||||
if (!conf) return { kind: "unknown", detail: null };
|
||||
const $ = loadHtml(html);
|
||||
if (SiteSpider.isLoginOrPermissionPage($)) return { kind: "login", detail: null };
|
||||
const bodyHead = $("body").text().replace(/\s+/g, " ").slice(0, 2000);
|
||||
if (conf.notFoundMarkers.some((m) => bodyHead.includes(m))) return { kind: "not_found", detail: null };
|
||||
const h1 = $(conf.title).first();
|
||||
if (h1.length === 0) return { kind: "unknown", detail: null };
|
||||
|
||||
const row = (labels: string[]) => {
|
||||
let found: ReturnType<CheerioAPI> | null = null;
|
||||
$(conf.rowLabel).each((_, el) => {
|
||||
const label = $(el).text().replace(/\s+/g, "");
|
||||
if (!found && labels.some((l) => label.startsWith(l.replace(/\s+/g, "")))) {
|
||||
found = $(el).nextAll(conf.rowValue).first();
|
||||
}
|
||||
});
|
||||
return found as ReturnType<CheerioAPI> | null;
|
||||
};
|
||||
|
||||
const title = h1
|
||||
.contents()
|
||||
.filter((_, n) => n.type === "text")
|
||||
.text()
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
const subtitle = row(conf.labels.subtitle)?.text().trim();
|
||||
const basicText = row(conf.labels.basicInfo)?.text().replace(/\s+/g, " ") ?? "";
|
||||
const sizeText = /(?:大小|Size)\s*[:：]\s*([\d.,]+\s*[KMGTP]?i?B)/i.exec(basicText)?.[1] ?? null;
|
||||
const catText = /(?:类型|Type)\s*[:：]\s*(\S+)/i.exec(basicText)?.[1] ?? null;
|
||||
const catId = catText
|
||||
? (Object.entries(def.umt.categoryNames ?? {}).find(([, name]) => name === catText)?.[0] ?? null)
|
||||
: null;
|
||||
const peersText = row(conf.labels.peers)?.text().replace(/\s+/g, " ") ?? "";
|
||||
const seeders = /(\d[\d,]*)\s*个?做种者|(\d[\d,]*)\s*seeder/i.exec(peersText);
|
||||
const leechers = /(\d[\d,]*)\s*个?下载者|(\d[\d,]*)\s*leecher/i.exec(peersText);
|
||||
const descEl = $(conf.description).first();
|
||||
const descText = descEl.text().trim();
|
||||
const descHtml = descEl.html() ?? "";
|
||||
|
||||
// 促销/HR:只用这四个字段在详情页头部取值
|
||||
const fields = def.torrents.fields;
|
||||
const promoDef: IndexerDefinition = {
|
||||
...def,
|
||||
torrents: {
|
||||
list: def.torrents.list,
|
||||
fields: Object.fromEntries(
|
||||
(["downloadvolumefactor", "uploadvolumefactor", "freedate", "hr"] as const)
|
||||
.filter((k) => fields[k])
|
||||
.map((k) => [k, fields[k]]),
|
||||
),
|
||||
},
|
||||
browse: undefined,
|
||||
};
|
||||
const promo = new SiteSpider(promoDef).infoFrom($, $(conf.headerScope).first());
|
||||
const timeTitle = $("span[title]")
|
||||
.filter((_, el) => /\d{4}-\d{2}-\d{2} \d{2}:\d{2}/.test($(el).attr("title") ?? ""))
|
||||
.first()
|
||||
.attr("title");
|
||||
|
||||
const detail: SiteReleaseDetail = {
|
||||
siteId: def.id,
|
||||
siteReleaseId: id,
|
||||
title,
|
||||
category: { id: catId, name: catText ?? categoryName(def, catId ?? undefined) },
|
||||
sizeBytes: sizeText ? parseSizeMp(sizeText) : null,
|
||||
seeders: seeders ? parseCount(seeders[1] ?? seeders[2]) : null,
|
||||
leechers: leechers ? parseCount(leechers[1] ?? leechers[2]) : null,
|
||||
snatched: null,
|
||||
promotion: promotionFromFactors(
|
||||
promo.downloadvolumefactor,
|
||||
promo.uploadvolumefactor,
|
||||
promo.freedate ? parseSiteTime(promo.freedate, def.umt.timezone) : null,
|
||||
),
|
||||
hr: promo.hit_and_run === undefined ? "unknown" : promo.hit_and_run,
|
||||
uploadedAt: parseSiteTime(timeTitle, def.umt.timezone),
|
||||
detailUrl: new URL(`${def.umt.detailPath}?id=${id}`, def.domain).toString(),
|
||||
downloadUrl: new URL(`${def.umt.downloadPath}?id=${id}`, def.domain).toString(),
|
||||
tags: [],
|
||||
raw: { basicInfoText: basicText.trim().slice(0, 500), peersText: peersText.trim(), ...promo },
|
||||
observedAt,
|
||||
};
|
||||
if (subtitle) detail.subtitle = subtitle;
|
||||
if (descText) detail.descriptionText = descText.slice(0, 20_000);
|
||||
const imdb = extractImdbId(descHtml) ?? extractImdbId(descText);
|
||||
if (imdb) detail.imdbId = imdb;
|
||||
const douban = extractDoubanId(descHtml);
|
||||
if (douban) detail.doubanId = douban;
|
||||
return { kind: "ok", detail };
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import { redactUrl } from "../../rss/redact";
|
||||
import type { PromotionType, SitePromotion, SiteRelease } from "../types";
|
||||
import { parseSiteTime } from "../util";
|
||||
import type { IndexerDefinition } from "./schema";
|
||||
import type { SpiderTorrent } from "./spider";
|
||||
|
||||
/**
|
||||
* MP 系数 → 统一促销类型。MP 只有 downloadvolumefactor/uploadvolumefactor/freedate 三个字段,
|
||||
* 类型名由系数组合推出;字段未配置(undefined)时为 unknown。
|
||||
*/
|
||||
export function promotionFromFactors(
|
||||
dl: number | undefined,
|
||||
ul: number | undefined,
|
||||
freedate: string | null,
|
||||
): SitePromotion {
|
||||
if (dl === undefined && ul === undefined)
|
||||
return { type: "unknown", downloadFactor: null, uploadFactor: null, expiresAt: null };
|
||||
const d = dl ?? 1;
|
||||
const u = ul ?? 1;
|
||||
let type: PromotionType;
|
||||
if (d === 0) type = u >= 2 ? "2xfree" : "free";
|
||||
else if (d === 0.5) type = u >= 2 ? "2x50%" : "50%";
|
||||
else if (d === 0.3) type = "30%";
|
||||
else if (d === 1 && u >= 2) type = "2x";
|
||||
else if (d === 1 && u === 1) type = "none";
|
||||
else type = "unknown";
|
||||
return { type, downloadFactor: dl ?? null, uploadFactor: ul ?? null, expiresAt: type === "none" ? null : freedate };
|
||||
}
|
||||
|
||||
/** 从详情链接取站点资源 ID(NexusPHP `details.php?id=`)。 */
|
||||
export function releaseIdFromUrl(url: string | undefined): string | null {
|
||||
if (!url) return null;
|
||||
return /[?&]id=(\d+)/.exec(url)?.[1] ?? null;
|
||||
}
|
||||
|
||||
export function categoryName(def: IndexerDefinition, id: string | undefined): string | null {
|
||||
if (!id) return null;
|
||||
const named = def.umt.categoryNames?.[id];
|
||||
if (named) return named;
|
||||
for (const k of ["movie", "tv", "music"] as const) {
|
||||
const hit = (def.category?.[k] ?? []).find((c) => String(c.id) === id);
|
||||
if (hit?.desc) return hit.desc;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** SpiderTorrent(MP TorrentInfo 字段)→ SiteRelease。raw 保存完整 MP 字段(enclosure 脱敏)。 */
|
||||
export function spiderTorrentToRelease(
|
||||
t: SpiderTorrent,
|
||||
def: IndexerDefinition,
|
||||
observedAt: string,
|
||||
): SiteRelease | null {
|
||||
const id = releaseIdFromUrl(t.page_url) ?? releaseIdFromUrl(t.enclosure);
|
||||
if (!id || !t.title) return null;
|
||||
const tz = def.umt.timezone;
|
||||
const freedate = t.freedate ? parseSiteTime(t.freedate, tz) : null;
|
||||
const raw: Record<string, unknown> = { ...t };
|
||||
if (t.enclosure) raw.enclosure = redactUrl(t.enclosure);
|
||||
const release: SiteRelease = {
|
||||
siteId: def.id,
|
||||
siteReleaseId: id,
|
||||
title: t.title,
|
||||
category: { id: t.category_id ?? null, name: categoryName(def, t.category_id) },
|
||||
sizeBytes: t.size ?? null,
|
||||
seeders: t.seeders ?? null,
|
||||
leechers: t.peers ?? null,
|
||||
snatched: t.grabs ?? null,
|
||||
promotion: promotionFromFactors(t.downloadvolumefactor, t.uploadvolumefactor, freedate),
|
||||
hr: t.hit_and_run === undefined ? "unknown" : t.hit_and_run,
|
||||
uploadedAt: parseSiteTime(t.pubdate, tz),
|
||||
detailUrl: t.page_url ?? new URL(`${def.umt.detailPath}?id=${id}`, def.domain).toString(),
|
||||
downloadUrl: t.enclosure
|
||||
? redactUrl(t.enclosure)
|
||||
: new URL(`${def.umt.downloadPath}?id=${id}`, def.domain).toString(),
|
||||
tags: t.labels ?? [],
|
||||
raw,
|
||||
observedAt,
|
||||
};
|
||||
if (t.description) release.subtitle = t.description;
|
||||
if (t.imdbid) release.imdbId = t.imdbid;
|
||||
return release;
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
/**
|
||||
* 站点索引定义(indexer)schema。
|
||||
*
|
||||
* 来源:MoviePilot e6820a8 `app/modules/indexer/spider/__init__.py` `SiteSpider.__init__`/`__get_search_url`/`get_info`
|
||||
* 读取的全部键,以及 `app/application/site/sites.pyi` `SitesHelper.get_indexer(domain) -> dict`。
|
||||
* MP 的真实站点定义在加密资源 user.sites.v3.bin 中不可读,本项目的站点定义按同一 schema 手写
|
||||
* (见 ../definitions/*.json)。`umt` 键是本项目扩展(限流、时区、登录判定、详情页、分类名),MP 无对应。
|
||||
*/
|
||||
|
||||
/** 字段过滤器(MP `SiteSpider.__filter_text`)。 */
|
||||
export type SpiderFilter =
|
||||
| { name: "re_search"; args: [string, number] }
|
||||
| { name: "split"; args: [string, number] }
|
||||
| { name: "replace"; args: [string, string] }
|
||||
| { name: "dateparse"; args: string }
|
||||
| { name: "strip"; args?: unknown }
|
||||
| { name: "appendleft"; args: string }
|
||||
| { name: "querystring"; args: string };
|
||||
|
||||
/** 字段选择器(MP `_safe_query` / `__attribute_or_text` / `__index` / `__remove` 读取的键)。 */
|
||||
export interface SpiderFieldSelector {
|
||||
selector?: string;
|
||||
/** description 专用:selectors 等价 selector */
|
||||
selectors?: string;
|
||||
attribute?: string;
|
||||
index?: number;
|
||||
contents?: number;
|
||||
/** 逗号+空格分隔的待移除子元素选择器 */
|
||||
remove?: string;
|
||||
filters?: SpiderFilter[];
|
||||
/** Jinja 模板(title/description/date) */
|
||||
text?: string;
|
||||
/** 下载/上传系数:按声明顺序首个命中的选择器 → 系数 */
|
||||
case?: Record<string, number>;
|
||||
}
|
||||
|
||||
export interface SpiderCategoryItem {
|
||||
id: string | number;
|
||||
cat?: string;
|
||||
desc?: string;
|
||||
/** 分类参数名(如 HHClub 的 cat[]) */
|
||||
param?: string;
|
||||
/** 参数值(缺省用 id) */
|
||||
value?: string | number;
|
||||
/** 附加开关 */
|
||||
params?: Record<string, string | number>;
|
||||
}
|
||||
|
||||
export interface SpiderCategoryConfig {
|
||||
movie?: SpiderCategoryItem[];
|
||||
tv?: SpiderCategoryItem[];
|
||||
music?: SpiderCategoryItem[];
|
||||
field?: string;
|
||||
delimiter?: string;
|
||||
param?: string;
|
||||
}
|
||||
|
||||
export interface SpiderListConfig {
|
||||
selector: string;
|
||||
}
|
||||
|
||||
export interface SpiderFields {
|
||||
[name: string]: SpiderFieldSelector | undefined;
|
||||
}
|
||||
|
||||
export interface IndexerDefinition {
|
||||
id: string;
|
||||
name: string;
|
||||
domain: string;
|
||||
encoding?: string;
|
||||
public?: boolean;
|
||||
proxy?: boolean;
|
||||
language?: string;
|
||||
/** 专用解析器(mTorrent 等);缺省走通用 SiteSpider */
|
||||
parser?: string;
|
||||
result_num?: number;
|
||||
timeout?: number;
|
||||
/** 站点只提供某一媒体类型时填写(电影/电视剧/音乐) */
|
||||
media_type?: string;
|
||||
search?: {
|
||||
paths: Array<{ path: string; type?: "all" | "movie" | "tv" | "music" }>;
|
||||
params?: Record<string, string | number>;
|
||||
imdbid_format?: string;
|
||||
result_media_type?: "requested";
|
||||
};
|
||||
batch?: { delimiter?: string; space_replace?: string };
|
||||
browse?: { path: string; start?: number; list?: SpiderListConfig; fields?: SpiderFields };
|
||||
category?: SpiderCategoryConfig;
|
||||
torrents: { list: SpiderListConfig; fields: SpiderFields };
|
||||
/** 本项目扩展(MP 无对应) */
|
||||
umt: UmtSiteExtension;
|
||||
}
|
||||
|
||||
export interface UmtSiteExtension {
|
||||
/** 定义版本,入 site_snapshot */
|
||||
version: string;
|
||||
kind: "nexusphp";
|
||||
displayName?: string;
|
||||
/** 站点时间时区偏移,如 "+08:00";pubdate/freedate 按此转 UTC */
|
||||
timezone: string;
|
||||
rateLimit: { minIntervalMs: number; burst: number };
|
||||
capabilities: { list: boolean; search: boolean; detail: boolean; download: boolean; rss: boolean };
|
||||
/** 原始分类 ID → 站点分类名(列表只有 ID 时补名称;不做投影) */
|
||||
categoryNames?: Record<string, string>;
|
||||
/** 登录成功标志;缺失且命中登录页判定时为 auth_expired */
|
||||
loginSuccess?: string;
|
||||
/** 列表为空时用于区分"确实无结果"与"结构变化"的提示文本 */
|
||||
emptyMarkers?: string[];
|
||||
/** 分页链接选择器(估算总页数) */
|
||||
pagination?: string;
|
||||
/** 详情页解析(MP 通用 spider 不含详情页,本项目扩展) */
|
||||
detail?: {
|
||||
path: string;
|
||||
title: string;
|
||||
rowLabel: string;
|
||||
rowValue: string;
|
||||
labels: { subtitle: string[]; basicInfo: string[]; peers: string[] };
|
||||
description: string;
|
||||
/** 促销/HR 标记所在范围 */
|
||||
headerScope: string;
|
||||
notFoundMarkers: string[];
|
||||
};
|
||||
/** 下载路径(按资源 ID 拼接) */
|
||||
downloadPath: string;
|
||||
detailPath: string;
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
audiencesDefinition,
|
||||
createSiteRateLimiter,
|
||||
createSpiderSiteAdapter,
|
||||
type IndexerDefinition,
|
||||
type SiteAdapter,
|
||||
SiteError,
|
||||
SiteSpider,
|
||||
staticSecretResolver,
|
||||
} from "../index";
|
||||
|
||||
let site: MockNexusPhpSite | undefined;
|
||||
afterEach(async () => {
|
||||
await site?.close();
|
||||
site = undefined;
|
||||
});
|
||||
|
||||
function adapterFor(s: MockNexusPhpSite, def = audiencesDefinition, cookie: string | null = s.cookie): SiteAdapter {
|
||||
return createSpiderSiteAdapter(def, {
|
||||
baseUrl: `${s.baseUrl}/`,
|
||||
secrets: staticSecretResolver(cookie ? { [def.id]: { cookie } } : {}),
|
||||
limiter: createSiteRateLimiter({ minIntervalMs: 0 }),
|
||||
});
|
||||
}
|
||||
|
||||
async function expectSiteError(p: Promise<unknown>, kind: string): Promise<SiteError> {
|
||||
try {
|
||||
await p;
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(SiteError);
|
||||
expect((e as SiteError).kind).toBe(kind);
|
||||
return e as SiteError;
|
||||
}
|
||||
throw new Error(`expected SiteError ${kind}`);
|
||||
}
|
||||
|
||||
describe("SiteSpider(MP spider 迁移)", () => {
|
||||
const def: IndexerDefinition = {
|
||||
...audiencesDefinition,
|
||||
domain: "https://example.com/",
|
||||
category: {
|
||||
movie: [{ id: "401" }],
|
||||
tv: [{ id: "402" }, { id: "403" }],
|
||||
},
|
||||
};
|
||||
|
||||
it("__get_search_url:默认参数、search_area 仅 IMDb 保留、mtype 选分类组、cat 白名单", () => {
|
||||
const u1 = new URL(new SiteSpider(def, { keyword: "Dune", page: 2 }).getSearchUrl());
|
||||
expect(u1.pathname).toBe("/torrents.php");
|
||||
expect(Object.fromEntries(u1.searchParams)).toEqual({
|
||||
search_mode: "0",
|
||||
search_area: "0",
|
||||
page: "2",
|
||||
notnewword: "1",
|
||||
search: "Dune",
|
||||
cat401: "1",
|
||||
cat402: "1",
|
||||
cat403: "1",
|
||||
});
|
||||
const u2 = new URL(new SiteSpider(def, { keyword: "tt15239678" }).getSearchUrl());
|
||||
expect(u2.searchParams.get("search_area")).toBe("4");
|
||||
const u3 = new URL(new SiteSpider(def, { keyword: "x", mtype: "tv", cat: "403" }).getSearchUrl());
|
||||
expect([...u3.searchParams.keys()].filter((k) => k.startsWith("cat"))).toEqual(["cat403"]);
|
||||
const u4 = new URL(new SiteSpider(def, { keyword: ["A B", "C"] }).getSearchUrl());
|
||||
expect(u4.searchParams.get("search")).toBe("A_B C");
|
||||
expect(u4.searchParams.get("search_mode")).toBe("1");
|
||||
// 浏览:browse.path + start
|
||||
expect(new SiteSpider(def, { page: 3 }).getSearchUrl()).toBe("https://example.com/torrents.php?page=3");
|
||||
});
|
||||
|
||||
it("分类参数 param(cat[] 重复)与 field 拼接两种写法", () => {
|
||||
const withParam: IndexerDefinition = { ...def, category: { param: "cat[]", tv: [{ id: "402" }, { id: "403" }] } };
|
||||
const u = new URL(new SiteSpider(withParam, { keyword: "x" }).getSearchUrl());
|
||||
expect(u.searchParams.getAll("cat[]")).toEqual(["402", "403"]);
|
||||
const withField: IndexerDefinition = { ...def, category: { field: "cat", delimiter: ",", tv: [{ id: "402" }] } };
|
||||
expect(new URL(new SiteSpider(withField, { keyword: "x" }).getSearchUrl()).searchParams.get("cat")).toBe(",402");
|
||||
});
|
||||
|
||||
it("get_info 字段 DSL:模板标题、remove、dateparse、case 系数、querystring、hr 配置后两态", () => {
|
||||
const html = `<html><head><title>t</title></head><body><table class="torrents"><tr><td>h</td></tr>
|
||||
<tr><td><a href="?cat=402"><img></a></td><td><table class="torrentname"><tr><td class="embedded">
|
||||
<a href="details.php?id=100" title="Full.Title.2025.2160p">Full.Title…</a> <img class="free">
|
||||
<font class="subtitle">副标题 <span>[置顶]</span><a>x</a></font><span class="label">中字</span><span class="label">DIY</span>
|
||||
</td></tr></table></td><td>0</td><td><span title="2025-05-01 12:13:14">1天</span></td><td>1.5 GB</td><td>1,234</td><td>5/7</td><td>9</td></tr>
|
||||
<tr><td><a href="?cat=401"></a></td><td><table class="torrentname"><tr><td class="embedded">
|
||||
<a href="details.php?id=101">No.Title.Attr</a> <img class="hitandrun">
|
||||
</td></tr></table></td><td>0</td><td><span>3 小时</span></td><td>--</td><td>x</td><td></td><td></td></tr></table></body></html>`;
|
||||
const d: IndexerDefinition = {
|
||||
...def,
|
||||
torrents: {
|
||||
list: { selector: 'table.torrents > tr:has("table.torrentname")' },
|
||||
fields: {
|
||||
...def.torrents.fields,
|
||||
description: { selector: "font.subtitle", remove: "span,a" },
|
||||
labels: { selector: "span.label" },
|
||||
hr: { selector: "img.hitandrun" },
|
||||
downloadvolumefactor: { case: { "img.free": 0, "*": 1 } },
|
||||
},
|
||||
},
|
||||
};
|
||||
const { torrents } = new SiteSpider(d).parse(html);
|
||||
expect(torrents[0]).toMatchObject({
|
||||
title: "Full.Title.2025.2160p",
|
||||
description: "副标题",
|
||||
page_url: "https://example.com/details.php?id=100",
|
||||
size: 1610612736,
|
||||
seeders: 1234,
|
||||
peers: 5,
|
||||
grabs: 9,
|
||||
downloadvolumefactor: 0,
|
||||
uploadvolumefactor: 1,
|
||||
pubdate: "2025-05-01 12:13:14",
|
||||
labels: ["中字", "DIY"],
|
||||
hit_and_run: false,
|
||||
category: "电视剧",
|
||||
category_id: "402",
|
||||
});
|
||||
expect(torrents[1]).toMatchObject({
|
||||
title: "No.Title.Attr",
|
||||
hit_and_run: true,
|
||||
size: 0,
|
||||
seeders: 0,
|
||||
category: "电影",
|
||||
});
|
||||
expect(torrents[1]!.pubdate).toBeUndefined(); // 相对时间不写入 pubdate(MP __is_invalid_pubdate_text)
|
||||
});
|
||||
|
||||
it("登录/权限页判定(MP __is_login_or_permission_page)", () => {
|
||||
const r = new SiteSpider(def).parse("<html><head><title>观众 :: 登录</title></head><body></body></html>");
|
||||
expect(r).toMatchObject({ isError: true, loginPage: true, torrents: [] });
|
||||
const r2 = new SiteSpider(def).parse("<html><body><p>该页面必须在登录后才能访问</p></body></html>");
|
||||
expect(r2.loginPage).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("观众适配器(spider + mock NexusPHP)", () => {
|
||||
it("列表:page=1 发送 page=0;免费+截止、时间、原始分类、副标题;H&R 未配置为 unknown", async () => {
|
||||
site = await createMockNexusPhpSite();
|
||||
const page = await adapterFor(site).list(1);
|
||||
const req = site.requests[0]!;
|
||||
expect(req.path).toBe("/torrents.php?page=0");
|
||||
expect(req.headers.cookie).toBe(site.cookie);
|
||||
expect(page.items).toHaveLength(5);
|
||||
expect(page.totalPages).toBe(1);
|
||||
const dune = page.items[0]!;
|
||||
expect(dune).toMatchObject({
|
||||
siteId: "audiences",
|
||||
siteReleaseId: "812345",
|
||||
title: "Dune.Part.Two.2024.2160p.UHD.BluRay.REMUX.DV.HDR.HEVC.TrueHD.Atmos.7.1-FraMeSToR",
|
||||
subtitle: "沙丘2 / 沙丘：第二部 | 杜比视界 国英双语 中字",
|
||||
category: { id: "401", name: "电影" },
|
||||
sizeBytes: 84202833838,
|
||||
seeders: 1204,
|
||||
leechers: 17,
|
||||
snatched: 3388,
|
||||
promotion: { type: "free", downloadFactor: 0, uploadFactor: 1, expiresAt: "2026-10-01T04:00:00.000Z" },
|
||||
hr: "unknown",
|
||||
uploadedAt: "2026-09-26T13:14:03.000Z",
|
||||
});
|
||||
expect(dune.raw).toMatchObject({ category: "电影", category_id: "401", pubdate: "2026-09-26 21:14:03" });
|
||||
expect(dune.detailUrl).toBe(`${site.baseUrl}/details.php?id=812345&hit=1`);
|
||||
expect(JSON.stringify(page)).not.toContain("mock-session-secret");
|
||||
});
|
||||
|
||||
it("列表:综艺保留 403、50%+2x、2x、截断标题、TB/KB", async () => {
|
||||
site = await createMockNexusPhpSite();
|
||||
const { items } = await adapterFor(site).list(1);
|
||||
const byId = Object.fromEntries(items.map((i) => [i.siteReleaseId, i]));
|
||||
expect(byId["812346"]!.category).toEqual({ id: "403", name: "综艺" });
|
||||
expect(byId["812346"]!.raw.category).toBe("电视剧");
|
||||
expect(byId["812346"]!.promotion.type).toBe("none");
|
||||
expect(byId["812347"]!.promotion).toMatchObject({ type: "2x50%", downloadFactor: 0.5, uploadFactor: 2 });
|
||||
expect(byId["812347"]!.promotion.expiresAt).toBe("2026-09-29T00:30:00.000Z");
|
||||
expect(byId["812347"]!.sizeBytes).toBe(1294529987);
|
||||
expect(byId["812348"]!.title).toBe("The.Last.of.Us.S02.2160p.MAX.WEB-DL.DDP5.1.Atmos.D…");
|
||||
expect(byId["812348"]!.sizeBytes).toBe(2308974418330);
|
||||
expect(byId["812349"]!.promotion).toMatchObject({
|
||||
type: "2x",
|
||||
downloadFactor: 1,
|
||||
uploadFactor: 2,
|
||||
expiresAt: null,
|
||||
});
|
||||
expect(byId["812349"]!.category).toEqual({ id: "406", name: "纪录片" });
|
||||
});
|
||||
|
||||
it("配置 hr 字段后 H&R 为 true/false", async () => {
|
||||
site = await createMockNexusPhpSite();
|
||||
const def: IndexerDefinition = {
|
||||
...audiencesDefinition,
|
||||
torrents: {
|
||||
...audiencesDefinition.torrents,
|
||||
fields: { ...audiencesDefinition.torrents.fields, hr: { selector: "img.hitandrun" } },
|
||||
},
|
||||
};
|
||||
const { items } = await adapterFor(site, def).list(1);
|
||||
expect(items.find((i) => i.siteReleaseId === "812346")!.hr).toBe(true);
|
||||
expect(items.find((i) => i.siteReleaseId === "812345")!.hr).toBe(false);
|
||||
});
|
||||
|
||||
it("搜索:关键词清洗与分类下推;无结果返回空页", async () => {
|
||||
site = await createMockNexusPhpSite();
|
||||
const a = adapterFor(site);
|
||||
const r = await a.search("奔跑吧!", { categoryIds: ["403"] });
|
||||
const u = new URL(site.requests[0]!.path, site.baseUrl);
|
||||
expect(u.searchParams.get("search")).toBe("奔跑吧");
|
||||
expect(u.searchParams.get("cat403")).toBe("1");
|
||||
expect(u.searchParams.get("cat401")).toBeNull();
|
||||
expect(r.items.map((i) => i.siteReleaseId)).toEqual(["812346"]);
|
||||
expect((await a.search("不存在的片名")).items).toEqual([]);
|
||||
});
|
||||
|
||||
it("分页:pageSize=2 时 page=2 发送 page=1,总页数由分页链接估计", async () => {
|
||||
site = await createMockNexusPhpSite({ pageSize: 2 });
|
||||
const p2 = await adapterFor(site).list(2);
|
||||
expect(site.requests[0]!.path).toBe("/torrents.php?page=1");
|
||||
expect(p2).toMatchObject({ totalPages: 3, totalPagesEstimated: true });
|
||||
expect(p2.items.map((i) => i.siteReleaseId)).toEqual(["812347", "812348"]);
|
||||
});
|
||||
|
||||
it("详情:标题/副标题/体积/同伴/促销/IMDb/豆瓣;不存在 → not_found", async () => {
|
||||
site = await createMockNexusPhpSite();
|
||||
const a = adapterFor(site);
|
||||
const d = await a.detail("812345");
|
||||
expect(d).toMatchObject({
|
||||
title: "Dune.Part.Two.2024.2160p.UHD.BluRay.REMUX.DV.HDR.HEVC.TrueHD.Atmos.7.1-FraMeSToR",
|
||||
subtitle: "沙丘2 / 沙丘：第二部 | 杜比视界 国英双语 中字",
|
||||
sizeBytes: 84202833838,
|
||||
seeders: 1204,
|
||||
leechers: 17,
|
||||
imdbId: "tt15239678",
|
||||
doubanId: "35575567",
|
||||
uploadedAt: "2026-09-26T13:14:03.000Z",
|
||||
category: { id: "401", name: "电影" },
|
||||
promotion: { type: "free", expiresAt: "2026-10-01T04:00:00.000Z" },
|
||||
});
|
||||
await expectSiteError(a.detail("999999"), "not_found");
|
||||
});
|
||||
|
||||
it("下载目标:带 Cookie 取 .torrent 并校验 infohash", async () => {
|
||||
site = await createMockNexusPhpSite();
|
||||
const t = await adapterFor(site).downloadTarget("812345");
|
||||
expect(t.kind).toBe("torrent");
|
||||
if (t.kind !== "torrent") return;
|
||||
expect(t.infoHash).toBe(site.infoHashOf("812345"));
|
||||
expect(t.name).toContain("Dune.Part.Two");
|
||||
expect(t.fileName).toContain(".torrent");
|
||||
});
|
||||
|
||||
it("登录失效识别:列表/详情/下载抛 auth_expired,健康检查报告", async () => {
|
||||
site = await createMockNexusPhpSite();
|
||||
const a = adapterFor(site);
|
||||
expect((await a.healthCheck()).status).toBe("ok");
|
||||
site.setLoggedIn(false);
|
||||
const err = await expectSiteError(a.list(1), "auth_expired");
|
||||
expect(err.isAuthError).toBe(true);
|
||||
await expectSiteError(a.detail("812345"), "auth_expired");
|
||||
await expectSiteError(a.downloadTarget("812345"), "auth_expired");
|
||||
expect(await a.healthCheck()).toMatchObject({ ok: false, status: "auth_expired" });
|
||||
});
|
||||
|
||||
it("登录失效(302 login.php)与缺少 Cookie", async () => {
|
||||
site = await createMockNexusPhpSite({ loginMode: "redirect" });
|
||||
await expectSiteError(
|
||||
adapterFor(site, audiencesDefinition, "c_secure_uid=1; c_secure_pass=wrong").list(1),
|
||||
"auth_expired",
|
||||
);
|
||||
const before = site.requests.length;
|
||||
await expectSiteError(adapterFor(site, audiencesDefinition, null).list(1), "auth_missing");
|
||||
expect(site.requests.length).toBe(before);
|
||||
});
|
||||
|
||||
it("页面改版 → parse_error;429 + Retry-After → rate_limited", async () => {
|
||||
site = await createMockNexusPhpSite();
|
||||
const a = adapterFor(site);
|
||||
site.setBrokenLayout(true);
|
||||
await expectSiteError(a.list(1), "parse_error");
|
||||
site.setBrokenLayout(false);
|
||||
site.failNext(1, 429, "7");
|
||||
expect((await expectSiteError(a.list(1), "rate_limited")).retryAfterMs).toBe(7000);
|
||||
});
|
||||
|
||||
it("站点级限流:列表/搜索/详情/下载共享同一令牌桶", async () => {
|
||||
site = await createMockNexusPhpSite();
|
||||
let clock = 0;
|
||||
const sleeps: number[] = [];
|
||||
const limiter = createSiteRateLimiter({
|
||||
minIntervalMs: 3000,
|
||||
now: () => clock,
|
||||
sleep: async (ms) => {
|
||||
sleeps.push(ms);
|
||||
clock += ms;
|
||||
},
|
||||
});
|
||||
const a = createSpiderSiteAdapter(audiencesDefinition, {
|
||||
baseUrl: `${site.baseUrl}/`,
|
||||
secrets: staticSecretResolver({ audiences: { cookie: site.cookie } }),
|
||||
limiter,
|
||||
});
|
||||
await a.list(1);
|
||||
await a.search("Dune");
|
||||
await a.detail("812345");
|
||||
await a.downloadTarget("812345");
|
||||
expect(sleeps).toEqual([3000, 3000, 3000]);
|
||||
expect(limiter.stats.granted).toBe(4);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,755 @@
|
||||
/**
|
||||
* 通用站点爬虫(配置驱动)。
|
||||
*
|
||||
* 迁移自 MoviePilot e6820a8 `app/modules/indexer/spider/__init__.py`:
|
||||
* - `select_media_categories` / `resolve_category_media_type` → 同名函数
|
||||
* - `SiteSpider.__get_search_url` / `__format_search_word` → `SiteSpider.getSearchUrl` / `formatSearchWord`
|
||||
* - `SiteSpider.get_info` 及 `__get_title/__get_description/__get_detail/__get_download/__get_grabs/__get_leechers/
|
||||
* __get_seeders/__get_size/__get_imdbid/__get_downloadvolumefactor/__get_uploadvolumefactor/__get_pubdate/__get_date/
|
||||
* __get_date_from_cell/__get_date_elapsed/__get_free_date/__get_labels/__get_hit_and_run/__get_category`
|
||||
* → `SiteSpider.getInfo` 与同名私有方法
|
||||
* - `_safe_query / __filter_text / __remove / __attribute_or_text / __index` → 同名方法
|
||||
* - `__is_login_or_permission_page` → `isLoginOrPermissionPage`
|
||||
* - `parse` → `SiteSpider.parse`(不含 Rust 加速分支与字幕分支)
|
||||
* - `app/foundation/size.py parse_size` → `parseSizeMp`
|
||||
*
|
||||
* 与 MP 的差异(均在 README 与迁移账本登记):
|
||||
* 1. 额外输出 `category_id`(原始分类 ID),`category` 仍是 MP 的媒体类型投影(REQ §4.5 原始分类不得丢失)。
|
||||
* 2. 系数 `selector` 模式用浮点而非 int()(MP 对 0.5 抛异常导致整条丢失,迁移账本 SA4)。
|
||||
* 3. 不发请求:HTTP/Cookie/UA 由 adapter 注入,本类只负责 URL 构造与 HTML 解析(便于限流与脱敏统一)。
|
||||
*/
|
||||
import { type CheerioAPI, load } from "cheerio";
|
||||
import type { IndexerDefinition, SpiderFieldSelector, SpiderFields, SpiderFilter, SpiderListConfig } from "./schema";
|
||||
import { renderTemplate } from "./template";
|
||||
|
||||
type Sel = ReturnType<CheerioAPI>;
|
||||
|
||||
/** MP `MediaType` 取值 */
|
||||
export type MpMediaType = "电影" | "电视剧" | "音乐" | "未知";
|
||||
export type MediaKind = "movie" | "tv" | "music";
|
||||
|
||||
const MEDIA_VALUE: Record<MediaKind, MpMediaType> = { movie: "电影", tv: "电视剧", music: "音乐" };
|
||||
|
||||
/**
|
||||
* 以 htmlparser2(HTML 模式)加载:与 MP 的 lxml.html 一致不自动插入 <tbody>,
|
||||
* 使 `table.torrents > tr` 这类 MP 风格选择器保持原语义。
|
||||
*/
|
||||
export function loadHtml(html: string): CheerioAPI {
|
||||
return load(html, { xml: { xmlMode: false, decodeEntities: true } });
|
||||
}
|
||||
|
||||
/**
|
||||
* PyQuery(cssselect)接受 `:has("sel")` 引号写法,cheerio-select 不接受;统一去引号以兼容 MP 风格定义。
|
||||
*/
|
||||
export function normalizeSelector(sel: string): string {
|
||||
return sel.replace(/:has\((["'])(.*?)\1\)/g, ":has($2)");
|
||||
}
|
||||
|
||||
/** 解析结果:键名与 MP `TorrentInfo` 字段一致,`category_id` 为本项目扩展。 */
|
||||
export interface SpiderTorrent {
|
||||
title?: string;
|
||||
description?: string;
|
||||
page_url?: string;
|
||||
enclosure?: string;
|
||||
grabs?: number;
|
||||
peers?: number;
|
||||
seeders?: number;
|
||||
size?: number;
|
||||
imdbid?: string;
|
||||
downloadvolumefactor?: number;
|
||||
uploadvolumefactor?: number;
|
||||
pubdate?: string;
|
||||
date_elapsed?: string;
|
||||
freedate?: string;
|
||||
labels?: string[];
|
||||
hit_and_run?: boolean;
|
||||
category?: MpMediaType;
|
||||
category_id?: string;
|
||||
}
|
||||
|
||||
/** MP `select_media_categories` */
|
||||
export function selectMediaCategories(category: IndexerDefinition["category"], mtype?: MediaKind) {
|
||||
if (!category) return [];
|
||||
if (mtype === "tv") return category.tv ?? [];
|
||||
if (mtype === "movie") return category.movie ?? [];
|
||||
if (mtype === "music") return category.music ?? [];
|
||||
return [...(category.movie ?? []), ...(category.tv ?? []), ...(category.music ?? [])];
|
||||
}
|
||||
|
||||
/** MP `resolve_category_media_type`:同一 ID 只落在一个媒体类型里才投影,否则未知。 */
|
||||
export function resolveCategoryMediaType(value: unknown, category: IndexerDefinition["category"]): MpMediaType {
|
||||
if (value == null || !category) return "未知";
|
||||
const id = String(value);
|
||||
const matches = (["movie", "tv", "music"] as const).filter((k) =>
|
||||
(category[k] ?? []).some((item) => item.id != null && String(item.id) === id),
|
||||
);
|
||||
return matches.length === 1 ? MEDIA_VALUE[matches[0]!] : "未知";
|
||||
}
|
||||
|
||||
/** MP `size_tools.parse_size`:二进制单位,round。 */
|
||||
export function parseSizeMp(text: string | number | null | undefined): number {
|
||||
if (!text) return 0;
|
||||
const s = String(text);
|
||||
if (/^\d+$/.test(s)) return Number(s);
|
||||
const normalized = s.replace(/,/g, "").replace(/ /g, "").toUpperCase();
|
||||
const sizeText = normalized.replace(/[KMGTPI]*B?/gi, "");
|
||||
const size = Number(sizeText);
|
||||
if (sizeText === "" || !Number.isFinite(size)) return 0;
|
||||
let v = size;
|
||||
if (normalized.includes("PB") || normalized.includes("PIB")) v *= 1024 ** 5;
|
||||
else if (normalized.includes("TB") || normalized.includes("TIB")) v *= 1024 ** 4;
|
||||
else if (normalized.includes("GB") || normalized.includes("GIB")) v *= 1024 ** 3;
|
||||
else if (normalized.includes("MB") || normalized.includes("MIB")) v *= 1024 ** 2;
|
||||
else if (normalized.includes("KB") || normalized.includes("KIB")) v *= 1024;
|
||||
return Math.round(v);
|
||||
}
|
||||
|
||||
/** strptime 结果(MP dateparse 过滤器返回 datetime) */
|
||||
class ParsedDateTime {
|
||||
constructor(readonly text: string) {}
|
||||
}
|
||||
|
||||
const STRPTIME_TOKENS: Record<string, string> = {
|
||||
"%Y": "(\\d{4})",
|
||||
"%m": "(\\d{1,2})",
|
||||
"%d": "(\\d{1,2})",
|
||||
"%H": "(\\d{1,2})",
|
||||
"%M": "(\\d{1,2})",
|
||||
"%S": "(\\d{1,2})",
|
||||
};
|
||||
|
||||
/** Python strptime 的最小子集(%Y %m %d %H %M %S),失败抛错(与 MP 一致由调用方吞掉)。 */
|
||||
function strptime(text: string, fmt: string): ParsedDateTime {
|
||||
const order: string[] = [];
|
||||
const re = fmt.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/%[YmdHMS]/g, (t) => {
|
||||
order.push(t);
|
||||
return STRPTIME_TOKENS[t]!;
|
||||
});
|
||||
const m = new RegExp(`^${re}$`).exec(text);
|
||||
if (!m) throw new Error(`time data '${text}' does not match format '${fmt}'`);
|
||||
const v: Record<string, number> = { "%Y": 1900, "%m": 1, "%d": 1, "%H": 0, "%M": 0, "%S": 0 };
|
||||
order.forEach((t, i) => {
|
||||
v[t] = Number(m[i + 1]);
|
||||
});
|
||||
const p = (n: number) => String(n).padStart(2, "0");
|
||||
return new ParsedDateTime(`${v["%Y"]}-${p(v["%m"]!)}-${p(v["%d"]!)} ${p(v["%H"]!)}:${p(v["%M"]!)}:${p(v["%S"]!)}`);
|
||||
}
|
||||
|
||||
const STD_TIME = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/;
|
||||
|
||||
/** MP `time_tools.normalize_datetime` 的保守子集:只接受可确定解析的绝对时间。 */
|
||||
function normalizeDatetime(value: string): string {
|
||||
const m = /^(\d{4})[-/年.](\d{1,2})[-/月.](\d{1,2})日?(?:[ T](\d{1,2}):(\d{1,2})(?::(\d{1,2}))?)?$/.exec(
|
||||
value.trim(),
|
||||
);
|
||||
if (!m) return value;
|
||||
const p = (n: string | undefined) => String(Number(n ?? 0)).padStart(2, "0");
|
||||
return `${m[1]}-${p(m[2])}-${p(m[3])} ${p(m[4])}:${p(m[5])}:${p(m[6])}`;
|
||||
}
|
||||
|
||||
function isRelativePubdateText(pubdate: string | null | undefined): boolean {
|
||||
if (!pubdate) return false;
|
||||
const text = String(pubdate).trim().toLowerCase();
|
||||
if (/\d{4}[-/年]\d{1,2}/.test(text)) return false;
|
||||
if (text.includes("ago")) return true;
|
||||
return /\d+\s*(秒|分钟|分|小时|天|周|月|年)/.test(text);
|
||||
}
|
||||
|
||||
function isInvalidPubdateText(pubdate: string | null | undefined): boolean {
|
||||
if (!pubdate) return true;
|
||||
const text = String(pubdate).trim();
|
||||
if (text.toLowerCase() === "now" || text === "0") return true;
|
||||
if (isRelativePubdateText(text)) return true;
|
||||
return !STD_TIME.test(text);
|
||||
}
|
||||
|
||||
function pyIndex<T>(arr: T[], i: number): T | undefined {
|
||||
return i < 0 ? arr[arr.length + i] : arr[i];
|
||||
}
|
||||
|
||||
/** Python `parse_qs(urlparse(url).query).get(key)[0]` */
|
||||
function querystringValue(text: string, key: string): string {
|
||||
const q = text.includes("?") ? text.slice(text.indexOf("?") + 1) : "";
|
||||
const v = new URLSearchParams(q.split("#")[0]).getAll(key).filter((x) => x !== "");
|
||||
return v[0] ?? "";
|
||||
}
|
||||
|
||||
/** PyQuery `.text()` 近似:`<br>` 视为换行,行内空白折叠。 */
|
||||
function pqText(el: Sel): string {
|
||||
const c = el.clone();
|
||||
c.find("br").replaceWith("\n");
|
||||
return c
|
||||
.text()
|
||||
.split("\n")
|
||||
.map((l) => l.replace(/[ \t\r\f\v\u00a0]+/g, " ").trim())
|
||||
.filter((l) => l !== "")
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
export interface SiteSpiderQuery {
|
||||
keyword?: string | string[];
|
||||
mtype?: MediaKind;
|
||||
/** 逗号分隔的分类 ID 白名单(MP `cat`) */
|
||||
cat?: string;
|
||||
/** 站点内部页码(MP 0 起) */
|
||||
page?: number;
|
||||
}
|
||||
|
||||
export class SiteSpider {
|
||||
readonly def: IndexerDefinition;
|
||||
readonly keyword: string | string[] | undefined;
|
||||
readonly mtype: MediaKind | undefined;
|
||||
readonly cat: string | undefined;
|
||||
readonly page: number;
|
||||
readonly domain: string;
|
||||
readonly resultNum: number;
|
||||
readonly siteMediaType: MediaKind | undefined;
|
||||
private list: SpiderListConfig;
|
||||
private fields: SpiderFields;
|
||||
private requestedResultMediaType: MediaKind | undefined;
|
||||
private $!: CheerioAPI;
|
||||
private info: SpiderTorrent = {};
|
||||
|
||||
constructor(def: IndexerDefinition, q: SiteSpiderQuery = {}) {
|
||||
this.def = def;
|
||||
this.keyword = q.keyword;
|
||||
this.mtype = q.mtype;
|
||||
this.cat = q.cat;
|
||||
this.page = q.page ?? 0;
|
||||
this.siteMediaType = fromAgent(def.media_type);
|
||||
this.list = def.torrents.list;
|
||||
this.fields = def.torrents.fields ?? {};
|
||||
if (!q.keyword && def.browse) {
|
||||
this.list = def.browse.list ?? this.list;
|
||||
this.fields = def.browse.fields ?? this.fields;
|
||||
}
|
||||
this.domain = def.domain.endsWith("/") ? def.domain : `${def.domain}/`;
|
||||
this.resultNum = Number(def.result_num ?? 100);
|
||||
}
|
||||
|
||||
/** MP `__can_search` */
|
||||
canSearch(): boolean {
|
||||
if (this.siteMediaType && this.mtype && this.siteMediaType !== this.mtype) return false;
|
||||
return Boolean(this.def.search && this.domain);
|
||||
}
|
||||
|
||||
/** MP `__format_search_word` */
|
||||
private formatSearchWord(word: string): string {
|
||||
if (word && /^tt\d+$/.test(word) && this.def.search?.imdbid_format) {
|
||||
return this.def.search.imdbid_format
|
||||
.replace(/\{keyword\}/g, word)
|
||||
.replace(/\{imdbid\}/g, word)
|
||||
.replace(/\{imdbid_num\}/g, word.slice(2));
|
||||
}
|
||||
return word;
|
||||
}
|
||||
|
||||
/** MP `__get_search_url` */
|
||||
getSearchUrl(): string {
|
||||
const search = this.def.search ?? { paths: [] };
|
||||
const paths = search.paths ?? [];
|
||||
let torrentspath = "";
|
||||
let typedPathSelected = false;
|
||||
let categoryFilterSelected = false;
|
||||
if (paths.length === 1) {
|
||||
torrentspath = paths[0]!.path ?? "";
|
||||
} else {
|
||||
let fallback = "";
|
||||
const expected = this.mtype;
|
||||
for (const p of paths) {
|
||||
if (p.type === "all" && !fallback) fallback = p.path ?? "";
|
||||
if ((expected && p.type === expected) || (!expected && p.type === "all")) {
|
||||
torrentspath = p.path ?? "";
|
||||
typedPathSelected = Boolean(expected && p.type === expected);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!torrentspath) torrentspath = fallback;
|
||||
}
|
||||
|
||||
let searchurl: string;
|
||||
if (this.keyword && (typeof this.keyword === "string" || this.keyword.length > 0)) {
|
||||
let searchWord: string;
|
||||
let searchMode: string;
|
||||
if (Array.isArray(this.keyword)) {
|
||||
if (this.def.batch) {
|
||||
const delimiter = this.def.batch.delimiter || " ";
|
||||
const spaceReplace = this.def.batch.space_replace || " ";
|
||||
searchWord = this.keyword.map((k) => String(k).replace(/ /g, spaceReplace)).join(delimiter);
|
||||
} else {
|
||||
searchWord = this.keyword.join(" ");
|
||||
}
|
||||
searchMode = "1";
|
||||
} else {
|
||||
searchWord = this.keyword;
|
||||
searchMode = "0";
|
||||
}
|
||||
const isImdb = typeof this.keyword === "string" && /^tt\d+$/.test(this.keyword);
|
||||
searchWord = this.formatSearchWord(searchWord);
|
||||
const indexerParams: Record<string, string | number> = { ...(search.params ?? {}) };
|
||||
if (Object.keys(indexerParams).length > 0) {
|
||||
if (indexerParams.search_area && !isImdb) delete indexerParams.search_area;
|
||||
const params: Record<string, string | number | Array<string | number>> = {
|
||||
search_mode: searchMode,
|
||||
search_area: 0,
|
||||
page: this.page || 0,
|
||||
notnewword: 1,
|
||||
};
|
||||
for (const [k, v] of Object.entries(indexerParams)) params[k] = String(v).replace(/\{keyword\}/g, searchWord);
|
||||
const category = this.def.category;
|
||||
if (category) {
|
||||
const cats = selectMediaCategories(category, this.mtype);
|
||||
const allowed = this.cat ? new Set(this.cat.split(",")) : null;
|
||||
for (const c of cats) {
|
||||
if (allowed && !allowed.has(String(c.id))) continue;
|
||||
if (category.field) {
|
||||
categoryFilterSelected = true;
|
||||
const cur = String(params[category.field] ?? "");
|
||||
params[category.field] = cur + (category.delimiter ?? " ") + String(c.id);
|
||||
} else {
|
||||
const categoryParam = c.param || category.param;
|
||||
if (categoryParam) {
|
||||
categoryFilterSelected = true;
|
||||
const cid = c.value ?? c.id;
|
||||
const cur = params[categoryParam];
|
||||
if (cur === undefined) params[categoryParam] = cid;
|
||||
else if (Array.isArray(cur)) cur.push(cid);
|
||||
else params[categoryParam] = [cur, cid];
|
||||
} else {
|
||||
categoryFilterSelected = true;
|
||||
params[`cat${c.id}`] = 1;
|
||||
}
|
||||
}
|
||||
if (c.params) Object.assign(params, c.params);
|
||||
}
|
||||
}
|
||||
searchurl = combineUrl(this.domain, torrentspath, params);
|
||||
} else {
|
||||
searchurl =
|
||||
this.domain +
|
||||
torrentspath
|
||||
.replace(/\{keyword\}/g, encodeURIComponent(searchWord))
|
||||
.replace(/\{page\}/g, String(this.page || 0));
|
||||
}
|
||||
} else {
|
||||
let page = this.page || 0;
|
||||
if (this.def.browse && !typedPathSelected) {
|
||||
torrentspath = this.def.browse.path;
|
||||
if (this.def.browse.start) page = Number(this.def.browse.start) + Number(this.page || 0);
|
||||
} else if (this.page && !torrentspath.includes("{page}")) {
|
||||
torrentspath += `${torrentspath.includes("?") ? "&" : "?"}page=${this.page}`;
|
||||
}
|
||||
searchurl = this.domain + torrentspath.replace(/\{page\}/g, String(page)).replace(/\{keyword\}/g, "");
|
||||
}
|
||||
|
||||
this.requestedResultMediaType =
|
||||
search.result_media_type === "requested" && this.mtype && (typedPathSelected || categoryFilterSelected)
|
||||
? this.mtype
|
||||
: undefined;
|
||||
return searchurl;
|
||||
}
|
||||
|
||||
/** MP `__is_login_or_permission_page` */
|
||||
static isLoginOrPermissionPage($: CheerioAPI): boolean {
|
||||
const title = $("title").first().text().trim();
|
||||
const pageText = $.root().text().split(/\s+/).filter(Boolean).join(" ").slice(0, 1000);
|
||||
if (title === "登录" || title.includes(":: 登录")) return true;
|
||||
return ["未登录", "登录 / 注册", "必须在登录后才能访问", "你需要启用cookies才能登录"].some((m) =>
|
||||
pageText.includes(m),
|
||||
);
|
||||
}
|
||||
|
||||
/** MP `parse`:登录/权限页返回 isError=true 与空列表。 */
|
||||
parse(html: string): { isError: boolean; loginPage: boolean; torrents: SpiderTorrent[]; $: CheerioAPI | null } {
|
||||
if (!html) return { isError: true, loginPage: false, torrents: [], $: null };
|
||||
const $ = loadHtml(html);
|
||||
if (SiteSpider.isLoginOrPermissionPage($)) return { isError: true, loginPage: true, torrents: [], $ };
|
||||
this.$ = $;
|
||||
const out: SpiderTorrent[] = [];
|
||||
const rows = $(normalizeSelector(this.list.selector ?? ""));
|
||||
rows.each((i, el) => {
|
||||
if (i >= this.resultNum) return false;
|
||||
const info = this.getInfo($(el));
|
||||
if (Object.keys(info).length > 0) out.push(this.applyRequestedMediaType(info));
|
||||
return undefined;
|
||||
});
|
||||
return { isError: false, loginPage: false, torrents: out, $ };
|
||||
}
|
||||
|
||||
/** 对任意片段(如详情页头部)按字段配置取值 */
|
||||
infoFrom($: CheerioAPI, el: Sel): SpiderTorrent {
|
||||
this.$ = $;
|
||||
return this.getInfo(el);
|
||||
}
|
||||
|
||||
private applyRequestedMediaType(t: SpiderTorrent): SpiderTorrent {
|
||||
if (this.requestedResultMediaType) t.category = MEDIA_VALUE[this.requestedResultMediaType];
|
||||
return t;
|
||||
}
|
||||
|
||||
/** MP `get_info`:任一字段异常则整条返回 {}。 */
|
||||
getInfo(row: Sel): SpiderTorrent {
|
||||
this.info = {};
|
||||
try {
|
||||
this.getTitle(row);
|
||||
this.getDescription(row);
|
||||
this.getDetail(row);
|
||||
this.getDownload(row);
|
||||
this.getGrabs(row);
|
||||
this.getLeechers(row);
|
||||
this.getSeeders(row);
|
||||
this.getSize(row);
|
||||
this.getImdbid(row);
|
||||
this.getVolumeFactor(row, "downloadvolumefactor");
|
||||
this.getVolumeFactor(row, "uploadvolumefactor");
|
||||
this.getPubdate(row);
|
||||
this.getDateElapsed(row);
|
||||
this.getFreeDate(row);
|
||||
this.getLabels(row);
|
||||
this.getHitAndRun(row);
|
||||
this.getCategory(row);
|
||||
return { ...this.info };
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 字段 ----
|
||||
|
||||
private getTitle(row: Sel) {
|
||||
if (!("title" in this.fields)) return;
|
||||
const selector = this.fields.title ?? {};
|
||||
let title: string | null | undefined;
|
||||
if (selector.selector) {
|
||||
title = this.safeQuery(row, selector);
|
||||
} else if (selector.text) {
|
||||
const render: Record<string, unknown> = {};
|
||||
if ("title_default" in this.fields) render.title_default = this.safeQuery(row, this.fields.title_default);
|
||||
if ("title_optional" in this.fields) render.title_optional = this.safeQuery(row, this.fields.title_optional);
|
||||
title = renderTemplate(selector.text, render);
|
||||
}
|
||||
const v = this.filterText(title, selector.filters);
|
||||
if (v != null) this.info.title = String(v);
|
||||
}
|
||||
|
||||
private getDescription(row: Sel) {
|
||||
if (!("description" in this.fields)) return;
|
||||
const selector = this.fields.description ?? {};
|
||||
let desc: string | null | undefined;
|
||||
if (selector.selector || selector.selectors) {
|
||||
const s: SpiderFieldSelector = { ...selector };
|
||||
if (selector.selectors && !selector.selector) s.selector = selector.selectors;
|
||||
desc = this.safeQuery(row, s);
|
||||
} else if (selector.text) {
|
||||
const render: Record<string, unknown> = {};
|
||||
for (const k of ["tags", "subject", "description_free_forever", "description_normal"]) {
|
||||
if (k in this.fields) render[k] = this.safeQuery(row, this.fields[k]);
|
||||
}
|
||||
desc = renderTemplate(selector.text, render);
|
||||
}
|
||||
const v = this.filterText(desc, selector.filters);
|
||||
if (v != null) this.info.description = String(v);
|
||||
}
|
||||
|
||||
private normalizeLink(link: string): string {
|
||||
if (link.startsWith("http")) return link;
|
||||
if (link.startsWith("//")) return `${this.domain.split(":")[0]}:${link}`;
|
||||
if (link.startsWith("/")) return this.domain + link.slice(1);
|
||||
return this.domain + link;
|
||||
}
|
||||
|
||||
private getDetail(row: Sel) {
|
||||
if (!("details" in this.fields)) return;
|
||||
const selector = this.fields.details ?? {};
|
||||
const link = this.filterText(this.safeQuery(row, selector), selector.filters);
|
||||
if (link) this.info.page_url = this.normalizeLink(String(link));
|
||||
}
|
||||
|
||||
private getDownload(row: Sel) {
|
||||
if (!("download" in this.fields)) return;
|
||||
const selector = this.fields.download ?? {};
|
||||
const raw = this.filterText(this.safeQuery(row, selector), selector.filters);
|
||||
if (!raw) return;
|
||||
const link = String(raw);
|
||||
if (!link.startsWith("http") && !link.startsWith("magnet")) {
|
||||
const [scheme, host] = splitNetloc(this.domain);
|
||||
if (host && link.includes(host)) {
|
||||
this.info.enclosure = link.startsWith("/") ? `${scheme}:${link}` : `${scheme}://${link}`;
|
||||
} else {
|
||||
this.info.enclosure = link.startsWith("/") ? `${this.domain}${link.slice(1)}` : `${this.domain}${link}`;
|
||||
}
|
||||
} else {
|
||||
this.info.enclosure = link;
|
||||
}
|
||||
}
|
||||
|
||||
private countField(row: Sel, name: string, key: "grabs" | "peers" | "seeders", allowZeroText: boolean) {
|
||||
if (!(name in this.fields)) return;
|
||||
const selector = this.fields[name] ?? {};
|
||||
const item = this.safeQuery(row, selector);
|
||||
const present = allowZeroText ? item != null && item !== "" : Boolean(item);
|
||||
if (present) {
|
||||
let v: unknown = String(item).split("/")[0]!.replace(/,/g, "");
|
||||
v = this.filterText(v as string, selector.filters);
|
||||
this.info[key] = typeof v === "string" && /^\d+$/.test(v) ? Number(v) : 0;
|
||||
} else {
|
||||
this.info[key] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private getGrabs(row: Sel) {
|
||||
this.countField(row, "grabs", "grabs", true);
|
||||
}
|
||||
private getLeechers(row: Sel) {
|
||||
this.countField(row, "leechers", "peers", false);
|
||||
}
|
||||
private getSeeders(row: Sel) {
|
||||
this.countField(row, "seeders", "seeders", false);
|
||||
}
|
||||
|
||||
private getSize(row: Sel) {
|
||||
if (!("size" in this.fields)) return;
|
||||
const selector = this.fields.size ?? {};
|
||||
const item = this.safeQuery(row, selector);
|
||||
if (item != null && item !== "") {
|
||||
const v = this.filterText(item.replace(/\n/g, "").trim(), selector.filters);
|
||||
this.info.size = parseSizeMp(v == null ? "" : String(v));
|
||||
} else {
|
||||
this.info.size = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private getImdbid(row: Sel) {
|
||||
if (!("imdbid" in this.fields)) return;
|
||||
const selector = this.fields.imdbid ?? {};
|
||||
const v = this.filterText(this.safeQuery(row, selector), selector.filters);
|
||||
if (v != null) this.info.imdbid = String(v);
|
||||
}
|
||||
|
||||
private getVolumeFactor(row: Sel, key: "downloadvolumefactor" | "uploadvolumefactor") {
|
||||
const selector = this.fields[key];
|
||||
if (!selector || Object.keys(selector).length === 0) return;
|
||||
this.info[key] = 1;
|
||||
if (selector.case) {
|
||||
for (const [css, value] of Object.entries(selector.case)) {
|
||||
if (row.find(normalizeSelector(css)).length > 0) {
|
||||
this.info[key] = value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (selector.selector) {
|
||||
const item = this.safeQuery(row, selector);
|
||||
const m = item ? /(\d+\.?\d*)/.exec(item) : null;
|
||||
// 偏离 MP:MP 用 int() 转换,"0.5" 抛异常导致整条丢失(迁移账本 SA4)
|
||||
if (m) this.info[key] = Number.parseFloat(m[1]!);
|
||||
}
|
||||
}
|
||||
|
||||
private getPubdate(row: Sel) {
|
||||
if (!("date_added" in this.fields) && !("date" in this.fields)) return;
|
||||
let selector: SpiderFieldSelector = this.fields.date_added ?? {};
|
||||
let pubdate: string | null | undefined = this.safeQuery(row, selector);
|
||||
if (!pubdate) {
|
||||
selector = this.fields.date ?? {};
|
||||
pubdate = this.getDate(row, selector);
|
||||
}
|
||||
if (pubdate) pubdate = pubdate.replace(/\n/g, " ").trim();
|
||||
const filtered = this.filterText(pubdate, selector.filters);
|
||||
let value: string | undefined;
|
||||
if (filtered instanceof ParsedDateTime) value = filtered.text;
|
||||
else if (filtered) value = STD_TIME.test(String(filtered)) ? String(filtered) : normalizeDatetime(String(filtered));
|
||||
if (value && !isInvalidPubdateText(value)) this.info.pubdate = value;
|
||||
else delete this.info.pubdate;
|
||||
}
|
||||
|
||||
private getDate(row: Sel, selector: SpiderFieldSelector): string | null {
|
||||
if (!selector || Object.keys(selector).length === 0) return null;
|
||||
if (selector.selector) return this.safeQuery(row, selector) ?? null;
|
||||
if (!selector.text) return null;
|
||||
const render: Record<string, string | null> = {};
|
||||
for (const name of ["date_elapsed", "date_added"]) {
|
||||
const fs = this.fields[name] ?? {};
|
||||
let v = this.safeQuery(row, fs) ?? null;
|
||||
if (!v) v = this.getDateFromCell(row, fs);
|
||||
render[name] = v;
|
||||
}
|
||||
if (!Object.values(render).some(Boolean)) return null;
|
||||
const s = renderTemplate(selector.text, render);
|
||||
if (s === "now" || isRelativePubdateText(s)) return null;
|
||||
return s;
|
||||
}
|
||||
|
||||
private getDateFromCell(row: Sel, selector: SpiderFieldSelector): string | null {
|
||||
const sel = selector.selector?.trim();
|
||||
if (!sel?.includes("> span")) return null;
|
||||
return this.safeQuery(row, { selector: sel.split("> span")[0]!.trim() }) ?? null;
|
||||
}
|
||||
|
||||
private getDateElapsed(row: Sel) {
|
||||
if (!("date_elapsed" in this.fields)) return;
|
||||
const selector = this.fields.date_elapsed ?? {};
|
||||
const v = this.filterText(this.safeQuery(row, selector), selector.filters);
|
||||
if (v != null) this.info.date_elapsed = String(v);
|
||||
}
|
||||
|
||||
private getFreeDate(row: Sel) {
|
||||
if (!("freedate" in this.fields)) return;
|
||||
const selector = this.fields.freedate ?? {};
|
||||
const v = this.filterText(this.safeQuery(row, selector), selector.filters);
|
||||
if (v != null) this.info.freedate = v instanceof ParsedDateTime ? v.text : String(v);
|
||||
}
|
||||
|
||||
private getLabels(row: Sel) {
|
||||
if (!("labels" in this.fields)) return;
|
||||
const selector = this.fields.labels ?? {};
|
||||
if (!selector.selector) {
|
||||
this.info.labels = [];
|
||||
return;
|
||||
}
|
||||
const q = row.find(normalizeSelector(selector.selector)).clone();
|
||||
this.remove(q, selector);
|
||||
this.info.labels = this.attributeOrText(q, selector).filter((x): x is string => Boolean(x));
|
||||
}
|
||||
|
||||
private getHitAndRun(row: Sel) {
|
||||
if (!("hr" in this.fields)) return;
|
||||
const selector = this.fields.hr ?? {};
|
||||
this.info.hit_and_run = selector.selector ? row.find(normalizeSelector(selector.selector)).length > 0 : false;
|
||||
}
|
||||
|
||||
private getCategory(row: Sel) {
|
||||
if (this.requestedResultMediaType) {
|
||||
this.info.category = MEDIA_VALUE[this.requestedResultMediaType];
|
||||
return;
|
||||
}
|
||||
if (!("category" in this.fields)) {
|
||||
if (this.siteMediaType) this.info.category = MEDIA_VALUE[this.siteMediaType];
|
||||
return;
|
||||
}
|
||||
const selector = this.fields.category ?? {};
|
||||
const value = this.filterText(this.safeQuery(row, selector), selector.filters);
|
||||
if (value != null && value !== "") this.info.category_id = String(value);
|
||||
let resolved = resolveCategoryMediaType(value, this.def.category);
|
||||
if (resolved === "未知" && this.siteMediaType) resolved = MEDIA_VALUE[this.siteMediaType];
|
||||
this.info.category = resolved;
|
||||
}
|
||||
|
||||
// ---- 通用查询 ----
|
||||
|
||||
/** MP `_safe_query` */
|
||||
safeQuery(row: Sel, config: SpiderFieldSelector | undefined): string | null | undefined {
|
||||
if (!config?.selector) return null;
|
||||
let q = row.find(normalizeSelector(config.selector));
|
||||
if (config.remove) q = q.clone();
|
||||
this.remove(q, config);
|
||||
return this.index(this.attributeOrText(q, config), config);
|
||||
}
|
||||
|
||||
private remove(q: Sel, config: SpiderFieldSelector) {
|
||||
if (!config.remove) return;
|
||||
for (const v of config.remove.split(", ")) q.find(normalizeSelector(v)).remove();
|
||||
}
|
||||
|
||||
private attributeOrText(q: Sel, config: SpiderFieldSelector): Array<string | undefined> {
|
||||
if (q.length === 0) return [];
|
||||
const out: Array<string | undefined> = [];
|
||||
q.each((_, el) => {
|
||||
const item = this.$(el);
|
||||
out.push(config.attribute ? item.attr(config.attribute) : pqText(item));
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
private index(items: Array<string | undefined>, config: SpiderFieldSelector): string | undefined {
|
||||
if (items.length === 0) return undefined;
|
||||
if (config.contents !== undefined && items.length > Number(config.contents)) {
|
||||
return pyIndex((items[0] ?? "").split("\n"), Number(config.contents));
|
||||
}
|
||||
if (config.index !== undefined && items.length > Number(config.index)) return pyIndex(items, Number(config.index));
|
||||
return items[0];
|
||||
}
|
||||
|
||||
/** MP `__filter_text`;单个过滤器失败只跳过该过滤器。 */
|
||||
filterText(
|
||||
text: string | null | undefined,
|
||||
filters: SpiderFilter[] | undefined,
|
||||
): string | ParsedDateTime | null | undefined {
|
||||
if (!text || !filters || !Array.isArray(filters)) return text;
|
||||
let t: string | ParsedDateTime = String(text);
|
||||
for (const f of filters) {
|
||||
if (!t) break;
|
||||
try {
|
||||
if (t instanceof ParsedDateTime) break;
|
||||
const s: string = t;
|
||||
switch (f.name) {
|
||||
case "re_search": {
|
||||
if (!Array.isArray(f.args)) break;
|
||||
const m = new RegExp(f.args[0]).exec(s);
|
||||
if (m) t = m[Number(f.args[f.args.length - 1])] ?? "";
|
||||
break;
|
||||
}
|
||||
case "split":
|
||||
if (Array.isArray(f.args)) {
|
||||
const parts = s.split(f.args[0]);
|
||||
const v = pyIndex(parts, Number(f.args[f.args.length - 1]));
|
||||
if (v === undefined) throw new Error("list index out of range");
|
||||
t = v;
|
||||
}
|
||||
break;
|
||||
case "replace":
|
||||
if (Array.isArray(f.args)) t = s.split(String(f.args[0])).join(String(f.args[f.args.length - 1]));
|
||||
break;
|
||||
case "dateparse":
|
||||
if (typeof f.args === "string") t = strptime(s.replace(/\n/g, " ").trim(), f.args);
|
||||
break;
|
||||
case "strip":
|
||||
t = s.trim();
|
||||
break;
|
||||
case "appendleft":
|
||||
t = `${f.args}${s}`;
|
||||
break;
|
||||
case "querystring":
|
||||
t = querystringValue(s, String(f.args));
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// MP 仅 debug 日志,继续下一个过滤器
|
||||
}
|
||||
}
|
||||
return typeof t === "string" ? t.trim() : t;
|
||||
}
|
||||
}
|
||||
|
||||
/** MP `MediaType.from_agent` 的子集 */
|
||||
function fromAgent(v: string | undefined): MediaKind | undefined {
|
||||
if (!v) return undefined;
|
||||
const s = v.toLowerCase();
|
||||
if (s === "movie" || v === "电影") return "movie";
|
||||
if (s === "tv" || v === "电视剧") return "tv";
|
||||
if (s === "music" || v === "音乐") return "music";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** MP `url.split_netloc` */
|
||||
function splitNetloc(url: string): [string, string] {
|
||||
if (!url) return ["", ""];
|
||||
if (!url.startsWith("http")) return ["http", url];
|
||||
const u = new URL(url);
|
||||
return [u.protocol.replace(":", ""), u.host];
|
||||
}
|
||||
|
||||
/** MP `UrlUtils.combine_url`:urljoin + 合并已有 query + urlencode(doseq=True)。 */
|
||||
export function combineUrl(
|
||||
host: string,
|
||||
path: string,
|
||||
query: Record<string, string | number | Array<string | number>>,
|
||||
): string {
|
||||
const base = host.endsWith("/") ? host : `${host}/`;
|
||||
const url = new URL(path || "/", base);
|
||||
for (const [k, v] of Object.entries(query)) {
|
||||
url.searchParams.delete(k);
|
||||
if (Array.isArray(v)) for (const x of v) url.searchParams.append(k, String(x));
|
||||
else url.searchParams.set(k, String(v));
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import nunjucks from "nunjucks";
|
||||
|
||||
/**
|
||||
* Jinja 模板渲染(MP `SiteSpider.__build_field_templates` 用 jinja2.Template)。
|
||||
* nunjucks 兼容 Jinja 语法(`{% if %}`、`fields['x']`、`a if b else c`);模板只来自随产品发布的站点定义。
|
||||
*/
|
||||
const env = new nunjucks.Environment(null, { autoescape: false, throwOnUndefined: false });
|
||||
const cache = new Map<string, nunjucks.Template>();
|
||||
|
||||
export function renderTemplate(text: string, fields: Record<string, unknown>): string {
|
||||
let tpl = cache.get(text);
|
||||
if (!tpl) {
|
||||
tpl = new nunjucks.Template(text, env);
|
||||
cache.set(text, tpl);
|
||||
}
|
||||
return tpl.render({ fields });
|
||||
}
|
||||
Reference in new issue
Block a user