430 lines
18 KiB
TypeScript
430 lines
18 KiB
TypeScript
/**
|
||
* @mp-source app/modules/indexer/__init__.py @ 3b156a5
|
||
* 通用 spider 站点适配器(NexusPHP 等 HTML 站)。
|
||
*
|
||
* 对应 MoviePilot v3.2.4:
|
||
* - `app/modules/indexer/__init__.py IndexerModule.search_torrents / refresh_torrents` → `search` / `list`
|
||
* (`__search_check` 的英文站不搜中文 → search 前检查;`__clear_search_text` → `clearSearchText`;
|
||
* `SitesHelper.check(domain)` 流控 → 站点令牌桶)
|
||
* - `SiteSpider.get_torrents`(RequestUtils GET + cookie + ua)→ `fetchPage`
|
||
* - `app/chain/download/submission.py download_torrent`(带 cookie GET enclosure,磁力原样返回)→ `downloadTarget`
|
||
*
|
||
* 本项目调整:登录页不再"返回空列表",而是抛 `auth_expired`(REQ §4.7.1 不展示空列表冒充无内容);
|
||
* 3xx 不自动跟随(避免把 cookie 带到站外,并把跳转 login 识别为失效);凭证经 SecretResolver 注入。
|
||
*/
|
||
|
||
import { inspectTorrentIsolated } from "../../isolation.ts";
|
||
import { redactText } from "../../rss/redact.ts";
|
||
import { challengeOutcomeText, challengeRuleOfUnrecognizedPage } from "../challenge.ts";
|
||
import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
|
||
import { buildFailureSample, type SiteRequestKind } from "../failure-sample.ts";
|
||
import { readBody, readText, siteFetch } from "../http.ts";
|
||
import {
|
||
type DownloadTarget,
|
||
type SiteAdapter,
|
||
type SiteCallOptions,
|
||
SiteError,
|
||
type SiteHealth,
|
||
type SiteHealthDetail,
|
||
type SiteListFilters,
|
||
type SitePage,
|
||
type SiteReleaseDetail,
|
||
} from "../types.ts";
|
||
import { parseDetailPage } from "./detail.ts";
|
||
import { filesFromFileListHtml } from "./detail-page.ts";
|
||
import { spiderTorrentToRelease } from "./map.ts";
|
||
import type { IndexerDefinition } from "./schema.ts";
|
||
import { loadHtml, normalizeSelector, SiteSpider } from "./spider.ts";
|
||
|
||
/** MP `text_tools.remove_punctuation(text, replacement=" ", allow_space=True)` */
|
||
export function clearSearchText(text: string): string {
|
||
return text
|
||
.replace(/[-]/g, "")
|
||
.replace(/[、.。,，·:：;；!！?？'’"“”()（）[\]【】「」\-—―+|\\_/&#～~]/g, " ")
|
||
.replace(/\s+/g, " ")
|
||
.trim();
|
||
}
|
||
|
||
function containsChinese(s: string): boolean {
|
||
return /[一-鿿]/.test(s);
|
||
}
|
||
|
||
export function healthFromError(siteId: string, e: unknown, checkedAt: string): SiteHealth {
|
||
if (e instanceof SiteError) {
|
||
const status: SiteHealth["status"] =
|
||
e.kind === "network"
|
||
? "unreachable"
|
||
: e.kind === "not_found" || e.kind === "unsupported"
|
||
? "parse_error"
|
||
: e.kind;
|
||
const detail = e.detail ?? (e.httpStatus !== undefined ? { reason: "http", httpStatus: e.httpStatus } : undefined);
|
||
return {
|
||
siteId,
|
||
status,
|
||
ok: false,
|
||
checkedAt,
|
||
latencyMs: null,
|
||
message: e.message,
|
||
...(detail ? { detail } : {}),
|
||
};
|
||
}
|
||
return { siteId, status: "unreachable", ok: false, checkedAt, latencyMs: null, message: redactText(String(e)) };
|
||
}
|
||
|
||
/**
|
||
* 健康检查的首页登录判定:登录页 / 权限页 → 未登录;登录标志选择器命中 → 已登录;
|
||
* 否则区分"疑似人机验证"与"页面里找不到登录标志"。detail 是原始判定(不含页面里的个人数据)。
|
||
*/
|
||
export function judgeLoginPage(
|
||
html: string,
|
||
loginSuccess: string | undefined,
|
||
): { ok: true; detail: SiteHealthDetail } | { ok: false; message: string; detail: SiteHealthDetail } {
|
||
const $ = loadHtml(html);
|
||
const base = { path: "index.php", title: $("title").first().text().trim().slice(0, 80) };
|
||
const login = SiteSpider.loginPageRule($);
|
||
if (login) {
|
||
return {
|
||
ok: false,
|
||
message:
|
||
login.kind === "login_page"
|
||
? "健康检查:首页返回登录页,Cookie 已失效"
|
||
: "健康检查:首页返回权限提示页(要求先登录)",
|
||
detail: { ...base, reason: login.kind, rule: login.rule },
|
||
};
|
||
}
|
||
if (!loginSuccess || $(loginSuccess).length > 0) {
|
||
return { ok: true, detail: { ...base, reason: "logged_in", ...(loginSuccess ? { selector: loginSuccess } : {}) } };
|
||
}
|
||
const challenge = SiteSpider.challengeRule($, html);
|
||
if (challenge) {
|
||
return {
|
||
ok: false,
|
||
message: "健康检查:疑似人机验证(Cloudflare),无法确认登录态",
|
||
detail: { ...base, reason: "challenge", rule: challenge },
|
||
};
|
||
}
|
||
return {
|
||
ok: false,
|
||
message: `健康检查:首页里找不到登录标志(选择器 ${loginSuccess})`,
|
||
detail: { ...base, reason: "marker_missing", selector: loginSuccess, length: html.length },
|
||
};
|
||
}
|
||
|
||
/** 抓到的页面:正文与留样用的响应信息 */
|
||
interface FetchedPage {
|
||
html: string;
|
||
url: string;
|
||
status: number;
|
||
headers: Headers;
|
||
}
|
||
|
||
/** 错误文案里的页面名 */
|
||
const PAGE_LABEL: Record<SiteRequestKind, string> = {
|
||
search: "搜索页",
|
||
list: "列表页",
|
||
detail: "详情页",
|
||
health: "首页",
|
||
other: "页面",
|
||
};
|
||
|
||
export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdapterDeps): SiteAdapter {
|
||
const def: IndexerDefinition = deps.baseUrl ? { ...input, domain: deps.baseUrl } : input;
|
||
const ext = def.umt;
|
||
const http = buildHttpContext(def.id, deps, ext.rateLimit);
|
||
const base = def.domain.endsWith("/") ? def.domain : `${def.domain}/`;
|
||
|
||
async function cookie(): Promise<string> {
|
||
const c = await deps.secrets.resolve({ siteId: def.id, kind: "cookie" });
|
||
if (!c && def.public) return "";
|
||
if (!c) throw new SiteError(def.id, "auth_missing", "未配置站点 Cookie");
|
||
return c;
|
||
}
|
||
|
||
/** 留样:交给调用方保存;回调出错不影响请求结果 */
|
||
function report(e: SiteError): SiteError {
|
||
if (e.sample && deps.onFailureSample) {
|
||
try {
|
||
deps.onFailureSample(e.sample);
|
||
} catch {
|
||
// 留样失败不影响请求
|
||
}
|
||
}
|
||
return e;
|
||
}
|
||
|
||
/** 页面认不出来时的错误:像人机验证就报 challenge,否则 parse_error;都附页面样本 */
|
||
function unrecognized(page: FetchedPage, what: string, kind: SiteRequestKind, parseMessage: string): SiteError {
|
||
const rule = challengeRuleOfUnrecognizedPage(page.html);
|
||
const title = redactText(loadHtml(page.html)("title").first().text().replace(/\s+/g, " ").trim()).slice(0, 120);
|
||
const message = rule
|
||
? `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText("unhandled")}`
|
||
: `${parseMessage}（HTTP ${page.status}${title ? `，页面标题：${title}` : ""}；页面样本见站点页技术信息）`;
|
||
const sample = buildFailureSample({
|
||
kind: rule ? "challenge" : "parse_error",
|
||
requestKind: kind,
|
||
url: page.url,
|
||
httpStatus: page.status,
|
||
headers: page.headers,
|
||
message,
|
||
rule,
|
||
html: page.html,
|
||
now: http.now(),
|
||
});
|
||
return report(
|
||
new SiteError(def.id, rule ? "challenge" : "parse_error", message, {
|
||
httpStatus: page.status,
|
||
...(rule ? { detail: { reason: "challenge", rule, httpStatus: page.status } } : {}),
|
||
sample,
|
||
}),
|
||
);
|
||
}
|
||
|
||
async function fetchPage(
|
||
url: string,
|
||
what: string,
|
||
opts: SiteCallOptions,
|
||
kind: SiteRequestKind,
|
||
): Promise<FetchedPage> {
|
||
let res: Response;
|
||
try {
|
||
res = await siteFetch(http, url, {
|
||
method: "GET",
|
||
headers: {
|
||
cookie: await cookie(),
|
||
accept: "text/html,application/xhtml+xml,application/rss+xml,application/xml",
|
||
referer: base,
|
||
},
|
||
redirect: "manual",
|
||
...(opts.signal ? { signal: opts.signal } : {}),
|
||
});
|
||
} catch (e) {
|
||
if (e instanceof SiteError && e.kind === "challenge") {
|
||
// 网络层判出的人机验证:换成带请求类型的文案,样本补上请求类型
|
||
const outcome = typeof e.detail?.outcome === "string" ? e.detail.outcome : null;
|
||
const message = `${what}：${PAGE_LABEL[kind]}需要人机验证（Cloudflare 等），${challengeOutcomeText(outcome)}`;
|
||
throw report(
|
||
new SiteError(def.id, "challenge", message, {
|
||
...(e.httpStatus !== undefined ? { httpStatus: e.httpStatus } : {}),
|
||
...(e.detail ? { detail: e.detail } : {}),
|
||
...(e.sample ? { sample: { ...e.sample, requestKind: kind, message } } : {}),
|
||
}),
|
||
);
|
||
}
|
||
throw e;
|
||
}
|
||
if (res.status >= 300 && res.status < 400) {
|
||
const loc = res.headers.get("location") ?? "";
|
||
await res.body?.cancel().catch(() => {});
|
||
if (/login/i.test(loc)) {
|
||
throw new SiteError(def.id, "auth_expired", `${what}:跳转登录页,Cookie 已失效`, {
|
||
httpStatus: res.status,
|
||
detail: { reason: "redirect_login", httpStatus: res.status, location: redactText(loc).slice(0, 200) },
|
||
});
|
||
}
|
||
throw new SiteError(def.id, "parse_error", redactText(`${what}:意外重定向到 ${loc}`), { httpStatus: res.status });
|
||
}
|
||
// Cloudflare 挑战响应带 cf-mitigated: challenge(403 / 503)
|
||
if (res.headers.get("cf-mitigated") === "challenge") {
|
||
await res.body?.cancel().catch(() => {});
|
||
throw new SiteError(def.id, "challenge", `${what}:疑似人机验证(Cloudflare)`, {
|
||
httpStatus: res.status,
|
||
detail: { reason: "challenge", rule: "header cf-mitigated=challenge", httpStatus: res.status },
|
||
});
|
||
}
|
||
if (res.status === 401 || res.status === 403) {
|
||
await res.body?.cancel().catch(() => {});
|
||
throw new SiteError(def.id, "auth_invalid", `${what}:HTTP ${res.status}`, { httpStatus: res.status });
|
||
}
|
||
if (!res.ok) {
|
||
await res.body?.cancel().catch(() => {});
|
||
throw new SiteError(def.id, res.status === 404 ? "not_found" : "network", `${what}:HTTP ${res.status}`, {
|
||
httpStatus: res.status,
|
||
});
|
||
}
|
||
return { html: await readText(http, res), url, status: res.status, headers: res.headers };
|
||
}
|
||
|
||
async function run(
|
||
page: number,
|
||
filters: SiteListFilters,
|
||
keyword: string | undefined,
|
||
opts: SiteCallOptions,
|
||
): Promise<SitePage> {
|
||
const what = keyword ? "搜索" : `列表第 ${page} 页`;
|
||
const spider = new SiteSpider(def, {
|
||
...(keyword ? { keyword } : {}),
|
||
...(filters.mtype ? { mtype: filters.mtype } : {}),
|
||
...(filters.categoryIds?.length ? { cat: filters.categoryIds.join(",") } : {}),
|
||
page: Math.max(1, page) - 1, // 对外 1 起,MP 内部 0 起
|
||
});
|
||
if (!spider.canSearch()) throw new SiteError(def.id, "unsupported", `${what}:站点定义不支持该媒体类型`);
|
||
const kind: SiteRequestKind = keyword ? "search" : "list";
|
||
const page$ = await fetchPage(spider.getSearchUrl(), what, opts, kind);
|
||
const parsed = spider.parse(page$.html);
|
||
if (parsed.loginPage) throw new SiteError(def.id, "auth_expired", `${what}:返回登录或权限提示页`);
|
||
if (parsed.isError || !parsed.$) throw unrecognized(page$, what, kind, `${what}:响应为空`);
|
||
const $ = parsed.$;
|
||
if (parsed.torrents.length === 0) {
|
||
const text = $("body").text();
|
||
const hasList =
|
||
$(normalizeSelector(spider.def.torrents.list.selector)).length > 0 ||
|
||
$("table.torrents, rss > channel, feed").length > 0;
|
||
const empty = (ext.emptyMarkers ?? []).some((m) => text.includes(m));
|
||
// 认不出列表:先看是不是人机验证页(搜索页单独加盾、间歇触发),不是才报改版
|
||
if (!hasList && !empty) throw unrecognized(page$, what, kind, `${what}:页面结构无法识别(可能改版)`);
|
||
if (ext.loginSuccess && $(ext.loginSuccess).length === 0 && !empty) {
|
||
throw new SiteError(def.id, "auth_expired", `${what}:未找到登录标志`);
|
||
}
|
||
}
|
||
const observedAt = http.now().toISOString();
|
||
let items = parsed.torrents
|
||
.map((t) => spiderTorrentToRelease(t, def, observedAt))
|
||
.filter((r): r is NonNullable<typeof r> => r !== null);
|
||
if (filters.freeOnly) items = items.filter((r) => r.promotion.downloadFactor === 0);
|
||
if (!keyword && filters.categoryIds?.length) {
|
||
// MP 浏览模式不带分类参数;本地按原始分类过滤
|
||
const set = new Set(filters.categoryIds);
|
||
items = items.filter((r) => r.category.id !== null && set.has(r.category.id));
|
||
}
|
||
let maxRaw: number | null = null;
|
||
if (ext.pagination) {
|
||
$(ext.pagination).each((_, el) => {
|
||
const p = Number(/[?&]page=(\d+)/.exec($(el).attr("href") ?? "")?.[1]);
|
||
if (Number.isInteger(p) && (maxRaw === null || p > maxRaw)) maxRaw = p;
|
||
});
|
||
}
|
||
const start = keyword ? 0 : Number(def.browse?.start ?? 0);
|
||
return {
|
||
items,
|
||
page: Math.max(1, page),
|
||
totalPages: parsed.totalPages ?? (maxRaw === null ? null : Math.max(page, (maxRaw as number) - start + 1)),
|
||
totalPagesEstimated: parsed.totalPages === undefined,
|
||
fetchedAt: observedAt,
|
||
};
|
||
}
|
||
|
||
return {
|
||
id: def.id,
|
||
kind: ext.kind,
|
||
displayName: ext.displayName ?? def.name,
|
||
capabilities: ext.capabilities,
|
||
definitionVersion: ext.version,
|
||
|
||
list(page, filters = {}, opts = {}) {
|
||
return run(page, filters, undefined, opts);
|
||
},
|
||
|
||
async search(query, filters = {}, opts = {}) {
|
||
const kw = /^tt\d+$/.test(query.trim()) ? query.trim() : clearSearchText(query);
|
||
if (kw && def.language === "en" && containsChinese(kw)) {
|
||
throw new SiteError(def.id, "unsupported", "该站点不支持中文搜索");
|
||
}
|
||
return run(filters.page ?? 1, filters, kw, opts);
|
||
},
|
||
|
||
async detail(siteReleaseId, opts = {}): Promise<SiteReleaseDetail> {
|
||
if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法");
|
||
if (!ext.detail) throw new SiteError(def.id, "unsupported", "站点定义未配置详情页解析");
|
||
const url = new URL(`${ext.detail.path}?id=${siteReleaseId}&hit=1`, base).toString();
|
||
const detailPage = await fetchPage(url, `详情 ${siteReleaseId}`, opts, "detail");
|
||
const r = parseDetailPage(detailPage.html, def, siteReleaseId, http.now().toISOString());
|
||
if (r.kind === "login") throw new SiteError(def.id, "auth_expired", `详情 ${siteReleaseId}:返回登录页`);
|
||
if (r.kind === "not_found") throw new SiteError(def.id, "not_found", `详情 ${siteReleaseId}:资源不存在`);
|
||
if (!r.detail) {
|
||
throw unrecognized(detailPage, `详情 ${siteReleaseId}`, "detail", `详情 ${siteReleaseId}:页面结构无法识别`);
|
||
}
|
||
// 文件清单不在页面上(NexusPHP 点「查看列表」才 AJAX 取)→ 按需 1 次;失败只是没有文件清单
|
||
const flPath = ext.detail.page?.fileListPath;
|
||
if (flPath && r.detail.page && !r.detail.page.files.length) {
|
||
try {
|
||
const flUrl = new URL(flPath.replace("{id}", siteReleaseId), base).toString();
|
||
const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts, "detail");
|
||
r.detail.page.files = filesFromFileListHtml(loadHtml(frag.html));
|
||
} catch (e) {
|
||
if (e instanceof SiteError && e.isAuthError) throw e;
|
||
}
|
||
}
|
||
return r.detail;
|
||
},
|
||
|
||
async downloadTarget(siteReleaseId, opts = {}): Promise<DownloadTarget> {
|
||
if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法");
|
||
const what = `下载 ${siteReleaseId}`;
|
||
const url = new URL(`${ext.downloadPath}?id=${siteReleaseId}`, base).toString();
|
||
const res = await siteFetch(http, url, {
|
||
method: "GET",
|
||
headers: { cookie: await cookie(), referer: base },
|
||
redirect: "manual",
|
||
...(opts.signal ? { signal: opts.signal } : {}),
|
||
});
|
||
const verifiedAt = http.now().toISOString();
|
||
if (res.status >= 300 && res.status < 400) {
|
||
const loc = res.headers.get("location") ?? "";
|
||
await res.body?.cancel().catch(() => {});
|
||
// MP download_torrent:磁力链原样返回
|
||
if (loc.startsWith("magnet:")) return { kind: "magnet", uri: loc, verifiedAt };
|
||
if (/login/i.test(loc)) throw new SiteError(def.id, "auth_expired", `${what}:跳转登录页,Cookie 已失效`);
|
||
throw new SiteError(def.id, "parse_error", `${what}:意外重定向`, { httpStatus: res.status });
|
||
}
|
||
if (res.headers.get("cf-mitigated") === "challenge") {
|
||
await res.body?.cancel().catch(() => {});
|
||
throw new SiteError(def.id, "challenge", `${what}:疑似人机验证(Cloudflare)`, {
|
||
httpStatus: res.status,
|
||
detail: { reason: "challenge", rule: "header cf-mitigated=challenge", httpStatus: res.status },
|
||
});
|
||
}
|
||
if (!res.ok) {
|
||
await res.body?.cancel().catch(() => {});
|
||
throw new SiteError(def.id, res.status === 404 ? "not_found" : "network", `${what}:HTTP ${res.status}`, {
|
||
httpStatus: res.status,
|
||
});
|
||
}
|
||
const bytes = await readBody(http, res);
|
||
const info = await inspectTorrentIsolated(bytes);
|
||
if (!info) {
|
||
// 站点在 cookie 失效时常返回 200 + 登录页 HTML
|
||
const $ = loadHtml(new TextDecoder().decode(bytes.subarray(0, 200_000)));
|
||
if (SiteSpider.isLoginOrPermissionPage($)) throw new SiteError(def.id, "auth_expired", `${what}:返回登录页`);
|
||
const text = $("body").text();
|
||
if (ext.detail?.notFoundMarkers.some((m) => text.includes(m))) {
|
||
throw new SiteError(def.id, "not_found", `${what}:资源不存在`);
|
||
}
|
||
throw new SiteError(def.id, "parse_error", `${what}:返回内容不是 .torrent`);
|
||
}
|
||
const disposition = res.headers.get("content-disposition") ?? "";
|
||
const fileName =
|
||
/filename\*=UTF-8''([^;]+)/i.exec(disposition)?.[1] ?? /filename="?([^";]+)"?/i.exec(disposition)?.[1] ?? null;
|
||
return {
|
||
kind: "torrent",
|
||
bytes,
|
||
infoHash: info.infoHash,
|
||
name: info.name,
|
||
fileName: fileName ? decodeURIComponent(fileName) : null,
|
||
verifiedAt,
|
||
};
|
||
},
|
||
|
||
async healthCheck(opts = {}): Promise<SiteHealth> {
|
||
const started = Date.now();
|
||
const checkedAt = http.now().toISOString();
|
||
try {
|
||
const home = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts, "health");
|
||
const v = judgeLoginPage(home.html, ext.loginSuccess);
|
||
const latencyMs = Date.now() - started;
|
||
if (v.ok) return { siteId: def.id, status: "ok", ok: true, checkedAt, latencyMs, detail: v.detail };
|
||
return {
|
||
siteId: def.id,
|
||
status: v.detail.reason === "challenge" ? "challenge" : "auth_expired",
|
||
ok: false,
|
||
checkedAt,
|
||
latencyMs,
|
||
message: v.message,
|
||
detail: v.detail,
|
||
};
|
||
} catch (e) {
|
||
return healthFromError(def.id, e, checkedAt);
|
||
}
|
||
},
|
||
};
|
||
}
|