/** * 通用 spider 站点适配器(NexusPHP 等 HTML 站)。 * * 对应 MoviePilot e6820a8: * - `app/modules/indexer/__init__.py IndexerModule.search_torrents / refresh_torrents` → `search` / `list` * (`__search_check` 的英文站不搜中文 → search 前检查;`__clear_search_text` → `clearSearchText`; * `SitesHelper.check(domain)` 流控 → 站点令牌桶) * - `SiteSpider.get_torrents`(RequestUtils GET + cookie + ua)→ `fetchPage` * - `app/chain/download/submission.py download_torrent`(带 cookie GET enclosure,磁力原样返回)→ `downloadTarget` * * 本项目调整:登录页不再"返回空列表",而是抛 `auth_expired`(REQ §4.7.1 不展示空列表冒充无内容); * 3xx 不自动跟随(避免把 cookie 带到站外,并把跳转 login 识别为失效);凭证经 SecretResolver 注入。 */ import { redactText } from "../../rss/redact.ts"; import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts"; import { readBody, readText, siteFetch } from "../http.ts"; import { type DownloadTarget, type SiteAdapter, type SiteCallOptions, SiteError, type SiteHealth, type SiteListFilters, type SitePage, type SiteReleaseDetail, } from "../types.ts"; import { inspectTorrent } from "../util.ts"; import { parseDetailPage } from "./detail.ts"; import { filesFromFileListHtml } from "./detail-page.ts"; import { spiderTorrentToRelease } from "./map.ts"; import type { IndexerDefinition } from "./schema.ts"; import { loadHtml, normalizeSelector, SiteSpider } from "./spider.ts"; /** MP `text_tools.remove_punctuation(text, replacement=" ", allow_space=True)` */ export function clearSearchText(text: string): string { return text .replace(/[​-‍]/g, "") .replace(/[、.。,,·::;;!!??'’"“”()()[\]【】「」\-—―+|\\_/&#~~]/g, " ") .replace(/\s+/g, " ") .trim(); } function containsChinese(s: string): boolean { return /[一-鿿]/.test(s); } export function healthFromError(siteId: string, e: unknown, checkedAt: string): SiteHealth { if (e instanceof SiteError) { const status: SiteHealth["status"] = e.kind === "network" ? "unreachable" : e.kind === "not_found" || e.kind === "unsupported" ? "parse_error" : e.kind; return { siteId, status, ok: false, checkedAt, latencyMs: null, message: e.message }; } return { siteId, status: "unreachable", ok: false, checkedAt, latencyMs: null, message: redactText(String(e)) }; } export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdapterDeps): SiteAdapter { const def: IndexerDefinition = deps.baseUrl ? { ...input, domain: deps.baseUrl } : input; const ext = def.umt; const http = buildHttpContext(def.id, deps, ext.rateLimit); const base = def.domain.endsWith("/") ? def.domain : `${def.domain}/`; async function cookie(): Promise { const c = await deps.secrets.resolve({ siteId: def.id, kind: "cookie" }); if (!c) throw new SiteError(def.id, "auth_missing", "未配置站点 Cookie"); return c; } async function fetchPage(url: string, what: string, opts: SiteCallOptions): Promise { const res = await siteFetch(http, url, { method: "GET", headers: { cookie: await cookie(), accept: "text/html,application/xhtml+xml", referer: base }, redirect: "manual", ...(opts.signal ? { signal: opts.signal } : {}), }); if (res.status >= 300 && res.status < 400) { const loc = res.headers.get("location") ?? ""; await res.body?.cancel().catch(() => {}); if (/login/i.test(loc)) throw new SiteError(def.id, "auth_expired", `${what}:跳转登录页,Cookie 已失效`); throw new SiteError(def.id, "parse_error", redactText(`${what}:意外重定向到 ${loc}`), { httpStatus: res.status }); } if (res.status === 401 || res.status === 403) { await res.body?.cancel().catch(() => {}); throw new SiteError(def.id, "auth_invalid", `${what}:HTTP ${res.status}`, { httpStatus: res.status }); } if (!res.ok) { await res.body?.cancel().catch(() => {}); throw new SiteError(def.id, res.status === 404 ? "not_found" : "network", `${what}:HTTP ${res.status}`, { httpStatus: res.status, }); } return readText(http, res); } async function run( page: number, filters: SiteListFilters, keyword: string | undefined, opts: SiteCallOptions, ): Promise { const what = keyword ? "搜索" : `列表第 ${page} 页`; const spider = new SiteSpider(def, { ...(keyword ? { keyword } : {}), ...(filters.mtype ? { mtype: filters.mtype } : {}), ...(filters.categoryIds?.length ? { cat: filters.categoryIds.join(",") } : {}), page: Math.max(1, page) - 1, // 对外 1 起,MP 内部 0 起 }); if (!spider.canSearch()) throw new SiteError(def.id, "unsupported", `${what}:站点定义不支持该媒体类型`); const html = await fetchPage(spider.getSearchUrl(), what, opts); const parsed = spider.parse(html); if (parsed.loginPage) throw new SiteError(def.id, "auth_expired", `${what}:返回登录或权限提示页`); if (parsed.isError || !parsed.$) throw new SiteError(def.id, "parse_error", `${what}:响应为空`); const $ = parsed.$; if (parsed.torrents.length === 0) { const text = $("body").text(); const hasList = $(normalizeSelector(spider.def.torrents.list.selector)).length > 0 || $("table.torrents").length > 0; const empty = (ext.emptyMarkers ?? []).some((m) => text.includes(m)); if (!hasList && !empty) throw new SiteError(def.id, "parse_error", `${what}:页面结构无法识别(可能改版)`); if (ext.loginSuccess && $(ext.loginSuccess).length === 0 && !empty) { throw new SiteError(def.id, "auth_expired", `${what}:未找到登录标志`); } } const observedAt = http.now().toISOString(); let items = parsed.torrents .map((t) => spiderTorrentToRelease(t, def, observedAt)) .filter((r): r is NonNullable => r !== null); if (filters.freeOnly) items = items.filter((r) => r.promotion.downloadFactor === 0); if (!keyword && filters.categoryIds?.length) { // MP 浏览模式不带分类参数;本地按原始分类过滤 const set = new Set(filters.categoryIds); items = items.filter((r) => r.category.id !== null && set.has(r.category.id)); } let maxRaw: number | null = null; if (ext.pagination) { $(ext.pagination).each((_, el) => { const p = Number(/[?&]page=(\d+)/.exec($(el).attr("href") ?? "")?.[1]); if (Number.isInteger(p) && (maxRaw === null || p > maxRaw)) maxRaw = p; }); } const start = keyword ? 0 : Number(def.browse?.start ?? 0); return { items, page: Math.max(1, page), totalPages: maxRaw === null ? null : Math.max(page, (maxRaw as number) - start + 1), totalPagesEstimated: true, fetchedAt: observedAt, }; } return { id: def.id, kind: ext.kind, displayName: ext.displayName ?? def.name, capabilities: ext.capabilities, definitionVersion: ext.version, list(page, filters = {}, opts = {}) { return run(page, filters, undefined, opts); }, async search(query, filters = {}, opts = {}) { const kw = /^tt\d+$/.test(query.trim()) ? query.trim() : clearSearchText(query); if (kw && def.language === "en" && containsChinese(kw)) { throw new SiteError(def.id, "unsupported", "该站点不支持中文搜索"); } return run(filters.page ?? 1, filters, kw, opts); }, async detail(siteReleaseId, opts = {}): Promise { if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法"); if (!ext.detail) throw new SiteError(def.id, "unsupported", "站点定义未配置详情页解析"); const url = new URL(`${ext.detail.path}?id=${siteReleaseId}&hit=1`, base).toString(); const html = await fetchPage(url, `详情 ${siteReleaseId}`, opts); const r = parseDetailPage(html, def, siteReleaseId, http.now().toISOString()); if (r.kind === "login") throw new SiteError(def.id, "auth_expired", `详情 ${siteReleaseId}:返回登录页`); if (r.kind === "not_found") throw new SiteError(def.id, "not_found", `详情 ${siteReleaseId}:资源不存在`); if (!r.detail) throw new SiteError(def.id, "parse_error", `详情 ${siteReleaseId}:页面结构无法识别`); // 文件清单不在页面上(NexusPHP 点「查看列表」才 AJAX 取)→ 按需 1 次;失败只是没有文件清单 const flPath = ext.detail.page?.fileListPath; if (flPath && r.detail.page && !r.detail.page.files.length) { try { const flUrl = new URL(flPath.replace("{id}", siteReleaseId), base).toString(); const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts); r.detail.page.files = filesFromFileListHtml(loadHtml(frag)); } catch (e) { if (e instanceof SiteError && e.isAuthError) throw e; } } return r.detail; }, async downloadTarget(siteReleaseId, opts = {}): Promise { if (!/^\d+$/.test(siteReleaseId)) throw new SiteError(def.id, "not_found", "资源 ID 非法"); const what = `下载 ${siteReleaseId}`; const url = new URL(`${ext.downloadPath}?id=${siteReleaseId}`, base).toString(); const res = await siteFetch(http, url, { method: "GET", headers: { cookie: await cookie(), referer: base }, redirect: "manual", ...(opts.signal ? { signal: opts.signal } : {}), }); const verifiedAt = http.now().toISOString(); if (res.status >= 300 && res.status < 400) { const loc = res.headers.get("location") ?? ""; await res.body?.cancel().catch(() => {}); // MP download_torrent:磁力链原样返回 if (loc.startsWith("magnet:")) return { kind: "magnet", uri: loc, verifiedAt }; if (/login/i.test(loc)) throw new SiteError(def.id, "auth_expired", `${what}:跳转登录页,Cookie 已失效`); throw new SiteError(def.id, "parse_error", `${what}:意外重定向`, { httpStatus: res.status }); } if (!res.ok) { await res.body?.cancel().catch(() => {}); throw new SiteError(def.id, res.status === 404 ? "not_found" : "network", `${what}:HTTP ${res.status}`, { httpStatus: res.status, }); } const bytes = await readBody(http, res); const info = inspectTorrent(bytes); if (!info) { // 站点在 cookie 失效时常返回 200 + 登录页 HTML const $ = loadHtml(new TextDecoder().decode(bytes.subarray(0, 200_000))); if (SiteSpider.isLoginOrPermissionPage($)) throw new SiteError(def.id, "auth_expired", `${what}:返回登录页`); const text = $("body").text(); if (ext.detail?.notFoundMarkers.some((m) => text.includes(m))) { throw new SiteError(def.id, "not_found", `${what}:资源不存在`); } throw new SiteError(def.id, "parse_error", `${what}:返回内容不是 .torrent`); } const disposition = res.headers.get("content-disposition") ?? ""; const fileName = /filename\*=UTF-8''([^;]+)/i.exec(disposition)?.[1] ?? /filename="?([^";]+)"?/i.exec(disposition)?.[1] ?? null; return { kind: "torrent", bytes, infoHash: info.infoHash, name: info.name, fileName: fileName ? decodeURIComponent(fileName) : null, verifiedAt, }; }, async healthCheck(opts = {}): Promise { const started = Date.now(); const checkedAt = http.now().toISOString(); try { const html = await fetchPage(new URL("index.php", base).toString(), "健康检查", opts); const $ = loadHtml(html); if (SiteSpider.isLoginOrPermissionPage($) || (ext.loginSuccess && $(ext.loginSuccess).length === 0)) { throw new SiteError(def.id, "auth_expired", "健康检查:未登录"); } return { siteId: def.id, status: "ok", ok: true, checkedAt, latencyMs: Date.now() - started }; } catch (e) { return healthFromError(def.id, e, checkedAt); } }, }; }