/** * 通用站点爬虫(配置驱动)。 * * 迁移自 MoviePilot e6820a8 `app/modules/indexer/spider/__init__.py`: * - `select_media_categories` / `resolve_category_media_type` → 同名函数 * - `SiteSpider.__get_search_url` / `__format_search_word` → `SiteSpider.getSearchUrl` / `formatSearchWord` * - `SiteSpider.get_info` 及 `__get_title/__get_description/__get_detail/__get_download/__get_grabs/__get_leechers/ * __get_seeders/__get_size/__get_imdbid/__get_downloadvolumefactor/__get_uploadvolumefactor/__get_pubdate/__get_date/ * __get_date_from_cell/__get_date_elapsed/__get_free_date/__get_labels/__get_hit_and_run/__get_category` * → `SiteSpider.getInfo` 与同名私有方法 * - `_safe_query / __filter_text / __remove / __attribute_or_text / __index` → 同名方法 * - `__is_login_or_permission_page` → `isLoginOrPermissionPage` * - `parse` → `SiteSpider.parse`(不含 Rust 加速分支);`search_type=subtitles` 分支:`get_subtitle_info / * __get_language_icon / __get_report_url / __get_subtitle_field / __fill_subtitle_ids` → `getSubtitleInfo` 等同名方法 * - `app/foundation/size.py parse_size` → `parseSizeMp`(Python round 为银行家舍入,同样实现) * * 与 MP 的差异(均在 README 与迁移账本登记): * 1. 额外输出 `category_id`(原始分类 ID),`category` 仍是 MP 的媒体类型投影(REQ §4.5 原始分类不得丢失)。 * 2. 系数 `selector` 模式用浮点而非 int()(MP 对 0.5 抛异常导致整条丢失,迁移账本 SA4)。 * 3. 不发请求:HTTP/Cookie/UA 由 adapter 注入,本类只负责 URL 构造与 HTML 解析(便于限流与脱敏统一)。 * 4. 查询范围:PyQuery 在行元素上查询是 descendant-or-self(行自身也会命中),MP Rust 快路径与 cheerio `.find` 只查后代。 * 本项目取折中:促销 case / hr / labels 这三类"行级标记"(NexusPHP 把 free_bg、sticky 等 class 挂在 上) * 包含行自身,与 Python 一致;取值字段(safeQuery)只查后代,避免 `div:nth-child(3)` 这类选择器命中行本身取到整行文本。 */ import { type CheerioAPI, load } from "cheerio"; import type { IndexerDefinition, SpiderCategoryConfig, SpiderFieldSelector, SpiderFields, SpiderFilter, SpiderListConfig, } from "./schema.ts"; import { renderTemplate } from "./template.ts"; type Sel = ReturnType; /** MP `MediaType` 取值 */ export type MpMediaType = "电影" | "电视剧" | "音乐" | "未知"; export type MediaKind = "movie" | "tv" | "music"; const MEDIA_VALUE: Record = { movie: "电影", tv: "电视剧", music: "音乐" }; /** * 以 htmlparser2(HTML 模式)加载:与 MP 的 lxml.html 一致不自动插入 , * 使 `table.torrents > tr` 这类 MP 风格选择器保持原语义。 */ export function loadHtml(html: string): CheerioAPI { return load(html, { xml: { xmlMode: false, decodeEntities: true } }); } /** * PyQuery(cssselect)接受 `:has("sel")` 引号写法,cheerio-select 不接受;统一去引号以兼容 MP 风格定义。 */ export function normalizeSelector(sel: string): string { return sel.replace(/:has\((["'])(.*?)\1\)/g, ":has($2)"); } /** 解析结果:键名与 MP `TorrentInfo` 字段一致,`category_id` 为本项目扩展。 */ export interface SpiderTorrent { title?: string; description?: string; page_url?: string; enclosure?: string; grabs?: number; peers?: number; seeders?: number; size?: number; imdbid?: string; downloadvolumefactor?: number; uploadvolumefactor?: number; pubdate?: string; date_elapsed?: string; freedate?: string; labels?: string[]; hit_and_run?: boolean; category?: MpMediaType; category_id?: string; // ---- 字幕分支(search_type=subtitles)专有字段,键名与 MP 一致 ---- language_icon?: string; report_url?: string; language?: string; uploader?: string; torrent_id?: string; subtitle_id?: string; file_name?: string; } export type SpiderSearchType = "torrents" | "subtitles"; /** MP `select_media_categories` */ export function selectMediaCategories(category: IndexerDefinition["category"], mtype?: MediaKind) { if (!category) return []; if (mtype === "tv") return category.tv ?? []; if (mtype === "movie") return category.movie ?? []; if (mtype === "music") return category.music ?? []; return [...(category.movie ?? []), ...(category.tv ?? []), ...(category.music ?? [])]; } /** MP `resolve_category_media_type`:同一 ID 只落在一个媒体类型里才投影,否则未知。 */ export function resolveCategoryMediaType(value: unknown, category: IndexerDefinition["category"]): MpMediaType { if (value == null || !category) return "未知"; const id = String(value); const matches = (["movie", "tv", "music"] as const).filter((k) => (category[k] ?? []).some((item) => item.id != null && String(item.id) === id), ); return matches.length === 1 ? MEDIA_VALUE[matches[0]!] : "未知"; } /** Python 3 `round()`:恰好 .5 时取偶数(银行家舍入) */ function roundHalfEven(v: number): number { const f = Math.floor(v); const diff = v - f; if (diff === 0.5) return f % 2 === 0 ? f : f + 1; return Math.round(v); } /** MP `size_tools.parse_size`:二进制单位,round(银行家舍入)。 */ export function parseSizeMp(text: string | number | null | undefined): number { if (!text) return 0; const s = String(text); if (/^\d+$/.test(s)) return Number(s); const normalized = s.replace(/,/g, "").replace(/ /g, "").toUpperCase(); const sizeText = normalized.replace(/[KMGTPI]*B?/gi, ""); const size = Number(sizeText); if (sizeText === "" || !Number.isFinite(size)) return 0; let v = size; if (normalized.includes("PB") || normalized.includes("PIB")) v *= 1024 ** 5; else if (normalized.includes("TB") || normalized.includes("TIB")) v *= 1024 ** 4; else if (normalized.includes("GB") || normalized.includes("GIB")) v *= 1024 ** 3; else if (normalized.includes("MB") || normalized.includes("MIB")) v *= 1024 ** 2; else if (normalized.includes("KB") || normalized.includes("KIB")) v *= 1024; return roundHalfEven(v); } /** strptime 结果(MP dateparse 过滤器返回 datetime) */ class ParsedDateTime { constructor(readonly text: string) {} } const STRPTIME_TOKENS: Record = { "%Y": "(\\d{4})", "%m": "(\\d{1,2})", "%d": "(\\d{1,2})", "%H": "(\\d{1,2})", "%M": "(\\d{1,2})", "%S": "(\\d{1,2})", }; /** Python strptime 的最小子集(%Y %m %d %H %M %S),失败抛错(与 MP 一致由调用方吞掉)。 */ function strptime(text: string, fmt: string): ParsedDateTime { const order: string[] = []; const re = fmt.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/%[YmdHMS]/g, (t) => { order.push(t); return STRPTIME_TOKENS[t]!; }); const m = new RegExp(`^${re}$`).exec(text); if (!m) throw new Error(`time data '${text}' does not match format '${fmt}'`); const v: Record = { "%Y": 1900, "%m": 1, "%d": 1, "%H": 0, "%M": 0, "%S": 0 }; order.forEach((t, i) => { v[t] = Number(m[i + 1]); }); const p = (n: number) => String(n).padStart(2, "0"); return new ParsedDateTime(`${v["%Y"]}-${p(v["%m"]!)}-${p(v["%d"]!)} ${p(v["%H"]!)}:${p(v["%M"]!)}:${p(v["%S"]!)}`); } const STD_TIME = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/; /** MP `time_tools.normalize_datetime` 的保守子集:只接受可确定解析的绝对时间。 */ function normalizeDatetime(value: string): string { const m = /^(\d{4})[-/年.](\d{1,2})[-/月.](\d{1,2})日?(?:[ T](\d{1,2}):(\d{1,2})(?::(\d{1,2}))?)?$/.exec( value.trim(), ); if (!m) return value; const p = (n: string | undefined) => String(Number(n ?? 0)).padStart(2, "0"); return `${m[1]}-${p(m[2])}-${p(m[3])} ${p(m[4])}:${p(m[5])}:${p(m[6])}`; } function isRelativePubdateText(pubdate: string | null | undefined): boolean { if (!pubdate) return false; const text = String(pubdate).trim().toLowerCase(); if (/\d{4}[-/年]\d{1,2}/.test(text)) return false; if (text.includes("ago")) return true; return /\d+\s*(秒|分钟|分|小时|天|周|月|年)/.test(text); } function isInvalidPubdateText(pubdate: string | null | undefined): boolean { if (!pubdate) return true; const text = String(pubdate).trim(); if (text.toLowerCase() === "now" || text === "0") return true; if (isRelativePubdateText(text)) return true; return !STD_TIME.test(text); } function pyIndex(arr: T[], i: number): T | undefined { return i < 0 ? arr[arr.length + i] : arr[i]; } /** Python `parse_qs(urlparse(url).query).get(key)[0]` */ function querystringValue(text: string, key: string): string { const q = text.includes("?") ? text.slice(text.indexOf("?") + 1) : ""; const v = new URLSearchParams(q.split("#")[0]).getAll(key).filter((x) => x !== ""); return v[0] ?? ""; } /** PyQuery `.text()` 近似:`
` 视为换行,行内空白折叠。 */ function pqText(el: Sel): string { const c = el.clone(); c.find("br").replaceWith("\n"); return c .text() .split("\n") .map((l) => l.replace(/[ \t\r\f\v\u00a0]+/g, " ").trim()) .filter((l) => l !== "") .join("\n"); } export interface SiteSpiderQuery { keyword?: string | string[]; mtype?: MediaKind; /** 逗号分隔的分类 ID 白名单(MP `cat`) */ cat?: string; /** 站点内部页码(MP 0 起) */ page?: number; /** MP `search_type`:subtitles 时改用 `def.subtitles` 的配置 */ searchType?: SpiderSearchType; } export class SiteSpider { readonly def: IndexerDefinition; readonly keyword: string | string[] | undefined; readonly mtype: MediaKind | undefined; readonly cat: string | undefined; readonly page: number; readonly domain: string; readonly resultNum: number; readonly siteMediaType: MediaKind | undefined; readonly searchType: SpiderSearchType; private list: SpiderListConfig; private fields: SpiderFields; private searchCfg: IndexerDefinition["search"]; private batchCfg: IndexerDefinition["batch"]; private browseCfg: IndexerDefinition["browse"]; private categoryCfg: SpiderCategoryConfig | undefined; private requestedResultMediaType: MediaKind | undefined; private $!: CheerioAPI; private info: SpiderTorrent = {}; constructor(def: IndexerDefinition, q: SiteSpiderQuery = {}) { this.def = def; this.keyword = q.keyword; this.mtype = q.mtype; this.cat = q.cat; this.page = q.page ?? 0; this.siteMediaType = fromAgent(def.media_type); this.searchType = q.searchType ?? "torrents"; if (this.searchType === "subtitles") { const sc = def.subtitles ?? {}; this.searchCfg = sc.search; this.batchCfg = sc.batch; this.browseCfg = sc.browse; this.categoryCfg = sc.category; this.list = sc.list ?? { selector: "" }; this.fields = sc.fields ?? {}; this.resultNum = Number(sc.result_num ?? def.result_num ?? 100); } else { this.searchCfg = def.search; this.batchCfg = def.batch; this.browseCfg = def.browse; this.categoryCfg = def.category; this.list = def.torrents.list; this.fields = def.torrents.fields ?? {}; if (!q.keyword && def.browse) { this.list = def.browse.list ?? this.list; this.fields = def.browse.fields ?? this.fields; } this.resultNum = Number(def.result_num ?? 100); } this.domain = def.domain.endsWith("/") ? def.domain : `${def.domain}/`; } /** MP `__can_search` */ canSearch(): boolean { if (this.siteMediaType && this.mtype && this.siteMediaType !== this.mtype) return false; return Boolean(this.searchCfg && this.domain); } /** MP `__format_search_word` */ private formatSearchWord(word: string): string { if (word && /^tt\d+$/.test(word) && this.searchCfg?.imdbid_format) { return this.searchCfg.imdbid_format .replace(/\{keyword\}/g, word) .replace(/\{imdbid\}/g, word) .replace(/\{imdbid_num\}/g, word.slice(2)); } return word; } /** MP `__get_search_url` */ getSearchUrl(): string { const search = this.searchCfg ?? { paths: [] }; const paths = search.paths ?? []; let torrentspath = ""; let typedPathSelected = false; let categoryFilterSelected = false; if (paths.length === 1) { torrentspath = paths[0]!.path ?? ""; } else { let fallback = ""; const expected = this.mtype; for (const p of paths) { if (p.type === "all" && !fallback) fallback = p.path ?? ""; if ((expected && p.type === expected) || (!expected && p.type === "all")) { torrentspath = p.path ?? ""; typedPathSelected = Boolean(expected && p.type === expected); break; } } if (!torrentspath) torrentspath = fallback; } let searchurl: string; if (this.keyword && (typeof this.keyword === "string" || this.keyword.length > 0)) { let searchWord: string; let searchMode: string; if (Array.isArray(this.keyword)) { if (this.batchCfg) { const delimiter = this.batchCfg.delimiter || " "; const spaceReplace = this.batchCfg.space_replace || " "; searchWord = this.keyword.map((k) => String(k).replace(/ /g, spaceReplace)).join(delimiter); } else { searchWord = this.keyword.join(" "); } searchMode = "1"; } else { searchWord = this.keyword; searchMode = "0"; } const isImdb = typeof this.keyword === "string" && /^tt\d+$/.test(this.keyword); searchWord = this.formatSearchWord(searchWord); const indexerParams: Record = { ...(search.params ?? {}) }; if (Object.keys(indexerParams).length > 0) { if (indexerParams.search_area && !isImdb) delete indexerParams.search_area; const params: Record> = { search_mode: searchMode, search_area: 0, page: this.page || 0, notnewword: 1, }; for (const [k, v] of Object.entries(indexerParams)) params[k] = String(v).replace(/\{keyword\}/g, searchWord); const category = this.categoryCfg; if (category) { const cats = selectMediaCategories(category, this.mtype); const allowed = this.cat ? new Set(this.cat.split(",")) : null; for (const c of cats) { if (allowed && !allowed.has(String(c.id))) continue; if (category.field) { categoryFilterSelected = true; const cur = String(params[category.field] ?? ""); params[category.field] = cur + (category.delimiter ?? " ") + String(c.id); } else { const categoryParam = c.param || category.param; if (categoryParam) { categoryFilterSelected = true; const cid = c.value ?? c.id; const cur = params[categoryParam]; if (cur === undefined) params[categoryParam] = cid; else if (Array.isArray(cur)) cur.push(cid); else params[categoryParam] = [cur, cid]; } else { categoryFilterSelected = true; params[`cat${c.id}`] = 1; } } if (c.params) Object.assign(params, c.params); } } searchurl = combineUrl(this.domain, torrentspath, params); } else { searchurl = this.domain + torrentspath .replace(/\{keyword\}/g, encodeURIComponent(searchWord)) .replace(/\{page\}/g, String(this.page || 0)); } } else { let page = this.page || 0; if (this.browseCfg && !typedPathSelected) { torrentspath = this.browseCfg.path; if (this.browseCfg.start) page = Number(this.browseCfg.start) + Number(this.page || 0); } else if (this.page && !torrentspath.includes("{page}")) { torrentspath += `${torrentspath.includes("?") ? "&" : "?"}page=${this.page}`; } searchurl = this.domain + torrentspath.replace(/\{page\}/g, String(page)).replace(/\{keyword\}/g, ""); } this.requestedResultMediaType = search.result_media_type === "requested" && this.mtype && (typedPathSelected || categoryFilterSelected) ? this.mtype : undefined; return searchurl; } /** MP `__is_login_or_permission_page` */ static isLoginOrPermissionPage($: CheerioAPI): boolean { const title = $("title").first().text().trim(); const pageText = $.root().text().split(/\s+/).filter(Boolean).join(" ").slice(0, 1000); if (title === "登录" || title.includes(":: 登录")) return true; return ["未登录", "登录 / 注册", "必须在登录后才能访问", "你需要启用cookies才能登录"].some((m) => pageText.includes(m), ); } /** MP `parse`:登录/权限页返回 isError=true 与空列表。 */ parse(html: string): { isError: boolean; loginPage: boolean; torrents: SpiderTorrent[]; $: CheerioAPI | null } { if (!html) return { isError: true, loginPage: false, torrents: [], $: null }; const $ = loadHtml(html); if (SiteSpider.isLoginOrPermissionPage($)) return { isError: true, loginPage: true, torrents: [], $ }; this.$ = $; const out: SpiderTorrent[] = []; const rows = $(normalizeSelector(this.list.selector ?? "")); rows.each((i, el) => { if (i >= this.resultNum) return false; const info = this.searchType === "subtitles" ? this.getSubtitleInfo($(el)) : this.getInfo($(el)); if (Object.keys(info).length > 0) out.push(this.applyRequestedMediaType(info)); return undefined; }); return { isError: false, loginPage: false, torrents: out, $ }; } /** 对任意片段(如详情页头部)按字段配置取值 */ infoFrom($: CheerioAPI, el: Sel): SpiderTorrent { this.$ = $; return this.getInfo(el); } private applyRequestedMediaType(t: SpiderTorrent): SpiderTorrent { if (this.requestedResultMediaType) t.category = MEDIA_VALUE[this.requestedResultMediaType]; return t; } /** MP `get_info`:任一字段异常则整条返回 {}。 */ getInfo(row: Sel): SpiderTorrent { this.info = {}; try { this.getTitle(row); this.getDescription(row); this.getDetail(row); this.getDownload(row); this.getGrabs(row); this.getLeechers(row); this.getSeeders(row); this.getSize(row); this.getImdbid(row); this.getVolumeFactor(row, "downloadvolumefactor"); this.getVolumeFactor(row, "uploadvolumefactor"); this.getPubdate(row); this.getDateElapsed(row); this.getFreeDate(row); this.getLabels(row); this.getHitAndRun(row); this.getCategory(row); return { ...this.info }; } catch { return {}; } } /** MP `get_subtitle_info`:缺标题或下载链接的行丢弃;任一字段异常整条返回 {}。 */ getSubtitleInfo(row: Sel): SpiderTorrent { this.info = {}; try { this.getTitle(row); this.getDescription(row); this.getDetail(row); this.getDownload(row); this.getSize(row); this.getPubdate(row); this.getDateElapsed(row); this.getGrabs(row); this.getLinkField(row, "language_icon", "language_icon"); this.getLinkField(row, "report", "report_url"); for (const name of ["language", "uploader", "torrent_id", "subtitle_id", "file_name"] as const) { this.getSubtitleField(row, name); } this.fillSubtitleIds(); if (!this.info.title || !this.info.enclosure) return {}; return { ...this.info }; } catch { return {}; } } /** MP `__get_language_icon` / `__get_report_url`:相对链接补全为绝对链接 */ private getLinkField(row: Sel, name: string, key: "language_icon" | "report_url") { if (!(name in this.fields)) return; const selector = this.fields[name] ?? {}; const link = this.filterText(this.safeQuery(row, selector), selector.filters); if (link) this.info[key] = this.normalizeLink(String(link)); } /** MP `__get_subtitle_field` */ private getSubtitleField(row: Sel, name: "language" | "uploader" | "torrent_id" | "subtitle_id" | "file_name") { const selector = this.fields[name]; if (!selector || Object.keys(selector).length === 0) return; const v = this.filterText(this.safeQuery(row, selector), selector.filters); if (v != null) this.info[name] = v instanceof ParsedDateTime ? v.text : String(v); } /** MP `__fill_subtitle_ids`:从下载链接 query 补种子 ID(torrentid/torrent_id)与字幕 ID(subid/subtitle) */ private fillSubtitleIds() { const enclosure = this.info.enclosure; if (!enclosure) return; const q = enclosure.includes("?") ? enclosure.slice(enclosure.indexOf("?") + 1).split("#")[0]! : ""; const params = new URLSearchParams(q); const first = (...keys: string[]) => { for (const k of keys) { const v = params.getAll(k).find((x) => x !== ""); if (v !== undefined) return v; } return undefined; }; if (!this.info.torrent_id) { const v = first("torrentid", "torrent_id"); if (v !== undefined) this.info.torrent_id = v; } if (!this.info.subtitle_id) { const v = first("subid", "subtitle"); if (v !== undefined) this.info.subtitle_id = v; } } // ---- 字段 ---- private getTitle(row: Sel) { if (!("title" in this.fields)) return; const selector = this.fields.title ?? {}; let title: string | null | undefined; if (selector.selector) { title = this.safeQuery(row, selector); } else if (selector.text) { const render: Record = {}; if ("title_default" in this.fields) render.title_default = this.safeQuery(row, this.fields.title_default); if ("title_optional" in this.fields) render.title_optional = this.safeQuery(row, this.fields.title_optional); title = renderTemplate(selector.text, render); } const v = this.filterText(title, selector.filters); if (v != null) this.info.title = String(v); } private getDescription(row: Sel) { if (!("description" in this.fields)) return; const selector = this.fields.description ?? {}; let desc: string | null | undefined; if (selector.selector || selector.selectors) { const s: SpiderFieldSelector = { ...selector }; if (selector.selectors && !selector.selector) s.selector = selector.selectors; desc = this.safeQuery(row, s); } else if (selector.text) { const render: Record = {}; for (const k of ["tags", "subject", "description_free_forever", "description_normal"]) { if (k in this.fields) render[k] = this.safeQuery(row, this.fields[k]); } desc = renderTemplate(selector.text, render); } const v = this.filterText(desc, selector.filters); if (v != null) this.info.description = String(v); } private normalizeLink(link: string): string { if (link.startsWith("http")) return link; if (link.startsWith("//")) return `${this.domain.split(":")[0]}:${link}`; if (link.startsWith("/")) return this.domain + link.slice(1); return this.domain + link; } private getDetail(row: Sel) { if (!("details" in this.fields)) return; const selector = this.fields.details ?? {}; const link = this.filterText(this.safeQuery(row, selector), selector.filters); if (link) this.info.page_url = this.normalizeLink(String(link)); } private getDownload(row: Sel) { if (!("download" in this.fields)) return; const selector = this.fields.download ?? {}; const raw = this.filterText(this.safeQuery(row, selector), selector.filters); if (!raw) return; const link = String(raw); if (!link.startsWith("http") && !link.startsWith("magnet")) { const [scheme, host] = splitNetloc(this.domain); if (host && link.includes(host)) { this.info.enclosure = link.startsWith("/") ? `${scheme}:${link}` : `${scheme}://${link}`; } else { this.info.enclosure = link.startsWith("/") ? `${this.domain}${link.slice(1)}` : `${this.domain}${link}`; } } else { this.info.enclosure = link; } } private countField(row: Sel, name: string, key: "grabs" | "peers" | "seeders", allowZeroText: boolean) { if (!(name in this.fields)) return; const selector = this.fields[name] ?? {}; const item = this.safeQuery(row, selector); const present = allowZeroText ? item != null && item !== "" : Boolean(item); if (present) { let v: unknown = String(item).split("/")[0]!.replace(/,/g, ""); v = this.filterText(v as string, selector.filters); this.info[key] = typeof v === "string" && /^\d+$/.test(v) ? Number(v) : 0; } else { this.info[key] = 0; } } private getGrabs(row: Sel) { this.countField(row, "grabs", "grabs", true); } private getLeechers(row: Sel) { this.countField(row, "leechers", "peers", false); } private getSeeders(row: Sel) { this.countField(row, "seeders", "seeders", false); } private getSize(row: Sel) { if (!("size" in this.fields)) return; const selector = this.fields.size ?? {}; const item = this.safeQuery(row, selector); if (item != null && item !== "") { const v = this.filterText(item.replace(/\n/g, "").trim(), selector.filters); this.info.size = parseSizeMp(v == null ? "" : String(v)); } else { this.info.size = 0; } } private getImdbid(row: Sel) { if (!("imdbid" in this.fields)) return; const selector = this.fields.imdbid ?? {}; const v = this.filterText(this.safeQuery(row, selector), selector.filters); if (v != null) this.info.imdbid = String(v); } private getVolumeFactor(row: Sel, key: "downloadvolumefactor" | "uploadvolumefactor") { const selector = this.fields[key]; if (!selector || Object.keys(selector).length === 0) return; this.info[key] = 1; if (selector.case) { for (const [css, value] of Object.entries(selector.case)) { if (this.rowScope(row, css).length > 0) { this.info[key] = value; break; } } } else if (selector.selector) { const item = this.safeQuery(row, selector); const m = item ? /(\d+\.?\d*)/.exec(item) : null; // 偏离 MP:MP 用 int() 转换,"0.5" 抛异常导致整条丢失(迁移账本 SA4) if (m) this.info[key] = Number.parseFloat(m[1]!); } } private getPubdate(row: Sel) { if (!("date_added" in this.fields) && !("date" in this.fields)) return; let selector: SpiderFieldSelector = this.fields.date_added ?? {}; let pubdate: string | null | undefined = this.safeQuery(row, selector); if (!pubdate) { selector = this.fields.date ?? {}; pubdate = this.getDate(row, selector); } if (pubdate) pubdate = pubdate.replace(/\n/g, " ").trim(); const filtered = this.filterText(pubdate, selector.filters); let value: string | undefined; if (filtered instanceof ParsedDateTime) value = filtered.text; else if (filtered) value = STD_TIME.test(String(filtered)) ? String(filtered) : normalizeDatetime(String(filtered)); if (value && !isInvalidPubdateText(value)) this.info.pubdate = value; else delete this.info.pubdate; } private getDate(row: Sel, selector: SpiderFieldSelector): string | null { if (!selector || Object.keys(selector).length === 0) return null; if (selector.selector) return this.safeQuery(row, selector) ?? null; if (!selector.text) return null; const render: Record = {}; for (const name of ["date_elapsed", "date_added"]) { const fs = this.fields[name] ?? {}; let v = this.safeQuery(row, fs) ?? null; if (!v) v = this.getDateFromCell(row, fs); render[name] = v; } if (!Object.values(render).some(Boolean)) return null; const s = renderTemplate(selector.text, render); if (s === "now" || isRelativePubdateText(s)) return null; return s; } private getDateFromCell(row: Sel, selector: SpiderFieldSelector): string | null { const sel = selector.selector?.trim(); if (!sel?.includes("> span")) return null; return this.safeQuery(row, { selector: sel.split("> span")[0]!.trim() }) ?? null; } private getDateElapsed(row: Sel) { if (!("date_elapsed" in this.fields)) return; const selector = this.fields.date_elapsed ?? {}; const v = this.filterText(this.safeQuery(row, selector), selector.filters); if (v != null) this.info.date_elapsed = String(v); } private getFreeDate(row: Sel) { if (!("freedate" in this.fields)) return; const selector = this.fields.freedate ?? {}; const v = this.filterText(this.safeQuery(row, selector), selector.filters); if (v != null) this.info.freedate = v instanceof ParsedDateTime ? v.text : String(v); } private getLabels(row: Sel) { if (!("labels" in this.fields)) return; const selector = this.fields.labels ?? {}; if (!selector.selector) { this.info.labels = []; return; } const q = this.rowScope(row, selector.selector).clone(); this.remove(q, selector); this.info.labels = this.attributeOrText(q, selector).filter((x): x is string => Boolean(x)); } private getHitAndRun(row: Sel) { if (!("hr" in this.fields)) return; const selector = this.fields.hr ?? {}; this.info.hit_and_run = selector.selector ? this.rowScope(row, selector.selector).length > 0 : false; } private getCategory(row: Sel) { if (this.requestedResultMediaType) { this.info.category = MEDIA_VALUE[this.requestedResultMediaType]; return; } if (!("category" in this.fields)) { if (this.siteMediaType) this.info.category = MEDIA_VALUE[this.siteMediaType]; return; } const selector = this.fields.category ?? {}; const value = this.filterText(this.safeQuery(row, selector), selector.filters); if (value != null && value !== "") this.info.category_id = String(value); let resolved = resolveCategoryMediaType(value, this.categoryCfg); if (resolved === "未知" && this.siteMediaType) resolved = MEDIA_VALUE[this.siteMediaType]; this.info.category = resolved; } // ---- 通用查询 ---- /** 行级标记查询:行自身 + 后代(PyQuery descendant-or-self,见文件头差异 4);`:scope` 选择器只查后代 */ private rowScope(row: Sel, selector: string): Sel { const css = normalizeSelector(selector); const desc = row.find(css); if (css.includes(":scope")) return desc; return row.filter(css).add(desc); } /** MP `_safe_query` */ safeQuery(row: Sel, config: SpiderFieldSelector | undefined): string | null | undefined { if (!config?.selector) return null; let q = row.find(normalizeSelector(config.selector)); if (config.remove) q = q.clone(); this.remove(q, config); return this.index(this.attributeOrText(q, config), config); } private remove(q: Sel, config: SpiderFieldSelector) { if (!config.remove) return; for (const v of config.remove.split(", ")) q.find(normalizeSelector(v)).remove(); } private attributeOrText(q: Sel, config: SpiderFieldSelector): Array { if (q.length === 0) return []; const out: Array = []; q.each((_, el) => { const item = this.$(el); out.push(config.attribute ? item.attr(config.attribute) : pqText(item)); }); return out; } private index(items: Array, config: SpiderFieldSelector): string | undefined { if (items.length === 0) return undefined; if (config.contents !== undefined && items.length > Number(config.contents)) { return pyIndex((items[0] ?? "").split("\n"), Number(config.contents)); } if (config.index !== undefined && items.length > Number(config.index)) return pyIndex(items, Number(config.index)); return items[0]; } /** MP `__filter_text`;单个过滤器失败只跳过该过滤器。 */ filterText( text: string | null | undefined, filters: SpiderFilter[] | undefined, ): string | ParsedDateTime | null | undefined { if (!text || !filters || !Array.isArray(filters)) return text; let t: string | ParsedDateTime = String(text); for (const f of filters) { if (!t) break; try { if (t instanceof ParsedDateTime) break; const s: string = t; switch (f.name) { case "re_search": { if (!Array.isArray(f.args)) break; const m = new RegExp(f.args[0]).exec(s); if (m) t = m[Number(f.args[f.args.length - 1])] ?? ""; break; } case "split": if (Array.isArray(f.args)) { const parts = s.split(f.args[0]); const v = pyIndex(parts, Number(f.args[f.args.length - 1])); if (v === undefined) throw new Error("list index out of range"); t = v; } break; case "replace": if (Array.isArray(f.args)) t = s.split(String(f.args[0])).join(String(f.args[f.args.length - 1])); break; case "dateparse": if (typeof f.args === "string") t = strptime(s.replace(/\n/g, " ").trim(), f.args); break; case "strip": t = s.trim(); break; case "appendleft": t = `${f.args}${s}`; break; case "querystring": t = querystringValue(s, String(f.args)); break; } } catch { // MP 仅 debug 日志,继续下一个过滤器 } } return typeof t === "string" ? t.trim() : t; } } /** MP `MediaType.from_agent` 的子集 */ function fromAgent(v: string | undefined): MediaKind | undefined { if (!v) return undefined; const s = v.toLowerCase(); if (s === "movie" || v === "电影") return "movie"; if (s === "tv" || v === "电视剧") return "tv"; if (s === "music" || v === "音乐") return "music"; return undefined; } /** MP `url.split_netloc` */ function splitNetloc(url: string): [string, string] { if (!url) return ["", ""]; if (!url.startsWith("http")) return ["http", url]; const u = new URL(url); return [u.protocol.replace(":", ""), u.host]; } /** MP `UrlUtils.combine_url`:urljoin + 合并已有 query + urlencode(doseq=True)。 */ export function combineUrl( host: string, path: string, query: Record>, ): string { const base = host.endsWith("/") ? host : `${host}/`; const url = new URL(path || "/", base); for (const [k, v] of Object.entries(query)) { url.searchParams.delete(k); if (Array.isArray(v)) for (const x of v) url.searchParams.append(k, String(x)); else url.searchParams.set(k, String(v)); } return url.toString(); }