rootandClaude Opus 5.5
36dc048e89
Title recognition (parseRelease / parseReleasePath):
- anime prefix tags (★N月新番, [国漫], 【22年日剧】), category lead bracket, `A / B` cn aliases
- `Name [01]`, `[TV 04]`, `#13`, `名称 12.mkv`, `S01 13-14`, `E13-14` episodes
- lead bracket as title vs group, trailing [Group], YTS.MX/Hall_of_C style groups, 字幕组 suffix
- `[dot.release.name][…]` unbracketing, `Season 2017` as year, leading date not airDate,
date-only parent dir does not merge as TV
- original disc: Blu-ray + AVC/HEVC/VC-1/MPEG-2 without x26x/H.26x -> BluRay-Disc
- `{[tmdbid=;type=;s=;e=]}` explicit block, bangumi/anilist ids
- bare [1080]/[5.1], DTS-HD MA5.1, ￡ multi-group, 合集, pinyin title -> cn from subtitle,
streaming platform table, CHS/CHT/GB/BIG5 in subtitle
zh-map: add 迴滾魷瑯臥駭; document one-to-many chars kept out.
Sites: tbody branch + header-row tightening via movieclaw-import rowSelector (18 definitions),
row-level case/hr/labels include the row itself, subtitle branch (searchType: "subtitles"),
parseSizeMp banker's rounding.
Differential snapshots regenerated; MP test sample sources registered in ledger.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
880 lines
34 KiB
TypeScript
880 lines
34 KiB
TypeScript
/**
|
|
* 通用站点爬虫(配置驱动)。
|
|
*
|
|
* 迁移自 MoviePilot e6820a8 `app/modules/indexer/spider/__init__.py`:
|
|
* - `select_media_categories` / `resolve_category_media_type` → 同名函数
|
|
* - `SiteSpider.__get_search_url` / `__format_search_word` → `SiteSpider.getSearchUrl` / `formatSearchWord`
|
|
* - `SiteSpider.get_info` 及 `__get_title/__get_description/__get_detail/__get_download/__get_grabs/__get_leechers/
|
|
* __get_seeders/__get_size/__get_imdbid/__get_downloadvolumefactor/__get_uploadvolumefactor/__get_pubdate/__get_date/
|
|
* __get_date_from_cell/__get_date_elapsed/__get_free_date/__get_labels/__get_hit_and_run/__get_category`
|
|
* → `SiteSpider.getInfo` 与同名私有方法
|
|
* - `_safe_query / __filter_text / __remove / __attribute_or_text / __index` → 同名方法
|
|
* - `__is_login_or_permission_page` → `isLoginOrPermissionPage`
|
|
* - `parse` → `SiteSpider.parse`(不含 Rust 加速分支);`search_type=subtitles` 分支:`get_subtitle_info /
|
|
* __get_language_icon / __get_report_url / __get_subtitle_field / __fill_subtitle_ids` → `getSubtitleInfo` 等同名方法
|
|
* - `app/foundation/size.py parse_size` → `parseSizeMp`(Python round 为银行家舍入,同样实现)
|
|
*
|
|
* 与 MP 的差异(均在 README 与迁移账本登记):
|
|
* 1. 额外输出 `category_id`(原始分类 ID),`category` 仍是 MP 的媒体类型投影(REQ §4.5 原始分类不得丢失)。
|
|
* 2. 系数 `selector` 模式用浮点而非 int()(MP 对 0.5 抛异常导致整条丢失,迁移账本 SA4)。
|
|
* 3. 不发请求:HTTP/Cookie/UA 由 adapter 注入,本类只负责 URL 构造与 HTML 解析(便于限流与脱敏统一)。
|
|
* 4. 查询范围:PyQuery 在行元素上查询是 descendant-or-self(行自身也会命中),MP Rust 快路径与 cheerio `.find` 只查后代。
|
|
* 本项目取折中:促销 case / hr / labels 这三类"行级标记"(NexusPHP 把 free_bg、sticky 等 class 挂在 <tr> 上)
|
|
* 包含行自身,与 Python 一致;取值字段(safeQuery)只查后代,避免 `div:nth-child(3)` 这类选择器命中行本身取到整行文本。
|
|
*/
|
|
import { type CheerioAPI, load } from "cheerio";
|
|
import type {
|
|
IndexerDefinition,
|
|
SpiderCategoryConfig,
|
|
SpiderFieldSelector,
|
|
SpiderFields,
|
|
SpiderFilter,
|
|
SpiderListConfig,
|
|
} from "./schema.ts";
|
|
import { renderTemplate } from "./template.ts";
|
|
|
|
type Sel = ReturnType<CheerioAPI>;
|
|
|
|
/** MP `MediaType` 取值 */
|
|
export type MpMediaType = "电影" | "电视剧" | "音乐" | "未知";
|
|
export type MediaKind = "movie" | "tv" | "music";
|
|
|
|
const MEDIA_VALUE: Record<MediaKind, MpMediaType> = { movie: "电影", tv: "电视剧", music: "音乐" };
|
|
|
|
/**
|
|
* 以 htmlparser2(HTML 模式)加载:与 MP 的 lxml.html 一致不自动插入 <tbody>,
|
|
* 使 `table.torrents > tr` 这类 MP 风格选择器保持原语义。
|
|
*/
|
|
export function loadHtml(html: string): CheerioAPI {
|
|
return load(html, { xml: { xmlMode: false, decodeEntities: true } });
|
|
}
|
|
|
|
/**
|
|
* PyQuery(cssselect)接受 `:has("sel")` 引号写法,cheerio-select 不接受;统一去引号以兼容 MP 风格定义。
|
|
*/
|
|
export function normalizeSelector(sel: string): string {
|
|
return sel.replace(/:has\((["'])(.*?)\1\)/g, ":has($2)");
|
|
}
|
|
|
|
/** 解析结果:键名与 MP `TorrentInfo` 字段一致,`category_id` 为本项目扩展。 */
|
|
export interface SpiderTorrent {
|
|
title?: string;
|
|
description?: string;
|
|
page_url?: string;
|
|
enclosure?: string;
|
|
grabs?: number;
|
|
peers?: number;
|
|
seeders?: number;
|
|
size?: number;
|
|
imdbid?: string;
|
|
downloadvolumefactor?: number;
|
|
uploadvolumefactor?: number;
|
|
pubdate?: string;
|
|
date_elapsed?: string;
|
|
freedate?: string;
|
|
labels?: string[];
|
|
hit_and_run?: boolean;
|
|
category?: MpMediaType;
|
|
category_id?: string;
|
|
// ---- 字幕分支(search_type=subtitles)专有字段,键名与 MP 一致 ----
|
|
language_icon?: string;
|
|
report_url?: string;
|
|
language?: string;
|
|
uploader?: string;
|
|
torrent_id?: string;
|
|
subtitle_id?: string;
|
|
file_name?: string;
|
|
}
|
|
|
|
export type SpiderSearchType = "torrents" | "subtitles";
|
|
|
|
/** MP `select_media_categories` */
|
|
export function selectMediaCategories(category: IndexerDefinition["category"], mtype?: MediaKind) {
|
|
if (!category) return [];
|
|
if (mtype === "tv") return category.tv ?? [];
|
|
if (mtype === "movie") return category.movie ?? [];
|
|
if (mtype === "music") return category.music ?? [];
|
|
return [...(category.movie ?? []), ...(category.tv ?? []), ...(category.music ?? [])];
|
|
}
|
|
|
|
/** MP `resolve_category_media_type`:同一 ID 只落在一个媒体类型里才投影,否则未知。 */
|
|
export function resolveCategoryMediaType(value: unknown, category: IndexerDefinition["category"]): MpMediaType {
|
|
if (value == null || !category) return "未知";
|
|
const id = String(value);
|
|
const matches = (["movie", "tv", "music"] as const).filter((k) =>
|
|
(category[k] ?? []).some((item) => item.id != null && String(item.id) === id),
|
|
);
|
|
return matches.length === 1 ? MEDIA_VALUE[matches[0]!] : "未知";
|
|
}
|
|
|
|
/** Python 3 `round()`:恰好 .5 时取偶数(银行家舍入) */
|
|
function roundHalfEven(v: number): number {
|
|
const f = Math.floor(v);
|
|
const diff = v - f;
|
|
if (diff === 0.5) return f % 2 === 0 ? f : f + 1;
|
|
return Math.round(v);
|
|
}
|
|
|
|
/** MP `size_tools.parse_size`:二进制单位,round(银行家舍入)。 */
|
|
export function parseSizeMp(text: string | number | null | undefined): number {
|
|
if (!text) return 0;
|
|
const s = String(text);
|
|
if (/^\d+$/.test(s)) return Number(s);
|
|
const normalized = s.replace(/,/g, "").replace(/ /g, "").toUpperCase();
|
|
const sizeText = normalized.replace(/[KMGTPI]*B?/gi, "");
|
|
const size = Number(sizeText);
|
|
if (sizeText === "" || !Number.isFinite(size)) return 0;
|
|
let v = size;
|
|
if (normalized.includes("PB") || normalized.includes("PIB")) v *= 1024 ** 5;
|
|
else if (normalized.includes("TB") || normalized.includes("TIB")) v *= 1024 ** 4;
|
|
else if (normalized.includes("GB") || normalized.includes("GIB")) v *= 1024 ** 3;
|
|
else if (normalized.includes("MB") || normalized.includes("MIB")) v *= 1024 ** 2;
|
|
else if (normalized.includes("KB") || normalized.includes("KIB")) v *= 1024;
|
|
return roundHalfEven(v);
|
|
}
|
|
|
|
/** strptime 结果(MP dateparse 过滤器返回 datetime) */
|
|
class ParsedDateTime {
|
|
constructor(readonly text: string) {}
|
|
}
|
|
|
|
const STRPTIME_TOKENS: Record<string, string> = {
|
|
"%Y": "(\\d{4})",
|
|
"%m": "(\\d{1,2})",
|
|
"%d": "(\\d{1,2})",
|
|
"%H": "(\\d{1,2})",
|
|
"%M": "(\\d{1,2})",
|
|
"%S": "(\\d{1,2})",
|
|
};
|
|
|
|
/** Python strptime 的最小子集(%Y %m %d %H %M %S),失败抛错(与 MP 一致由调用方吞掉)。 */
|
|
function strptime(text: string, fmt: string): ParsedDateTime {
|
|
const order: string[] = [];
|
|
const re = fmt.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/%[YmdHMS]/g, (t) => {
|
|
order.push(t);
|
|
return STRPTIME_TOKENS[t]!;
|
|
});
|
|
const m = new RegExp(`^${re}$`).exec(text);
|
|
if (!m) throw new Error(`time data '${text}' does not match format '${fmt}'`);
|
|
const v: Record<string, number> = { "%Y": 1900, "%m": 1, "%d": 1, "%H": 0, "%M": 0, "%S": 0 };
|
|
order.forEach((t, i) => {
|
|
v[t] = Number(m[i + 1]);
|
|
});
|
|
const p = (n: number) => String(n).padStart(2, "0");
|
|
return new ParsedDateTime(`${v["%Y"]}-${p(v["%m"]!)}-${p(v["%d"]!)} ${p(v["%H"]!)}:${p(v["%M"]!)}:${p(v["%S"]!)}`);
|
|
}
|
|
|
|
const STD_TIME = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/;
|
|
|
|
/** MP `time_tools.normalize_datetime` 的保守子集:只接受可确定解析的绝对时间。 */
|
|
function normalizeDatetime(value: string): string {
|
|
const m = /^(\d{4})[-/年.](\d{1,2})[-/月.](\d{1,2})日?(?:[ T](\d{1,2}):(\d{1,2})(?::(\d{1,2}))?)?$/.exec(
|
|
value.trim(),
|
|
);
|
|
if (!m) return value;
|
|
const p = (n: string | undefined) => String(Number(n ?? 0)).padStart(2, "0");
|
|
return `${m[1]}-${p(m[2])}-${p(m[3])} ${p(m[4])}:${p(m[5])}:${p(m[6])}`;
|
|
}
|
|
|
|
function isRelativePubdateText(pubdate: string | null | undefined): boolean {
|
|
if (!pubdate) return false;
|
|
const text = String(pubdate).trim().toLowerCase();
|
|
if (/\d{4}[-/年]\d{1,2}/.test(text)) return false;
|
|
if (text.includes("ago")) return true;
|
|
return /\d+\s*(秒|分钟|分|小时|天|周|月|年)/.test(text);
|
|
}
|
|
|
|
function isInvalidPubdateText(pubdate: string | null | undefined): boolean {
|
|
if (!pubdate) return true;
|
|
const text = String(pubdate).trim();
|
|
if (text.toLowerCase() === "now" || text === "0") return true;
|
|
if (isRelativePubdateText(text)) return true;
|
|
return !STD_TIME.test(text);
|
|
}
|
|
|
|
function pyIndex<T>(arr: T[], i: number): T | undefined {
|
|
return i < 0 ? arr[arr.length + i] : arr[i];
|
|
}
|
|
|
|
/** Python `parse_qs(urlparse(url).query).get(key)[0]` */
|
|
function querystringValue(text: string, key: string): string {
|
|
const q = text.includes("?") ? text.slice(text.indexOf("?") + 1) : "";
|
|
const v = new URLSearchParams(q.split("#")[0]).getAll(key).filter((x) => x !== "");
|
|
return v[0] ?? "";
|
|
}
|
|
|
|
/** PyQuery `.text()` 近似:`<br>` 视为换行,行内空白折叠。 */
|
|
function pqText(el: Sel): string {
|
|
const c = el.clone();
|
|
c.find("br").replaceWith("\n");
|
|
return c
|
|
.text()
|
|
.split("\n")
|
|
.map((l) => l.replace(/[ \t\r\f\v\u00a0]+/g, " ").trim())
|
|
.filter((l) => l !== "")
|
|
.join("\n");
|
|
}
|
|
|
|
export interface SiteSpiderQuery {
|
|
keyword?: string | string[];
|
|
mtype?: MediaKind;
|
|
/** 逗号分隔的分类 ID 白名单(MP `cat`) */
|
|
cat?: string;
|
|
/** 站点内部页码(MP 0 起) */
|
|
page?: number;
|
|
/** MP `search_type`:subtitles 时改用 `def.subtitles` 的配置 */
|
|
searchType?: SpiderSearchType;
|
|
}
|
|
|
|
export class SiteSpider {
|
|
readonly def: IndexerDefinition;
|
|
readonly keyword: string | string[] | undefined;
|
|
readonly mtype: MediaKind | undefined;
|
|
readonly cat: string | undefined;
|
|
readonly page: number;
|
|
readonly domain: string;
|
|
readonly resultNum: number;
|
|
readonly siteMediaType: MediaKind | undefined;
|
|
readonly searchType: SpiderSearchType;
|
|
private list: SpiderListConfig;
|
|
private fields: SpiderFields;
|
|
private searchCfg: IndexerDefinition["search"];
|
|
private batchCfg: IndexerDefinition["batch"];
|
|
private browseCfg: IndexerDefinition["browse"];
|
|
private categoryCfg: SpiderCategoryConfig | undefined;
|
|
private requestedResultMediaType: MediaKind | undefined;
|
|
private $!: CheerioAPI;
|
|
private info: SpiderTorrent = {};
|
|
|
|
constructor(def: IndexerDefinition, q: SiteSpiderQuery = {}) {
|
|
this.def = def;
|
|
this.keyword = q.keyword;
|
|
this.mtype = q.mtype;
|
|
this.cat = q.cat;
|
|
this.page = q.page ?? 0;
|
|
this.siteMediaType = fromAgent(def.media_type);
|
|
this.searchType = q.searchType ?? "torrents";
|
|
if (this.searchType === "subtitles") {
|
|
const sc = def.subtitles ?? {};
|
|
this.searchCfg = sc.search;
|
|
this.batchCfg = sc.batch;
|
|
this.browseCfg = sc.browse;
|
|
this.categoryCfg = sc.category;
|
|
this.list = sc.list ?? { selector: "" };
|
|
this.fields = sc.fields ?? {};
|
|
this.resultNum = Number(sc.result_num ?? def.result_num ?? 100);
|
|
} else {
|
|
this.searchCfg = def.search;
|
|
this.batchCfg = def.batch;
|
|
this.browseCfg = def.browse;
|
|
this.categoryCfg = def.category;
|
|
this.list = def.torrents.list;
|
|
this.fields = def.torrents.fields ?? {};
|
|
if (!q.keyword && def.browse) {
|
|
this.list = def.browse.list ?? this.list;
|
|
this.fields = def.browse.fields ?? this.fields;
|
|
}
|
|
this.resultNum = Number(def.result_num ?? 100);
|
|
}
|
|
this.domain = def.domain.endsWith("/") ? def.domain : `${def.domain}/`;
|
|
}
|
|
|
|
/** MP `__can_search` */
|
|
canSearch(): boolean {
|
|
if (this.siteMediaType && this.mtype && this.siteMediaType !== this.mtype) return false;
|
|
return Boolean(this.searchCfg && this.domain);
|
|
}
|
|
|
|
/** MP `__format_search_word` */
|
|
private formatSearchWord(word: string): string {
|
|
if (word && /^tt\d+$/.test(word) && this.searchCfg?.imdbid_format) {
|
|
return this.searchCfg.imdbid_format
|
|
.replace(/\{keyword\}/g, word)
|
|
.replace(/\{imdbid\}/g, word)
|
|
.replace(/\{imdbid_num\}/g, word.slice(2));
|
|
}
|
|
return word;
|
|
}
|
|
|
|
/** MP `__get_search_url` */
|
|
getSearchUrl(): string {
|
|
const search = this.searchCfg ?? { paths: [] };
|
|
const paths = search.paths ?? [];
|
|
let torrentspath = "";
|
|
let typedPathSelected = false;
|
|
let categoryFilterSelected = false;
|
|
if (paths.length === 1) {
|
|
torrentspath = paths[0]!.path ?? "";
|
|
} else {
|
|
let fallback = "";
|
|
const expected = this.mtype;
|
|
for (const p of paths) {
|
|
if (p.type === "all" && !fallback) fallback = p.path ?? "";
|
|
if ((expected && p.type === expected) || (!expected && p.type === "all")) {
|
|
torrentspath = p.path ?? "";
|
|
typedPathSelected = Boolean(expected && p.type === expected);
|
|
break;
|
|
}
|
|
}
|
|
if (!torrentspath) torrentspath = fallback;
|
|
}
|
|
|
|
let searchurl: string;
|
|
if (this.keyword && (typeof this.keyword === "string" || this.keyword.length > 0)) {
|
|
let searchWord: string;
|
|
let searchMode: string;
|
|
if (Array.isArray(this.keyword)) {
|
|
if (this.batchCfg) {
|
|
const delimiter = this.batchCfg.delimiter || " ";
|
|
const spaceReplace = this.batchCfg.space_replace || " ";
|
|
searchWord = this.keyword.map((k) => String(k).replace(/ /g, spaceReplace)).join(delimiter);
|
|
} else {
|
|
searchWord = this.keyword.join(" ");
|
|
}
|
|
searchMode = "1";
|
|
} else {
|
|
searchWord = this.keyword;
|
|
searchMode = "0";
|
|
}
|
|
const isImdb = typeof this.keyword === "string" && /^tt\d+$/.test(this.keyword);
|
|
searchWord = this.formatSearchWord(searchWord);
|
|
const indexerParams: Record<string, string | number> = { ...(search.params ?? {}) };
|
|
if (Object.keys(indexerParams).length > 0) {
|
|
if (indexerParams.search_area && !isImdb) delete indexerParams.search_area;
|
|
const params: Record<string, string | number | Array<string | number>> = {
|
|
search_mode: searchMode,
|
|
search_area: 0,
|
|
page: this.page || 0,
|
|
notnewword: 1,
|
|
};
|
|
for (const [k, v] of Object.entries(indexerParams)) params[k] = String(v).replace(/\{keyword\}/g, searchWord);
|
|
const category = this.categoryCfg;
|
|
if (category) {
|
|
const cats = selectMediaCategories(category, this.mtype);
|
|
const allowed = this.cat ? new Set(this.cat.split(",")) : null;
|
|
for (const c of cats) {
|
|
if (allowed && !allowed.has(String(c.id))) continue;
|
|
if (category.field) {
|
|
categoryFilterSelected = true;
|
|
const cur = String(params[category.field] ?? "");
|
|
params[category.field] = cur + (category.delimiter ?? " ") + String(c.id);
|
|
} else {
|
|
const categoryParam = c.param || category.param;
|
|
if (categoryParam) {
|
|
categoryFilterSelected = true;
|
|
const cid = c.value ?? c.id;
|
|
const cur = params[categoryParam];
|
|
if (cur === undefined) params[categoryParam] = cid;
|
|
else if (Array.isArray(cur)) cur.push(cid);
|
|
else params[categoryParam] = [cur, cid];
|
|
} else {
|
|
categoryFilterSelected = true;
|
|
params[`cat${c.id}`] = 1;
|
|
}
|
|
}
|
|
if (c.params) Object.assign(params, c.params);
|
|
}
|
|
}
|
|
searchurl = combineUrl(this.domain, torrentspath, params);
|
|
} else {
|
|
searchurl =
|
|
this.domain +
|
|
torrentspath
|
|
.replace(/\{keyword\}/g, encodeURIComponent(searchWord))
|
|
.replace(/\{page\}/g, String(this.page || 0));
|
|
}
|
|
} else {
|
|
let page = this.page || 0;
|
|
if (this.browseCfg && !typedPathSelected) {
|
|
torrentspath = this.browseCfg.path;
|
|
if (this.browseCfg.start) page = Number(this.browseCfg.start) + Number(this.page || 0);
|
|
} else if (this.page && !torrentspath.includes("{page}")) {
|
|
torrentspath += `${torrentspath.includes("?") ? "&" : "?"}page=${this.page}`;
|
|
}
|
|
searchurl = this.domain + torrentspath.replace(/\{page\}/g, String(page)).replace(/\{keyword\}/g, "");
|
|
}
|
|
|
|
this.requestedResultMediaType =
|
|
search.result_media_type === "requested" && this.mtype && (typedPathSelected || categoryFilterSelected)
|
|
? this.mtype
|
|
: undefined;
|
|
return searchurl;
|
|
}
|
|
|
|
/** MP `__is_login_or_permission_page` */
|
|
static isLoginOrPermissionPage($: CheerioAPI): boolean {
|
|
const title = $("title").first().text().trim();
|
|
const pageText = $.root().text().split(/\s+/).filter(Boolean).join(" ").slice(0, 1000);
|
|
if (title === "登录" || title.includes(":: 登录")) return true;
|
|
return ["未登录", "登录 / 注册", "必须在登录后才能访问", "你需要启用cookies才能登录"].some((m) =>
|
|
pageText.includes(m),
|
|
);
|
|
}
|
|
|
|
/** MP `parse`:登录/权限页返回 isError=true 与空列表。 */
|
|
parse(html: string): { isError: boolean; loginPage: boolean; torrents: SpiderTorrent[]; $: CheerioAPI | null } {
|
|
if (!html) return { isError: true, loginPage: false, torrents: [], $: null };
|
|
const $ = loadHtml(html);
|
|
if (SiteSpider.isLoginOrPermissionPage($)) return { isError: true, loginPage: true, torrents: [], $ };
|
|
this.$ = $;
|
|
const out: SpiderTorrent[] = [];
|
|
const rows = $(normalizeSelector(this.list.selector ?? ""));
|
|
rows.each((i, el) => {
|
|
if (i >= this.resultNum) return false;
|
|
const info = this.searchType === "subtitles" ? this.getSubtitleInfo($(el)) : this.getInfo($(el));
|
|
if (Object.keys(info).length > 0) out.push(this.applyRequestedMediaType(info));
|
|
return undefined;
|
|
});
|
|
return { isError: false, loginPage: false, torrents: out, $ };
|
|
}
|
|
|
|
/** 对任意片段(如详情页头部)按字段配置取值 */
|
|
infoFrom($: CheerioAPI, el: Sel): SpiderTorrent {
|
|
this.$ = $;
|
|
return this.getInfo(el);
|
|
}
|
|
|
|
private applyRequestedMediaType(t: SpiderTorrent): SpiderTorrent {
|
|
if (this.requestedResultMediaType) t.category = MEDIA_VALUE[this.requestedResultMediaType];
|
|
return t;
|
|
}
|
|
|
|
/** MP `get_info`:任一字段异常则整条返回 {}。 */
|
|
getInfo(row: Sel): SpiderTorrent {
|
|
this.info = {};
|
|
try {
|
|
this.getTitle(row);
|
|
this.getDescription(row);
|
|
this.getDetail(row);
|
|
this.getDownload(row);
|
|
this.getGrabs(row);
|
|
this.getLeechers(row);
|
|
this.getSeeders(row);
|
|
this.getSize(row);
|
|
this.getImdbid(row);
|
|
this.getVolumeFactor(row, "downloadvolumefactor");
|
|
this.getVolumeFactor(row, "uploadvolumefactor");
|
|
this.getPubdate(row);
|
|
this.getDateElapsed(row);
|
|
this.getFreeDate(row);
|
|
this.getLabels(row);
|
|
this.getHitAndRun(row);
|
|
this.getCategory(row);
|
|
return { ...this.info };
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
/** MP `get_subtitle_info`:缺标题或下载链接的行丢弃;任一字段异常整条返回 {}。 */
|
|
getSubtitleInfo(row: Sel): SpiderTorrent {
|
|
this.info = {};
|
|
try {
|
|
this.getTitle(row);
|
|
this.getDescription(row);
|
|
this.getDetail(row);
|
|
this.getDownload(row);
|
|
this.getSize(row);
|
|
this.getPubdate(row);
|
|
this.getDateElapsed(row);
|
|
this.getGrabs(row);
|
|
this.getLinkField(row, "language_icon", "language_icon");
|
|
this.getLinkField(row, "report", "report_url");
|
|
for (const name of ["language", "uploader", "torrent_id", "subtitle_id", "file_name"] as const) {
|
|
this.getSubtitleField(row, name);
|
|
}
|
|
this.fillSubtitleIds();
|
|
if (!this.info.title || !this.info.enclosure) return {};
|
|
return { ...this.info };
|
|
} catch {
|
|
return {};
|
|
}
|
|
}
|
|
|
|
/** MP `__get_language_icon` / `__get_report_url`:相对链接补全为绝对链接 */
|
|
private getLinkField(row: Sel, name: string, key: "language_icon" | "report_url") {
|
|
if (!(name in this.fields)) return;
|
|
const selector = this.fields[name] ?? {};
|
|
const link = this.filterText(this.safeQuery(row, selector), selector.filters);
|
|
if (link) this.info[key] = this.normalizeLink(String(link));
|
|
}
|
|
|
|
/** MP `__get_subtitle_field` */
|
|
private getSubtitleField(row: Sel, name: "language" | "uploader" | "torrent_id" | "subtitle_id" | "file_name") {
|
|
const selector = this.fields[name];
|
|
if (!selector || Object.keys(selector).length === 0) return;
|
|
const v = this.filterText(this.safeQuery(row, selector), selector.filters);
|
|
if (v != null) this.info[name] = v instanceof ParsedDateTime ? v.text : String(v);
|
|
}
|
|
|
|
/** MP `__fill_subtitle_ids`:从下载链接 query 补种子 ID(torrentid/torrent_id)与字幕 ID(subid/subtitle) */
|
|
private fillSubtitleIds() {
|
|
const enclosure = this.info.enclosure;
|
|
if (!enclosure) return;
|
|
const q = enclosure.includes("?") ? enclosure.slice(enclosure.indexOf("?") + 1).split("#")[0]! : "";
|
|
const params = new URLSearchParams(q);
|
|
const first = (...keys: string[]) => {
|
|
for (const k of keys) {
|
|
const v = params.getAll(k).find((x) => x !== "");
|
|
if (v !== undefined) return v;
|
|
}
|
|
return undefined;
|
|
};
|
|
if (!this.info.torrent_id) {
|
|
const v = first("torrentid", "torrent_id");
|
|
if (v !== undefined) this.info.torrent_id = v;
|
|
}
|
|
if (!this.info.subtitle_id) {
|
|
const v = first("subid", "subtitle");
|
|
if (v !== undefined) this.info.subtitle_id = v;
|
|
}
|
|
}
|
|
|
|
// ---- 字段 ----
|
|
|
|
private getTitle(row: Sel) {
|
|
if (!("title" in this.fields)) return;
|
|
const selector = this.fields.title ?? {};
|
|
let title: string | null | undefined;
|
|
if (selector.selector) {
|
|
title = this.safeQuery(row, selector);
|
|
} else if (selector.text) {
|
|
const render: Record<string, unknown> = {};
|
|
if ("title_default" in this.fields) render.title_default = this.safeQuery(row, this.fields.title_default);
|
|
if ("title_optional" in this.fields) render.title_optional = this.safeQuery(row, this.fields.title_optional);
|
|
title = renderTemplate(selector.text, render);
|
|
}
|
|
const v = this.filterText(title, selector.filters);
|
|
if (v != null) this.info.title = String(v);
|
|
}
|
|
|
|
private getDescription(row: Sel) {
|
|
if (!("description" in this.fields)) return;
|
|
const selector = this.fields.description ?? {};
|
|
let desc: string | null | undefined;
|
|
if (selector.selector || selector.selectors) {
|
|
const s: SpiderFieldSelector = { ...selector };
|
|
if (selector.selectors && !selector.selector) s.selector = selector.selectors;
|
|
desc = this.safeQuery(row, s);
|
|
} else if (selector.text) {
|
|
const render: Record<string, unknown> = {};
|
|
for (const k of ["tags", "subject", "description_free_forever", "description_normal"]) {
|
|
if (k in this.fields) render[k] = this.safeQuery(row, this.fields[k]);
|
|
}
|
|
desc = renderTemplate(selector.text, render);
|
|
}
|
|
const v = this.filterText(desc, selector.filters);
|
|
if (v != null) this.info.description = String(v);
|
|
}
|
|
|
|
private normalizeLink(link: string): string {
|
|
if (link.startsWith("http")) return link;
|
|
if (link.startsWith("//")) return `${this.domain.split(":")[0]}:${link}`;
|
|
if (link.startsWith("/")) return this.domain + link.slice(1);
|
|
return this.domain + link;
|
|
}
|
|
|
|
private getDetail(row: Sel) {
|
|
if (!("details" in this.fields)) return;
|
|
const selector = this.fields.details ?? {};
|
|
const link = this.filterText(this.safeQuery(row, selector), selector.filters);
|
|
if (link) this.info.page_url = this.normalizeLink(String(link));
|
|
}
|
|
|
|
private getDownload(row: Sel) {
|
|
if (!("download" in this.fields)) return;
|
|
const selector = this.fields.download ?? {};
|
|
const raw = this.filterText(this.safeQuery(row, selector), selector.filters);
|
|
if (!raw) return;
|
|
const link = String(raw);
|
|
if (!link.startsWith("http") && !link.startsWith("magnet")) {
|
|
const [scheme, host] = splitNetloc(this.domain);
|
|
if (host && link.includes(host)) {
|
|
this.info.enclosure = link.startsWith("/") ? `${scheme}:${link}` : `${scheme}://${link}`;
|
|
} else {
|
|
this.info.enclosure = link.startsWith("/") ? `${this.domain}${link.slice(1)}` : `${this.domain}${link}`;
|
|
}
|
|
} else {
|
|
this.info.enclosure = link;
|
|
}
|
|
}
|
|
|
|
private countField(row: Sel, name: string, key: "grabs" | "peers" | "seeders", allowZeroText: boolean) {
|
|
if (!(name in this.fields)) return;
|
|
const selector = this.fields[name] ?? {};
|
|
const item = this.safeQuery(row, selector);
|
|
const present = allowZeroText ? item != null && item !== "" : Boolean(item);
|
|
if (present) {
|
|
let v: unknown = String(item).split("/")[0]!.replace(/,/g, "");
|
|
v = this.filterText(v as string, selector.filters);
|
|
this.info[key] = typeof v === "string" && /^\d+$/.test(v) ? Number(v) : 0;
|
|
} else {
|
|
this.info[key] = 0;
|
|
}
|
|
}
|
|
|
|
private getGrabs(row: Sel) {
|
|
this.countField(row, "grabs", "grabs", true);
|
|
}
|
|
private getLeechers(row: Sel) {
|
|
this.countField(row, "leechers", "peers", false);
|
|
}
|
|
private getSeeders(row: Sel) {
|
|
this.countField(row, "seeders", "seeders", false);
|
|
}
|
|
|
|
private getSize(row: Sel) {
|
|
if (!("size" in this.fields)) return;
|
|
const selector = this.fields.size ?? {};
|
|
const item = this.safeQuery(row, selector);
|
|
if (item != null && item !== "") {
|
|
const v = this.filterText(item.replace(/\n/g, "").trim(), selector.filters);
|
|
this.info.size = parseSizeMp(v == null ? "" : String(v));
|
|
} else {
|
|
this.info.size = 0;
|
|
}
|
|
}
|
|
|
|
private getImdbid(row: Sel) {
|
|
if (!("imdbid" in this.fields)) return;
|
|
const selector = this.fields.imdbid ?? {};
|
|
const v = this.filterText(this.safeQuery(row, selector), selector.filters);
|
|
if (v != null) this.info.imdbid = String(v);
|
|
}
|
|
|
|
private getVolumeFactor(row: Sel, key: "downloadvolumefactor" | "uploadvolumefactor") {
|
|
const selector = this.fields[key];
|
|
if (!selector || Object.keys(selector).length === 0) return;
|
|
this.info[key] = 1;
|
|
if (selector.case) {
|
|
for (const [css, value] of Object.entries(selector.case)) {
|
|
if (this.rowScope(row, css).length > 0) {
|
|
this.info[key] = value;
|
|
break;
|
|
}
|
|
}
|
|
} else if (selector.selector) {
|
|
const item = this.safeQuery(row, selector);
|
|
const m = item ? /(\d+\.?\d*)/.exec(item) : null;
|
|
// 偏离 MP:MP 用 int() 转换,"0.5" 抛异常导致整条丢失(迁移账本 SA4)
|
|
if (m) this.info[key] = Number.parseFloat(m[1]!);
|
|
}
|
|
}
|
|
|
|
private getPubdate(row: Sel) {
|
|
if (!("date_added" in this.fields) && !("date" in this.fields)) return;
|
|
let selector: SpiderFieldSelector = this.fields.date_added ?? {};
|
|
let pubdate: string | null | undefined = this.safeQuery(row, selector);
|
|
if (!pubdate) {
|
|
selector = this.fields.date ?? {};
|
|
pubdate = this.getDate(row, selector);
|
|
}
|
|
if (pubdate) pubdate = pubdate.replace(/\n/g, " ").trim();
|
|
const filtered = this.filterText(pubdate, selector.filters);
|
|
let value: string | undefined;
|
|
if (filtered instanceof ParsedDateTime) value = filtered.text;
|
|
else if (filtered) value = STD_TIME.test(String(filtered)) ? String(filtered) : normalizeDatetime(String(filtered));
|
|
if (value && !isInvalidPubdateText(value)) this.info.pubdate = value;
|
|
else delete this.info.pubdate;
|
|
}
|
|
|
|
private getDate(row: Sel, selector: SpiderFieldSelector): string | null {
|
|
if (!selector || Object.keys(selector).length === 0) return null;
|
|
if (selector.selector) return this.safeQuery(row, selector) ?? null;
|
|
if (!selector.text) return null;
|
|
const render: Record<string, string | null> = {};
|
|
for (const name of ["date_elapsed", "date_added"]) {
|
|
const fs = this.fields[name] ?? {};
|
|
let v = this.safeQuery(row, fs) ?? null;
|
|
if (!v) v = this.getDateFromCell(row, fs);
|
|
render[name] = v;
|
|
}
|
|
if (!Object.values(render).some(Boolean)) return null;
|
|
const s = renderTemplate(selector.text, render);
|
|
if (s === "now" || isRelativePubdateText(s)) return null;
|
|
return s;
|
|
}
|
|
|
|
private getDateFromCell(row: Sel, selector: SpiderFieldSelector): string | null {
|
|
const sel = selector.selector?.trim();
|
|
if (!sel?.includes("> span")) return null;
|
|
return this.safeQuery(row, { selector: sel.split("> span")[0]!.trim() }) ?? null;
|
|
}
|
|
|
|
private getDateElapsed(row: Sel) {
|
|
if (!("date_elapsed" in this.fields)) return;
|
|
const selector = this.fields.date_elapsed ?? {};
|
|
const v = this.filterText(this.safeQuery(row, selector), selector.filters);
|
|
if (v != null) this.info.date_elapsed = String(v);
|
|
}
|
|
|
|
private getFreeDate(row: Sel) {
|
|
if (!("freedate" in this.fields)) return;
|
|
const selector = this.fields.freedate ?? {};
|
|
const v = this.filterText(this.safeQuery(row, selector), selector.filters);
|
|
if (v != null) this.info.freedate = v instanceof ParsedDateTime ? v.text : String(v);
|
|
}
|
|
|
|
private getLabels(row: Sel) {
|
|
if (!("labels" in this.fields)) return;
|
|
const selector = this.fields.labels ?? {};
|
|
if (!selector.selector) {
|
|
this.info.labels = [];
|
|
return;
|
|
}
|
|
const q = this.rowScope(row, selector.selector).clone();
|
|
this.remove(q, selector);
|
|
this.info.labels = this.attributeOrText(q, selector).filter((x): x is string => Boolean(x));
|
|
}
|
|
|
|
private getHitAndRun(row: Sel) {
|
|
if (!("hr" in this.fields)) return;
|
|
const selector = this.fields.hr ?? {};
|
|
this.info.hit_and_run = selector.selector ? this.rowScope(row, selector.selector).length > 0 : false;
|
|
}
|
|
|
|
private getCategory(row: Sel) {
|
|
if (this.requestedResultMediaType) {
|
|
this.info.category = MEDIA_VALUE[this.requestedResultMediaType];
|
|
return;
|
|
}
|
|
if (!("category" in this.fields)) {
|
|
if (this.siteMediaType) this.info.category = MEDIA_VALUE[this.siteMediaType];
|
|
return;
|
|
}
|
|
const selector = this.fields.category ?? {};
|
|
const value = this.filterText(this.safeQuery(row, selector), selector.filters);
|
|
if (value != null && value !== "") this.info.category_id = String(value);
|
|
let resolved = resolveCategoryMediaType(value, this.categoryCfg);
|
|
if (resolved === "未知" && this.siteMediaType) resolved = MEDIA_VALUE[this.siteMediaType];
|
|
this.info.category = resolved;
|
|
}
|
|
|
|
// ---- 通用查询 ----
|
|
|
|
/** 行级标记查询:行自身 + 后代(PyQuery descendant-or-self,见文件头差异 4);`:scope` 选择器只查后代 */
|
|
private rowScope(row: Sel, selector: string): Sel {
|
|
const css = normalizeSelector(selector);
|
|
const desc = row.find(css);
|
|
if (css.includes(":scope")) return desc;
|
|
return row.filter(css).add(desc);
|
|
}
|
|
|
|
/** MP `_safe_query` */
|
|
safeQuery(row: Sel, config: SpiderFieldSelector | undefined): string | null | undefined {
|
|
if (!config?.selector) return null;
|
|
let q = row.find(normalizeSelector(config.selector));
|
|
if (config.remove) q = q.clone();
|
|
this.remove(q, config);
|
|
return this.index(this.attributeOrText(q, config), config);
|
|
}
|
|
|
|
private remove(q: Sel, config: SpiderFieldSelector) {
|
|
if (!config.remove) return;
|
|
for (const v of config.remove.split(", ")) q.find(normalizeSelector(v)).remove();
|
|
}
|
|
|
|
private attributeOrText(q: Sel, config: SpiderFieldSelector): Array<string | undefined> {
|
|
if (q.length === 0) return [];
|
|
const out: Array<string | undefined> = [];
|
|
q.each((_, el) => {
|
|
const item = this.$(el);
|
|
out.push(config.attribute ? item.attr(config.attribute) : pqText(item));
|
|
});
|
|
return out;
|
|
}
|
|
|
|
private index(items: Array<string | undefined>, config: SpiderFieldSelector): string | undefined {
|
|
if (items.length === 0) return undefined;
|
|
if (config.contents !== undefined && items.length > Number(config.contents)) {
|
|
return pyIndex((items[0] ?? "").split("\n"), Number(config.contents));
|
|
}
|
|
if (config.index !== undefined && items.length > Number(config.index)) return pyIndex(items, Number(config.index));
|
|
return items[0];
|
|
}
|
|
|
|
/** MP `__filter_text`;单个过滤器失败只跳过该过滤器。 */
|
|
filterText(
|
|
text: string | null | undefined,
|
|
filters: SpiderFilter[] | undefined,
|
|
): string | ParsedDateTime | null | undefined {
|
|
if (!text || !filters || !Array.isArray(filters)) return text;
|
|
let t: string | ParsedDateTime = String(text);
|
|
for (const f of filters) {
|
|
if (!t) break;
|
|
try {
|
|
if (t instanceof ParsedDateTime) break;
|
|
const s: string = t;
|
|
switch (f.name) {
|
|
case "re_search": {
|
|
if (!Array.isArray(f.args)) break;
|
|
const m = new RegExp(f.args[0]).exec(s);
|
|
if (m) t = m[Number(f.args[f.args.length - 1])] ?? "";
|
|
break;
|
|
}
|
|
case "split":
|
|
if (Array.isArray(f.args)) {
|
|
const parts = s.split(f.args[0]);
|
|
const v = pyIndex(parts, Number(f.args[f.args.length - 1]));
|
|
if (v === undefined) throw new Error("list index out of range");
|
|
t = v;
|
|
}
|
|
break;
|
|
case "replace":
|
|
if (Array.isArray(f.args)) t = s.split(String(f.args[0])).join(String(f.args[f.args.length - 1]));
|
|
break;
|
|
case "dateparse":
|
|
if (typeof f.args === "string") t = strptime(s.replace(/\n/g, " ").trim(), f.args);
|
|
break;
|
|
case "strip":
|
|
t = s.trim();
|
|
break;
|
|
case "appendleft":
|
|
t = `${f.args}${s}`;
|
|
break;
|
|
case "querystring":
|
|
t = querystringValue(s, String(f.args));
|
|
break;
|
|
}
|
|
} catch {
|
|
// MP 仅 debug 日志,继续下一个过滤器
|
|
}
|
|
}
|
|
return typeof t === "string" ? t.trim() : t;
|
|
}
|
|
}
|
|
|
|
/** MP `MediaType.from_agent` 的子集 */
|
|
function fromAgent(v: string | undefined): MediaKind | undefined {
|
|
if (!v) return undefined;
|
|
const s = v.toLowerCase();
|
|
if (s === "movie" || v === "电影") return "movie";
|
|
if (s === "tv" || v === "电视剧") return "tv";
|
|
if (s === "music" || v === "音乐") return "music";
|
|
return undefined;
|
|
}
|
|
|
|
/** MP `url.split_netloc` */
|
|
function splitNetloc(url: string): [string, string] {
|
|
if (!url) return ["", ""];
|
|
if (!url.startsWith("http")) return ["http", url];
|
|
const u = new URL(url);
|
|
return [u.protocol.replace(":", ""), u.host];
|
|
}
|
|
|
|
/** MP `UrlUtils.combine_url`:urljoin + 合并已有 query + urlencode(doseq=True)。 */
|
|
export function combineUrl(
|
|
host: string,
|
|
path: string,
|
|
query: Record<string, string | number | Array<string | number>>,
|
|
): string {
|
|
const base = host.endsWith("/") ? host : `${host}/`;
|
|
const url = new URL(path || "/", base);
|
|
for (const [k, v] of Object.entries(query)) {
|
|
url.searchParams.delete(k);
|
|
if (Array.isArray(v)) for (const x of v) url.searchParams.append(k, String(x));
|
|
else url.searchParams.set(k, String(v));
|
|
}
|
|
return url.toString();
|
|
}
|