feat(parser,sites): W5-D port title/zhconv/site parsing gaps found by MP differential
Title recognition (parseRelease / parseReleasePath):
- anime prefix tags (★N月新番, [国漫], 【22年日剧】), category lead bracket, `A / B` cn aliases
- `Name [01]`, `[TV 04]`, `#13`, `名称 12.mkv`, `S01 13-14`, `E13-14` episodes
- lead bracket as title vs group, trailing [Group], YTS.MX/Hall_of_C style groups, 字幕组 suffix
- `[dot.release.name][…]` unbracketing, `Season 2017` as year, leading date not airDate,
date-only parent dir does not merge as TV
- original disc: Blu-ray + AVC/HEVC/VC-1/MPEG-2 without x26x/H.26x -> BluRay-Disc
- `{[tmdbid=;type=;s=;e=]}` explicit block, bangumi/anilist ids
- bare [1080]/[5.1], DTS-HD MA5.1, £ multi-group, 合集, pinyin title -> cn from subtitle,
streaming platform table, CHS/CHT/GB/BIG5 in subtitle
zh-map: add 迴滾魷瑯臥駭; document one-to-many chars kept out.
Sites: tbody branch + header-row tightening via movieclaw-import rowSelector (18 definitions),
row-level case/hr/labels include the row itself, subtitle branch (searchType: "subtitles"),
parseSizeMp banker's rounding.
Differential snapshots regenerated; MP test sample sources registered in ledger.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
8352e5e881
commit
36dc048e89
41 files changed
+3111
-2372
No files matched your search
@@ -78,7 +78,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr:has(table.torrentname)"
|
||||
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -71,7 +71,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr:has(table.torrentname)"
|
||||
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -102,7 +102,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr, table.torrents > tbody > tr"
|
||||
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -66,7 +66,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr, table.torrents > tbody > tr"
|
||||
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -70,7 +70,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr, table.torrents > tbody > tr"
|
||||
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -206,7 +206,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr:has(table.torrentname)"
|
||||
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -70,7 +70,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr, table.torrents > tbody > tr"
|
||||
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -70,7 +70,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents tr"
|
||||
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -78,7 +78,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr:has(table.torrentname)"
|
||||
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -43,7 +43,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr, table.torrents > tbody > tr"
|
||||
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -83,7 +83,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr:has(table.torrentname)"
|
||||
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -66,7 +66,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr, table.torrents > tbody > tr"
|
||||
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -66,7 +66,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr:has(table.torrentname)"
|
||||
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -66,7 +66,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr:has(table.torrentname)"
|
||||
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -62,7 +62,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr, table.torrents > tbody > tr"
|
||||
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -70,7 +70,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr, table.torrents > tbody > tr"
|
||||
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -70,7 +70,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr:has(table.torrentname)"
|
||||
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -43,7 +43,7 @@
|
||||
},
|
||||
"torrents": {
|
||||
"list": {
|
||||
"selector": "table.torrents > tr:not(:first-child)"
|
||||
"selector": "table.torrents > tr:not(:first-child), table.torrents > tbody > tr:not(:first-child)"
|
||||
},
|
||||
"fields": {
|
||||
"title_default": {
|
||||
|
||||
@@ -63,6 +63,17 @@ export interface SpiderFields {
|
||||
[name: string]: SpiderFieldSelector | undefined;
|
||||
}
|
||||
|
||||
/** MP `indexer.subtitles`:字幕搜索页的独立配置(字段额外支持 language_icon/report/language/uploader/torrent_id/subtitle_id/file_name) */
|
||||
export interface SpiderSubtitleConfig {
|
||||
search?: IndexerDefinition["search"];
|
||||
batch?: IndexerDefinition["batch"];
|
||||
browse?: IndexerDefinition["browse"];
|
||||
category?: SpiderCategoryConfig;
|
||||
list?: SpiderListConfig;
|
||||
fields?: SpiderFields;
|
||||
result_num?: number;
|
||||
}
|
||||
|
||||
/** 定义来源(MovieClaw v0.27.0 MIT 转换 / 手写) */
|
||||
export interface DefinitionProvenance {
|
||||
source: "movieclaw" | "handwritten";
|
||||
@@ -104,6 +115,8 @@ export interface IndexerDefinition {
|
||||
browse?: { path: string; start?: number; list?: SpiderListConfig; fields?: SpiderFields };
|
||||
category?: SpiderCategoryConfig;
|
||||
torrents: { list: SpiderListConfig; fields: SpiderFields };
|
||||
/** 字幕页配置(MP `indexer.subtitles`,search_type=subtitles 时替换 search/batch/browse/category/list/fields) */
|
||||
subtitles?: SpiderSubtitleConfig;
|
||||
/** 本项目扩展(MP 无对应) */
|
||||
umt: UmtSiteExtension;
|
||||
}
|
||||
|
||||
@@ -1,10 +1,18 @@
|
||||
import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit";
|
||||
import {
|
||||
CHDBITS_LIST_PAGE,
|
||||
createMockNexusPhpSite,
|
||||
type MockNexusPhpSite,
|
||||
NEXUSPHP_SUBTITLES_CONFIG,
|
||||
NEXUSPHP_SUBTITLES_PAGE,
|
||||
} from "@umt/testkit";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
audiencesDefinition,
|
||||
BUILTIN_SITE_DEFINITIONS,
|
||||
createSiteRateLimiter,
|
||||
createSpiderSiteAdapter,
|
||||
type IndexerDefinition,
|
||||
parseSizeMp,
|
||||
type SiteAdapter,
|
||||
SiteError,
|
||||
SiteSpider,
|
||||
@@ -306,3 +314,82 @@ describe("观众适配器(spider + mock NexusPHP)", () => {
|
||||
expect(limiter.stats.granted).toBe(4);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SiteSpider 差分补迁(W5-D)", () => {
|
||||
const withTbody = (html: string) =>
|
||||
html.replace(/(<table class="torrents"[^>]*>)([\s\S]*?)(<\/table>\s*<\/body>)/, "$1<tbody>$2</tbody>$3");
|
||||
|
||||
it("列表选择器:显式 <tbody> 的页面也能取到行,表头行不再产出空标题", () => {
|
||||
const chd = BUILTIN_SITE_DEFINITIONS.chdbits as IndexerDefinition;
|
||||
const plain = new SiteSpider(chd, { keyword: "x" }).parse(CHDBITS_LIST_PAGE).torrents;
|
||||
const tbody = new SiteSpider(chd, { keyword: "x" }).parse(withTbody(CHDBITS_LIST_PAGE)).torrents;
|
||||
expect(plain.length).toBeGreaterThan(0);
|
||||
expect(tbody.map((t) => t.title)).toEqual(plain.map((t) => t.title));
|
||||
const hdarea = BUILTIN_SITE_DEFINITIONS.hdarea as IndexerDefinition;
|
||||
const rows = new SiteSpider(hdarea, { keyword: "x" }).parse(CHDBITS_LIST_PAGE).torrents;
|
||||
expect(rows.length).toBe(plain.length);
|
||||
expect(rows.every((t) => Boolean(t.title))).toBe(true);
|
||||
});
|
||||
|
||||
it("行级标记 case/hr/labels 包含行自身(PyQuery descendant-or-self),取值字段只查后代", () => {
|
||||
const def: IndexerDefinition = {
|
||||
id: "self",
|
||||
name: "self",
|
||||
domain: "https://self.example/",
|
||||
torrents: {
|
||||
list: { selector: "table.t > tr" },
|
||||
fields: {
|
||||
title: { selector: "a.t" },
|
||||
downloadvolumefactor: { case: { ".free_bg": 0, "*": 1 } },
|
||||
hr: { selector: ".hr_row" },
|
||||
labels: { selector: ".sticky", attribute: "class" },
|
||||
},
|
||||
},
|
||||
umt: {} as IndexerDefinition["umt"],
|
||||
};
|
||||
const html =
|
||||
'<table class="t"><tr class="free_bg hr_row sticky"><td><a class="t">A</a></td></tr><tr><td><a class="t">B</a></td></tr></table>';
|
||||
const [a, b] = new SiteSpider(def, { keyword: "x" }).parse(html).torrents;
|
||||
expect(a).toMatchObject({
|
||||
title: "A",
|
||||
downloadvolumefactor: 0,
|
||||
hit_and_run: true,
|
||||
labels: ["free_bg hr_row sticky"],
|
||||
});
|
||||
expect(b).toMatchObject({ title: "B", downloadvolumefactor: 1, hit_and_run: false, labels: [] });
|
||||
});
|
||||
|
||||
it("parseSizeMp 与 Python round 一致(银行家舍入)", () => {
|
||||
expect(parseSizeMp("2.5 B")).toBe(2);
|
||||
expect(parseSizeMp("3.5 B")).toBe(4);
|
||||
expect(parseSizeMp("1.5 KB")).toBe(1536);
|
||||
});
|
||||
|
||||
it("字幕分支:get_subtitle_info + __fill_subtitle_ids,缺下载链接的行丢弃", () => {
|
||||
const def: IndexerDefinition = {
|
||||
id: "subs",
|
||||
name: "subs",
|
||||
domain: "https://hhanclub.net/",
|
||||
torrents: { list: { selector: "none" }, fields: {} },
|
||||
subtitles: NEXUSPHP_SUBTITLES_CONFIG,
|
||||
umt: {} as IndexerDefinition["umt"],
|
||||
};
|
||||
const spider = new SiteSpider(def, { keyword: "Example Show", searchType: "subtitles" });
|
||||
const { isError, torrents } = spider.parse(NEXUSPHP_SUBTITLES_PAGE);
|
||||
expect(isError).toBe(false);
|
||||
expect(torrents).toHaveLength(3);
|
||||
expect(torrents[0]).toEqual({
|
||||
title: "Example Show S01E01 1080p WEB-DL CHS",
|
||||
enclosure: "https://hhanclub.net/downloadsubs.php?torrentid=0&subid=1000",
|
||||
size: 114678,
|
||||
pubdate: "2026-04-21 20:54:37",
|
||||
date_elapsed: "1月18天",
|
||||
language_icon: "https://hhanclub.net/pic/flag/china.gif",
|
||||
report_url: "https://hhanclub.net/report.php?subtitle=1000",
|
||||
uploader: "tester0",
|
||||
torrent_id: "0",
|
||||
subtitle_id: "1000",
|
||||
});
|
||||
expect(torrents[2]).toMatchObject({ torrent_id: "2", subtitle_id: "1002", size: 114678 });
|
||||
});
|
||||
});
|
||||
@@ -10,16 +10,27 @@
|
||||
* → `SiteSpider.getInfo` 与同名私有方法
|
||||
* - `_safe_query / __filter_text / __remove / __attribute_or_text / __index` → 同名方法
|
||||
* - `__is_login_or_permission_page` → `isLoginOrPermissionPage`
|
||||
* - `parse` → `SiteSpider.parse`(不含 Rust 加速分支与字幕分支)
|
||||
* - `app/foundation/size.py parse_size` → `parseSizeMp`
|
||||
* - `parse` → `SiteSpider.parse`(不含 Rust 加速分支);`search_type=subtitles` 分支:`get_subtitle_info /
|
||||
* __get_language_icon / __get_report_url / __get_subtitle_field / __fill_subtitle_ids` → `getSubtitleInfo` 等同名方法
|
||||
* - `app/foundation/size.py parse_size` → `parseSizeMp`(Python round 为银行家舍入,同样实现)
|
||||
*
|
||||
* 与 MP 的差异(均在 README 与迁移账本登记):
|
||||
* 1. 额外输出 `category_id`(原始分类 ID),`category` 仍是 MP 的媒体类型投影(REQ §4.5 原始分类不得丢失)。
|
||||
* 2. 系数 `selector` 模式用浮点而非 int()(MP 对 0.5 抛异常导致整条丢失,迁移账本 SA4)。
|
||||
* 3. 不发请求:HTTP/Cookie/UA 由 adapter 注入,本类只负责 URL 构造与 HTML 解析(便于限流与脱敏统一)。
|
||||
* 4. 查询范围:PyQuery 在行元素上查询是 descendant-or-self(行自身也会命中),MP Rust 快路径与 cheerio `.find` 只查后代。
|
||||
* 本项目取折中:促销 case / hr / labels 这三类"行级标记"(NexusPHP 把 free_bg、sticky 等 class 挂在 <tr> 上)
|
||||
* 包含行自身,与 Python 一致;取值字段(safeQuery)只查后代,避免 `div:nth-child(3)` 这类选择器命中行本身取到整行文本。
|
||||
*/
|
||||
import { type CheerioAPI, load } from "cheerio";
|
||||
import type { IndexerDefinition, SpiderFieldSelector, SpiderFields, SpiderFilter, SpiderListConfig } from "./schema.ts";
|
||||
import type {
|
||||
IndexerDefinition,
|
||||
SpiderCategoryConfig,
|
||||
SpiderFieldSelector,
|
||||
SpiderFields,
|
||||
SpiderFilter,
|
||||
SpiderListConfig,
|
||||
} from "./schema.ts";
|
||||
import { renderTemplate } from "./template.ts";
|
||||
|
||||
type Sel = ReturnType<CheerioAPI>;
|
||||
@@ -65,8 +76,18 @@ export interface SpiderTorrent {
|
||||
hit_and_run?: boolean;
|
||||
category?: MpMediaType;
|
||||
category_id?: string;
|
||||
// ---- 字幕分支(search_type=subtitles)专有字段,键名与 MP 一致 ----
|
||||
language_icon?: string;
|
||||
report_url?: string;
|
||||
language?: string;
|
||||
uploader?: string;
|
||||
torrent_id?: string;
|
||||
subtitle_id?: string;
|
||||
file_name?: string;
|
||||
}
|
||||
|
||||
export type SpiderSearchType = "torrents" | "subtitles";
|
||||
|
||||
/** MP `select_media_categories` */
|
||||
export function selectMediaCategories(category: IndexerDefinition["category"], mtype?: MediaKind) {
|
||||
if (!category) return [];
|
||||
@@ -86,7 +107,15 @@ export function resolveCategoryMediaType(value: unknown, category: IndexerDefini
|
||||
return matches.length === 1 ? MEDIA_VALUE[matches[0]!] : "未知";
|
||||
}
|
||||
|
||||
/** MP `size_tools.parse_size`:二进制单位,round。 */
|
||||
/** Python 3 `round()`:恰好 .5 时取偶数(银行家舍入) */
|
||||
function roundHalfEven(v: number): number {
|
||||
const f = Math.floor(v);
|
||||
const diff = v - f;
|
||||
if (diff === 0.5) return f % 2 === 0 ? f : f + 1;
|
||||
return Math.round(v);
|
||||
}
|
||||
|
||||
/** MP `size_tools.parse_size`:二进制单位,round(银行家舍入)。 */
|
||||
export function parseSizeMp(text: string | number | null | undefined): number {
|
||||
if (!text) return 0;
|
||||
const s = String(text);
|
||||
@@ -101,7 +130,7 @@ export function parseSizeMp(text: string | number | null | undefined): number {
|
||||
else if (normalized.includes("GB") || normalized.includes("GIB")) v *= 1024 ** 3;
|
||||
else if (normalized.includes("MB") || normalized.includes("MIB")) v *= 1024 ** 2;
|
||||
else if (normalized.includes("KB") || normalized.includes("KIB")) v *= 1024;
|
||||
return Math.round(v);
|
||||
return roundHalfEven(v);
|
||||
}
|
||||
|
||||
/** strptime 结果(MP dateparse 过滤器返回 datetime) */
|
||||
@@ -193,6 +222,8 @@ export interface SiteSpiderQuery {
|
||||
cat?: string;
|
||||
/** 站点内部页码(MP 0 起) */
|
||||
page?: number;
|
||||
/** MP `search_type`:subtitles 时改用 `def.subtitles` 的配置 */
|
||||
searchType?: SpiderSearchType;
|
||||
}
|
||||
|
||||
export class SiteSpider {
|
||||
@@ -204,8 +235,13 @@ export class SiteSpider {
|
||||
readonly domain: string;
|
||||
readonly resultNum: number;
|
||||
readonly siteMediaType: MediaKind | undefined;
|
||||
readonly searchType: SpiderSearchType;
|
||||
private list: SpiderListConfig;
|
||||
private fields: SpiderFields;
|
||||
private searchCfg: IndexerDefinition["search"];
|
||||
private batchCfg: IndexerDefinition["batch"];
|
||||
private browseCfg: IndexerDefinition["browse"];
|
||||
private categoryCfg: SpiderCategoryConfig | undefined;
|
||||
private requestedResultMediaType: MediaKind | undefined;
|
||||
private $!: CheerioAPI;
|
||||
private info: SpiderTorrent = {};
|
||||
@@ -217,26 +253,42 @@ export class SiteSpider {
|
||||
this.cat = q.cat;
|
||||
this.page = q.page ?? 0;
|
||||
this.siteMediaType = fromAgent(def.media_type);
|
||||
this.list = def.torrents.list;
|
||||
this.fields = def.torrents.fields ?? {};
|
||||
if (!q.keyword && def.browse) {
|
||||
this.list = def.browse.list ?? this.list;
|
||||
this.fields = def.browse.fields ?? this.fields;
|
||||
this.searchType = q.searchType ?? "torrents";
|
||||
if (this.searchType === "subtitles") {
|
||||
const sc = def.subtitles ?? {};
|
||||
this.searchCfg = sc.search;
|
||||
this.batchCfg = sc.batch;
|
||||
this.browseCfg = sc.browse;
|
||||
this.categoryCfg = sc.category;
|
||||
this.list = sc.list ?? { selector: "" };
|
||||
this.fields = sc.fields ?? {};
|
||||
this.resultNum = Number(sc.result_num ?? def.result_num ?? 100);
|
||||
} else {
|
||||
this.searchCfg = def.search;
|
||||
this.batchCfg = def.batch;
|
||||
this.browseCfg = def.browse;
|
||||
this.categoryCfg = def.category;
|
||||
this.list = def.torrents.list;
|
||||
this.fields = def.torrents.fields ?? {};
|
||||
if (!q.keyword && def.browse) {
|
||||
this.list = def.browse.list ?? this.list;
|
||||
this.fields = def.browse.fields ?? this.fields;
|
||||
}
|
||||
this.resultNum = Number(def.result_num ?? 100);
|
||||
}
|
||||
this.domain = def.domain.endsWith("/") ? def.domain : `${def.domain}/`;
|
||||
this.resultNum = Number(def.result_num ?? 100);
|
||||
}
|
||||
|
||||
/** MP `__can_search` */
|
||||
canSearch(): boolean {
|
||||
if (this.siteMediaType && this.mtype && this.siteMediaType !== this.mtype) return false;
|
||||
return Boolean(this.def.search && this.domain);
|
||||
return Boolean(this.searchCfg && this.domain);
|
||||
}
|
||||
|
||||
/** MP `__format_search_word` */
|
||||
private formatSearchWord(word: string): string {
|
||||
if (word && /^tt\d+$/.test(word) && this.def.search?.imdbid_format) {
|
||||
return this.def.search.imdbid_format
|
||||
if (word && /^tt\d+$/.test(word) && this.searchCfg?.imdbid_format) {
|
||||
return this.searchCfg.imdbid_format
|
||||
.replace(/\{keyword\}/g, word)
|
||||
.replace(/\{imdbid\}/g, word)
|
||||
.replace(/\{imdbid_num\}/g, word.slice(2));
|
||||
@@ -246,7 +298,7 @@ export class SiteSpider {
|
||||
|
||||
/** MP `__get_search_url` */
|
||||
getSearchUrl(): string {
|
||||
const search = this.def.search ?? { paths: [] };
|
||||
const search = this.searchCfg ?? { paths: [] };
|
||||
const paths = search.paths ?? [];
|
||||
let torrentspath = "";
|
||||
let typedPathSelected = false;
|
||||
@@ -272,9 +324,9 @@ export class SiteSpider {
|
||||
let searchWord: string;
|
||||
let searchMode: string;
|
||||
if (Array.isArray(this.keyword)) {
|
||||
if (this.def.batch) {
|
||||
const delimiter = this.def.batch.delimiter || " ";
|
||||
const spaceReplace = this.def.batch.space_replace || " ";
|
||||
if (this.batchCfg) {
|
||||
const delimiter = this.batchCfg.delimiter || " ";
|
||||
const spaceReplace = this.batchCfg.space_replace || " ";
|
||||
searchWord = this.keyword.map((k) => String(k).replace(/ /g, spaceReplace)).join(delimiter);
|
||||
} else {
|
||||
searchWord = this.keyword.join(" ");
|
||||
@@ -296,7 +348,7 @@ export class SiteSpider {
|
||||
notnewword: 1,
|
||||
};
|
||||
for (const [k, v] of Object.entries(indexerParams)) params[k] = String(v).replace(/\{keyword\}/g, searchWord);
|
||||
const category = this.def.category;
|
||||
const category = this.categoryCfg;
|
||||
if (category) {
|
||||
const cats = selectMediaCategories(category, this.mtype);
|
||||
const allowed = this.cat ? new Set(this.cat.split(",")) : null;
|
||||
@@ -333,9 +385,9 @@ export class SiteSpider {
|
||||
}
|
||||
} else {
|
||||
let page = this.page || 0;
|
||||
if (this.def.browse && !typedPathSelected) {
|
||||
torrentspath = this.def.browse.path;
|
||||
if (this.def.browse.start) page = Number(this.def.browse.start) + Number(this.page || 0);
|
||||
if (this.browseCfg && !typedPathSelected) {
|
||||
torrentspath = this.browseCfg.path;
|
||||
if (this.browseCfg.start) page = Number(this.browseCfg.start) + Number(this.page || 0);
|
||||
} else if (this.page && !torrentspath.includes("{page}")) {
|
||||
torrentspath += `${torrentspath.includes("?") ? "&" : "?"}page=${this.page}`;
|
||||
}
|
||||
@@ -369,7 +421,7 @@ export class SiteSpider {
|
||||
const rows = $(normalizeSelector(this.list.selector ?? ""));
|
||||
rows.each((i, el) => {
|
||||
if (i >= this.resultNum) return false;
|
||||
const info = this.getInfo($(el));
|
||||
const info = this.searchType === "subtitles" ? this.getSubtitleInfo($(el)) : this.getInfo($(el));
|
||||
if (Object.keys(info).length > 0) out.push(this.applyRequestedMediaType(info));
|
||||
return undefined;
|
||||
});
|
||||
@@ -414,6 +466,70 @@ export class SiteSpider {
|
||||
}
|
||||
}
|
||||
|
||||
/** MP `get_subtitle_info`:缺标题或下载链接的行丢弃;任一字段异常整条返回 {}。 */
|
||||
getSubtitleInfo(row: Sel): SpiderTorrent {
|
||||
this.info = {};
|
||||
try {
|
||||
this.getTitle(row);
|
||||
this.getDescription(row);
|
||||
this.getDetail(row);
|
||||
this.getDownload(row);
|
||||
this.getSize(row);
|
||||
this.getPubdate(row);
|
||||
this.getDateElapsed(row);
|
||||
this.getGrabs(row);
|
||||
this.getLinkField(row, "language_icon", "language_icon");
|
||||
this.getLinkField(row, "report", "report_url");
|
||||
for (const name of ["language", "uploader", "torrent_id", "subtitle_id", "file_name"] as const) {
|
||||
this.getSubtitleField(row, name);
|
||||
}
|
||||
this.fillSubtitleIds();
|
||||
if (!this.info.title || !this.info.enclosure) return {};
|
||||
return { ...this.info };
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/** MP `__get_language_icon` / `__get_report_url`:相对链接补全为绝对链接 */
|
||||
private getLinkField(row: Sel, name: string, key: "language_icon" | "report_url") {
|
||||
if (!(name in this.fields)) return;
|
||||
const selector = this.fields[name] ?? {};
|
||||
const link = this.filterText(this.safeQuery(row, selector), selector.filters);
|
||||
if (link) this.info[key] = this.normalizeLink(String(link));
|
||||
}
|
||||
|
||||
/** MP `__get_subtitle_field` */
|
||||
private getSubtitleField(row: Sel, name: "language" | "uploader" | "torrent_id" | "subtitle_id" | "file_name") {
|
||||
const selector = this.fields[name];
|
||||
if (!selector || Object.keys(selector).length === 0) return;
|
||||
const v = this.filterText(this.safeQuery(row, selector), selector.filters);
|
||||
if (v != null) this.info[name] = v instanceof ParsedDateTime ? v.text : String(v);
|
||||
}
|
||||
|
||||
/** MP `__fill_subtitle_ids`:从下载链接 query 补种子 ID(torrentid/torrent_id)与字幕 ID(subid/subtitle) */
|
||||
private fillSubtitleIds() {
|
||||
const enclosure = this.info.enclosure;
|
||||
if (!enclosure) return;
|
||||
const q = enclosure.includes("?") ? enclosure.slice(enclosure.indexOf("?") + 1).split("#")[0]! : "";
|
||||
const params = new URLSearchParams(q);
|
||||
const first = (...keys: string[]) => {
|
||||
for (const k of keys) {
|
||||
const v = params.getAll(k).find((x) => x !== "");
|
||||
if (v !== undefined) return v;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
if (!this.info.torrent_id) {
|
||||
const v = first("torrentid", "torrent_id");
|
||||
if (v !== undefined) this.info.torrent_id = v;
|
||||
}
|
||||
if (!this.info.subtitle_id) {
|
||||
const v = first("subid", "subtitle");
|
||||
if (v !== undefined) this.info.subtitle_id = v;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 字段 ----
|
||||
|
||||
private getTitle(row: Sel) {
|
||||
@@ -532,7 +648,7 @@ export class SiteSpider {
|
||||
this.info[key] = 1;
|
||||
if (selector.case) {
|
||||
for (const [css, value] of Object.entries(selector.case)) {
|
||||
if (row.find(normalizeSelector(css)).length > 0) {
|
||||
if (this.rowScope(row, css).length > 0) {
|
||||
this.info[key] = value;
|
||||
break;
|
||||
}
|
||||
@@ -606,7 +722,7 @@ export class SiteSpider {
|
||||
this.info.labels = [];
|
||||
return;
|
||||
}
|
||||
const q = row.find(normalizeSelector(selector.selector)).clone();
|
||||
const q = this.rowScope(row, selector.selector).clone();
|
||||
this.remove(q, selector);
|
||||
this.info.labels = this.attributeOrText(q, selector).filter((x): x is string => Boolean(x));
|
||||
}
|
||||
@@ -614,7 +730,7 @@ export class SiteSpider {
|
||||
private getHitAndRun(row: Sel) {
|
||||
if (!("hr" in this.fields)) return;
|
||||
const selector = this.fields.hr ?? {};
|
||||
this.info.hit_and_run = selector.selector ? row.find(normalizeSelector(selector.selector)).length > 0 : false;
|
||||
this.info.hit_and_run = selector.selector ? this.rowScope(row, selector.selector).length > 0 : false;
|
||||
}
|
||||
|
||||
private getCategory(row: Sel) {
|
||||
@@ -629,13 +745,21 @@ export class SiteSpider {
|
||||
const selector = this.fields.category ?? {};
|
||||
const value = this.filterText(this.safeQuery(row, selector), selector.filters);
|
||||
if (value != null && value !== "") this.info.category_id = String(value);
|
||||
let resolved = resolveCategoryMediaType(value, this.def.category);
|
||||
let resolved = resolveCategoryMediaType(value, this.categoryCfg);
|
||||
if (resolved === "未知" && this.siteMediaType) resolved = MEDIA_VALUE[this.siteMediaType];
|
||||
this.info.category = resolved;
|
||||
}
|
||||
|
||||
// ---- 通用查询 ----
|
||||
|
||||
/** 行级标记查询:行自身 + 后代(PyQuery descendant-or-self,见文件头差异 4);`:scope` 选择器只查后代 */
|
||||
private rowScope(row: Sel, selector: string): Sel {
|
||||
const css = normalizeSelector(selector);
|
||||
const desc = row.find(css);
|
||||
if (css.includes(":scope")) return desc;
|
||||
return row.filter(css).add(desc);
|
||||
}
|
||||
|
||||
/** MP `_safe_query` */
|
||||
safeQuery(row: Sel, config: SpiderFieldSelector | undefined): string | null | undefined {
|
||||
if (!config?.selector) return null;
|
||||
|
||||
Reference in new issue
Block a user