feat(parser,sites): W5-D port title/zhconv/site parsing gaps found by MP differential

Title recognition (parseRelease / parseReleasePath):
- anime prefix tags (★N月新番, [国漫], 【22年日剧】), category lead bracket, `A / B` cn aliases
- `Name [01]`, `[TV 04]`, `#13`, `名称 12.mkv`, `S01 13-14`, `E13-14` episodes
- lead bracket as title vs group, trailing [Group], YTS.MX/Hall_of_C style groups, 字幕组 suffix
- `[dot.release.name][…]` unbracketing, `Season 2017` as year, leading date not airDate,
  date-only parent dir does not merge as TV
- original disc: Blu-ray + AVC/HEVC/VC-1/MPEG-2 without x26x/H.26x -> BluRay-Disc
- `{[tmdbid=;type=;s=;e=]}` explicit block, bangumi/anilist ids
- bare [1080]/[5.1], DTS-HD MA5.1, £ multi-group, 合集, pinyin title -> cn from subtitle,
  streaming platform table, CHS/CHT/GB/BIG5 in subtitle
zh-map: add 迴滾魷瑯臥駭; document one-to-many chars kept out.
Sites: tbody branch + header-row tightening via movieclaw-import rowSelector (18 definitions),
row-level case/hr/labels include the row itself, subtitle branch (searchType: "subtitles"),
parseSizeMp banker's rounding.
Differential snapshots regenerated; MP test sample sources registered in ledger.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-27 23:09:14 +08:00
1 parent 8352e5e881
commit 36dc048e89
41 files changed
+3111 -2372

No files matched your search

@@ -78,7 +78,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr:has(table.torrentname)"
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
},
"fields": {
"title_default": {
@@ -71,7 +71,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr:has(table.torrentname)"
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
},
"fields": {
"title_default": {
@@ -102,7 +102,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr, table.torrents > tbody > tr"
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
},
"fields": {
"title_default": {
@@ -66,7 +66,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr, table.torrents > tbody > tr"
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
},
"fields": {
"title_default": {
@@ -70,7 +70,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr, table.torrents > tbody > tr"
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
},
"fields": {
"title_default": {
@@ -206,7 +206,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr:has(table.torrentname)"
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
},
"fields": {
"title_default": {
@@ -70,7 +70,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr, table.torrents > tbody > tr"
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
},
"fields": {
"title_default": {
@@ -70,7 +70,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents tr"
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
},
"fields": {
"title_default": {
@@ -78,7 +78,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr:has(table.torrentname)"
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
},
"fields": {
"title_default": {
@@ -43,7 +43,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr, table.torrents > tbody > tr"
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
},
"fields": {
"title_default": {
@@ -83,7 +83,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr:has(table.torrentname)"
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
},
"fields": {
"title_default": {
@@ -66,7 +66,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr, table.torrents > tbody > tr"
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
},
"fields": {
"title_default": {
@@ -66,7 +66,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr:has(table.torrentname)"
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
},
"fields": {
"title_default": {
@@ -66,7 +66,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr:has(table.torrentname)"
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
},
"fields": {
"title_default": {
@@ -62,7 +62,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr, table.torrents > tbody > tr"
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
},
"fields": {
"title_default": {
@@ -70,7 +70,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr, table.torrents > tbody > tr"
"selector": "table.torrents > tr:has(td.rowfollow), table.torrents > tbody > tr:has(td.rowfollow)"
},
"fields": {
"title_default": {
@@ -70,7 +70,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr:has(table.torrentname)"
"selector": "table.torrents > tr:has(table.torrentname), table.torrents > tbody > tr:has(table.torrentname)"
},
"fields": {
"title_default": {
@@ -43,7 +43,7 @@
},
"torrents": {
"list": {
"selector": "table.torrents > tr:not(:first-child)"
"selector": "table.torrents > tr:not(:first-child), table.torrents > tbody > tr:not(:first-child)"
},
"fields": {
"title_default": {
@@ -63,6 +63,17 @@ export interface SpiderFields {
[name: string]: SpiderFieldSelector | undefined;
}
/** MP `indexer.subtitles`:字幕搜索页的独立配置(字段额外支持 language_icon/report/language/uploader/torrent_id/subtitle_id/file_name) */
export interface SpiderSubtitleConfig {
search?: IndexerDefinition["search"];
batch?: IndexerDefinition["batch"];
browse?: IndexerDefinition["browse"];
category?: SpiderCategoryConfig;
list?: SpiderListConfig;
fields?: SpiderFields;
result_num?: number;
}
/** 定义来源(MovieClaw v0.27.0 MIT 转换 / 手写) */
export interface DefinitionProvenance {
source: "movieclaw" | "handwritten";
@@ -104,6 +115,8 @@ export interface IndexerDefinition {
browse?: { path: string; start?: number; list?: SpiderListConfig; fields?: SpiderFields };
category?: SpiderCategoryConfig;
torrents: { list: SpiderListConfig; fields: SpiderFields };
/** 字幕页配置(MP `indexer.subtitles`,search_type=subtitles 时替换 search/batch/browse/category/list/fields) */
subtitles?: SpiderSubtitleConfig;
/** 本项目扩展(MP 无对应) */
umt: UmtSiteExtension;
}
@@ -1,10 +1,18 @@
import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit";
import {
CHDBITS_LIST_PAGE,
createMockNexusPhpSite,
type MockNexusPhpSite,
NEXUSPHP_SUBTITLES_CONFIG,
NEXUSPHP_SUBTITLES_PAGE,
} from "@umt/testkit";
import { afterEach, describe, expect, it } from "vitest";
import {
audiencesDefinition,
BUILTIN_SITE_DEFINITIONS,
createSiteRateLimiter,
createSpiderSiteAdapter,
type IndexerDefinition,
parseSizeMp,
type SiteAdapter,
SiteError,
SiteSpider,
@@ -306,3 +314,82 @@ describe("观众适配器(spider + mock NexusPHP)", () => {
expect(limiter.stats.granted).toBe(4);
});
});
describe("SiteSpider 差分补迁(W5-D)", () => {
const withTbody = (html: string) =>
html.replace(/(<table class="torrents"[^>]*>)([\s\S]*?)(<\/table>\s*<\/body>)/, "$1<tbody>$2</tbody>$3");
it("列表选择器:显式 <tbody> 的页面也能取到行,表头行不再产出空标题", () => {
const chd = BUILTIN_SITE_DEFINITIONS.chdbits as IndexerDefinition;
const plain = new SiteSpider(chd, { keyword: "x" }).parse(CHDBITS_LIST_PAGE).torrents;
const tbody = new SiteSpider(chd, { keyword: "x" }).parse(withTbody(CHDBITS_LIST_PAGE)).torrents;
expect(plain.length).toBeGreaterThan(0);
expect(tbody.map((t) => t.title)).toEqual(plain.map((t) => t.title));
const hdarea = BUILTIN_SITE_DEFINITIONS.hdarea as IndexerDefinition;
const rows = new SiteSpider(hdarea, { keyword: "x" }).parse(CHDBITS_LIST_PAGE).torrents;
expect(rows.length).toBe(plain.length);
expect(rows.every((t) => Boolean(t.title))).toBe(true);
});
it("行级标记 case/hr/labels 包含行自身(PyQuery descendant-or-self),取值字段只查后代", () => {
const def: IndexerDefinition = {
id: "self",
name: "self",
domain: "https://self.example/",
torrents: {
list: { selector: "table.t > tr" },
fields: {
title: { selector: "a.t" },
downloadvolumefactor: { case: { ".free_bg": 0, "*": 1 } },
hr: { selector: ".hr_row" },
labels: { selector: ".sticky", attribute: "class" },
},
},
umt: {} as IndexerDefinition["umt"],
};
const html =
'<table class="t"><tr class="free_bg hr_row sticky"><td><a class="t">A</a></td></tr><tr><td><a class="t">B</a></td></tr></table>';
const [a, b] = new SiteSpider(def, { keyword: "x" }).parse(html).torrents;
expect(a).toMatchObject({
title: "A",
downloadvolumefactor: 0,
hit_and_run: true,
labels: ["free_bg hr_row sticky"],
});
expect(b).toMatchObject({ title: "B", downloadvolumefactor: 1, hit_and_run: false, labels: [] });
});
it("parseSizeMp 与 Python round 一致(银行家舍入)", () => {
expect(parseSizeMp("2.5 B")).toBe(2);
expect(parseSizeMp("3.5 B")).toBe(4);
expect(parseSizeMp("1.5 KB")).toBe(1536);
});
it("字幕分支:get_subtitle_info + __fill_subtitle_ids,缺下载链接的行丢弃", () => {
const def: IndexerDefinition = {
id: "subs",
name: "subs",
domain: "https://hhanclub.net/",
torrents: { list: { selector: "none" }, fields: {} },
subtitles: NEXUSPHP_SUBTITLES_CONFIG,
umt: {} as IndexerDefinition["umt"],
};
const spider = new SiteSpider(def, { keyword: "Example Show", searchType: "subtitles" });
const { isError, torrents } = spider.parse(NEXUSPHP_SUBTITLES_PAGE);
expect(isError).toBe(false);
expect(torrents).toHaveLength(3);
expect(torrents[0]).toEqual({
title: "Example Show S01E01 1080p WEB-DL CHS",
enclosure: "https://hhanclub.net/downloadsubs.php?torrentid=0&subid=1000",
size: 114678,
pubdate: "2026-04-21 20:54:37",
date_elapsed: "1月18天",
language_icon: "https://hhanclub.net/pic/flag/china.gif",
report_url: "https://hhanclub.net/report.php?subtitle=1000",
uploader: "tester0",
torrent_id: "0",
subtitle_id: "1000",
});
expect(torrents[2]).toMatchObject({ torrent_id: "2", subtitle_id: "1002", size: 114678 });
});
});
+151 -27
View File
@@ -10,16 +10,27 @@
* → `SiteSpider.getInfo` 与同名私有方法
* - `_safe_query / __filter_text / __remove / __attribute_or_text / __index` → 同名方法
* - `__is_login_or_permission_page` → `isLoginOrPermissionPage`
* - `parse` → `SiteSpider.parse`(不含 Rust 加速分支与字幕分支)
* - `app/foundation/size.py parse_size` → `parseSizeMp`
* - `parse` → `SiteSpider.parse`(不含 Rust 加速分支);`search_type=subtitles` 分支:`get_subtitle_info /
* __get_language_icon / __get_report_url / __get_subtitle_field / __fill_subtitle_ids` → `getSubtitleInfo` 等同名方法
* - `app/foundation/size.py parse_size` → `parseSizeMp`(Python round 为银行家舍入,同样实现)
*
* 与 MP 的差异(均在 README 与迁移账本登记):
* 1. 额外输出 `category_id`(原始分类 ID),`category` 仍是 MP 的媒体类型投影(REQ §4.5 原始分类不得丢失)。
* 2. 系数 `selector` 模式用浮点而非 int()(MP 对 0.5 抛异常导致整条丢失,迁移账本 SA4)。
* 3. 不发请求:HTTP/Cookie/UA 由 adapter 注入,本类只负责 URL 构造与 HTML 解析(便于限流与脱敏统一)。
* 4. 查询范围:PyQuery 在行元素上查询是 descendant-or-self(行自身也会命中),MP Rust 快路径与 cheerio `.find` 只查后代。
* 本项目取折中:促销 case / hr / labels 这三类"行级标记"(NexusPHP 把 free_bg、sticky 等 class 挂在 <tr> 上)
* 包含行自身,与 Python 一致;取值字段(safeQuery)只查后代,避免 `div:nth-child(3)` 这类选择器命中行本身取到整行文本。
*/
import { type CheerioAPI, load } from "cheerio";
import type { IndexerDefinition, SpiderFieldSelector, SpiderFields, SpiderFilter, SpiderListConfig } from "./schema.ts";
import type {
IndexerDefinition,
SpiderCategoryConfig,
SpiderFieldSelector,
SpiderFields,
SpiderFilter,
SpiderListConfig,
} from "./schema.ts";
import { renderTemplate } from "./template.ts";
type Sel = ReturnType<CheerioAPI>;
@@ -65,8 +76,18 @@ export interface SpiderTorrent {
hit_and_run?: boolean;
category?: MpMediaType;
category_id?: string;
// ---- 字幕分支(search_type=subtitles)专有字段,键名与 MP 一致 ----
language_icon?: string;
report_url?: string;
language?: string;
uploader?: string;
torrent_id?: string;
subtitle_id?: string;
file_name?: string;
}
export type SpiderSearchType = "torrents" | "subtitles";
/** MP `select_media_categories` */
export function selectMediaCategories(category: IndexerDefinition["category"], mtype?: MediaKind) {
if (!category) return [];
@@ -86,7 +107,15 @@ export function resolveCategoryMediaType(value: unknown, category: IndexerDefini
return matches.length === 1 ? MEDIA_VALUE[matches[0]!] : "未知";
}
/** MP `size_tools.parse_size`:二进制单位,round。 */
/** Python 3 `round()`:恰好 .5 时取偶数(银行家舍入) */
function roundHalfEven(v: number): number {
const f = Math.floor(v);
const diff = v - f;
if (diff === 0.5) return f % 2 === 0 ? f : f + 1;
return Math.round(v);
}
/** MP `size_tools.parse_size`:二进制单位,round(银行家舍入)。 */
export function parseSizeMp(text: string | number | null | undefined): number {
if (!text) return 0;
const s = String(text);
@@ -101,7 +130,7 @@ export function parseSizeMp(text: string | number | null | undefined): number {
else if (normalized.includes("GB") || normalized.includes("GIB")) v *= 1024 ** 3;
else if (normalized.includes("MB") || normalized.includes("MIB")) v *= 1024 ** 2;
else if (normalized.includes("KB") || normalized.includes("KIB")) v *= 1024;
return Math.round(v);
return roundHalfEven(v);
}
/** strptime 结果(MP dateparse 过滤器返回 datetime) */
@@ -193,6 +222,8 @@ export interface SiteSpiderQuery {
cat?: string;
/** 站点内部页码(MP 0 起) */
page?: number;
/** MP `search_type`:subtitles 时改用 `def.subtitles` 的配置 */
searchType?: SpiderSearchType;
}
export class SiteSpider {
@@ -204,8 +235,13 @@ export class SiteSpider {
readonly domain: string;
readonly resultNum: number;
readonly siteMediaType: MediaKind | undefined;
readonly searchType: SpiderSearchType;
private list: SpiderListConfig;
private fields: SpiderFields;
private searchCfg: IndexerDefinition["search"];
private batchCfg: IndexerDefinition["batch"];
private browseCfg: IndexerDefinition["browse"];
private categoryCfg: SpiderCategoryConfig | undefined;
private requestedResultMediaType: MediaKind | undefined;
private $!: CheerioAPI;
private info: SpiderTorrent = {};
@@ -217,26 +253,42 @@ export class SiteSpider {
this.cat = q.cat;
this.page = q.page ?? 0;
this.siteMediaType = fromAgent(def.media_type);
this.list = def.torrents.list;
this.fields = def.torrents.fields ?? {};
if (!q.keyword && def.browse) {
this.list = def.browse.list ?? this.list;
this.fields = def.browse.fields ?? this.fields;
this.searchType = q.searchType ?? "torrents";
if (this.searchType === "subtitles") {
const sc = def.subtitles ?? {};
this.searchCfg = sc.search;
this.batchCfg = sc.batch;
this.browseCfg = sc.browse;
this.categoryCfg = sc.category;
this.list = sc.list ?? { selector: "" };
this.fields = sc.fields ?? {};
this.resultNum = Number(sc.result_num ?? def.result_num ?? 100);
} else {
this.searchCfg = def.search;
this.batchCfg = def.batch;
this.browseCfg = def.browse;
this.categoryCfg = def.category;
this.list = def.torrents.list;
this.fields = def.torrents.fields ?? {};
if (!q.keyword && def.browse) {
this.list = def.browse.list ?? this.list;
this.fields = def.browse.fields ?? this.fields;
}
this.resultNum = Number(def.result_num ?? 100);
}
this.domain = def.domain.endsWith("/") ? def.domain : `${def.domain}/`;
this.resultNum = Number(def.result_num ?? 100);
}
/** MP `__can_search` */
canSearch(): boolean {
if (this.siteMediaType && this.mtype && this.siteMediaType !== this.mtype) return false;
return Boolean(this.def.search && this.domain);
return Boolean(this.searchCfg && this.domain);
}
/** MP `__format_search_word` */
private formatSearchWord(word: string): string {
if (word && /^tt\d+$/.test(word) && this.def.search?.imdbid_format) {
return this.def.search.imdbid_format
if (word && /^tt\d+$/.test(word) && this.searchCfg?.imdbid_format) {
return this.searchCfg.imdbid_format
.replace(/\{keyword\}/g, word)
.replace(/\{imdbid\}/g, word)
.replace(/\{imdbid_num\}/g, word.slice(2));
@@ -246,7 +298,7 @@ export class SiteSpider {
/** MP `__get_search_url` */
getSearchUrl(): string {
const search = this.def.search ?? { paths: [] };
const search = this.searchCfg ?? { paths: [] };
const paths = search.paths ?? [];
let torrentspath = "";
let typedPathSelected = false;
@@ -272,9 +324,9 @@ export class SiteSpider {
let searchWord: string;
let searchMode: string;
if (Array.isArray(this.keyword)) {
if (this.def.batch) {
const delimiter = this.def.batch.delimiter || " ";
const spaceReplace = this.def.batch.space_replace || " ";
if (this.batchCfg) {
const delimiter = this.batchCfg.delimiter || " ";
const spaceReplace = this.batchCfg.space_replace || " ";
searchWord = this.keyword.map((k) => String(k).replace(/ /g, spaceReplace)).join(delimiter);
} else {
searchWord = this.keyword.join(" ");
@@ -296,7 +348,7 @@ export class SiteSpider {
notnewword: 1,
};
for (const [k, v] of Object.entries(indexerParams)) params[k] = String(v).replace(/\{keyword\}/g, searchWord);
const category = this.def.category;
const category = this.categoryCfg;
if (category) {
const cats = selectMediaCategories(category, this.mtype);
const allowed = this.cat ? new Set(this.cat.split(",")) : null;
@@ -333,9 +385,9 @@ export class SiteSpider {
}
} else {
let page = this.page || 0;
if (this.def.browse && !typedPathSelected) {
torrentspath = this.def.browse.path;
if (this.def.browse.start) page = Number(this.def.browse.start) + Number(this.page || 0);
if (this.browseCfg && !typedPathSelected) {
torrentspath = this.browseCfg.path;
if (this.browseCfg.start) page = Number(this.browseCfg.start) + Number(this.page || 0);
} else if (this.page && !torrentspath.includes("{page}")) {
torrentspath += `${torrentspath.includes("?") ? "&" : "?"}page=${this.page}`;
}
@@ -369,7 +421,7 @@ export class SiteSpider {
const rows = $(normalizeSelector(this.list.selector ?? ""));
rows.each((i, el) => {
if (i >= this.resultNum) return false;
const info = this.getInfo($(el));
const info = this.searchType === "subtitles" ? this.getSubtitleInfo($(el)) : this.getInfo($(el));
if (Object.keys(info).length > 0) out.push(this.applyRequestedMediaType(info));
return undefined;
});
@@ -414,6 +466,70 @@ export class SiteSpider {
}
}
/** MP `get_subtitle_info`:缺标题或下载链接的行丢弃;任一字段异常整条返回 {}。 */
getSubtitleInfo(row: Sel): SpiderTorrent {
this.info = {};
try {
this.getTitle(row);
this.getDescription(row);
this.getDetail(row);
this.getDownload(row);
this.getSize(row);
this.getPubdate(row);
this.getDateElapsed(row);
this.getGrabs(row);
this.getLinkField(row, "language_icon", "language_icon");
this.getLinkField(row, "report", "report_url");
for (const name of ["language", "uploader", "torrent_id", "subtitle_id", "file_name"] as const) {
this.getSubtitleField(row, name);
}
this.fillSubtitleIds();
if (!this.info.title || !this.info.enclosure) return {};
return { ...this.info };
} catch {
return {};
}
}
/** MP `__get_language_icon` / `__get_report_url`:相对链接补全为绝对链接 */
private getLinkField(row: Sel, name: string, key: "language_icon" | "report_url") {
if (!(name in this.fields)) return;
const selector = this.fields[name] ?? {};
const link = this.filterText(this.safeQuery(row, selector), selector.filters);
if (link) this.info[key] = this.normalizeLink(String(link));
}
/** MP `__get_subtitle_field` */
private getSubtitleField(row: Sel, name: "language" | "uploader" | "torrent_id" | "subtitle_id" | "file_name") {
const selector = this.fields[name];
if (!selector || Object.keys(selector).length === 0) return;
const v = this.filterText(this.safeQuery(row, selector), selector.filters);
if (v != null) this.info[name] = v instanceof ParsedDateTime ? v.text : String(v);
}
/** MP `__fill_subtitle_ids`:从下载链接 query 补种子 ID(torrentid/torrent_id)与字幕 ID(subid/subtitle) */
private fillSubtitleIds() {
const enclosure = this.info.enclosure;
if (!enclosure) return;
const q = enclosure.includes("?") ? enclosure.slice(enclosure.indexOf("?") + 1).split("#")[0]! : "";
const params = new URLSearchParams(q);
const first = (...keys: string[]) => {
for (const k of keys) {
const v = params.getAll(k).find((x) => x !== "");
if (v !== undefined) return v;
}
return undefined;
};
if (!this.info.torrent_id) {
const v = first("torrentid", "torrent_id");
if (v !== undefined) this.info.torrent_id = v;
}
if (!this.info.subtitle_id) {
const v = first("subid", "subtitle");
if (v !== undefined) this.info.subtitle_id = v;
}
}
// ---- 字段 ----
private getTitle(row: Sel) {
@@ -532,7 +648,7 @@ export class SiteSpider {
this.info[key] = 1;
if (selector.case) {
for (const [css, value] of Object.entries(selector.case)) {
if (row.find(normalizeSelector(css)).length > 0) {
if (this.rowScope(row, css).length > 0) {
this.info[key] = value;
break;
}
@@ -606,7 +722,7 @@ export class SiteSpider {
this.info.labels = [];
return;
}
const q = row.find(normalizeSelector(selector.selector)).clone();
const q = this.rowScope(row, selector.selector).clone();
this.remove(q, selector);
this.info.labels = this.attributeOrText(q, selector).filter((x): x is string => Boolean(x));
}
@@ -614,7 +730,7 @@ export class SiteSpider {
private getHitAndRun(row: Sel) {
if (!("hr" in this.fields)) return;
const selector = this.fields.hr ?? {};
this.info.hit_and_run = selector.selector ? row.find(normalizeSelector(selector.selector)).length > 0 : false;
this.info.hit_and_run = selector.selector ? this.rowScope(row, selector.selector).length > 0 : false;
}
private getCategory(row: Sel) {
@@ -629,13 +745,21 @@ export class SiteSpider {
const selector = this.fields.category ?? {};
const value = this.filterText(this.safeQuery(row, selector), selector.filters);
if (value != null && value !== "") this.info.category_id = String(value);
let resolved = resolveCategoryMediaType(value, this.def.category);
let resolved = resolveCategoryMediaType(value, this.categoryCfg);
if (resolved === "未知" && this.siteMediaType) resolved = MEDIA_VALUE[this.siteMediaType];
this.info.category = resolved;
}
// ---- 通用查询 ----
/** 行级标记查询:行自身 + 后代(PyQuery descendant-or-self,见文件头差异 4);`:scope` 选择器只查后代 */
private rowScope(row: Sel, selector: string): Sel {
const css = normalizeSelector(selector);
const desc = row.find(css);
if (css.includes(":scope")) return desc;
return row.filter(css).add(desc);
}
/** MP `_safe_query` */
safeQuery(row: Sel, config: SpiderFieldSelector | undefined): string | null | undefined {
if (!config?.selector) return null;