feat(adapters): port MoviePilot open site framework (7 spiders, 20 user parsers) + MovieClaw v0.27.0 site definitions + mp-sync tools (W1-3c)
- sites/spiders: Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema (+mTorrent wrapper), specialized SiteAdapter, indirect download resolved at download time - sites/user-parsers: SiteParserBase + all 20 MP SiteSchema parsers (lxml XPath via xpath+xmldom), refreshSiteUserData with schema fallback - sites/definitions: zod SiteDefinition schemas, 23 MovieClaw v0.27.0 (MIT) definitions via scripts/mp-sync/movieclaw-import.mjs, registry getSiteDefinition/listSiteDefinitions, NOTICE - scripts/mp-sync: ledger.json (upstream sha256), check.mjs, build-ledger.mjs; docs/porting/mp-sync-ledger.md - license-and-provenance: add copied-mit kind, re-anchor MovieClaw rows to v0.27.0 Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
cf85481d95
commit
d9f2af2f3c
166 files changed
+16577
-186
No files matched your search
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/gazelle.py @ e6820a8
|
||||
*
|
||||
* Gazelle 站点用户数据解析器(MP `GazelleSiteUserInfo`)。
|
||||
* 函数对照:`_parse_logged_in` → `_parseLoggedIn`、`_parse_user_base_info` → `_parseUserBaseInfo`、
|
||||
* `_parse_site_page` → `_parseSitePage`(空)、`_parse_user_detail_info` → `_parseUserDetailInfo`、
|
||||
* `_parse_user_torrent_seeding_info` → `_parseUserTorrentSeedingInfo`、`_parse_user_traffic_info`(空)、
|
||||
* `_parse_message_unread_links` / `_parse_message_content`(不支持)。XPath 与正则逐字保留。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase } from "./base.ts";
|
||||
import { normalizeDatetime, parseFloatMp, parseIntMp, reSearch } from "./common.ts";
|
||||
import { hasChildElements, parseHtml } from "./dom.ts";
|
||||
import { round3 } from "./nexus-php.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
export class GazelleSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.Gazelle;
|
||||
|
||||
/** 兼容不展示退出入口的站点首页 */
|
||||
protected override _parseLoggedIn(htmlText: unknown): boolean | Promise<boolean> {
|
||||
const html = parseHtml(String(htmlText ?? ""));
|
||||
if (hasChildElements(html) && html.nodes('//a[contains(@href, "user.php?id=")]').length) return true;
|
||||
return super._parseLoggedIn(htmlText);
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
const html = parseHtml(this._prepareHtmlText(htmlText));
|
||||
if (!html) return;
|
||||
let tmps = html.nodes('//a[contains(@href, "user.php?id=") or contains(@href, "user?id=")]');
|
||||
if (tmps.length) {
|
||||
const m = reSearch(/user(?:\.php)?\?id=(\d+)/, tmps[0]!.attr("href"));
|
||||
if (m?.[0].trim()) {
|
||||
this.userid = m[1]!;
|
||||
this._torrentSeedingPage = `torrents.php?type=seeding&userid=${this.userid}`;
|
||||
this._userDetailPage = `user.php?id=${this.userid}`;
|
||||
// lxml `el.text`:首个子节点为文本时取之
|
||||
const first = tmps[0]!.node.firstChild;
|
||||
this.username = first && first.nodeType === 3 ? (first.nodeValue ?? "").trim() : null;
|
||||
}
|
||||
}
|
||||
|
||||
let vals = html.strings('//*[@id="header-uploaded-value"]/@data-value');
|
||||
if (vals.length) this.upload = this.numFilesize(vals[0]);
|
||||
else {
|
||||
vals = html.strings('//li[@id="stats_seeding"]/span/text()');
|
||||
if (vals.length) this.upload = this.numFilesize(vals[0]);
|
||||
}
|
||||
vals = html.strings('//*[@id="header-downloaded-value"]/@data-value');
|
||||
if (vals.length) this.download = this.numFilesize(vals[0]);
|
||||
else {
|
||||
vals = html.strings('//li[@id="stats_leeching"]/span/text()');
|
||||
if (vals.length) this.download = this.numFilesize(vals[0]);
|
||||
}
|
||||
this.ratio = this.download <= 0.0 ? 0.0 : round3(this.upload / this.download);
|
||||
|
||||
vals = html.strings('//a[contains(@href, "bonus")]/@data-tooltip');
|
||||
if (vals.length) {
|
||||
const m = reSearch(/([\d,.]+)/, vals[0]);
|
||||
if (m?.[1]?.trim()) this.bonus = parseFloatMp(m[1]);
|
||||
} else {
|
||||
tmps = html.nodes('//a[contains(@href, "bonus")]');
|
||||
if (tmps.length) {
|
||||
const m = reSearch(/([\d,.]+)/, tmps[0]!.text());
|
||||
if (m?.[1]?.trim()) this.bonus = parseFloatMp(m[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected _parseSitePage(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserDetailInfo(htmlText: string): void {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
// 用户等级
|
||||
let levels = html.strings('//*[@id="class-value"]/@data-value');
|
||||
if (levels.length) this.userLevel = levels[0]!.trim();
|
||||
else {
|
||||
levels = html.strings('//li[contains(text(), "用户等级")]/text()');
|
||||
if (levels.length) {
|
||||
const part = levels[0]!.split(":")[1];
|
||||
if (part === undefined) throw new Error("IndexError: list index out of range");
|
||||
this.userLevel = part.trim();
|
||||
}
|
||||
}
|
||||
// 加入日期
|
||||
let joinAt = html.strings('//*[@id="join-date-value"]/@data-value');
|
||||
if (joinAt.length) this.joinAt = normalizeDatetime(joinAt[0]!.trim()) ?? null;
|
||||
else {
|
||||
joinAt = html.strings(
|
||||
'//div[contains(@class, "box_userinfo_stats")]//li[contains(text(), "加入时间")]/span/text()',
|
||||
);
|
||||
if (joinAt.length) this.joinAt = normalizeDatetime(joinAt[0]!.trim()) ?? null;
|
||||
}
|
||||
// 兼容以文本形式展示上传/下载的站点(如 JPopsuki)
|
||||
if (!this.upload) {
|
||||
const up = html.nodes('//li[starts-with(normalize-space(text()), "Uploaded:")]');
|
||||
if (up.length) {
|
||||
const m = reSearch(/([\d.,]+\s*[GMKT]?i?B)/i, up[0]!.text());
|
||||
if (m) this.upload = this.numFilesize(m[1]);
|
||||
}
|
||||
}
|
||||
if (!this.download) {
|
||||
const down = html.nodes('//li[starts-with(normalize-space(text()), "Downloaded:")]');
|
||||
if (down.length) {
|
||||
const m = reSearch(/([\d.,]+\s*[GMKT]?i?B)/i, down[0]!.text());
|
||||
if (m) this.download = this.numFilesize(m[1]);
|
||||
}
|
||||
}
|
||||
if (!this.ratio && this.upload && this.download) this.ratio = round3(this.upload / this.download);
|
||||
}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(htmlText: string, multiPage = false): NextPage {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return null;
|
||||
let sizeCol = 3;
|
||||
const headCells = html.nodes('//table[contains(@id, "torrent")]//tr[1]/td');
|
||||
if (headCells.length) sizeCol = headCells.length - 3;
|
||||
const seedersCol = sizeCol + 2;
|
||||
|
||||
let pageSeeding = 0;
|
||||
let pageSeedingSize = 0;
|
||||
const pageSeedingInfo: Array<[number, number]> = [];
|
||||
const semanticRows = html.nodes(
|
||||
'//table[contains(@id, "torrent")]//tr[contains(concat(" ", normalize-space(@class), " "), " TableTorrent-rowTitle ")]',
|
||||
);
|
||||
if (semanticRows.length) {
|
||||
// GPW 等现代 Gazelle 页面以语义 class 标记列,逐行按 class 配对读取
|
||||
for (const row of semanticRows) {
|
||||
const sizeCells = row.nodes(
|
||||
'./td[contains(concat(" ", normalize-space(@class), " "), " TableTorrent-cellStatSize ")]',
|
||||
);
|
||||
const seederCells = row.nodes(
|
||||
'./td[contains(concat(" ", normalize-space(@class), " "), " TableTorrent-cellStatSeeders ")]',
|
||||
);
|
||||
if (!sizeCells.length || !seederCells.length) continue;
|
||||
const size = this.numFilesize(sizeCells[0]!.text().trim());
|
||||
const seeders = parseIntMp(seederCells[0]!.text().trim());
|
||||
pageSeeding += 1;
|
||||
pageSeedingSize += size;
|
||||
pageSeedingInfo.push([seeders, size]);
|
||||
}
|
||||
} else {
|
||||
const sizes = html.nodes(`//table[contains(@id, "torrent")]//tr[position()>1]/td[${sizeCol}]`);
|
||||
const seeders = html.strings(`//table[contains(@id, "torrent")]//tr[position()>1]/td[${seedersCol}]/text()`);
|
||||
if (sizes.length && seeders.length) {
|
||||
pageSeeding = sizes.length;
|
||||
for (let i = 0; i < sizes.length; i++) {
|
||||
const size = this.numFilesize(sizes[i]!.text().trim());
|
||||
pageSeedingSize += size;
|
||||
pageSeedingInfo.push([parseIntMp(seeders[i]), size]);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (multiPage) {
|
||||
this.seeding += pageSeeding;
|
||||
this.seedingSize += pageSeedingSize;
|
||||
this.seedingInfo.push(...pageSeedingInfo);
|
||||
} else {
|
||||
if (!this.seeding) this.seeding = pageSeeding;
|
||||
if (!this.seedingSize) this.seedingSize = pageSeedingSize;
|
||||
if (!this.seedingInfo.length) this.seedingInfo = pageSeedingInfo;
|
||||
}
|
||||
const next = html.strings(
|
||||
'//a[contains(.//text(), "Next") or contains(.//text(), "下一页") or contains(@title, "下一页") or contains(@title, "Next")]/@href',
|
||||
);
|
||||
return next.length ? next[next.length - 1]!.trim() : null;
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user