feat(adapters): port MoviePilot open site framework (7 spiders, 20 user parsers) + MovieClaw v0.27.0 site definitions + mp-sync tools (W1-3c)
- sites/spiders: Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema (+mTorrent wrapper), specialized SiteAdapter, indirect download resolved at download time - sites/user-parsers: SiteParserBase + all 20 MP SiteSchema parsers (lxml XPath via xpath+xmldom), refreshSiteUserData with schema fallback - sites/definitions: zod SiteDefinition schemas, 23 MovieClaw v0.27.0 (MIT) definitions via scripts/mp-sync/movieclaw-import.mjs, registry getSiteDefinition/listSiteDefinitions, NOTICE - scripts/mp-sync: ledger.json (upstream sha256), check.mjs, build-ledger.mjs; docs/porting/mp-sync-ledger.md - license-and-provenance: add copied-mit kind, re-anchor MovieClaw rows to v0.27.0 Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
cf85481d95
commit
d9f2af2f3c
166 files changed
+16577
-186
No files matched your search
@@ -0,0 +1,448 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/__init__.py @ e6820a8
|
||||
*
|
||||
* 站点用户数据解析器基类。
|
||||
*
|
||||
* 迁移自 MoviePilot e6820a8(GPLv3)`app/modules/indexer/parser/__init__.py` `SiteParserBase`,
|
||||
* 函数一一对应(Python 私有 `_x_y` → TS `_xY`):
|
||||
* - `parse` → `parse`(页面获取顺序:首页/登录判定 → `_parse_site_page` → 基础信息 → 详情 → 未读消息 → 流量 → 做种分页)
|
||||
* - `_pase_unread_msgs` → `_paseUnreadMsgs`(保留 MP 拼写)、`_parse_seeding_pages` → `_parseSeedingPages`
|
||||
* - `_prepare_html_text` → `prepareHtmlText`、`num_filesize` → `numFilesize`
|
||||
* - `_get_page_content` → `_getPageContent`(cookie/apikey 两种请求模式、JSON Accept、200/500/403 视为有内容、Cloudflare 挑战返回空)
|
||||
* - `_parse_logged_in` → `_parseLoggedIn`、`clear` / `to_dict` → `clear` / `toDict`
|
||||
* - 抽象方法 `_parse_site_page/_parse_user_base_info/_parse_user_traffic_info/_parse_user_torrent_seeding_info/
|
||||
* _parse_user_detail_info/_parse_message_unread_links/_parse_message_content` → 同名 camelCase
|
||||
*
|
||||
* 与 MP 的差异(安全与健壮性):
|
||||
* 1. 凭据经构造参数注入,只放请求头;`toDict()`/`raw` 不含 cookie/apikey/token。
|
||||
* 2. 做种页与消息页翻页设上限(`MAX_PAGES`=50),防止站点返回自引用"下一页"导致死循环(MP 无上限)。
|
||||
* 3. 网络异常与非 200/500/403 状态:与 MP 一样返回空串继续,另把脱敏后的原因记在 `httpErrors`(入 raw)。
|
||||
* 4. 响应解码:按 Content-Type / `<meta charset>` 选 TextDecoder(MP `get_decoded_html_content` 用 chardet 探测)。
|
||||
*/
|
||||
import { redactText } from "../../rss/redact.ts";
|
||||
import { parseSizeMp } from "../spider/spider.ts";
|
||||
import { extractDomain, isLoggedIn, underChallenge } from "./common.ts";
|
||||
import type { SiteSchemaValue, SiteUnreadMessage, SiteUserInfo, SiteUserParserOptions } from "./types.ts";
|
||||
|
||||
export type Params = Record<string, unknown>;
|
||||
export type Headers_ = Record<string, string>;
|
||||
/** 做种解析返回:下一页地址;null/undefined/false 表示结束("" 表示同地址再取一页,MTorrent 用法) */
|
||||
export type NextPage = string | null | undefined | false;
|
||||
|
||||
const MAX_PAGES = 50;
|
||||
|
||||
/** Python `urljoin` */
|
||||
export function urljoin(base: string, path: string): string {
|
||||
try {
|
||||
return new URL(path, base).toString();
|
||||
} catch {
|
||||
return path;
|
||||
}
|
||||
}
|
||||
|
||||
function detectCharset(contentType: string | null, bytes: Uint8Array): string {
|
||||
const fromHeader = /charset=["']?([\w-]+)/i.exec(contentType ?? "")?.[1];
|
||||
if (fromHeader) return fromHeader.toLowerCase();
|
||||
const head = new TextDecoder("latin1").decode(bytes.subarray(0, 2048));
|
||||
const fromMeta = /<meta[^>]+charset=["']?([\w-]+)/i.exec(head)?.[1];
|
||||
return (fromMeta ?? "utf-8").toLowerCase();
|
||||
}
|
||||
|
||||
function decode(bytes: Uint8Array, charset: string): string {
|
||||
try {
|
||||
return new TextDecoder(charset).decode(bytes);
|
||||
} catch {
|
||||
return new TextDecoder("utf-8").decode(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
export abstract class SiteParserBase {
|
||||
/** 站点模版(MP `schema`) */
|
||||
static schema: SiteSchemaValue | null = null;
|
||||
/** 请求模式 cookie/apikey(MP `request_mode`) */
|
||||
requestMode: "cookie" | "apikey" = "cookie";
|
||||
|
||||
// 站点信息
|
||||
apikey: string | null;
|
||||
token: string | null;
|
||||
protected _siteName: string;
|
||||
protected _siteUrl: string;
|
||||
protected _siteDomain: string;
|
||||
protected _baseUrl: string;
|
||||
protected _siteCookie: string | null;
|
||||
protected _apiUrl: string | null;
|
||||
protected _ua: string | null;
|
||||
protected _indexHtml: string | Record<string, never> = "";
|
||||
protected readonly _opts: SiteUserParserOptions;
|
||||
|
||||
// 用户信息
|
||||
username: string | null = null;
|
||||
userid: string | null = null;
|
||||
userLevel: string | null = null;
|
||||
joinAt: string | null = null;
|
||||
bonus = 0.0;
|
||||
// 流量信息
|
||||
upload = 0;
|
||||
download = 0;
|
||||
ratio = 0;
|
||||
// 做种信息
|
||||
seeding = 0;
|
||||
leeching = 0;
|
||||
seedingSize = 0;
|
||||
leechingSize = 0;
|
||||
uploaded = 0;
|
||||
completed = 0;
|
||||
incomplete = 0;
|
||||
uploadedSize = 0;
|
||||
completedSize = 0;
|
||||
incompleteSize = 0;
|
||||
/** 做种人数, 种子大小 */
|
||||
seedingInfo: Array<[number, number]> = [];
|
||||
// 未读消息
|
||||
messageUnread = 0;
|
||||
messageUnreadContents: Array<[string | null, string | null, string | null]> = [];
|
||||
messageReadForce = false;
|
||||
// 全局附加请求头
|
||||
protected _additionHeaders: Headers_ | null = null;
|
||||
// 用户基础信息
|
||||
protected _userBasicPage: string | null = null;
|
||||
protected _userBasicParams: Params | null = null;
|
||||
protected _userBasicMethod: string | null = null;
|
||||
protected _userBasicHeaders: Headers_ | null = null;
|
||||
// 用户详情
|
||||
protected _userDetailPage: string | null = "userdetails.php?id=";
|
||||
protected _userDetailParams: Params | null = null;
|
||||
protected _userDetailHeaders: Headers_ | null = null;
|
||||
// 流量
|
||||
protected _userTrafficPage: string | null = "index.php";
|
||||
protected _userTrafficParams: Params | null = null;
|
||||
protected _userTrafficHeaders: Headers_ | null = null;
|
||||
// 未读消息
|
||||
protected _userMailUnreadPage: string | null = "messages.php?action=viewmailbox&box=1&unread=yes";
|
||||
protected _sysMailUnreadPage: string | null = "messages.php?action=viewmailbox&box=-2&unread=yes";
|
||||
protected _mailUnreadParams: Params | null = null;
|
||||
protected _mailUnreadHeaders: Headers_ | null = null;
|
||||
protected _mailContentParams: Params | null = null;
|
||||
protected _mailContentHeaders: Headers_ | null = null;
|
||||
// 做种
|
||||
protected _torrentSeedingPage: string | null = "getusertorrentlistajax.php?userid=";
|
||||
protected _torrentSeedingParams: Params | null = null;
|
||||
protected _torrentSeedingHeaders: Headers_ | null = null;
|
||||
/** 错误信息(MP `err_msg`) */
|
||||
errMsg: string | null = null;
|
||||
/** 本项目扩展:请求失败原因(已脱敏) */
|
||||
httpErrors: string[] = [];
|
||||
|
||||
constructor(opts: SiteUserParserOptions) {
|
||||
this._opts = opts;
|
||||
this.apikey = opts.apikey ?? null;
|
||||
this.token = opts.token ?? null;
|
||||
this._siteName = opts.siteName;
|
||||
this._siteUrl = opts.url;
|
||||
const u = new URL(opts.url);
|
||||
this._siteDomain = u.host;
|
||||
this._baseUrl = `${u.protocol}//${u.host}`;
|
||||
this._siteCookie = opts.cookie ?? null;
|
||||
this._apiUrl = opts.apiUrl ?? null;
|
||||
this._ua = opts.ua ?? null;
|
||||
}
|
||||
|
||||
get schema(): SiteSchemaValue {
|
||||
return (this.constructor as typeof SiteParserBase).schema!;
|
||||
}
|
||||
|
||||
/** MP `site_schema` */
|
||||
siteSchema(): SiteSchemaValue {
|
||||
return this.schema;
|
||||
}
|
||||
|
||||
/** MP `num_filesize` = `size_tools.parse_size` */
|
||||
static numFilesize(text: string | number | null | undefined): number {
|
||||
return parseSizeMp(text);
|
||||
}
|
||||
numFilesize(text: string | number | null | undefined): number {
|
||||
return parseSizeMp(text);
|
||||
}
|
||||
|
||||
/** MP `parse` */
|
||||
async parse(): Promise<void> {
|
||||
if (this.requestMode === "apikey") {
|
||||
if (!this.apikey && !this.token) {
|
||||
this.errMsg = this.errMsg ?? "未设置cookie 或 apikey/token";
|
||||
return;
|
||||
}
|
||||
this._indexHtml = {};
|
||||
} else {
|
||||
this._indexHtml = await this._getPageContent(this._siteUrl);
|
||||
if (!(await this._parseLoggedIn(this._indexHtml))) return;
|
||||
}
|
||||
await this._parseSitePage(this._indexHtml as string);
|
||||
if (this._userBasicPage) {
|
||||
await this._parseUserBaseInfo(
|
||||
await this._getPageContent(
|
||||
urljoin(this._baseUrl, this._userBasicPage),
|
||||
this._userBasicParams,
|
||||
this._userBasicHeaders,
|
||||
this._userBasicMethod,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
await this._parseUserBaseInfo(this._indexHtml as string);
|
||||
}
|
||||
if (this._userDetailPage) {
|
||||
await this._parseUserDetailInfo(
|
||||
await this._getPageContent(
|
||||
urljoin(this._baseUrl, this._userDetailPage),
|
||||
this._userDetailParams,
|
||||
this._userDetailHeaders,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (this._opts.fetchMessages) await this._paseUnreadMsgs();
|
||||
if (this._userTrafficPage) {
|
||||
await this._parseUserTrafficInfo(
|
||||
await this._getPageContent(
|
||||
urljoin(this._baseUrl, this._userTrafficPage),
|
||||
this._userTrafficParams,
|
||||
this._userTrafficHeaders,
|
||||
),
|
||||
);
|
||||
}
|
||||
await this._parseSeedingPages();
|
||||
}
|
||||
|
||||
/** MP `_pase_unread_msgs` */
|
||||
protected async _paseUnreadMsgs(): Promise<void> {
|
||||
const unreadMsgLinks: string[] = [];
|
||||
if (this.messageUnread > 0 || this.messageReadForce) {
|
||||
const links = new Set([this._userMailUnreadPage, this._sysMailUnreadPage]);
|
||||
for (const link of links) {
|
||||
if (!link) continue;
|
||||
const msgLinks: string[] = [];
|
||||
let nextPage = await this._parseMessageUnreadLinks(
|
||||
await this._getPageContent(urljoin(this._baseUrl, link), this._mailUnreadParams, this._mailUnreadHeaders),
|
||||
msgLinks,
|
||||
);
|
||||
let guard = 0;
|
||||
while (nextPage && guard++ < MAX_PAGES) {
|
||||
nextPage = await this._parseMessageUnreadLinks(
|
||||
await this._getPageContent(
|
||||
urljoin(this._baseUrl, nextPage),
|
||||
this._mailUnreadParams,
|
||||
this._mailUnreadHeaders,
|
||||
),
|
||||
msgLinks,
|
||||
);
|
||||
}
|
||||
unreadMsgLinks.push(...msgLinks);
|
||||
}
|
||||
}
|
||||
if (unreadMsgLinks.length && !this.messageUnread) this.messageUnread = unreadMsgLinks.length;
|
||||
for (const msgLink of unreadMsgLinks) {
|
||||
const [head, date, content] = await this._parseMessageContent(
|
||||
await this._getPageContent(urljoin(this._baseUrl, msgLink), this._mailContentParams, this._mailContentHeaders),
|
||||
);
|
||||
this.messageUnreadContents.push([head, date, content]);
|
||||
}
|
||||
}
|
||||
|
||||
/** MP `_parse_seeding_pages` */
|
||||
protected async _parseSeedingPages(): Promise<void> {
|
||||
if (!this._torrentSeedingPage) return;
|
||||
let nextPage: NextPage = await this._parseUserTorrentSeedingInfo(
|
||||
await this._getPageContent(
|
||||
urljoin(this._baseUrl, this._torrentSeedingPage),
|
||||
this._torrentSeedingParams,
|
||||
this._torrentSeedingHeaders,
|
||||
),
|
||||
);
|
||||
let guard = 0;
|
||||
while (nextPage !== null && nextPage !== undefined && nextPage !== false && guard++ < MAX_PAGES) {
|
||||
nextPage = await this._parseUserTorrentSeedingInfo(
|
||||
await this._getPageContent(
|
||||
urljoin(urljoin(this._baseUrl, this._torrentSeedingPage), nextPage),
|
||||
this._torrentSeedingParams,
|
||||
this._torrentSeedingHeaders,
|
||||
),
|
||||
true,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** MP `_prepare_html_text`:去掉 `\d+px` 与 `#\d+` 干扰 */
|
||||
static prepareHtmlText(htmlText: string): string {
|
||||
return String(htmlText ?? "")
|
||||
.replace(/\d+px/g, "")
|
||||
.replace(/#\d+/g, "");
|
||||
}
|
||||
protected _prepareHtmlText(htmlText: string): string {
|
||||
return SiteParserBase.prepareHtmlText(htmlText);
|
||||
}
|
||||
|
||||
/** MP `_get_page_content` */
|
||||
protected async _getPageContent(
|
||||
url: string,
|
||||
params?: Params | null,
|
||||
headers?: Headers_ | null,
|
||||
method?: string | null,
|
||||
): Promise<string> {
|
||||
let reqHeaders: Headers_ | null = null;
|
||||
if (this._ua || headers || this._additionHeaders) {
|
||||
reqHeaders = this.requestMode === "apikey" ? {} : { "User-Agent": `${this._ua}` };
|
||||
if (headers) Object.assign(reqHeaders, headers);
|
||||
else reqHeaders["Content-Type"] = "application/x-www-form-urlencoded; charset=UTF-8";
|
||||
if (this._additionHeaders) Object.assign(reqHeaders, this._additionHeaders);
|
||||
}
|
||||
const h = new Headers();
|
||||
for (const [k, v] of Object.entries(reqHeaders ?? {})) if (v != null) h.set(k, String(v));
|
||||
if (this.requestMode !== "apikey" && this._siteCookie) h.set("cookie", this._siteCookie);
|
||||
|
||||
let init: RequestInit;
|
||||
if (method === "post" || (params && Object.keys(params).length > 0)) {
|
||||
if (reqHeaders?.["Content-Type"] === "application/json") {
|
||||
init = { method: "POST", headers: h, body: JSON.stringify(params ?? {}) };
|
||||
} else {
|
||||
if (!h.has("content-type")) h.set("content-type", "application/x-www-form-urlencoded; charset=UTF-8");
|
||||
const form = new URLSearchParams();
|
||||
for (const [k, v] of Object.entries(params ?? {})) {
|
||||
if (v == null) continue;
|
||||
if (Array.isArray(v)) for (const x of v) form.append(k, String(x));
|
||||
else form.append(k, String(v));
|
||||
}
|
||||
init = { method: "POST", headers: h, body: form.toString() };
|
||||
}
|
||||
} else {
|
||||
init = { method: "GET", headers: h };
|
||||
}
|
||||
|
||||
const fetchImpl = this._opts.fetch ?? globalThis.fetch;
|
||||
const timeout = AbortSignal.timeout(this._opts.timeoutMs ?? 60_000);
|
||||
const signal = this._opts.signal ? AbortSignal.any([this._opts.signal, timeout]) : timeout;
|
||||
let res: Response;
|
||||
try {
|
||||
await this._opts.limiter?.acquire(this._opts.signal);
|
||||
res = await fetchImpl(url, { ...init, signal });
|
||||
} catch (e) {
|
||||
if (this._opts.signal?.aborted) throw e;
|
||||
this.httpErrors.push(redactText(`${init.method} ${url}:${timeout.aborted ? "超时" : (e as Error).message}`));
|
||||
return "";
|
||||
}
|
||||
if (![200, 500, 403].includes(res.status)) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
this.httpErrors.push(redactText(`${init.method} ${url}:HTTP ${res.status}`));
|
||||
return "";
|
||||
}
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
if (reqHeaders && String(reqHeaders.Accept ?? "").includes("application/json")) {
|
||||
try {
|
||||
return JSON.stringify(JSON.parse(new TextDecoder("utf-8").decode(bytes)));
|
||||
} catch {
|
||||
this.httpErrors.push(redactText(`${init.method} ${url}:API 响应 JSON 解析失败`));
|
||||
return "";
|
||||
}
|
||||
}
|
||||
const text = decode(bytes, detectCharset(res.headers.get("content-type"), bytes));
|
||||
if (underChallenge(text)) {
|
||||
this.httpErrors.push(redactText(`${url}:检测到 Cloudflare,请更新 Cookie 和 UA`));
|
||||
return "";
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
/** MP `_parse_logged_in` */
|
||||
protected _parseLoggedIn(htmlText: unknown): boolean | Promise<boolean> {
|
||||
const loggedIn = isLoggedIn(String(htmlText ?? ""));
|
||||
if (!loggedIn) this.errMsg = "未检测到已登陆，请检查cookies是否过期";
|
||||
return loggedIn;
|
||||
}
|
||||
|
||||
protected abstract _parseSitePage(htmlText: string): void | Promise<void>;
|
||||
protected abstract _parseUserBaseInfo(htmlText: string): void | Promise<void>;
|
||||
protected abstract _parseUserTrafficInfo(htmlText: string): void | Promise<void>;
|
||||
protected abstract _parseUserTorrentSeedingInfo(htmlText: string, multiPage?: boolean): NextPage | Promise<NextPage>;
|
||||
protected abstract _parseUserDetailInfo(htmlText: string): void | Promise<void>;
|
||||
protected abstract _parseMessageUnreadLinks(htmlText: string, msgLinks: string[]): NextPage | Promise<NextPage>;
|
||||
protected abstract _parseMessageContent(
|
||||
htmlText: string,
|
||||
): [string | null, string | null, string | null] | Promise<[string | null, string | null, string | null]>;
|
||||
|
||||
/** MP `clear` */
|
||||
clear(): void {
|
||||
this._indexHtml = "";
|
||||
this.seedingInfo = [];
|
||||
this.messageUnreadContents = [];
|
||||
}
|
||||
|
||||
/** MP `to_dict`:公开字段(snake_case),不含凭据 */
|
||||
toDict(): Record<string, unknown> {
|
||||
return {
|
||||
schema: this.schema,
|
||||
userid: this.userid,
|
||||
username: this.username,
|
||||
user_level: this.userLevel,
|
||||
join_at: this.joinAt,
|
||||
bonus: this.bonus,
|
||||
upload: this.upload,
|
||||
download: this.download,
|
||||
ratio: this.ratio,
|
||||
seeding: this.seeding,
|
||||
leeching: this.leeching,
|
||||
seeding_size: this.seedingSize,
|
||||
leeching_size: this.leechingSize,
|
||||
uploaded: this.uploaded,
|
||||
completed: this.completed,
|
||||
incomplete: this.incomplete,
|
||||
uploaded_size: this.uploadedSize,
|
||||
completed_size: this.completedSize,
|
||||
incomplete_size: this.incompleteSize,
|
||||
seeding_info: this.seedingInfo.map((x) => [...x]),
|
||||
message_unread: this.messageUnread,
|
||||
message_unread_contents: this.messageUnreadContents.map((x) => [...x]),
|
||||
message_read_force: this.messageReadForce,
|
||||
err_msg: this.errMsg,
|
||||
http_errors: [...this.httpErrors],
|
||||
};
|
||||
}
|
||||
|
||||
/** MP `refresh_userdata` 里构造 `SiteUserData` 的部分 */
|
||||
toUserInfo(): SiteUserInfo {
|
||||
const num = (v: unknown) => {
|
||||
const n = typeof v === "number" ? v : Number(v);
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
};
|
||||
const messages: SiteUnreadMessage[] = this.messageUnreadContents.map(([head, date, content]) => ({
|
||||
head,
|
||||
date,
|
||||
content,
|
||||
}));
|
||||
return {
|
||||
schema: this.schema,
|
||||
domain: extractDomain(this._siteUrl) || null,
|
||||
userId: this.userid == null || this.userid === "" ? null : String(this.userid),
|
||||
username: this.username,
|
||||
userLevel: this.userLevel,
|
||||
joinAt: this.joinAt,
|
||||
upload: num(this.upload),
|
||||
download: num(this.download),
|
||||
ratio: num(this.ratio),
|
||||
bonus: num(this.bonus),
|
||||
seeding: num(this.seeding),
|
||||
seedingSize: num(this.seedingSize),
|
||||
leeching: num(this.leeching),
|
||||
leechingSize: num(this.leechingSize),
|
||||
seedingInfo: this.seedingInfo.map(([a, b]) => [num(a), num(b)] as [number, number]),
|
||||
messageUnread: num(this.messageUnread),
|
||||
messageUnreadContents: messages,
|
||||
errMsg: this.errMsg,
|
||||
raw: this.toDict(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export type SiteParserClass = (new (
|
||||
opts: SiteUserParserOptions,
|
||||
) => SiteParserBase) & {
|
||||
schema: SiteSchemaValue | null;
|
||||
};
|
||||
@@ -0,0 +1,47 @@
|
||||
import {
|
||||
BITPT_USER_DETAILS,
|
||||
BITPT_USER_INDEX,
|
||||
BITPT_USER_SEEDING_EMPTY,
|
||||
BITPT_USER_SEEDING_P1,
|
||||
createMockFetch,
|
||||
} from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BitptSiteUserInfo } from "./bitpt.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
const GiB = 1024 ** 3;
|
||||
|
||||
describe("BitptSiteUserInfo(MP bitpt.py)", () => {
|
||||
it("#userinfotop 登录判定 → frmtable 键值 → browse.php myseed 分页", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: BITPT_USER_INDEX },
|
||||
"/userdetails.php?uid=55": { body: BITPT_USER_DETAILS },
|
||||
"/browse.php?p=1": { body: BITPT_USER_SEEDING_P1 },
|
||||
"/browse.php?p=2": { body: BITPT_USER_SEEDING_EMPTY },
|
||||
});
|
||||
const info = await runParser(BitptSiteUserInfo, { fetch: m.fetch, cookie: "bitpt=secret" });
|
||||
expect(info).toMatchObject({
|
||||
schema: "Bitpt",
|
||||
userId: "55",
|
||||
username: "gina",
|
||||
userLevel: "Elite",
|
||||
joinAt: "2021-06-07 08:09:10",
|
||||
upload: 3 * 1024 ** 4,
|
||||
download: 1024 ** 4,
|
||||
ratio: 3,
|
||||
bonus: 2048.5,
|
||||
seeding: 2,
|
||||
seedingSize: 6 * GiB,
|
||||
});
|
||||
const seedReq = m.requests.find((r) => r.url.pathname === "/browse.php")!;
|
||||
expect(seedReq.url.search).toBe("?t=myseed&st=2&d=desc&p=1");
|
||||
expect(JSON.stringify(info)).not.toContain("secret");
|
||||
});
|
||||
|
||||
it("未登录(无 #userinfotop):只请求首页", async () => {
|
||||
const m = createMockFetch({ "/": { body: "<html><body>login</body></html>" } });
|
||||
const info = await runParser(BitptSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info.userId).toBeNull();
|
||||
expect(m.requests).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/bitpt.py @ e6820a8
|
||||
*
|
||||
* 极速之星(bitpt.cn)站点用户信息解析器(原作者 ThedoRap)。
|
||||
*
|
||||
* 函数对照(`BitptSiteUserInfo`):`_parse_site_page` → `_parseSitePage`、`_parse_logged_in` → `_parseLoggedIn`
|
||||
* (`#userinfotop`)、`_parse_user_base_info` → `_parseUserBaseInfo`(`table.frmtable` 两列键值)、
|
||||
* `_parse_user_torrent_seeding_page_info` → `parseUserTorrentSeedingPageInfo`(`table.torrenttable tr.btr td.r`)、
|
||||
* `parse` → `parse`(基类流程后取 uid → `userdetails.php?uid=` → `browse.php?t=myseed&st=2&d=desc&p=n` 逐页累加)。
|
||||
* BeautifulSoup 以 cheerio 等价实现。
|
||||
*
|
||||
* 与 MP 的偏离:末尾不把字段转字符串(同 zhixing.ts);做种翻页上限 50 页。
|
||||
*/
|
||||
import { load } from "cheerio";
|
||||
import { type NextPage, SiteParserBase, urljoin } from "./base.ts";
|
||||
import { normalizeDatetime, pyFloat, pyInt } from "./common.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
const MAX_PAGES = 50;
|
||||
|
||||
export class BitptSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.Bitpt;
|
||||
|
||||
protected _torrentSeedingBase: string | undefined;
|
||||
|
||||
protected _parseSitePage(_htmlText: string): void {
|
||||
this._userBasicPage = "userdetails.php?uid={uid}";
|
||||
this._userDetailPage = null;
|
||||
this._userBasicParams = {};
|
||||
this._userTrafficPage = null;
|
||||
this._sysMailUnreadPage = null;
|
||||
this._userMailUnreadPage = null;
|
||||
this._mailUnreadParams = {};
|
||||
this._torrentSeedingBase = "browse.php";
|
||||
this._torrentSeedingParams = { t: "myseed", st: "2", d: "desc" };
|
||||
this._torrentSeedingHeaders = {};
|
||||
this._additionHeaders = {};
|
||||
}
|
||||
|
||||
protected override _parseLoggedIn(htmlText: unknown): boolean {
|
||||
return load(String(htmlText ?? ""))("#userinfotop").length > 0;
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
if (!htmlText) return;
|
||||
const $ = load(htmlText);
|
||||
const table = $("table.frmtable").first();
|
||||
if (!table.length) return;
|
||||
const info: Record<string, string> = {};
|
||||
table.find("tr").each((_, row) => {
|
||||
const cells = $(row).find("td");
|
||||
if (cells.length === 2) info[cells.eq(0).text().trim()] = cells.eq(1).text().trim();
|
||||
});
|
||||
this.userid = info.UID ?? null;
|
||||
this.username = "用户名" in info ? info.用户名!.split(" ")[0]! : null;
|
||||
this.userLevel = info.用户级别 ?? null;
|
||||
this.joinAt = "注册时间" in info ? (normalizeDatetime(info.注册时间) ?? null) : null;
|
||||
this.upload = "上传流量" in info ? this.numFilesize(info.上传流量) : 0;
|
||||
this.download = "下载流量" in info ? this.numFilesize(info.下载流量) : 0;
|
||||
this.ratio = "共享率" in info ? pyFloat(info.共享率) : 0;
|
||||
const bonusMatch = /累计([\d.]+)/.exec(info.星辰 ?? "");
|
||||
this.bonus = bonusMatch ? pyFloat(bonusMatch[1]) : 0;
|
||||
this.messageUnread = 0;
|
||||
if (this._torrentSeedingBase) {
|
||||
this.seeding = 0;
|
||||
this.seedingSize = 0;
|
||||
} else {
|
||||
const seedingInfo = $("div").filter(
|
||||
(_, el) =>
|
||||
$(el).attr("style") ===
|
||||
"margin:0 auto;width:90%;font-size:14px;margin-top:10px;margin-bottom:10px;text-align:center;",
|
||||
);
|
||||
if (seedingInfo.length) {
|
||||
const links = seedingInfo.first().find("a");
|
||||
const text = links.length > 1 ? links.eq(1).text() : "";
|
||||
const m = /当前上传的种子\((\d+)个, 共([\d.]+ [KMGT]B)\)/.exec(text);
|
||||
this.seeding = m ? pyInt(m[1]) : 0;
|
||||
this.seedingSize = m ? this.numFilesize(m[2]) : 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
protected _parseUserDetailInfo(_htmlText: string): void {}
|
||||
|
||||
/** 单页做种:返回 [数量, 大小] */
|
||||
parseUserTorrentSeedingPageInfo(htmlText: string): [number, number] {
|
||||
if (!htmlText) return [0, 0];
|
||||
const $ = load(htmlText);
|
||||
const table = $("table.torrenttable").first();
|
||||
if (!table.length) return [0, 0];
|
||||
const rows = table.find("tr").toArray();
|
||||
if (rows.length <= 1) return [0, 0];
|
||||
let count = 0;
|
||||
let size = 0;
|
||||
for (const row of rows.slice(1)) {
|
||||
if (!$(row).hasClass("btr")) continue;
|
||||
const td = $(row).find("td.r").first();
|
||||
if (!td.length) continue;
|
||||
const a = td.find("a").first();
|
||||
const sizeText = a.length ? a.text().trim() : td.text().trim();
|
||||
if (sizeText) {
|
||||
count += 1;
|
||||
size += this.numFilesize(sizeText);
|
||||
}
|
||||
}
|
||||
return [count, size];
|
||||
}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
protected _parseUserTorrentSeedingInfo(_htmlText: string): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
override async parse(): Promise<void> {
|
||||
await super.parse();
|
||||
if (this._indexHtml) {
|
||||
const $ = load(String(this._indexHtml));
|
||||
const link = $("a")
|
||||
.filter((_, el) => /userdetails\.php\?uid=\d+/.test($(el).attr("href") ?? ""))
|
||||
.first();
|
||||
if (link.length) {
|
||||
const m = /uid=(\d+)/.exec(link.attr("href") ?? "");
|
||||
if (m) this.userid = m[1]!;
|
||||
}
|
||||
}
|
||||
if (this.userid && this._userBasicPage) {
|
||||
const basicUrl = this._userBasicPage.replace("{uid}", this.userid);
|
||||
this._parseUserBaseInfo(await this._getPageContent(urljoin(this._baseUrl, basicUrl)));
|
||||
}
|
||||
if (this._torrentSeedingBase) {
|
||||
const baseUrl = urljoin(this._baseUrl, this._torrentSeedingBase);
|
||||
const params: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(this._torrentSeedingParams ?? {})) params[k] = String(v);
|
||||
for (let page = 1; page <= MAX_PAGES; page++) {
|
||||
params.p = String(page);
|
||||
const [n, s] = this.parseUserTorrentSeedingPageInfo(
|
||||
await this._getPageContent(`${baseUrl}?${new URLSearchParams(params).toString()}`),
|
||||
);
|
||||
this.seeding += n;
|
||||
this.seedingSize += s;
|
||||
if (n === 0) break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/foundation/text.py @ e6820a8
|
||||
* @mp-source app/foundation/temporal.py @ e6820a8
|
||||
* @mp-source app/domain/site.py @ e6820a8
|
||||
* @mp-source app/adapters/network/cloudflare.py @ e6820a8
|
||||
*
|
||||
* 解析器公共工具:MoviePilot e6820a8(GPLv3)
|
||||
* - `app/foundation/text.py` `parse_int/parse_float/contains_chinese` → `parseIntMp/parseFloatMp/containsChinese`
|
||||
* - `app/foundation/temporal.py` `normalize_datetime/format_timestamp` → 同名(dateparser 的保守子集)
|
||||
* - `app/domain/site.py` `SiteUtils.is_logged_in` → `isLoggedIn`
|
||||
* - `app/adapters/network/cloudflare.py` `under_challenge` → `underChallenge`
|
||||
* - `app/domain/site.py` `extract_domain` → `extractDomain`(与 mteam/definition.ts `rootDomain` 同义)
|
||||
*/
|
||||
import { load } from "cheerio";
|
||||
import { hasChildElements, parseHtml } from "./dom.ts";
|
||||
|
||||
/** MP `parse_int`:去千分位,失败返回 0 */
|
||||
export function parseIntMp(value: string | number | null | undefined): number {
|
||||
if (value == null) return 0;
|
||||
const s = String(value).trim().replace(/,/g, "");
|
||||
if (!/^[+-]?\d+$/.test(s)) return 0;
|
||||
return Number.parseInt(s, 10);
|
||||
}
|
||||
|
||||
/** MP `parse_float`:去千分位,失败返回 0 */
|
||||
export function parseFloatMp(value: string | number | null | undefined): number {
|
||||
if (value == null) return 0;
|
||||
const s = String(value).trim().replace(/,/g, "");
|
||||
if (s === "") return 0;
|
||||
if (!/^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/.test(s) && !/^[+-]?(inf|infinity|nan)$/i.test(s)) return 0;
|
||||
const n = Number(s);
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
}
|
||||
|
||||
/** Python `int(str)`(带 strip);非法抛错,与 MP 一致交给调用方 */
|
||||
export function pyInt(value: unknown): number {
|
||||
if (typeof value === "number") return Math.trunc(value);
|
||||
if (typeof value === "boolean") return value ? 1 : 0;
|
||||
const s = String(value ?? "").trim();
|
||||
if (!/^[+-]?\d+$/.test(s)) throw new Error(`invalid literal for int(): '${s}'`);
|
||||
return Number.parseInt(s, 10);
|
||||
}
|
||||
|
||||
/** Python `float(str)`;非法抛错 */
|
||||
export function pyFloat(value: unknown): number {
|
||||
if (typeof value === "number") return value;
|
||||
const s = String(value ?? "").trim();
|
||||
const n = Number(s);
|
||||
if (s === "" || Number.isNaN(n)) throw new Error(`could not convert string to float: '${s}'`);
|
||||
return n;
|
||||
}
|
||||
|
||||
export function containsChinese(value: string | string[] | null | undefined): boolean {
|
||||
if (!value) return false;
|
||||
const s = Array.isArray(value) ? value.join(" ") : value;
|
||||
return /[一-鿿]/.test(s);
|
||||
}
|
||||
|
||||
const MONTHS: Record<string, number> = {
|
||||
jan: 1,
|
||||
feb: 2,
|
||||
mar: 3,
|
||||
apr: 4,
|
||||
may: 5,
|
||||
jun: 6,
|
||||
jul: 7,
|
||||
aug: 8,
|
||||
sep: 9,
|
||||
sept: 9,
|
||||
oct: 10,
|
||||
nov: 11,
|
||||
dec: 12,
|
||||
january: 1,
|
||||
february: 2,
|
||||
march: 3,
|
||||
april: 4,
|
||||
june: 6,
|
||||
july: 7,
|
||||
august: 8,
|
||||
september: 9,
|
||||
october: 10,
|
||||
november: 11,
|
||||
december: 12,
|
||||
};
|
||||
|
||||
const pad = (n: number | string) => String(Number(n)).padStart(2, "0");
|
||||
|
||||
/**
|
||||
* MP `normalize_datetime`(dateparser.parse 后 strftime("%Y-%m-%d %H:%M:%S"),失败原样返回)。
|
||||
* 这里只实现确定性的绝对时间格式:
|
||||
* - `YYYY-MM-DD[ T]HH:MM[:SS][.fff][Z|±HH:MM]`、`YYYY/MM/DD`、`YYYY年MM月DD日`(时区后缀按 dateparser 行为丢弃,保留墙钟时间)
|
||||
* - `DD Mon YYYY[, HH:MM]`、`Mon DD[,] YYYY [HH:MM]`(英文月份)
|
||||
* 相对时间("3 天前"/"2 hours ago")dateparser 会以当前时间换算,本实现不做(原样返回,标记待核验)。
|
||||
*/
|
||||
export function normalizeDatetime(value: string | null | undefined): string | null | undefined {
|
||||
if (!value) return value;
|
||||
const v = String(value).trim();
|
||||
let m =
|
||||
/^(\d{4})[-/.年](\d{1,2})[-/.月](\d{1,2})日?(?:[ T]+(\d{1,2}):(\d{1,2})(?::(\d{1,2}))?(?:\.\d+)?)?\s*(?:Z|[+-]\d{2}:?\d{2}|UTC|GMT)?$/i.exec(
|
||||
v,
|
||||
);
|
||||
if (m) return `${m[1]}-${pad(m[2]!)}-${pad(m[3]!)} ${pad(m[4] ?? 0)}:${pad(m[5] ?? 0)}:${pad(m[6] ?? 0)}`;
|
||||
m =
|
||||
/^(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3,9})\.?,?\s+(\d{4})(?:,?\s+(?:at\s+)?(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(am|pm)?)?$/i.exec(
|
||||
v,
|
||||
);
|
||||
if (m && MONTHS[m[2]!.toLowerCase()]) {
|
||||
const h = ampm(m[4], m[7]);
|
||||
return `${m[3]}-${pad(MONTHS[m[2]!.toLowerCase()]!)}-${pad(m[1]!)} ${pad(h)}:${pad(m[5] ?? 0)}:${pad(m[6] ?? 0)}`;
|
||||
}
|
||||
m =
|
||||
/^([A-Za-z]{3,9})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})(?:,?\s+(?:at\s+)?(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(am|pm)?)?$/i.exec(
|
||||
v,
|
||||
);
|
||||
if (m && MONTHS[m[1]!.toLowerCase()]) {
|
||||
const h = ampm(m[4], m[7]);
|
||||
return `${m[3]}-${pad(MONTHS[m[1]!.toLowerCase()]!)}-${pad(m[2]!)} ${pad(h)}:${pad(m[5] ?? 0)}:${pad(m[6] ?? 0)}`;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function ampm(h: string | undefined, suffix: string | undefined): number {
|
||||
let n = Number(h ?? 0);
|
||||
if (suffix?.toLowerCase() === "pm" && n < 12) n += 12;
|
||||
if (suffix?.toLowerCase() === "am" && n === 12) n = 0;
|
||||
return n;
|
||||
}
|
||||
|
||||
/**
|
||||
* MP `format_timestamp`:非纯数字字符串原样返回;Unix 秒 → 本地时间 "YYYY-MM-DD HH:MM:SS"。
|
||||
* MP 用进程本地时区;这里用显式 `tzOffsetMinutes`(默认 +480,即 Asia/Shanghai,MP 容器默认 TZ)。
|
||||
*/
|
||||
export function formatTimestamp(timestamp: unknown, tzOffsetMinutes = 480): string | undefined {
|
||||
if (timestamp == null) return undefined;
|
||||
if (typeof timestamp === "string" && !/^\d+$/.test(timestamp)) return timestamp;
|
||||
const n = Number(timestamp);
|
||||
if (!Number.isFinite(n)) return String(timestamp);
|
||||
const d = new Date(Math.trunc(n) * 1000 + tzOffsetMinutes * 60_000);
|
||||
return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
|
||||
}
|
||||
|
||||
/** MP `SiteUtils.is_logged_in` */
|
||||
export function isLoggedIn(htmlText: string): boolean {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return false;
|
||||
if (html.nodes("//input[@type='password']").length) return false;
|
||||
const xpaths = [
|
||||
'//a[contains(@href, "logout") or contains(@data-url, "logout") or contains(@href, "mybonus") or contains(@onclick, "logout") or contains(@href, "usercp") or contains(@lay-on, "logout")]',
|
||||
'//form[contains(@action, "logout")]',
|
||||
'//div[@class="user-info-side"]',
|
||||
'//a[@id="myitem"]',
|
||||
];
|
||||
return xpaths.some((x) => html.nodes(x).length > 0);
|
||||
}
|
||||
|
||||
const CHALLENGE_TITLES = ["Just a moment...", "请稍候…", "DDOS-GUARD"];
|
||||
const CHALLENGE_SELECTORS = [
|
||||
"#cf-challenge-running",
|
||||
".ray_id",
|
||||
".attack-box",
|
||||
"#cf-please-wait",
|
||||
"#challenge-spinner",
|
||||
"#trk_jschal_js",
|
||||
"td.info #js_info",
|
||||
"div.vc div.text-box h2",
|
||||
];
|
||||
|
||||
/** MP `under_challenge`:Cloudflare / DDoS-Guard 挑战页 */
|
||||
export function underChallenge(htmlText: string | null | undefined): boolean {
|
||||
if (!htmlText) return false;
|
||||
const $ = load(htmlText);
|
||||
const title = $("title").text().trim().toLowerCase();
|
||||
if (CHALLENGE_TITLES.some((t) => t.toLowerCase() === title)) return true;
|
||||
return CHALLENGE_SELECTORS.some((s) => $(s).length > 0);
|
||||
}
|
||||
|
||||
const SPECIAL_SITE_DOMAINS = ["u2.dmhy.org", "pt.ecust.pp.ua", "pt.gtkpw.xyz", "pt.gtk.pw"];
|
||||
|
||||
/** MP `site_rules.extract_domain`:特殊域名表 → netloc 超过 3 级原样 → 否则取最后两级 */
|
||||
export function extractDomain(url: string | null | undefined): string {
|
||||
if (!url) return "";
|
||||
for (const d of SPECIAL_SITE_DOMAINS) if (url.includes(d)) return d;
|
||||
let netloc: string;
|
||||
if (!url.startsWith("http")) netloc = url;
|
||||
else {
|
||||
try {
|
||||
netloc = new URL(url).host;
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
if (!netloc) return "";
|
||||
const labels = netloc.split(".");
|
||||
if (labels.length > 3) return netloc;
|
||||
return labels.slice(-2).join(".");
|
||||
}
|
||||
|
||||
/** Python `re.search(...)`: 返回 match 或 null(显式 helper,方便逐行对照) */
|
||||
export function reSearch(pattern: RegExp, text: string | null | undefined): RegExpExecArray | null {
|
||||
if (text == null) return null;
|
||||
const re = new RegExp(pattern.source, pattern.flags.replace("g", ""));
|
||||
return re.exec(text);
|
||||
}
|
||||
|
||||
/** Python `re.findall`(带分组时返回分组数组) */
|
||||
export function reFindall(pattern: RegExp, text: string): string[][] {
|
||||
const re = new RegExp(pattern.source, pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`);
|
||||
return [...text.matchAll(re)].map((m) => m.slice(1));
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import { createMockFetch, DISCUZ_USER_INDEX, DISCUZ_USER_SEEDING, DISCUZ_USER_SPACE } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { DiscuzUserInfo } from "./discuz.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
describe("DiscuzUserInfo(MP discuz.py)", () => {
|
||||
it("&uid= 链接定位用户,空间页取等级/积分/流量,做种表按表头定位列", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: DISCUZ_USER_INDEX },
|
||||
"/home.php?uid=42": { body: DISCUZ_USER_SPACE },
|
||||
"/forum.php?mod=torrents": { body: DISCUZ_USER_SEEDING },
|
||||
});
|
||||
const info = await runParser(DiscuzUserInfo, { fetch: m.fetch });
|
||||
expect(info).toMatchObject({
|
||||
schema: "DiscuzX",
|
||||
userId: "42",
|
||||
username: "frank",
|
||||
userLevel: "Lv3",
|
||||
joinAt: "2018-02-03 04:05:00",
|
||||
bonus: 520,
|
||||
upload: 2 * 1024 ** 4,
|
||||
download: 500 * 1024 ** 3,
|
||||
seeding: 1,
|
||||
seedingInfo: [[6, 8 * 1024 ** 3]],
|
||||
});
|
||||
});
|
||||
|
||||
it("未登录(无登出链接):errMsg", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: "<html><body><a href='member.php?mod=logging&action=login'>登录</a></body></html>" },
|
||||
});
|
||||
const info = await runParser(DiscuzUserInfo, { fetch: m.fetch });
|
||||
expect(info.errMsg).toContain("未检测到已登陆");
|
||||
expect(info.userId).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/discuz.py @ e6820a8
|
||||
*
|
||||
* Discuz!X 站点用户信息解析器。
|
||||
*
|
||||
* 函数对照(`DiscuzUserInfo`):`_parse_user_base_info` → `_parseUserBaseInfo`、`_parse_site_page` → 空、
|
||||
* `_parse_user_detail_info` → `_parseUserDetailInfo`、`_parse_user_torrent_seeding_info` →
|
||||
* `_parseUserTorrentSeedingInfo`、`_parse_user_traffic_info` / `_parse_message_unread_links` / `_parse_message_content` → 空。
|
||||
* XPath 与正则逐字保留(含 MP 把"分享率"括号值写入 bonus 的原行为,随后被"积分"覆盖)。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase } from "./base.ts";
|
||||
import { normalizeDatetime, parseFloatMp, parseIntMp, reSearch } from "./common.ts";
|
||||
import { hasChildElements, parseHtml } from "./dom.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
export class DiscuzUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.DiscuzX;
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
const html = parseHtml(this._prepareHtmlText(htmlText));
|
||||
if (!html) return;
|
||||
const userInfo = html.nodes('//a[contains(@href, "&uid=")]');
|
||||
if (!userInfo.length) return;
|
||||
const href = userInfo[0]!.attr("href") ?? "";
|
||||
const m = reSearch(/&uid=(\d+)/, href);
|
||||
if (m?.[0].trim()) {
|
||||
this.userid = m[1]!;
|
||||
this._torrentSeedingPage = "forum.php?&mod=torrents&cat_5up=on";
|
||||
this._userDetailPage = href;
|
||||
// lxml `.text`:首个子节点之前的文本
|
||||
const firstChild = userInfo[0]!.node.firstChild;
|
||||
this.username = firstChild && firstChild.nodeType === 3 ? (firstChild.nodeValue ?? "").trim() : "";
|
||||
}
|
||||
}
|
||||
|
||||
protected _parseSitePage(_htmlText: string): void {}
|
||||
|
||||
/** 解析用户额外信息,加入时间,等级 */
|
||||
protected _parseUserDetailInfo(htmlText: string): void {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
const levels = html.strings('//a[contains(@href, "usergroup")]/text()');
|
||||
if (levels.length) this.userLevel = levels[levels.length - 1]!.trim();
|
||||
const joinAt = html.strings('//li[em[text()="注册时间"]]/text()');
|
||||
if (joinAt.length) this.joinAt = normalizeDatetime(joinAt[0]!.trim()) ?? null;
|
||||
const ratioText = html.strings('//li[contains(.//text(), "分享率")]//text()');
|
||||
if (ratioText.length) {
|
||||
const m = reSearch(/\(([\d,.]+)\)/, ratioText[0]);
|
||||
if (m?.[1]?.trim()) this.bonus = parseFloatMp(m[1]);
|
||||
}
|
||||
const bonusText = html.strings('//li[em[text()="积分"]]/text()');
|
||||
if (bonusText.length) this.bonus = parseFloatMp(bonusText[0]!.trim());
|
||||
const upload = html.strings('//li[em[contains(text(),"上传量")]]/text()');
|
||||
if (upload.length) this.upload = this.numFilesize(upload[0]!.trim().split("/").pop());
|
||||
const download = html.strings('//li[em[contains(text(),"下载量")]]/text()');
|
||||
if (download.length) this.download = this.numFilesize(download[0]!.trim().split("/").pop());
|
||||
}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(htmlText: string): NextPage {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return null;
|
||||
let sizeCol = 3;
|
||||
let seedersCol = 4;
|
||||
if (html.nodes('//tr[position()=1]/td[.//img[@class="size"] and .//img[@alt="size"]]').length) {
|
||||
sizeCol =
|
||||
html.nodes('//tr[position()=1]/td[.//img[@class="size"] and .//img[@alt="size"]]/preceding-sibling::td')
|
||||
.length + 1;
|
||||
}
|
||||
if (html.nodes('//tr[position()=1]/td[.//img[@class="seeders"] and .//img[@alt="seeders"]]').length) {
|
||||
seedersCol =
|
||||
html.nodes('//tr[position()=1]/td[.//img[@class="seeders"] and .//img[@alt="seeders"]]/preceding-sibling::td')
|
||||
.length + 1;
|
||||
}
|
||||
let pageSeeding = 0;
|
||||
let pageSeedingSize = 0;
|
||||
const pageSeedingInfo: Array<[number, number]> = [];
|
||||
const sizes = html.nodes(`//tr[position()>1]/td[${sizeCol}]`);
|
||||
const seeders = html.strings(`//tr[position()>1]/td[${seedersCol}]//text()`);
|
||||
if (sizes.length && seeders.length) {
|
||||
pageSeeding = sizes.length;
|
||||
for (let i = 0; i < sizes.length; i++) {
|
||||
const size = this.numFilesize(sizes[i]!.text().trim());
|
||||
pageSeedingSize += size;
|
||||
pageSeedingInfo.push([parseIntMp(seeders[i]), size]);
|
||||
}
|
||||
}
|
||||
this.seeding += pageSeeding;
|
||||
this.seedingSize += pageSeedingSize;
|
||||
this.seedingInfo.push(...pageSeedingInfo);
|
||||
const next = html.strings('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁")]/@href');
|
||||
return next.length ? next[next.length - 1]!.trim() : null;
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/foundation/dom.py @ e6820a8
|
||||
*
|
||||
* lxml `etree.HTML(...).xpath(...)` 的 TS 等价层(站点用户信息解析器专用)。
|
||||
*
|
||||
* MP 解析器(app/modules/indexer/parser/*.py,提交 e6820a8,GPLv3)全部用 lxml XPath 1.0 取值;
|
||||
* 为了让 XPath 表达式能逐字迁移,本层:
|
||||
* 1. 用 htmlparser2(经 cheerio,HTML 模式)解析——与 libxml2 HTML 解析器一样**不插入 <tbody>**,
|
||||
* 标签/属性名小写,实体解码(` ` → U+00A0);
|
||||
* 2. 把 DOM 复制成 @xmldom/xmldom 文档,交给 `xpath` 包求值(XPath 1.0,与 lxml 同语义);
|
||||
* 3. 近似 libxml2 的 "blank" 规则:html/head/table/thead/tbody/tfoot/tr/ul/ol/select 下的纯空白文本节点丢弃
|
||||
* (影响 `//text()[0]` 类取值)。其他 libxml2 容错修复(自动补 <p>/<body> 等)未模拟。
|
||||
*
|
||||
* 结果约定同 lxml:元素 → XNode;`@attr` → 属性值字符串;`text()` → 文本字符串;`string(.)`/`count()` → 标量。
|
||||
*/
|
||||
import { DOMImplementation } from "@xmldom/xmldom";
|
||||
import { load } from "cheerio";
|
||||
import * as xpath from "xpath";
|
||||
|
||||
type XmlNode = Node;
|
||||
|
||||
/** cheerio/domhandler 节点的结构子集(避免直接依赖 domhandler 类型) */
|
||||
interface DhNode {
|
||||
type: string;
|
||||
name?: string;
|
||||
data?: string;
|
||||
attribs?: Record<string, string>;
|
||||
children?: DhNode[];
|
||||
}
|
||||
|
||||
const BLANK_DROP_PARENTS = new Set(["html", "head", "table", "thead", "tbody", "tfoot", "tr", "ul", "ol", "select"]);
|
||||
|
||||
function copyInto(doc: Document, parent: XmlNode, src: DhNode, parentName: string): void {
|
||||
if (src.type === "text") {
|
||||
const data = src.data ?? "";
|
||||
if (data.trim() === "" && BLANK_DROP_PARENTS.has(parentName)) return;
|
||||
parent.appendChild(doc.createTextNode(data));
|
||||
return;
|
||||
}
|
||||
if (src.type === "comment") {
|
||||
parent.appendChild(doc.createComment(src.data ?? ""));
|
||||
return;
|
||||
}
|
||||
if (src.type === "tag" || src.type === "script" || src.type === "style") {
|
||||
const name = (src.name ?? "div").toLowerCase();
|
||||
let el: Element;
|
||||
try {
|
||||
el = doc.createElement(name);
|
||||
} catch {
|
||||
el = doc.createElement("x-invalid");
|
||||
}
|
||||
for (const [k, v] of Object.entries(src.attribs ?? {})) {
|
||||
try {
|
||||
el.setAttribute(k.toLowerCase(), v);
|
||||
} catch {
|
||||
// 非法属性名(如 @click、:class)在 lxml 下同样取不到,跳过
|
||||
}
|
||||
}
|
||||
parent.appendChild(el);
|
||||
for (const c of src.children ?? []) copyInto(doc, el, c, name);
|
||||
return;
|
||||
}
|
||||
// root / directive(doctype) / cdata:仅递归子节点
|
||||
for (const c of src.children ?? []) copyInto(doc, parent, c, parentName);
|
||||
}
|
||||
|
||||
/** 可 XPath 查询的节点包装(对应 lxml `_Element`)。 */
|
||||
export class XNode {
|
||||
constructor(readonly node: XmlNode) {}
|
||||
|
||||
/** lxml `el.xpath(expr)`:节点集 → (XNode|string)[];标量 → [标量] */
|
||||
xpath(expr: string): Array<XNode | string> {
|
||||
const r = xpath.select(expr, this.node);
|
||||
if (Array.isArray(r)) {
|
||||
return r.map((n) => {
|
||||
if (n.nodeType === 1 || n.nodeType === 9) return new XNode(n);
|
||||
if (n.nodeType === 2) return (n as Attr).value;
|
||||
return n.nodeValue ?? "";
|
||||
});
|
||||
}
|
||||
if (r == null) return [];
|
||||
return [String(r)];
|
||||
}
|
||||
|
||||
/** 只取字符串结果(文本/属性);元素结果按 `string(.)` 转文本 */
|
||||
strings(expr: string): string[] {
|
||||
return this.xpath(expr).map((v) => (typeof v === "string" ? v : v.text()));
|
||||
}
|
||||
|
||||
/** 只取元素结果 */
|
||||
nodes(expr: string): XNode[] {
|
||||
return this.xpath(expr).filter((v): v is XNode => v instanceof XNode);
|
||||
}
|
||||
|
||||
/** lxml `el.xpath("string(.)")` */
|
||||
text(): string {
|
||||
return String(xpath.select("string(.)", this.node, true) ?? "");
|
||||
}
|
||||
|
||||
/** 标量求值(string()/count()/boolean()) */
|
||||
evaluate(expr: string): string | number | boolean | null {
|
||||
const r = xpath.select(expr, this.node, true);
|
||||
if (r == null) return null;
|
||||
if (typeof r === "object") return (r as XmlNode).nodeValue ?? new XNode(r as XmlNode).text();
|
||||
return r;
|
||||
}
|
||||
|
||||
get tagName(): string {
|
||||
return (this.node as Element).tagName ?? "";
|
||||
}
|
||||
|
||||
attr(name: string): string | null {
|
||||
const el = this.node as Element;
|
||||
return typeof el.getAttribute === "function" && el.hasAttribute(name) ? el.getAttribute(name) : null;
|
||||
}
|
||||
|
||||
/** lxml `len(el)`:子元素个数 */
|
||||
get childElementCount(): number {
|
||||
let n = 0;
|
||||
const cs = this.node.childNodes;
|
||||
for (let i = 0; i < cs.length; i++) if (cs[i]!.nodeType === 1) n++;
|
||||
return n;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* lxml `etree.HTML(text)`:空文本返回 null(lxml 返回 None);否则返回文档节点。
|
||||
* 与 MP `DomUtils.has_child_elements(html)` 搭配使用 → `hasChildElements`。
|
||||
*/
|
||||
export function parseHtml(text: string | null | undefined): XNode | null {
|
||||
if (text == null || String(text).trim() === "") return null;
|
||||
const $ = load(String(text), { xml: { xmlMode: false, decodeEntities: true } });
|
||||
const doc = new DOMImplementation().createDocument(null, "", null) as unknown as Document;
|
||||
const root = doc.createElement("html");
|
||||
doc.appendChild(root);
|
||||
const dhRoot = $.root()[0] as unknown as DhNode;
|
||||
const kids = dhRoot.children ?? [];
|
||||
// 输入自带 <html> 时展开到根,避免 html/html 嵌套(lxml 会合并)
|
||||
const htmlEl = kids.find((k) => k.type === "tag" && k.name?.toLowerCase() === "html");
|
||||
if (htmlEl) {
|
||||
for (const [k, v] of Object.entries(htmlEl.attribs ?? {})) {
|
||||
try {
|
||||
root.setAttribute(k.toLowerCase(), v);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
for (const k of kids) {
|
||||
if (k === htmlEl) for (const c of htmlEl.children ?? []) copyInto(doc, root, c, "html");
|
||||
else if (k.type !== "text" || (k.data ?? "").trim() !== "") copyInto(doc, root, k, "html");
|
||||
}
|
||||
} else {
|
||||
for (const k of kids) copyInto(doc, root, k, "html");
|
||||
}
|
||||
return new XNode(root);
|
||||
}
|
||||
|
||||
/** MP `DomUtils.has_child_elements`:存在且至少一个子元素 */
|
||||
export function hasChildElements(el: XNode | null | undefined): el is XNode {
|
||||
return el != null && el.childElementCount > 0;
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { createMockFetch, FILELIST_USER_DETAIL, FILELIST_USER_INDEX, FILELIST_USER_SEEDING } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { FileListSiteUserInfo } from "./file-list.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
describe("FileListSiteUserInfo(MP file_list.py)", () => {
|
||||
it("详情页流量/分享率/做种/等级/加入日期/积分 + snatchlist 做种人数", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: FILELIST_USER_INDEX },
|
||||
"/userdetails.php": { body: FILELIST_USER_DETAIL },
|
||||
"/snatchlist.php": { body: FILELIST_USER_SEEDING },
|
||||
});
|
||||
const info = await runParser(FileListSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info).toMatchObject({
|
||||
schema: "FileList",
|
||||
userId: "555",
|
||||
username: "erin",
|
||||
upload: 3 * 1024 ** 4,
|
||||
download: 1024 ** 4,
|
||||
ratio: 3,
|
||||
seeding: 42,
|
||||
seedingSize: Math.round(2.5 * 1024 ** 4),
|
||||
userLevel: "Elite User",
|
||||
joinAt: "2017-07-08 09:10:11",
|
||||
bonus: 1000.5,
|
||||
seedingInfo: [[4, 1024 ** 3]],
|
||||
});
|
||||
expect(m.requests.find((r) => r.url.pathname === "/snatchlist.php")!.url.search).toBe(
|
||||
"?id=555&action=torrents&type=seeding",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/file_list.py @ e6820a8
|
||||
*
|
||||
* FileList 站点用户数据解析器(MP `FileListSiteUserInfo`)。
|
||||
* 函数对照:`_parse_site_page` → `_parseSitePage`、`_parse_user_base_info` → `_parseUserBaseInfo`、
|
||||
* `_parse_user_traffic_info`(空)、`_parse_user_detail_info` → `_parseUserDetailInfo`、
|
||||
* `_parse_user_torrent_seeding_info` → `_parseUserTorrentSeedingInfo`、消息两函数(不支持)。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase } from "./base.ts";
|
||||
import { normalizeDatetime, parseFloatMp, parseIntMp, reSearch } from "./common.ts";
|
||||
import { hasChildElements, parseHtml } from "./dom.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
function at<T>(arr: T[], i: number): T {
|
||||
const v = arr[i];
|
||||
if (v === undefined) throw new Error("IndexError: list index out of range");
|
||||
return v;
|
||||
}
|
||||
|
||||
export class FileListSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.FileList;
|
||||
|
||||
protected _parseSitePage(htmlText: string): void {
|
||||
const text = this._prepareHtmlText(htmlText);
|
||||
const m = reSearch(/userdetails.php\?id=(\d+)/, text);
|
||||
if (m?.[0].trim()) {
|
||||
this._userDetailPage = m[0].trim().replace(/^\/+/, "");
|
||||
this.userid = m[1]!;
|
||||
}
|
||||
// MP 此处无条件拼接(userid 为 None 时得到 "id=None")
|
||||
this._torrentSeedingPage = `snatchlist.php?id=${this.userid ?? "None"}&action=torrents&type=seeding`;
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
const html = parseHtml(this._prepareHtmlText(htmlText));
|
||||
if (!html) return;
|
||||
const ret = html.strings(`//a[contains(@href, "userdetails") and contains(@href, "${this.userid}")]//text()`);
|
||||
if (ret.length) this.username = String(ret[0]);
|
||||
}
|
||||
|
||||
/** 上传/下载/分享率 [做种数/魔力值](FileList 在详情页解析) */
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserDetailInfo(htmlText: string): void {
|
||||
const html = parseHtml(this._prepareHtmlText(htmlText));
|
||||
if (!html) return;
|
||||
const up = html.strings('//table//tr/td[text()="Uploaded"]/following-sibling::td//text()');
|
||||
if (up.length) this.upload = this.numFilesize(up[0]);
|
||||
const down = html.strings('//table//tr/td[text()="Downloaded"]/following-sibling::td//text()');
|
||||
if (down.length) this.download = this.numFilesize(down[0]);
|
||||
const ratio = html.strings('//table//tr/td[text()="Share ratio"]/following-sibling::td//text()');
|
||||
const shareRatio = ratio.length ? parseFloatMp(ratio[0]) : 0;
|
||||
this.ratio = this.download === 0 ? 0 : shareRatio;
|
||||
|
||||
const seed = html.strings('//table//tr/td[text()="Seed bonus"]/following-sibling::td//text()');
|
||||
if (seed.length) {
|
||||
this.seeding = parseIntMp(at(seed, 1));
|
||||
this.seedingSize = this.numFilesize(at(seed, 3));
|
||||
}
|
||||
const level = html.strings('//table//tr/td[text()="Class"]/following-sibling::td//text()');
|
||||
if (level.length) this.userLevel = level[0]!.trim();
|
||||
const joinAt = html.strings('//table//tr/td[contains(text(), "Join")]/following-sibling::td//text()');
|
||||
if (joinAt.length) this.joinAt = normalizeDatetime(joinAt[0]!.split("(")[0]!.trim()) ?? null;
|
||||
const bonus = html.nodes('//a[contains(@href, "shop.php")]');
|
||||
if (bonus.length) this.bonus = parseFloatMp(bonus[0]!.text().trim());
|
||||
}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(htmlText: string, _multiPage = false): NextPage {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return null;
|
||||
const sizeCol = 6;
|
||||
const seedersCol = 7;
|
||||
const pageSeedingInfo: Array<[number, number]> = [];
|
||||
const sizes = html.nodes(`//table/tr[position()>1]/td[${sizeCol}]`);
|
||||
const seeders = html.nodes(`//table/tr[position()>1]/td[${seedersCol}]`);
|
||||
if (sizes.length && seeders.length) {
|
||||
for (let i = 0; i < sizes.length; i++) {
|
||||
const size = this.numFilesize(sizes[i]!.text().trim());
|
||||
pageSeedingInfo.push([parseIntMp(seeders[i]?.text().trim()), size]);
|
||||
}
|
||||
}
|
||||
this.seedingInfo.push(...pageSeedingInfo);
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import {
|
||||
createMockFetch,
|
||||
GAZELLE_LOGIN_PAGE,
|
||||
GAZELLE_USER_DETAIL,
|
||||
GAZELLE_USER_INDEX,
|
||||
GAZELLE_USER_SEEDING,
|
||||
} from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { GazelleSiteUserInfo } from "./gazelle.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
const GB = 1024 ** 3;
|
||||
|
||||
describe("GazelleSiteUserInfo(MP gazelle.py)", () => {
|
||||
it("首页用户链接/流量/积分 + 详情等级/加入时间 + 做种表", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: GAZELLE_USER_INDEX },
|
||||
"/user.php": { body: GAZELLE_USER_DETAIL },
|
||||
"/index.php": { body: GAZELLE_USER_INDEX },
|
||||
"/torrents.php": { body: GAZELLE_USER_SEEDING },
|
||||
});
|
||||
const info = await runParser(GazelleSiteUserInfo, { fetch: m.fetch, cookie: "session=secret" });
|
||||
expect(info).toMatchObject({
|
||||
schema: "Gazelle",
|
||||
userId: "4242",
|
||||
username: "bob",
|
||||
userLevel: "Power User",
|
||||
joinAt: "2019-05-06 07:08:09",
|
||||
upload: Math.round(120.5 * GB),
|
||||
download: Math.round(40.25 * GB),
|
||||
ratio: Math.round((120.5 / 40.25) * 1000) / 1000,
|
||||
bonus: 1234.5,
|
||||
seeding: 2,
|
||||
seedingSize: 11 * GB,
|
||||
seedingInfo: [
|
||||
[7, 10 * GB],
|
||||
[2, GB],
|
||||
],
|
||||
});
|
||||
expect(m.requests.find((r) => r.url.pathname === "/torrents.php")!.url.search).toBe("?type=seeding&userid=4242");
|
||||
expect(JSON.stringify(info)).not.toContain("secret");
|
||||
});
|
||||
|
||||
it("登录页:errMsg 提示未登录,不再请求", async () => {
|
||||
const m = createMockFetch({ "/": { body: GAZELLE_LOGIN_PAGE } });
|
||||
const info = await runParser(GazelleSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info.errMsg).toContain("未检测到已登陆");
|
||||
expect(m.requests).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/gazelle.py @ e6820a8
|
||||
*
|
||||
* Gazelle 站点用户数据解析器(MP `GazelleSiteUserInfo`)。
|
||||
* 函数对照:`_parse_logged_in` → `_parseLoggedIn`、`_parse_user_base_info` → `_parseUserBaseInfo`、
|
||||
* `_parse_site_page` → `_parseSitePage`(空)、`_parse_user_detail_info` → `_parseUserDetailInfo`、
|
||||
* `_parse_user_torrent_seeding_info` → `_parseUserTorrentSeedingInfo`、`_parse_user_traffic_info`(空)、
|
||||
* `_parse_message_unread_links` / `_parse_message_content`(不支持)。XPath 与正则逐字保留。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase } from "./base.ts";
|
||||
import { normalizeDatetime, parseFloatMp, parseIntMp, reSearch } from "./common.ts";
|
||||
import { hasChildElements, parseHtml } from "./dom.ts";
|
||||
import { round3 } from "./nexus-php.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
export class GazelleSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.Gazelle;
|
||||
|
||||
/** 兼容不展示退出入口的站点首页 */
|
||||
protected override _parseLoggedIn(htmlText: unknown): boolean | Promise<boolean> {
|
||||
const html = parseHtml(String(htmlText ?? ""));
|
||||
if (hasChildElements(html) && html.nodes('//a[contains(@href, "user.php?id=")]').length) return true;
|
||||
return super._parseLoggedIn(htmlText);
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
const html = parseHtml(this._prepareHtmlText(htmlText));
|
||||
if (!html) return;
|
||||
let tmps = html.nodes('//a[contains(@href, "user.php?id=") or contains(@href, "user?id=")]');
|
||||
if (tmps.length) {
|
||||
const m = reSearch(/user(?:\.php)?\?id=(\d+)/, tmps[0]!.attr("href"));
|
||||
if (m?.[0].trim()) {
|
||||
this.userid = m[1]!;
|
||||
this._torrentSeedingPage = `torrents.php?type=seeding&userid=${this.userid}`;
|
||||
this._userDetailPage = `user.php?id=${this.userid}`;
|
||||
// lxml `el.text`:首个子节点为文本时取之
|
||||
const first = tmps[0]!.node.firstChild;
|
||||
this.username = first && first.nodeType === 3 ? (first.nodeValue ?? "").trim() : null;
|
||||
}
|
||||
}
|
||||
|
||||
let vals = html.strings('//*[@id="header-uploaded-value"]/@data-value');
|
||||
if (vals.length) this.upload = this.numFilesize(vals[0]);
|
||||
else {
|
||||
vals = html.strings('//li[@id="stats_seeding"]/span/text()');
|
||||
if (vals.length) this.upload = this.numFilesize(vals[0]);
|
||||
}
|
||||
vals = html.strings('//*[@id="header-downloaded-value"]/@data-value');
|
||||
if (vals.length) this.download = this.numFilesize(vals[0]);
|
||||
else {
|
||||
vals = html.strings('//li[@id="stats_leeching"]/span/text()');
|
||||
if (vals.length) this.download = this.numFilesize(vals[0]);
|
||||
}
|
||||
this.ratio = this.download <= 0.0 ? 0.0 : round3(this.upload / this.download);
|
||||
|
||||
vals = html.strings('//a[contains(@href, "bonus")]/@data-tooltip');
|
||||
if (vals.length) {
|
||||
const m = reSearch(/([\d,.]+)/, vals[0]);
|
||||
if (m?.[1]?.trim()) this.bonus = parseFloatMp(m[1]);
|
||||
} else {
|
||||
tmps = html.nodes('//a[contains(@href, "bonus")]');
|
||||
if (tmps.length) {
|
||||
const m = reSearch(/([\d,.]+)/, tmps[0]!.text());
|
||||
if (m?.[1]?.trim()) this.bonus = parseFloatMp(m[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected _parseSitePage(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserDetailInfo(htmlText: string): void {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
// 用户等级
|
||||
let levels = html.strings('//*[@id="class-value"]/@data-value');
|
||||
if (levels.length) this.userLevel = levels[0]!.trim();
|
||||
else {
|
||||
levels = html.strings('//li[contains(text(), "用户等级")]/text()');
|
||||
if (levels.length) {
|
||||
const part = levels[0]!.split(":")[1];
|
||||
if (part === undefined) throw new Error("IndexError: list index out of range");
|
||||
this.userLevel = part.trim();
|
||||
}
|
||||
}
|
||||
// 加入日期
|
||||
let joinAt = html.strings('//*[@id="join-date-value"]/@data-value');
|
||||
if (joinAt.length) this.joinAt = normalizeDatetime(joinAt[0]!.trim()) ?? null;
|
||||
else {
|
||||
joinAt = html.strings(
|
||||
'//div[contains(@class, "box_userinfo_stats")]//li[contains(text(), "加入时间")]/span/text()',
|
||||
);
|
||||
if (joinAt.length) this.joinAt = normalizeDatetime(joinAt[0]!.trim()) ?? null;
|
||||
}
|
||||
// 兼容以文本形式展示上传/下载的站点(如 JPopsuki)
|
||||
if (!this.upload) {
|
||||
const up = html.nodes('//li[starts-with(normalize-space(text()), "Uploaded:")]');
|
||||
if (up.length) {
|
||||
const m = reSearch(/([\d.,]+\s*[GMKT]?i?B)/i, up[0]!.text());
|
||||
if (m) this.upload = this.numFilesize(m[1]);
|
||||
}
|
||||
}
|
||||
if (!this.download) {
|
||||
const down = html.nodes('//li[starts-with(normalize-space(text()), "Downloaded:")]');
|
||||
if (down.length) {
|
||||
const m = reSearch(/([\d.,]+\s*[GMKT]?i?B)/i, down[0]!.text());
|
||||
if (m) this.download = this.numFilesize(m[1]);
|
||||
}
|
||||
}
|
||||
if (!this.ratio && this.upload && this.download) this.ratio = round3(this.upload / this.download);
|
||||
}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(htmlText: string, multiPage = false): NextPage {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return null;
|
||||
let sizeCol = 3;
|
||||
const headCells = html.nodes('//table[contains(@id, "torrent")]//tr[1]/td');
|
||||
if (headCells.length) sizeCol = headCells.length - 3;
|
||||
const seedersCol = sizeCol + 2;
|
||||
|
||||
let pageSeeding = 0;
|
||||
let pageSeedingSize = 0;
|
||||
const pageSeedingInfo: Array<[number, number]> = [];
|
||||
const semanticRows = html.nodes(
|
||||
'//table[contains(@id, "torrent")]//tr[contains(concat(" ", normalize-space(@class), " "), " TableTorrent-rowTitle ")]',
|
||||
);
|
||||
if (semanticRows.length) {
|
||||
// GPW 等现代 Gazelle 页面以语义 class 标记列,逐行按 class 配对读取
|
||||
for (const row of semanticRows) {
|
||||
const sizeCells = row.nodes(
|
||||
'./td[contains(concat(" ", normalize-space(@class), " "), " TableTorrent-cellStatSize ")]',
|
||||
);
|
||||
const seederCells = row.nodes(
|
||||
'./td[contains(concat(" ", normalize-space(@class), " "), " TableTorrent-cellStatSeeders ")]',
|
||||
);
|
||||
if (!sizeCells.length || !seederCells.length) continue;
|
||||
const size = this.numFilesize(sizeCells[0]!.text().trim());
|
||||
const seeders = parseIntMp(seederCells[0]!.text().trim());
|
||||
pageSeeding += 1;
|
||||
pageSeedingSize += size;
|
||||
pageSeedingInfo.push([seeders, size]);
|
||||
}
|
||||
} else {
|
||||
const sizes = html.nodes(`//table[contains(@id, "torrent")]//tr[position()>1]/td[${sizeCol}]`);
|
||||
const seeders = html.strings(`//table[contains(@id, "torrent")]//tr[position()>1]/td[${seedersCol}]/text()`);
|
||||
if (sizes.length && seeders.length) {
|
||||
pageSeeding = sizes.length;
|
||||
for (let i = 0; i < sizes.length; i++) {
|
||||
const size = this.numFilesize(sizes[i]!.text().trim());
|
||||
pageSeedingSize += size;
|
||||
pageSeedingInfo.push([parseIntMp(seeders[i]), size]);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (multiPage) {
|
||||
this.seeding += pageSeeding;
|
||||
this.seedingSize += pageSeedingSize;
|
||||
this.seedingInfo.push(...pageSeedingInfo);
|
||||
} else {
|
||||
if (!this.seeding) this.seeding = pageSeeding;
|
||||
if (!this.seedingSize) this.seedingSize = pageSeedingSize;
|
||||
if (!this.seedingInfo.length) this.seedingInfo = pageSeedingInfo;
|
||||
}
|
||||
const next = html.strings(
|
||||
'//a[contains(.//text(), "Next") or contains(.//text(), "下一页") or contains(@title, "下一页") or contains(@title, "Next")]/@href',
|
||||
);
|
||||
return next.length ? next[next.length - 1]!.trim() : null;
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { createMockFetch, HDDOLBY_USER_DATA, HDDOLBY_USER_ERROR, HDDOLBY_USER_PEERS } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { HDDolbySiteUserInfo } from "./hddolby.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
const API = "https://api.hddolby.com";
|
||||
|
||||
describe("HDDolbySiteUserInfo(MP hddolby.py)", () => {
|
||||
it("api/v1/user/data + api/v1/user/peers(GET,x-api-key)", async () => {
|
||||
const m = createMockFetch({
|
||||
[`GET ${API}/api/v1/user/data`]: { body: HDDOLBY_USER_DATA },
|
||||
[`GET ${API}/api/v1/user/peers`]: { body: HDDOLBY_USER_PEERS },
|
||||
});
|
||||
const info = await runParser(HDDolbySiteUserInfo, {
|
||||
fetch: m.fetch,
|
||||
url: "https://www.hddolby.com/",
|
||||
apikey: "dolby-key",
|
||||
});
|
||||
expect(info).toMatchObject({
|
||||
schema: "HDDolby",
|
||||
userId: "1",
|
||||
username: "ivy",
|
||||
userLevel: "Veteran User",
|
||||
joinAt: "2019-03-03 15:30:36",
|
||||
upload: 852071699418,
|
||||
download: 1885536536,
|
||||
ratio: Math.round((852071699418 / 1885536536) * 100) / 100,
|
||||
bonus: 99774.5,
|
||||
messageUnread: 2,
|
||||
seeding: 2,
|
||||
seedingSize: 1500,
|
||||
seedingInfo: [
|
||||
[4, 1000],
|
||||
[1, 500],
|
||||
],
|
||||
});
|
||||
expect(m.requests[0]!.headers["x-api-key"]).toBe("dolby-key");
|
||||
expect(JSON.stringify(info)).not.toContain("dolby-key");
|
||||
});
|
||||
|
||||
it("业务错误:不填用户数据;无 cookie 时 fetchMessages 也不读网页消息", async () => {
|
||||
const m = createMockFetch({
|
||||
[`GET ${API}/api/v1/user/data`]: { body: HDDOLBY_USER_ERROR },
|
||||
[`GET ${API}/api/v1/user/peers`]: { body: HDDOLBY_USER_ERROR },
|
||||
});
|
||||
const info = await runParser(HDDolbySiteUserInfo, {
|
||||
fetch: m.fetch,
|
||||
url: "https://www.hddolby.com/",
|
||||
apikey: "k",
|
||||
fetchMessages: true,
|
||||
});
|
||||
expect(info.userId).toBeNull();
|
||||
expect(m.requests.map((r) => r.url.pathname)).toEqual(["/api/v1/user/data", "/api/v1/user/peers"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/hddolby.py @ e6820a8
|
||||
*
|
||||
* HDDolby 用户数据解析器(MP `HDDolbySiteUserInfo`,apikey 模式)。
|
||||
* 函数对照:`HDDolby_sysRoleList` → `HDDOLBY_SYS_ROLE_LIST`、`_parse_site_page` → `_parseSitePage`(api.<域>)、
|
||||
* `_parse_logged_in`(恒 True)、`_parse_user_base_info` → `_parseUserBaseInfo`(api/v1/user/data)、
|
||||
* `_parse_user_torrent_seeding_info` → `_parseUserTorrentSeedingInfo`(api/v1/user/peers)、
|
||||
* `_pase_unread_msgs` → `_paseUnreadMsgs`(API 只给未读数,正文借 NexusPHP 解析器用 Cookie 读网页)。
|
||||
* 注:`_user_basic_params = {}` 在 Python 中为假值,故 profile/peers 两个接口实际为 GET(与 MP 一致)。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase } from "./base.ts";
|
||||
import { extractDomain, pyFloat, pyInt } from "./common.ts";
|
||||
import { NexusPhpSiteUserInfo } from "./nexus-php.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
/** 用户级别字典(MP `HDDolby_sysRoleList`) */
|
||||
export const HDDOLBY_SYS_ROLE_LIST: Record<string, string> = {
|
||||
"0": "Peasant",
|
||||
"1": "User",
|
||||
"2": "Power User",
|
||||
"3": "Elite User",
|
||||
"4": "Crazy User",
|
||||
"5": "Insane User",
|
||||
"6": "Veteran User",
|
||||
"7": "Extreme User",
|
||||
"8": "Ultimate User",
|
||||
"9": "Nexus Master",
|
||||
"10": "VIP",
|
||||
"11": "Retiree",
|
||||
"12": "Helper",
|
||||
"13": "Seeder",
|
||||
"14": "Transferrer",
|
||||
"15": "Uploader",
|
||||
"16": "Torrent Manager",
|
||||
"17": "Forum Moderator",
|
||||
"18": "Coder",
|
||||
"19": "Moderator",
|
||||
"20": "Administrator",
|
||||
"21": "Sysop",
|
||||
"22": "Staff Leader",
|
||||
};
|
||||
|
||||
export class HDDolbySiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.HDDolby;
|
||||
override requestMode = "apikey" as const;
|
||||
|
||||
protected _parseSitePage(_htmlText: string): void {
|
||||
this._baseUrl = `https://api.${extractDomain(this._baseUrl)}`;
|
||||
this._userTrafficPage = null;
|
||||
this._userDetailPage = null;
|
||||
this._userBasicPage = "api/v1/user/data";
|
||||
this._userBasicParams = {};
|
||||
this._userBasicHeaders = { "Content-Type": "application/json", Accept: "application/json, text/plain, */*" };
|
||||
this._sysMailUnreadPage = null;
|
||||
this._userMailUnreadPage = null;
|
||||
this._mailUnreadParams = {};
|
||||
this._torrentSeedingPage = "api/v1/user/peers";
|
||||
this._torrentSeedingParams = {};
|
||||
this._torrentSeedingHeaders = { "Content-Type": "application/json", Accept: "application/json, text/plain, */*" };
|
||||
this._additionHeaders = { "x-api-key": this.apikey ?? "" };
|
||||
}
|
||||
|
||||
protected override _parseLoggedIn(_htmlText: unknown): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
if (!htmlText) return;
|
||||
const detail = JSON.parse(htmlText) as Json | null;
|
||||
if (detail?.status !== 0) return;
|
||||
const infos = detail.data as Json[] | null;
|
||||
if (!infos?.length) return;
|
||||
const u = infos[0]!;
|
||||
this.userid = u.id == null ? null : String(u.id);
|
||||
this.username = (u.username as string) ?? null;
|
||||
this.userLevel = HDDOLBY_SYS_ROLE_LIST[String(u.class || "1")] ?? null;
|
||||
this.joinAt = (u.added as string) ?? null;
|
||||
this.upload = pyInt(u.uploaded || "0");
|
||||
this.download = pyInt(u.downloaded || "0");
|
||||
this.ratio = this.download ? Math.round((this.upload / this.download) * 100) / 100 : 0;
|
||||
this.bonus = pyFloat(u.seedbonus || "0");
|
||||
this.messageUnread = pyInt(u.unread_messages || "0");
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserDetailInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(htmlText: string, _multiPage = false): NextPage {
|
||||
if (!htmlText) return null;
|
||||
const seedingInfo = JSON.parse(htmlText) as Json | null;
|
||||
if (seedingInfo?.status !== 0) return null;
|
||||
const torrents = (seedingInfo.data ?? []) as Json[];
|
||||
let pageSeedingSize = 0;
|
||||
const pageSeedingInfo: Array<[number, number]> = [];
|
||||
for (const info of torrents) {
|
||||
const size = Number(info.size ?? 0);
|
||||
const seeder = Number(info.seeders || 1);
|
||||
pageSeedingSize += size;
|
||||
pageSeedingInfo.push([seeder, size]);
|
||||
}
|
||||
this.seeding += torrents.length;
|
||||
this.seedingSize += pageSeedingSize;
|
||||
this.seedingInfo.push(...pageSeedingInfo);
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
|
||||
/** API 仅返回未读数量,正文需通过 NexusPHP 网页接口读取(需 Cookie) */
|
||||
protected override async _paseUnreadMsgs(): Promise<void> {
|
||||
if (!this._siteCookie) return;
|
||||
const nexus = new NexusPhpSiteUserInfo({ ...this._opts, cookie: this._siteCookie });
|
||||
nexus.messageUnread = this.messageUnread;
|
||||
nexus.messageReadForce = this.messageUnread > 0;
|
||||
await (nexus as unknown as { _paseUnreadMsgs(): Promise<void> })._paseUnreadMsgs();
|
||||
this.messageUnreadContents = [...nexus.messageUnreadContents];
|
||||
this.httpErrors.push(...nexus.httpErrors);
|
||||
if (this.messageUnreadContents.length && !this.messageUnread) {
|
||||
this.messageUnread = this.messageUnreadContents.length;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
export { SiteParserBase, type SiteParserClass, urljoin } from "./base.ts";
|
||||
export { BitptSiteUserInfo } from "./bitpt.ts";
|
||||
export {
|
||||
containsChinese,
|
||||
extractDomain,
|
||||
formatTimestamp,
|
||||
isLoggedIn,
|
||||
normalizeDatetime,
|
||||
parseFloatMp,
|
||||
parseIntMp,
|
||||
underChallenge,
|
||||
} from "./common.ts";
|
||||
export { DiscuzUserInfo } from "./discuz.ts";
|
||||
export { hasChildElements, parseHtml, XNode } from "./dom.ts";
|
||||
export { FileListSiteUserInfo } from "./file-list.ts";
|
||||
export { GazelleSiteUserInfo } from "./gazelle.ts";
|
||||
export { HDDolbySiteUserInfo } from "./hddolby.ts";
|
||||
export { IptSiteUserInfo } from "./ipt.ts";
|
||||
export { MTorrentSiteUserInfo } from "./mtorrent.ts";
|
||||
export { NexusAudiencesSiteUserInfo } from "./nexus-audiences.ts";
|
||||
export { NexusHhanclubSiteUserInfo } from "./nexus-hhanclub.ts";
|
||||
export { NexusPhpSiteUserInfo } from "./nexus-php.ts";
|
||||
export { NexusProjectSiteUserInfo } from "./nexus-project.ts";
|
||||
export { NexusRabbitSiteUserInfo } from "./nexus-rabbit.ts";
|
||||
export {
|
||||
getUserParserClass,
|
||||
refreshSiteUserData,
|
||||
type SiteUserDataRequest,
|
||||
USER_PARSER_CLASSES,
|
||||
} from "./registry.ts";
|
||||
export { RousiSiteUserInfo } from "./rousi.ts";
|
||||
export { SmallHorseSiteUserInfo } from "./small-horse.ts";
|
||||
export { SunnyPTSiteUserInfo } from "./sunnypt.ts";
|
||||
export { TNodeSiteUserInfo } from "./tnode.ts";
|
||||
export { TorrentLeechSiteUserInfo } from "./torrent-leech.ts";
|
||||
export {
|
||||
SiteSchema,
|
||||
type SiteSchemaValue,
|
||||
type SiteUnreadMessage,
|
||||
type SiteUserInfo,
|
||||
type SiteUserParserOptions,
|
||||
} from "./types.ts";
|
||||
export { Unit3dSiteUserInfo } from "./unit3d.ts";
|
||||
export { YemaSiteUserInfo } from "./yema.ts";
|
||||
export { ZhixingSiteUserInfo } from "./zhixing.ts";
|
||||
@@ -0,0 +1,31 @@
|
||||
import { createMockFetch, IPT_USER_DETAIL, IPT_USER_INDEX } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { IptSiteUserInfo } from "./ipt.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
describe("IptSiteUserInfo(MP ipt.py)", () => {
|
||||
it("统计块:分享率/上传/下载/做种/下载中/积分;详情页等级与加入日期", async () => {
|
||||
const m = createMockFetch({ "/": { body: IPT_USER_INDEX }, "/user.php": { body: IPT_USER_DETAIL } });
|
||||
const info = await runParser(IptSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info).toMatchObject({
|
||||
schema: "IPTorrents",
|
||||
userId: "777",
|
||||
username: "dave",
|
||||
ratio: 2.5,
|
||||
upload: Math.round(1.2 * 1024 ** 4),
|
||||
download: 480 * 1024 ** 3,
|
||||
seeding: 15,
|
||||
leeching: 3,
|
||||
bonus: 4321.5,
|
||||
userLevel: "Power User",
|
||||
joinAt: "2018-01-02 03:04:05",
|
||||
});
|
||||
expect(m.requests.find((r) => r.url.pathname === "/user.php")!.url.search).toBe("?u=777");
|
||||
});
|
||||
|
||||
it("统计块结构缺列时与 MP 一样抛 IndexError", async () => {
|
||||
const broken = IPT_USER_INDEX.replace("<span>480.00 GB</span>", "");
|
||||
const m = createMockFetch({ "/": { body: broken } });
|
||||
await expect(runParser(IptSiteUserInfo, { fetch: m.fetch })).rejects.toThrow(/IndexError/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,101 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/ipt.py @ e6820a8
|
||||
*
|
||||
* IPTorrents 站点用户数据解析器(MP `IptSiteUserInfo`)。
|
||||
* 函数对照:`_parse_user_base_info` → `_parseUserBaseInfo`、`_parse_site_page`(空)、
|
||||
* `_parse_user_detail_info` → `_parseUserDetailInfo`、`_parse_user_torrent_seeding_info` → `_parseUserTorrentSeedingInfo`、
|
||||
* 其余为空实现。XPath 逐字保留;统计块下标越界时与 MP 一样抛 IndexError。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase } from "./base.ts";
|
||||
import { normalizeDatetime, parseFloatMp, parseIntMp, reSearch } from "./common.ts";
|
||||
import { hasChildElements, parseHtml } from "./dom.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
/** Python 列表下标(越界抛 IndexError,保持 MP 失败语义) */
|
||||
function at<T>(arr: T[], i: number): T {
|
||||
const v = i < 0 ? arr[arr.length + i] : arr[i];
|
||||
if (v === undefined) throw new Error("IndexError: list index out of range");
|
||||
return v;
|
||||
}
|
||||
|
||||
export class IptSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.Ipt;
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
const html = parseHtml(this._prepareHtmlText(htmlText));
|
||||
if (!html) return;
|
||||
const names = html.strings('//a[contains(@href, "/u/")]//text()');
|
||||
const ids = html.strings('//a[contains(@href, "/u/")]/@href');
|
||||
if (names.length) this.username = String(at(names, -1));
|
||||
if (ids.length) {
|
||||
const m = reSearch(/\/u\/(\d+)/, ids[0]);
|
||||
if (m?.[0].trim()) {
|
||||
this.userid = m[1]!;
|
||||
this._userDetailPage = `user.php?u=${this.userid}`;
|
||||
this._torrentSeedingPage = `peers?u=${this.userid}`;
|
||||
}
|
||||
}
|
||||
const stats = html.nodes('//div[@class = "stats"]/div/div');
|
||||
if (stats.length) {
|
||||
const s = stats[0]!;
|
||||
const spans = s.strings("span/text()");
|
||||
const links = s.nodes("a");
|
||||
this.upload = this.numFilesize(String(at(spans, 1)).trim());
|
||||
this.download = this.numFilesize(String(at(spans, 2)).trim());
|
||||
this.seeding = parseIntMp(at(at(links, 2).strings("text()"), 0));
|
||||
this.leeching = parseIntMp(at(at(links, 2).strings("text()"), 1));
|
||||
this.ratio = parseFloatMp(String(at(spans, 0)).trim().replaceAll("-", "0"));
|
||||
this.bonus = parseFloatMp(at(at(links, 3).strings("text()"), 0));
|
||||
}
|
||||
}
|
||||
|
||||
protected _parseSitePage(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserDetailInfo(htmlText: string): void {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
const levels = html.strings('//tr/th[text()="Class"]/following-sibling::td[1]/text()');
|
||||
if (levels.length) this.userLevel = levels[0]!.trim();
|
||||
const joinAt = html.strings('//tr/th[text()="Join date"]/following-sibling::td[1]/text()');
|
||||
if (joinAt.length) this.joinAt = normalizeDatetime(joinAt[0]!.split(" (")[0]) ?? null;
|
||||
}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(htmlText: string, _multiPage = false): NextPage {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return null;
|
||||
let seedingEndPos = 3;
|
||||
if (html.nodes('//tr/td[text() = "Leechers"]').length) {
|
||||
seedingEndPos = html.nodes('//tr/td[text() = "Leechers"]/../preceding-sibling::tr').length + 1;
|
||||
seedingEndPos = seedingEndPos - 3;
|
||||
}
|
||||
let pageSeeding = 0;
|
||||
let pageSeedingSize = 0;
|
||||
const torrents = html.strings('//tr/td[text() = "Seeders"]/../following-sibling::tr/td[position()=6]/text()');
|
||||
if (torrents.length) {
|
||||
pageSeeding = seedingEndPos;
|
||||
// Python 切片 [:n](n 为负时从尾部截)
|
||||
for (let perSize of torrents.slice(0, seedingEndPos)) {
|
||||
if (perSize.includes("(") && perSize.includes(")")) {
|
||||
perSize = perSize.split("(").at(-1)!;
|
||||
perSize = perSize.split(")")[0]!;
|
||||
}
|
||||
pageSeedingSize += this.numFilesize(perSize);
|
||||
}
|
||||
}
|
||||
this.seeding = pageSeeding;
|
||||
this.seedingSize = pageSeedingSize;
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import {
|
||||
createMockFetch,
|
||||
MTORRENT_USER_KEY_INVALID,
|
||||
MTORRENT_USER_MESSAGES,
|
||||
MTORRENT_USER_PEER_STATUS,
|
||||
MTORRENT_USER_PROFILE,
|
||||
MTORRENT_USER_SEEDING_PAGE1,
|
||||
MTORRENT_USER_SEEDING_PAGE2,
|
||||
} from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { MTorrentSiteUserInfo } from "./mtorrent.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
const API = "https://api.m-team.cc";
|
||||
|
||||
function routes() {
|
||||
return createMockFetch({
|
||||
[`POST ${API}/api/member/profile`]: { body: MTORRENT_USER_PROFILE },
|
||||
[`POST ${API}/api/member/getUserTorrentList`]: (req) => ({
|
||||
body: JSON.parse(req.body ?? "{}").pageNumber === 1 ? MTORRENT_USER_SEEDING_PAGE1 : MTORRENT_USER_SEEDING_PAGE2,
|
||||
}),
|
||||
[`POST ${API}/api/tracker/myPeerStatus`]: { body: MTORRENT_USER_PEER_STATUS },
|
||||
[`POST ${API}/api/msg/search`]: { body: MTORRENT_USER_MESSAGES },
|
||||
[`POST ${API}/api/msg/markRead`]: { body: { code: "0" } },
|
||||
});
|
||||
}
|
||||
|
||||
describe("MTorrentSiteUserInfo(MP mtorrent.py)", () => {
|
||||
it("profile + 做种分页(按 myPeerStatus 总数翻页);默认不读消息、不标记已读", async () => {
|
||||
const m = routes();
|
||||
const info = await runParser(MTorrentSiteUserInfo, {
|
||||
fetch: m.fetch,
|
||||
url: "https://kp.m-team.cc/",
|
||||
apikey: "KEY-secret",
|
||||
});
|
||||
expect(info).toMatchObject({
|
||||
schema: "MTorrent",
|
||||
domain: "m-team.cc",
|
||||
userId: "31337",
|
||||
username: "henry",
|
||||
userLevel: "Veteran User",
|
||||
joinAt: "2020-10-11 12:13:14",
|
||||
upload: 3000,
|
||||
download: 1000,
|
||||
ratio: 3,
|
||||
bonus: 4567.8,
|
||||
seeding: 3,
|
||||
seedingSize: 600,
|
||||
seedingInfo: [
|
||||
[5, 100],
|
||||
[2, 200],
|
||||
[1, 300],
|
||||
],
|
||||
messageUnreadContents: [],
|
||||
});
|
||||
const profile = m.requests[0]!;
|
||||
expect(profile.url.href).toBe(`${API}/api/member/profile`);
|
||||
expect(profile.headers["x-api-key"]).toBe("KEY-secret");
|
||||
expect(m.requests.filter((r) => r.url.pathname === "/api/member/getUserTorrentList")).toHaveLength(2);
|
||||
expect(m.requests.some((r) => r.url.pathname === "/api/msg/markRead")).toBe(false);
|
||||
expect(JSON.stringify(info)).not.toContain("KEY-secret");
|
||||
});
|
||||
|
||||
it("fetchMessages=true:读出未读消息并逐条 markRead(写操作)", async () => {
|
||||
const m = routes();
|
||||
const info = await runParser(MTorrentSiteUserInfo, {
|
||||
fetch: m.fetch,
|
||||
url: "https://kp.m-team.cc/",
|
||||
apikey: "k",
|
||||
fetchMessages: true,
|
||||
});
|
||||
expect(info.messageUnreadContents).toEqual([{ head: "欢迎", date: "2026-09-01 10:00:00", content: "hello" }]);
|
||||
// 与 MP 一致:消息直接写入 contents,不经 msg_links,故 messageUnread 不回填
|
||||
expect(info.messageUnread).toBe(0);
|
||||
const mark = m.requests.filter((r) => r.url.pathname === "/api/msg/markRead");
|
||||
expect(mark).toHaveLength(1);
|
||||
expect(mark[0]!.body).toBe("msgId=m1");
|
||||
});
|
||||
|
||||
it("key 无效(业务 code≠0):不填用户数据;未配置 key 时不发请求", async () => {
|
||||
const m = createMockFetch({ [`POST ${API}/api/member/profile`]: { body: MTORRENT_USER_KEY_INVALID } });
|
||||
const info = await runParser(MTorrentSiteUserInfo, { fetch: m.fetch, url: "https://kp.m-team.cc/", apikey: "bad" });
|
||||
expect(info.userId).toBeNull();
|
||||
expect(info.upload).toBe(0);
|
||||
|
||||
const m2 = createMockFetch({});
|
||||
const info2 = await runParser(MTorrentSiteUserInfo, { fetch: m2.fetch, url: "https://kp.m-team.cc/" });
|
||||
expect(m2.requests).toHaveLength(0);
|
||||
expect(info2.errMsg).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/mtorrent.py @ e6820a8
|
||||
*
|
||||
* M-Team(mTorrent API)用户数据解析器(MP `MTorrentSiteUserInfo`,apikey 模式)。
|
||||
* 函数对照:`MTeam_sysRoleList` → `MTEAM_SYS_ROLE_LIST`、`_parse_site_page` → `_parseSitePage`(改用 api.<域> 接口)、
|
||||
* `_parse_logged_in` → `_parseLoggedIn`(恒 True)、`_parse_user_base_info` → `_parseUserBaseInfo`(api/member/profile)、
|
||||
* `_parse_user_torrent_seeding_info` → `_parseUserTorrentSeedingInfo`(api/member/getUserTorrentList 分页,
|
||||
* 以 api/tracker/myPeerStatus 的 seeder 总数判断是否还有下一页)、
|
||||
* `_parse_message_unread_links` → `_parseMessageUnreadLinks`(api/msg/search,直接读出正文)。
|
||||
*
|
||||
* 注意(写操作):与 MP 一样,读取未读消息后会调用 `api/msg/markRead` 把消息标记为已读。
|
||||
* 该流程只在 `fetchMessages=true`(MP `SITE_MESSAGE`)时执行,默认不触发。
|
||||
* API Key 只放在 `x-api-key` 请求头,不进入结果。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase, urljoin } from "./base.ts";
|
||||
import { extractDomain, pyInt } from "./common.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
/** 用户级别字典(MP `MTeam_sysRoleList`) */
|
||||
export const MTEAM_SYS_ROLE_LIST: Record<string, string> = {
|
||||
"1": "User",
|
||||
"2": "Power User",
|
||||
"3": "Elite User",
|
||||
"4": "Crazy User",
|
||||
"5": "Insane User",
|
||||
"6": "Veteran User",
|
||||
"7": "Extreme User",
|
||||
"8": "Ultimate User",
|
||||
"9": "Nexus Master",
|
||||
"10": "VIP",
|
||||
"11": "Retiree",
|
||||
"12": "Uploader",
|
||||
"13": "Moderator",
|
||||
"14": "Administrator",
|
||||
"15": "Sysop",
|
||||
"16": "Staff",
|
||||
"17": "Offer memberStaff",
|
||||
"18": "Bet memberStaff",
|
||||
};
|
||||
|
||||
export class MTorrentSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.MTorrent;
|
||||
override requestMode = "apikey" as const;
|
||||
|
||||
protected _parseSitePage(_htmlText: string): void {
|
||||
// 更换 api 地址
|
||||
this._baseUrl = `https://api.${extractDomain(this._baseUrl)}`;
|
||||
this._userTrafficPage = null;
|
||||
this._userDetailPage = null;
|
||||
this._userBasicPage = "api/member/profile";
|
||||
this._userBasicParams = { uid: this.userid };
|
||||
this._sysMailUnreadPage = null;
|
||||
this._userMailUnreadPage = "api/msg/search";
|
||||
this._mailUnreadParams = { keyword: "", box: "-2", type: "pageNumber", pageSize: 100 };
|
||||
this._torrentSeedingPage = "api/member/getUserTorrentList";
|
||||
this._torrentSeedingHeaders = {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json, text/plain, */*",
|
||||
};
|
||||
this._additionHeaders = { "x-api-key": this.apikey ?? "" };
|
||||
}
|
||||
|
||||
/** 暂时跳过检测(与 MP 一致) */
|
||||
protected override _parseLoggedIn(_htmlText: unknown): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
/** 基础信息:合并了流量与详情解析 */
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
if (!htmlText) return;
|
||||
const detail = JSON.parse(htmlText) as Json | null;
|
||||
if (detail?.code !== "0") return;
|
||||
const u = (detail.data ?? {}) as Json;
|
||||
const memberCount = (u.memberCount ?? {}) as Json;
|
||||
this.userid = u.id == null ? null : String(u.id);
|
||||
this.username = (u.username as string) ?? null;
|
||||
this.userLevel = MTEAM_SYS_ROLE_LIST[String(u.role || "1")] ?? null;
|
||||
this.joinAt = ((u.memberStatus ?? {}) as Json).createdDate as string | null;
|
||||
this.upload = pyInt(memberCount.uploaded || "0");
|
||||
this.download = pyInt(memberCount.downloaded || "0");
|
||||
this.ratio = (memberCount.shareRate as number) || 0;
|
||||
this.bonus = (memberCount.bonus as number) || 0;
|
||||
this.messageReadForce = true;
|
||||
this._torrentSeedingParams = { pageNumber: 1, pageSize: 200, type: "SEEDING", userid: this.userid };
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserDetailInfo(_htmlText: string): void {}
|
||||
|
||||
protected async _parseUserTorrentSeedingInfo(htmlText: string, _multiPage = false): Promise<NextPage> {
|
||||
if (!htmlText) return null;
|
||||
const seedingInfo = JSON.parse(htmlText) as Json | null;
|
||||
if (seedingInfo?.code !== "0") return null;
|
||||
const seedingData = seedingInfo.data || {};
|
||||
if (typeof seedingData !== "object" || Array.isArray(seedingData)) return null;
|
||||
const torrents = (seedingData as Json).data || [];
|
||||
if (!Array.isArray(torrents)) return null;
|
||||
let pageSeeding = 0;
|
||||
let pageSeedingSize = 0;
|
||||
const pageSeedingInfo: Array<[number, number]> = [];
|
||||
for (const info of torrents as unknown[]) {
|
||||
if (!info || typeof info !== "object" || Array.isArray(info)) continue;
|
||||
const torrent = (info as Json).torrent;
|
||||
if (!torrent || typeof torrent !== "object" || Array.isArray(torrent)) continue;
|
||||
const size = pyInt((torrent as Json).size || "0");
|
||||
const seeders = pyInt((torrent as Json).source || "0");
|
||||
pageSeeding += 1;
|
||||
pageSeedingSize += size;
|
||||
pageSeedingInfo.push([seeders, size]);
|
||||
}
|
||||
this.seeding += pageSeeding;
|
||||
this.seedingSize += pageSeedingSize;
|
||||
this.seedingInfo.push(...pageSeedingInfo);
|
||||
|
||||
// 查询总做种数
|
||||
let seederCount = 0;
|
||||
try {
|
||||
const result = await this._getPageContent(urljoin(this._baseUrl, "api/tracker/myPeerStatus"), {
|
||||
uid: this.userid,
|
||||
});
|
||||
if (result) {
|
||||
const seederInfo = JSON.parse(result) as Json;
|
||||
seederCount = pyInt(((seederInfo.data ?? {}) as Json).seeder || 0);
|
||||
}
|
||||
} catch {
|
||||
// MP:记录日志后按 0 处理
|
||||
}
|
||||
if (!seederCount) return null;
|
||||
if (this.seeding >= seederCount) return null;
|
||||
// 还有下一页
|
||||
(this._torrentSeedingParams as Json).pageNumber = Number((this._torrentSeedingParams as Json).pageNumber) + 1;
|
||||
return "";
|
||||
}
|
||||
|
||||
/** 解析未读消息链接,这里直接读出详情(并标记已读,见文件头) */
|
||||
protected async _parseMessageUnreadLinks(htmlText: string, _msgLinks: string[]): Promise<NextPage> {
|
||||
if (!htmlText) return null;
|
||||
const messagesInfo = JSON.parse(htmlText) as Json | null;
|
||||
if (messagesInfo?.code !== "0") return null;
|
||||
const messages = (((messagesInfo.data ?? {}) as Json).data ?? []) as Json[];
|
||||
for (const message of messages) {
|
||||
if (!message.unread) continue;
|
||||
const head = message.title as string | null;
|
||||
const date = message.createdDate as string | null;
|
||||
const content = message.context as string | null;
|
||||
if (head && date && content) {
|
||||
this.messageUnreadContents.push([head, date, content]);
|
||||
// 设置已读
|
||||
await this._getPageContent(urljoin(this._baseUrl, "api/msg/markRead"), { msgId: message.id });
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import {
|
||||
AUDIENCES_USER_DETAILS,
|
||||
AUDIENCES_USER_INDEX,
|
||||
AUDIENCES_USER_MAILBOX,
|
||||
AUDIENCES_USER_MESSAGE_901,
|
||||
AUDIENCES_USER_SEEDING,
|
||||
createMockFetch,
|
||||
NEXUS_PHP_LOGIN_PAGE,
|
||||
} from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { NexusAudiencesSiteUserInfo } from "./nexus-audiences.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
const GiB = 1024 ** 3;
|
||||
const TiB = 1024 ** 4;
|
||||
|
||||
function routes() {
|
||||
return createMockFetch({
|
||||
"/": { body: AUDIENCES_USER_INDEX },
|
||||
"/userdetails.php": { body: AUDIENCES_USER_DETAILS },
|
||||
"/getusertorrentlistajax.php": { body: AUDIENCES_USER_SEEDING },
|
||||
"/messages.php": { body: AUDIENCES_USER_MAILBOX },
|
||||
"/messageviewer.php?id=901": { body: AUDIENCES_USER_MESSAGE_901 },
|
||||
});
|
||||
}
|
||||
|
||||
describe("NexusAudiencesSiteUserInfo(MP nexus_audiences.py)", () => {
|
||||
it("userbar data-uploader-* + compact metric + Total 行汇总做种", async () => {
|
||||
const m = routes();
|
||||
const info = await runParser(NexusAudiencesSiteUserInfo, { fetch: m.fetch, cookie: "c_secure=abc" });
|
||||
expect(info).toMatchObject({
|
||||
schema: "NexusAudiences",
|
||||
userId: "233",
|
||||
username: "bob",
|
||||
userLevel: "Elite User",
|
||||
joinAt: "2020-05-06 07:08:09",
|
||||
upload: 2 * TiB,
|
||||
download: 100 * GiB,
|
||||
ratio: 20.48,
|
||||
bonus: 8888.5,
|
||||
leeching: 1,
|
||||
seeding: 31,
|
||||
seedingSize: Math.round(3.5 * TiB),
|
||||
messageUnread: 2,
|
||||
messageUnreadContents: [],
|
||||
});
|
||||
const seedReq = m.requests.find((r) => r.url.pathname === "/getusertorrentlistajax.php")!;
|
||||
expect(seedReq.headers.referer).toBe("https://pt.example.com/userdetails.php?id=233");
|
||||
expect(m.requests.some((r) => r.url.pathname === "/messages.php")).toBe(false);
|
||||
});
|
||||
|
||||
it("读取未读私信:首页不带 page、详情解析、详情为空用列表预览兜底、达到未读数停止翻页", async () => {
|
||||
const m = routes();
|
||||
const info = await runParser(NexusAudiencesSiteUserInfo, { fetch: m.fetch, fetchMessages: true });
|
||||
const mailReqs = m.requests.filter((r) => r.url.pathname === "/messages.php");
|
||||
expect(mailReqs.map((r) => r.url.search)).toEqual(["?action=viewmailbox&box=1&unread=yes"]);
|
||||
expect(info.messageUnreadContents).toEqual([
|
||||
{ head: "种子被删除", date: "2026-09-01 10:00:00", content: "你的种子 X 因重复已被删除。" },
|
||||
{ head: "欢迎", date: "2026-09-02 11:00:00", content: "欢迎加入观众" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("未登录:只请求首页", async () => {
|
||||
const m = createMockFetch({ "/": { body: NEXUS_PHP_LOGIN_PAGE } });
|
||||
const info = await runParser(NexusAudiencesSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info.userId).toBeNull();
|
||||
expect(m.requests).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("收件箱入口解析:总数/未读、角标、仅未读标记", () => {
|
||||
expect(NexusAudiencesSiteUserInfo.extractInboxUnreadPair("收件箱 1,749 / 172")).toBe(172);
|
||||
expect(NexusAudiencesSiteUserInfo.extractInboxUnreadBadge(" 5 ")).toBe(5);
|
||||
expect(NexusAudiencesSiteUserInfo.extractInboxUnreadBadge("新")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,401 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/nexus_audiences.py @ e6820a8
|
||||
*
|
||||
* 观众(Audiences)站点用户信息解析器,继承 NexusPHP 通用解析。
|
||||
*
|
||||
* 函数对照(`NexusAudiencesSiteUserInfo`):
|
||||
* `__init__` → constructor、`_parse_message_unread` → `_parseMessageUnread`、
|
||||
* `_parse_message_unread_links` → `_parseMessageUnreadLinks`、`_parse_message_content` → `_parseMessageContent`、
|
||||
* `_pase_unread_msgs` → `_paseUnreadMsgs`、`__parse_unread_message_list_page` → `parseUnreadMessageListPage`、
|
||||
* `__reset_unread_message_parse_state` → `resetUnreadMessageParseState`、`__filter_new_message_links` → `filterNewMessageLinks`、
|
||||
* `__parse_table_unread_message_links` → `parseTableUnreadMessageLinks`、`__parse_pm_item_unread_message_links` →
|
||||
* `parsePmItemUnreadMessageLinks`、`__cache_pm_item_preview` → `cachePmItemPreview`、`__extract_pm_item_text` /
|
||||
* `__extract_first_text` / `__extract_pm_view_meta` / `__normalize_text` → 同名、`__fill_empty_message_content_from_list` →
|
||||
* `fillEmptyMessageContentFromList`、`__should_fetch_next_unread_page` / `__has_reached_expected_unread_count` /
|
||||
* `__is_empty_message_content` / `__build_unread_mailbox_page` / `__build_next_unread_mailbox_page` → 同名、
|
||||
* `_parse_user_traffic_info` / `_parse_user_detail_info` → 同名(追加 `__parse_userbar_info`)、
|
||||
* `__parse_user_identity` / `__parse_uploader_stats` / `__parse_compact_metric` / `__set_metric_value` /
|
||||
* `__parse_inbox_unread` / `__extract_inbox_unread_pair` / `__extract_inbox_unread_badge` /
|
||||
* `__has_inbox_unread_marker` → 同名、`_parse_seeding_pages` → `_parseSeedingPages`(Total 行汇总)。
|
||||
* XPath 与正则逐字保留。
|
||||
*/
|
||||
import { urljoin } from "./base.ts";
|
||||
import { parseFloatMp, parseIntMp, reSearch } from "./common.ts";
|
||||
import { hasChildElements, parseHtml, type XNode } from "./dom.ts";
|
||||
import { NexusPhpSiteUserInfo } from "./nexus-php.ts";
|
||||
import { SiteSchema, type SiteSchemaValue, type SiteUserParserOptions } from "./types.ts";
|
||||
|
||||
type Msg = [string | null, string | null, string | null];
|
||||
|
||||
const UNKNOWN_UNREAD_COUNT = 99999;
|
||||
const MAX_MAIL_PAGES = 50;
|
||||
|
||||
function normalizeText(text: string | null | undefined): string | null {
|
||||
if (!text) return null;
|
||||
const t = text.replaceAll(" ", " ").replace(/\s+/g, " ").trim();
|
||||
return t || null;
|
||||
}
|
||||
|
||||
export class NexusAudiencesSiteUserInfo extends NexusPhpSiteUserInfo {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.NexusAudiences;
|
||||
|
||||
private nextMailPage = 1;
|
||||
private readonly seenUnreadMessageLinks = new Set<string>();
|
||||
private readonly messageListPreviews = new Map<string, Msg>();
|
||||
|
||||
constructor(opts: SiteUserParserOptions) {
|
||||
super(opts);
|
||||
this._userMailUnreadPage = NexusAudiencesSiteUserInfo.buildUnreadMailboxPage(1);
|
||||
this._sysMailUnreadPage = null;
|
||||
}
|
||||
|
||||
/** 解析新版顶部用户栏中的未读消息数 */
|
||||
protected override _parseMessageUnread(htmlText: string): void {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) {
|
||||
super._parseMessageUnread(htmlText);
|
||||
return;
|
||||
}
|
||||
const messageTools = html.nodes(
|
||||
'//a[contains(@class, "site-userbar__compact-tool") and contains(@href, "messages.php") ' +
|
||||
'and (contains(@class, "site-userbar__compact-tool--has-unread") ' +
|
||||
'or .//*[contains(@class, "site-userbar__compact-tool-badge--unread")])]' +
|
||||
'|//a[contains(@href, "messages.php") ' +
|
||||
'and (contains(@title, "收件箱") or contains(@aria-label, "收件箱"))]',
|
||||
);
|
||||
for (const link of messageTools) {
|
||||
const unread = NexusAudiencesSiteUserInfo.parseInboxUnread(link);
|
||||
if (unread !== null) {
|
||||
this.messageUnread = unread;
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (messageTools.length) return;
|
||||
super._parseMessageUnread(htmlText);
|
||||
}
|
||||
|
||||
protected override _parseMessageUnreadLinks(htmlText: string, msgLinks: string[]): string | null {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return null;
|
||||
const messageLinks = NexusAudiencesSiteUserInfo.parseTableUnreadMessageLinks(html);
|
||||
messageLinks.push(...this.parsePmItemUnreadMessageLinks(html));
|
||||
const newLinks = this.filterNewMessageLinks(messageLinks);
|
||||
msgLinks.push(...newLinks);
|
||||
return this.buildNextUnreadMailboxPage(this.shouldFetchNextUnreadPage(newLinks));
|
||||
}
|
||||
|
||||
protected override _parseMessageContent(htmlText: string): Msg {
|
||||
const html = parseHtml(htmlText);
|
||||
if (hasChildElements(html)) {
|
||||
const head = NexusAudiencesSiteUserInfo.extractFirstText(
|
||||
html,
|
||||
'//*[contains(concat(" ", normalize-space(@class), " "), " pm-hero__title ")]',
|
||||
);
|
||||
const date = NexusAudiencesSiteUserInfo.extractPmViewMeta(html, "日期");
|
||||
const content = NexusAudiencesSiteUserInfo.extractFirstText(
|
||||
html,
|
||||
'//*[contains(concat(" ", normalize-space(@class), " "), " pm-view__body ")]',
|
||||
);
|
||||
if (!NexusAudiencesSiteUserInfo.isEmptyMessageContent(head, date, content)) return [head, date, content];
|
||||
}
|
||||
return super._parseMessageContent(htmlText);
|
||||
}
|
||||
|
||||
/** 避免异常分页重复通知和空详情通知 */
|
||||
protected override async _paseUnreadMsgs(): Promise<void> {
|
||||
this.resetUnreadMessageParseState();
|
||||
const unreadMsgLinks: string[] = [];
|
||||
if (this.messageUnread > 0 || this.messageReadForce) {
|
||||
let nextPage = await this.parseUnreadMessageListPage(this._userMailUnreadPage, unreadMsgLinks);
|
||||
let guard = 0;
|
||||
while (nextPage && guard++ < MAX_MAIL_PAGES) {
|
||||
nextPage = await this.parseUnreadMessageListPage(nextPage, unreadMsgLinks);
|
||||
}
|
||||
}
|
||||
if (this.messageUnread === UNKNOWN_UNREAD_COUNT) this.messageUnread = unreadMsgLinks.length;
|
||||
else if (unreadMsgLinks.length && !this.messageUnread) this.messageUnread = unreadMsgLinks.length;
|
||||
for (const msgLink of unreadMsgLinks) {
|
||||
let [head, date, content] = this._parseMessageContent(
|
||||
await this._getPageContent(urljoin(this._baseUrl, msgLink), this._mailContentParams, this._mailContentHeaders),
|
||||
);
|
||||
[head, date, content] = this.fillEmptyMessageContentFromList(msgLink, head, date, content);
|
||||
if (NexusAudiencesSiteUserInfo.isEmptyMessageContent(head, date, content)) continue;
|
||||
this.messageUnreadContents.push([head, date, content]);
|
||||
}
|
||||
}
|
||||
|
||||
private async parseUnreadMessageListPage(link: string | null, unreadMsgLinks: string[]): Promise<string | null> {
|
||||
if (!link) return null;
|
||||
return this._parseMessageUnreadLinks(
|
||||
await this._getPageContent(urljoin(this._baseUrl, link), this._mailUnreadParams, this._mailUnreadHeaders),
|
||||
unreadMsgLinks,
|
||||
);
|
||||
}
|
||||
|
||||
private resetUnreadMessageParseState(): void {
|
||||
this.nextMailPage = 1;
|
||||
this.seenUnreadMessageLinks.clear();
|
||||
this.messageListPreviews.clear();
|
||||
}
|
||||
|
||||
private filterNewMessageLinks(messageLinks: string[]): string[] {
|
||||
const out: string[] = [];
|
||||
for (const link of messageLinks) {
|
||||
const key = urljoin(this._baseUrl, link);
|
||||
if (this.seenUnreadMessageLinks.has(key)) continue;
|
||||
this.seenUnreadMessageLinks.add(key);
|
||||
out.push(link);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static parseTableUnreadMessageLinks(html: XNode): string[] {
|
||||
return html.strings(
|
||||
'//tr[.//img[contains(concat(" ", normalize-space(@class), " "), " unreadpm ") ' +
|
||||
'or @alt="Unread" or @title="未读"]]/td/a[contains(@href, "viewmessage")]/@href',
|
||||
);
|
||||
}
|
||||
|
||||
private parsePmItemUnreadMessageLinks(html: XNode): string[] {
|
||||
const links: string[] = [];
|
||||
let rows = html.nodes(
|
||||
'//*[contains(concat(" ", normalize-space(@class), " "), " pm-item-row ") ' +
|
||||
'and contains(concat(" ", normalize-space(@class), " "), " is-unread ")]',
|
||||
);
|
||||
if (!rows.length) {
|
||||
rows = html.nodes(
|
||||
'//*[contains(concat(" ", normalize-space(@class), " "), " pm-item-row ") ' +
|
||||
'and .//*[contains(concat(" ", normalize-space(@class), " "), " pm-item__status--unread ") ' +
|
||||
'or @title="未读"]]',
|
||||
);
|
||||
}
|
||||
for (const row of rows) {
|
||||
const rowLinks = row.strings('.//a[contains(@href, "viewmessage")]/@href');
|
||||
if (!rowLinks.length) continue;
|
||||
const link = rowLinks[0]!.trim();
|
||||
if (!link) continue;
|
||||
links.push(link);
|
||||
this.cachePmItemPreview(link, row);
|
||||
}
|
||||
return links;
|
||||
}
|
||||
|
||||
private cachePmItemPreview(link: string, row: XNode): void {
|
||||
const f = (cls: string) =>
|
||||
NexusAudiencesSiteUserInfo.extractPmItemText(
|
||||
row,
|
||||
`.//*[contains(concat(" ", normalize-space(@class), " "), " ${cls} ")]`,
|
||||
);
|
||||
this.messageListPreviews.set(urljoin(this._baseUrl, link), [
|
||||
f("pm-item__subject"),
|
||||
f("pm-item__time"),
|
||||
f("pm-item__preview"),
|
||||
]);
|
||||
}
|
||||
|
||||
static extractPmItemText(row: XNode, xp: string): string | null {
|
||||
const nodes = row.nodes(xp);
|
||||
if (!nodes.length) return null;
|
||||
return normalizeText(nodes[0]!.text());
|
||||
}
|
||||
|
||||
static extractFirstText(html: XNode, xp: string): string | null {
|
||||
const nodes = html.nodes(xp);
|
||||
if (!nodes.length) return null;
|
||||
return normalizeText(nodes[0]!.text());
|
||||
}
|
||||
|
||||
static extractPmViewMeta(html: XNode, label: string): string | null {
|
||||
const values = html.nodes(
|
||||
'//*[contains(concat(" ", normalize-space(@class), " "), " pm-view__meta ") ' +
|
||||
`and .//*[contains(concat(" ", normalize-space(@class), " "), " pm-view__label ") ` +
|
||||
`and normalize-space()="${label}"]]` +
|
||||
'//*[contains(concat(" ", normalize-space(@class), " "), " pm-view__value ")]',
|
||||
);
|
||||
if (!values.length) return null;
|
||||
return normalizeText(values[0]!.text());
|
||||
}
|
||||
|
||||
private fillEmptyMessageContentFromList(
|
||||
msgLink: string,
|
||||
head: string | null,
|
||||
date: string | null,
|
||||
content: string | null,
|
||||
): Msg {
|
||||
const preview = this.messageListPreviews.get(urljoin(this._baseUrl, msgLink));
|
||||
if (!preview) return [head, date, content];
|
||||
return [head || preview[0], date || preview[1], content || preview[2]];
|
||||
}
|
||||
|
||||
private shouldFetchNextUnreadPage(newLinks: string[]): boolean {
|
||||
if (!newLinks.length) return false;
|
||||
return !this.hasReachedExpectedUnreadCount();
|
||||
}
|
||||
|
||||
private hasReachedExpectedUnreadCount(): boolean {
|
||||
return (
|
||||
!this.messageReadForce &&
|
||||
this.messageUnread > 0 &&
|
||||
this.messageUnread !== UNKNOWN_UNREAD_COUNT &&
|
||||
this.seenUnreadMessageLinks.size >= this.messageUnread
|
||||
);
|
||||
}
|
||||
|
||||
static isEmptyMessageContent(head: string | null, date: string | null, content: string | null): boolean {
|
||||
return ![head, date, content].some((x) => x != null && String(x).trim() !== "");
|
||||
}
|
||||
|
||||
static buildUnreadMailboxPage(box: number): string {
|
||||
return `messages.php?action=viewmailbox&box=${box}&unread=yes`;
|
||||
}
|
||||
|
||||
/** 首页不能带 page 参数;之后 page=1,2,... */
|
||||
private buildNextUnreadMailboxPage(hasUnread: boolean): string | null {
|
||||
if (!hasUnread) return null;
|
||||
const next = this.nextMailPage;
|
||||
this.nextMailPage += 1;
|
||||
return `${this._userMailUnreadPage}&page=${next}`;
|
||||
}
|
||||
|
||||
protected override _parseUserTrafficInfo(htmlText: string): void {
|
||||
super._parseUserTrafficInfo(htmlText);
|
||||
this.parseUserbarInfo(htmlText);
|
||||
}
|
||||
|
||||
protected override _parseUserDetailInfo(htmlText: string): void {
|
||||
super._parseUserDetailInfo(htmlText);
|
||||
this.parseUserbarInfo(htmlText);
|
||||
}
|
||||
|
||||
/** 新版顶部用户栏,覆盖 NexusPHP 通用正则的误判 */
|
||||
private parseUserbarInfo(htmlText: string): void {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
for (const node of html.nodes("//*[@data-uploader-url or @data-uploader-stats]")) {
|
||||
this.parseUserIdentity(node);
|
||||
this.parseUploaderStats(node.attr("data-uploader-stats"));
|
||||
}
|
||||
// data-uploader-stats 不包含分享率,需从 compact metric 的 class 中读取
|
||||
this.parseCompactMetric(html, "ratio", "ratio");
|
||||
this.parseCompactMetric(html, "uploaded", "upload");
|
||||
this.parseCompactMetric(html, "downloaded", "download");
|
||||
this.parseCompactMetric(html, "bonus", "bonus");
|
||||
this.parseCompactMetric(html, "active", "active");
|
||||
}
|
||||
|
||||
private parseUserIdentity(node: XNode): void {
|
||||
const userUrl = node.attr("data-uploader-url") ?? "";
|
||||
const m = reSearch(/userdetails\.php\?id=(\d+)/, userUrl);
|
||||
if (m?.[1]?.trim()) this.userid = m[1].trim();
|
||||
const username = node.attr("data-uploader-label");
|
||||
if (username?.trim()) this.username = username.trim();
|
||||
const level = node.attr("data-uploader-badge");
|
||||
if (level?.trim()) this.userLevel = level.trim();
|
||||
}
|
||||
|
||||
private parseUploaderStats(statsText: string | null): void {
|
||||
if (!statsText) return;
|
||||
let stats: unknown;
|
||||
try {
|
||||
stats = JSON.parse(statsText);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!Array.isArray(stats)) return;
|
||||
for (const item of stats) {
|
||||
if (!item || typeof item !== "object" || Array.isArray(item)) continue;
|
||||
const it = item as Record<string, unknown>;
|
||||
const label = String(it.label || "").replace(/^[ ：:]+|[ ：:]+$/g, "");
|
||||
const tone = String(it.tone || "").trim();
|
||||
const value = String(it.value || "").trim();
|
||||
this.setMetricValue(value, label, tone);
|
||||
}
|
||||
}
|
||||
|
||||
private parseCompactMetric(html: XNode, metric: string, field: string): void {
|
||||
let values = html.strings(
|
||||
`//*[contains(concat(" ", normalize-space(@class), " "), " site-userbar__compact-metric--${metric} ")]` +
|
||||
"//span[normalize-space()][last()]/text()",
|
||||
);
|
||||
if (!values.length) {
|
||||
values = html.strings(
|
||||
`//*[contains(concat(" ", normalize-space(@class), " "), " site-userbar__compact-metric--${metric} ")]` +
|
||||
"/text()",
|
||||
);
|
||||
}
|
||||
if (values.length) this.setMetricValue(values[values.length - 1]!.trim(), null, null, field);
|
||||
}
|
||||
|
||||
private setMetricValue(value: string, label?: string | null, tone?: string | null, field?: string | null): void {
|
||||
if (!value) return;
|
||||
const key = field || tone || label || "";
|
||||
if (["uploaded", "上传量", "upload"].includes(key)) this.upload = this.numFilesize(value);
|
||||
else if (["downloaded", "下载量", "download"].includes(key)) this.download = this.numFilesize(value);
|
||||
else if (["bonus", "爆米花"].includes(key)) this.bonus = parseFloatMp(value);
|
||||
else if (key === "ratio") this.ratio = parseFloatMp(value);
|
||||
else if (["active", "活跃"].includes(key)) {
|
||||
const m = reSearch(/↑\s*(\d+)\s*\/\s*↓\s*(\d+)/, value);
|
||||
if (m) {
|
||||
this.seeding = parseIntMp(m[1]);
|
||||
this.leeching = parseIntMp(m[2]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static parseInboxUnread(link: XNode): number | null {
|
||||
for (const t of [link.attr("title"), link.attr("aria-label")]) {
|
||||
const u = NexusAudiencesSiteUserInfo.extractInboxUnreadPair(t);
|
||||
if (u !== null) return u;
|
||||
}
|
||||
for (const t of link.strings('.//*[contains(@class, "site-userbar__compact-tool-badge--unread")]/text()')) {
|
||||
const u = NexusAudiencesSiteUserInfo.extractInboxUnreadBadge(t);
|
||||
if (u !== null) return u;
|
||||
}
|
||||
if (NexusAudiencesSiteUserInfo.hasInboxUnreadMarker(link)) return UNKNOWN_UNREAD_COUNT;
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 总数/未读数格式,如 1749/172 */
|
||||
static extractInboxUnreadPair(text: string | null | undefined): number | null {
|
||||
const t = normalizeText(text);
|
||||
if (!t) return null;
|
||||
const m = reSearch(/(?:收件箱\s*)?(\d[\d,]*)\s*\/\s*(\d[\d,]*)/, t);
|
||||
return m ? parseIntMp(m[2]) : null;
|
||||
}
|
||||
|
||||
static extractInboxUnreadBadge(text: string | null | undefined): number | null {
|
||||
const pair = NexusAudiencesSiteUserInfo.extractInboxUnreadPair(text);
|
||||
if (pair !== null) return pair;
|
||||
const t = normalizeText(text);
|
||||
if (!t) return null;
|
||||
const m = /^(\d[\d,]*)$/.exec(t);
|
||||
return m ? parseIntMp(m[1]) : null;
|
||||
}
|
||||
|
||||
static hasInboxUnreadMarker(link: XNode): boolean {
|
||||
if ((link.attr("class") ?? "").includes("site-userbar__compact-tool--has-unread")) return true;
|
||||
return link.nodes('.//*[contains(@class, "site-userbar__compact-tool-badge--unread")]').length > 0;
|
||||
}
|
||||
|
||||
/** 做种页只读 Total 汇总行 */
|
||||
protected override async _parseSeedingPages(): Promise<void> {
|
||||
if (!this._torrentSeedingPage) return;
|
||||
this._torrentSeedingHeaders = { Referer: urljoin(this._baseUrl, this._userDetailPage ?? "") };
|
||||
const htmlText = await this._getPageContent(
|
||||
urljoin(this._baseUrl, this._torrentSeedingPage),
|
||||
this._torrentSeedingParams,
|
||||
this._torrentSeedingHeaders,
|
||||
);
|
||||
if (!htmlText) return;
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
const totalRow = html.nodes('//table[@class="table table-bordered"]//tr[td[1][normalize-space()="Total"]]');
|
||||
if (!totalRow.length) return;
|
||||
const count = totalRow[0]!.strings("./td[2]/text()");
|
||||
const size = totalRow[0]!.strings("./td[3]/text()");
|
||||
this.seeding = count.length ? parseIntMp(count[0]) : 0;
|
||||
this.seedingSize = size.length ? this.numFilesize(size[0]!.trim()) : 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { createMockFetch, HHCLUB_USER_DETAILS, HHCLUB_USER_INDEX, HHCLUB_USER_INDEX_NO_PANEL } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { NexusHhanclubSiteUserInfo } from "./nexus-hhanclub.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
describe("NexusHhanclubSiteUserInfo(MP nexus_hhanclub.py)", () => {
|
||||
it("#user-info-panel 取上传/下载/分享率,详情取加入日期与 _Name 等级", async () => {
|
||||
const m = createMockFetch({ "/": { body: HHCLUB_USER_INDEX }, "/userdetails.php": { body: HHCLUB_USER_DETAILS } });
|
||||
const info = await runParser(NexusHhanclubSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info).toMatchObject({
|
||||
schema: "NexusHhanclub",
|
||||
userId: "4567",
|
||||
username: "hana",
|
||||
upload: Math.round(1.2 * 1024 ** 4),
|
||||
download: 300 * 1024 ** 3,
|
||||
ratio: 3.5,
|
||||
bonus: 6789,
|
||||
joinAt: "2022-02-02 02:02:02",
|
||||
userLevel: "Power User",
|
||||
});
|
||||
});
|
||||
|
||||
it("面板缺失(改版):与 MP 一样抛 IndexError", async () => {
|
||||
const m = createMockFetch({ "/": { body: HHCLUB_USER_INDEX_NO_PANEL } });
|
||||
await expect(runParser(NexusHhanclubSiteUserInfo, { fetch: m.fetch })).rejects.toThrow(/IndexError/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/nexus_hhanclub.py @ e6820a8
|
||||
*
|
||||
* 憨憨(HHClub)站点用户信息解析器,继承 NexusPHP 通用解析。
|
||||
*
|
||||
* 函数对照(`NexusHhanclubSiteUserInfo`):`_parse_user_traffic_info` → `_parseUserTrafficInfo`、
|
||||
* `_parse_user_detail_info` → `_parseUserDetailInfo`、`_get_user_level` → `_getUserLevel`。XPath 与正则逐字保留。
|
||||
*
|
||||
* 与 MP 一致:`#user-info-panel` 结构缺失时 `xpath(...)[0]` 抛错并中断整站解析(交给 schema 回退)。
|
||||
*/
|
||||
|
||||
import { normalizeDatetime, parseFloatMp, reSearch } from "./common.ts";
|
||||
import { hasChildElements, parseHtml, type XNode } from "./dom.ts";
|
||||
import { NexusPhpSiteUserInfo, round3 } from "./nexus-php.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
function first(values: string[], expr: string): string {
|
||||
if (!values.length) throw new Error(`IndexError: list index out of range (${expr})`);
|
||||
return values[0]!;
|
||||
}
|
||||
|
||||
export class NexusHhanclubSiteUserInfo extends NexusPhpSiteUserInfo {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.NexusHhanclub;
|
||||
|
||||
protected override _parseUserTrafficInfo(htmlText: string): void {
|
||||
super._parseUserTrafficInfo(htmlText);
|
||||
const text = this._prepareHtmlText(htmlText);
|
||||
const html = parseHtml(text);
|
||||
if (!html) throw new Error("AttributeError: 'NoneType' object has no attribute 'xpath'");
|
||||
const x = (e: string) => first(html.strings(e), e);
|
||||
// 上传、下载、分享率
|
||||
const uploadMatch = reSearch(
|
||||
/[_<>/a-zA-Z-="'\s#;]+([\d,.\s]+[KMGTPI]*B)/,
|
||||
x('//*[@id="user-info-panel"]/div[2]/div[2]/div[4]/text()'),
|
||||
);
|
||||
const downloadMatch = reSearch(
|
||||
/[_<>/a-zA-Z-="'\s#;]+([\d,.\s]+[KMGTPI]*B)/,
|
||||
x('//*[@id="user-info-panel"]/div[2]/div[2]/div[5]/text()'),
|
||||
);
|
||||
const ratioMatch = reSearch(
|
||||
/分享率\][:：_<>/a-zA-Z-="'\s#;]+([\d,.\s]+)/,
|
||||
x('//*[@id="user-info-panel"]/div[2]/div[1]/div[1]/div/text()'),
|
||||
);
|
||||
this.upload = uploadMatch ? this.numFilesize(uploadMatch[1]!.trim()) : 0;
|
||||
this.download = downloadMatch ? this.numFilesize(downloadMatch[1]!.trim()) : 0;
|
||||
const calcRatio = this.download <= 0.0 ? 0.0 : round3(this.upload / this.download);
|
||||
this.ratio = ratioMatch?.[1]?.trim() ? parseFloatMp(ratioMatch[1]) : calcRatio;
|
||||
}
|
||||
|
||||
protected override _parseUserDetailInfo(htmlText: string): void {
|
||||
super._parseUserDetailInfo(htmlText);
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
const joinAt = html.strings('//span[contains(text(), "加入日期")]/following-sibling::span/span/@title');
|
||||
if (joinAt.length) this.joinAt = normalizeDatetime(joinAt[0]!.trim()) ?? null;
|
||||
}
|
||||
|
||||
protected override _getUserLevel(html: XNode): void {
|
||||
super._getUserLevel(html);
|
||||
const level = html.strings('//b[contains(@class, "_Name")]/text()');
|
||||
if (level.length) this.userLevel = level[0]!;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import {
|
||||
createMockFetch,
|
||||
NEXUS_PHP_CF_CHALLENGE,
|
||||
NEXUS_PHP_INDEX,
|
||||
NEXUS_PHP_LOGIN_PAGE,
|
||||
NEXUS_PHP_SEEDING,
|
||||
NEXUS_PHP_USERDETAILS,
|
||||
} from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { NexusPhpSiteUserInfo } from "./nexus-php.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
describe("NexusPhpSiteUserInfo(MP nexus_php.py)", () => {
|
||||
it("首页 + 详情 + 做种页:流量/魔力/等级/加入日期/做种统计", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: NEXUS_PHP_INDEX },
|
||||
"/userdetails.php": { body: NEXUS_PHP_USERDETAILS },
|
||||
"/getusertorrentlistajax.php": { body: NEXUS_PHP_SEEDING },
|
||||
});
|
||||
const info = await runParser(NexusPhpSiteUserInfo, { fetch: m.fetch, cookie: "uid=1; pass=secret" });
|
||||
expect(info).toMatchObject({
|
||||
schema: "NexusPhp",
|
||||
domain: "example.com",
|
||||
userId: "10086",
|
||||
username: "alice",
|
||||
userLevel: "Veteran User",
|
||||
joinAt: "2021-03-04 05:06:07",
|
||||
upload: Math.round(1.5 * 1024 ** 4),
|
||||
download: Math.round(477.3 * 1024 ** 3),
|
||||
ratio: 3.217,
|
||||
bonus: 12345.6,
|
||||
leeching: 2,
|
||||
seeding: 2,
|
||||
seedingSize: 10 * 1024 ** 3 + Math.round(1.5 * 1024 ** 4),
|
||||
seedingInfo: [
|
||||
[5, 10 * 1024 ** 3],
|
||||
[12, Math.round(1.5 * 1024 ** 4)],
|
||||
],
|
||||
messageUnread: 3,
|
||||
errMsg: null,
|
||||
});
|
||||
// 请求顺序:首页 → 用户详情 → 做种(流量页已合并到首页解析)
|
||||
expect(m.requests.map((r) => r.url.pathname)).toEqual(["/", "/userdetails.php", "/getusertorrentlistajax.php"]);
|
||||
expect(m.requests[2]!.url.search).toBe("?userid=10086&type=seeding");
|
||||
expect(m.requests[0]!.headers.cookie).toBe("uid=1; pass=secret");
|
||||
expect(JSON.stringify(info)).not.toContain("secret");
|
||||
});
|
||||
|
||||
it("登录页(含密码框):不再继续请求,errMsg 提示 cookie 过期", async () => {
|
||||
const m = createMockFetch({ "/": { body: NEXUS_PHP_LOGIN_PAGE } });
|
||||
const info = await runParser(NexusPhpSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info.userId).toBeNull();
|
||||
expect(info.errMsg).toContain("未检测到已登陆");
|
||||
expect(m.requests).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("Cloudflare 挑战页按空内容处理并记录原因", async () => {
|
||||
const m = createMockFetch({ "/": { body: NEXUS_PHP_CF_CHALLENGE, status: 403 } });
|
||||
const info = await runParser(NexusPhpSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info.errMsg).toContain("未检测到已登陆");
|
||||
expect(info.raw.http_errors).toEqual([expect.stringContaining("Cloudflare")]);
|
||||
});
|
||||
|
||||
it("_fixup_next_page_url:补 userid/type,已有 userid 原样,无 userid 停止", () => {
|
||||
expect(NexusPhpSiteUserInfo.fixupNextPageUrl("getusertorrentlistajax.php?page=1", "7")).toBe(
|
||||
"getusertorrentlistajax.php?page=1&userid=7&type=seeding",
|
||||
);
|
||||
expect(NexusPhpSiteUserInfo.fixupNextPageUrl("x.php?userid=3&page=2", "7")).toBe("x.php?userid=3&page=2");
|
||||
expect(NexusPhpSiteUserInfo.fixupNextPageUrl("x.php?page=2", null)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,361 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/nexus_php.py @ e6820a8
|
||||
*
|
||||
* NexusPHP 站点用户信息解析器。
|
||||
*
|
||||
* 迁移自 MoviePilot e6820a8(GPLv3)`app/modules/indexer/parser/nexus_php.py` `NexusPhpSiteUserInfo`,
|
||||
* 函数对照:`_parse_site_page` → `_parseSitePage`、`_parse_message_unread` → `_parseMessageUnread`、
|
||||
* `_parse_user_base_info` → `_parseUserBaseInfo`、`_parse_user_traffic_info` → `_parseUserTrafficInfo`、
|
||||
* `_parse_ucoin` → `parseUcoin`、`_parse_user_torrent_seeding_info` → `_parseUserTorrentSeedingInfo`、
|
||||
* `_fixup_next_page_url` → `fixupNextPageUrl`、`_parse_user_detail_info` → `_parseUserDetailInfo`、
|
||||
* `_fixup_torrent_seeding_page` → `_fixupTorrentSeedingPage`、`_get_user_level` → `_getUserLevel`、
|
||||
* `_parse_message_unread_links` → `_parseMessageUnreadLinks`、`_parse_message_content` → `_parseMessageContent`、
|
||||
* `_fixup_traffic_info` → `_fixupTrafficInfo`。XPath 与正则逐字保留。
|
||||
*
|
||||
* 差异:做种表"大小列/做种人数列"条数不一致时 MP 抛 IndexError 中断整站解析,这里缺失的做种人数记 0。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase } from "./base.ts";
|
||||
import { normalizeDatetime, parseFloatMp, parseIntMp, reFindall, reSearch } from "./common.ts";
|
||||
import { hasChildElements, parseHtml, type XNode } from "./dom.ts";
|
||||
import { SiteSchema } from "./types.ts";
|
||||
|
||||
/** Python `round(x, 3)` 的近似 */
|
||||
export function round3(x: number): number {
|
||||
return Math.round(x * 1000) / 1000;
|
||||
}
|
||||
|
||||
export class NexusPhpSiteUserInfo extends SiteParserBase {
|
||||
static override schema: import("./types.ts").SiteSchemaValue | null = SiteSchema.NexusPhp;
|
||||
|
||||
protected _parseSitePage(htmlText: string): void {
|
||||
const text = this._prepareHtmlText(htmlText);
|
||||
const userDetail = reSearch(/userdetails.php\?id=(\d+)/, text);
|
||||
if (userDetail?.[0].trim()) {
|
||||
this._userDetailPage = userDetail[0].trim().replace(/^\/+/, "");
|
||||
this.userid = userDetail[1]!;
|
||||
this._torrentSeedingPage = `getusertorrentlistajax.php?userid=${this.userid}&type=seeding`;
|
||||
} else {
|
||||
const m = reSearch(/(userdetails)/, text);
|
||||
if (m?.[0].trim()) {
|
||||
this._userDetailPage = m[0].trim().replace(/^\/+/, "");
|
||||
this.userid = null;
|
||||
this._torrentSeedingPage = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 解析未读短消息数量 */
|
||||
protected _parseMessageUnread(htmlText: string): void {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
const messageLabels = html.nodes('//a[@href="messages.php"]/..');
|
||||
messageLabels.push(...html.nodes('//a[contains(@href, "messages.php")]/..'));
|
||||
if (messageLabels.length) {
|
||||
const messageText = messageLabels[0]!.text();
|
||||
const matches = reFindall(/[^Date](信息箱\s*|\((?![^)]*:)|你有 )(\d+)/, messageText);
|
||||
if (matches.length && matches[matches.length - 1]!.length === 2) {
|
||||
this.messageUnread = parseIntMp(matches[matches.length - 1]![1]);
|
||||
} else if (/^\d+$/.test(messageText)) {
|
||||
this.messageUnread = parseIntMp(messageText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
// 合并解析,减少额外请求调用
|
||||
this._parseUserTrafficInfo(htmlText);
|
||||
this._userTrafficPage = null;
|
||||
this._parseMessageUnread(htmlText);
|
||||
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
let ret = html.strings(`//a[contains(@href, "userdetails") and contains(@href, "${this.userid}")]//b//text()`);
|
||||
if (ret.length) {
|
||||
this.username = String(ret[0]);
|
||||
return;
|
||||
}
|
||||
ret = html.strings(`//a[contains(@href, "userdetails") and contains(@href, "${this.userid}")]//text()`);
|
||||
if (ret.length) this.username = String(ret[0]);
|
||||
ret = html.strings('//a[contains(@href, "userdetails")]//strong//text()');
|
||||
if (ret.length) this.username = String(ret[0]);
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(htmlText: string): void {
|
||||
const text = this._prepareHtmlText(htmlText);
|
||||
const uploadMatch = reSearch(/[^总]上[传傳]量?[:：_<>/a-zA-Z-="'\s#;]+([\d,.\s]+[KMGTPI]*B)/i, text);
|
||||
this.upload = uploadMatch ? this.numFilesize(uploadMatch[1]!.trim()) : 0;
|
||||
const downloadMatch = reSearch(/[^总子影力]下[载載]量?[:：_<>/a-zA-Z-="'\s#;]+([\d,.\s]+[KMGTPI]*B)/i, text);
|
||||
this.download = downloadMatch ? this.numFilesize(downloadMatch[1]!.trim()) : 0;
|
||||
const ratioMatch = reSearch(/分享率[:：_<>/a-zA-Z-="'\s#;]+([\d,.\s]+)/, text);
|
||||
const calcRatio = this.download <= 0.0 ? 0.0 : round3(this.upload / this.download);
|
||||
this.ratio = ratioMatch && ratioMatch[1]!.trim() ? parseFloatMp(ratioMatch[1]) : calcRatio;
|
||||
const leechingMatch = reSearch(/(Torrents leeching|下载中)[一-龥\D\s]+(\d+)[\s\S]+</, text);
|
||||
this.leeching = leechingMatch && leechingMatch[2]!.trim() ? parseIntMp(leechingMatch[2]) : 0;
|
||||
|
||||
const html = parseHtml(text);
|
||||
const [hasUcoin, bonus] = NexusPhpSiteUserInfo.parseUcoin(html);
|
||||
this.bonus = bonus;
|
||||
if (hasUcoin) return;
|
||||
const tmps = html ? html.strings('//a[contains(@href,"mybonus")]/text()') : [];
|
||||
if (tmps.length) {
|
||||
const bonusText = String(tmps[0]).trim();
|
||||
const m = reSearch(/([\d,.]+)/, bonusText);
|
||||
if (m && m[1]!.trim()) {
|
||||
this.bonus = parseFloatMp(m[1]);
|
||||
return;
|
||||
}
|
||||
}
|
||||
let bonusMatch = reSearch(/mybonus.[[\]:：<>/a-zA-Z_\-="'\s#;.(使用魔力值豆]+\s*([\d,.]+)[<()&\s]/, text);
|
||||
if (bonusMatch && bonusMatch[1]!.trim()) {
|
||||
this.bonus = parseFloatMp(bonusMatch[1]);
|
||||
return;
|
||||
}
|
||||
bonusMatch = reSearch(/[魔力值|\]][[\]:：<>/a-zA-Z_\-="'\s#;]+\s*([\d,.]+|"[\d,.]+")[<>()&\s]/s, text);
|
||||
if (bonusMatch && bonusMatch[1]!.trim()) {
|
||||
this.bonus = parseFloatMp(bonusMatch[1]!.replace(/^"+|"+$/g, ""));
|
||||
}
|
||||
}
|
||||
|
||||
/** MP `_parse_ucoin`:金/银/铜币统一换算为铜币 */
|
||||
static parseUcoin(html: XNode | null): [boolean, number] {
|
||||
if (hasChildElements(html)) {
|
||||
let gold: number | null = null;
|
||||
let silver: number | null = null;
|
||||
let copper: number | null = null;
|
||||
const golds = html.strings('//span[@class = "ucoin-symbol ucoin-gold"]//text()');
|
||||
if (golds.length) gold = parseFloatMp(golds[golds.length - 1]);
|
||||
const silvers = html.strings('//span[@class = "ucoin-symbol ucoin-silver"]//text()');
|
||||
if (silvers.length) silver = parseFloatMp(silvers[silvers.length - 1]);
|
||||
const coppers = html.strings('//span[@class = "ucoin-symbol ucoin-copper"]//text()');
|
||||
if (coppers.length) copper = parseFloatMp(coppers[coppers.length - 1]);
|
||||
if (gold || silver || copper) {
|
||||
return [true, (gold || 0) * 100 * 100 + (silver || 0) * 100 + (copper || 0)];
|
||||
}
|
||||
}
|
||||
return [false, 0.0];
|
||||
}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(htmlText: string, multiPage = false): NextPage {
|
||||
const html = parseHtml(String(htmlText ?? "").replaceAll("\\/", "/"));
|
||||
if (!hasChildElements(html)) return null;
|
||||
|
||||
// 首页存在扩展链接,使用扩展链接
|
||||
const seedingUrlText = html.strings('//a[contains(@href,"torrents.php") and contains(@href,"seeding")]/@href');
|
||||
if (multiPage === false && seedingUrlText.length && seedingUrlText[0]!.trim()) {
|
||||
this._torrentSeedingPage = seedingUrlText[0]!.trim();
|
||||
return this._torrentSeedingPage;
|
||||
}
|
||||
|
||||
let sizeCol = 3;
|
||||
let seedersCol = 4;
|
||||
const sizeColXpath =
|
||||
'//tr[position()=1]/td[(img[@class="size"] and img[@alt="size"]) or (text() = "大小") or (a/img[@class="size" and @alt="size"])]';
|
||||
if (html.nodes(sizeColXpath).length) {
|
||||
sizeCol = html.nodes(`${sizeColXpath}/preceding-sibling::td`).length + 1;
|
||||
}
|
||||
const seedersColXpath =
|
||||
'//tr[position()=1]/td[(img[@class="seeders"] and img[@alt="seeders"]) or (text() = "在做种") or (a/img[@class="seeders" and @alt="seeders"])]';
|
||||
if (html.nodes(seedersColXpath).length) {
|
||||
seedersCol = html.nodes(`${seedersColXpath}/preceding-sibling::td`).length + 1;
|
||||
}
|
||||
|
||||
let pageSeeding = 0;
|
||||
let pageSeedingSize = 0;
|
||||
const pageSeedingInfo: Array<[number, number]> = [];
|
||||
const tableClass = html.nodes('//table[@class="torrents"]').length ? '//table[@class="torrents"]' : "";
|
||||
const seedingSizes = html.nodes(`${tableClass}//tr[position()>1]/td[${sizeCol}]`);
|
||||
let seedingSeeders = html.strings(`${tableClass}//tr[position()>1]/td[${seedersCol}]/b/a/text()`);
|
||||
if (!seedingSeeders.length)
|
||||
seedingSeeders = html.strings(`${tableClass}//tr[position()>1]/td[${seedersCol}]//text()`);
|
||||
if (seedingSizes.length && seedingSeeders.length) {
|
||||
pageSeeding = seedingSizes.length;
|
||||
for (let i = 0; i < seedingSizes.length; i++) {
|
||||
const size = this.numFilesize(seedingSizes[i]!.text().trim());
|
||||
const seeders = parseIntMp(seedingSeeders[i]);
|
||||
pageSeedingSize += size;
|
||||
pageSeedingInfo.push([seeders, size]);
|
||||
}
|
||||
}
|
||||
this.seeding += pageSeeding;
|
||||
this.seedingSize += pageSeedingSize;
|
||||
this.seedingInfo.push(...pageSeedingInfo);
|
||||
|
||||
// 是否存在下页数据
|
||||
let nextPage: string | null = null;
|
||||
const nextPageText = html.strings(
|
||||
'//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁") or contains(.//text(), ">")]/@href',
|
||||
);
|
||||
// 防止识别到详情页
|
||||
while (nextPageText.length) {
|
||||
nextPage = nextPageText.pop()!.trim();
|
||||
if (!nextPage.startsWith("details.php")) break;
|
||||
nextPage = null;
|
||||
}
|
||||
if (nextPage) nextPage = NexusPhpSiteUserInfo.fixupNextPageUrl(nextPage, this.userid);
|
||||
return nextPage;
|
||||
}
|
||||
|
||||
/** MP `_fixup_next_page_url`:补齐 userid/type,无法补齐时停止翻页 */
|
||||
static fixupNextPageUrl(nextPage: string, userid: string | null): string | null {
|
||||
const hashIdx = nextPage.indexOf("#");
|
||||
const fragment = hashIdx >= 0 ? nextPage.slice(hashIdx) : "";
|
||||
const noFrag = hashIdx >= 0 ? nextPage.slice(0, hashIdx) : nextPage;
|
||||
const qIdx = noFrag.indexOf("?");
|
||||
const path = qIdx >= 0 ? noFrag.slice(0, qIdx) : noFrag;
|
||||
const query = qIdx >= 0 ? noFrag.slice(qIdx + 1) : "";
|
||||
// parse_qsl → dict:同名键后者覆盖,保留插入顺序
|
||||
const params = new Map<string, string>();
|
||||
for (const [k, v] of new URLSearchParams(query)) params.set(k, v);
|
||||
if (params.get("userid")) return nextPage;
|
||||
if (!userid) return null;
|
||||
params.set("userid", userid);
|
||||
if (!params.has("type")) params.set("type", "seeding");
|
||||
const qs = new URLSearchParams([...params.entries()]).toString();
|
||||
return `${path}${qs ? `?${qs}` : ""}${fragment}`;
|
||||
}
|
||||
|
||||
protected _parseUserDetailInfo(htmlText: string): void {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
this._getUserLevel(html);
|
||||
this._fixupTrafficInfo(html);
|
||||
|
||||
// 加入日期
|
||||
const joinAtText = html.strings(
|
||||
'//tr/td[text()="加入日期" or text()="注册日期" or *[text()="加入日期"]]/following-sibling::td[1]//text()' +
|
||||
'|//div/b[text()="加入日期"]/../text()' +
|
||||
'|//*[@id="outer"]/table/tr/td/div/div[1]/div[2]/div[3]/span[1]/span/@title',
|
||||
);
|
||||
if (joinAtText.length) {
|
||||
this.joinAt = normalizeDatetime(joinAtText[0]!.split(" (")[0]!.trim()) ?? null;
|
||||
}
|
||||
|
||||
// 做种体积 & 做种数(seeding 页面获取不到的话,此处再获取一次)
|
||||
const seedingSizes = html.nodes(
|
||||
'//tr/td[text()="当前上传"]/following-sibling::td[1]//table[tr[1][td[4 and text()="尺寸"]]]//tr[position()>1]/td[4]',
|
||||
);
|
||||
const seedingSeeders = html.strings(
|
||||
'//tr/td[text()="当前上传"]/following-sibling::td[1]//table[tr[1][td[5 and text()="做种者"]]]//tr[position()>1]/td[5]//text()',
|
||||
);
|
||||
let tmpSeeding = seedingSizes.length;
|
||||
let tmpSeedingSize = 0;
|
||||
const tmpSeedingInfo: Array<[number, number]> = [];
|
||||
for (let i = 0; i < seedingSizes.length; i++) {
|
||||
const size = this.numFilesize(seedingSizes[i]!.text().trim());
|
||||
const seeders = parseIntMp(seedingSeeders[i]);
|
||||
tmpSeedingSize += size;
|
||||
tmpSeedingInfo.push([seeders, size]);
|
||||
}
|
||||
if (!this.seedingSize) this.seedingSize = tmpSeedingSize;
|
||||
if (!this.seeding) this.seeding = tmpSeeding;
|
||||
if (!this.seedingInfo.length) this.seedingInfo = tmpSeedingInfo;
|
||||
|
||||
const stat = html.strings('//tr/td[text()="做种统计"]/following-sibling::td[1]//text()');
|
||||
if (stat.length) {
|
||||
const seedingMatch = reSearch(/总做种数:\s+(\d+)/i, stat[0]);
|
||||
const seedingSizeMatch = reSearch(/总做种体积:\s+([\d,.\s]+[KMGTPI]*B)/i, stat[0]);
|
||||
tmpSeeding = seedingMatch?.[1] ? parseIntMp(seedingMatch[1]) : 0;
|
||||
tmpSeedingSize = seedingSizeMatch ? this.numFilesize(seedingSizeMatch[1]!.trim()) : 0;
|
||||
}
|
||||
if (!this.seedingSize) this.seedingSize = tmpSeedingSize;
|
||||
if (!this.seeding) this.seeding = tmpSeeding;
|
||||
|
||||
this._fixupTorrentSeedingPage(html);
|
||||
}
|
||||
|
||||
/** MP `_fixup_torrent_seeding_page` */
|
||||
protected _fixupTorrentSeedingPage(html: XNode): void {
|
||||
let seedingUrlText = html.strings(
|
||||
'//a[contains(@href,"getusertorrentlist.php") and contains(@href,"seeding")]/@href',
|
||||
);
|
||||
if (seedingUrlText.length) this._torrentSeedingPage = seedingUrlText[0]!.trim();
|
||||
// 从JS调用种获取用户ID
|
||||
seedingUrlText = html.strings(
|
||||
'//a[contains(@href, "javascript: getusertorrentlistajax") and contains(@href,"seeding")]/@href',
|
||||
);
|
||||
const csrfText = html.strings('//meta[@name="x-csrf"]/@content');
|
||||
if (!this._torrentSeedingPage && seedingUrlText.length) {
|
||||
const userJs = reSearch(/javascript: getusertorrentlistajax\(\s*'(\d+)/, seedingUrlText[0]);
|
||||
if (userJs?.[1]?.trim()) {
|
||||
this.userid = userJs[1].trim();
|
||||
this._torrentSeedingPage = `getusertorrentlistajax.php?userid=${this.userid}&type=seeding`;
|
||||
}
|
||||
} else if (seedingUrlText.length && csrfText.length) {
|
||||
if (csrfText[0]!.trim()) {
|
||||
this._torrentSeedingPage = "ajax_getusertorrentlist.php";
|
||||
this._torrentSeedingParams = { userid: this.userid, type: "seeding", csrf: csrfText[0]!.trim() };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** MP `_get_user_level`:图片等级取 title,否则取文本 */
|
||||
protected _getUserLevel(html: XNode): void {
|
||||
let levels = html.strings(
|
||||
'//tr/td[text()="等級" or text()="等级" or *[text()="等级"]]/following-sibling::td[1]/img[1]/@title',
|
||||
);
|
||||
if (levels.length) {
|
||||
this.userLevel = levels[0]!.trim();
|
||||
return;
|
||||
}
|
||||
let nodes = html.nodes(
|
||||
'//tr/td[text()="等級" or text()="等级"]/following-sibling::td[1 and not(img)]' +
|
||||
'|//tr/td[text()="等級" or text()="等级"]/following-sibling::td[1 and img[not(@title)]]',
|
||||
);
|
||||
if (nodes.length) {
|
||||
this.userLevel = nodes[0]!.text().trim();
|
||||
return;
|
||||
}
|
||||
nodes = html.nodes('//tr/td[text()="等級" or text()="等级"]/following-sibling::td[1]');
|
||||
if (nodes.length) {
|
||||
this.userLevel = nodes[0]!.text().trim();
|
||||
return;
|
||||
}
|
||||
levels = html.strings('//a[contains(@href, "userdetails")]/text()');
|
||||
if (!this.userLevel && levels.length) {
|
||||
for (const t of levels) {
|
||||
const m = reSearch(/\[(.*)]/, t);
|
||||
if (m?.[1]?.trim()) {
|
||||
this.userLevel = m[1].trim();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected _parseMessageUnreadLinks(htmlText: string, msgLinks: string[]): NextPage {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return null;
|
||||
msgLinks.push(...html.strings('//tr[not(./td/img[@alt="Read"])]/td/a[contains(@href, "viewmessage")]/@href'));
|
||||
const nextPageText = html.strings('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁")]/@href');
|
||||
return nextPageText.length ? nextPageText[nextPageText.length - 1]!.trim() : null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(htmlText: string): [string | null, string | null, string | null] {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return [null, null, null];
|
||||
let head: string | null = null;
|
||||
const messageHead = html.strings('//h1/text()|//div[@class="layui-card-header"]/span[1]/text()');
|
||||
if (messageHead.length) head = messageHead[messageHead.length - 1]!.trim();
|
||||
let date: string | null = null;
|
||||
const messageDate = html.nodes(
|
||||
'//h1/following-sibling::table[.//tr/td[@class="colhead"]]//tr[2]/td[2]|//div[@class="layui-card-header"]/span[2]/span[2]',
|
||||
);
|
||||
if (messageDate.length) date = messageDate[0]!.text().trim();
|
||||
let content: string | null = null;
|
||||
const messageContent = html.nodes(
|
||||
'//h1/following-sibling::table[.//tr/td[@class="colhead"]]//tr[3]/td|//div[contains(@class,"layui-card-body")]',
|
||||
);
|
||||
if (messageContent.length) content = messageContent[0]!.text().trim();
|
||||
return [head, date, content];
|
||||
}
|
||||
|
||||
/** MP `_fixup_traffic_info` */
|
||||
protected _fixupTrafficInfo(html: XNode): void {
|
||||
if (!this.bonus) {
|
||||
const bonusText = html.strings('//tr/td[text()="魔力值" or text()="猫粮"]/following-sibling::td[1]/text()');
|
||||
if (bonusText.length) this.bonus = parseFloatMp(bonusText[0]!.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { createMockFetch, NEXUS_PHP_INDEX, NEXUS_PROJECT_USER_INDEX } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { round3 } from "./nexus-php.ts";
|
||||
import { NexusProjectSiteUserInfo } from "./nexus-project.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
describe("NexusProjectSiteUserInfo(MP nexus_project.py)", () => {
|
||||
it("user_detail.php?uid= 变种 + span#user_info 无 B 后缀流量", async () => {
|
||||
const m = createMockFetch({ "/": { body: NEXUS_PROJECT_USER_INDEX } });
|
||||
const info = await runParser(NexusProjectSiteUserInfo, { fetch: m.fetch });
|
||||
const up = Math.round(445.11 * 1024 ** 3);
|
||||
const down = Math.round(29.61 * 1024 ** 3);
|
||||
expect(info).toMatchObject({
|
||||
schema: "NexusProject",
|
||||
userId: "77",
|
||||
username: "carol",
|
||||
upload: up,
|
||||
download: down,
|
||||
ratio: round3(up / down),
|
||||
});
|
||||
expect(m.requests.map((r) => r.url.pathname + r.url.search)).toContain("/viewusertorrents.php?id=77&show=seeding");
|
||||
expect(m.requests.map((r) => r.url.pathname)).toContain("/user_detail.php"); // 与 MP 一致:正则只取 user_detail.php?uid=,不含 p_user/ 前缀
|
||||
});
|
||||
|
||||
it("标准 NexusPHP 首页:回落通用流量解析", async () => {
|
||||
const m = createMockFetch({ "/": { body: NEXUS_PHP_INDEX } });
|
||||
const info = await runParser(NexusProjectSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info).toMatchObject({ userId: "10086", upload: Math.round(1.5 * 1024 ** 4), ratio: 3.217 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/nexus_project.py @ e6820a8
|
||||
*
|
||||
* NexusProject 站点用户信息解析器,继承 NexusPHP 通用解析。
|
||||
*
|
||||
* 函数对照(`NexusProjectSiteUserInfo`):`_parse_site_page` → `_parseSitePage`(兼容 `user_detail.php?uid=` 变种)、
|
||||
* `_parse_user_traffic_info` → `_parseUserTrafficInfo`(兼容 `span#user_info` 无 B 后缀的流量文本)。正则逐字保留。
|
||||
*/
|
||||
import { reSearch } from "./common.ts";
|
||||
import { parseHtml } from "./dom.ts";
|
||||
import { NexusPhpSiteUserInfo, round3 } from "./nexus-php.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
export class NexusProjectSiteUserInfo extends NexusPhpSiteUserInfo {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.NexusProject;
|
||||
|
||||
protected override _parseSitePage(htmlText: string): void {
|
||||
const text = this._prepareHtmlText(htmlText);
|
||||
let userDetail = reSearch(/userdetails\.php\?id=(\d+)/, text);
|
||||
if (userDetail?.[0].trim()) {
|
||||
this._userDetailPage = userDetail[0].trim().replace(/^\/+/, "");
|
||||
this.userid = userDetail[1]!;
|
||||
} else {
|
||||
// 兼容部分 NexusProject 变种站点(如 star-space.net)的 p_user/user_detail.php?uid= 用户定位格式
|
||||
userDetail = reSearch(/user_detail\.php\?uid=(\d+)/, text);
|
||||
if (userDetail?.[0].trim()) {
|
||||
this._userDetailPage = userDetail[0].trim().replace(/^\/+/, "");
|
||||
this.userid = userDetail[1]!;
|
||||
const uname = reSearch(/user_detail\.php\?uid=\d+[^>]*?>\s*<[^>]*>([^<]+)</, text);
|
||||
if (uname) this.username = uname[1]!.trim();
|
||||
}
|
||||
}
|
||||
this._torrentSeedingPage = `viewusertorrents.php?id=${this.userid}&show=seeding`;
|
||||
}
|
||||
|
||||
protected override _parseUserTrafficInfo(htmlText: string): void {
|
||||
// 兼容 span#user_info / span#user_info_no_hover:"上传：445.11 G" / "下载：29.61 G"(单位无 B 后缀)
|
||||
try {
|
||||
const html = parseHtml(htmlText);
|
||||
if (html) {
|
||||
const bodyText = html
|
||||
.strings('//span[@id="user_info"]//text() | //span[@id="user_info_no_hover"]//text()')
|
||||
.join(" ");
|
||||
let m = reSearch(/上传[：:]\s*([\d.,]+)\s*([GMKT]?i?B?)/i, bodyText);
|
||||
if (m) this.upload = this.numFilesize(`${m[1]!.replaceAll(",", "")} ${m[2]!.toUpperCase()}B`);
|
||||
m = reSearch(/下载[：:]\s*([\d.,]+)\s*([GMKT]?i?B?)/i, bodyText);
|
||||
if (m) this.download = this.numFilesize(`${m[1]!.replaceAll(",", "")} ${m[2]!.toUpperCase()}B`);
|
||||
if (this.upload && this.download) this.ratio = round3(this.upload / this.download);
|
||||
if (this.upload || this.download) return;
|
||||
}
|
||||
} catch {
|
||||
// 与 MP 一致:吞掉异常回落到通用解析
|
||||
}
|
||||
super._parseUserTrafficInfo(htmlText);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import {
|
||||
createMockFetch,
|
||||
NEXUS_PHP_LOGIN_PAGE,
|
||||
RABBIT_USER_DETAILS,
|
||||
RABBIT_USER_INDEX,
|
||||
RABBIT_USER_SEEDING_JSON,
|
||||
} from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { NexusRabbitSiteUserInfo } from "./nexus-rabbit.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
describe("NexusRabbitSiteUserInfo(MP nexus_rabbit.py)", () => {
|
||||
it("首页奶糖余额 + 详情 layui-hares + api/general JSON 做种", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: RABBIT_USER_INDEX },
|
||||
"/user.php": { body: RABBIT_USER_DETAILS },
|
||||
"POST /api/general": { body: RABBIT_USER_SEEDING_JSON },
|
||||
});
|
||||
const info = await runParser(NexusRabbitSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info).toMatchObject({
|
||||
schema: "NexusRabbit",
|
||||
userId: "5",
|
||||
username: "dave",
|
||||
userLevel: "Power User",
|
||||
joinAt: "2022-01-02 03:04:05",
|
||||
upload: Math.round(1.5 * 1024 ** 4),
|
||||
download: 500 * 1024 ** 3,
|
||||
ratio: 3.072,
|
||||
bonus: 1024.5,
|
||||
seeding: 2,
|
||||
seedingSize: Math.round(3.5 * 1024 ** 3),
|
||||
seedingInfo: [
|
||||
[3, 1024 ** 3],
|
||||
[7, Math.round(2.5 * 1024 ** 3)],
|
||||
],
|
||||
});
|
||||
const api = m.requests.find((r) => r.url.pathname === "/api/general")!;
|
||||
expect(api.headers["x-requested-with"]).toBe("XMLHttpRequest");
|
||||
expect(JSON.parse(api.body!)).toMatchObject({ action: "userTorrentsList", data: { type: "seeding", id: 5 } });
|
||||
});
|
||||
|
||||
it("未登录:errMsg 且不请求 API", async () => {
|
||||
const m = createMockFetch({ "/": { body: NEXUS_PHP_LOGIN_PAGE } });
|
||||
const info = await runParser(NexusRabbitSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info.errMsg).toContain("未检测到已登陆");
|
||||
expect(m.requests).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/nexus_rabbit.py @ e6820a8
|
||||
*
|
||||
* NexusRabbit(兔子/layui-hares 模板)站点用户信息解析器。
|
||||
*
|
||||
* 函数对照(`NexusRabbitSiteUserInfo`):`_parse_site_page` → `_parseSitePage`(做种/消息改走 `api/general` JSON)、
|
||||
* `_parse_user_torrent_seeding_info` → `_parseUserTorrentSeedingInfo`、`_parse_message_unread_links` →
|
||||
* `_parseMessageUnreadLinks`(直接读出消息并调用 readMessage 标记已读)、`_parse_user_base_info` → `_parseUserBaseInfo`
|
||||
* (奶糖余额)、`_parse_user_detail_info` → `_parseUserDetailInfo`、`_parse_message_content` / `_parse_user_traffic_info` → 空实现。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase, urljoin } from "./base.ts";
|
||||
import { normalizeDatetime, parseFloatMp, pyInt, reSearch } from "./common.ts";
|
||||
import { hasChildElements, parseHtml } from "./dom.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
const JSON_HEADERS = {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json, text/plain, */*",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
};
|
||||
|
||||
function removePrefix(s: string, p: string): string {
|
||||
return s.startsWith(p) ? s.slice(p.length) : s;
|
||||
}
|
||||
|
||||
export class NexusRabbitSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.NexusRabbit;
|
||||
|
||||
protected _parseSitePage(htmlText: string): void {
|
||||
const text = this._prepareHtmlText(htmlText);
|
||||
const userDetail = reSearch(/user.php\?id=(\d+)/, text);
|
||||
if (!userDetail?.[0].trim()) return;
|
||||
this.userid = userDetail[1]!;
|
||||
this._userDetailPage = `user.php?id=${this.userid}`;
|
||||
this._userTrafficPage = null;
|
||||
this._torrentSeedingPage = "api/general";
|
||||
this._torrentSeedingParams = {
|
||||
page: 1,
|
||||
limit: 5000000,
|
||||
action: "userTorrentsList",
|
||||
data: { type: "seeding", id: pyInt(this.userid) },
|
||||
};
|
||||
// X-Requested-With 必须要加上,不然返回的是空数据
|
||||
this._torrentSeedingHeaders = { ...JSON_HEADERS };
|
||||
this._userMailUnreadPage = null;
|
||||
this._sysMailUnreadPage = "api/general";
|
||||
this._mailUnreadParams = { page: 1, limit: 5000000, action: "getMessageIn" };
|
||||
this._mailUnreadHeaders = { ...JSON_HEADERS };
|
||||
}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(htmlText: string): NextPage {
|
||||
let torrents: Array<Record<string, unknown>>;
|
||||
try {
|
||||
const data = (JSON.parse(htmlText) as { data?: unknown }).data;
|
||||
torrents = (data ?? []) as Array<Record<string, unknown>>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
let seedingSize = 0;
|
||||
const seedingInfo: Array<[number, number]> = [];
|
||||
for (const t of torrents) {
|
||||
const seeders = pyInt(t.seeders ?? 0);
|
||||
const size = this.numFilesize(t.size as string);
|
||||
seedingSize += size;
|
||||
seedingInfo.push([seeders, size]);
|
||||
}
|
||||
this.seeding = torrents.length;
|
||||
this.seedingSize = seedingSize;
|
||||
this.seedingInfo = seedingInfo;
|
||||
return null;
|
||||
}
|
||||
|
||||
protected async _parseMessageUnreadLinks(htmlText: string, _msgLinks: string[]): Promise<NextPage> {
|
||||
const unreadIds: unknown[] = [];
|
||||
let messages: Array<Record<string, unknown>>;
|
||||
try {
|
||||
messages = ((JSON.parse(htmlText) as { data?: unknown }).data ?? []) as Array<Record<string, unknown>>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
for (const msg of messages) {
|
||||
const id = msg.id;
|
||||
const unread = msg.unread;
|
||||
if (!(id && unread) || unread === "no") continue;
|
||||
unreadIds.push(id);
|
||||
const head = msg.subject as string | null;
|
||||
const date = msg.added as string | null;
|
||||
const content = msg.msg as string | null;
|
||||
if (head && date && content) this.messageUnreadContents.push([head, date, content]);
|
||||
}
|
||||
this.messageUnread = unreadIds.length;
|
||||
if (unreadIds.length) {
|
||||
await this._getPageContent(
|
||||
urljoin(this._baseUrl, "api/general?loading=true"),
|
||||
{ action: "readMessage", data: { ids: unreadIds } },
|
||||
{ ...JSON_HEADERS },
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 只有奶糖余额需要在 base 中获取,其它均可以在详情页拿到 */
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
const bonus = html.strings('//div[contains(text(), "奶糖余额")]/following-sibling::div[1]/text()');
|
||||
if (bonus.length) this.bonus = parseFloatMp(bonus[0]!.trim());
|
||||
}
|
||||
|
||||
protected _parseUserDetailInfo(htmlText: string): void {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
const infos = html.nodes('//div[contains(@class, "layui-hares-user-info-right")]');
|
||||
if (!infos.length) return;
|
||||
const info = infos[0]!;
|
||||
const username = info.strings('.//span[contains(text(), "用户名")]/a/span/text()');
|
||||
if (username.length) this.username = username[0]!.trim();
|
||||
const level = info.strings('.//span[contains(text(), "等级")]/b/text()');
|
||||
if (level.length) this.userLevel = level[0]!.trim();
|
||||
const joinDate = info.strings('.//span[contains(text(), "注册日期")]/text()');
|
||||
if (joinDate.length) {
|
||||
const d = removePrefix(joinDate[0]!.trim().split("\r")[0]!, "注册日期：");
|
||||
this.joinAt = normalizeDatetime(d) ?? null;
|
||||
}
|
||||
const upload = info.strings('.//span[contains(text(), "上传量")]/text()');
|
||||
if (upload.length) this.upload = this.numFilesize(removePrefix(upload[0]!.trim(), "上传量："));
|
||||
const download = info.strings('.//span[contains(text(), "下载量")]/text()');
|
||||
if (download.length) this.download = this.numFilesize(removePrefix(download[0]!.trim(), "下载量："));
|
||||
const ratio = info.strings('.//span[contains(text(), "分享率")]/em/text()');
|
||||
if (ratio.length) this.ratio = parseFloatMp(ratio[0]!.trim());
|
||||
}
|
||||
|
||||
/** 已在 _parseMessageUnreadLinks 内实现 */
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
|
||||
/** 已在 _parseUserDetailInfo 内实现 */
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { createMockFetch, NEXUS_PHP_INDEX, NEXUS_PHP_LOGIN_PAGE, NEXUS_PHP_USERDETAILS } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { getUserParserClass, refreshSiteUserData, USER_PARSER_CLASSES } from "./registry.ts";
|
||||
import { SiteSchema } from "./types.ts";
|
||||
|
||||
describe("用户信息解析器注册表(MP refresh_userdata)", () => {
|
||||
it("覆盖 MP SiteSchema 全部 20 个值,且一一对应", () => {
|
||||
const schemas = USER_PARSER_CLASSES.map((c) => c.schema);
|
||||
expect(new Set(schemas)).toEqual(new Set(Object.values(SiteSchema)));
|
||||
expect(schemas).toHaveLength(20);
|
||||
expect(getUserParserClass("Small Horse")?.name).toBe("SmallHorseSiteUserInfo");
|
||||
expect(getUserParserClass("nope")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("未知 schema 返回 null", async () => {
|
||||
const m = createMockFetch({});
|
||||
expect(
|
||||
await refreshSiteUserData({ name: "x", url: "https://x.example.com/", schema: "Nope", fetch: m.fetch }),
|
||||
).toBeNull();
|
||||
expect(m.requests).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("声明 schema 解析不出 userid 时按序回退到能解析的 schema(Unit3d 标错 → NexusPhp)", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: NEXUS_PHP_INDEX },
|
||||
"/userdetails.php": { body: NEXUS_PHP_USERDETAILS },
|
||||
});
|
||||
const info = await refreshSiteUserData({
|
||||
name: "demo",
|
||||
url: "https://pt.example.com/",
|
||||
schema: SiteSchema.Unit3d,
|
||||
cookie: "c=1",
|
||||
fetch: m.fetch,
|
||||
});
|
||||
expect(info).toMatchObject({ userId: "10086", username: "alice", domain: "example.com" });
|
||||
expect(info!.schema).not.toBe("Unit3d");
|
||||
});
|
||||
|
||||
it("全部回退失败:返回声明 schema 的结果与 errMsg", async () => {
|
||||
const m = createMockFetch({ "/": { body: NEXUS_PHP_LOGIN_PAGE } });
|
||||
const info = await refreshSiteUserData({
|
||||
name: "demo",
|
||||
url: "https://pt.example.com/",
|
||||
schema: SiteSchema.NexusPhp,
|
||||
fetch: m.fetch,
|
||||
});
|
||||
expect(info).toMatchObject({ schema: "NexusPhp", userId: null });
|
||||
expect(info!.errMsg).toContain("未检测到已登陆");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/__init__.py @ e6820a8
|
||||
*
|
||||
* 站点用户信息解析器注册表与刷新入口。
|
||||
* - `IndexerModule.__init__` 里 `ModuleHelper.load('app.modules.indexer.parser', filter=有 schema)` → `USER_PARSER_CLASSES`
|
||||
* (按模块文件名字母序,与 pkgutil 遍历顺序一致;回退时按此顺序尝试)
|
||||
* - `IndexerModule.refresh_userdata` → `refreshSiteUserData`:按站点声明的 schema 取解析器;解析后 userid 为空且非公开站时,
|
||||
* 依次尝试其他 schema(兼容 schema 标注错误/变种站),其他 schema 的异常只记录不抛出。
|
||||
*
|
||||
* 与 MP 一致:声明 schema 解析器的异常原样抛出(由调用方记录),只有回退尝试的异常被吞掉。
|
||||
* 差异:结果不含凭据;`fetchMessages` 默认 false(MP 由全局 SITE_MESSAGE 控制)。
|
||||
*/
|
||||
import type { SiteParserBase, SiteParserClass } from "./base.ts";
|
||||
import { BitptSiteUserInfo } from "./bitpt.ts";
|
||||
import { extractDomain } from "./common.ts";
|
||||
import { DiscuzUserInfo } from "./discuz.ts";
|
||||
import { FileListSiteUserInfo } from "./file-list.ts";
|
||||
import { GazelleSiteUserInfo } from "./gazelle.ts";
|
||||
import { HDDolbySiteUserInfo } from "./hddolby.ts";
|
||||
import { IptSiteUserInfo } from "./ipt.ts";
|
||||
import { MTorrentSiteUserInfo } from "./mtorrent.ts";
|
||||
import { NexusAudiencesSiteUserInfo } from "./nexus-audiences.ts";
|
||||
import { NexusHhanclubSiteUserInfo } from "./nexus-hhanclub.ts";
|
||||
import { NexusPhpSiteUserInfo } from "./nexus-php.ts";
|
||||
import { NexusProjectSiteUserInfo } from "./nexus-project.ts";
|
||||
import { NexusRabbitSiteUserInfo } from "./nexus-rabbit.ts";
|
||||
import { RousiSiteUserInfo } from "./rousi.ts";
|
||||
import { SmallHorseSiteUserInfo } from "./small-horse.ts";
|
||||
import { SunnyPTSiteUserInfo } from "./sunnypt.ts";
|
||||
import { TNodeSiteUserInfo } from "./tnode.ts";
|
||||
import { TorrentLeechSiteUserInfo } from "./torrent-leech.ts";
|
||||
import type { SiteSchemaValue, SiteUserInfo, SiteUserParserOptions } from "./types.ts";
|
||||
import { Unit3dSiteUserInfo } from "./unit3d.ts";
|
||||
import { YemaSiteUserInfo } from "./yema.ts";
|
||||
import { ZhixingSiteUserInfo } from "./zhixing.ts";
|
||||
|
||||
/** 全部解析器(模块字母序:bitpt, discuz, file_list, gazelle, hddolby, ipt, mtorrent, nexus_*, rousi, …) */
|
||||
export const USER_PARSER_CLASSES: readonly SiteParserClass[] = [
|
||||
BitptSiteUserInfo,
|
||||
DiscuzUserInfo,
|
||||
FileListSiteUserInfo,
|
||||
GazelleSiteUserInfo,
|
||||
HDDolbySiteUserInfo,
|
||||
IptSiteUserInfo,
|
||||
MTorrentSiteUserInfo,
|
||||
NexusAudiencesSiteUserInfo,
|
||||
NexusHhanclubSiteUserInfo,
|
||||
NexusPhpSiteUserInfo,
|
||||
NexusProjectSiteUserInfo,
|
||||
NexusRabbitSiteUserInfo,
|
||||
RousiSiteUserInfo,
|
||||
SmallHorseSiteUserInfo,
|
||||
SunnyPTSiteUserInfo,
|
||||
TNodeSiteUserInfo,
|
||||
TorrentLeechSiteUserInfo,
|
||||
Unit3dSiteUserInfo,
|
||||
YemaSiteUserInfo,
|
||||
ZhixingSiteUserInfo,
|
||||
] as unknown as SiteParserClass[];
|
||||
|
||||
export function getUserParserClass(schema: string | null | undefined): SiteParserClass | undefined {
|
||||
if (!schema) return undefined;
|
||||
return USER_PARSER_CLASSES.find((c) => c.schema === schema);
|
||||
}
|
||||
|
||||
/** refresh_userdata 的站点入参(MP site dict 子集) */
|
||||
export interface SiteUserDataRequest extends Omit<SiteUserParserOptions, "siteName"> {
|
||||
name: string;
|
||||
/** 站点声明的解析 schema(SiteSchema 值) */
|
||||
schema?: string | null;
|
||||
/** 公开站:不报"未找到解析器",也不做 schema 回退 */
|
||||
public?: boolean;
|
||||
}
|
||||
|
||||
/** MP `refresh_userdata`:未找到解析器返回 null */
|
||||
export async function refreshSiteUserData(site: SiteUserDataRequest): Promise<SiteUserInfo | null> {
|
||||
const make = (schema: string | null | undefined): SiteParserBase | null => {
|
||||
const cls = getUserParserClass(schema);
|
||||
if (!cls) return null;
|
||||
const { name, schema: _s, public: _p, ...rest } = site;
|
||||
return new cls({ ...rest, siteName: name });
|
||||
};
|
||||
let obj = make(site.schema);
|
||||
if (!obj) return null;
|
||||
try {
|
||||
await obj.parse();
|
||||
if (!obj.userid && !site.public) {
|
||||
const tried = new Set<string>([String(site.schema)]);
|
||||
for (const cls of USER_PARSER_CLASSES) {
|
||||
const value = cls.schema as SiteSchemaValue | null;
|
||||
if (!value || tried.has(value)) continue;
|
||||
tried.add(value);
|
||||
const alt = make(value);
|
||||
if (!alt) continue;
|
||||
try {
|
||||
await alt.parse();
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (alt.userid) {
|
||||
obj = alt;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
const info = obj.toUserInfo();
|
||||
info.domain = extractDomain(site.url) || null;
|
||||
return info;
|
||||
} finally {
|
||||
obj.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { createMockFetch, ROUSI_USER_ERROR, ROUSI_USER_PROFILE } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { RousiSiteUserInfo } from "./rousi.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
describe("RousiSiteUserInfo(MP rousi.py)", () => {
|
||||
it("api/v1/profile 单接口:流量/karma/做种与下载中统计", async () => {
|
||||
const m = createMockFetch({ "GET https://rousi.pro/api/v1/profile": { body: ROUSI_USER_PROFILE } });
|
||||
const info = await runParser(RousiSiteUserInfo, { fetch: m.fetch, url: "https://rousi.pro/", apikey: "rk" });
|
||||
expect(info).toMatchObject({
|
||||
schema: "RousiPro",
|
||||
userId: "1",
|
||||
username: "jack",
|
||||
userLevel: "Lv.5",
|
||||
joinAt: "2024-01-01 00:00:00",
|
||||
upload: 1073741824,
|
||||
download: 536870912,
|
||||
ratio: 2,
|
||||
bonus: 1000.5,
|
||||
seeding: 10,
|
||||
seedingSize: 10737418240,
|
||||
leeching: 2,
|
||||
leechingSize: 2147483648,
|
||||
});
|
||||
expect(m.requests[0]!.headers.authorization).toBe("Bearer rk");
|
||||
expect(m.requests).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("API 错误:errMsg 为站点 message", async () => {
|
||||
const m = createMockFetch({ "GET https://rousi.pro/api/v1/profile": { body: ROUSI_USER_ERROR } });
|
||||
const info = await runParser(RousiSiteUserInfo, { fetch: m.fetch, url: "https://rousi.pro/", apikey: "bad" });
|
||||
expect(info.userId).toBeNull();
|
||||
expect(info.errMsg).toBe("API Key 无效");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/rousi.py @ e6820a8
|
||||
*
|
||||
* Rousi.pro(PeerGo)用户数据解析器(MP `RousiSiteUserInfo`,apikey 模式)。
|
||||
* 函数对照:`_parse_site_page` → `_parseSitePage`(api/v1/profile,Bearer API Key)、`_parse_logged_in`(恒 True)、
|
||||
* `_parse_user_base_info` → `_parseUserBaseInfo`(单接口含流量/做种/下载中统计)、
|
||||
* `_pase_unread_msgs` → `_paseUnreadMsgs`(网页接口 api/messages,需 token;页码从 0 起,照 MP)。
|
||||
*
|
||||
* 注意(写操作):与 MP 一样,读取未读消息后调用 `POST api/messages/read-all` 标记全部已读;
|
||||
* 只在 `fetchMessages=true` 时执行。API Key / token 只放 Authorization 头,不进入结果。
|
||||
*/
|
||||
import { redactText } from "../../rss/redact.ts";
|
||||
import { type NextPage, SiteParserBase, urljoin } from "./base.ts";
|
||||
import { extractDomain, normalizeDatetime, pyFloat, pyInt } from "./common.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
export class RousiSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.RousiPro;
|
||||
override requestMode = "apikey" as const;
|
||||
|
||||
protected _parseSitePage(_htmlText: string): void {
|
||||
this._baseUrl = `https://${extractDomain(this._siteUrl)}`;
|
||||
this._userBasicPage = "api/v1/profile";
|
||||
this._userBasicParams = {};
|
||||
this._userBasicHeaders = {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json",
|
||||
Authorization: `Bearer ${this.apikey}`,
|
||||
};
|
||||
this._userTrafficPage = null;
|
||||
this._userDetailPage = null;
|
||||
this._torrentSeedingPage = null;
|
||||
this._userMailUnreadPage = null;
|
||||
this._sysMailUnreadPage = null;
|
||||
}
|
||||
|
||||
protected override _parseLoggedIn(_htmlText: unknown): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
if (!htmlText) return;
|
||||
let data: unknown;
|
||||
try {
|
||||
data = JSON.parse(htmlText);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!data || typeof data !== "object" || Array.isArray(data)) {
|
||||
this.errMsg = "用户数据响应结构无效";
|
||||
return;
|
||||
}
|
||||
const d = data as Json;
|
||||
if (d.code !== 0) {
|
||||
this.errMsg = String(d.message || "未知错误");
|
||||
return;
|
||||
}
|
||||
const u = d.data;
|
||||
if (!u || typeof u !== "object" || Array.isArray(u)) return;
|
||||
const info = u as Json;
|
||||
this.userid = info.id == null ? null : String(info.id);
|
||||
this.username = (info.username as string) ?? null;
|
||||
this.userLevel = ((info.level_text || info.role_text) as string) ?? null;
|
||||
const registeredAt = info.registered_at;
|
||||
const joinAt = typeof registeredAt === "string" ? normalizeDatetime(registeredAt) : null;
|
||||
if (joinAt) this.joinAt = joinAt.length >= 19 ? joinAt.slice(0, 19) : joinAt;
|
||||
this.upload = pyInt(info.uploaded || 0);
|
||||
this.download = pyInt(info.downloaded || 0);
|
||||
this.ratio = Math.round(pyFloat(info.ratio || 0) * 100) / 100;
|
||||
this.bonus = pyFloat(info.karma || 0);
|
||||
const sl = (info.seeding_leeching_data ?? {}) as Json;
|
||||
this.seeding = pyInt(sl.seeding_count || 0);
|
||||
this.seedingSize = pyInt(sl.seeding_size || 0);
|
||||
this.leeching = pyInt(sl.leeching_count || 0);
|
||||
this.leechingSize = pyInt(sl.leeching_size || 0);
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserDetailInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(_htmlText: string, _multiPage = false): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
|
||||
/** 直接请求(MP 此处绕过 `_get_page_content`,用 RequestUtils 带 query GET / POST) */
|
||||
private async _rawJson(method: "GET" | "POST", url: string, headers: Record<string, string>): Promise<Json | null> {
|
||||
const fetchImpl = this._opts.fetch ?? globalThis.fetch;
|
||||
const timeout = AbortSignal.timeout(this._opts.timeoutMs ?? 60_000);
|
||||
const signal = this._opts.signal ? AbortSignal.any([this._opts.signal, timeout]) : timeout;
|
||||
try {
|
||||
await this._opts.limiter?.acquire(this._opts.signal);
|
||||
const res = await fetchImpl(url, { method, headers, signal });
|
||||
if (res.status !== 200) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
this.httpErrors.push(redactText(`${method} ${url}:HTTP ${res.status}`));
|
||||
return null;
|
||||
}
|
||||
return (await res.json()) as Json;
|
||||
} catch (e) {
|
||||
if (this._opts.signal?.aborted) throw e;
|
||||
this.httpErrors.push(redactText(`${method} ${url}:${(e as Error).message}`));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** 解析所有未读消息标题和内容(网页接口) */
|
||||
protected override async _paseUnreadMsgs(): Promise<void> {
|
||||
if (!this.token) return;
|
||||
const headers: Record<string, string> = {
|
||||
"User-Agent": `${this._ua}`,
|
||||
Accept: "application/json, text/plain, */*",
|
||||
Authorization: this.token.startsWith("Bearer ") ? this.token : `Bearer ${this.token}`,
|
||||
};
|
||||
const getMessageList = async (page: number): Promise<Json> => {
|
||||
const qs = new URLSearchParams({ page: String(page), page_size: "100", unread_only: "true" });
|
||||
const res = await this._rawJson("GET", `${urljoin(this._baseUrl, "api/messages")}?${qs}`, headers);
|
||||
if (!res || (res.code ?? -1) !== 0) return { messages: [], total_pages: 0 };
|
||||
return (res.data ?? {}) as Json;
|
||||
};
|
||||
let page = 0;
|
||||
let res = await getMessageList(page);
|
||||
page += 1;
|
||||
const messages = [...((res.messages ?? []) as Json[])];
|
||||
const totalPages = Number(res.total_pages ?? 0);
|
||||
while (page < totalPages && page < 50) {
|
||||
res = await getMessageList(page);
|
||||
messages.push(...((res.messages ?? []) as Json[]));
|
||||
page += 1;
|
||||
}
|
||||
this.messageUnread = messages.length;
|
||||
for (const m of messages) {
|
||||
this.messageUnreadContents.push([
|
||||
(m.title as string) ?? null,
|
||||
normalizeDatetime(m.created_at as string) ?? null,
|
||||
(m.content as string) ?? null,
|
||||
]);
|
||||
}
|
||||
// 更新消息为已读(写操作,见文件头)
|
||||
await this._rawJson("POST", urljoin(this._baseUrl, "api/messages/read-all"), headers);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import {
|
||||
createMockFetch,
|
||||
SMALL_HORSE_USER_INDEX,
|
||||
SMALL_HORSE_USER_PAGE,
|
||||
SMALL_HORSE_USER_SEEDING_P1,
|
||||
SMALL_HORSE_USER_SEEDING_P2,
|
||||
} from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { SmallHorseSiteUserInfo } from "./small-horse.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
const GiB = 1024 ** 3;
|
||||
|
||||
describe("SmallHorseSiteUserInfo(MP small_horse.py)", () => {
|
||||
it("stats nobullet 按下标取流量/等级,做种表按分页累加", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: SMALL_HORSE_USER_INDEX },
|
||||
"/user.php": { body: SMALL_HORSE_USER_PAGE },
|
||||
"/torrents.php": { body: SMALL_HORSE_USER_SEEDING_P1 },
|
||||
"/torrents.php?page=2": { body: SMALL_HORSE_USER_SEEDING_P2 },
|
||||
});
|
||||
const info = await runParser(SmallHorseSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info).toMatchObject({
|
||||
schema: "Small Horse",
|
||||
userId: "9",
|
||||
username: "eve",
|
||||
joinAt: "2019-01-01 00:00:00",
|
||||
upload: 1024 ** 4,
|
||||
download: 200 * GiB,
|
||||
ratio: 5.12,
|
||||
bonus: 1234.5,
|
||||
userLevel: "Power User",
|
||||
leeching: 3,
|
||||
seeding: 3,
|
||||
seedingSize: 16 * GiB,
|
||||
seedingInfo: [
|
||||
[4, 10 * GiB],
|
||||
[2, 5 * GiB],
|
||||
[9, GiB],
|
||||
],
|
||||
});
|
||||
expect(m.requests.filter((r) => r.url.pathname === "/torrents.php").map((r) => r.url.search)).toEqual([
|
||||
"?type=seeding&userid=9",
|
||||
"?type=seeding&userid=9&page=2",
|
||||
]);
|
||||
});
|
||||
|
||||
it("统计列表缺项:与 MP 一样抛 IndexError", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: SMALL_HORSE_USER_INDEX },
|
||||
"/user.php": { body: '<ul class="stats nobullet"><li>x</li></ul><ul class="stats nobullet"><li>y</li></ul>' },
|
||||
});
|
||||
await expect(runParser(SmallHorseSiteUserInfo, { fetch: m.fetch })).rejects.toThrow(/IndexError/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/small_horse.py @ e6820a8
|
||||
*
|
||||
* Small Horse(Gazelle 变种)站点用户信息解析器。
|
||||
*
|
||||
* 函数对照(`SmallHorseSiteUserInfo`):`_parse_site_page` → `_parseSitePage`、`_parse_user_base_info` →
|
||||
* `_parseUserBaseInfo`、`_parse_user_traffic_info` → `_parseUserTrafficInfo`(`ul.stats nobullet` 按下标取值)、
|
||||
* `_parse_user_detail_info` → 空、`_parse_user_torrent_seeding_info` → `_parseUserTorrentSeedingInfo`、
|
||||
* `_parse_message_unread_links` / `_parse_message_content` → 空。XPath 逐字保留。
|
||||
*
|
||||
* 与 MP 一致:统计列表下标越界时抛错(MP IndexError)。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase } from "./base.ts";
|
||||
import { normalizeDatetime, parseFloatMp, parseIntMp, reSearch } from "./common.ts";
|
||||
import { hasChildElements, parseHtml, type XNode } from "./dom.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
function at<T>(arr: T[], i: number): T {
|
||||
if (i >= arr.length) throw new Error(`IndexError: list index out of range [${i}]`);
|
||||
return arr[i]!;
|
||||
}
|
||||
|
||||
/** Python `str.split(":")[1]` */
|
||||
function after(s: string): string {
|
||||
return at(s.split(":"), 1);
|
||||
}
|
||||
|
||||
export class SmallHorseSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.SmallHorse;
|
||||
|
||||
protected _parseSitePage(htmlText: string): void {
|
||||
const text = this._prepareHtmlText(htmlText);
|
||||
const userDetail = reSearch(/user.php\?id=(\d+)/, text);
|
||||
if (userDetail?.[0].trim()) {
|
||||
this._userDetailPage = userDetail[0].trim().replace(/^\/+/, "");
|
||||
this.userid = userDetail[1]!;
|
||||
this._torrentSeedingPage = `torrents.php?type=seeding&userid=${this.userid}`;
|
||||
}
|
||||
this._userTrafficPage = `user.php?id=${this.userid}`;
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
const html = parseHtml(this._prepareHtmlText(htmlText));
|
||||
if (!html) return;
|
||||
const ret = html.strings('//a[contains(@href, "user.php")]//text()');
|
||||
if (ret.length) this.username = String(ret[0]);
|
||||
}
|
||||
|
||||
/** 上传/下载/分享率 [做种数/魔力值] */
|
||||
protected _parseUserTrafficInfo(htmlText: string): void {
|
||||
const html = parseHtml(this._prepareHtmlText(htmlText));
|
||||
if (!html) return;
|
||||
const tmps = html.nodes('//ul[@class = "stats nobullet"]');
|
||||
if (!tmps.length) return;
|
||||
const li = (n: XNode) => n.nodes("li");
|
||||
const stats1 = li(at(tmps, 1));
|
||||
if (stats1.length && stats1[0]!.strings("span//text()").length) {
|
||||
this.joinAt = normalizeDatetime(stats1[0]!.strings("span//text()")[0]!) ?? null;
|
||||
}
|
||||
this.upload = this.numFilesize(after(at(at(stats1, 2).strings("text()"), 0)).trim());
|
||||
this.download = this.numFilesize(after(at(at(stats1, 3).strings("text()"), 0)).trim());
|
||||
const ratioSpan = at(stats1, 4).strings("span//text()");
|
||||
if (ratioSpan.length) this.ratio = parseFloatMp(ratioSpan[0]!.replaceAll("∞", "0"));
|
||||
else this.ratio = parseFloatMp(after(at(at(stats1, 5).strings("text()"), 0)));
|
||||
this.bonus = parseFloatMp(after(at(at(stats1, 5).strings("text()"), 0)));
|
||||
this.userLevel = after(at(at(li(at(tmps, 3)), 0).strings("text()"), 0)).trim();
|
||||
this.leeching = parseIntMp(after(at(at(li(at(tmps, 4)), 6).strings("text()"), 0)).replaceAll("[", ""));
|
||||
}
|
||||
|
||||
protected _parseUserDetailInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(htmlText: string): NextPage {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return null;
|
||||
const sizeCol = 6;
|
||||
const seedersCol = 8;
|
||||
let pageSeeding = 0;
|
||||
let pageSeedingSize = 0;
|
||||
const pageSeedingInfo: Array<[number, number]> = [];
|
||||
const sizes = html.nodes(`//table[@id="torrent_table"]//tr[position()>1]/td[${sizeCol}]`);
|
||||
const seeders = html.nodes(`//table[@id="torrent_table"]//tr[position()>1]/td[${seedersCol}]`);
|
||||
if (sizes.length && seeders.length) {
|
||||
pageSeeding = sizes.length;
|
||||
for (let i = 0; i < sizes.length; i++) {
|
||||
const size = this.numFilesize(sizes[i]!.text().trim());
|
||||
const s = parseIntMp(seeders[i]?.text().trim());
|
||||
pageSeedingSize += size;
|
||||
pageSeedingInfo.push([s, size]);
|
||||
}
|
||||
}
|
||||
this.seeding += pageSeeding;
|
||||
this.seedingSize += pageSeedingSize;
|
||||
this.seedingInfo.push(...pageSeedingInfo);
|
||||
let nextPage: string | null = null;
|
||||
const nextPages = html.nodes('//ul[@class="pagination"]/li[contains(@class,"active")]/following-sibling::li');
|
||||
if (nextPages.length > 1) {
|
||||
const pageNum = nextPages[0]!.text().trim();
|
||||
if (/^\d+$/.test(pageNum)) nextPage = `${this._torrentSeedingPage}&page=${pageNum}`;
|
||||
}
|
||||
return nextPage;
|
||||
}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { createMockFetch, SUNNYPT_USER_ERROR, SUNNYPT_USER_MESSAGES, SUNNYPT_USER_PROFILE } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { SunnyPTSiteUserInfo } from "./sunnypt.ts";
|
||||
import { runParser } from "./testing.ts";
|
||||
|
||||
const API = "https://api.sunnypt.top/api/v1/mp";
|
||||
|
||||
describe("SunnyPTSiteUserInfo(MP sunnypt.py)", () => {
|
||||
it("profile + 未读消息分页(不标记已读)", async () => {
|
||||
const m = createMockFetch({
|
||||
[`GET ${API}/profile`]: { body: SUNNYPT_USER_PROFILE },
|
||||
[`GET ${API}/messages?unread_only=true&page=1`]: { body: SUNNYPT_USER_MESSAGES },
|
||||
});
|
||||
const info = await runParser(SunnyPTSiteUserInfo, {
|
||||
fetch: m.fetch,
|
||||
url: "https://sunnypt.top/",
|
||||
apikey: "sk",
|
||||
fetchMessages: true,
|
||||
});
|
||||
expect(info).toMatchObject({
|
||||
schema: "SunnyPT",
|
||||
userId: "12",
|
||||
username: "kate",
|
||||
userLevel: "Elite User",
|
||||
joinAt: "2023-04-05 06:07:08",
|
||||
upload: 4000,
|
||||
download: 2000,
|
||||
ratio: 2,
|
||||
bonus: 88.8,
|
||||
seeding: 7,
|
||||
seedingSize: 7000,
|
||||
leeching: 1,
|
||||
leechingSize: 100,
|
||||
messageUnread: 1,
|
||||
messageUnreadContents: [{ head: "通知", date: "2026-09-20 08:00:00", content: "内容" }],
|
||||
});
|
||||
expect(m.requests[0]!.headers["x-api-key"]).toBe("sk");
|
||||
expect(m.requests.every((r) => r.method === "GET")).toBe(true);
|
||||
});
|
||||
|
||||
it("API 错误:errMsg 为站点 msg", async () => {
|
||||
const m = createMockFetch({ [`GET ${API}/profile`]: { body: SUNNYPT_USER_ERROR } });
|
||||
const info = await runParser(SunnyPTSiteUserInfo, { fetch: m.fetch, url: "https://sunnypt.top/", apikey: "x" });
|
||||
expect(info.userId).toBeNull();
|
||||
expect(info.errMsg).toBe("API Key 无效");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/sunnypt.py @ e6820a8
|
||||
*
|
||||
* SunnyPT MoviePilot API 用户数据解析器(MP `SunnyPTSiteUserInfo`,apikey 模式)。
|
||||
* 函数对照:`_parse_site_page` → `_parseSitePage`(api_url,缺省 https://api.sunnypt.top/api/v1/mp)、
|
||||
* `_load_api_payload` → `_loadApiPayload`、`_parse_user_base_info` → `_parseUserBaseInfo`(profile)、
|
||||
* `_pase_unread_msgs` → `_paseUnreadMsgs`(messages?unread_only=true 分页,不标记已读)、
|
||||
* `_parse_message_unread_links` → `_parseMessageUnreadLinks`,其余为空实现。
|
||||
*
|
||||
* 差异:MP 消息元组带第 4 项 `sunnypt-message:<id>` 来源标识;本项目统一消息结构只有 (head, date, content),
|
||||
* 来源标识未保留。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase, urljoin } from "./base.ts";
|
||||
import { normalizeDatetime, pyFloat, pyInt } from "./common.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
export class SunnyPTSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.SunnyPT;
|
||||
override requestMode = "apikey" as const;
|
||||
|
||||
protected _parseSitePage(_htmlText: string): void {
|
||||
this._baseUrl = `${String(this._apiUrl || "https://api.sunnypt.top/api/v1/mp").replace(/\/+$/, "")}/`;
|
||||
this._userBasicPage = "profile";
|
||||
this._userDetailPage = null;
|
||||
this._userTrafficPage = null;
|
||||
this._torrentSeedingPage = null;
|
||||
this._userMailUnreadPage = "messages";
|
||||
this._sysMailUnreadPage = null;
|
||||
this._additionHeaders = {
|
||||
Accept: "application/json",
|
||||
"User-Agent": `${this._ua}`,
|
||||
"X-API-Key": this.apikey ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
/** 解析并校验 SunnyPT 通用 JSON 响应,成功返回 data */
|
||||
protected _loadApiPayload(htmlText: string, operation: string): Json | null {
|
||||
if (!htmlText) {
|
||||
this.errMsg = `${operation}失败，未收到 API 响应`;
|
||||
return null;
|
||||
}
|
||||
let payload: unknown;
|
||||
try {
|
||||
payload = JSON.parse(htmlText);
|
||||
} catch {
|
||||
this.errMsg = `${operation}失败，API 响应不是有效 JSON`;
|
||||
return null;
|
||||
}
|
||||
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
|
||||
this.errMsg = `${operation}失败，API 响应结构无效`;
|
||||
return null;
|
||||
}
|
||||
const p = payload as Json;
|
||||
if (String(p.code) !== "0") {
|
||||
this.errMsg = (p.msg as string) || `${operation}失败`;
|
||||
return null;
|
||||
}
|
||||
const data = p.data;
|
||||
return data && typeof data === "object" && !Array.isArray(data) ? (data as Json) : null;
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
const u = this._loadApiPayload(htmlText, "获取用户信息");
|
||||
if (!u) return;
|
||||
this.userid = u.id == null ? null : String(u.id);
|
||||
this.username = (u.username as string) ?? null;
|
||||
this.userLevel = (u.level as string) || String(u.class || "");
|
||||
this.joinAt = (normalizeDatetime(u.registered_at as string) as string | null | undefined) ?? null;
|
||||
this.upload = pyInt(u.uploaded || 0);
|
||||
this.download = pyInt(u.downloaded || 0);
|
||||
this.ratio = pyFloat(u.ratio || 0);
|
||||
this.bonus = pyFloat(u.bonus || 0);
|
||||
this.seeding = pyInt(u.seeding_count || 0);
|
||||
this.seedingSize = pyInt(u.seeding_size || 0);
|
||||
this.leeching = pyInt(u.leeching_count || 0);
|
||||
this.leechingSize = pyInt(u.leeching_size || 0);
|
||||
this.messageUnread = pyInt(u.unread_messages || 0);
|
||||
}
|
||||
|
||||
/** 分页读取未读消息;解析阶段不标记已读 */
|
||||
protected override async _paseUnreadMsgs(): Promise<void> {
|
||||
let page = 1;
|
||||
for (let guard = 0; guard < 50; guard++) {
|
||||
const query = new URLSearchParams({ unread_only: "true", page: String(page), page_size: "100" });
|
||||
const htmlText = await this._getPageContent(urljoin(this._baseUrl, `messages?${query}`));
|
||||
const hasMore = await this._parseMessageUnreadLinks(htmlText, []);
|
||||
if (!hasMore) return;
|
||||
page += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/** 解析未读消息列表并直接保存消息正文;有下一页返回占位串 */
|
||||
protected _parseMessageUnreadLinks(htmlText: string, _msgLinks: string[]): NextPage {
|
||||
const data = this._loadApiPayload(htmlText, "获取未读消息");
|
||||
if (!data) return null;
|
||||
this.messageUnread = pyInt(data.unread_count || 0);
|
||||
for (const m of (data.items || []) as unknown[]) {
|
||||
if (!m || typeof m !== "object" || Array.isArray(m) || !(m as Json).unread) continue;
|
||||
const msg = m as Json;
|
||||
const title = msg.title as string | null;
|
||||
const content = msg.content as string | null;
|
||||
const createdAt = normalizeDatetime(msg.created_at as string);
|
||||
if (title && content && createdAt) this.messageUnreadContents.push([title, createdAt, content]);
|
||||
}
|
||||
return data.has_more ? "next" : null;
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserDetailInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(_htmlText: string, _multiPage = false): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
/** 测试辅助:构造解析器并跑完整 parse 流程(非公开 API,仅测试使用)。 */
|
||||
import type { SiteParserClass } from "./base.ts";
|
||||
import type { SiteUserInfo, SiteUserParserOptions } from "./types.ts";
|
||||
|
||||
export async function runParser(
|
||||
cls: SiteParserClass,
|
||||
opts: Partial<SiteUserParserOptions> & { fetch: typeof globalThis.fetch },
|
||||
): Promise<SiteUserInfo> {
|
||||
const p = new cls({ siteName: "demo", url: "https://pt.example.com/", ...opts });
|
||||
await p.parse();
|
||||
return p.toUserInfo();
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { createMockFetch, TNODE_USER_INDEX, TNODE_USER_MAIN_INFO, TNODE_USER_SEEDING } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { runParser } from "./testing.ts";
|
||||
import { TNodeSiteUserInfo } from "./tnode.ts";
|
||||
|
||||
describe("TNodeSiteUserInfo(MP tnode.py)", () => {
|
||||
it("首页 csrf → getMainInfo / listTorrentActivity JSON", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: TNODE_USER_INDEX },
|
||||
"/api/user/getMainInfo": { body: TNODE_USER_MAIN_INFO },
|
||||
"/api/user/listTorrentActivity": { body: TNODE_USER_SEEDING },
|
||||
});
|
||||
const info = await runParser(TNodeSiteUserInfo, { fetch: m.fetch, ua: "UA/1" });
|
||||
expect(info).toMatchObject({
|
||||
schema: "TNode",
|
||||
userId: "99",
|
||||
username: "gina",
|
||||
userLevel: "Elite",
|
||||
joinAt: "2022-01-02 03:04:05",
|
||||
upload: 2000,
|
||||
download: 1000,
|
||||
ratio: 2,
|
||||
bonus: 12.5,
|
||||
messageUnread: 3,
|
||||
seeding: 2,
|
||||
seedingSize: 150,
|
||||
seedingInfo: [
|
||||
[3, 100],
|
||||
[1, 50],
|
||||
],
|
||||
});
|
||||
const main = m.requests.find((r) => r.url.pathname === "/api/user/getMainInfo")!;
|
||||
expect(main.headers["x-csrf-token"]).toBe("csrf-abc");
|
||||
});
|
||||
|
||||
it("接口 status≠200:不填用户数据", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: TNODE_USER_INDEX },
|
||||
"/api/user/getMainInfo": { body: { status: 401, msg: "unauthorized" } },
|
||||
"/api/user/listTorrentActivity": { body: { status: 401 } },
|
||||
});
|
||||
const info = await runParser(TNodeSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info.userId).toBeNull();
|
||||
expect(info.seeding).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/tnode.py @ e6820a8
|
||||
*
|
||||
* TNode 站点用户数据解析器(MP `TNodeSiteUserInfo`)。
|
||||
* 函数对照:`_parse_site_page` → `_parseSitePage`(首页取 x-csrf-token,改走 JSON 接口)、
|
||||
* `_parse_logged_in` → `_parseLoggedIn`(恒 True)、`_parse_user_base_info` → `_parseUserBaseInfo`、
|
||||
* `_parse_user_detail_info` → `_parseUserDetailInfo`(api/user/getMainInfo)、
|
||||
* `_parse_user_torrent_seeding_info` → `_parseUserTorrentSeedingInfo`(api/user/listTorrentActivity)、
|
||||
* 消息两函数(不支持)。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase } from "./base.ts";
|
||||
import { normalizeDatetime, reSearch } from "./common.ts";
|
||||
import { round3 } from "./nexus-php.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
function tryJson(text: string): unknown {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export class TNodeSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.TNode;
|
||||
|
||||
protected _parseSitePage(htmlText: string): void {
|
||||
const text = this._prepareHtmlText(htmlText);
|
||||
const csrf = reSearch(/<meta name="x-csrf-token" content="(.+?)">/, text);
|
||||
if (csrf) {
|
||||
this._additionHeaders = { "X-CSRF-TOKEN": csrf[1]! };
|
||||
this._userDetailPage = "api/user/getMainInfo";
|
||||
this._torrentSeedingPage = "api/user/listTorrentActivity?id=&type=seeding&page=1&size=20000";
|
||||
}
|
||||
}
|
||||
|
||||
/** 暂时跳过检测(与 MP 一致) */
|
||||
protected override _parseLoggedIn(_htmlText: unknown): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(_htmlText: string): void {
|
||||
this.username = this.userid;
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserDetailInfo(htmlText: string): void {
|
||||
const detail = tryJson(htmlText) as Json | undefined;
|
||||
if (detail === undefined || detail === null || typeof detail !== "object") return;
|
||||
if (detail.status !== 200) return;
|
||||
const u = (detail.data ?? {}) as Json;
|
||||
this.userid = u.id == null ? null : String(u.id);
|
||||
this.username = (u.username as string) ?? null;
|
||||
this.userLevel = (((u.class ?? {}) as Json).name as string) ?? null;
|
||||
this.joinAt = normalizeDatetime(String(u.regTime ?? 0)) ?? null;
|
||||
this.upload = Number(u.upload ?? 0);
|
||||
this.download = Number(u.download ?? 0);
|
||||
this.ratio = this.download <= 0 ? 0 : round3(this.upload / this.download);
|
||||
this.bonus = Number(u.bonus ?? 0);
|
||||
this.messageUnread = Number(u.unreadAdmin ?? 0) + Number(u.unreadInbox ?? 0) + Number(u.unreadSystem ?? 0);
|
||||
}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(htmlText: string, _multiPage = false): NextPage {
|
||||
const info = tryJson(htmlText);
|
||||
if (info === undefined) return null;
|
||||
if (!info || typeof info !== "object" || Array.isArray(info)) return null;
|
||||
const seedingInfo = info as Json;
|
||||
if (seedingInfo.status !== 200) return null;
|
||||
const torrents = (((seedingInfo.data ?? {}) as Json).torrents ?? []) as Json[];
|
||||
let pageSeedingSize = 0;
|
||||
const pageSeedingInfo: Array<[number, number]> = [];
|
||||
for (const t of torrents) {
|
||||
const size = Number(t.size ?? 0);
|
||||
const seeders = Number(t.seeding ?? 0);
|
||||
pageSeedingSize += size;
|
||||
pageSeedingInfo.push([seeders, size]);
|
||||
}
|
||||
this.seeding += torrents.length;
|
||||
this.seedingSize += pageSeedingSize;
|
||||
this.seedingInfo.push(...pageSeedingInfo);
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 系统/收件箱/管理员信息接口(api/message/listSystem 等)MP 未实现 */
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import {
|
||||
createMockFetch,
|
||||
TORRENTLEECH_USER_INDEX,
|
||||
TORRENTLEECH_USER_PROFILE,
|
||||
TORRENTLEECH_USER_SEEDING,
|
||||
} from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { runParser } from "./testing.ts";
|
||||
import { TorrentLeechSiteUserInfo } from "./torrent-leech.ts";
|
||||
|
||||
describe("TorrentLeechSiteUserInfo(MP torrent_leech.py)", () => {
|
||||
it("顶栏 onclick 取用户 ID;资料页流量/等级/注册/积分;tbody 做种表", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: TORRENTLEECH_USER_INDEX },
|
||||
"/profile/frank/view": { body: TORRENTLEECH_USER_PROFILE },
|
||||
"/profile/frank/seeding": { body: TORRENTLEECH_USER_SEEDING },
|
||||
});
|
||||
const info = await runParser(TorrentLeechSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info).toMatchObject({
|
||||
schema: "TorrentLeech",
|
||||
userId: "frank",
|
||||
username: "frank",
|
||||
upload: 5 * 1024 ** 4,
|
||||
download: 1024 ** 4,
|
||||
ratio: 5,
|
||||
userLevel: "VIP",
|
||||
joinAt: "2016-06-07 08:09:10",
|
||||
bonus: 98765.4,
|
||||
seeding: 1,
|
||||
seedingSize: 2 * 1024 ** 3,
|
||||
seedingInfo: [[11, 2 * 1024 ** 3]],
|
||||
});
|
||||
// 流量页置空,不请求 index.php
|
||||
expect(m.requests.map((r) => r.url.pathname)).toEqual(["/", "/profile/frank/view", "/profile/frank/seeding"]);
|
||||
});
|
||||
|
||||
it("已登录但取不到用户 ID:errMsg=未获取到用户ID,不再请求", async () => {
|
||||
const m = createMockFetch({ "/": { body: `<html><body><a href="/logout">Logout</a></body></html>` } });
|
||||
const info = await runParser(TorrentLeechSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info.errMsg).toBe("未获取到用户ID");
|
||||
expect(m.requests).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/torrent_leech.py @ e6820a8
|
||||
*
|
||||
* TorrentLeech 站点用户信息解析器(MP `TorrentLeechSiteUserInfo`)。
|
||||
* 函数对照:`_parse_site_page` → `_parseSitePage`、`_parse_user_base_info` → `_parseUserBaseInfo`、
|
||||
* `_parse_user_traffic_info` → `_parseUserTrafficInfo`、`_parse_user_detail_info` → `_parseUserDetailInfo`、
|
||||
* `_parse_user_torrent_seeding_info` → `_parseUserTorrentSeedingInfo`、消息两函数(不支持)。
|
||||
* 注意:做种表 XPath 使用 `//tbody/tr`,本项目解析器不自动插入 tbody,须站点原文带 tbody(与 lxml 行为一致)。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase } from "./base.ts";
|
||||
import { normalizeDatetime, parseFloatMp, parseIntMp, reSearch } from "./common.ts";
|
||||
import { hasChildElements, parseHtml } from "./dom.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
export class TorrentLeechSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.TorrentLeech;
|
||||
|
||||
/** 解析当前用户 ID 并初始化用户资料页面地址 */
|
||||
protected _parseSitePage(htmlText: string): void {
|
||||
const text = this._prepareHtmlText(htmlText);
|
||||
const html = parseHtml(text);
|
||||
let currentUserid: string | null = null;
|
||||
if (hasChildElements(html)) {
|
||||
const routes = html.strings(
|
||||
'//span[contains(concat(" ", normalize-space(@class), " "), " centerTopBar ")]//*[@onclick]/@onclick',
|
||||
);
|
||||
for (const route of routes) {
|
||||
const m = reSearch(/\/profile\/([^/]+)\/view/, route);
|
||||
if (m) {
|
||||
currentUserid = m[1]!.trim();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
const detail = reSearch(/\/profile\/([^/]+)\//, text);
|
||||
const fallback = detail ? detail[1]!.trim() : null;
|
||||
this.userid = currentUserid || fallback;
|
||||
if (!this.userid) {
|
||||
this.errMsg = "未获取到用户ID";
|
||||
this._userDetailPage = null;
|
||||
this._userTrafficPage = null;
|
||||
this._torrentSeedingPage = null;
|
||||
return;
|
||||
}
|
||||
this._userDetailPage = `profile/${this.userid}/view`;
|
||||
this._userTrafficPage = null;
|
||||
this._torrentSeedingPage = `profile/${this.userid}/seeding`;
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(_htmlText: string): void {
|
||||
this.username = this.userid;
|
||||
}
|
||||
|
||||
/** 解析用户资料页中的用户名、流量、等级、注册时间和积分 */
|
||||
protected _parseUserTrafficInfo(htmlText: string): void {
|
||||
const html = parseHtml(this._prepareHtmlText(htmlText));
|
||||
if (!hasChildElements(html)) return;
|
||||
let username = html.strings(
|
||||
'//div[contains(concat(" ", normalize-space(@class), " "), " profile-username ")]/text()',
|
||||
);
|
||||
if (!username.length) {
|
||||
username = html.strings(
|
||||
'//table[contains(@class, "profileViewTable")]//tr/td[normalize-space()="Username"]/following-sibling::td[1]/text()',
|
||||
);
|
||||
}
|
||||
if (username.length && username[0]!.trim()) this.username = username[0]!.trim();
|
||||
const up = html.strings('//div[contains(@class,"profile-uploaded")]//span/text()');
|
||||
if (up.length) this.upload = this.numFilesize(up[0]);
|
||||
const down = html.strings('//div[contains(@class,"profile-downloaded")]//span/text()');
|
||||
if (down.length) this.download = this.numFilesize(down[0]);
|
||||
const ratio = html.strings('//div[contains(@class,"profile-ratio")]//span/text()');
|
||||
if (ratio.length) this.ratio = parseFloatMp(ratio[0]!.replaceAll("∞", "0"));
|
||||
const level = html.strings(
|
||||
'//table[contains(@class, "profileViewTable")]//tr/td[normalize-space()="Class"]/following-sibling::td[1]/text()',
|
||||
);
|
||||
if (level.length) this.userLevel = level[0]!.trim();
|
||||
const joinAt = html.strings(
|
||||
'//table[contains(@class, "profileViewTable")]//tr/td[normalize-space()="Registration date"]/following-sibling::td[1]/text()',
|
||||
);
|
||||
if (joinAt.length) this.joinAt = normalizeDatetime(joinAt[0]!.trim()) ?? null;
|
||||
const bonus = html.strings('//span[contains(@class, "total-TL-points")]/text()');
|
||||
if (bonus.length) this.bonus = parseFloatMp(bonus[0]!.trim());
|
||||
}
|
||||
|
||||
protected _parseUserDetailInfo(htmlText: string): void {
|
||||
this._parseUserTrafficInfo(htmlText);
|
||||
}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(htmlText: string, _multiPage = false): NextPage {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return null;
|
||||
const sizeCol = 2;
|
||||
const seedersCol = 7;
|
||||
let pageSeeding = 0;
|
||||
let pageSeedingSize = 0;
|
||||
const pageSeedingInfo: Array<[number, number]> = [];
|
||||
const sizes = html.nodes(`//tbody/tr/td[${sizeCol}]`);
|
||||
const seeders = html.strings(`//tbody/tr/td[${seedersCol}]/text()`);
|
||||
if (sizes.length && seeders.length) {
|
||||
pageSeeding = sizes.length;
|
||||
for (let i = 0; i < sizes.length; i++) {
|
||||
const size = this.numFilesize(sizes[i]!.text().trim());
|
||||
pageSeedingSize += size;
|
||||
pageSeedingInfo.push([parseIntMp(seeders[i]), size]);
|
||||
}
|
||||
}
|
||||
this.seeding += pageSeeding;
|
||||
this.seedingSize += pageSeedingSize;
|
||||
this.seedingInfo.push(...pageSeedingInfo);
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/__init__.py @ e6820a8
|
||||
* @mp-source app/schemas/site.py @ e6820a8
|
||||
*
|
||||
* 站点用户信息解析类型。
|
||||
*
|
||||
* 来源:MoviePilot e6820a8(GPLv3)`app/modules/indexer/parser/__init__.py` `SiteSchema`、
|
||||
* `app/schemas/site.py` `SiteUserData`、`app/modules/indexer/__init__.py` `refresh_userdata`。
|
||||
*/
|
||||
|
||||
/** MP `SiteSchema` 枚举(值与 MP 完全一致,站点定义 `schema` 字段直接写这些值)。 */
|
||||
export const SiteSchema = {
|
||||
DiscuzX: "DiscuzX",
|
||||
Gazelle: "Gazelle",
|
||||
Ipt: "IPTorrents",
|
||||
NexusPhp: "NexusPhp",
|
||||
NexusProject: "NexusProject",
|
||||
NexusRabbit: "NexusRabbit",
|
||||
NexusHhanclub: "NexusHhanclub",
|
||||
NexusAudiences: "NexusAudiences",
|
||||
SmallHorse: "Small Horse",
|
||||
Unit3d: "Unit3d",
|
||||
TorrentLeech: "TorrentLeech",
|
||||
FileList: "FileList",
|
||||
TNode: "TNode",
|
||||
MTorrent: "MTorrent",
|
||||
Yema: "Yema",
|
||||
HDDolby: "HDDolby",
|
||||
Zhixing: "Zhixing",
|
||||
Bitpt: "Bitpt",
|
||||
RousiPro: "RousiPro",
|
||||
SunnyPT: "SunnyPT",
|
||||
} as const;
|
||||
export type SiteSchemaValue = (typeof SiteSchema)[keyof typeof SiteSchema];
|
||||
|
||||
/** MP `SiteUnreadMessage`/`message_unread_contents` 元素 (head, date, content) */
|
||||
export interface SiteUnreadMessage {
|
||||
head: string | null;
|
||||
date: string | null;
|
||||
content: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 统一用户信息(MP `SiteUserData` 的 camelCase 投影)。
|
||||
* 数值字段沿用 MP 语义:解析不到为 0(MP 默认值),`userLevel/joinAt/username` 为 null。
|
||||
* `raw` 保存解析器实例的全部公开字段(MP `to_dict()` 等价,snake_case),不含任何凭据。
|
||||
*/
|
||||
export interface SiteUserInfo {
|
||||
/** 解析时使用的 schema(可能是回退后的 schema) */
|
||||
schema: SiteSchemaValue;
|
||||
domain: string | null;
|
||||
userId: string | null;
|
||||
username: string | null;
|
||||
userLevel: string | null;
|
||||
joinAt: string | null;
|
||||
upload: number;
|
||||
download: number;
|
||||
ratio: number;
|
||||
bonus: number;
|
||||
seeding: number;
|
||||
seedingSize: number;
|
||||
leeching: number;
|
||||
leechingSize: number;
|
||||
/** MP `seeding_info`:[做种人数, 种子大小] */
|
||||
seedingInfo: Array<[number, number]>;
|
||||
messageUnread: number;
|
||||
messageUnreadContents: SiteUnreadMessage[];
|
||||
/** H&R 未达标数:MP 解析器不提供,保留字段供站点扩展 */
|
||||
hrCount?: number;
|
||||
/** 解析错误说明(MP `err_msg`,如"未检测到已登陆") */
|
||||
errMsg: string | null;
|
||||
raw: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** 请求上下文(MP `SiteParserBase.__init__` 参数;凭据只在内存中使用,不进入结果/日志)。 */
|
||||
export interface SiteUserParserOptions {
|
||||
siteName: string;
|
||||
/** 站点前端地址(MP `url`) */
|
||||
url: string;
|
||||
cookie?: string | null;
|
||||
apikey?: string | null;
|
||||
token?: string | null;
|
||||
ua?: string | null;
|
||||
/** 站点独立 API Base URL(MP `api_url`,SunnyPT 等) */
|
||||
apiUrl?: string | null;
|
||||
/** 是否读取未读消息正文(MP 全局设置 `SITE_MESSAGE`);默认 false。注意 MTorrent 读取后会标记已读 */
|
||||
fetchMessages?: boolean;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
/** 单请求超时(MP 固定 60s) */
|
||||
timeoutMs?: number;
|
||||
/** 共享站点限流器(与 list/search 共用) */
|
||||
limiter?: { acquire(signal?: AbortSignal): Promise<void> };
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { createMockFetch, UNIT3D_USER_DETAIL, UNIT3D_USER_INDEX, UNIT3D_USER_SEEDING } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { runParser } from "./testing.ts";
|
||||
import { Unit3dSiteUserInfo } from "./unit3d.ts";
|
||||
|
||||
const GiB = 1024 ** 3;
|
||||
|
||||
describe("Unit3dSiteUserInfo(MP unit3d.py)", () => {
|
||||
it("用户名来自 settings 链接;流量/等级/注册日期/做种列按 thead 定位", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: UNIT3D_USER_INDEX },
|
||||
"/users/carol": { body: UNIT3D_USER_DETAIL },
|
||||
"/index.php": { body: UNIT3D_USER_INDEX },
|
||||
"/users/carol/active": { body: UNIT3D_USER_SEEDING },
|
||||
});
|
||||
const info = await runParser(Unit3dSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info).toMatchObject({
|
||||
schema: "Unit3d",
|
||||
username: "carol",
|
||||
userLevel: "Elite",
|
||||
joinAt: "2020-02-03 04:05:06",
|
||||
bonus: 5678.9,
|
||||
upload: 2 * 1024 ** 4,
|
||||
download: 500 * GiB,
|
||||
ratio: 4.1,
|
||||
seeding: 2,
|
||||
seedingSize: 4 * GiB,
|
||||
seedingInfo: [
|
||||
[9, 3 * GiB],
|
||||
[1, GiB],
|
||||
],
|
||||
});
|
||||
expect(m.requests.find((r) => r.url.pathname === "/users/carol/active")!.url.searchParams.get("seeding")).toBe(
|
||||
"include",
|
||||
);
|
||||
});
|
||||
|
||||
it("首页取不到内容(HTTP 404):未登录并记录请求失败", async () => {
|
||||
const m = createMockFetch({});
|
||||
const info = await runParser(Unit3dSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info.username).toBeNull();
|
||||
expect(info.errMsg).toContain("未检测到已登陆");
|
||||
expect(info.raw.http_errors).toEqual([expect.stringContaining("HTTP 404")]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/unit3d.py @ e6820a8
|
||||
*
|
||||
* UNIT3D 站点用户数据解析器(MP `Unit3dSiteUserInfo`)。
|
||||
* 函数对照:`_parse_user_base_info` → `_parseUserBaseInfo`、`_parse_site_page`(空)、
|
||||
* `_parse_user_detail_info` → `_parseUserDetailInfo`、`_parse_user_torrent_seeding_info` → `_parseUserTorrentSeedingInfo`、
|
||||
* `_parse_user_traffic_info` → `_parseUserTrafficInfo`、消息两函数(不支持)。XPath 与正则逐字保留。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase } from "./base.ts";
|
||||
import { normalizeDatetime, parseFloatMp, parseIntMp, reSearch } from "./common.ts";
|
||||
import { hasChildElements, parseHtml } from "./dom.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
export class Unit3dSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.Unit3d;
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
const html = parseHtml(this._prepareHtmlText(htmlText));
|
||||
if (!html) return;
|
||||
const hrefs = html.strings('//a[contains(@href, "/users/") and contains(@href, "settings")]/@href');
|
||||
if (hrefs.length) {
|
||||
const m = reSearch(/\/users\/(.+)\/settings/, hrefs[0]);
|
||||
if (m?.[0].trim()) {
|
||||
this.username = m[1]!;
|
||||
this._torrentSeedingPage = `/users/${this.username}/active?perPage=100&client=&seeding=include`;
|
||||
this._userDetailPage = `/users/${this.username}`;
|
||||
}
|
||||
}
|
||||
const bonus = html.nodes('//a[contains(@href, "bonus/earnings")]');
|
||||
if (bonus.length) {
|
||||
const m = reSearch(/([\d,.]+)/, bonus[0]!.text());
|
||||
if (m?.[1]?.trim()) this.bonus = parseFloatMp(m[1]);
|
||||
}
|
||||
}
|
||||
|
||||
protected _parseSitePage(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserDetailInfo(htmlText: string): void {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return;
|
||||
const levels = html.strings('//div[contains(@class, "content")]//span[contains(@class, "badge-user")]/text()');
|
||||
if (levels.length) this.userLevel = levels[0]!.trim();
|
||||
const joinAt = html.strings(
|
||||
'//div[contains(@class, "content")]//h4[contains(text(), "注册日期") or contains(text(), "註冊日期") or contains(text(), "Registration date")]/text()',
|
||||
);
|
||||
if (joinAt.length) {
|
||||
this.joinAt =
|
||||
normalizeDatetime(
|
||||
joinAt[0]!.replaceAll("注册日期", "").replaceAll("註冊日期", "").replaceAll("Registration date", ""),
|
||||
) ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(htmlText: string, _multiPage = false): NextPage {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return null;
|
||||
let sizeCol = 9;
|
||||
let seedersCol = 2;
|
||||
if (html.nodes('//thead//th[contains(@class,"size")]').length) {
|
||||
sizeCol = html.nodes('//thead//th[contains(@class,"size")][1]/preceding-sibling::th').length + 1;
|
||||
}
|
||||
if (html.nodes('//thead//th[contains(@class,"seeders")]').length) {
|
||||
seedersCol = html.nodes('//thead//th[contains(@class,"seeders")]/preceding-sibling::th').length + 1;
|
||||
}
|
||||
let pageSeeding = 0;
|
||||
let pageSeedingSize = 0;
|
||||
const pageSeedingInfo: Array<[number, number]> = [];
|
||||
const sizes = html.nodes(`//tr[position()]/td[${sizeCol}]`);
|
||||
const seeders = html.nodes(`//tr[position()]/td[${seedersCol}]`);
|
||||
if (sizes.length && seeders.length) {
|
||||
pageSeeding = sizes.length;
|
||||
for (let i = 0; i < sizes.length; i++) {
|
||||
const size = this.numFilesize(sizes[i]!.text().trim());
|
||||
pageSeedingSize += size;
|
||||
pageSeedingInfo.push([parseIntMp(seeders[i]?.text().trim()), size]);
|
||||
}
|
||||
}
|
||||
this.seeding += pageSeeding;
|
||||
this.seedingSize += pageSeedingSize;
|
||||
this.seedingInfo.push(...pageSeedingInfo);
|
||||
|
||||
let nextPage: string | null = null;
|
||||
const nextPages = html.nodes('//ul[@class="pagination"]/li[contains(@class,"active")]/following-sibling::li');
|
||||
if (nextPages.length > 1) {
|
||||
const pageNum = nextPages[0]!.text().trim();
|
||||
if (/^\d+$/.test(pageNum)) nextPage = `${this._torrentSeedingPage}&page=${pageNum}`;
|
||||
}
|
||||
return nextPage;
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(htmlText: string): void {
|
||||
const text = this._prepareHtmlText(htmlText);
|
||||
const up = reSearch(/[^总]上[传傳]量?[:：_<>/a-zA-Z-="'\s#;]+([\d,.\s]+[KMGTPI]*B)/i, text);
|
||||
this.upload = up ? this.numFilesize(up[1]!.trim()) : 0;
|
||||
const down = reSearch(/[^总子影力]下[载載]量?[:：_<>/a-zA-Z-="'\s#;]+([\d,.\s]+[KMGTPI]*B)/i, text);
|
||||
this.download = down ? this.numFilesize(down[1]!.trim()) : 0;
|
||||
const ratio = reSearch(/分享率[:：_<>/a-zA-Z-="'\s#;]+([\d,.\s]+)/, text);
|
||||
this.ratio = ratio?.[1]?.trim() ? parseFloatMp(ratio[1]) : 0.0;
|
||||
}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { createMockFetch, YEMA_USER_BASIC, YEMA_USER_ERROR } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { runParser } from "./testing.ts";
|
||||
import { YemaSiteUserInfo } from "./yema.ts";
|
||||
|
||||
const URL_ = "https://www.yemapt.org/openApi/user/fetchBasicInfo.json";
|
||||
|
||||
describe("YemaSiteUserInfo(MP yema.py)", () => {
|
||||
it("POST fetchBasicInfo(JSON 体,Authorization=AuthKey):促销流量为上传/下载", async () => {
|
||||
const m = createMockFetch({ [`POST ${URL_}`]: { body: YEMA_USER_BASIC } });
|
||||
const info = await runParser(YemaSiteUserInfo, { fetch: m.fetch, url: "https://www.yemapt.org/", apikey: "ak" });
|
||||
expect(info).toMatchObject({
|
||||
schema: "Yema",
|
||||
userId: "321",
|
||||
username: "leo",
|
||||
userLevel: "3",
|
||||
joinAt: "2024-05-06 07:08:09",
|
||||
upload: 3000,
|
||||
download: 1000,
|
||||
ratio: 3,
|
||||
bonus: 12.3,
|
||||
});
|
||||
const req = m.requests[0]!;
|
||||
expect(req.headers.authorization).toBe("ak");
|
||||
expect(req.headers["content-type"]).toBe("application/json");
|
||||
expect(req.body).toBe("{}");
|
||||
});
|
||||
|
||||
it("success=false:errMsg 为 errorMessage", async () => {
|
||||
const m = createMockFetch({ [`POST ${URL_}`]: { body: YEMA_USER_ERROR } });
|
||||
const info = await runParser(YemaSiteUserInfo, { fetch: m.fetch, url: "https://www.yemapt.org/", apikey: "x" });
|
||||
expect(info.userId).toBeNull();
|
||||
expect(info.errMsg).toBe("AuthKey 无效");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/yema.py @ e6820a8
|
||||
*
|
||||
* YemaPT 开放 API 用户数据解析器(MP `YemaSiteUserInfo`,apikey 模式)。
|
||||
* 函数对照:`_parse_site_page` → `_parseSitePage`(POST openApi/user/fetchBasicInfo.json,Authorization 头为 AuthKey)、
|
||||
* `_parse_user_base_info` → `_parseUserBaseInfo`(促销流量作为上传/下载),其余为空实现(开放 API 不提供)。
|
||||
*/
|
||||
import { type NextPage, SiteParserBase } from "./base.ts";
|
||||
import { normalizeDatetime, pyFloat, pyInt } from "./common.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
export class YemaSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.Yema;
|
||||
override requestMode = "apikey" as const;
|
||||
|
||||
protected _parseSitePage(_htmlText: string): void {
|
||||
this._userBasicPage = "openApi/user/fetchBasicInfo.json";
|
||||
this._userBasicParams = {};
|
||||
this._userBasicMethod = "post";
|
||||
this._userDetailPage = null;
|
||||
this._userTrafficPage = null;
|
||||
this._torrentSeedingPage = null;
|
||||
this._sysMailUnreadPage = null;
|
||||
this._userMailUnreadPage = null;
|
||||
this._additionHeaders = {
|
||||
Authorization: this.apikey ?? "",
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json, text/plain, */*",
|
||||
"User-Agent": `${this._ua}`,
|
||||
};
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
if (!htmlText) {
|
||||
this.errMsg = "获取用户信息失败，未收到开放 API 响应";
|
||||
return;
|
||||
}
|
||||
let payload: unknown;
|
||||
try {
|
||||
payload = JSON.parse(htmlText);
|
||||
} catch {
|
||||
this.errMsg = "获取用户信息失败，开放 API 响应不是有效 JSON";
|
||||
return;
|
||||
}
|
||||
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
|
||||
this.errMsg = "获取用户信息失败，开放 API 响应结构无效";
|
||||
return;
|
||||
}
|
||||
const p = payload as Json;
|
||||
if (!p.success || !p.data || typeof p.data !== "object" || Array.isArray(p.data)) {
|
||||
this.errMsg = (p.errorMessage as string) || "获取用户信息失败";
|
||||
return;
|
||||
}
|
||||
const u = p.data as Json;
|
||||
this.userid = u.id == null ? null : String(u.id);
|
||||
this.username = (u.name as string) ?? null;
|
||||
this.userLevel = u.level != null ? String(u.level) : null;
|
||||
this.joinAt = (normalizeDatetime(u.registerTime as string) as string | null | undefined) ?? null;
|
||||
this.upload = pyInt(u.promotionUploadSize || 0);
|
||||
this.download = pyInt(u.promotionDownloadSize || 0);
|
||||
this.ratio = Math.round((this.upload / (this.download || 1)) * 100) / 100;
|
||||
this.bonus = pyFloat(u.bonus || 0);
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserDetailInfo(_htmlText: string): void {}
|
||||
|
||||
protected _parseUserTorrentSeedingInfo(_htmlText: string, _multiPage = false): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import {
|
||||
createMockFetch,
|
||||
ZHIXING_LOGIN_PAGE,
|
||||
ZHIXING_USER_BASIC,
|
||||
ZHIXING_USER_INDEX,
|
||||
ZHIXING_USER_SEEDING_EMPTY,
|
||||
ZHIXING_USER_SEEDING_P1,
|
||||
} from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { runParser } from "./testing.ts";
|
||||
import { ZhixingSiteUserInfo } from "./zhixing.ts";
|
||||
|
||||
const GiB = 1024 ** 3;
|
||||
|
||||
describe("ZhixingSiteUserInfo(MP zhixing.py)", () => {
|
||||
it("#um 登录判定 → user/{uid}/ 键值 → seeding/p{n} 逐页累加", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: ZHIXING_USER_INDEX },
|
||||
"/user/321/": { body: ZHIXING_USER_BASIC },
|
||||
"/user/321/seeding/p1": { body: ZHIXING_USER_SEEDING_P1 },
|
||||
"/user/321/seeding/p2": { body: ZHIXING_USER_SEEDING_EMPTY },
|
||||
});
|
||||
const info = await runParser(ZhixingSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info).toMatchObject({
|
||||
schema: "Zhixing",
|
||||
userId: "321",
|
||||
username: "zed",
|
||||
userLevel: "Lv2",
|
||||
joinAt: "2023-01-01 10:00:00",
|
||||
upload: Math.round(1.5 * 1024 ** 4),
|
||||
download: 123456,
|
||||
ratio: 12.5,
|
||||
bonus: 88.8,
|
||||
seeding: 2,
|
||||
seedingSize: 30 * GiB,
|
||||
});
|
||||
});
|
||||
|
||||
it("做种页全空:回落基础信息的当前保种数量/容量", async () => {
|
||||
const m = createMockFetch({
|
||||
"/": { body: ZHIXING_USER_INDEX },
|
||||
"/user/321/": { body: ZHIXING_USER_BASIC },
|
||||
"/user/321/seeding/p1": { body: ZHIXING_USER_SEEDING_EMPTY },
|
||||
});
|
||||
const info = await runParser(ZhixingSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info).toMatchObject({ seeding: 7, seedingSize: 70 * GiB });
|
||||
});
|
||||
|
||||
it("未登录(无 #um):只请求首页", async () => {
|
||||
const m = createMockFetch({ "/": { body: ZHIXING_LOGIN_PAGE } });
|
||||
const info = await runParser(ZhixingSiteUserInfo, { fetch: m.fetch });
|
||||
expect(info.userId).toBeNull();
|
||||
expect(m.requests).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,159 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/modules/indexer/parser/zhixing.py @ e6820a8
|
||||
*
|
||||
* 知行(pt.zhixing.bjtu.edu.cn)站点用户信息解析器(原作者 ThedoRap)。
|
||||
*
|
||||
* 函数对照(`ZhixingSiteUserInfo`):`_parse_site_page` → `_parseSitePage`、`_parse_logged_in` → `_parseLoggedIn`
|
||||
* (`#um`)、`_parse_user_base_info` → `_parseUserBaseInfo`(`div.user-details-tabs p` 键值)、
|
||||
* `_parse_user_torrent_seeding_page_info` → `parseUserTorrentSeedingPageInfo`、`parse` → `parse`(先走基类流程,
|
||||
* 再从首页取 uid → `user/{uid}/` → `user/{uid}/seeding/p{n}` 逐页累加,空页结束,无结果回落基础信息)。
|
||||
* BeautifulSoup 以 cheerio 等价实现。
|
||||
*
|
||||
* 与 MP 的偏离:
|
||||
* 1. MP 末尾把全部字段转成字符串(避免其 UI join 报错);这里保持数值类型(`toUserInfo` 语义相同)。
|
||||
* 2. 做种翻页设上限 50 页(MP `while True`)。
|
||||
*/
|
||||
import { load } from "cheerio";
|
||||
import { type NextPage, SiteParserBase, urljoin } from "./base.ts";
|
||||
import { normalizeDatetime, pyFloat, pyInt } from "./common.ts";
|
||||
import { SiteSchema, type SiteSchemaValue } from "./types.ts";
|
||||
|
||||
const MAX_PAGES = 50;
|
||||
|
||||
export class ZhixingSiteUserInfo extends SiteParserBase {
|
||||
static override schema: SiteSchemaValue | null = SiteSchema.Zhixing;
|
||||
|
||||
protected _torrentSeedingBase: string | undefined;
|
||||
protected _basicInfo: Record<string, string> | undefined;
|
||||
|
||||
protected _parseSitePage(_htmlText: string): void {
|
||||
this._userBasicPage = "user/{uid}/";
|
||||
this._userDetailPage = null;
|
||||
this._userBasicParams = {};
|
||||
this._userTrafficPage = null;
|
||||
this._sysMailUnreadPage = null;
|
||||
this._userMailUnreadPage = null;
|
||||
this._mailUnreadParams = {};
|
||||
this._torrentSeedingBase = "user/{uid}/seeding";
|
||||
this._torrentSeedingParams = {};
|
||||
this._torrentSeedingHeaders = {};
|
||||
this._additionHeaders = {};
|
||||
}
|
||||
|
||||
/** 通过是否存在 #um 用户信息判断登录 */
|
||||
protected override _parseLoggedIn(htmlText: unknown): boolean {
|
||||
return load(String(htmlText ?? ""))("#um").length > 0;
|
||||
}
|
||||
|
||||
private numFilesizeSafe(s: string | undefined): number {
|
||||
let v = s;
|
||||
if (v) {
|
||||
v = v.trim();
|
||||
if (/^\d+(\.\d+)?$/.test(v)) v += " B";
|
||||
}
|
||||
return v ? this.numFilesize(v) : 0;
|
||||
}
|
||||
|
||||
protected _parseUserBaseInfo(htmlText: string): void {
|
||||
if (!htmlText) return;
|
||||
const $ = load(htmlText);
|
||||
const info: Record<string, string> = {};
|
||||
$("div.user-details-tabs").each((_, tab) => {
|
||||
$(tab)
|
||||
.find("p")
|
||||
.each((_i, p) => {
|
||||
const text = $(p).text().trim();
|
||||
let parts: string[];
|
||||
if (text.includes("：")) parts = [text.slice(0, text.indexOf("：")), text.slice(text.indexOf("：") + 1)];
|
||||
else if (text.includes(":")) parts = [text.slice(0, text.indexOf(":")), text.slice(text.indexOf(":") + 1)];
|
||||
else return;
|
||||
const key = parts[0]!.trim();
|
||||
const valueText = parts[1]!.trim();
|
||||
info[key] = valueText.split(/\s*\(/)[0]!.trim().split("查看")[0]!.trim();
|
||||
});
|
||||
});
|
||||
this._basicInfo = info;
|
||||
this.userid = info.UID ?? null;
|
||||
this.username = info.用户名 ?? null;
|
||||
this.userLevel = info.用户组 ?? null;
|
||||
this.joinAt = "注册时间" in info ? (normalizeDatetime(info.注册时间) ?? null) : null;
|
||||
this.upload = "上传流量" in info ? this.numFilesizeSafe(info.上传流量) : 0;
|
||||
this.download = "下载流量" in info ? this.numFilesizeSafe(info.下载流量) : 0;
|
||||
this.ratio = "共享率" in info ? pyFloat(info.共享率) : 0;
|
||||
this.bonus = "保种积分" in info ? pyFloat(info.保种积分) : 0.0;
|
||||
this.messageUnread = 0; // 暂无消息解析
|
||||
this.seeding = "当前保种数量" in info ? pyInt(info.当前保种数量) : 0;
|
||||
this.seedingSize = "当前保种容量" in info ? this.numFilesizeSafe(info.当前保种容量) : 0;
|
||||
}
|
||||
|
||||
protected _parseUserTrafficInfo(_htmlText: string): void {}
|
||||
protected _parseUserDetailInfo(_htmlText: string): void {}
|
||||
|
||||
/** 单页做种:返回 [数量, 大小] */
|
||||
parseUserTorrentSeedingPageInfo(htmlText: string): [number, number] {
|
||||
if (!htmlText) return [0, 0];
|
||||
const $ = load(htmlText);
|
||||
let count = 0;
|
||||
let size = 0;
|
||||
$("tr")
|
||||
.filter((_, el) => /^t\d+/.test($(el).attr("id") ?? ""))
|
||||
.each((_, tr) => {
|
||||
const td = $(tr).find("td.r").first();
|
||||
if (td.length) {
|
||||
const a = td.find("a").first();
|
||||
const sizeText = a.length ? a.text() : td.text().trim();
|
||||
count += 1;
|
||||
size += this.numFilesize(sizeText);
|
||||
}
|
||||
});
|
||||
return [count, size];
|
||||
}
|
||||
|
||||
protected _parseMessageUnreadLinks(_htmlText: string, _msgLinks: string[]): NextPage {
|
||||
return null;
|
||||
}
|
||||
protected _parseMessageContent(_htmlText: string): [string | null, string | null, string | null] {
|
||||
return [null, null, null];
|
||||
}
|
||||
protected _parseUserTorrentSeedingInfo(_htmlText: string): NextPage {
|
||||
return null;
|
||||
}
|
||||
|
||||
override async parse(): Promise<void> {
|
||||
await super.parse();
|
||||
// 先从首页解析 userid
|
||||
if (this._indexHtml) {
|
||||
const $ = load(String(this._indexHtml));
|
||||
const link = $("a")
|
||||
.filter((_, el) => /\/user\/\d+\//.test($(el).attr("href") ?? ""))
|
||||
.first();
|
||||
if (link.length) {
|
||||
const m = /\/user\/(\d+)\//.exec(link.attr("href") ?? "");
|
||||
if (m) this.userid = m[1]!;
|
||||
}
|
||||
}
|
||||
if (this.userid) {
|
||||
if (this._userBasicPage) {
|
||||
const basicUrl = this._userBasicPage.replace("{uid}", this.userid);
|
||||
this._parseUserBaseInfo(await this._getPageContent(urljoin(this._baseUrl, basicUrl)));
|
||||
}
|
||||
if (this._torrentSeedingBase) {
|
||||
this.seeding = 0;
|
||||
this.seedingSize = 0;
|
||||
const baseUrl = urljoin(this._baseUrl, this._torrentSeedingBase.replace("{uid}", this.userid!));
|
||||
for (let page = 1; page <= MAX_PAGES; page++) {
|
||||
const [n, s] = this.parseUserTorrentSeedingPageInfo(await this._getPageContent(`${baseUrl}/p${page}`));
|
||||
this.seeding += n;
|
||||
this.seedingSize += s;
|
||||
if (n === 0) break;
|
||||
}
|
||||
if (this.seeding === 0 && this._basicInfo) {
|
||||
this.seeding = pyInt(this._basicInfo.当前保种数量 ?? 0);
|
||||
this.seedingSize = this.numFilesizeSafe(this._basicInfo.当前保种容量 ?? "");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user