feat(adapters): port MoviePilot open site framework (7 spiders, 20 user parsers) + MovieClaw v0.27.0 site definitions + mp-sync tools (W1-3c)
- sites/spiders: Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema (+mTorrent wrapper), specialized SiteAdapter, indirect download resolved at download time - sites/user-parsers: SiteParserBase + all 20 MP SiteSchema parsers (lxml XPath via xpath+xmldom), refreshSiteUserData with schema fallback - sites/definitions: zod SiteDefinition schemas, 23 MovieClaw v0.27.0 (MIT) definitions via scripts/mp-sync/movieclaw-import.mjs, registry getSiteDefinition/listSiteDefinitions, NOTICE - scripts/mp-sync: ledger.json (upstream sha256), check.mjs, build-ledger.mjs; docs/porting/mp-sync-ledger.md - license-and-provenance: add copied-mit kind, re-anchor MovieClaw rows to v0.27.0 Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
cf85481d95
commit
d9f2af2f3c
166 files changed
+16577
-186
No files matched your search
@@ -0,0 +1,211 @@
|
||||
/**
|
||||
* SPDX-License-Identifier: GPL-3.0-only
|
||||
* 移植自 MoviePilot(https://github.com/jxxghp/MoviePilot,GPLv3),本文件按 GPLv3 条款提供。
|
||||
* @mp-source app/foundation/text.py @ e6820a8
|
||||
* @mp-source app/foundation/temporal.py @ e6820a8
|
||||
* @mp-source app/domain/site.py @ e6820a8
|
||||
* @mp-source app/adapters/network/cloudflare.py @ e6820a8
|
||||
*
|
||||
* 解析器公共工具:MoviePilot e6820a8(GPLv3)
|
||||
* - `app/foundation/text.py` `parse_int/parse_float/contains_chinese` → `parseIntMp/parseFloatMp/containsChinese`
|
||||
* - `app/foundation/temporal.py` `normalize_datetime/format_timestamp` → 同名(dateparser 的保守子集)
|
||||
* - `app/domain/site.py` `SiteUtils.is_logged_in` → `isLoggedIn`
|
||||
* - `app/adapters/network/cloudflare.py` `under_challenge` → `underChallenge`
|
||||
* - `app/domain/site.py` `extract_domain` → `extractDomain`(与 mteam/definition.ts `rootDomain` 同义)
|
||||
*/
|
||||
import { load } from "cheerio";
|
||||
import { hasChildElements, parseHtml } from "./dom.ts";
|
||||
|
||||
/** MP `parse_int`:去千分位,失败返回 0 */
|
||||
export function parseIntMp(value: string | number | null | undefined): number {
|
||||
if (value == null) return 0;
|
||||
const s = String(value).trim().replace(/,/g, "");
|
||||
if (!/^[+-]?\d+$/.test(s)) return 0;
|
||||
return Number.parseInt(s, 10);
|
||||
}
|
||||
|
||||
/** MP `parse_float`:去千分位,失败返回 0 */
|
||||
export function parseFloatMp(value: string | number | null | undefined): number {
|
||||
if (value == null) return 0;
|
||||
const s = String(value).trim().replace(/,/g, "");
|
||||
if (s === "") return 0;
|
||||
if (!/^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/.test(s) && !/^[+-]?(inf|infinity|nan)$/i.test(s)) return 0;
|
||||
const n = Number(s);
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
}
|
||||
|
||||
/** Python `int(str)`(带 strip);非法抛错,与 MP 一致交给调用方 */
|
||||
export function pyInt(value: unknown): number {
|
||||
if (typeof value === "number") return Math.trunc(value);
|
||||
if (typeof value === "boolean") return value ? 1 : 0;
|
||||
const s = String(value ?? "").trim();
|
||||
if (!/^[+-]?\d+$/.test(s)) throw new Error(`invalid literal for int(): '${s}'`);
|
||||
return Number.parseInt(s, 10);
|
||||
}
|
||||
|
||||
/** Python `float(str)`;非法抛错 */
|
||||
export function pyFloat(value: unknown): number {
|
||||
if (typeof value === "number") return value;
|
||||
const s = String(value ?? "").trim();
|
||||
const n = Number(s);
|
||||
if (s === "" || Number.isNaN(n)) throw new Error(`could not convert string to float: '${s}'`);
|
||||
return n;
|
||||
}
|
||||
|
||||
export function containsChinese(value: string | string[] | null | undefined): boolean {
|
||||
if (!value) return false;
|
||||
const s = Array.isArray(value) ? value.join(" ") : value;
|
||||
return /[一-鿿]/.test(s);
|
||||
}
|
||||
|
||||
const MONTHS: Record<string, number> = {
|
||||
jan: 1,
|
||||
feb: 2,
|
||||
mar: 3,
|
||||
apr: 4,
|
||||
may: 5,
|
||||
jun: 6,
|
||||
jul: 7,
|
||||
aug: 8,
|
||||
sep: 9,
|
||||
sept: 9,
|
||||
oct: 10,
|
||||
nov: 11,
|
||||
dec: 12,
|
||||
january: 1,
|
||||
february: 2,
|
||||
march: 3,
|
||||
april: 4,
|
||||
june: 6,
|
||||
july: 7,
|
||||
august: 8,
|
||||
september: 9,
|
||||
october: 10,
|
||||
november: 11,
|
||||
december: 12,
|
||||
};
|
||||
|
||||
const pad = (n: number | string) => String(Number(n)).padStart(2, "0");
|
||||
|
||||
/**
|
||||
* MP `normalize_datetime`(dateparser.parse 后 strftime("%Y-%m-%d %H:%M:%S"),失败原样返回)。
|
||||
* 这里只实现确定性的绝对时间格式:
|
||||
* - `YYYY-MM-DD[ T]HH:MM[:SS][.fff][Z|±HH:MM]`、`YYYY/MM/DD`、`YYYY年MM月DD日`(时区后缀按 dateparser 行为丢弃,保留墙钟时间)
|
||||
* - `DD Mon YYYY[, HH:MM]`、`Mon DD[,] YYYY [HH:MM]`(英文月份)
|
||||
* 相对时间("3 天前"/"2 hours ago")dateparser 会以当前时间换算,本实现不做(原样返回,标记待核验)。
|
||||
*/
|
||||
export function normalizeDatetime(value: string | null | undefined): string | null | undefined {
|
||||
if (!value) return value;
|
||||
const v = String(value).trim();
|
||||
let m =
|
||||
/^(\d{4})[-/.年](\d{1,2})[-/.月](\d{1,2})日?(?:[ T]+(\d{1,2}):(\d{1,2})(?::(\d{1,2}))?(?:\.\d+)?)?\s*(?:Z|[+-]\d{2}:?\d{2}|UTC|GMT)?$/i.exec(
|
||||
v,
|
||||
);
|
||||
if (m) return `${m[1]}-${pad(m[2]!)}-${pad(m[3]!)} ${pad(m[4] ?? 0)}:${pad(m[5] ?? 0)}:${pad(m[6] ?? 0)}`;
|
||||
m =
|
||||
/^(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]{3,9})\.?,?\s+(\d{4})(?:,?\s+(?:at\s+)?(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(am|pm)?)?$/i.exec(
|
||||
v,
|
||||
);
|
||||
if (m && MONTHS[m[2]!.toLowerCase()]) {
|
||||
const h = ampm(m[4], m[7]);
|
||||
return `${m[3]}-${pad(MONTHS[m[2]!.toLowerCase()]!)}-${pad(m[1]!)} ${pad(h)}:${pad(m[5] ?? 0)}:${pad(m[6] ?? 0)}`;
|
||||
}
|
||||
m =
|
||||
/^([A-Za-z]{3,9})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})(?:,?\s+(?:at\s+)?(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(am|pm)?)?$/i.exec(
|
||||
v,
|
||||
);
|
||||
if (m && MONTHS[m[1]!.toLowerCase()]) {
|
||||
const h = ampm(m[4], m[7]);
|
||||
return `${m[3]}-${pad(MONTHS[m[1]!.toLowerCase()]!)}-${pad(m[2]!)} ${pad(h)}:${pad(m[5] ?? 0)}:${pad(m[6] ?? 0)}`;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function ampm(h: string | undefined, suffix: string | undefined): number {
|
||||
let n = Number(h ?? 0);
|
||||
if (suffix?.toLowerCase() === "pm" && n < 12) n += 12;
|
||||
if (suffix?.toLowerCase() === "am" && n === 12) n = 0;
|
||||
return n;
|
||||
}
|
||||
|
||||
/**
|
||||
* MP `format_timestamp`:非纯数字字符串原样返回;Unix 秒 → 本地时间 "YYYY-MM-DD HH:MM:SS"。
|
||||
* MP 用进程本地时区;这里用显式 `tzOffsetMinutes`(默认 +480,即 Asia/Shanghai,MP 容器默认 TZ)。
|
||||
*/
|
||||
export function formatTimestamp(timestamp: unknown, tzOffsetMinutes = 480): string | undefined {
|
||||
if (timestamp == null) return undefined;
|
||||
if (typeof timestamp === "string" && !/^\d+$/.test(timestamp)) return timestamp;
|
||||
const n = Number(timestamp);
|
||||
if (!Number.isFinite(n)) return String(timestamp);
|
||||
const d = new Date(Math.trunc(n) * 1000 + tzOffsetMinutes * 60_000);
|
||||
return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}`;
|
||||
}
|
||||
|
||||
/** MP `SiteUtils.is_logged_in` */
|
||||
export function isLoggedIn(htmlText: string): boolean {
|
||||
const html = parseHtml(htmlText);
|
||||
if (!hasChildElements(html)) return false;
|
||||
if (html.nodes("//input[@type='password']").length) return false;
|
||||
const xpaths = [
|
||||
'//a[contains(@href, "logout") or contains(@data-url, "logout") or contains(@href, "mybonus") or contains(@onclick, "logout") or contains(@href, "usercp") or contains(@lay-on, "logout")]',
|
||||
'//form[contains(@action, "logout")]',
|
||||
'//div[@class="user-info-side"]',
|
||||
'//a[@id="myitem"]',
|
||||
];
|
||||
return xpaths.some((x) => html.nodes(x).length > 0);
|
||||
}
|
||||
|
||||
const CHALLENGE_TITLES = ["Just a moment...", "请稍候…", "DDOS-GUARD"];
|
||||
const CHALLENGE_SELECTORS = [
|
||||
"#cf-challenge-running",
|
||||
".ray_id",
|
||||
".attack-box",
|
||||
"#cf-please-wait",
|
||||
"#challenge-spinner",
|
||||
"#trk_jschal_js",
|
||||
"td.info #js_info",
|
||||
"div.vc div.text-box h2",
|
||||
];
|
||||
|
||||
/** MP `under_challenge`:Cloudflare / DDoS-Guard 挑战页 */
|
||||
export function underChallenge(htmlText: string | null | undefined): boolean {
|
||||
if (!htmlText) return false;
|
||||
const $ = load(htmlText);
|
||||
const title = $("title").text().trim().toLowerCase();
|
||||
if (CHALLENGE_TITLES.some((t) => t.toLowerCase() === title)) return true;
|
||||
return CHALLENGE_SELECTORS.some((s) => $(s).length > 0);
|
||||
}
|
||||
|
||||
const SPECIAL_SITE_DOMAINS = ["u2.dmhy.org", "pt.ecust.pp.ua", "pt.gtkpw.xyz", "pt.gtk.pw"];
|
||||
|
||||
/** MP `site_rules.extract_domain`:特殊域名表 → netloc 超过 3 级原样 → 否则取最后两级 */
|
||||
export function extractDomain(url: string | null | undefined): string {
|
||||
if (!url) return "";
|
||||
for (const d of SPECIAL_SITE_DOMAINS) if (url.includes(d)) return d;
|
||||
let netloc: string;
|
||||
if (!url.startsWith("http")) netloc = url;
|
||||
else {
|
||||
try {
|
||||
netloc = new URL(url).host;
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
if (!netloc) return "";
|
||||
const labels = netloc.split(".");
|
||||
if (labels.length > 3) return netloc;
|
||||
return labels.slice(-2).join(".");
|
||||
}
|
||||
|
||||
/** Python `re.search(...)`: 返回 match 或 null(显式 helper,方便逐行对照) */
|
||||
export function reSearch(pattern: RegExp, text: string | null | undefined): RegExpExecArray | null {
|
||||
if (text == null) return null;
|
||||
const re = new RegExp(pattern.source, pattern.flags.replace("g", ""));
|
||||
return re.exec(text);
|
||||
}
|
||||
|
||||
/** Python `re.findall`(带分组时返回分组数组) */
|
||||
export function reFindall(pattern: RegExp, text: string): string[][] {
|
||||
const re = new RegExp(pattern.source, pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`);
|
||||
return [...text.matchAll(re)].map((m) => m.slice(1));
|
||||
}
|
||||
Reference in new issue
Block a user