#!/usr/bin/env -S deno run -A /** * MovieClaw v0.27.0(MIT)站点 YAML → 本项目 SiteDefinition JSON 转换器(可重复执行)。 * * 用法: * deno run -A scripts/mp-sync/movieclaw-import.mjs [--repo spikes/movieclaw-src/repo] [--ref v0.27.0] [--check] * * - 只用 `git show :` 读取文件(固定 tag,不读工作区,避免混入许可切换后的内容)。 * - 校验 解析到的提交与 LICENSE 为 MIT;ref 不是 v0.27.0 时要求 --allow-ref(许可需另行核对)。 * - 输出:packages/adapters/src/sites/definitions/.json 与 manifest.generated.ts。 * - 手工覆盖层:scripts/mp-sync/movieclaw-overrides/.json 深合并到转换结果之上(覆盖层优先; * `torrents.fields.` 与数组整体替换)。 * - `--check`:只比较,不写文件;有差异时退出码 1(CI/跟随更新用)。 * * 选择器默认值来自 MovieClaw `movieclaw_tracker/selectors.py` NexusPHPSelectors(MIT,见 * packages/adapters/src/sites/definitions/NOTICE-MovieClaw.md)。 */ import { execFileSync } from "node:child_process"; import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import YAML from "yaml"; const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const args = process.argv.slice(2); const opt = (name, def) => { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : def; }; const REPO = resolve(ROOT, opt("--repo", "spikes/movieclaw-src/repo")); const REF = opt("--ref", "v0.27.0"); const CHECK = args.includes("--check"); const EXPECTED = { "v0.27.0": "c79bb1b065c826823fc1c15bcb99b3c282c3fce0" }; const OUT_DIR = join(ROOT, "packages/adapters/src/sites/definitions"); const OVERRIDES = join(ROOT, "scripts/mp-sync/movieclaw-overrides"); const CONFIG_DIR = "src/movieclaw_tracker/sites/configs"; /** 主力站:默认启用;其余 enabled:false(未实测) */ const PRIMARY = new Set(["audiences", "hhanclub", "chdbits", "mteam"]); /** MP 用户信息解析器 schema(app/modules/indexer/parser SiteSchema 值) */ const MP_SCHEMA = { audiences: "NexusAudiences", hhanclub: "NexusHhanclub", mteam: "MTorrent", hddolby: "HDDolby" }; const CATEGORY_LABELS = { movie: "电影", tv: "剧集", documentary: "纪录片", anime: "动漫", music: "音乐", game: "游戏", av: "成人", other: "其他", }; const TZ = { "Asia/Shanghai": "+08:00", "Asia/Hong_Kong": "+08:00", "Asia/Taipei": "+08:00", UTC: "+00:00" }; // ---- MovieClaw NexusPHPSelectors 默认值(v0.27.0 selectors.py) ---- const MC_DEFAULTS = { login_path: "login.php", login_username_field: "username", login_password_field: "password", login_error_css: "td.text", login_success_css: "a[href*='logout']", login_extra_form_data: [], login_select_defaults: [], torrent_list_path: "torrents.php", torrent_row_css: "table.torrents > tr:not(:first-child)", torrent_id_css: "a[href*='details.php']::attr(href)", torrent_title_css: "a[href*='details.php']", torrent_subtitle_css: "td.embedded > span", torrent_category_css: "td:nth-child(1) img::attr(class)", torrent_size_css: "td:nth-child(5)", torrent_seeders_css: "td:nth-child(6)", torrent_leechers_css: "td:nth-child(7)", torrent_snatched_css: "td:nth-child(8)", torrent_time_css: "td:nth-child(4) > span::attr(title)", torrent_time_fallback_css: "td:nth-child(4)", torrent_time_fmt: "%Y-%m-%d %H:%M:%S", torrent_uploader_css: "td:nth-child(9) a", promo_download_rules: [ ["img.pro_free", 0.0], ["img.pro_free2up", 0.0], ["img.pro_50pctdown", 0.5], ["img.pro_50pctdown2up", 0.5], ["img.pro_30pctdown", 0.3], ], promo_upload_rules: [ ["img.pro_free2up", 2.0], ["img.pro_50pctdown2up", 2.0], ["img.pro_2up", 2.0], ], torrent_hr_css: "", torrent_promo_deadline_css: "", torrent_promo_deadline_attr: "title", torrent_promo_deadline_re: "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}", torrent_promo_deadline_fmt: "%Y-%m-%d %H:%M:%S", torrent_detail_url_css: "a[href*='details.php']::attr(href)", torrent_download_url_css: "a[href*='download.php']::attr(href)", page_offset: 0, search_path: "torrents.php", search_keyword_param: "search", detail_title_css: "#top::text", detail_subtitle_css: "td.rowhead:contains('副标题') + td::text", detail_description_css: "#kdescr", detail_download_css: "a[href*='download.php']::attr(href)", detail_imdb_css: "a[href*='imdb.com']::attr(href)", detail_douban_css: "a[href*='douban.com']::attr(href)", detail_size_css: "td.rowhead:contains('大小') + td::text", detail_file_list_css: "td.rowfollow > ul > li::text", profile_path: "userdetails.php", profile_uid_css: "", profile_username_css: "h1::text", profile_class_css: "td.rowhead:contains('等级') + td::text", profile_uploaded_css: "td.rowhead:contains('上传量') + td::text", profile_downloaded_css: "td.rowhead:contains('下载量') + td::text", profile_ratio_css: "td.rowhead:contains('分享率') + td::text", profile_bonus_css: "td.rowhead:contains('魔力') + td::text", profile_join_date_css: "td.rowhead:contains('加入日期') + td::text", profile_seeding_css: "td.rowhead:contains('当前做种') + td::text", profile_leeching_css: "td.rowhead:contains('当前下载') + td::text", profile_vip_css: "", download_path: "download.php", }; /** MovieClaw custom_class 的行为差异 → 定义层表达(只收录能用定义表达的部分) */ const CUSTOM_CLASS_RULES = { "movieclaw_tracker.sites.custom.ourbits.OurBitsSite": (def) => { // 分类参数为重复的 cat[](custom/ourbits.py _apply_category_params) def.category.param = "cat[]"; return true; // 已完全由定义表达 }, "movieclaw_tracker.sites.custom.ttg.TTGSite": () => false, // 分类/ID/做种列均非标准,需自定义适配器 "movieclaw_tracker.sites.custom.mteam.MTeamSite": () => true, // 由 mteam/* 实现 }; function git(...a) { return execFileSync("git", ["-C", REPO, ...a], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); } function stripPseudo(css) { return css.replace(/::(attr\([^)]*\)|text|next_sibling_text)$/, ""); } /** MovieClaw `X::attr(a)` / `X::text` / `X` → SiteSpider 字段选择器 */ function field(css, extra = {}) { if (!css) return undefined; const m = /^(.*)::attr\(([^)]+)\)$/.exec(css); if (m) return { selector: m[1].trim(), attribute: m[2].trim(), ...extra }; return { selector: stripPseudo(css).trim(), ...extra }; } function caseTable(rules) { const out = {}; for (const [css, factor] of rules) out[css] = Number(factor); out["*"] = 1; return out; } function rulesFrom(v, fallback) { if (v == null) return fallback; if (Array.isArray(v)) return v; return Object.entries(v); } function convertNexus(y, file) { const s = { ...MC_DEFAULTS, ...(y.selectors ?? {}) }; s.promo_download_rules = rulesFrom(y.selectors?.promo_download_rules, MC_DEFAULTS.promo_download_rules); s.promo_upload_rules = rulesFrom(y.selectors?.promo_upload_rules, MC_DEFAULTS.promo_upload_rules); const id = y.site_id; const tz = TZ[y.timezone ?? "Asia/Shanghai"]; if (!tz) throw new Error(`${file}: 未知时区 ${y.timezone}`); const cats = y.categories ?? {}; const item = (k) => (cats[k] ?? []).map((cid) => ({ id: String(cid), desc: CATEGORY_LABELS[k] })); const categoryNames = {}; for (const [k, ids] of Object.entries(cats)) for (const cid of ids ?? []) categoryNames[String(cid)] ??= CATEGORY_LABELS[k] ?? k; const titleCss = stripPseudo(s.torrent_title_css); const fields = { title_default: { selector: titleCss }, title_optional: { selector: `${titleCss}[title]`, attribute: "title" }, title: { text: "{% if fields['title_optional'] %}{{ fields['title_optional'] }}{% else %}{{ fields['title_default'] }}{% endif %}", }, details: field(s.torrent_detail_url_css), download: field(s.torrent_download_url_css), imdbid: { selector: 'a[href*="imdb.com/title/tt"]', attribute: "href", filters: [{ name: "re_search", args: ["tt\\d+", 0] }], }, date_elapsed: field(s.torrent_time_fallback_css), date_added: field(s.torrent_time_css), date: { text: "{% if fields['date_elapsed'] or fields['date_added'] %}{{ fields['date_added'] if fields['date_added'] else fields['date_elapsed'] }}{% else %}now{% endif %}", filters: [{ name: "dateparse", args: s.torrent_time_fmt }], }, size: field(s.torrent_size_css), seeders: field(s.torrent_seeders_css), leechers: field(s.torrent_leechers_css), grabs: field(s.torrent_snatched_css), downloadvolumefactor: { case: caseTable(s.promo_download_rules) }, uploadvolumefactor: { case: caseTable(s.promo_upload_rules) }, description: field(s.torrent_subtitle_css), category: field(s.torrent_category_css, { filters: [{ name: "re_search", args: ["\\d+", 0] }] }), }; if (s.torrent_promo_deadline_css) { fields.freedate = { selector: stripPseudo(s.torrent_promo_deadline_css), attribute: s.torrent_promo_deadline_attr || "title", filters: [{ name: "re_search", args: [s.torrent_promo_deadline_re, 0] }], }; } if (s.torrent_hr_css) fields.hr = { selector: s.torrent_hr_css }; const profile = {}; for (const [k, v] of Object.entries(s)) { if (k.startsWith("profile_") && typeof v === "string" && v) profile[k.slice(8)] = v; } const def = { id, name: y.display_name, enabled: PRIMARY.has(id), schema: MP_SCHEMA[id] ?? "NexusPhp", provenance: { source: "movieclaw", ref: REF, commit: COMMIT.slice(0, 8), license: "MIT", file: `${CONFIG_DIR}/${file}`, }, domain: `${String(y.base_url).replace(/\/+$/, "")}/`, encoding: "UTF-8", public: false, proxy: false, language: "zh", result_num: 100, timeout: Number(y.http?.timeout ?? 30), search: { paths: [{ path: s.search_path, type: "all" }], params: { [s.search_keyword_param]: "{keyword}" } }, batch: { delimiter: " ", space_replace: "_" }, browse: { path: `${s.torrent_list_path}?page={page}`, start: Number(s.page_offset) === -1 ? 0 : 1 }, category: { movie: item("movie"), tv: item("tv"), music: item("music") }, torrents: { list: { selector: rowSelector(s.torrent_row_css) }, fields }, umt: { version: `${id}@mc-${REF}`, kind: "nexusphp", displayName: y.display_name, timezone: tz, rateLimit: { minIntervalMs: Math.round(Number(y.min_request_interval ?? 3) * 1000), burst: 1 }, capabilities: { list: true, search: true, detail: true, download: true, rss: false }, categoryNames, loginSuccess: s.login_success_css, emptyMarkers: ["没有种子", "没有结果", "Nothing found"], pagination: 'a[href*="page="]', detailPath: "details.php", downloadPath: s.download_path, ...(y.auth?.supported ? { authSupported: y.auth.supported } : {}), login: { path: s.login_path, usernameField: s.login_username_field, passwordField: s.login_password_field, errorCss: s.login_error_css, ...(s.login_extra_form_data?.length ? { extraFormData: pairs(s.login_extra_form_data) } : {}), ...(s.login_select_defaults && Object.keys(s.login_select_defaults).length ? { selectDefaults: pairs(s.login_select_defaults) } : {}), }, profile, detail: { path: "details.php", title: stripPseudo(s.detail_title_css), rowLabel: "td.rowhead", rowValue: "td.rowfollow", labels: { subtitle: ["副标题", "Small Description"], basicInfo: ["基本信息", "Basic Info"], peers: ["同伴", "Peers"], }, description: s.detail_description_css, headerScope: stripPseudo(s.detail_title_css), notFoundMarkers: ["没有该ID的种子", "No torrent with ID"], fields: { title: s.detail_title_css, subtitle: s.detail_subtitle_css, size: s.detail_size_css, description: s.detail_description_css, download: s.detail_download_css, imdb: s.detail_imdb_css, douban: s.detail_douban_css, fileList: s.detail_file_list_css, }, }, }, }; if (y.custom_class) { def.umt.customClass = y.custom_class; const rule = CUSTOM_CLASS_RULES[y.custom_class]; if (!rule) throw new Error(`${file}: 未登记的 custom_class ${y.custom_class}`); if (!rule(def)) def.enabled = false; } if (y.hr_seed_hours) def.umt.hrSeedHours = Number(y.hr_seed_hours); return def; } function pairs(v) { if (Array.isArray(v)) return v.map((p) => [String(p[0]), String(p[1])]); return Object.entries(v).map(([k, x]) => [k, String(x)]); } function convertApi(y, file) { if (y.site_id !== "mteam") throw new Error(`${file}: 暂只支持 mteam 这一 API 站`); const categoryNames = {}; for (const [k, ids] of Object.entries(y.categories ?? {})) for (const cid of ids ?? []) categoryNames[String(cid)] ??= CATEGORY_LABELS[k] ?? k; return { id: y.site_id, name: y.display_name, enabled: PRIMARY.has(y.site_id), schema: MP_SCHEMA[y.site_id], provenance: { source: "movieclaw", ref: REF, commit: COMMIT.slice(0, 8), license: "MIT", file: `${CONFIG_DIR}/${file}`, }, domain: `${String(y.web_base_url ?? y.base_url).replace(/\/+$/, "")}/`, parser: "mTorrent", timeout: Number(y.http?.timeout ?? 15), umt: { version: `${y.site_id}@mc-${REF}`, kind: "mteam", displayName: y.display_name, apiBaseUrl: String(y.base_url).replace(/\/+$/, ""), timezone: TZ[y.timezone ?? "Asia/Shanghai"], rateLimit: { minIntervalMs: Math.round(Number(y.min_request_interval ?? 3) * 1000), burst: 1 }, capabilities: { list: true, search: true, detail: true, download: true, rss: false }, categoryNames, ...(y.auth?.supported ? { authSupported: y.auth.supported } : {}), ...(y.custom_class ? { customClass: y.custom_class } : {}), }, }; } function isObj(v) { return v && typeof v === "object" && !Array.isArray(v); } /** * 列表行选择器规范化(W5-D,见 docs/porting/mp-rust-logic-parity.md §5): * - `table.torrents > tr…` 在显式带 的页面上取不到行(cheerio 与 MP PyQuery 同),补 `> tbody > tr…` 分支; * - 不带条件的 `table.torrents > tr` / `table.torrents tr` 会命中表头行,产出标题为空的垃圾行,收紧为"含 td.rowfollow 数据单元格的行"(NexusPHP 表头是 td.colhead;MP PyQuery 的 :has 只接受简单选择器,不能写属性选择器)。 */ const ROW_HAS_DETAILS = ":has(td.rowfollow)"; function rowSelector(css) { if (!css || !/^\s*table\.torrents[\s>]/.test(css)) return css; const parts = []; const add = (p) => { if (!parts.includes(p)) parts.push(p); }; for (const raw of String(css).split(",")) { const part = raw.trim(); const m = /^table\.torrents\s*(?:>\s*)?(?:tbody\s*>\s*)?tr(.*)$/.exec(part); if (!m) { add(part); continue; } const cond = m[1].trim() === "" ? ROW_HAS_DETAILS : m[1].trim(); add(`table.torrents > tr${cond}`); add(`table.torrents > tbody > tr${cond}`); } return parts.join(", "); } function deepMerge(base, over, path = "") { if (!isObj(base) || !isObj(over)) return over; const out = { ...base }; for (const [k, v] of Object.entries(over)) { const p = path ? `${path}.${k}` : k; if (v === null) delete out[k]; else if (/^torrents\.fields\.[^.]+$/.test(p) || /^umt\.detail\.fields$/.test(p)) out[k] = v; // 字段整体替换 else out[k] = isObj(v) && isObj(out[k]) ? deepMerge(out[k], v, p) : v; } return out; } /** 用仓库的 biome 格式化(与 lint 一致,保证 --check 幂等);biome 不可用时退回 JSON.stringify */ function biomeFormat(text, fakePath) { const bins = [join(ROOT, "node_modules/.bin/biome")]; for (const bin of bins) { if (!existsSync(bin)) continue; try { return execFileSync(bin, ["format", `--stdin-file-path=${fakePath}`], { input: text, encoding: "utf8", cwd: ROOT, }); } catch { /* fallthrough */ } } return text; } /** 输出键顺序固定,方便 diff */ function stable(v, fakePath) { return biomeFormat(`${JSON.stringify(v, null, 2)}\n`, fakePath); } // ---- main ---- const COMMIT = git("rev-parse", `${REF}^{commit}`).trim(); if (EXPECTED[REF] && EXPECTED[REF] !== COMMIT) throw new Error(`${REF} 解析到 ${COMMIT},与登记的 ${EXPECTED[REF]} 不符`); if (!EXPECTED[REF] && !args.includes("--allow-ref")) throw new Error(`ref ${REF} 未登记;许可需另行核对后加 --allow-ref`); const license = git("show", `${REF}:LICENSE`); if (!/^MIT License/m.test(license)) throw new Error(`${REF}:LICENSE 不是 MIT,停止(许可切换后的内容不得复制)`); const files = git("ls-tree", "--name-only", `${REF}:${CONFIG_DIR}`) .split("\n") .filter((f) => f.endsWith(".yaml") && !f.startsWith("_")); const ids = []; let diffs = 0; for (const file of files) { const y = YAML.parse(git("show", `${REF}:${CONFIG_DIR}/${file}`)); let def = y.framework === "api" ? convertApi(y, file) : convertNexus(y, file); const ovPath = join(OVERRIDES, `${y.site_id}.json`); if (existsSync(ovPath)) { const ov = JSON.parse(readFileSync(ovPath, "utf8")); delete ov.$comment; def = deepMerge(def, ov); def.provenance = { ...def.provenance, overrides: `scripts/mp-sync/movieclaw-overrides/${y.site_id}.json` }; } // 分类项 desc 以最终 categoryNames 为准(覆盖层可能给出更细的站点分类名) for (const k of ["movie", "tv", "music"]) { for (const c of def.category?.[k] ?? []) { const n = def.umt?.categoryNames?.[String(c.id)]; if (n) c.desc = n; } } const out = join(OUT_DIR, `${y.site_id}.json`); const text = stable(def, `packages/adapters/src/sites/definitions/${y.site_id}.json`); const old = existsSync(out) ? readFileSync(out, "utf8") : null; if (old !== text) { diffs++; if (CHECK) console.log(`差异:${y.site_id}.json`); else writeFileSync(out, text); } ids.push(y.site_id); } const ident = (id) => `${id.replace(/[^A-Za-z0-9_]/g, "_")}Json`; const manifest = `// 由 scripts/mp-sync/movieclaw-import.mjs 生成,请勿手改(MovieClaw ${REF} ${COMMIT.slice(0, 8)},MIT) ${ids .sort() .map((id) => `import ${ident(id)} from "./${id}.json" with { type: "json" };`) .join("\n")} /** MovieClaw 转换来的站点定义原始 JSON(未校验;经 loadSiteDefinition 校验后使用) */ export const MOVIECLAW_DEFINITION_JSON: Record = { ${ids .sort() .map((id) => ` ${JSON.stringify(id)}: ${ident(id)},`) .join("\n")} }; `; const manifestPath = join(OUT_DIR, "manifest.generated.ts"); const manifestText = biomeFormat(manifest, "packages/adapters/src/sites/definitions/manifest.generated.ts"); if ((existsSync(manifestPath) ? readFileSync(manifestPath, "utf8") : null) !== manifestText) { diffs++; if (CHECK) console.log("差异:manifest.generated.ts"); else writeFileSync(manifestPath, manifestText); } const stale = readdirSync(OUT_DIR).filter((f) => f.endsWith(".json") && !ids.includes(f.replace(/\.json$/, ""))); if (stale.length) console.log(`注意:定义目录中存在非 MovieClaw 来源的 JSON(保留不动):${stale.join(", ")}`); console.log( `${CHECK ? "检查" : "转换"}完成:${ids.length} 个站点,${diffs} 个文件${CHECK ? "有差异" : "已更新"}(${REF} ${COMMIT.slice(0, 8)})`, ); if (CHECK && diffs) process.exit(1);