UHD_Media_Tool/scripts/mp-sync/movieclaw-import.mjs

#!/usr/bin/env -S deno run -A
/**
 * MovieClaw v0.27.0(MIT)站点 YAML → 本项目 SiteDefinition JSON 转换器(可重复执行)。
 *
 * 用法:
 *   deno run -A scripts/mp-sync/movieclaw-import.mjs [--repo spikes/movieclaw-src/repo] [--ref v0.27.0] [--check]
 *
 * - 只用 `git show <ref>:<path>` 读取文件(固定 tag,不读工作区,避免混入许可切换后的内容)。
 * - 校验 <ref> 解析到的提交与 LICENSE 为 MIT;ref 不是 v0.27.0 时要求 --allow-ref(许可需另行核对)。
 * - 输出:packages/adapters/src/sites/definitions/<site_id>.json 与 manifest.generated.ts。
 * - 手工覆盖层:scripts/mp-sync/movieclaw-overrides/<site_id>.json 深合并到转换结果之上(覆盖层优先;
 *   `torrents.fields.<name>` 与数组整体替换)。
 * - `--check`:只比较,不写文件;有差异时退出码 1(CI/跟随更新用)。
 *
 * 选择器默认值来自 MovieClaw `movieclaw_tracker/selectors.py` NexusPHPSelectors(MIT,见
 * packages/adapters/src/sites/definitions/NOTICE-MovieClaw.md)。
 */
import { execFileSync } from "node:child_process";
import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import YAML from "yaml";

const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../..");

const args = process.argv.slice(2);
const opt = (name, def) => {
  const i = args.indexOf(name);
  return i >= 0 ? args[i + 1] : def;
};
const REPO = resolve(ROOT, opt("--repo", "spikes/movieclaw-src/repo"));
const REF = opt("--ref", "v0.27.0");
const CHECK = args.includes("--check");
const EXPECTED = { "v0.27.0": "c79bb1b065c826823fc1c15bcb99b3c282c3fce0" };
const OUT_DIR = join(ROOT, "packages/adapters/src/sites/definitions");
const OVERRIDES = join(ROOT, "scripts/mp-sync/movieclaw-overrides");
const CONFIG_DIR = "src/movieclaw_tracker/sites/configs";
/** 主力站:默认启用;其余 enabled:false(未实测) */
const PRIMARY = new Set(["audiences", "hhanclub", "chdbits", "mteam"]);
/** MP 用户信息解析器 schema(app/modules/indexer/parser SiteSchema 值) */
const MP_SCHEMA = { audiences: "NexusAudiences", hhanclub: "NexusHhanclub", mteam: "MTorrent", hddolby: "HDDolby" };
const CATEGORY_LABELS = {
  movie: "电影",
  tv: "剧集",
  documentary: "纪录片",
  anime: "动漫",
  music: "音乐",
  game: "游戏",
  av: "成人",
  other: "其他",
};
const TZ = { "Asia/Shanghai": "+08:00", "Asia/Hong_Kong": "+08:00", "Asia/Taipei": "+08:00", UTC: "+00:00" };

// ---- MovieClaw NexusPHPSelectors 默认值(v0.27.0 selectors.py) ----
const MC_DEFAULTS = {
  login_path: "login.php",
  login_username_field: "username",
  login_password_field: "password",
  login_error_css: "td.text",
  login_success_css: "a[href*='logout']",
  login_extra_form_data: [],
  login_select_defaults: [],
  torrent_list_path: "torrents.php",
  torrent_row_css: "table.torrents > tr:not(:first-child)",
  torrent_id_css: "a[href*='details.php']::attr(href)",
  torrent_title_css: "a[href*='details.php']",
  torrent_subtitle_css: "td.embedded > span",
  torrent_category_css: "td:nth-child(1) img::attr(class)",
  torrent_size_css: "td:nth-child(5)",
  torrent_seeders_css: "td:nth-child(6)",
  torrent_leechers_css: "td:nth-child(7)",
  torrent_snatched_css: "td:nth-child(8)",
  torrent_time_css: "td:nth-child(4) > span::attr(title)",
  torrent_time_fallback_css: "td:nth-child(4)",
  torrent_time_fmt: "%Y-%m-%d %H:%M:%S",
  torrent_uploader_css: "td:nth-child(9) a",
  promo_download_rules: [
    ["img.pro_free", 0.0],
    ["img.pro_free2up", 0.0],
    ["img.pro_50pctdown", 0.5],
    ["img.pro_50pctdown2up", 0.5],
    ["img.pro_30pctdown", 0.3],
  ],
  promo_upload_rules: [
    ["img.pro_free2up", 2.0],
    ["img.pro_50pctdown2up", 2.0],
    ["img.pro_2up", 2.0],
  ],
  torrent_hr_css: "",
  torrent_promo_deadline_css: "",
  torrent_promo_deadline_attr: "title",
  torrent_promo_deadline_re: "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}",
  torrent_promo_deadline_fmt: "%Y-%m-%d %H:%M:%S",
  torrent_detail_url_css: "a[href*='details.php']::attr(href)",
  torrent_download_url_css: "a[href*='download.php']::attr(href)",
  page_offset: 0,
  search_path: "torrents.php",
  search_keyword_param: "search",
  detail_title_css: "#top::text",
  detail_subtitle_css: "td.rowhead:contains('副标题') + td::text",
  detail_description_css: "#kdescr",
  detail_download_css: "a[href*='download.php']::attr(href)",
  detail_imdb_css: "a[href*='imdb.com']::attr(href)",
  detail_douban_css: "a[href*='douban.com']::attr(href)",
  detail_size_css: "td.rowhead:contains('大小') + td::text",
  detail_file_list_css: "td.rowfollow > ul > li::text",
  profile_path: "userdetails.php",
  profile_uid_css: "",
  profile_username_css: "h1::text",
  profile_class_css: "td.rowhead:contains('等级') + td::text",
  profile_uploaded_css: "td.rowhead:contains('上传量') + td::text",
  profile_downloaded_css: "td.rowhead:contains('下载量') + td::text",
  profile_ratio_css: "td.rowhead:contains('分享率') + td::text",
  profile_bonus_css: "td.rowhead:contains('魔力') + td::text",
  profile_join_date_css: "td.rowhead:contains('加入日期') + td::text",
  profile_seeding_css: "td.rowhead:contains('当前做种') + td::text",
  profile_leeching_css: "td.rowhead:contains('当前下载') + td::text",
  profile_vip_css: "",
  download_path: "download.php",
};

/** MovieClaw custom_class 的行为差异 → 定义层表达(只收录能用定义表达的部分) */
const CUSTOM_CLASS_RULES = {
  "movieclaw_tracker.sites.custom.ourbits.OurBitsSite": (def) => {
    // 分类参数为重复的 cat[](custom/ourbits.py _apply_category_params)
    def.category.param = "cat[]";
    return true; // 已完全由定义表达
  },
  "movieclaw_tracker.sites.custom.ttg.TTGSite": () => false, // 分类/ID/做种列均非标准,需自定义适配器
  "movieclaw_tracker.sites.custom.mteam.MTeamSite": () => true, // 由 mteam/* 实现
};

function git(...a) {
  return execFileSync("git", ["-C", REPO, ...a], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
}

function stripPseudo(css) {
  return css.replace(/::(attr\([^)]*\)|text|next_sibling_text)$/, "");
}

/** MovieClaw `X::attr(a)` / `X::text` / `X` → SiteSpider 字段选择器 */
function field(css, extra = {}) {
  if (!css) return undefined;
  const m = /^(.*)::attr\(([^)]+)\)$/.exec(css);
  if (m) return { selector: m[1].trim(), attribute: m[2].trim(), ...extra };
  return { selector: stripPseudo(css).trim(), ...extra };
}

function caseTable(rules) {
  const out = {};
  for (const [css, factor] of rules) out[css] = Number(factor);
  out["*"] = 1;
  return out;
}

function rulesFrom(v, fallback) {
  if (v == null) return fallback;
  if (Array.isArray(v)) return v;
  return Object.entries(v);
}

function convertNexus(y, file) {
  const s = { ...MC_DEFAULTS, ...(y.selectors ?? {}) };
  s.promo_download_rules = rulesFrom(y.selectors?.promo_download_rules, MC_DEFAULTS.promo_download_rules);
  s.promo_upload_rules = rulesFrom(y.selectors?.promo_upload_rules, MC_DEFAULTS.promo_upload_rules);
  const id = y.site_id;
  const tz = TZ[y.timezone ?? "Asia/Shanghai"];
  if (!tz) throw new Error(`${file}: 未知时区 ${y.timezone}`);
  const cats = y.categories ?? {};
  const item = (k) => (cats[k] ?? []).map((cid) => ({ id: String(cid), desc: CATEGORY_LABELS[k] }));
  const categoryNames = {};
  for (const [k, ids] of Object.entries(cats))
    for (const cid of ids ?? []) categoryNames[String(cid)] ??= CATEGORY_LABELS[k] ?? k;

  const titleCss = stripPseudo(s.torrent_title_css);
  const fields = {
    title_default: { selector: titleCss },
    title_optional: { selector: `${titleCss}[title]`, attribute: "title" },
    title: {
      text: "{% if fields['title_optional'] %}{{ fields['title_optional'] }}{% else %}{{ fields['title_default'] }}{% endif %}",
    },
    details: field(s.torrent_detail_url_css),
    download: field(s.torrent_download_url_css),
    imdbid: {
      selector: 'a[href*="imdb.com/title/tt"]',
      attribute: "href",
      filters: [{ name: "re_search", args: ["tt\\d+", 0] }],
    },
    date_elapsed: field(s.torrent_time_fallback_css),
    date_added: field(s.torrent_time_css),
    date: {
      text: "{% if fields['date_elapsed'] or fields['date_added'] %}{{ fields['date_added'] if fields['date_added'] else fields['date_elapsed'] }}{% else %}now{% endif %}",
      filters: [{ name: "dateparse", args: s.torrent_time_fmt }],
    },
    size: field(s.torrent_size_css),
    seeders: field(s.torrent_seeders_css),
    leechers: field(s.torrent_leechers_css),
    grabs: field(s.torrent_snatched_css),
    downloadvolumefactor: { case: caseTable(s.promo_download_rules) },
    uploadvolumefactor: { case: caseTable(s.promo_upload_rules) },
    description: field(s.torrent_subtitle_css),
    category: field(s.torrent_category_css, { filters: [{ name: "re_search", args: ["\\d+", 0] }] }),
  };
  if (s.torrent_promo_deadline_css) {
    fields.freedate = {
      selector: stripPseudo(s.torrent_promo_deadline_css),
      attribute: s.torrent_promo_deadline_attr || "title",
      filters: [{ name: "re_search", args: [s.torrent_promo_deadline_re, 0] }],
    };
  }
  if (s.torrent_hr_css) fields.hr = { selector: s.torrent_hr_css };

  const profile = {};
  for (const [k, v] of Object.entries(s)) {
    if (k.startsWith("profile_") && typeof v === "string" && v) profile[k.slice(8)] = v;
  }

  const def = {
    id,
    name: y.display_name,
    enabled: PRIMARY.has(id),
    schema: MP_SCHEMA[id] ?? "NexusPhp",
    provenance: {
      source: "movieclaw",
      ref: REF,
      commit: COMMIT.slice(0, 8),
      license: "MIT",
      file: `${CONFIG_DIR}/${file}`,
    },
    domain: `${String(y.base_url).replace(/\/+$/, "")}/`,
    encoding: "UTF-8",
    public: false,
    proxy: false,
    language: "zh",
    result_num: 100,
    timeout: Number(y.http?.timeout ?? 30),
    search: { paths: [{ path: s.search_path, type: "all" }], params: { [s.search_keyword_param]: "{keyword}" } },
    batch: { delimiter: " ", space_replace: "_" },
    browse: { path: `${s.torrent_list_path}?page={page}`, start: Number(s.page_offset) === -1 ? 0 : 1 },
    category: { movie: item("movie"), tv: item("tv"), music: item("music") },
    torrents: { list: { selector: rowSelector(s.torrent_row_css) }, fields },
    umt: {
      version: `${id}@mc-${REF}`,
      kind: "nexusphp",
      displayName: y.display_name,
      timezone: tz,
      rateLimit: { minIntervalMs: Math.round(Number(y.min_request_interval ?? 3) * 1000), burst: 1 },
      capabilities: { list: true, search: true, detail: true, download: true, rss: false },
      categoryNames,
      loginSuccess: s.login_success_css,
      emptyMarkers: ["没有种子", "没有结果", "Nothing found"],
      pagination: 'a[href*="page="]',
      detailPath: "details.php",
      downloadPath: s.download_path,
      ...(y.auth?.supported ? { authSupported: y.auth.supported } : {}),
      login: {
        path: s.login_path,
        usernameField: s.login_username_field,
        passwordField: s.login_password_field,
        errorCss: s.login_error_css,
        ...(s.login_extra_form_data?.length ? { extraFormData: pairs(s.login_extra_form_data) } : {}),
        ...(s.login_select_defaults && Object.keys(s.login_select_defaults).length
          ? { selectDefaults: pairs(s.login_select_defaults) }
          : {}),
      },
      profile,
      detail: {
        path: "details.php",
        title: stripPseudo(s.detail_title_css),
        rowLabel: "td.rowhead",
        rowValue: "td.rowfollow",
        labels: {
          subtitle: ["副标题", "Small Description"],
          basicInfo: ["基本信息", "Basic Info"],
          peers: ["同伴", "Peers"],
        },
        description: s.detail_description_css,
        headerScope: stripPseudo(s.detail_title_css),
        notFoundMarkers: ["没有该ID的种子", "No torrent with ID"],
        fields: {
          title: s.detail_title_css,
          subtitle: s.detail_subtitle_css,
          size: s.detail_size_css,
          description: s.detail_description_css,
          download: s.detail_download_css,
          imdb: s.detail_imdb_css,
          douban: s.detail_douban_css,
          fileList: s.detail_file_list_css,
        },
      },
    },
  };
  if (y.custom_class) {
    def.umt.customClass = y.custom_class;
    const rule = CUSTOM_CLASS_RULES[y.custom_class];
    if (!rule) throw new Error(`${file}: 未登记的 custom_class ${y.custom_class}`);
    if (!rule(def)) def.enabled = false;
  }
  if (y.hr_seed_hours) def.umt.hrSeedHours = Number(y.hr_seed_hours);
  return def;
}

function pairs(v) {
  if (Array.isArray(v)) return v.map((p) => [String(p[0]), String(p[1])]);
  return Object.entries(v).map(([k, x]) => [k, String(x)]);
}

function convertApi(y, file) {
  if (y.site_id !== "mteam") throw new Error(`${file}: 暂只支持 mteam 这一 API 站`);
  const categoryNames = {};
  for (const [k, ids] of Object.entries(y.categories ?? {}))
    for (const cid of ids ?? []) categoryNames[String(cid)] ??= CATEGORY_LABELS[k] ?? k;
  return {
    id: y.site_id,
    name: y.display_name,
    enabled: PRIMARY.has(y.site_id),
    schema: MP_SCHEMA[y.site_id],
    provenance: {
      source: "movieclaw",
      ref: REF,
      commit: COMMIT.slice(0, 8),
      license: "MIT",
      file: `${CONFIG_DIR}/${file}`,
    },
    domain: `${String(y.web_base_url ?? y.base_url).replace(/\/+$/, "")}/`,
    parser: "mTorrent",
    timeout: Number(y.http?.timeout ?? 15),
    umt: {
      version: `${y.site_id}@mc-${REF}`,
      kind: "mteam",
      displayName: y.display_name,
      apiBaseUrl: String(y.base_url).replace(/\/+$/, ""),
      timezone: TZ[y.timezone ?? "Asia/Shanghai"],
      rateLimit: { minIntervalMs: Math.round(Number(y.min_request_interval ?? 3) * 1000), burst: 1 },
      capabilities: { list: true, search: true, detail: true, download: true, rss: false },
      categoryNames,
      ...(y.auth?.supported ? { authSupported: y.auth.supported } : {}),
      ...(y.custom_class ? { customClass: y.custom_class } : {}),
    },
  };
}

function isObj(v) {
  return v && typeof v === "object" && !Array.isArray(v);
}
/**
 * 列表行选择器规范化(W5-D,见 docs/porting/mp-rust-logic-parity.md §5):
 * - `table.torrents > tr…` 在显式带 <tbody> 的页面上取不到行(cheerio 与 MP PyQuery 同),补 `> tbody > tr…` 分支;
 * - 不带条件的 `table.torrents > tr` / `table.torrents tr` 会命中表头行,产出标题为空的垃圾行,收紧为"含 td.rowfollow 数据单元格的行"(NexusPHP 表头是 td.colhead;MP PyQuery 的 :has 只接受简单选择器,不能写属性选择器)。
 */
const ROW_HAS_DETAILS = ":has(td.rowfollow)";
function rowSelector(css) {
  if (!css || !/^\s*table\.torrents[\s>]/.test(css)) return css;
  const parts = [];
  const add = (p) => {
    if (!parts.includes(p)) parts.push(p);
  };
  for (const raw of String(css).split(",")) {
    const part = raw.trim();
    const m = /^table\.torrents\s*(?:>\s*)?(?:tbody\s*>\s*)?tr(.*)$/.exec(part);
    if (!m) {
      add(part);
      continue;
    }
    const cond = m[1].trim() === "" ? ROW_HAS_DETAILS : m[1].trim();
    add(`table.torrents > tr${cond}`);
    add(`table.torrents > tbody > tr${cond}`);
  }
  return parts.join(", ");
}

function deepMerge(base, over, path = "") {
  if (!isObj(base) || !isObj(over)) return over;
  const out = { ...base };
  for (const [k, v] of Object.entries(over)) {
    const p = path ? `${path}.${k}` : k;
    if (v === null) delete out[k];
    else if (/^torrents\.fields\.[^.]+$/.test(p) || /^umt\.detail\.fields$/.test(p))
      out[k] = v; // 字段整体替换
    else out[k] = isObj(v) && isObj(out[k]) ? deepMerge(out[k], v, p) : v;
  }
  return out;
}

/** 用仓库的 biome 格式化(与 lint 一致,保证 --check 幂等);biome 不可用时退回 JSON.stringify */
function biomeFormat(text, fakePath) {
  const bins = [join(ROOT, "node_modules/.bin/biome")];
  for (const bin of bins) {
    if (!existsSync(bin)) continue;
    try {
      return execFileSync(bin, ["format", `--stdin-file-path=${fakePath}`], {
        input: text,
        encoding: "utf8",
        cwd: ROOT,
      });
    } catch {
      /* fallthrough */
    }
  }
  return text;
}

/** 输出键顺序固定,方便 diff */
function stable(v, fakePath) {
  return biomeFormat(`${JSON.stringify(v, null, 2)}\n`, fakePath);
}

// ---- main ----
const COMMIT = git("rev-parse", `${REF}^{commit}`).trim();
if (EXPECTED[REF] && EXPECTED[REF] !== COMMIT)
  throw new Error(`${REF} 解析到 ${COMMIT},与登记的 ${EXPECTED[REF]} 不符`);
if (!EXPECTED[REF] && !args.includes("--allow-ref"))
  throw new Error(`ref ${REF} 未登记;许可需另行核对后加 --allow-ref`);
const license = git("show", `${REF}:LICENSE`);
if (!/^MIT License/m.test(license)) throw new Error(`${REF}:LICENSE 不是 MIT,停止(许可切换后的内容不得复制)`);

const files = git("ls-tree", "--name-only", `${REF}:${CONFIG_DIR}`)
  .split("\n")
  .filter((f) => f.endsWith(".yaml") && !f.startsWith("_"));
const ids = [];
let diffs = 0;
for (const file of files) {
  const y = YAML.parse(git("show", `${REF}:${CONFIG_DIR}/${file}`));
  let def = y.framework === "api" ? convertApi(y, file) : convertNexus(y, file);
  const ovPath = join(OVERRIDES, `${y.site_id}.json`);
  if (existsSync(ovPath)) {
    const ov = JSON.parse(readFileSync(ovPath, "utf8"));
    delete ov.$comment;
    def = deepMerge(def, ov);
    def.provenance = { ...def.provenance, overrides: `scripts/mp-sync/movieclaw-overrides/${y.site_id}.json` };
  }
  // 分类项 desc 以最终 categoryNames 为准(覆盖层可能给出更细的站点分类名)
  for (const k of ["movie", "tv", "music"]) {
    for (const c of def.category?.[k] ?? []) {
      const n = def.umt?.categoryNames?.[String(c.id)];
      if (n) c.desc = n;
    }
  }
  const out = join(OUT_DIR, `${y.site_id}.json`);
  const text = stable(def, `packages/adapters/src/sites/definitions/${y.site_id}.json`);
  const old = existsSync(out) ? readFileSync(out, "utf8") : null;
  if (old !== text) {
    diffs++;
    if (CHECK) console.log(`差异:${y.site_id}.json`);
    else writeFileSync(out, text);
  }
  ids.push(y.site_id);
}

const ident = (id) => `${id.replace(/[^A-Za-z0-9_]/g, "_")}Json`;
const manifest = `// 由 scripts/mp-sync/movieclaw-import.mjs 生成,请勿手改(MovieClaw ${REF} ${COMMIT.slice(0, 8)},MIT)
${ids
  .sort()
  .map((id) => `import ${ident(id)} from "./${id}.json" with { type: "json" };`)
  .join("\n")}

/** MovieClaw 转换来的站点定义原始 JSON(未校验;经 loadSiteDefinition 校验后使用) */
export const MOVIECLAW_DEFINITION_JSON: Record<string, unknown> = {
${ids
  .sort()
  .map((id) => `  ${JSON.stringify(id)}: ${ident(id)},`)
  .join("\n")}
};
`;
const manifestPath = join(OUT_DIR, "manifest.generated.ts");
const manifestText = biomeFormat(manifest, "packages/adapters/src/sites/definitions/manifest.generated.ts");
if ((existsSync(manifestPath) ? readFileSync(manifestPath, "utf8") : null) !== manifestText) {
  diffs++;
  if (CHECK) console.log("差异:manifest.generated.ts");
  else writeFileSync(manifestPath, manifestText);
}
const stale = readdirSync(OUT_DIR).filter((f) => f.endsWith(".json") && !ids.includes(f.replace(/\.json$/, "")));
if (stale.length) console.log(`注意:定义目录中存在非 MovieClaw 来源的 JSON(保留不动):${stale.join(", ")}`);
console.log(
  `${CHECK ? "检查" : "转换"}完成:${ids.length} 个站点,${diffs} 个文件${CHECK ? "有差异" : "已更新"}(${REF} ${COMMIT.slice(0, 8)})`,
);
if (CHECK && diffs) process.exit(1);