feat(adapters): port MoviePilot open site framework (7 spiders, 20 user parsers) + MovieClaw v0.27.0 site definitions + mp-sync tools (W1-3c)

- sites/spiders: Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema (+mTorrent wrapper), specialized SiteAdapter, indirect download resolved at download time - sites/user-parsers: SiteParserBase + all 20 MP SiteSchema parsers (lxml XPath via xpath+xmldom), refreshSiteUserData with schema fallback - sites/definitions: zod SiteDefinition schemas, 23 MovieClaw v0.27.0 (MIT) definitions via scripts/mp-sync/movieclaw-import.mjs, registry getSiteDefinition/listSiteDefinitions, NOTICE - scripts/mp-sync: ledger.json (upstream sha256), check.mjs, build-ledger.mjs; docs/porting/mp-sync-ledger.md - license-and-provenance: add copied-mit kind, re-anchor MovieClaw rows to v0.27.0 Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-09-27 18:39:27 +08:00
1 parent cf85481d95
commit d9f2af2f3c
@@ -11,9 +11,9 @@
 * 本项目调整:登录页不再"返回空列表",而是抛 `auth_expired`(REQ §4.7.1 不展示空列表冒充无内容);
 * 3xx 不自动跟随(避免把 cookie 带到站外,并把跳转 login 识别为失效);凭证经 SecretResolver 注入。
 */
-import { redactText } from "../../rss/redact";
-import { buildHttpContext, type SiteAdapterDeps } from "../deps";
-import { readBody, readText, siteFetch } from "../http";
+import { redactText } from "../../rss/redact.ts";
+import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
+import { readBody, readText, siteFetch } from "../http.ts";
 import {
  type DownloadTarget,
  type SiteAdapter,
@@ -23,12 +23,12 @@ import {
  type SiteListFilters,
  type SitePage,
  type SiteReleaseDetail,
-} from "../types";
-import { inspectTorrent } from "../util";
-import { parseDetailPage } from "./detail";
-import { spiderTorrentToRelease } from "./map";
-import type { IndexerDefinition } from "./schema";
-import { loadHtml, normalizeSelector, SiteSpider } from "./spider";
+} from "../types.ts";
+import { inspectTorrent } from "../util.ts";
+import { parseDetailPage } from "./detail.ts";
+import { spiderTorrentToRelease } from "./map.ts";
+import type { IndexerDefinition } from "./schema.ts";
+import { loadHtml, normalizeSelector, SiteSpider } from "./spider.ts";

 /** MP `text_tools.remove_punctuation(text, replacement=" ", allow_space=True)` */
 export function clearSearchText(text: string): string {
@@ -4,11 +4,12 @@
 * 促销/H&R 复用站点定义中 downloadvolumefactor / uploadvolumefactor / freedate / hr 字段(与列表同语义)。
 */
 import type { CheerioAPI } from "cheerio";
-import type { SiteReleaseDetail } from "../types";
-import { extractDoubanId, extractImdbId, parseCount, parseSiteTime } from "../util";
-import { categoryName, promotionFromFactors } from "./map";
-import type { IndexerDefinition } from "./schema";
-import { loadHtml, parseSizeMp, SiteSpider } from "./spider";
+import type { SiteReleaseDetail } from "../types.ts";
+import { extractDoubanId, extractImdbId, parseCount, parseSiteTime } from "../util.ts";
+import { categoryName, promotionFromFactors } from "./map.ts";
+import { mcSelect } from "./mc-selector.ts";
+import type { IndexerDefinition } from "./schema.ts";
+import { loadHtml, parseSizeMp, SiteSpider } from "./spider.ts";

 export type DetailPageKind = "ok" | "login" | "not_found" | "unknown";

@@ -38,15 +39,23 @@ export function parseDetailPage(
    return found as ReturnType<CheerioAPI> | null;
  };

-  const title = h1
-    .contents()
-    .filter((_, n) => n.type === "text")
-    .text()
+  // 逐字段选择器(MovieClaw detail_*_css)优先于 rowhead 表格模型
+  const f = conf.fields ?? {};
+  const title = (
+    f.title
+      ? (mcSelect($, null, f.title) ?? "")
+      : h1
+          .contents()
+          .filter((_, n) => n.type === "text")
+          .text()
+  )
    .replace(/\s+/g, " ")
    .trim();
-  const subtitle = row(conf.labels.subtitle)?.text().trim();
+  const subtitle = f.subtitle ? (mcSelect($, null, f.subtitle) ?? undefined) : row(conf.labels.subtitle)?.text().trim();
  const basicText = row(conf.labels.basicInfo)?.text().replace(/\s+/g, " ") ?? "";
-  const sizeText = /(?:大小|Size)\s*[:：]\s*([\d.,]+\s*[KMGTP]?i?B)/i.exec(basicText)?.[1] ?? null;
+  const sizeText = f.size
+    ? (/([\d.,]+\s*[KMGTP]?i?B)/i.exec(mcSelect($, null, f.size) ?? "")?.[1] ?? null)
+    : (/(?:大小|Size)\s*[:：]\s*([\d.,]+\s*[KMGTP]?i?B)/i.exec(basicText)?.[1] ?? null);
  const catText = /(?:类型|Type)\s*[:：]\s*(\S+)/i.exec(basicText)?.[1] ?? null;
  const catId = catText
    ? (Object.entries(def.umt.categoryNames ?? {}).find(([, name]) => name === catText)?.[0] ?? null)
@@ -102,9 +111,9 @@ export function parseDetailPage(
  };
  if (subtitle) detail.subtitle = subtitle;
  if (descText) detail.descriptionText = descText.slice(0, 20_000);
-  const imdb = extractImdbId(descHtml) ?? extractImdbId(descText);
+  const imdb = extractImdbId(mcSelect($, null, f.imdb)) ?? extractImdbId(descHtml) ?? extractImdbId(descText);
  if (imdb) detail.imdbId = imdb;
-  const douban = extractDoubanId(descHtml);
+  const douban = extractDoubanId(mcSelect($, null, f.douban)) ?? extractDoubanId(descHtml);
  if (douban) detail.doubanId = douban;
  return { kind: "ok", detail };
 }
@@ -1,8 +1,8 @@
-import { redactUrl } from "../../rss/redact";
-import type { PromotionType, SitePromotion, SiteRelease } from "../types";
-import { parseSiteTime } from "../util";
-import type { IndexerDefinition } from "./schema";
-import type { SpiderTorrent } from "./spider";
+import { redactUrl } from "../../rss/redact.ts";
+import type { PromotionType, SitePromotion, SiteRelease } from "../types.ts";
+import { parseSiteTime } from "../util.ts";
+import type { IndexerDefinition } from "./schema.ts";
+import type { SpiderTorrent } from "./spider.ts";

 /**
 * MP 系数 → 统一促销类型。MP 只有 downloadvolumefactor/uploadvolumefactor/freedate 三个字段,
@@ -0,0 +1,52 @@
+/**
+ * MovieClaw 选择器语法求值(parsel/cssselect 扩展伪元素),用于 MovieClaw v0.27.0 转换来的
+ * `umt.detail.fields` / `umt.profile` 选择器。语义对照 MovieClaw `NexusPHPSite._css_text`(MIT,见
+ * ../definitions/NOTICE-MovieClaw.md):
+ * - 空选择器 → null
+ * - `X::next_sibling_text` → X 首个命中元素之后第一个文本兄弟节点(XPath `following-sibling::text()[1]`),trim
+ * - `X::attr(a)` → 首个命中元素的属性 a;`X::text` → 首个命中元素的直接文本节点(parsel `::text` 取第一个)
+ * - 普通选择器 → 所有命中元素的全部后代文本,逐段 trim 后以空格连接
+ * `:contains()` 由 cheerio-select 原生支持。
+ */
+import type { CheerioAPI } from "cheerio";
+
+type Scope = ReturnType<CheerioAPI>;
+
+export function mcSelect($: CheerioAPI, scope: Scope | null, css: string | null | undefined): string | null {
+  if (!css) return null;
+  const find = (sel: string) => (scope ? scope.find(sel) : $(sel));
+  if (css.endsWith("::next_sibling_text")) {
+    const el = find(css.slice(0, -"::next_sibling_text".length)).first();
+    if (el.length === 0) return null;
+    let n = el[0]!.next;
+    while (n && n.type !== "text") n = n.next;
+    const t = n && n.type === "text" ? (n as unknown as { data: string }).data.trim() : "";
+    return t || null;
+  }
+  const attr = /::attr\(([^)]+)\)$/.exec(css);
+  if (attr) {
+    const v = find(css.slice(0, attr.index)).first().attr(attr[1]!.trim());
+    return v?.trim() || null;
+  }
+  if (css.endsWith("::text")) {
+    const el = find(css.slice(0, -"::text".length)).first();
+    const t = el
+      .contents()
+      .filter((_, n) => n.type === "text")
+      .first()
+      .text()
+      .trim();
+    return t || null;
+  }
+  const texts: string[] = [];
+  find(css).each((_, el) => {
+    const walk = (n: { type: string; data?: string; children?: unknown[] }) => {
+      if (n.type === "text") {
+        const t = (n.data ?? "").trim();
+        if (t) texts.push(t);
+      } else for (const c of (n.children ?? []) as Array<typeof n>) walk(c);
+    };
+    walk(el as unknown as { type: string });
+  });
+  return texts.length ? texts.join(" ") : null;
+}
@@ -63,8 +63,25 @@ export interface SpiderFields {
  [name: string]: SpiderFieldSelector | undefined;
 }

+/** 定义来源(MovieClaw v0.27.0 MIT 转换 / 手写) */
+export interface DefinitionProvenance {
+  source: "movieclaw" | "handwritten";
+  ref?: string;
+  commit?: string;
+  license?: string;
+  file?: string;
+  /** 本项目手工覆盖层(深合并,优先于来源值) */
+  overrides?: string;
+  note?: string;
+}
+
 export interface IndexerDefinition {
  id: string;
+  /** 是否随产品启用(未实测站点为 false) */
+  enabled?: boolean;
+  /** MP 用户信息解析器 schema(`SiteSchema` 值,如 NexusPhp / NexusHhanclub) */
+  schema?: string;
+  provenance?: DefinitionProvenance;
  name: string;
  domain: string;
  encoding?: string;
@@ -95,6 +112,26 @@ export interface UmtSiteExtension {
  /** 定义版本,入 site_snapshot */
  version: string;
  kind: "nexusphp";
+  /** 需要自定义代码的站点(MovieClaw custom_class,如 TTG);通用 spider 结果不可靠 */
+  customClass?: string;
+  /** H&R 考核做种小时数(MovieClaw hr_seed_hours;暂无站点配置) */
+  hrSeedHours?: number;
+  /** 支持的凭据方式(MovieClaw auth.supported:cookie / credential / apikey) */
+  authSupported?: string[];
+  /** 登录表单配置(MovieClaw login_*;M2 账号密码登录用) */
+  login?: {
+    path: string;
+    usernameField: string;
+    passwordField: string;
+    errorCss: string;
+    extraFormData?: Array<[string, string]>;
+    selectDefaults?: Array<[string, string]>;
+  };
+  /**
+   * 用户资料选择器(MovieClaw profile_*,选择器语法见 spider/mc-selector.ts:支持 ::attr(x) / ::text / ::next_sibling_text)。
+   * 主路径仍是 MP 用户信息解析器(`schema`),此处为备用数据,M2 使用。
+   */
+  profile?: Record<string, string>;
  displayName?: string;
  /** 站点时间时区偏移,如 "+08:00";pubdate/freedate 按此转 UTC */
  timezone: string;
@@ -119,6 +156,13 @@ export interface UmtSiteExtension {
    /** 促销/HR 标记所在范围 */
    headerScope: string;
    notFoundMarkers: string[];
+    /**
+     * 逐字段选择器(MovieClaw detail_*_css,mc-selector 语法)。配置时优先于 rowhead 表格模型,
+     * 用于 HHClub 这类 div 详情页。
+     */
+    fields?: Partial<
+      Record<"title" | "subtitle" | "size" | "description" | "download" | "imdb" | "douban" | "fileList", string>
+    >;
  };
  /** 下载路径(按资源 ID 拼接) */
  downloadPath: string;
@@ -9,7 +9,7 @@ import {
  SiteError,
  SiteSpider,
  staticSecretResolver,
-} from "../index";
+} from "../index.ts";

 let site: MockNexusPhpSite | undefined;
 afterEach(async () => {
@@ -19,8 +19,8 @@
 * 3. 不发请求:HTTP/Cookie/UA 由 adapter 注入,本类只负责 URL 构造与 HTML 解析(便于限流与脱敏统一)。
 */
 import { type CheerioAPI, load } from "cheerio";
-import type { IndexerDefinition, SpiderFieldSelector, SpiderFields, SpiderFilter, SpiderListConfig } from "./schema";
-import { renderTemplate } from "./template";
+import type { IndexerDefinition, SpiderFieldSelector, SpiderFields, SpiderFilter, SpiderListConfig } from "./schema.ts";
+import { renderTemplate } from "./template.ts";

 type Sel = ReturnType<CheerioAPI>;