feat(adapters): port MoviePilot open site framework (7 spiders, 20 user parsers) + MovieClaw v0.27.0 site definitions + mp-sync tools (W1-3c)

- sites/spiders: Haidan/HDDolby/RousiPro/SunnyPT/TNode/TorrentLeech/Yema (+mTorrent wrapper), specialized SiteAdapter, indirect download resolved at download time
- sites/user-parsers: SiteParserBase + all 20 MP SiteSchema parsers (lxml XPath via xpath+xmldom), refreshSiteUserData with schema fallback
- sites/definitions: zod SiteDefinition schemas, 23 MovieClaw v0.27.0 (MIT) definitions via scripts/mp-sync/movieclaw-import.mjs, registry getSiteDefinition/listSiteDefinitions, NOTICE
- scripts/mp-sync: ledger.json (upstream sha256), check.mjs, build-ledger.mjs; docs/porting/mp-sync-ledger.md
- license-and-provenance: add copied-mit kind, re-anchor MovieClaw rows to v0.27.0

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-27 18:39:27 +08:00
1 parent cf85481d95
commit d9f2af2f3c
166 files changed
+16577 -186

No files matched your search

+17
View File
@@ -0,0 +1,17 @@
# scripts/mp-sync
跟随上游移植工具(说明与对照总表:`docs/porting/mp-sync-ledger.md`)。Node 24 或 `deno run -A` 均可运行,不依赖根 package.json / deno.json 任务。
| 脚本 | 作用 |
| --- | --- |
| `check.mjs [--mp <MP 仓库>] [--to <提交>] [--json] [--ledger <文件>]` | 比较 `ledger.json` 与 MP 目标提交:已变更 / 已删除 / `spider/`、`parser/` 新增文件,输出 `git diff` 命令;有变化退出码 1 |
| `build-ledger.mjs [--mp <MP 仓库>] [--commit <提交>]` | 扫描源码 `@mp-source <路径> @ <提交>` 标签 + `ledger.manual.json`,生成 `ledger.json`(含上游文件 sha256)并刷新文档表格 |
| `movieclaw-import.mjs [--repo <MC 仓库>] [--ref v0.27.0] [--check]` | MovieClaw v0.27.0(MIT)站点 YAML → `packages/adapters/src/sites/definitions/*.json` + `manifest.generated.ts`;覆盖层 `movieclaw-overrides/<id>.json`;`--check` 只比较 |
```bash
node scripts/mp-sync/check.mjs --to <新提交>
node scripts/mp-sync/build-ledger.mjs --commit <新提交>
node scripts/mp-sync/movieclaw-import.mjs --check
```
MovieClaw v0.27.0 之后为附条件许可:`movieclaw-import.mjs` 校验 ref→提交与 LICENSE=MIT,其他 ref 需 `--allow-ref` 并另行核对许可。
+88
View File
@@ -0,0 +1,88 @@
#!/usr/bin/env node
/**
* 生成 scripts/mp-sync/ledger.json 与 docs/porting/mp-sync-ledger.md 中的对照总表。
*
* node scripts/mp-sync/build-ledger.mjs [--mp /opt/moviepilot-plus/upstream/MoviePilot] [--commit e6820a8]
*
* 数据来源:源码文件头 `@mp-source <上游路径> @ <提交>` 标签 + ledger.manual.json(未打标签的既有移植)。
* 每条记录保存移植时的上游提交与该文件内容 sha256,供 check.mjs 比较。
* 增量移植完成后:把对应源码文件头的提交号改为新提交,再重跑本脚本。
*/
import { readFileSync, writeFileSync } from "node:fs";
import {
biomeFormat,
DEFAULT_MP_REPO,
DOC_PATH,
LEDGER_PATH,
MANUAL_PATH,
parseArgs,
resolveCommit,
scanTags,
upstreamHash,
WATCH_DIRS,
} from "./lib.mjs";
const args = parseArgs(process.argv.slice(2));
const repo = args.mp ?? DEFAULT_MP_REPO;
const defaultCommit = resolveCommit(repo, args.commit ?? "e6820a8");
const byUpstream = new Map();
function add(upstream, commitRef, ours, source) {
const commit = resolveCommit(repo, commitRef);
const key = upstream;
const e = byUpstream.get(key) ?? { upstream, commit, sha256: null, ours: [], sources: new Set() };
if (e.commit !== commit)
throw new Error(`${upstream} 在不同文件中登记了不同提交:${e.commit.slice(0, 7)} / ${commit.slice(0, 7)}`);
for (const o of ours) if (!e.ours.includes(o)) e.ours.push(o);
e.sources.add(source);
byUpstream.set(key, e);
}
for (const t of scanTags()) add(t.upstream, t.commit, [t.ours], "tag");
const manual = JSON.parse(readFileSync(MANUAL_PATH, "utf8"));
for (const m of manual.moviepilot ?? []) add(m.upstream, m.commit ?? defaultCommit, m.ours, "manual");
const entries = [...byUpstream.values()]
.map((e) => {
const h = upstreamHash(repo, e.commit, e.upstream);
if (!h) throw new Error(`上游 ${e.commit.slice(0, 7)}:${e.upstream} 不存在`);
return {
upstream: e.upstream,
commit: e.commit,
sha256: h,
ours: e.ours.sort(),
source: [...e.sources].sort().join("+"),
};
})
.sort((a, b) => a.upstream.localeCompare(b.upstream));
const ledger = {
$comment: "由 scripts/mp-sync/build-ledger.mjs 生成;不要手改(改源码文件头 @mp-source 标签或 ledger.manual.json)。",
upstream: { name: "MoviePilot", url: "https://github.com/jxxghp/MoviePilot", branch: "v3", license: "GPL-3.0" },
watchDirs: WATCH_DIRS,
entries,
};
writeFileSync(LEDGER_PATH, biomeFormat(`${JSON.stringify(ledger, null, 2)}\n`, "scripts/mp-sync/ledger.json"));
// 更新文档对照总表(标记之间的内容)
const BEGIN = "<!-- mp-sync:table:begin -->";
const END = "<!-- mp-sync:table:end -->";
const rows = entries.map(
(e) =>
`| \`${e.upstream}\` | ${e.ours.map((o) => `\`${o.replace(/^packages\/adapters\/src\//, "adapters/").replace(/^packages\/testkit\/src\//, "testkit/")}\``).join("<br>")} | \`${e.commit.slice(0, 7)}\` | \`${e.sha256.slice(0, 12)}\` | ${e.source} |`,
);
const table = [
BEGIN,
`共 ${entries.length} 个上游文件(\`adapters/\` = \`packages/adapters/src/\`)。`,
"",
"| 上游文件 | 本项目文件 | 移植提交 | sha256(前 12 位) | 来源 |",
"| --- | --- | --- | --- | --- |",
...rows,
END,
].join("\n");
const doc = readFileSync(DOC_PATH, "utf8");
const i = doc.indexOf(BEGIN);
const j = doc.indexOf(END);
if (i < 0 || j < 0) throw new Error(`${DOC_PATH} 缺少表格标记`);
writeFileSync(DOC_PATH, doc.slice(0, i) + table + doc.slice(j + END.length));
console.log(`ledger.json:${entries.length} 个上游文件(${defaultCommit.slice(0, 7)});文档表格已更新`);
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env node
/**
* 比较 ledger.json 与 MoviePilot 目标提交,列出需要跟随移植的上游变化。
*
* node scripts/mp-sync/check.mjs [--mp <MP 仓库路径>] [--to <目标提交,默认 HEAD>] [--json]
*
* 输出:已变更 / 已删除 / 未变化 的 ledger 文件,监视目录(spider/、parser/)中 ledger 未收录的新文件,
* 以及逐文件 `git diff` 命令。存在变化时退出码为 1(可用于 CI 或定时提醒)。
* 浅克隆需要先取目标提交:git -C <mp> fetch --depth=1 origin <commit>(或 fetch --unshallow)。
*/
import { readFileSync } from "node:fs";
import { DEFAULT_MP_REPO, LEDGER_PATH, listUpstreamDir, parseArgs, resolveCommit, upstreamHash } from "./lib.mjs";
const args = parseArgs(process.argv.slice(2));
const repo = args.mp ?? DEFAULT_MP_REPO;
const target = resolveCommit(repo, args.to ?? "HEAD");
const ledger = JSON.parse(readFileSync(args.ledger ?? LEDGER_PATH, "utf8"));
const changed = [];
const deleted = [];
const unchanged = [];
for (const e of ledger.entries) {
const h = upstreamHash(repo, target, e.upstream);
if (h === null) deleted.push(e);
else if (h !== e.sha256) changed.push({ ...e, newSha256: h });
else unchanged.push(e);
}
const known = new Set(ledger.entries.map((e) => e.upstream));
const added = [];
for (const dir of ledger.watchDirs ?? []) {
for (const f of listUpstreamDir(repo, target, dir)) if (!known.has(f)) added.push(f);
}
const short = (c) => c.slice(0, 7);
if (args.json) {
console.log(JSON.stringify({ target, changed, deleted, added, unchanged: unchanged.length }, null, 2));
} else {
console.log(`MoviePilot 目标提交 ${short(target)}(ledger ${ledger.entries.length} 个文件)\n`);
const section = (title, list, fmt) => {
console.log(`## ${title}(${list.length})`);
for (const x of list) console.log(fmt(x));
console.log("");
};
section("已变更", changed, (e) =>
[
`- ${e.upstream} (${short(e.commit)} → ${short(target)})`,
` 本项目:${e.ours.join(", ")}`,
` git -C ${repo} diff ${short(e.commit)} ${short(target)} -- ${e.upstream}`,
].join("\n"),
);
section("已删除 / 改名", deleted, (e) =>
[
`- ${e.upstream}(本项目:${e.ours.join(", ")})`,
` git -C ${repo} log --oneline --follow ${short(e.commit)}..${short(target)} -- ${e.upstream}`,
` git -C ${repo} diff --stat -M ${short(e.commit)} ${short(target)} -- ${e.upstream.replace(/\/[^/]+$/, "/")}`,
].join("\n"),
);
section("监视目录新增(ledger 未收录)", added, (f) => `- ${f}\n git -C ${repo} show ${short(target)}:${f}`);
console.log(`未变化:${unchanged.length}`);
if (changed.length || deleted.length || added.length) {
console.log(
"\n处理流程:按 diff 增量移植 → 把对应源码文件头 `@mp-source ... @ <提交>` 改为新提交(新文件新增标签)→ node scripts/mp-sync/build-ledger.mjs --commit <新提交> → 测试。",
);
}
}
process.exit(changed.length || deleted.length || added.length ? 1 : 0);
+385
View File
@@ -0,0 +1,385 @@
{
"$comment": "由 scripts/mp-sync/build-ledger.mjs 生成;不要手改(改源码文件头 @mp-source 标签或 ledger.manual.json)。",
"upstream": {
"name": "MoviePilot",
"url": "https://github.com/jxxghp/MoviePilot",
"branch": "v3",
"license": "GPL-3.0"
},
"watchDirs": ["app/modules/indexer/spider", "app/modules/indexer/parser"],
"entries": [
{
"upstream": "app/adapters/network/cloudflare.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "a23a2c44da051cd36f5dca87fe834a9518fee2019a1088aa2eecd88b3cafcca4",
"ours": ["packages/adapters/src/sites/user-parsers/common.ts"],
"source": "tag"
},
{
"upstream": "app/application/mediaserver.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "f034f0be8ba656e3b6d8148f467f39010f1e2fb51c21565ffef1b0ae8e088eac",
"ours": ["packages/adapters/src/emby/client.ts"],
"source": "manual"
},
{
"upstream": "app/application/rss.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "8ea70f712512486dca06a96da2d352f832ef1e1dbb430c815245bf13058f05eb",
"ours": [
"packages/adapters/src/rss/fetch.ts",
"packages/adapters/src/rss/parse.ts",
"packages/adapters/src/rss/rss-link.ts"
],
"source": "manual"
},
{
"upstream": "app/application/site/sites.pyi",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "8fea9a239354f2b7eebeaf912a569ce0fa161e18b9483c49a802d828c5b10c51",
"ours": [
"packages/adapters/src/sites/definitions/schema.ts",
"packages/adapters/src/sites/registry.ts",
"packages/adapters/src/sites/spider/schema.ts"
],
"source": "manual"
},
{
"upstream": "app/chain/download/submission.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "f7cb870e47d644c9a85c6c62e9284f5b3405da09148540bf1eec55479b7212c9",
"ours": [
"packages/adapters/src/sites/mteam/adapter.ts",
"packages/adapters/src/sites/spider/adapter.ts",
"packages/adapters/src/sites/spiders/common.ts"
],
"source": "manual+tag"
},
{
"upstream": "app/domain/site.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "650c795a4def0a11317572c9210aa111c51e564973928d3af3fce8ce2da66107",
"ours": ["packages/adapters/src/sites/user-parsers/common.ts"],
"source": "tag"
},
{
"upstream": "app/foundation/dom.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "6d6accf2cef132e572aabb15da071ee355a3a6b02957fcb9c2e0c4082f863f4e",
"ours": ["packages/adapters/src/sites/user-parsers/dom.ts"],
"source": "tag"
},
{
"upstream": "app/foundation/size.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "7b0b8f50588cbf9b827be2cfbe00044f493ea329730de8b61526cf159b5a1600",
"ours": ["packages/adapters/src/sites/spider/spider.ts"],
"source": "manual"
},
{
"upstream": "app/foundation/temporal.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "74941f460e4069f45cae85636ded26366f80f9869438b2ff0ca092ef5dddb892",
"ours": ["packages/adapters/src/sites/spiders/common.ts", "packages/adapters/src/sites/user-parsers/common.ts"],
"source": "tag"
},
{
"upstream": "app/foundation/text.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "a88c75dc843131cb4a71e041b59386bea1defd332c77087978be854b56177961",
"ours": ["packages/adapters/src/sites/user-parsers/common.ts"],
"source": "tag"
},
{
"upstream": "app/modules/__init__.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "2edada0ea59108b0e69913ae631beffa3f85f7d4675502cc790696b26cf47ab6",
"ours": ["packages/adapters/src/qbittorrent/paths.ts"],
"source": "manual"
},
{
"upstream": "app/modules/_base/downloader.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "84bd8946c8b26dd2bb7caa4983d4407c6a9b28385ff28df947fe412c3d49c3b5",
"ours": ["packages/adapters/src/qbittorrent/torrent.ts"],
"source": "manual"
},
{
"upstream": "app/modules/emby/emby.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "e6468ea34ef2d6beec7219bcf0d5d309578cc0b65c896b91e1a8b6e65967ed6c",
"ours": ["packages/adapters/src/emby/client.ts"],
"source": "manual"
},
{
"upstream": "app/modules/filemanager/storages/local.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "806a1970ca8eb1f500d5273f62e0a5eaa834568cdb1af7d7c5dd92fe0c3d1702",
"ours": ["packages/adapters/src/fs/copy.ts", "packages/adapters/src/fs/move.ts"],
"source": "manual"
},
{
"upstream": "app/modules/filemanager/transhandler.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "13e2bba491acfde28e1f6ed083a639f306a3bbaac585732b994eccae925b9bad",
"ours": ["packages/adapters/src/fs/move.ts"],
"source": "manual"
},
{
"upstream": "app/modules/indexer/__init__.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "e9c5d15fdaa53205e15210d1f19019b9b82716affe075f6b268f6a044d676641",
"ours": [
"packages/adapters/src/sites/registry.ts",
"packages/adapters/src/sites/spider/adapter.ts",
"packages/adapters/src/sites/spiders/adapter.ts",
"packages/adapters/src/sites/spiders/index.ts",
"packages/adapters/src/sites/spiders/map.ts",
"packages/adapters/src/sites/spiders/types.ts",
"packages/adapters/src/sites/user-parsers/registry.ts"
],
"source": "manual+tag"
},
{
"upstream": "app/modules/indexer/parser/__init__.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "0e6c4215b713493a4efdedfb7f42ff9418ec297adc2567b4f596ebaec8120ea3",
"ours": ["packages/adapters/src/sites/user-parsers/base.ts", "packages/adapters/src/sites/user-parsers/types.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/bitpt.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "871d25e570f1fadcd8f214d8ae9f3b07653a053337a5e14a487a96c99a8da3a0",
"ours": ["packages/adapters/src/sites/user-parsers/bitpt.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/discuz.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "12277ba955ce7263c4b1862749443878c0900350ea8d5eb970bb57e1249c279d",
"ours": ["packages/adapters/src/sites/user-parsers/discuz.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/file_list.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "f92e65ed73278c8e68fdd86927cb307e418564580daf8fded3e7a467d7bea7ac",
"ours": ["packages/adapters/src/sites/user-parsers/file-list.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/gazelle.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "c10e5246724b51391fff60f90e17022c0fd5fa2a4151ae69d6c4ee9ad1c5f1c6",
"ours": ["packages/adapters/src/sites/user-parsers/gazelle.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/hddolby.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "a4336d92d603c06953bb957d166f354d7a847eda75d9eaf2bcc96212f6e58c56",
"ours": ["packages/adapters/src/sites/user-parsers/hddolby.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/ipt.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "2b9d04d76606329f32d7f726b5ae478997cdb7ee55f74ff89ab881c641e58bc0",
"ours": ["packages/adapters/src/sites/user-parsers/ipt.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/mtorrent.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "c5d5a3c6f9f5fa5a60e3d65426bdc6620e94e861e0cf27f718595afc61b86165",
"ours": ["packages/adapters/src/sites/user-parsers/mtorrent.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/nexus_audiences.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "1a2d104db7699c8dffbc79546a63dd40108e8fa171f1e7380e886521b0c3698f",
"ours": ["packages/adapters/src/sites/user-parsers/nexus-audiences.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/nexus_hhanclub.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "8850619069d4b9baaac2ef53fbdf5bac88b365b2e3009455ad40593c963c5ccb",
"ours": ["packages/adapters/src/sites/user-parsers/nexus-hhanclub.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/nexus_php.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "39303ae96d9cd198d0145e10539357ac17186be0127eaa92ada36ee3746296d2",
"ours": ["packages/adapters/src/sites/user-parsers/nexus-php.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/nexus_project.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "2b5508cfb2f89efa343515c5b91ae7af95fb9adf4624da6686c4f3b0004f07b0",
"ours": ["packages/adapters/src/sites/user-parsers/nexus-project.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/nexus_rabbit.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "09149b89a7c76b9880d34deedc7335aba3051a81910d333df237614a0a55a98f",
"ours": ["packages/adapters/src/sites/user-parsers/nexus-rabbit.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/rousi.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "1b1ea93de084c20b7001fc22994d87b4f99d5c04773ee25cb1482966fefca205",
"ours": ["packages/adapters/src/sites/user-parsers/rousi.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/small_horse.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "b73b5de97737ca60e46bfecfc580bad6d9c7ee573d01bf6002f0282bcdf229ab",
"ours": ["packages/adapters/src/sites/user-parsers/small-horse.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/sunnypt.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "937f8453fa2b73d06a690ff91e8939cd1478009b4b731f622d6585d9cdbc28cf",
"ours": ["packages/adapters/src/sites/user-parsers/sunnypt.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/tnode.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "c9cb809fc4bffc161f2595cbb6db97b919c83d437e3bdc29374e46e3c46ba74e",
"ours": ["packages/adapters/src/sites/user-parsers/tnode.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/torrent_leech.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "72d1473a3bba5398d1a283995b646f7c93683992c4fa9f6a0d5eb80272754b0b",
"ours": ["packages/adapters/src/sites/user-parsers/torrent-leech.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/unit3d.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "9a4d579deca593b445e80e560ed79c265e9b95cb631bef91e4dca859d020b912",
"ours": ["packages/adapters/src/sites/user-parsers/unit3d.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/yema.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "0f98fd580bad06edebaf81730deb5f11ad7c55ac5ae0d9ebc76ae50c5f828017",
"ours": ["packages/adapters/src/sites/user-parsers/yema.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/parser/zhixing.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "90c8954e73876d9437220bf3a26c8f863d6b4773b54d97ae07321ee0c1b1abc5",
"ours": ["packages/adapters/src/sites/user-parsers/zhixing.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/spider/__init__.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "2513d962fa437f9bf2ca06f5f908b7de8b9bb7b032f80eff283611af38a8b487",
"ours": [
"packages/adapters/src/sites/spider/map.ts",
"packages/adapters/src/sites/spider/schema.ts",
"packages/adapters/src/sites/spider/spider.ts",
"packages/adapters/src/sites/spider/template.ts"
],
"source": "manual"
},
{
"upstream": "app/modules/indexer/spider/haidan.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "75eb802f19f02cf64ed2b7b981026e47369a2d815c6a666c5a9fd70aa86a598e",
"ours": ["packages/adapters/src/sites/spiders/haidan.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/spider/hddolby.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "46a50bd8d49f83a21f59b6273b0378d2342feb78e7f2f5b953eb9d1311c38804",
"ours": ["packages/adapters/src/sites/spiders/hddolby.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/spider/mtorrent.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "a8ab2b822cb24a22c16a8db01e0ef68985435a25b5332d741992f9ff99c10de5",
"ours": [
"packages/adapters/src/sites/mteam/adapter.ts",
"packages/adapters/src/sites/mteam/client.ts",
"packages/adapters/src/sites/mteam/spider.ts",
"packages/adapters/src/sites/spiders/mtorrent.ts"
],
"source": "manual+tag"
},
{
"upstream": "app/modules/indexer/spider/rousi.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "aa13c386a560a97125ac3879b56187c50564d544e925ad9f221bbbefc15e712a",
"ours": ["packages/adapters/src/sites/spiders/rousi.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/spider/sunnypt.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "735b0d721e65ec4006d1ea309b1815635caa7e69e1897643faaa6ca8b84897b8",
"ours": ["packages/adapters/src/sites/spiders/sunnypt.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/spider/tnode.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "a3634a268dbad6cfe45a5c70606b94a4c2fe8f9b351d9b7d6c6499f4a8107fe3",
"ours": ["packages/adapters/src/sites/spiders/tnode.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/spider/torrentleech.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "c965f2bcaea4a50335bcb6e670a9ddb3a96a3f62f59bf3f93deff9cd9e7e28e5",
"ours": ["packages/adapters/src/sites/spiders/torrentleech.ts"],
"source": "tag"
},
{
"upstream": "app/modules/indexer/spider/yema.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "560b62c1ff102d3efa47ad9dc31ba1a5794eae804fc320dbb06f08b72a004f3f",
"ours": ["packages/adapters/src/sites/spiders/yema.ts"],
"source": "tag"
},
{
"upstream": "app/modules/qbittorrent/qbittorrent.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "ab02e97e986f5d4d5441419364ee1e04411f9c9b77323c713e059301d9746390",
"ours": ["packages/adapters/src/qbittorrent/client.ts", "packages/adapters/src/qbittorrent/state.ts"],
"source": "manual"
},
{
"upstream": "app/modules/themoviedb/tmdbapi.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "787553a6f57d8b21e04442d100e098694b9ff7835a2f8d7541b53511606caff8",
"ours": ["packages/adapters/src/tmdb/client.ts", "packages/adapters/src/tmdb/match.ts"],
"source": "manual"
},
{
"upstream": "app/schemas/site.py",
"commit": "e6820a83ab05e8a00eb0e14adf0f357faa965c7e",
"sha256": "e7952ed40b7b5a66d95fe1b07e1d3915601b47c2936ff10ec6880c8a030d0b8f",
"ours": ["packages/adapters/src/sites/user-parsers/types.ts"],
"source": "tag"
}
]
}
+64
View File
@@ -0,0 +1,64 @@
{
"$comment": "未在文件头打 @mp-source 标签的既有移植(W1-3a/W1-3b 及 sites 目录外文件)的人工对照;build-ledger.mjs 与源码标签合并生成 ledger.json。",
"moviepilot": [
{
"upstream": "app/modules/indexer/spider/__init__.py",
"ours": [
"packages/adapters/src/sites/spider/spider.ts",
"packages/adapters/src/sites/spider/schema.ts",
"packages/adapters/src/sites/spider/map.ts",
"packages/adapters/src/sites/spider/template.ts"
]
},
{
"upstream": "app/modules/indexer/spider/mtorrent.py",
"ours": [
"packages/adapters/src/sites/mteam/spider.ts",
"packages/adapters/src/sites/mteam/adapter.ts",
"packages/adapters/src/sites/mteam/client.ts"
]
},
{
"upstream": "app/modules/indexer/__init__.py",
"ours": ["packages/adapters/src/sites/spider/adapter.ts", "packages/adapters/src/sites/registry.ts"]
},
{
"upstream": "app/chain/download/submission.py",
"ours": ["packages/adapters/src/sites/spider/adapter.ts", "packages/adapters/src/sites/mteam/adapter.ts"]
},
{
"upstream": "app/application/site/sites.pyi",
"ours": [
"packages/adapters/src/sites/spider/schema.ts",
"packages/adapters/src/sites/definitions/schema.ts",
"packages/adapters/src/sites/registry.ts"
]
},
{ "upstream": "app/foundation/size.py", "ours": ["packages/adapters/src/sites/spider/spider.ts"] },
{
"upstream": "app/application/rss.py",
"ours": [
"packages/adapters/src/rss/fetch.ts",
"packages/adapters/src/rss/parse.ts",
"packages/adapters/src/rss/rss-link.ts"
]
},
{
"upstream": "app/modules/qbittorrent/qbittorrent.py",
"ours": ["packages/adapters/src/qbittorrent/client.ts", "packages/adapters/src/qbittorrent/state.ts"]
},
{ "upstream": "app/modules/_base/downloader.py", "ours": ["packages/adapters/src/qbittorrent/torrent.ts"] },
{ "upstream": "app/modules/__init__.py", "ours": ["packages/adapters/src/qbittorrent/paths.ts"] },
{
"upstream": "app/modules/themoviedb/tmdbapi.py",
"ours": ["packages/adapters/src/tmdb/client.ts", "packages/adapters/src/tmdb/match.ts"]
},
{
"upstream": "app/modules/filemanager/storages/local.py",
"ours": ["packages/adapters/src/fs/copy.ts", "packages/adapters/src/fs/move.ts"]
},
{ "upstream": "app/modules/filemanager/transhandler.py", "ours": ["packages/adapters/src/fs/move.ts"] },
{ "upstream": "app/modules/emby/emby.py", "ours": ["packages/adapters/src/emby/client.ts"] },
{ "upstream": "app/application/mediaserver.py", "ours": ["packages/adapters/src/emby/client.ts"] }
]
}
+106
View File
@@ -0,0 +1,106 @@
// mp-sync 公共函数(Node 24 / Deno 2 均可运行,无第三方依赖)
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
import { dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
export const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
export const LEDGER_PATH = join(ROOT, "scripts/mp-sync/ledger.json");
export const MANUAL_PATH = join(ROOT, "scripts/mp-sync/ledger.manual.json");
export const DOC_PATH = join(ROOT, "docs/porting/mp-sync-ledger.md");
export const DEFAULT_MP_REPO = "/opt/moviepilot-plus/upstream/MoviePilot";
/** 需要监视"新增文件"的上游目录(MP 开源站点框架) */
export const WATCH_DIRS = ["app/modules/indexer/spider", "app/modules/indexer/parser"];
/** 扫描 @mp-source 标签的本项目目录 */
export const SCAN_DIRS = ["packages/adapters/src", "packages/testkit/src"];
export function parseArgs(argv) {
const out = { _: [] };
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a.startsWith("--")) {
const k = a.slice(2);
const v = argv[i + 1] && !argv[i + 1].startsWith("--") ? argv[++i] : true;
out[k] = v;
} else out._.push(a);
}
return out;
}
export function git(repo, ...args) {
return execFileSync("git", ["-C", repo, ...args], {
encoding: "buffer",
maxBuffer: 256 * 1024 * 1024,
stdio: ["ignore", "pipe", "ignore"],
});
}
export function tryGit(repo, ...args) {
try {
return git(repo, ...args);
} catch {
return null;
}
}
export function sha256(buf) {
return createHash("sha256").update(buf).digest("hex");
}
export function resolveCommit(repo, ref) {
const out = tryGit(repo, "rev-parse", "--verify", `${ref}^{commit}`);
if (!out) throw new Error(`上游仓库 ${repo} 中找不到提交 ${ref}(浅克隆需先 git fetch --depth=1 origin <commit>)`);
return out.toString().trim();
}
/** 上游文件在某提交的内容 sha256;不存在返回 null */
export function upstreamHash(repo, commit, path) {
const buf = tryGit(repo, "show", `${commit}:${path}`);
return buf ? sha256(buf) : null;
}
export function listUpstreamDir(repo, commit, dir) {
const out = tryGit(repo, "ls-tree", "--name-only", `${commit}:${dir}`);
if (!out) return [];
return out
.toString()
.split("\n")
.filter((f) => f.endsWith(".py"))
.map((f) => `${dir}/${f}`);
}
function walk(dir, acc) {
for (const name of readdirSync(dir)) {
if (name === "node_modules") continue;
const p = join(dir, name);
if (statSync(p).isDirectory()) walk(p, acc);
else if (/\.(ts|mjs)$/.test(name)) acc.push(p);
}
return acc;
}
/** 扫描源码文件头 `@mp-source <path> @ <commit>` 标签 */
export function scanTags() {
const tags = [];
for (const d of SCAN_DIRS) {
for (const file of walk(join(ROOT, d), [])) {
const text = readFileSync(file, "utf8");
for (const m of text.matchAll(/@mp-source\s+(\S+)\s+@\s+([0-9a-f]{7,40})/g)) {
tags.push({ upstream: m[1], commit: m[2], ours: relative(ROOT, file) });
}
}
}
return tags;
}
/** 用仓库 biome 格式化(与 lint 一致,生成物重跑幂等);不可用时原样返回 */
export function biomeFormat(text, fakePath) {
const bin = join(ROOT, "node_modules/.bin/biome");
if (!existsSync(bin)) return text;
try {
return execFileSync(bin, ["format", `--stdin-file-path=${fakePath}`], { input: text, encoding: "utf8", cwd: ROOT });
} catch {
return text;
}
}
+468
View File
@@ -0,0 +1,468 @@
#!/usr/bin/env node
/**
* MovieClaw v0.27.0(MIT)站点 YAML → 本项目 SiteDefinition JSON 转换器(可重复执行)。
*
* 用法:
* node scripts/mp-sync/movieclaw-import.mjs [--repo spikes/movieclaw-src/repo] [--ref v0.27.0] [--check]
*
* - 只用 `git show <ref>:<path>` 读取文件(固定 tag,不读工作区,避免混入许可切换后的内容)。
* - 校验 <ref> 解析到的提交与 LICENSE 为 MIT;ref 不是 v0.27.0 时要求 --allow-ref(许可需另行核对)。
* - 输出:packages/adapters/src/sites/definitions/<site_id>.json 与 manifest.generated.ts。
* - 手工覆盖层:scripts/mp-sync/movieclaw-overrides/<site_id>.json 深合并到转换结果之上(覆盖层优先;
* `torrents.fields.<name>` 与数组整体替换)。
* - `--check`:只比较,不写文件;有差异时退出码 1(CI/跟随更新用)。
*
* 选择器默认值来自 MovieClaw `movieclaw_tracker/selectors.py` NexusPHPSelectors(MIT,见
* packages/adapters/src/sites/definitions/NOTICE-MovieClaw.md)。
*/
import { execFileSync } from "node:child_process";
import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
import { createRequire } from "node:module";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
/** yaml 解析器:Deno(根 deno.json import map)或 Node(node_modules)均可 */
async function loadYaml() {
try {
return (await import("yaml")).default;
} catch {
for (const base of ["packages/adapters/package.json", "package.json"]) {
try {
return createRequire(join(ROOT, base))("yaml");
} catch {
/* next */
}
}
}
throw new Error("找不到 yaml 包:请用 `deno run -A scripts/mp-sync/movieclaw-import.mjs` 运行");
}
const YAML = await loadYaml();
const args = process.argv.slice(2);
const opt = (name, def) => {
const i = args.indexOf(name);
return i >= 0 ? args[i + 1] : def;
};
const REPO = resolve(ROOT, opt("--repo", "spikes/movieclaw-src/repo"));
const REF = opt("--ref", "v0.27.0");
const CHECK = args.includes("--check");
const EXPECTED = { "v0.27.0": "c79bb1b065c826823fc1c15bcb99b3c282c3fce0" };
const OUT_DIR = join(ROOT, "packages/adapters/src/sites/definitions");
const OVERRIDES = join(ROOT, "scripts/mp-sync/movieclaw-overrides");
const CONFIG_DIR = "src/movieclaw_tracker/sites/configs";
/** 主力站:默认启用;其余 enabled:false(未实测) */
const PRIMARY = new Set(["audiences", "hhanclub", "chdbits", "mteam"]);
/** MP 用户信息解析器 schema(app/modules/indexer/parser SiteSchema 值) */
const MP_SCHEMA = { audiences: "NexusAudiences", hhanclub: "NexusHhanclub", mteam: "MTorrent", hddolby: "HDDolby" };
const CATEGORY_LABELS = {
movie: "电影",
tv: "剧集",
documentary: "纪录片",
anime: "动漫",
music: "音乐",
game: "游戏",
av: "成人",
other: "其他",
};
const TZ = { "Asia/Shanghai": "+08:00", "Asia/Hong_Kong": "+08:00", "Asia/Taipei": "+08:00", UTC: "+00:00" };
// ---- MovieClaw NexusPHPSelectors 默认值(v0.27.0 selectors.py) ----
const MC_DEFAULTS = {
login_path: "login.php",
login_username_field: "username",
login_password_field: "password",
login_error_css: "td.text",
login_success_css: "a[href*='logout']",
login_extra_form_data: [],
login_select_defaults: [],
torrent_list_path: "torrents.php",
torrent_row_css: "table.torrents > tr:not(:first-child)",
torrent_id_css: "a[href*='details.php']::attr(href)",
torrent_title_css: "a[href*='details.php']",
torrent_subtitle_css: "td.embedded > span",
torrent_category_css: "td:nth-child(1) img::attr(class)",
torrent_size_css: "td:nth-child(5)",
torrent_seeders_css: "td:nth-child(6)",
torrent_leechers_css: "td:nth-child(7)",
torrent_snatched_css: "td:nth-child(8)",
torrent_time_css: "td:nth-child(4) > span::attr(title)",
torrent_time_fallback_css: "td:nth-child(4)",
torrent_time_fmt: "%Y-%m-%d %H:%M:%S",
torrent_uploader_css: "td:nth-child(9) a",
promo_download_rules: [
["img.pro_free", 0.0],
["img.pro_free2up", 0.0],
["img.pro_50pctdown", 0.5],
["img.pro_50pctdown2up", 0.5],
["img.pro_30pctdown", 0.3],
],
promo_upload_rules: [
["img.pro_free2up", 2.0],
["img.pro_50pctdown2up", 2.0],
["img.pro_2up", 2.0],
],
torrent_hr_css: "",
torrent_promo_deadline_css: "",
torrent_promo_deadline_attr: "title",
torrent_promo_deadline_re: "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}",
torrent_promo_deadline_fmt: "%Y-%m-%d %H:%M:%S",
torrent_detail_url_css: "a[href*='details.php']::attr(href)",
torrent_download_url_css: "a[href*='download.php']::attr(href)",
page_offset: 0,
search_path: "torrents.php",
search_keyword_param: "search",
detail_title_css: "#top::text",
detail_subtitle_css: "td.rowhead:contains('副标题') + td::text",
detail_description_css: "#kdescr",
detail_download_css: "a[href*='download.php']::attr(href)",
detail_imdb_css: "a[href*='imdb.com']::attr(href)",
detail_douban_css: "a[href*='douban.com']::attr(href)",
detail_size_css: "td.rowhead:contains('大小') + td::text",
detail_file_list_css: "td.rowfollow > ul > li::text",
profile_path: "userdetails.php",
profile_uid_css: "",
profile_username_css: "h1::text",
profile_class_css: "td.rowhead:contains('等级') + td::text",
profile_uploaded_css: "td.rowhead:contains('上传量') + td::text",
profile_downloaded_css: "td.rowhead:contains('下载量') + td::text",
profile_ratio_css: "td.rowhead:contains('分享率') + td::text",
profile_bonus_css: "td.rowhead:contains('魔力') + td::text",
profile_join_date_css: "td.rowhead:contains('加入日期') + td::text",
profile_seeding_css: "td.rowhead:contains('当前做种') + td::text",
profile_leeching_css: "td.rowhead:contains('当前下载') + td::text",
profile_vip_css: "",
download_path: "download.php",
};
/** MovieClaw custom_class 的行为差异 → 定义层表达(只收录能用定义表达的部分) */
const CUSTOM_CLASS_RULES = {
"movieclaw_tracker.sites.custom.ourbits.OurBitsSite": (def) => {
// 分类参数为重复的 cat[](custom/ourbits.py _apply_category_params)
def.category.param = "cat[]";
return true; // 已完全由定义表达
},
"movieclaw_tracker.sites.custom.ttg.TTGSite": () => false, // 分类/ID/做种列均非标准,需自定义适配器
"movieclaw_tracker.sites.custom.mteam.MTeamSite": () => true, // 由 mteam/* 实现
};
function git(...a) {
return execFileSync("git", ["-C", REPO, ...a], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
}
function stripPseudo(css) {
return css.replace(/::(attr\([^)]*\)|text|next_sibling_text)$/, "");
}
/** MovieClaw `X::attr(a)` / `X::text` / `X` → SiteSpider 字段选择器 */
function field(css, extra = {}) {
if (!css) return undefined;
const m = /^(.*)::attr\(([^)]+)\)$/.exec(css);
if (m) return { selector: m[1].trim(), attribute: m[2].trim(), ...extra };
return { selector: stripPseudo(css).trim(), ...extra };
}
function caseTable(rules) {
const out = {};
for (const [css, factor] of rules) out[css] = Number(factor);
out["*"] = 1;
return out;
}
function rulesFrom(v, fallback) {
if (v == null) return fallback;
if (Array.isArray(v)) return v;
return Object.entries(v);
}
function convertNexus(y, file) {
const s = { ...MC_DEFAULTS, ...(y.selectors ?? {}) };
s.promo_download_rules = rulesFrom(y.selectors?.promo_download_rules, MC_DEFAULTS.promo_download_rules);
s.promo_upload_rules = rulesFrom(y.selectors?.promo_upload_rules, MC_DEFAULTS.promo_upload_rules);
const id = y.site_id;
const tz = TZ[y.timezone ?? "Asia/Shanghai"];
if (!tz) throw new Error(`${file}: 未知时区 ${y.timezone}`);
const cats = y.categories ?? {};
const item = (k) => (cats[k] ?? []).map((cid) => ({ id: String(cid), desc: CATEGORY_LABELS[k] }));
const categoryNames = {};
for (const [k, ids] of Object.entries(cats))
for (const cid of ids ?? []) categoryNames[String(cid)] ??= CATEGORY_LABELS[k] ?? k;
const titleCss = stripPseudo(s.torrent_title_css);
const fields = {
title_default: { selector: titleCss },
title_optional: { selector: `${titleCss}[title]`, attribute: "title" },
title: {
text: "{% if fields['title_optional'] %}{{ fields['title_optional'] }}{% else %}{{ fields['title_default'] }}{% endif %}",
},
details: field(s.torrent_detail_url_css),
download: field(s.torrent_download_url_css),
imdbid: {
selector: 'a[href*="imdb.com/title/tt"]',
attribute: "href",
filters: [{ name: "re_search", args: ["tt\\d+", 0] }],
},
date_elapsed: field(s.torrent_time_fallback_css),
date_added: field(s.torrent_time_css),
date: {
text: "{% if fields['date_elapsed'] or fields['date_added'] %}{{ fields['date_added'] if fields['date_added'] else fields['date_elapsed'] }}{% else %}now{% endif %}",
filters: [{ name: "dateparse", args: s.torrent_time_fmt }],
},
size: field(s.torrent_size_css),
seeders: field(s.torrent_seeders_css),
leechers: field(s.torrent_leechers_css),
grabs: field(s.torrent_snatched_css),
downloadvolumefactor: { case: caseTable(s.promo_download_rules) },
uploadvolumefactor: { case: caseTable(s.promo_upload_rules) },
description: field(s.torrent_subtitle_css),
category: field(s.torrent_category_css, { filters: [{ name: "re_search", args: ["\\d+", 0] }] }),
};
if (s.torrent_promo_deadline_css) {
fields.freedate = {
selector: stripPseudo(s.torrent_promo_deadline_css),
attribute: s.torrent_promo_deadline_attr || "title",
filters: [{ name: "re_search", args: [s.torrent_promo_deadline_re, 0] }],
};
}
if (s.torrent_hr_css) fields.hr = { selector: s.torrent_hr_css };
const profile = {};
for (const [k, v] of Object.entries(s)) {
if (k.startsWith("profile_") && typeof v === "string" && v) profile[k.slice(8)] = v;
}
const def = {
id,
name: y.display_name,
enabled: PRIMARY.has(id),
schema: MP_SCHEMA[id] ?? "NexusPhp",
provenance: {
source: "movieclaw",
ref: REF,
commit: COMMIT.slice(0, 8),
license: "MIT",
file: `${CONFIG_DIR}/${file}`,
},
domain: `${String(y.base_url).replace(/\/+$/, "")}/`,
encoding: "UTF-8",
public: false,
proxy: false,
language: "zh",
result_num: 100,
timeout: Number(y.http?.timeout ?? 30),
search: { paths: [{ path: s.search_path, type: "all" }], params: { [s.search_keyword_param]: "{keyword}" } },
batch: { delimiter: " ", space_replace: "_" },
browse: { path: `${s.torrent_list_path}?page={page}`, start: Number(s.page_offset) === -1 ? 0 : 1 },
category: { movie: item("movie"), tv: item("tv"), music: item("music") },
torrents: { list: { selector: s.torrent_row_css }, fields },
umt: {
version: `${id}@mc-${REF}`,
kind: "nexusphp",
displayName: y.display_name,
timezone: tz,
rateLimit: { minIntervalMs: Math.round(Number(y.min_request_interval ?? 3) * 1000), burst: 1 },
capabilities: { list: true, search: true, detail: true, download: true, rss: false },
categoryNames,
loginSuccess: s.login_success_css,
emptyMarkers: ["没有种子", "没有结果", "Nothing found"],
pagination: 'a[href*="page="]',
detailPath: "details.php",
downloadPath: s.download_path,
...(y.auth?.supported ? { authSupported: y.auth.supported } : {}),
login: {
path: s.login_path,
usernameField: s.login_username_field,
passwordField: s.login_password_field,
errorCss: s.login_error_css,
...(s.login_extra_form_data?.length ? { extraFormData: pairs(s.login_extra_form_data) } : {}),
...(s.login_select_defaults && Object.keys(s.login_select_defaults).length
? { selectDefaults: pairs(s.login_select_defaults) }
: {}),
},
profile,
detail: {
path: "details.php",
title: stripPseudo(s.detail_title_css),
rowLabel: "td.rowhead",
rowValue: "td.rowfollow",
labels: {
subtitle: ["副标题", "Small Description"],
basicInfo: ["基本信息", "Basic Info"],
peers: ["同伴", "Peers"],
},
description: s.detail_description_css,
headerScope: stripPseudo(s.detail_title_css),
notFoundMarkers: ["没有该ID的种子", "No torrent with ID"],
fields: {
title: s.detail_title_css,
subtitle: s.detail_subtitle_css,
size: s.detail_size_css,
description: s.detail_description_css,
download: s.detail_download_css,
imdb: s.detail_imdb_css,
douban: s.detail_douban_css,
fileList: s.detail_file_list_css,
},
},
},
};
if (y.custom_class) {
def.umt.customClass = y.custom_class;
const rule = CUSTOM_CLASS_RULES[y.custom_class];
if (!rule) throw new Error(`${file}: 未登记的 custom_class ${y.custom_class}`);
if (!rule(def)) def.enabled = false;
}
if (y.hr_seed_hours) def.umt.hrSeedHours = Number(y.hr_seed_hours);
return def;
}
function pairs(v) {
if (Array.isArray(v)) return v.map((p) => [String(p[0]), String(p[1])]);
return Object.entries(v).map(([k, x]) => [k, String(x)]);
}
function convertApi(y, file) {
if (y.site_id !== "mteam") throw new Error(`${file}: 暂只支持 mteam 这一 API 站`);
const categoryNames = {};
for (const [k, ids] of Object.entries(y.categories ?? {}))
for (const cid of ids ?? []) categoryNames[String(cid)] ??= CATEGORY_LABELS[k] ?? k;
return {
id: y.site_id,
name: y.display_name,
enabled: PRIMARY.has(y.site_id),
schema: MP_SCHEMA[y.site_id],
provenance: {
source: "movieclaw",
ref: REF,
commit: COMMIT.slice(0, 8),
license: "MIT",
file: `${CONFIG_DIR}/${file}`,
},
domain: `${String(y.web_base_url ?? y.base_url).replace(/\/+$/, "")}/`,
parser: "mTorrent",
timeout: Number(y.http?.timeout ?? 15),
umt: {
version: `${y.site_id}@mc-${REF}`,
kind: "mteam",
displayName: y.display_name,
apiBaseUrl: String(y.base_url).replace(/\/+$/, ""),
timezone: TZ[y.timezone ?? "Asia/Shanghai"],
rateLimit: { minIntervalMs: Math.round(Number(y.min_request_interval ?? 3) * 1000), burst: 1 },
capabilities: { list: true, search: true, detail: true, download: true, rss: false },
categoryNames,
...(y.auth?.supported ? { authSupported: y.auth.supported } : {}),
...(y.custom_class ? { customClass: y.custom_class } : {}),
},
};
}
function isObj(v) {
return v && typeof v === "object" && !Array.isArray(v);
}
function deepMerge(base, over, path = "") {
if (!isObj(base) || !isObj(over)) return over;
const out = { ...base };
for (const [k, v] of Object.entries(over)) {
const p = path ? `${path}.${k}` : k;
if (v === null) delete out[k];
else if (/^torrents\.fields\.[^.]+$/.test(p) || /^umt\.detail\.fields$/.test(p))
out[k] = v; // 字段整体替换
else out[k] = isObj(v) && isObj(out[k]) ? deepMerge(out[k], v, p) : v;
}
return out;
}
/** 用仓库的 biome 格式化(与 lint 一致,保证 --check 幂等);biome 不可用时退回 JSON.stringify */
function biomeFormat(text, fakePath) {
const bins = [join(ROOT, "node_modules/.bin/biome")];
for (const bin of bins) {
if (!existsSync(bin)) continue;
try {
return execFileSync(bin, ["format", `--stdin-file-path=${fakePath}`], {
input: text,
encoding: "utf8",
cwd: ROOT,
});
} catch {
/* fallthrough */
}
}
return text;
}
/** 输出键顺序固定,方便 diff */
function stable(v, fakePath) {
return biomeFormat(`${JSON.stringify(v, null, 2)}\n`, fakePath);
}
// ---- main ----
const COMMIT = git("rev-parse", `${REF}^{commit}`).trim();
if (EXPECTED[REF] && EXPECTED[REF] !== COMMIT)
throw new Error(`${REF} 解析到 ${COMMIT},与登记的 ${EXPECTED[REF]} 不符`);
if (!EXPECTED[REF] && !args.includes("--allow-ref"))
throw new Error(`ref ${REF} 未登记;许可需另行核对后加 --allow-ref`);
const license = git("show", `${REF}:LICENSE`);
if (!/^MIT License/m.test(license)) throw new Error(`${REF}:LICENSE 不是 MIT,停止(许可切换后的内容不得复制)`);
const files = git("ls-tree", "--name-only", `${REF}:${CONFIG_DIR}`)
.split("\n")
.filter((f) => f.endsWith(".yaml") && !f.startsWith("_"));
const ids = [];
let diffs = 0;
for (const file of files) {
const y = YAML.parse(git("show", `${REF}:${CONFIG_DIR}/${file}`));
let def = y.framework === "api" ? convertApi(y, file) : convertNexus(y, file);
const ovPath = join(OVERRIDES, `${y.site_id}.json`);
if (existsSync(ovPath)) {
const ov = JSON.parse(readFileSync(ovPath, "utf8"));
delete ov.$comment;
def = deepMerge(def, ov);
def.provenance = { ...def.provenance, overrides: `scripts/mp-sync/movieclaw-overrides/${y.site_id}.json` };
}
// 分类项 desc 以最终 categoryNames 为准(覆盖层可能给出更细的站点分类名)
for (const k of ["movie", "tv", "music"]) {
for (const c of def.category?.[k] ?? []) {
const n = def.umt?.categoryNames?.[String(c.id)];
if (n) c.desc = n;
}
}
const out = join(OUT_DIR, `${y.site_id}.json`);
const text = stable(def, `packages/adapters/src/sites/definitions/${y.site_id}.json`);
const old = existsSync(out) ? readFileSync(out, "utf8") : null;
if (old !== text) {
diffs++;
if (CHECK) console.log(`差异:${y.site_id}.json`);
else writeFileSync(out, text);
}
ids.push(y.site_id);
}
const ident = (id) => `${id.replace(/[^A-Za-z0-9_]/g, "_")}Json`;
const manifest = `// 由 scripts/mp-sync/movieclaw-import.mjs 生成,请勿手改(MovieClaw ${REF} ${COMMIT.slice(0, 8)},MIT)
${ids
.sort()
.map((id) => `import ${ident(id)} from "./${id}.json" with { type: "json" };`)
.join("\n")}
/** MovieClaw 转换来的站点定义原始 JSON(未校验;经 loadSiteDefinition 校验后使用) */
export const MOVIECLAW_DEFINITION_JSON: Record<string, unknown> = {
${ids
.sort()
.map((id) => ` ${JSON.stringify(id)}: ${ident(id)},`)
.join("\n")}
};
`;
const manifestPath = join(OUT_DIR, "manifest.generated.ts");
const manifestText = biomeFormat(manifest, "packages/adapters/src/sites/definitions/manifest.generated.ts");
if ((existsSync(manifestPath) ? readFileSync(manifestPath, "utf8") : null) !== manifestText) {
diffs++;
if (CHECK) console.log("差异:manifest.generated.ts");
else writeFileSync(manifestPath, manifestText);
}
const stale = readdirSync(OUT_DIR).filter((f) => f.endsWith(".json") && !ids.includes(f.replace(/\.json$/, "")));
if (stale.length) console.log(`注意:定义目录中存在非 MovieClaw 来源的 JSON(保留不动):${stale.join(", ")}`);
console.log(
`${CHECK ? "检查" : "转换"}完成:${ids.length} 个站点,${diffs} 个文件${CHECK ? "有差异" : "已更新"}(${REF} ${COMMIT.slice(0, 8)})`,
);
if (CHECK && diffs) process.exit(1);
@@ -0,0 +1,204 @@
{
"$comment": "W1-3b 手写观众定义(按 MP indexer schema;仅对 testkit mock 验证)。深合并到 MovieClaw v0.27.0 转换结果之上,本文件的值优先,MovieClaw 值只补空。 umt.detail.fields=null:保留 W1-3b rowhead 详情模型(已对 mock 验证),不采用 MovieClaw 默认逐字段选择器。",
"id": "audiences",
"name": "观众",
"domain": "https://audiences.me/",
"encoding": "UTF-8",
"public": false,
"proxy": false,
"language": "zh",
"result_num": 100,
"timeout": 30,
"search": {
"paths": [
{
"path": "torrents.php",
"type": "all"
}
],
"params": {
"search": "{keyword}",
"search_area": 4
}
},
"batch": {
"delimiter": " ",
"space_replace": "_"
},
"browse": {
"path": "torrents.php?page={page}",
"start": 0
},
"category": {
"movie": [
{
"id": "401",
"cat": "Movies",
"desc": "电影"
}
],
"tv": [
{
"id": "402",
"cat": "TV Series",
"desc": "剧集"
},
{
"id": "403",
"cat": "TV Shows",
"desc": "综艺"
}
],
"music": []
},
"torrents": {
"list": {
"selector": "table.torrents > tbody > tr:has(table.torrentname), table.torrents > tr:has(table.torrentname)"
},
"fields": {
"title_default": {
"selector": "a[href*=\"details.php?id=\"]"
},
"title_optional": {
"selector": "a[title][href*=\"details.php?id=\"]",
"attribute": "title"
},
"title": {
"text": "{% if fields['title_optional'] %}{{ fields['title_optional'] }}{% else %}{{ fields['title_default'] }}{% endif %}"
},
"details": {
"selector": "a[href*=\"details.php?id=\"]",
"attribute": "href"
},
"download": {
"selector": "a[href*=\"download.php?id=\"]",
"attribute": "href"
},
"imdbid": {
"selector": "a[href*=\"imdb.com/title/tt\"]",
"attribute": "href",
"filters": [
{
"name": "re_search",
"args": ["tt\\d+", 0]
}
]
},
"date_elapsed": {
"selector": "td:nth-child(4) > span"
},
"date_added": {
"selector": "td:nth-child(4) > span",
"attribute": "title"
},
"date": {
"text": "{% if fields['date_elapsed'] or fields['date_added'] %}{{ fields['date_added'] if fields['date_added'] else fields['date_elapsed'] }}{% else %}now{% endif %}",
"filters": [
{
"name": "dateparse",
"args": "%Y-%m-%d %H:%M:%S"
}
]
},
"size": {
"selector": "td:nth-child(5)"
},
"seeders": {
"selector": "td:nth-child(6)"
},
"leechers": {
"selector": "td:nth-child(7)"
},
"grabs": {
"selector": "td:nth-child(8)"
},
"downloadvolumefactor": {
"case": {
"img.pro_free": 0,
"img.pro_free2up": 0,
"img.pro_50pctdown": 0.5,
"img.pro_50pctdown2up": 0.5,
"img.pro_30pctdown": 0.3,
"*": 1
}
},
"uploadvolumefactor": {
"case": {
"img.pro_50pctdown2up": 2,
"img.pro_free2up": 2,
"img.pro_2up": 2,
"*": 1
}
},
"freedate": {
"selector": "img[class^=\"pro_\"]",
"attribute": "title",
"filters": [
{
"name": "re_search",
"args": ["\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}", 0]
}
]
},
"description": {
"selector": "span.torrent-subtitle-line"
},
"labels": {
"selector": "span.tags"
},
"category": {
"selector": "a[href*=\"?cat=\"]",
"attribute": "href",
"filters": [
{
"name": "querystring",
"args": "cat"
}
]
}
}
},
"umt": {
"version": "[email protected]",
"kind": "nexusphp",
"displayName": "观众",
"timezone": "+08:00",
"rateLimit": {
"minIntervalMs": 3000,
"burst": 1
},
"capabilities": {
"list": true,
"search": true,
"detail": true,
"download": true,
"rss": true
},
"categoryNames": {
"401": "电影",
"402": "剧集",
"403": "综艺",
"406": "纪录片"
},
"loginSuccess": "a[href*=\"logout\"]",
"emptyMarkers": ["没有种子", "没有结果", "Nothing found"],
"pagination": "a[href*=\"page=\"]",
"detailPath": "details.php",
"downloadPath": "download.php",
"detail": {
"path": "details.php",
"title": "h1#top",
"rowLabel": "td.rowhead",
"rowValue": "td.rowfollow",
"labels": {
"subtitle": ["副标题", "Small Description"],
"basicInfo": ["基本信息", "Basic Info"],
"peers": ["同伴", "Peers"]
},
"description": "#kdescr",
"headerScope": "h1#top",
"notFoundMarkers": ["没有该ID的种子", "No torrent with ID"],
"fields": null
}
}
}
@@ -0,0 +1,16 @@
{
"$comment": "CHDBits 分类名取自 MovieClaw chdbits.yaml 注释(TV Series / TV Shows / Sports / Music Videos / HQ Audio)。",
"umt": {
"categoryNames": {
"401": "电影",
"402": "剧集",
"403": "综艺",
"404": "纪录片",
"405": "动漫",
"406": "MV",
"407": "体育",
"408": "HQ Audio",
"409": "其他"
}
}
}
@@ -0,0 +1,17 @@
{
"$comment": "HHClub 分类名取自 MovieClaw hhanclub.yaml 注释(导航栏分类表);REQ §4.5 要求综艺/体育/短剧保留原始分类。",
"umt": {
"categoryNames": {
"401": "电影",
"402": "电视剧",
"403": "综艺",
"404": "纪录片",
"405": "动漫",
"407": "体育",
"409": "其他",
"410": "音乐",
"411": "MV",
"412": "短剧"
}
}
}
@@ -0,0 +1,26 @@
{
"$comment": "W1-3b 馒头定义(mteam/definition.ts 原值,按 MP mtorrent.py)。本文件优先,MovieClaw 值只补空;categoryNames 为本项目编制的粗分类名。",
"name": "馒头",
"timeout": 15,
"umt": {
"version": "[email protected]",
"displayName": "馒头",
"categoryNames": {
"401": "电影",
"419": "电影",
"420": "电影",
"421": "电影",
"439": "电影",
"402": "剧集",
"403": "综艺",
"435": "剧集",
"438": "剧集",
"404": "纪录",
"405": "动画",
"407": "体育",
"406": "音乐",
"408": "音乐",
"434": "音乐"
}
}
}