feat(scrape): 任意目录就地刮削,按路径识别作品,每次刮削一条记录(SCR-2)
- POST /files/scrape 改为对目录 / 视频文件刮削:已登记用作品,否则按路径识别(本地 + TMDB), 认不准返回候选,选定后再提交;未登记直接用 TMDB 字段,addToLibrary 顺手建作品 - 作品详情重新刮削、入库后自动刮削改为对作品目录刮削,同一实现 - 迁移 0058_scrape_record 替代 media_scrape;GET /files/scrape-records - 与 MP ScrapingChain 的文件布局差分(7 个场景,3 处登记偏离) Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
f60254e265
commit
87bf33b42b
29 files changed
+2272
-653
No files matched your search
@@ -0,0 +1,144 @@
|
||||
"""SCR-2 刮削差分(文件布局):用 MP 上游 ScrapingChain.scrape_metadata 的真实决策代码,在临时目录里对同一组目录结构
|
||||
"就地刮削",记录 MP 写了哪些文件,写 scrape-layout-mp.json。与 apps/server/test/file-manager/scrape-layout.test.ts 对照。
|
||||
|
||||
用法(MP 仓库根目录,用其 .venv):
|
||||
cd /opt/moviepilot-plus/upstream/MoviePilot && PYTHONPATH=. \
|
||||
.venv/bin/python /opt/uhd_media_tool/apps/server/test/scrape/gen-scrape-layout-mp.py
|
||||
|
||||
只记录行为输出,不复制 MP 源码。替身:
|
||||
- 存储:本地目录的最小实现(list_files / get_parent_item / get_file_item / upload_file / contains_bluray_subdirectories);
|
||||
- NFO 内容:固定占位(内容差分见 gen-scrape-mp.py);
|
||||
- 图片清单:按 MP themoviedb 模块 get_metadata_img 的规则给(作品 poster + backdrop;季 seasonNN-poster;集 episode-thumb);
|
||||
- 识别:recognize_media 原样返回传入的作品(路径识别不在此差分范围)。
|
||||
刮削策略用 MP 缺省(ScrapingConfig 默认全部 missingonly);手动刮削 overwrite=True(同 MP /media/scrape 接口)。
|
||||
"""
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import shutil
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
|
||||
import app.chain.scraping as scraping_mod
|
||||
from app.chain.scraping import ScrapingChain, ScrapingConfig
|
||||
from app.domain.context import MediaInfo
|
||||
from app.schemas.types import MediaSource, MediaType
|
||||
from app.schemas.workflow import FileItem
|
||||
|
||||
HERE = pathlib.Path(__file__).parent
|
||||
VIDEO_EXT = (".mkv", ".mp4", ".ts", ".m2ts", ".avi", ".iso")
|
||||
|
||||
|
||||
def item(p: pathlib.Path) -> FileItem:
|
||||
return FileItem(
|
||||
storage="local",
|
||||
type="dir" if p.is_dir() else "file",
|
||||
path=str(p) + ("/" if p.is_dir() else ""),
|
||||
name=p.name,
|
||||
basename=p.stem if p.is_file() else p.name,
|
||||
extension=p.suffix[1:] if p.is_file() else None,
|
||||
)
|
||||
|
||||
|
||||
class LocalStorage:
|
||||
def list_files(self, fileitem, recursion=False):
|
||||
return [item(c) for c in sorted(pathlib.Path(fileitem.path).iterdir())]
|
||||
|
||||
def get_parent_item(self, fileitem):
|
||||
return item(pathlib.Path(fileitem.path.rstrip("/")).parent)
|
||||
|
||||
def get_file_item(self, storage, path):
|
||||
p = pathlib.Path(path)
|
||||
return item(p) if p.exists() else None
|
||||
|
||||
def upload_file(self, fileitem, path, new_name=None):
|
||||
target = pathlib.Path(fileitem.path) / (new_name or path.name)
|
||||
shutil.copyfile(path, target)
|
||||
return item(target)
|
||||
|
||||
def contains_bluray_subdirectories(self, files):
|
||||
return any(f.type == "dir" and f.name.upper() in ("BDMV", "VIDEO_TS") for f in files)
|
||||
|
||||
|
||||
def make_chain() -> ScrapingChain:
|
||||
c = object.__new__(ScrapingChain)
|
||||
c.storagechain = LocalStorage()
|
||||
c.scraping_policies = ScrapingConfig()
|
||||
c.runtime_config = SimpleNamespace(
|
||||
video_extensions=VIDEO_EXT, audio_extensions=(".flac", ".mp3"), season_zero_names=("Specials", "SPs")
|
||||
)
|
||||
c.metadata_nfo = lambda **kw: "<nfo/>"
|
||||
|
||||
def metadata_img(mediainfo, season=None, episode=None):
|
||||
if season is not None:
|
||||
if episode:
|
||||
return {"episode-thumb.jpg": "https://img/still.jpg"}
|
||||
return {f"season{str(season).rjust(2, '0')}-poster.jpg": "https://img/season.jpg"}
|
||||
return {"poster.jpg": "https://img/poster.jpg", "backdrop.jpg": "https://img/backdrop.jpg"}
|
||||
|
||||
c.metadata_img = metadata_img
|
||||
|
||||
def save_image(fileitem, path, url):
|
||||
pathlib.Path(path).write_bytes(b"jpg")
|
||||
|
||||
c._download_and_save_image = save_image
|
||||
return c
|
||||
|
||||
|
||||
class FakeMediaChain:
|
||||
def recognize_media(self, meta=None, **kw):
|
||||
return CURRENT["media"]
|
||||
|
||||
def recognize_by_meta(self, meta):
|
||||
return CURRENT["media"]
|
||||
|
||||
|
||||
CURRENT: dict = {}
|
||||
scraping_mod.MediaChain = FakeMediaChain # 集文件重新识别季集后仍是同一作品
|
||||
|
||||
|
||||
def media(kind: MediaType, tmdb_id: int, title: str) -> MediaInfo:
|
||||
mi = MediaInfo()
|
||||
mi.type = kind
|
||||
mi.tmdb_id = tmdb_id
|
||||
mi.media_source = MediaSource.TMDB
|
||||
mi.media_id = str(tmdb_id)
|
||||
mi.title = title
|
||||
return mi
|
||||
|
||||
|
||||
SHOW = "人生切割术 (2022)"
|
||||
EP1 = f"{SHOW}/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.mkv"
|
||||
EP2 = f"{SHOW}/Season 1/人生切割术 - S01E02 - 半循环.mkv"
|
||||
MOVIE = "星际穿越 (2014)"
|
||||
MOVIE_FILE = f"{MOVIE}/星际穿越 (2014) - 1080p.mkv"
|
||||
LOOSE = "Interstellar.2014.1080p.BluRay.mkv"
|
||||
TREE = [EP1, EP2, MOVIE_FILE, LOOSE, "Heat (1995)/BDMV/index.bdmv"]
|
||||
|
||||
SCENARIOS = {
|
||||
"show_dir": (SHOW, MediaType.TV),
|
||||
"season_dir": (f"{SHOW}/Season 1", MediaType.TV),
|
||||
"episode_file": (EP2, MediaType.TV),
|
||||
"movie_dir": (MOVIE, MediaType.MOVIE),
|
||||
"movie_file": (MOVIE_FILE, MediaType.MOVIE),
|
||||
"movie_file_in_root": (LOOSE, MediaType.MOVIE),
|
||||
"bluray_dir": ("Heat (1995)", MediaType.MOVIE),
|
||||
}
|
||||
|
||||
out = {}
|
||||
for name, (target, kind) in SCENARIOS.items():
|
||||
root = pathlib.Path(tempfile.mkdtemp(prefix="mp-scr-"))
|
||||
for f in TREE:
|
||||
(root / f).parent.mkdir(parents=True, exist_ok=True)
|
||||
(root / f).write_bytes(b"video")
|
||||
before = {str(p.relative_to(root)) for p in root.rglob("*") if p.is_file()}
|
||||
CURRENT["media"] = media(kind, 95396 if kind == MediaType.TV else 157336, "x")
|
||||
chain = make_chain()
|
||||
ok, msg = chain.scrape_metadata(fileitem=item(root / target), mediainfo=CURRENT["media"], overwrite=True)
|
||||
after = {str(p.relative_to(root)) for p in root.rglob("*") if p.is_file()}
|
||||
out[name] = {"target": target, "ok": ok, "written": sorted(after - before)}
|
||||
shutil.rmtree(root)
|
||||
|
||||
(HERE / "scrape-layout-mp.json").write_text(json.dumps(out, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
# 生成后在本仓库跑一次格式化(lint 要求):deno run -A npm:@biomejs/biome check --write apps/server/test/scrape/scrape-layout-mp.json
|
||||
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"show_dir": {
|
||||
"target": "人生切割术 (2022)",
|
||||
"ok": true,
|
||||
"written": [
|
||||
"人生切割术 (2022)/Season 1/poster.jpg",
|
||||
"人生切割术 (2022)/Season 1/season.nfo",
|
||||
"人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.jpg",
|
||||
"人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.nfo",
|
||||
"人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.jpg",
|
||||
"人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.nfo",
|
||||
"人生切割术 (2022)/backdrop.jpg",
|
||||
"人生切割术 (2022)/fanart.jpg",
|
||||
"人生切割术 (2022)/poster.jpg",
|
||||
"人生切割术 (2022)/season01-poster.jpg",
|
||||
"人生切割术 (2022)/tvshow.nfo"
|
||||
]
|
||||
},
|
||||
"season_dir": {
|
||||
"target": "人生切割术 (2022)/Season 1",
|
||||
"ok": true,
|
||||
"written": [
|
||||
"人生切割术 (2022)/Season 1/poster.jpg",
|
||||
"人生切割术 (2022)/Season 1/season.nfo",
|
||||
"人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.jpg",
|
||||
"人生切割术 (2022)/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.nfo",
|
||||
"人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.jpg",
|
||||
"人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.nfo",
|
||||
"人生切割术 (2022)/season01-poster.jpg"
|
||||
]
|
||||
},
|
||||
"episode_file": {
|
||||
"target": "人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.mkv",
|
||||
"ok": true,
|
||||
"written": [
|
||||
"人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.jpg",
|
||||
"人生切割术 (2022)/Season 1/人生切割术 - S01E02 - 半循环.nfo"
|
||||
]
|
||||
},
|
||||
"movie_dir": {
|
||||
"target": "星际穿越 (2014)",
|
||||
"ok": true,
|
||||
"written": [
|
||||
"星际穿越 (2014)/backdrop.jpg",
|
||||
"星际穿越 (2014)/fanart.jpg",
|
||||
"星际穿越 (2014)/poster.jpg",
|
||||
"星际穿越 (2014)/星际穿越 (2014) - 1080p.nfo"
|
||||
]
|
||||
},
|
||||
"movie_file": {
|
||||
"target": "星际穿越 (2014)/星际穿越 (2014) - 1080p.mkv",
|
||||
"ok": true,
|
||||
"written": [
|
||||
"星际穿越 (2014)/backdrop.jpg",
|
||||
"星际穿越 (2014)/fanart.jpg",
|
||||
"星际穿越 (2014)/poster.jpg",
|
||||
"星际穿越 (2014)/星际穿越 (2014) - 1080p.nfo"
|
||||
]
|
||||
},
|
||||
"movie_file_in_root": {
|
||||
"target": "Interstellar.2014.1080p.BluRay.mkv",
|
||||
"ok": true,
|
||||
"written": ["Interstellar.2014.1080p.BluRay.nfo", "backdrop.jpg", "fanart.jpg", "poster.jpg"]
|
||||
},
|
||||
"bluray_dir": {
|
||||
"target": "Heat (1995)",
|
||||
"ok": true,
|
||||
"written": [
|
||||
"Heat (1995)/Heat (1995).nfo",
|
||||
"Heat (1995)/backdrop.jpg",
|
||||
"Heat (1995)/fanart.jpg",
|
||||
"Heat (1995)/poster.jpg"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, relative } from "node:path";
|
||||
import type { TmdbRichDetail, TmdbRichSeasonDetail } from "@umt/adapters";
|
||||
import type { ScrapeMatch } from "@umt/contracts";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { queueScrape, runScrapeRecord } from "../../src/modules/acquisition-organize/services/scrape.ts";
|
||||
import { createFmHarness, type FmHarness, pgAvailable } from "../file-manager/harness.ts";
|
||||
|
||||
// SCR-2 刮削文件布局差分:同一组目录结构,MP ScrapingChain.scrape_metadata(gen-scrape-layout-mp.py 生成
|
||||
// scrape-layout-mp.json)写哪些文件 ⇔ 我们就地刮削写哪些文件。偏离只允许下面 DEVIATIONS 里登记的几类
|
||||
// (理由见 docs/runbook/scrape.md「与 MP 的差分」)。
|
||||
|
||||
const skip = !(await pgAvailable());
|
||||
const MP = JSON.parse(await readFile(new URL("./scrape-layout-mp.json", import.meta.url), "utf8")) as Record<
|
||||
string,
|
||||
{ target: string; ok: boolean; written: string[] }
|
||||
>;
|
||||
|
||||
const SHOW = "人生切割术 (2022)";
|
||||
const EP1 = `${SHOW}/Season 1/人生切割术 - S01E01 - 有关地狱的好消息.mkv`;
|
||||
const EP2 = `${SHOW}/Season 1/人生切割术 - S01E02 - 半循环.mkv`;
|
||||
const MOVIE_FILE = "星际穿越 (2014)/星际穿越 (2014) - 1080p.mkv";
|
||||
const LOOSE = "Interstellar.2014.1080p.BluRay.mkv";
|
||||
const TREE = [EP1, EP2, MOVIE_FILE, LOOSE, "Heat (1995)/BDMV/index.bdmv"];
|
||||
|
||||
/** 登记的偏离:把 MP 的输出换算成我们应有的输出 */
|
||||
const DEVIATIONS: { id: string; apply: (files: string[], scenario: string) => string[] }[] = [
|
||||
// 1. 不写 backdrop.jpg 副本,只写 fanart.jpg(Emby / Jellyfin / Kodi 都认 fanart)
|
||||
{ id: "no_backdrop_copy", apply: (fs) => fs.filter((f) => !/(^|\/)backdrop\.jpg$/.test(f)) },
|
||||
// 2. 剧照 <视频名>-thumb.jpg(Kodi 约定;MP 写 <视频名>.jpg)
|
||||
{
|
||||
id: "episode_thumb_suffix",
|
||||
apply: (fs) => fs.map((f) => (/ - S\d+E\d+ - [^/]+\.jpg$/.test(f) ? f.replace(/\.jpg$/, "-thumb.jpg") : f)),
|
||||
},
|
||||
// 3. 视频直接在登记目录根:图片写 <视频名>-poster.jpg / -fanart.jpg,不往根里放 poster.jpg(MP 会)
|
||||
{
|
||||
id: "root_movie_images",
|
||||
apply: (fs, s) =>
|
||||
s !== "movie_file_in_root"
|
||||
? fs
|
||||
: fs.map((f) => (f === "poster.jpg" || f === "fanart.jpg" ? `${LOOSE.replace(/\.mkv$/, "")}-${f}` : f)),
|
||||
},
|
||||
];
|
||||
|
||||
const detail = (entityType: "movie" | "tv", externalId: string): TmdbRichDetail =>
|
||||
({
|
||||
entityType,
|
||||
externalId,
|
||||
title: "x",
|
||||
titleLanguage: "zh-CN",
|
||||
titleZh: "x",
|
||||
titleZhLanguage: "zh-CN",
|
||||
originalTitle: null,
|
||||
originalLanguage: "en",
|
||||
overview: null,
|
||||
overviewLanguage: null,
|
||||
tagline: null,
|
||||
year: 2022,
|
||||
releaseDate: null,
|
||||
lastAirDate: null,
|
||||
genres: [],
|
||||
voteAverage: null,
|
||||
voteCount: null,
|
||||
status: null,
|
||||
runtime: null,
|
||||
seasonCount: null,
|
||||
episodeCount: null,
|
||||
networks: [],
|
||||
originCountry: [],
|
||||
posterPath: "/poster.jpg",
|
||||
backdropPath: "/backdrop.jpg",
|
||||
cast: [],
|
||||
crew: [],
|
||||
externalIds: { imdb: null, tvdb: null, wikidata: null },
|
||||
translations: [],
|
||||
alternativeTitles: [],
|
||||
seasons: [],
|
||||
languages: ["zh-CN"],
|
||||
}) satisfies TmdbRichDetail;
|
||||
|
||||
const season1: TmdbRichSeasonDetail = {
|
||||
tvId: "95396",
|
||||
seasonNumber: 1,
|
||||
name: null,
|
||||
nameLanguage: null,
|
||||
overview: null,
|
||||
overviewLanguage: null,
|
||||
airDate: null,
|
||||
posterPath: "/season.jpg",
|
||||
fetchedLanguages: ["zh-CN"],
|
||||
episodes: [1, 2].map((n) => ({
|
||||
seasonNumber: 1,
|
||||
episodeNumber: n,
|
||||
name: null,
|
||||
nameLanguage: null,
|
||||
overview: null,
|
||||
overviewLanguage: null,
|
||||
airDate: null,
|
||||
runtime: null,
|
||||
stillPath: "/still.jpg",
|
||||
voteAverage: null,
|
||||
voteCount: null,
|
||||
})),
|
||||
};
|
||||
|
||||
async function files(root: string): Promise<Set<string>> {
|
||||
const out = new Set<string>();
|
||||
const walk = async (d: string) => {
|
||||
for (const n of await readdir(d).catch(() => [])) {
|
||||
if (n.startsWith(".")) continue;
|
||||
const p = join(d, n);
|
||||
if ((await stat(p)).isDirectory()) await walk(p);
|
||||
else out.add(relative(root, p));
|
||||
}
|
||||
};
|
||||
await walk(root);
|
||||
return out;
|
||||
}
|
||||
|
||||
describe.skipIf(skip)("刮削文件布局与 MP 差分", () => {
|
||||
let h: FmHarness;
|
||||
beforeAll(async () => {
|
||||
h = await createFmHarness(
|
||||
"scrlayout",
|
||||
{
|
||||
ao: {
|
||||
scrapeImage: async () => ({ content: new Uint8Array([0xff, 0xd8, 0xff]), mime: "image/jpeg" }),
|
||||
scrapeTmdbDetail: async (type, id) => detail(type, id),
|
||||
scrapeTmdbSeason: async (_tv, n) => (n === 1 ? season1 : null),
|
||||
},
|
||||
},
|
||||
{ runTransfers: false },
|
||||
);
|
||||
});
|
||||
afterAll(async () => h?.close());
|
||||
|
||||
for (const [scenario, mp] of Object.entries(MP)) {
|
||||
it(`${scenario}:${mp.target}`, async () => {
|
||||
// 每个场景一份独立目录树;"视频直接在根"的场景用下载根(整个根就是这棵树)
|
||||
const inRoot = scenario === "movie_file_in_root";
|
||||
const base = inRoot ? h.dl : join(h.lib, scenario);
|
||||
for (const f of TREE) {
|
||||
await mkdir(dirname(join(base, f)), { recursive: true });
|
||||
await writeFile(join(base, f), "video");
|
||||
}
|
||||
const before = await files(base);
|
||||
const tv = mp.target.startsWith(SHOW);
|
||||
const match: ScrapeMatch = {
|
||||
mediaEntityId: null,
|
||||
provider: "tmdb",
|
||||
externalId: tv ? "95396" : "157336",
|
||||
kind: tv ? "tv" : "movie",
|
||||
title: "x",
|
||||
year: null,
|
||||
source: "chosen",
|
||||
};
|
||||
const q = await queueScrape(h.module.deps().ao, {
|
||||
...(inRoot
|
||||
? { rootId: h.dlRoot, rootPath: h.dl, libraryRootId: null }
|
||||
: { rootId: h.libRoot, rootPath: h.lib, libraryRootId: h.libraryRootId }),
|
||||
path: inRoot ? mp.target : `${scenario}/${mp.target}`,
|
||||
match,
|
||||
mode: "overwrite",
|
||||
trigger: "manual",
|
||||
});
|
||||
const rec = await runScrapeRecord(h.module.deps().ao, q.recordId);
|
||||
expect(rec.status).toBe("succeeded");
|
||||
const after = await files(base);
|
||||
const ours = [...after].filter((f) => !before.has(f)).sort();
|
||||
const expected = DEVIATIONS.reduce((fs, d) => d.apply(fs, scenario), [...mp.written]).sort();
|
||||
expect(ours).toEqual(expected);
|
||||
});
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user