Merge branch 'scr2'
Co-Authored-By: Claude Fable 5.1 <[email protected]> # Conflicts: # CHANGELOG.md # apps/server/src/modules/file-manager/routes.ts
This commit is contained in:
commit
44f7c88be6
41 files changed
+2810
-826
No files matched your search
@@ -3,11 +3,11 @@ import { join, relative } from "node:path";
|
||||
import { createTempTree, type TempTree } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { executeRecord, planImport } from "../services/organize.ts";
|
||||
import { runScrape, SCRAPE_TASK } from "../services/scrape.ts";
|
||||
import { runScrapeRecord, SCRAPE_TASK, scrapeEntity } from "../services/scrape.ts";
|
||||
import { createHarness, type Harness, pgAvailable, seedBase, seedEntity } from "./harness.ts";
|
||||
|
||||
// SCR-1 刮削:入库成功后自动排 scrapeMedia → 写 NFO 与图片(临时库根);只补缺失 / 全部覆盖;图片失败不阻塞 NFO;
|
||||
// 只登记库根不写;作品详情「重新刮削」接口与刮削状态。
|
||||
// 刮削(SCR-1 / SCR-2):入库成功后自动对作品目录排 scrapeMedia → 写 NFO 与图片(临时库根);只补缺失 / 全部覆盖;
|
||||
// 图片失败不阻塞 NFO;只登记库根不写;作品详情「重新刮削」接口与刮削状态(最近一条刮削记录)。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const SIZE = 200_000;
|
||||
@@ -75,13 +75,26 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => {
|
||||
return plan.recordId!;
|
||||
}
|
||||
|
||||
/** 按引擎记录的入队项跑刮削(假引擎不执行) */
|
||||
/** 按引擎记录的入队项跑刮削(假引擎不执行):该作品最近一条刮削记录 */
|
||||
async function runQueued(entity: string) {
|
||||
const q = h.engine.enqueued.filter(
|
||||
(x) => x.task === SCRAPE_TASK && (x.input as { mediaEntityId: string }).mediaEntityId === entity,
|
||||
);
|
||||
expect(q.length).toBeGreaterThan(0);
|
||||
return runScrape(h.d, q[q.length - 1]!.input as Parameters<typeof runScrape>[1]);
|
||||
const recs = await h.db
|
||||
.selectFrom("scrape_record")
|
||||
.select(["id"])
|
||||
.where("media_entity_id", "=", entity)
|
||||
.where("status", "=", "queued")
|
||||
.orderBy("requested_at", "desc")
|
||||
.execute();
|
||||
expect(recs.length).toBeGreaterThan(0);
|
||||
const q = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK);
|
||||
expect(q.some((x) => (x.input as { recordId: string }).recordId === recs[0]!.id)).toBe(true);
|
||||
return runScrapeRecord(h.d, recs[0]!.id);
|
||||
}
|
||||
|
||||
/** 作品详情「重新刮削」的服务端做法:对作品目录排刮削并执行 */
|
||||
async function rescrape(entity: string, mode?: "missing" | "overwrite") {
|
||||
const q = await scrapeEntity(h.d, entity, { trigger: "manual", mode });
|
||||
expect(q.length).toBe(1);
|
||||
return runScrapeRecord(h.d, q[0]!.recordId);
|
||||
}
|
||||
|
||||
let series = "";
|
||||
@@ -165,8 +178,18 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => {
|
||||
const rec = await organize(series, "Severance.S01E01.1080p.WEB-DL.H264-GRP.mkv");
|
||||
await organize(series, "Severance.S01E02.1080p.WEB-DL.H264-GRP.mkv");
|
||||
const q = h.engine.enqueued.filter((x) => x.task === SCRAPE_TASK);
|
||||
expect(q[0]!.key).toBe(`${SCRAPE_TASK}:record:${rec}`);
|
||||
expect(q[0]!.input).toMatchObject({ mediaEntityId: series, trigger: "auto", mode: "missing", recordId: rec });
|
||||
expect(q[0]!.key).toBe(`${SCRAPE_TASK}:record:${rec}:0`);
|
||||
const first = await h.db
|
||||
.selectFrom("scrape_record")
|
||||
.selectAll()
|
||||
.where("id", "=", (q[0]!.input as { recordId: string }).recordId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(first).toMatchObject({
|
||||
media_entity_id: series,
|
||||
trigger: "auto",
|
||||
mode: "missing",
|
||||
organization_record_id: rec,
|
||||
});
|
||||
|
||||
const r = await runQueued(series);
|
||||
expect(r.status).toBe("partial"); // 背景图下载失败
|
||||
@@ -213,23 +236,23 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => {
|
||||
});
|
||||
const audit = await h.db
|
||||
.selectFrom("audit_event")
|
||||
.select(["action", "target_id"])
|
||||
.select(["action", "target_id", "target_type"])
|
||||
.where("action", "=", "organize.scrape")
|
||||
.execute();
|
||||
expect(audit.some((a) => a.target_id === series)).toBe(true);
|
||||
expect(audit.some((a) => a.target_type === "scrape_record" && a.target_id === st.body.data.scrape.id)).toBe(true);
|
||||
}, 60_000);
|
||||
|
||||
it("只补缺失:已有文件(包括别人写的)不动;全部覆盖:改写", async () => {
|
||||
const files = await tree(lib.root);
|
||||
const show = join(lib.root, files.find((p) => p.endsWith("tvshow.nfo"))!);
|
||||
await writeFile(show, "<tvshow>别的工具写的</tvshow>");
|
||||
const again = await runScrape(h.d, { mediaEntityId: series, trigger: "manual", mode: "missing" });
|
||||
const again = await rescrape(series, "missing");
|
||||
expect(await readFile(show, "utf8")).toBe("<tvshow>别的工具写的</tvshow>");
|
||||
expect(again.files.find((x) => x.role === "tvshow_nfo")).toMatchObject({
|
||||
action: "skipped",
|
||||
reason: "scrape_exists",
|
||||
});
|
||||
const over = await runScrape(h.d, { mediaEntityId: series, trigger: "manual", mode: "overwrite" });
|
||||
const over = await rescrape(series, "overwrite");
|
||||
expect(over.written).toBeGreaterThan(0);
|
||||
expect(await readFile(show, "utf8")).toContain("<title>人生切割术</title>");
|
||||
}, 60_000);
|
||||
@@ -276,24 +299,23 @@ describe.skipIf(!hasPg)("刮削:NFO 与图片写进媒体库目录", () => {
|
||||
expect(nfo).toContain("<runtime>169</runtime>");
|
||||
}, 60_000);
|
||||
|
||||
it("写入失败记原因;只登记库根不写文件;没有入库文件的作品 409", async () => {
|
||||
it("写入失败记原因;只登记库根不写文件;没有入库文件的作品 412", async () => {
|
||||
const files = await tree(lib.root);
|
||||
const video = files.find((p) => p.includes("星际穿越") && p.endsWith(".mkv"))!;
|
||||
const dirAbs = join(lib.root, video.slice(0, video.lastIndexOf("/")));
|
||||
if (process.getuid?.() !== 0) {
|
||||
await chmod(dirAbs, 0o555);
|
||||
const r = await runScrape(h.d, { mediaEntityId: movie, trigger: "manual", mode: "overwrite" });
|
||||
const r = await rescrape(movie, "overwrite");
|
||||
await chmod(dirAbs, 0o755);
|
||||
expect(r.files.some((x) => x.reason === "scrape_write_failed")).toBe(true);
|
||||
}
|
||||
await h.db.updateTable("library_root").set({ management_mode: "register_only" }).execute();
|
||||
const ro = await runScrape(h.d, { mediaEntityId: movie, trigger: "manual", mode: "overwrite" });
|
||||
const ro = await rescrape(movie, "overwrite");
|
||||
expect(ro).toMatchObject({ status: "failed", reason: "scrape_root_read_only", written: 0 });
|
||||
await h.db.updateTable("library_root").set({ management_mode: "managed" }).execute();
|
||||
|
||||
const empty = await seedEntity(h.db, { kind: "movie", title: "没入库", year: 2020 });
|
||||
expect((await h.api("POST", `/library/${empty}/scrape`, {})).status).toBe(412);
|
||||
const none = await runScrape(h.d, { mediaEntityId: empty, trigger: "manual" });
|
||||
expect(none).toMatchObject({ status: "failed", reason: "scrape_no_metadata" });
|
||||
expect(await scrapeEntity(h.d, empty, { trigger: "manual" })).toEqual([]);
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -5,7 +5,7 @@ import { type AoDeps, automationBlock, ensureTaskRow, markTask } from "./service
|
||||
import { cleanupDownloadCache, submitDownload } from "./services/downloads.ts";
|
||||
import { executeRecord, planImport } from "./services/organize.ts";
|
||||
import { reconcileLibrary } from "./services/reconcile.ts";
|
||||
import { runScrape, SCRAPE_TASK } from "./services/scrape.ts";
|
||||
import { runScrapeRecord, SCRAPE_TASK } from "./services/scrape.ts";
|
||||
import { errMessage } from "./util.ts";
|
||||
|
||||
// 持久任务处理器。输入只放业务 id;业务状态写业务表;副作用经 withAttemptToken(在 services 里)。
|
||||
@@ -188,22 +188,16 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
|
||||
);
|
||||
},
|
||||
}),
|
||||
// SCR-1 刮削:写 NFO 与图片到媒体库目录(文件道);结果写 media_scrape,不重试(失败的文件在作品详情里重新刮削)
|
||||
// 刮削(SCR-1 / SCR-2):按刮削记录在目录里就地写 NFO 与图片(文件道);结果写回 scrape_record,不重试(失败的重新刮削)
|
||||
defineTask({
|
||||
name: SCRAPE_TASK,
|
||||
version: 1,
|
||||
version: 2,
|
||||
lane: "file",
|
||||
inputSchema: z.object({
|
||||
mediaEntityId: id,
|
||||
trigger: z.enum(["auto", "manual"]),
|
||||
mode: z.enum(["missing", "overwrite"]).optional(),
|
||||
recordId: id.optional(),
|
||||
actor: z.object({ type: z.string(), id: z.string().nullable() }).optional(),
|
||||
}),
|
||||
inputSchema: z.object({ recordId: id }),
|
||||
retryPolicy: { maximumAttempts: 1 },
|
||||
handler: async (t) =>
|
||||
t.step.run("scrape", async () => {
|
||||
const r = await runScrape(d, { ...t.input, runId: t.runId }, t.abortSignal);
|
||||
const r = await runScrapeRecord(d, t.input.recordId, t.abortSignal);
|
||||
return { status: r.status, written: r.written, skipped: r.skipped, failed: r.failed, reason: r.reason };
|
||||
}),
|
||||
}),
|
||||
|
||||
@@ -6,6 +6,8 @@ import {
|
||||
type DownloaderClient,
|
||||
diskUsage,
|
||||
type EmbyClient,
|
||||
fetchRichDetail,
|
||||
fetchRichSeason,
|
||||
ffprobe,
|
||||
getSiteAdapter,
|
||||
getSiteDefinition,
|
||||
@@ -13,6 +15,8 @@ import {
|
||||
qbAuthConfig,
|
||||
type SecretResolver,
|
||||
SiteError,
|
||||
type TmdbRichDetail,
|
||||
type TmdbRichSeasonDetail,
|
||||
type TorrentSource,
|
||||
type VerifyMethod,
|
||||
} from "@umt/adapters";
|
||||
@@ -88,6 +92,12 @@ export interface AoPorts {
|
||||
* 失败抛错(由刮削记为该图片失败,不影响 NFO)。
|
||||
*/
|
||||
scrapeImage(tmdbPath: string): Promise<{ content: Uint8Array; mime: string }>;
|
||||
/**
|
||||
* SCR-2 刮削未登记的目录:直接取 TMDB 富详情 / 季详情(按设置的元数据语言链,经 TMDB 缓存)。
|
||||
* TMDB 没配置返回 null;条目不存在或上游失败抛错。
|
||||
*/
|
||||
scrapeTmdbDetail(type: "movie" | "tv", id: string): Promise<TmdbRichDetail | null>;
|
||||
scrapeTmdbSeason(tvId: string, season: number): Promise<TmdbRichSeasonDetail | null>;
|
||||
options: AoOptions;
|
||||
}
|
||||
|
||||
@@ -197,6 +207,16 @@ export function createPorts(
|
||||
});
|
||||
return images.bytes("original", tmdbPath);
|
||||
},
|
||||
async scrapeTmdbDetail(type, id) {
|
||||
const client = await rt().tmdb();
|
||||
if (!client) return null;
|
||||
return fetchRichDetail(client, type, id, await metadataLanguages(ctx), await metadataTextOptions(ctx));
|
||||
},
|
||||
async scrapeTmdbSeason(tvId, season) {
|
||||
const client = await rt().tmdb();
|
||||
if (!client) return null;
|
||||
return fetchRichSeason(client, tvId, season, await metadataLanguages(ctx), await metadataTextOptions(ctx));
|
||||
},
|
||||
diskUsage: (p) => diskUsage(p),
|
||||
probe: (p) => ffprobe(p),
|
||||
async mediaServer(inst) {
|
||||
|
||||
@@ -27,7 +27,7 @@ import { sql } from "@umt/db";
|
||||
import { ApiException } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
import { type AoDeps, CANDIDATE_ARTIFACT_TITLE } from "../services/common.ts";
|
||||
import { enqueueScrape, loadScrapeState } from "../services/scrape.ts";
|
||||
import { latestEntityScrape, scrapeEntity } from "../services/scrape.ts";
|
||||
import { notFound, toApi, toApiList } from "../util.ts";
|
||||
import { mount, paginate } from "./helpers.ts";
|
||||
import { registerLibraryVersionRoutes } from "./library-versions.ts";
|
||||
@@ -407,21 +407,21 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) {
|
||||
createdAt: new Date(r.created_at).toISOString(),
|
||||
finishedAt: r.finished_at ? new Date(r.finished_at).toISOString() : null,
|
||||
})),
|
||||
scrape: await loadScrapeState(d, id),
|
||||
scrape: await latestEntityScrape(d, id),
|
||||
};
|
||||
return detail;
|
||||
},
|
||||
{ tag: TAG },
|
||||
);
|
||||
|
||||
// SCR-1 刮削:作品详情「重新刮削」;与入库后自动刮削、文件管理「刮削」同一任务 scrapeMedia
|
||||
// 作品详情「重新刮削」「覆盖重写」:对作品的目录刮削(SCR-2:与文件管理「刮削」、入库后自动刮削同一实现)
|
||||
mount(
|
||||
app,
|
||||
{
|
||||
method: "post",
|
||||
path: "/library/{id}/scrape",
|
||||
scope: "organize.execute",
|
||||
summary: "刮削:把作品的 NFO 与图片写进媒体库目录(后台任务 scrapeMedia;覆盖方式缺省按设置)",
|
||||
summary: "刮削:对作品目录就地写 NFO 与图片(后台任务 scrapeMedia;覆盖方式缺省按设置)",
|
||||
params: IdParam,
|
||||
body: LibraryScrapeRequest,
|
||||
response: LibraryScrapeResult,
|
||||
@@ -429,22 +429,20 @@ export function registerLibraryRoutes(app: ModuleApp, d: AoDeps) {
|
||||
async (c, { params, body }) => {
|
||||
const e = await db().selectFrom("media_entity").select(["id"]).where("id", "=", params.id).executeTakeFirst();
|
||||
if (!e) notFound("作品");
|
||||
const has = await db()
|
||||
.selectFrom("media_asset")
|
||||
.select(["id"])
|
||||
.where("media_entity_id", "=", params.id)
|
||||
.where("status", "=", "active")
|
||||
.executeTakeFirst();
|
||||
if (!has) throw new ApiException("precondition_failed", "作品没有已入库文件。先整理入库,再刮削");
|
||||
const actor = c.get("actor");
|
||||
const req = (body ?? {}) as LibraryScrapeRequest;
|
||||
const r = await enqueueScrape(d, {
|
||||
mediaEntityId: params.id,
|
||||
const recs = await scrapeEntity(d, params.id, {
|
||||
trigger: "manual",
|
||||
mode: req.mode,
|
||||
actor: { type: actor?.type === "api_key" ? "api_key" : "user", id: actor?.id ?? null },
|
||||
});
|
||||
return { mediaEntityId: params.id, ...r };
|
||||
if (!recs.length) throw new ApiException("precondition_failed", "作品没有已入库文件。到文件管理里对目录刮削");
|
||||
return {
|
||||
mediaEntityId: params.id,
|
||||
taskId: recs[0]?.taskId ?? null,
|
||||
deduplicated: recs.every((r) => r.deduplicated),
|
||||
recordIds: recs.map((r) => r.recordId),
|
||||
};
|
||||
},
|
||||
{ tag: TAG, status: 202 },
|
||||
);
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import type { TmdbRichDetail, TmdbRichSeasonDetail } from "@umt/adapters";
|
||||
import type { NfoEpisode, NfoIds, NfoPerson, NfoSeason, NfoWork } from "@umt/domain";
|
||||
import { richEntityFields } from "../../sources-identity/service/metadata-rich.ts";
|
||||
|
||||
// SCR-1:本地作品 / 季 / 集的最终字段(media_entity.metadata、content_unit.metadata,字段级 {value, …})→ NFO 输入。
|
||||
// 纯函数,不查库;差分测试(apps/server/test/scrape/nfo-mp-diff.test.ts)直接调用。
|
||||
@@ -134,3 +136,58 @@ export function imagePathsOf(entityMeta: unknown) {
|
||||
}
|
||||
export const unitPosterPath = (m: unknown) => str(val(m, "posterPath"));
|
||||
export const unitStillPath = (m: unknown) => str(val(m, "stillPath"));
|
||||
|
||||
// ── SCR-2:未登记目录直接用 TMDB 字段(与刷新元数据同一套映射,形状同 media_entity.metadata / content_unit.metadata) ──
|
||||
|
||||
/** TMDB 富详情 → 作品(NFO 输入);字段映射同刷新元数据(richEntityFields) */
|
||||
export function tmdbEntity(r: TmdbRichDetail): { entity: EntityForNfo; ids: IdentityMap } {
|
||||
return {
|
||||
entity: {
|
||||
kind: r.entityType === "movie" ? "movie" : "series",
|
||||
title: r.title,
|
||||
original_title: r.originalTitle,
|
||||
year: r.year,
|
||||
metadata: richEntityFields(r),
|
||||
},
|
||||
ids: {
|
||||
tmdb: r.externalId,
|
||||
...(r.externalIds.imdb ? { imdb: r.externalIds.imdb } : {}),
|
||||
...(r.externalIds.tvdb ? { tvdb: r.externalIds.tvdb } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const fv = (value: unknown) => ({ value });
|
||||
|
||||
/** TMDB 季详情 → 季与各集(NFO 输入) */
|
||||
export function tmdbSeasonUnits(sd: TmdbRichSeasonDetail): { season: UnitForNfo; episodes: Map<number, UnitForNfo> } {
|
||||
const season: UnitForNfo = {
|
||||
season_number: sd.seasonNumber,
|
||||
episode_number: null,
|
||||
title: sd.name,
|
||||
air_date: sd.airDate,
|
||||
metadata: {
|
||||
name: fv(sd.name),
|
||||
overview: fv(sd.overview),
|
||||
airDate: fv(sd.airDate),
|
||||
posterPath: fv(sd.posterPath),
|
||||
},
|
||||
};
|
||||
const episodes = new Map<number, UnitForNfo>();
|
||||
for (const e of sd.episodes)
|
||||
episodes.set(e.episodeNumber, {
|
||||
season_number: e.seasonNumber,
|
||||
episode_number: e.episodeNumber,
|
||||
title: e.name,
|
||||
air_date: e.airDate,
|
||||
metadata: {
|
||||
name: fv(e.name),
|
||||
overview: fv(e.overview),
|
||||
airDate: fv(e.airDate?.slice(0, 10) ?? null),
|
||||
runtime: fv(e.runtime),
|
||||
stillPath: fv(e.stillPath),
|
||||
voteAverage: fv(e.voteAverage),
|
||||
},
|
||||
});
|
||||
return { season, episodes };
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -11,7 +11,14 @@ import {
|
||||
type OrganizationStatus,
|
||||
} from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { type IdentityCandidate, type ParsedRelease, parseReleasePath, queryTitles, rankCandidates } from "@umt/domain";
|
||||
import {
|
||||
type IdentityCandidate,
|
||||
type ParsedRelease,
|
||||
parseReleasePath,
|
||||
queryTitles,
|
||||
rankCandidates,
|
||||
resolveIdentity,
|
||||
} from "@umt/domain";
|
||||
import { localIdentityCandidate } from "@umt/mapping";
|
||||
import type { Principal } from "../../http/auth.ts";
|
||||
import { EXTERNAL_INVOKE_TIMEOUT_MS, sharedInvokeLimiter } from "../agent/invoke-limits.ts";
|
||||
@@ -177,13 +184,24 @@ export async function runIdentifyRemote(
|
||||
return payload;
|
||||
}
|
||||
|
||||
async function identifyCore(
|
||||
/** 候选键:本地作品 id 或 `provider:externalId`(与 FileIdentifyCandidate 对应) */
|
||||
export const candidateKey = (c: { mediaEntityId: string | null; provider: string; externalId: string | null }) =>
|
||||
c.mediaEntityId ?? `${c.provider}:${c.externalId}`;
|
||||
|
||||
export async function identifyCore(
|
||||
d: FmDeps,
|
||||
body: FilesIdentifyRequest,
|
||||
opts: CoreOpts,
|
||||
): Promise<{ result: FilesIdentifyResult; remotePending: number; pendingPaths: Set<string> }> {
|
||||
): Promise<{
|
||||
result: FilesIdentifyResult;
|
||||
remotePending: number;
|
||||
pendingPaths: Set<string>;
|
||||
/** SCR-2:每个视频按 resolveIdentity 能否自动确认(root\0path → 候选键;不能为 null) */
|
||||
autoPicks: Map<string, string | null>;
|
||||
}> {
|
||||
const db = d.ctx.db;
|
||||
const pendingPaths = new Set<string>();
|
||||
const autoPicks = new Map<string, string | null>();
|
||||
const { videos, truncated } = await collectVideos(d, body);
|
||||
// G2:最小体积(请求/向导 > settings.organize.minSizeBytes,缺省 50 MiB;设置读不到时不过滤)
|
||||
const orgSettings = (await d.ctx.settings.get("organize").catch(() => null)) as { minSizeBytes?: unknown } | null;
|
||||
@@ -411,6 +429,14 @@ async function identifyCore(
|
||||
),
|
||||
};
|
||||
});
|
||||
const res = resolveIdentity(parsed, all);
|
||||
const pick = res.status === "auto" && res.best ? res.best.candidate : null;
|
||||
autoPicks.set(
|
||||
`${v.root}\u0000${v.rel}`,
|
||||
pick && (!kindHint || (kindHint === "movie") === (pick.entityType === "movie"))
|
||||
? (pick.localEntityId ?? `${pick.provider}:${pick.externalId}`)
|
||||
: null,
|
||||
);
|
||||
const best = candidates[0];
|
||||
let contentUnitId: string | null = null;
|
||||
let contentUnit: string | null = null;
|
||||
@@ -454,7 +480,7 @@ async function identifyCore(
|
||||
}
|
||||
if (videos.length === 0 && body.items.length > 0 && !truncated)
|
||||
throw new FilesError("bad_request", "所选项中没有视频文件");
|
||||
return { result: { items, truncated, excludedBySize }, remotePending: remoteQueries, pendingPaths };
|
||||
return { result: { items, truncated, excludedBySize }, remotePending: remoteQueries, pendingPaths, autoPicks };
|
||||
}
|
||||
|
||||
/** TMDB 图片 CDN 缩略图(w154);已是完整 URL 的原样返回 */
|
||||
|
||||
@@ -8,21 +8,17 @@ import type {
|
||||
FilesProbeResult,
|
||||
FilesScrapeBatchRequest,
|
||||
FilesScrapeBatchResult,
|
||||
FilesScrapeRequest,
|
||||
FilesScrapeResult,
|
||||
} from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import type { Principal } from "../../http/auth.ts";
|
||||
import { enqueueScrape } from "../acquisition-organize/services/scrape.ts";
|
||||
import { FilesError, type FmDeps, jsonb, resolveIn } from "./common.ts";
|
||||
import { audit } from "./ledger.ts";
|
||||
import { kindOfName, mediaSummary } from "./list.ts";
|
||||
import { entitiesUnder } from "./records.ts";
|
||||
|
||||
// probe:ffprobe 摘要;已登记资产写回 media_asset.media_info.probe,未登记只缓存 24h(DATA_DIR/cache/probe)。
|
||||
// scrape:只对已登记作品,等价 POST /media-entities/{id}/refresh(入队 refreshMetadata,字段锁优先、不动路径);
|
||||
// writeNfo / images(SCR-1)另排 scrapeMedia,把 NFO 与图片写进媒体库目录(与作品详情「重新刮削」同一任务)。
|
||||
// scrape-batch write=true:文件管理「刮削」,按作品去重各排一次 scrapeMedia(不刷新元数据)。
|
||||
// scrape-batch:文件管理「刷新元数据」,按作品去重各排一次 refreshMetadata(字段锁优先、不动路径、不写文件)。
|
||||
// 刮削(写 NFO 与图片)见 scrape.ts。
|
||||
|
||||
const CACHE_TTL_MS = 24 * 3600_000;
|
||||
|
||||
@@ -104,62 +100,8 @@ export async function probeFiles(d: FmDeps, body: FilesProbeRequest): Promise<Fi
|
||||
return { items };
|
||||
}
|
||||
|
||||
export async function scrape(
|
||||
d: FmDeps,
|
||||
principal: Principal | undefined,
|
||||
body: FilesScrapeRequest,
|
||||
): Promise<FilesScrapeResult> {
|
||||
const db = d.ctx.db;
|
||||
const e = await db
|
||||
.selectFrom("media_entity")
|
||||
.select(["id", "status", "field_locks"])
|
||||
.where("id", "=", body.mediaEntityId)
|
||||
.executeTakeFirst();
|
||||
if (!e) throw new FilesError("not_found", "作品不存在");
|
||||
const registered = await db
|
||||
.selectFrom("media_asset")
|
||||
.select(["id"])
|
||||
.where("media_entity_id", "=", e.id)
|
||||
.where("status", "=", "active")
|
||||
.executeTakeFirst();
|
||||
if (!registered) throw new FilesError("bad_request", "作品没有已入库文件。先识别并整理入库,再刮削");
|
||||
const { runId } = await d.ctx.engine.enqueue(
|
||||
"refreshMetadata",
|
||||
{ mediaEntityId: e.id },
|
||||
{ idempotencyKey: `refreshMetadata:files:${body.idempotencyKey}`, traceId: randomUUID() },
|
||||
);
|
||||
const overrides = await db
|
||||
.selectFrom("field_override")
|
||||
.select(["field"])
|
||||
.where("media_entity_id", "=", e.id)
|
||||
.where("locked", "=", true)
|
||||
.execute();
|
||||
const lockedFields = [...new Set([...(e.field_locks ?? []), ...overrides.map((o) => o.field)])].sort();
|
||||
const scrapeTaskId =
|
||||
body.writeNfo || body.images
|
||||
? (
|
||||
await enqueueScrape(d.ao, {
|
||||
mediaEntityId: e.id,
|
||||
trigger: "manual",
|
||||
actor: { type: principal?.kind === "api_key" ? "api_key" : "user", id: principal?.id ?? null },
|
||||
})
|
||||
).taskId
|
||||
: null;
|
||||
const result: FilesScrapeResult = {
|
||||
mediaEntityId: e.id,
|
||||
taskId: runId,
|
||||
lockedFields,
|
||||
nfo: body.writeNfo ? "queued" : "not_requested",
|
||||
images: body.images ? "queued" : "not_requested",
|
||||
scrapeTaskId,
|
||||
};
|
||||
await audit(d, principal, "scrape", { ...result });
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* G5 批量刷新元数据:文件/目录归到其下已登记资产的作品,与显式作品合并去重,逐个入队 refreshMetadata。
|
||||
* 只对有 active 资产的作品提交;其余进 skipped(前端汇总"已提交 n 个作品,锁定字段不变")。
|
||||
*/
|
||||
export async function scrapeBatch(
|
||||
d: FmDeps,
|
||||
@@ -183,18 +125,6 @@ export async function scrapeBatch(
|
||||
const ents = list.length
|
||||
? await db.selectFrom("media_entity").select(["id", "title", "field_locks"]).where("id", "in", list).execute()
|
||||
: [];
|
||||
const active = new Set(
|
||||
list.length
|
||||
? (
|
||||
await db
|
||||
.selectFrom("media_asset")
|
||||
.select(["media_entity_id"])
|
||||
.where("media_entity_id", "in", list)
|
||||
.where("status", "=", "active")
|
||||
.execute()
|
||||
).map((a) => a.media_entity_id)
|
||||
: [],
|
||||
);
|
||||
const locks = list.length
|
||||
? await db
|
||||
.selectFrom("field_override")
|
||||
@@ -210,34 +140,16 @@ export async function scrapeBatch(
|
||||
skipped.push({ ref: id, reason: "not_found" });
|
||||
continue;
|
||||
}
|
||||
if (!active.has(id)) {
|
||||
skipped.push({ ref: id, reason: "not_registered" });
|
||||
continue;
|
||||
}
|
||||
const runId = body.write
|
||||
? (
|
||||
await enqueueScrape(d.ao, {
|
||||
mediaEntityId: id,
|
||||
trigger: "manual",
|
||||
actor: { type: principal?.kind === "api_key" ? "api_key" : "user", id: principal?.id ?? null },
|
||||
})
|
||||
).taskId
|
||||
: (
|
||||
await d.ctx.engine.enqueue(
|
||||
"refreshMetadata",
|
||||
{ mediaEntityId: id },
|
||||
{ idempotencyKey: `refreshMetadata:files-batch:${body.idempotencyKey}:${id}`, traceId: randomUUID() },
|
||||
)
|
||||
).runId;
|
||||
const { runId } = await d.ctx.engine.enqueue(
|
||||
"refreshMetadata",
|
||||
{ mediaEntityId: id },
|
||||
{ idempotencyKey: `refreshMetadata:files-batch:${body.idempotencyKey}:${id}`, traceId: randomUUID() },
|
||||
);
|
||||
const lockedFields = [
|
||||
...new Set([...(e.field_locks ?? []), ...locks.filter((l) => l.media_entity_id === id).map((l) => l.field)]),
|
||||
].sort();
|
||||
submitted.push({ mediaEntityId: id, title: e.title, taskId: runId, lockedFields });
|
||||
}
|
||||
await audit(d, principal, "scrape_batch", {
|
||||
write: body.write,
|
||||
submitted: submitted.map((s) => s.mediaEntityId),
|
||||
skipped,
|
||||
});
|
||||
await audit(d, principal, "scrape_batch", { submitted: submitted.map((s) => s.mediaEntityId), skipped });
|
||||
return { submitted, skipped };
|
||||
}
|
||||
@@ -15,9 +15,10 @@ import { listDir, listRoots, statEntry } from "./list.ts";
|
||||
import { mkdirIn, renameEntry } from "./mutate.ts";
|
||||
import { organizeFiles } from "./organize.ts";
|
||||
import { previewFile } from "./preview.ts";
|
||||
import { probeFiles, scrape, scrapeBatch } from "./probe.ts";
|
||||
import { probeFiles, scrapeBatch } from "./probe.ts";
|
||||
import { deleteRecords } from "./record-delete.ts";
|
||||
import { findRecords } from "./records.ts";
|
||||
import { scrapeFiles, scrapeRecords } from "./scrape.ts";
|
||||
import { suggestName } from "./suggest-name.ts";
|
||||
import { transfer } from "./transfer.ts";
|
||||
import { deleteFiles, listTrash, purgeTrash, restoreTrash } from "./trash.ts";
|
||||
@@ -111,7 +112,11 @@ export function registerFileRoutes(app: ModuleApp, d: FmDeps) {
|
||||
mount(app, F.identify, async (c, { body }) =>
|
||||
identify(d, c.get("principal"), body, { wait: /^(1|true)$/i.test(c.req.query("wait") ?? "") }),
|
||||
);
|
||||
mount(app, F.scrape, async (c, { body }) => idem(d, c, "scrape", body, () => scrape(d, c.get("principal"), body)));
|
||||
// SCR-2:任意目录 / 视频文件就地刮削(识别 → 写 NFO 与图片);认不准的项返回候选
|
||||
mount(app, F.scrape, async (c, { body }) =>
|
||||
idem(d, c, "scrape", body, () => scrapeFiles(d, c.get("principal"), body)),
|
||||
);
|
||||
mount(app, F.scrapeRecords, async (_c, { query }) => scrapeRecords(d, query.ids));
|
||||
mount(app, F.probe, async (_c, { body }) => probeFiles(d, body));
|
||||
mount(app, F.records, async (_c, { query }) => findRecords(d, query));
|
||||
mount(app, F.download, async (c, { query }) => downloadFile(d, c.get("principal"), query), true);
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
import type {
|
||||
FileIdentifyCandidate,
|
||||
FilesScrapeItem,
|
||||
FilesScrapeNeedsChoice,
|
||||
FilesScrapeRequest,
|
||||
FilesScrapeResult,
|
||||
ScrapeMatch,
|
||||
} from "@umt/contracts";
|
||||
import type { Principal } from "../../http/auth.ts";
|
||||
import {
|
||||
entityScrapeMatch,
|
||||
loadScrapeRecords,
|
||||
queueScrape,
|
||||
scrapeSettings,
|
||||
waitScrapeRecords,
|
||||
} from "../acquisition-organize/services/scrape.ts";
|
||||
import { actorOfP, FilesError, type FmDeps, resolveIn } from "./common.ts";
|
||||
import { candidateKey, identifyCore } from "./identify.ts";
|
||||
import { audit } from "./ledger.ts";
|
||||
import { kindOfName } from "./list.ts";
|
||||
import { entitiesUnder } from "./records.ts";
|
||||
|
||||
// SCR-2 文件管理「刮削」(照 MP 文件管理的刮削):任意目录 / 视频文件 → 识别作品 → 就地写 NFO 与图片。
|
||||
// 识别顺序:请求指定(选定的候选)> 目录里已登记文件所属的唯一作品 > 按路径识别(本地召回 + TMDB,resolveIdentity
|
||||
// 能自动确认且各视频一致时自动用)。认不准的项进 needsChoice(前 5 候选),选定后带 mediaEntityId / external 再提交。
|
||||
// 未登记也能刮削(直接用 TMDB 字段);addToLibrary 时顺手把 TMDB 条目加入媒体库。
|
||||
// 每项一条刮削记录,排 scrapeMedia;请求内最多等 transferWaitMs,没刮完的 pending=true,前端按 id 轮询。
|
||||
|
||||
const RECOGNIZE_TIMEOUT_MS = 15_000;
|
||||
|
||||
type Actor = { type: string; id: string | null };
|
||||
const actorOf = (p: Principal | undefined): Actor => ({
|
||||
type: p?.kind === "api_key" ? "api_key" : "user",
|
||||
id: p?.id ?? null,
|
||||
});
|
||||
|
||||
/** 路径识别:各视频都能自动确认且是同一作品时返回它;否则返回候选(按出现次数与等级合并,前 5) */
|
||||
async function recognize(
|
||||
d: FmDeps,
|
||||
principal: Principal | undefined,
|
||||
it: FilesScrapeItem,
|
||||
): Promise<{ pick: FileIdentifyCandidate | null; choice: Omit<FilesScrapeNeedsChoice, "root" | "path"> }> {
|
||||
const r = await identifyCore(
|
||||
d,
|
||||
{ items: [{ root: it.root, path: it.path }], minSizeBytes: 0 },
|
||||
{ mode: "remote", actorId: actorOfP(principal), signal: AbortSignal.timeout(RECOGNIZE_TIMEOUT_MS) },
|
||||
);
|
||||
const items = r.result.items;
|
||||
const picks = items.map((i) => r.autoPicks.get(`${i.root}\u0000${i.path}`) ?? null);
|
||||
const agreed = picks.length > 0 && picks.every((p) => p && p === picks[0]) ? picks[0] : null;
|
||||
const LV = { high: 3, medium: 2, low: 1 } as const;
|
||||
const merged = new Map<string, { c: FileIdentifyCandidate; n: number }>();
|
||||
for (const i of items)
|
||||
for (const c of i.candidates) {
|
||||
const k = candidateKey(c);
|
||||
const cur = merged.get(k);
|
||||
if (!cur) merged.set(k, { c, n: 1 });
|
||||
else {
|
||||
cur.n += 1;
|
||||
if (LV[c.level] > LV[cur.c.level]) cur.c = { ...c, posterUrl: c.posterUrl ?? cur.c.posterUrl };
|
||||
}
|
||||
}
|
||||
const ranked = [...merged.values()].sort((a, b) => b.n - a.n || LV[b.c.level] - LV[a.c.level]).map((x) => x.c);
|
||||
const pick = agreed ? (ranked.find((c) => candidateKey(c) === agreed) ?? null) : null;
|
||||
const first = items[0]?.parsed;
|
||||
return {
|
||||
pick,
|
||||
choice: {
|
||||
title: first?.title ?? null,
|
||||
candidates: ranked.slice(0, 5),
|
||||
warnings: [...new Set(items.flatMap((i) => i.warnings))].slice(0, 5),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** TMDB 条目 → 识别结果:已有本地作品(已接受的 TMDB 映射)就用它;addToLibrary 时建作品 */
|
||||
async function externalMatch(
|
||||
d: FmDeps,
|
||||
ext: { externalId: string; kind: "movie" | "tv"; title?: string; year?: number | null },
|
||||
source: ScrapeMatch["source"],
|
||||
addToLibrary: boolean,
|
||||
actor: Actor,
|
||||
entities: FilesScrapeResult["entities"],
|
||||
): Promise<ScrapeMatch> {
|
||||
const local = await d.ctx.db
|
||||
.selectFrom("external_identity")
|
||||
.select(["media_entity_id"])
|
||||
.where("provider", "=", "tmdb")
|
||||
.where("entity_type", "=", ext.kind)
|
||||
.where("external_id", "=", ext.externalId)
|
||||
.where("status", "=", "accepted")
|
||||
.where("content_unit_id", "is", null)
|
||||
.executeTakeFirst();
|
||||
let entityId = local?.media_entity_id ?? null;
|
||||
if (!entityId && addToLibrary) {
|
||||
entityId = await d.ao.ports.entityFromTmdb({ type: ext.kind, id: ext.externalId }, actor);
|
||||
const e = await d.ctx.db.selectFrom("media_entity").select(["title"]).where("id", "=", entityId).executeTakeFirst();
|
||||
entities.push({ mediaEntityId: entityId, title: e?.title ?? ext.title ?? ext.externalId, created: true });
|
||||
}
|
||||
if (entityId) {
|
||||
const m = await entityScrapeMatch(d.ao, entityId, source);
|
||||
if (m) return { ...m, provider: "tmdb", externalId: ext.externalId };
|
||||
}
|
||||
return {
|
||||
mediaEntityId: null,
|
||||
provider: "tmdb",
|
||||
externalId: ext.externalId,
|
||||
kind: ext.kind,
|
||||
title: ext.title ?? `TMDB ${ext.externalId}`,
|
||||
year: ext.year ?? null,
|
||||
source,
|
||||
};
|
||||
}
|
||||
|
||||
export async function scrapeFiles(
|
||||
d: FmDeps,
|
||||
principal: Principal | undefined,
|
||||
body: FilesScrapeRequest,
|
||||
): Promise<FilesScrapeResult> {
|
||||
const db = d.ctx.db;
|
||||
const actor = actorOf(principal);
|
||||
const mode = body.mode ?? (await scrapeSettings(d.ao)).scrapeMode;
|
||||
const recordIds: string[] = [];
|
||||
const needsChoice: FilesScrapeResult["needsChoice"] = [];
|
||||
const skipped: FilesScrapeResult["skipped"] = [];
|
||||
const entities: FilesScrapeResult["entities"] = [];
|
||||
for (const it of body.items) {
|
||||
if (it.mediaEntityId && it.external) throw new FilesError("bad_request", "mediaEntityId 与 external 只能给一个");
|
||||
const r = await resolveIn(d, it.root, it.path);
|
||||
const ref = { root: it.root, path: r.rel };
|
||||
if (r.def.kind === "library" && !r.def.modifiable) {
|
||||
skipped.push({ ...ref, reason: "scrape_root_read_only" });
|
||||
continue;
|
||||
}
|
||||
const isDir = !!r.lst?.isDirectory();
|
||||
if (!isDir && !(r.lst?.isFile() && kindOfName(r.abs) === "video")) {
|
||||
skipped.push({ ...ref, reason: "scrape_not_video" });
|
||||
continue;
|
||||
}
|
||||
let match: ScrapeMatch | null = null;
|
||||
if (it.mediaEntityId) {
|
||||
match = await entityScrapeMatch(d.ao, it.mediaEntityId, "chosen");
|
||||
if (!match) throw new FilesError("not_found", `作品不存在:${it.mediaEntityId}`);
|
||||
} else if (it.external) {
|
||||
match = await externalMatch(d, it.external, "chosen", body.addToLibrary, actor, entities);
|
||||
} else {
|
||||
// 目录里的文件已登记到唯一作品:直接用(重新刮削已入库的)
|
||||
const owners = r.def.libraryRootId ? await entitiesUnder(db, r.def.libraryRootId, r.rel, isDir) : [];
|
||||
if (owners.length === 1) match = await entityScrapeMatch(d.ao, owners[0]!, "library");
|
||||
if (!match) {
|
||||
let rec: Awaited<ReturnType<typeof recognize>>;
|
||||
try {
|
||||
rec = await recognize(d, principal, { ...it, path: r.rel });
|
||||
} catch (e) {
|
||||
if (e instanceof FilesError && e.reason === "bad_request") {
|
||||
skipped.push({ ...ref, reason: "scrape_no_videos" });
|
||||
continue;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
const c = rec.pick;
|
||||
if (c?.mediaEntityId) match = await entityScrapeMatch(d.ao, c.mediaEntityId, "auto");
|
||||
else if (c && c.provider === "tmdb" && c.externalId)
|
||||
match = await externalMatch(
|
||||
d,
|
||||
{ externalId: c.externalId, kind: c.kind === "movie" ? "movie" : "tv", title: c.title, year: c.year },
|
||||
"auto",
|
||||
body.addToLibrary,
|
||||
actor,
|
||||
entities,
|
||||
);
|
||||
if (!match) {
|
||||
needsChoice.push({ ...ref, ...rec.choice });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
const q = await queueScrape(d.ao, {
|
||||
rootId: r.def.id,
|
||||
rootPath: r.def.hostPath,
|
||||
libraryRootId: r.def.libraryRootId,
|
||||
path: r.rel,
|
||||
match,
|
||||
mode,
|
||||
trigger: "manual",
|
||||
actor,
|
||||
});
|
||||
recordIds.push(q.recordId);
|
||||
}
|
||||
const records = recordIds.length ? await waitScrapeRecords(d.ao, recordIds, d.options.transferWaitMs) : [];
|
||||
await audit(d, principal, "scrape", {
|
||||
items: body.items.map((i) => `${i.root}:${i.path}`).slice(0, 100),
|
||||
mode,
|
||||
recordIds,
|
||||
needsChoice: needsChoice.map((n) => `${n.root}:${n.path}`),
|
||||
skipped,
|
||||
});
|
||||
return {
|
||||
records,
|
||||
needsChoice,
|
||||
skipped,
|
||||
pending: records.some((r) => r.status === "queued" || r.status === "running"),
|
||||
entities,
|
||||
};
|
||||
}
|
||||
|
||||
export async function scrapeRecords(d: FmDeps, idsParam: string) {
|
||||
const ids = [...new Set(idsParam.split(",").map((s) => s.trim()))].filter((s) => /^[0-9a-f-]{36}$/i.test(s));
|
||||
if (!ids.length || ids.length > 100) throw new FilesError("bad_request", "ids 需要 1–100 个刮削记录 id");
|
||||
return { records: await loadScrapeRecords(d.ao, ids) };
|
||||
}
|
||||
Reference in new issue
Block a user