feat(server): scrapeMedia 刮削任务,入库后自动刮削(SCR-1)

- 文件道任务 scrapeMedia:NFO 原子写入、图片经图片缓存下载(并发 4,失败不阻塞 NFO)、只补缺失 / 全部覆盖
- 设置 organize.scrapeAfterImport / scrapeMode / scrapeImages / scrapeEpisodeThumbs
- POST /library/{id}/scrape、详情 scrape 状态;files/scrape writeNfo 与 scrape-batch write 走同一任务
- 迁移 0057_media_scrape、原因码、审计 organize.scrape
- 与 MP 输出的 NFO 差分测试(apps/server/test/scrape)

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-09-30 22:36:14 +08:00
1 parent 861f799615
commit f38eca419e
24 files changed
+1740 -15

No files matched your search

+9
View File
@@ -2238,6 +2238,15 @@ export const Api = {
query: libraryVersions.LibraryVersionsQuery,
response: libraryVersions.LibraryVersions,
}),
scrape: ep({
method: "post",
path: "/library/{id}/scrape",
scope: "organize.execute",
summary: "刮削:把作品的 NFO 与图片写进媒体库目录(后台任务 scrapeMedia;覆盖方式缺省按设置)",
params: IdParam,
body: organization.LibraryScrapeRequest,
response: organization.LibraryScrapeResult,
}),
reconcile: ep({
method: "post",
path: "/library/reconcile",
+8 -3
View File
@@ -685,6 +685,7 @@ export type IdentifyCompletedPayload = z.infer<typeof IdentifyCompletedPayload>;
export const FilesScrapeRequest = z.object({
idempotencyKey: IdempotencyKey,
mediaEntityId: Id,
/** SCR-1:true 时另排 scrapeMedia,把 NFO(与 images 时的图片)写进媒体库目录 */
writeNfo: z.boolean().default(false),
images: z.boolean().default(false),
});
@@ -695,9 +696,11 @@ export const FilesScrapeResult = z.object({
taskId: z.string(),
/** 当前被用户锁定、刷新不会改动的字段 */
lockedFields: z.array(z.string()),
/** NFO/图片投影:W5 实现前请求时为 not_implemented */
nfo: z.enum(["written", "skipped", "not_requested", "not_implemented"]),
images: z.enum(["written", "skipped", "not_requested", "not_implemented"]),
/** NFO/图片投影(SCR-1):queued = 已排 scrapeMedia,结果看作品刮削状态;not_implemented 已不再返回 */
nfo: z.enum(["written", "skipped", "queued", "not_requested", "not_implemented"]),
images: z.enum(["written", "skipped", "queued", "not_requested", "not_implemented"]),
/** scrapeMedia 引擎 run id(writeNfo / images 时) */
scrapeTaskId: z.string().nullable().optional(),
});
export type FilesScrapeResult = z.infer<typeof FilesScrapeResult>;
@@ -845,6 +848,8 @@ export const FilesScrapeBatchRequest = z.object({
idempotencyKey: IdempotencyKey,
items: z.array(FileRef).max(500).optional(),
mediaEntityIds: z.array(Id).max(200).optional(),
/** SCR-1:true = 刮削(排 scrapeMedia 写 NFO 与图片,覆盖方式按设置);false(缺省)= 刷新元数据 */
write: z.boolean().default(false),
});
export type FilesScrapeBatchRequest = z.infer<typeof FilesScrapeBatchRequest>;
export const FilesScrapeBatchResult = z.object({
+77
View File
@@ -370,6 +370,81 @@ export const LibraryDroppedPlanView = z.object({
});
export type LibraryDroppedPlanView = z.infer<typeof LibraryDroppedPlanView>;
/** SCR-1 刮削:一个写入目标(NFO / 图片)的结果;path 为库根内相对路径 */
export const ScrapeFileResult = z.object({
path: z.string(),
kind: z.enum(["nfo", "image"]),
/** movie_nfo / tvshow_nfo / season_nfo / episode_nfo / poster / fanart / season_poster / episode_thumb */
role: z.string(),
action: z.enum(["written", "skipped", "failed"]),
/** 跳过 / 失败原因码(reasons.ts:scrape_exists / scrape_write_failed / scrape_image_failed …) */
reason: z.string().nullable(),
});
export type ScrapeFileResult = z.infer<typeof ScrapeFileResult>;
export const ScrapeMode = z.enum(["missing", "overwrite"]);
export type ScrapeMode = z.infer<typeof ScrapeMode>;
export const ScrapeStatus = z.enum(["queued", "running", "succeeded", "partial", "failed"]);
export type ScrapeStatus = z.infer<typeof ScrapeStatus>;
export const ScrapeTrigger = z.enum(["auto", "manual"]);
export type ScrapeTrigger = z.infer<typeof ScrapeTrigger>;
/** 表 media_scrape 的行(一部作品一行,最近一次刮削) */
export const MediaScrape = z.object({
mediaEntityId: Id,
status: ScrapeStatus,
trigger: ScrapeTrigger,
mode: ScrapeMode,
recordId: Id.nullable(),
engineRunId: z.string().nullable(),
requestedAt: Timestamp,
startedAt: Timestamp.nullable(),
finishedAt: Timestamp.nullable(),
written: NonNegInt,
skipped: NonNegInt,
failed: NonNegInt,
reason: z.string().nullable(),
files: z.array(ScrapeFileResult),
createdAt: Timestamp,
updatedAt: Timestamp,
});
export type MediaScrape = z.infer<typeof MediaScrape>;
/** SCR-1 作品刮削状态(最近一次;表 media_scrape) */
export const LibraryScrapeState = z.object({
status: ScrapeStatus,
trigger: ScrapeTrigger,
mode: ScrapeMode,
requestedAt: Timestamp,
startedAt: Timestamp.nullable(),
finishedAt: Timestamp.nullable(),
written: NonNegInt,
skipped: NonNegInt,
failed: NonNegInt,
/** 整体失败原因码(没有文件、没有元数据等) */
reason: z.string().nullable(),
files: z.array(ScrapeFileResult),
/** 触发它的整理记录(入库后自动刮削) */
recordId: Id.nullable(),
/** 引擎 run id */
taskId: z.string().nullable(),
});
export type LibraryScrapeState = z.infer<typeof LibraryScrapeState>;
export const LibraryScrapeRequest = z.object({
/** 缺省按设置 organize.scrapeMode */
mode: ScrapeMode.optional(),
});
export type LibraryScrapeRequest = z.infer<typeof LibraryScrapeRequest>;
export const LibraryScrapeResult = z.object({
mediaEntityId: Id,
taskId: z.string(),
/** 同一作品已有排队中的刮削,直接复用 */
deduplicated: z.boolean(),
});
export type LibraryScrapeResult = z.infer<typeof LibraryScrapeResult>;
export const LibraryDetail = z.object({
entity: MediaEntity,
/** 身份映射状态(含 proposed/conflicted,前端区分展示) */
@@ -403,6 +478,8 @@ export const LibraryDetail = z.object({
}),
)
.optional(),
/** SCR-1:最近一次刮削(写 NFO 与图片);没刮削过为 null */
scrape: LibraryScrapeState.nullable().optional(),
});
export type LibraryDetail = z.infer<typeof LibraryDetail>;
+8
View File
@@ -87,6 +87,14 @@ const REASONS: Record<string, ReasonText> = {
read_only_reconcile: { text: "只读对账中,暂不提交下载" },
rejected: { text: "下载器拒绝了这个种子" },
source_failed: { text: "从站点取种子失败", action: "检查站点登录后重新下载" },
// SCR-1 刮削(作品刮削状态、逐文件结果)
scrape_no_files: { text: "作品没有已入库文件", action: "先整理入库" },
scrape_no_metadata: { text: "作品还没有元数据", action: "先刷新元数据" },
scrape_root_read_only: { text: "媒体库目录是只登记模式,不写文件" },
scrape_exists: { text: "文件已存在,按设置只补缺失" },
scrape_write_failed: { text: "写入媒体库目录失败", action: "检查目录权限后重新刮削" },
scrape_image_failed: { text: "图片没下载成功", action: "稍后重新刮削" },
scrape_no_work_dir: { text: "视频直接放在媒体库目录下,不写剧集和季的文件" },
};
/**
+23 -2
View File
@@ -72,6 +72,14 @@ export const OrganizeSettings = z.object({
allowCrossVolumeMove: z.boolean(),
/** 整理 / 文件管理识别时视频文件的最小体积(字节),低于它的视为样片跳过;请求(向导)里给的值优先 */
minSizeBytes: z.number().int().nonnegative(),
/** SCR-1:整理入库成功后自动刮削(往媒体库目录写 NFO 与图片);缺省关(同 MP 目录「刮削」缺省) */
scrapeAfterImport: z.boolean(),
/** SCR-1:目标文件已存在时 missing = 只补缺失(MP「仅缺失」)/ overwrite = 全部覆盖 */
scrapeMode: z.enum(["missing", "overwrite"]),
/** SCR-1:下载海报、背景图、季海报 */
scrapeImages: z.boolean(),
/** SCR-1:下载每集剧照(`<视频名>-thumb.jpg`) */
scrapeEpisodeThumbs: z.boolean(),
});
export type OrganizeSettings = z.infer<typeof OrganizeSettings>;
@@ -618,7 +626,15 @@ export const DEFAULT_SETTINGS: GlobalSettings = {
globalImageCache: false,
globalImageCacheDays: 7,
},
organize: { defaultMode: "copy", allowCrossVolumeMove: true, minSizeBytes: 50 * 1024 * 1024 },
organize: {
defaultMode: "copy",
allowCrossVolumeMove: true,
minSizeBytes: 50 * 1024 * 1024,
scrapeAfterImport: false,
scrapeMode: "missing",
scrapeImages: true,
scrapeEpisodeThumbs: true,
},
retention: { seedRetentionDays: 7, logRetentionDays: 180 },
pi: {
defaultUpstreamId: null,
@@ -927,17 +943,22 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [
{
id: "library",
label: "媒体库与目录",
description: "媒体库目录、默认整理方式、命名模板",
description: "媒体库目录、默认整理方式、刮削、命名模板",
legacyTabs: [{ tab: "library-roots", anchor: "roots" }],
anchors: [
{ id: "roots", label: "媒体库目录" },
{ id: "organize", label: "整理方式" },
{ id: "scrape", label: "刮削" },
{ id: "naming", label: "命名模板" },
],
fields: [
"organize.defaultMode",
"organize.allowCrossVolumeMove",
"organize.minSizeBytes",
"organize.scrapeAfterImport",
"organize.scrapeMode",
"organize.scrapeImages",
"organize.scrapeEpisodeThumbs",
"naming.movie",
"naming.tv",
"naming.variety",
+4
View File
@@ -63,6 +63,7 @@ const ROWS: Record<string, { row: Obj; dbOnly?: string[]; apiOnly?: string[] }>
LibraryRoot: { row: C.LibraryRoot },
MediaAsset: { row: C.MediaAsset },
MediaEntity: { row: C.MediaEntity },
MediaScrape: { row: C.MediaScrape },
MediaServerInstance: { row: C.MediaServerInstance },
Observation: { row: C.Observation },
OrganizationFileResult: { row: C.OrganizationFileResult },
@@ -209,6 +210,9 @@ const ENUMS: Record<string, z.ZodEnum> = {
MediaAssetStatus: C.AssetStatus,
MediaEntityKind: C.MediaKind,
MediaEntityStatus: C.MediaEntityStatus,
MediaScrapeMode: C.ScrapeMode,
MediaScrapeStatus: C.ScrapeStatus,
MediaScrapeTrigger: C.ScrapeTrigger,
MediaServerInstanceKind: C.MediaServerKind,
MediaServerInstanceHealthStatus: C.InstanceHealth,
ObservationStatus: C.ObservationStatus,
@@ -0,0 +1,38 @@
import type { Kysely } from "kysely";
import { execDdl } from "../src/migration-helpers.ts";
// SCR-1 刮削:每部作品最近一次刮削(往媒体库目录写 NFO 与图片)的状态与逐文件结果。
// 一部作品一行,重新刮削覆盖;历史看审计(organize.scrape)与引擎运行记录。
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
export async function up(db: Kysely<any>): Promise<void> {
await execDdl(
db,
`create table if not exists media_scrape (
media_entity_id uuid primary key references media_entity (id) on delete cascade,
status text not null default 'queued' check (status in ('queued', 'running', 'succeeded', 'partial', 'failed')),
trigger text not null default 'manual' check (trigger in ('auto', 'manual')),
mode text not null default 'missing' check (mode in ('missing', 'overwrite')),
record_id uuid references organization_record (id) on delete set null,
engine_run_id text,
requested_at timestamptz not null default now(),
started_at timestamptz,
finished_at timestamptz,
written integer not null default 0,
skipped integer not null default 0,
failed integer not null default 0,
reason text,
files jsonb not null default '[]'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
drop trigger if exists media_scrape_touch_updated_at on media_scrape;
create trigger media_scrape_touch_updated_at before update on media_scrape
for each row execute function umt_touch_updated_at();`,
);
}
// biome-ignore lint/suspicious/noExplicitAny: 同上
export async function down(db: Kysely<any>): Promise<void> {
await execDdl(db, "drop table if exists media_scrape;");
}
+26
View File
@@ -159,6 +159,9 @@ export type MediaAssetKind = "video" | "bluray_dir" | "subtitle" | "audio" | "nf
export type MediaAssetStatus = "active" | "replaced" | "retired" | "missing";
export type MediaEntityKind = "movie" | "series" | "variety" | "collection" | "other";
export type MediaEntityStatus = "provisional" | "active" | "merged" | "archived";
export type MediaScrapeStatus = "queued" | "running" | "succeeded" | "partial" | "failed";
export type MediaScrapeTrigger = "auto" | "manual";
export type MediaScrapeMode = "missing" | "overwrite";
export type MediaServerInstanceKind = "emby" | "jellyfin";
export type MediaServerInstanceHealthStatus = "unknown" | "healthy" | "degraded" | "offline";
export type MetadataEnrichmentStatus = "suggested" | "applied" | "failed";
@@ -915,6 +918,25 @@ export interface MediaEntityTable {
naming_override: Json | null;
}
export interface MediaScrapeTable {
media_entity_id: string;
status: Generated<MediaScrapeStatus>;
trigger: Generated<MediaScrapeTrigger>;
mode: Generated<MediaScrapeMode>;
record_id: string | null;
engine_run_id: string | null;
requested_at: GeneratedTimestamptz;
started_at: Timestamptz | null;
finished_at: Timestamptz | null;
written: Generated<number>;
skipped: Generated<number>;
failed: Generated<number>;
reason: string | null;
files: Generated<Json>;
created_at: GeneratedTimestamptz;
updated_at: GeneratedTimestamptz;
}
export interface MediaServerInstanceTable {
id: Generated<string>;
name: string;
@@ -1698,6 +1720,7 @@ export interface DB {
mcp_server_config: McpServerConfigTable;
media_asset: MediaAssetTable;
media_entity: MediaEntityTable;
media_scrape: MediaScrapeTable;
media_server_instance: MediaServerInstanceTable;
metadata_enrichment: MetadataEnrichmentTable;
metadata_field: MetadataFieldTable;
@@ -1858,6 +1881,9 @@ export type MediaAssetUpdate = Updateable<MediaAssetTable>;
export type MediaEntityRow = Selectable<MediaEntityTable>;
export type NewMediaEntity = Insertable<MediaEntityTable>;
export type MediaEntityUpdate = Updateable<MediaEntityTable>;
export type MediaScrapeRow = Selectable<MediaScrapeTable>;
export type NewMediaScrape = Insertable<MediaScrapeTable>;
export type MediaScrapeUpdate = Updateable<MediaScrapeTable>;
export type MediaServerInstanceRow = Selectable<MediaServerInstanceTable>;
export type NewMediaServerInstance = Insertable<MediaServerInstanceTable>;
export type MediaServerInstanceUpdate = Updateable<MediaServerInstanceTable>;
+3
View File
@@ -26,6 +26,7 @@ const EXPECTED_TABLES = [
"path_mapping",
// 内容 / 身份
"media_entity",
"media_scrape",
"content_unit",
"title_alias",
"external_identity",
@@ -153,6 +154,8 @@ const NON_ENTITY_TABLES = [
"kv_cache",
// 站点请求计数:(site_id, hour) 主键
"site_request_count",
// 作品最近一次刮削:media_entity_id 主键(SCR-1)
"media_scrape",
];
describe.skipIf(skip)("schema migrations (real PG)", () => {