feat(server): scrapeMedia 刮削任务,入库后自动刮削(SCR-1)
- 文件道任务 scrapeMedia:NFO 原子写入、图片经图片缓存下载(并发 4,失败不阻塞 NFO)、只补缺失 / 全部覆盖
- 设置 organize.scrapeAfterImport / scrapeMode / scrapeImages / scrapeEpisodeThumbs
- POST /library/{id}/scrape、详情 scrape 状态;files/scrape writeNfo 与 scrape-batch write 走同一任务
- 迁移 0057_media_scrape、原因码、审计 organize.scrape
- 与 MP 输出的 NFO 差分测试(apps/server/test/scrape)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
861f799615
commit
f38eca419e
24 files changed
+1740
-15
No files matched your search
@@ -2238,6 +2238,15 @@ export const Api = {
|
||||
query: libraryVersions.LibraryVersionsQuery,
|
||||
response: libraryVersions.LibraryVersions,
|
||||
}),
|
||||
scrape: ep({
|
||||
method: "post",
|
||||
path: "/library/{id}/scrape",
|
||||
scope: "organize.execute",
|
||||
summary: "刮削:把作品的 NFO 与图片写进媒体库目录(后台任务 scrapeMedia;覆盖方式缺省按设置)",
|
||||
params: IdParam,
|
||||
body: organization.LibraryScrapeRequest,
|
||||
response: organization.LibraryScrapeResult,
|
||||
}),
|
||||
reconcile: ep({
|
||||
method: "post",
|
||||
path: "/library/reconcile",
|
||||
|
||||
@@ -685,6 +685,7 @@ export type IdentifyCompletedPayload = z.infer<typeof IdentifyCompletedPayload>;
|
||||
export const FilesScrapeRequest = z.object({
|
||||
idempotencyKey: IdempotencyKey,
|
||||
mediaEntityId: Id,
|
||||
/** SCR-1:true 时另排 scrapeMedia,把 NFO(与 images 时的图片)写进媒体库目录 */
|
||||
writeNfo: z.boolean().default(false),
|
||||
images: z.boolean().default(false),
|
||||
});
|
||||
@@ -695,9 +696,11 @@ export const FilesScrapeResult = z.object({
|
||||
taskId: z.string(),
|
||||
/** 当前被用户锁定、刷新不会改动的字段 */
|
||||
lockedFields: z.array(z.string()),
|
||||
/** NFO/图片投影:W5 实现前请求时为 not_implemented */
|
||||
nfo: z.enum(["written", "skipped", "not_requested", "not_implemented"]),
|
||||
images: z.enum(["written", "skipped", "not_requested", "not_implemented"]),
|
||||
/** NFO/图片投影(SCR-1):queued = 已排 scrapeMedia,结果看作品刮削状态;not_implemented 已不再返回 */
|
||||
nfo: z.enum(["written", "skipped", "queued", "not_requested", "not_implemented"]),
|
||||
images: z.enum(["written", "skipped", "queued", "not_requested", "not_implemented"]),
|
||||
/** scrapeMedia 引擎 run id(writeNfo / images 时) */
|
||||
scrapeTaskId: z.string().nullable().optional(),
|
||||
});
|
||||
export type FilesScrapeResult = z.infer<typeof FilesScrapeResult>;
|
||||
|
||||
@@ -845,6 +848,8 @@ export const FilesScrapeBatchRequest = z.object({
|
||||
idempotencyKey: IdempotencyKey,
|
||||
items: z.array(FileRef).max(500).optional(),
|
||||
mediaEntityIds: z.array(Id).max(200).optional(),
|
||||
/** SCR-1:true = 刮削(排 scrapeMedia 写 NFO 与图片,覆盖方式按设置);false(缺省)= 刷新元数据 */
|
||||
write: z.boolean().default(false),
|
||||
});
|
||||
export type FilesScrapeBatchRequest = z.infer<typeof FilesScrapeBatchRequest>;
|
||||
export const FilesScrapeBatchResult = z.object({
|
||||
|
||||
@@ -370,6 +370,81 @@ export const LibraryDroppedPlanView = z.object({
|
||||
});
|
||||
export type LibraryDroppedPlanView = z.infer<typeof LibraryDroppedPlanView>;
|
||||
|
||||
/** SCR-1 刮削:一个写入目标(NFO / 图片)的结果;path 为库根内相对路径 */
|
||||
export const ScrapeFileResult = z.object({
|
||||
path: z.string(),
|
||||
kind: z.enum(["nfo", "image"]),
|
||||
/** movie_nfo / tvshow_nfo / season_nfo / episode_nfo / poster / fanart / season_poster / episode_thumb */
|
||||
role: z.string(),
|
||||
action: z.enum(["written", "skipped", "failed"]),
|
||||
/** 跳过 / 失败原因码(reasons.ts:scrape_exists / scrape_write_failed / scrape_image_failed …) */
|
||||
reason: z.string().nullable(),
|
||||
});
|
||||
export type ScrapeFileResult = z.infer<typeof ScrapeFileResult>;
|
||||
|
||||
export const ScrapeMode = z.enum(["missing", "overwrite"]);
|
||||
export type ScrapeMode = z.infer<typeof ScrapeMode>;
|
||||
|
||||
export const ScrapeStatus = z.enum(["queued", "running", "succeeded", "partial", "failed"]);
|
||||
export type ScrapeStatus = z.infer<typeof ScrapeStatus>;
|
||||
export const ScrapeTrigger = z.enum(["auto", "manual"]);
|
||||
export type ScrapeTrigger = z.infer<typeof ScrapeTrigger>;
|
||||
|
||||
/** 表 media_scrape 的行(一部作品一行,最近一次刮削) */
|
||||
export const MediaScrape = z.object({
|
||||
mediaEntityId: Id,
|
||||
status: ScrapeStatus,
|
||||
trigger: ScrapeTrigger,
|
||||
mode: ScrapeMode,
|
||||
recordId: Id.nullable(),
|
||||
engineRunId: z.string().nullable(),
|
||||
requestedAt: Timestamp,
|
||||
startedAt: Timestamp.nullable(),
|
||||
finishedAt: Timestamp.nullable(),
|
||||
written: NonNegInt,
|
||||
skipped: NonNegInt,
|
||||
failed: NonNegInt,
|
||||
reason: z.string().nullable(),
|
||||
files: z.array(ScrapeFileResult),
|
||||
createdAt: Timestamp,
|
||||
updatedAt: Timestamp,
|
||||
});
|
||||
export type MediaScrape = z.infer<typeof MediaScrape>;
|
||||
|
||||
/** SCR-1 作品刮削状态(最近一次;表 media_scrape) */
|
||||
export const LibraryScrapeState = z.object({
|
||||
status: ScrapeStatus,
|
||||
trigger: ScrapeTrigger,
|
||||
mode: ScrapeMode,
|
||||
requestedAt: Timestamp,
|
||||
startedAt: Timestamp.nullable(),
|
||||
finishedAt: Timestamp.nullable(),
|
||||
written: NonNegInt,
|
||||
skipped: NonNegInt,
|
||||
failed: NonNegInt,
|
||||
/** 整体失败原因码(没有文件、没有元数据等) */
|
||||
reason: z.string().nullable(),
|
||||
files: z.array(ScrapeFileResult),
|
||||
/** 触发它的整理记录(入库后自动刮削) */
|
||||
recordId: Id.nullable(),
|
||||
/** 引擎 run id */
|
||||
taskId: z.string().nullable(),
|
||||
});
|
||||
export type LibraryScrapeState = z.infer<typeof LibraryScrapeState>;
|
||||
|
||||
export const LibraryScrapeRequest = z.object({
|
||||
/** 缺省按设置 organize.scrapeMode */
|
||||
mode: ScrapeMode.optional(),
|
||||
});
|
||||
export type LibraryScrapeRequest = z.infer<typeof LibraryScrapeRequest>;
|
||||
export const LibraryScrapeResult = z.object({
|
||||
mediaEntityId: Id,
|
||||
taskId: z.string(),
|
||||
/** 同一作品已有排队中的刮削,直接复用 */
|
||||
deduplicated: z.boolean(),
|
||||
});
|
||||
export type LibraryScrapeResult = z.infer<typeof LibraryScrapeResult>;
|
||||
|
||||
export const LibraryDetail = z.object({
|
||||
entity: MediaEntity,
|
||||
/** 身份映射状态(含 proposed/conflicted,前端区分展示) */
|
||||
@@ -403,6 +478,8 @@ export const LibraryDetail = z.object({
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
/** SCR-1:最近一次刮削(写 NFO 与图片);没刮削过为 null */
|
||||
scrape: LibraryScrapeState.nullable().optional(),
|
||||
});
|
||||
export type LibraryDetail = z.infer<typeof LibraryDetail>;
|
||||
|
||||
|
||||
@@ -87,6 +87,14 @@ const REASONS: Record<string, ReasonText> = {
|
||||
read_only_reconcile: { text: "只读对账中,暂不提交下载" },
|
||||
rejected: { text: "下载器拒绝了这个种子" },
|
||||
source_failed: { text: "从站点取种子失败", action: "检查站点登录后重新下载" },
|
||||
// SCR-1 刮削(作品刮削状态、逐文件结果)
|
||||
scrape_no_files: { text: "作品没有已入库文件", action: "先整理入库" },
|
||||
scrape_no_metadata: { text: "作品还没有元数据", action: "先刷新元数据" },
|
||||
scrape_root_read_only: { text: "媒体库目录是只登记模式,不写文件" },
|
||||
scrape_exists: { text: "文件已存在,按设置只补缺失" },
|
||||
scrape_write_failed: { text: "写入媒体库目录失败", action: "检查目录权限后重新刮削" },
|
||||
scrape_image_failed: { text: "图片没下载成功", action: "稍后重新刮削" },
|
||||
scrape_no_work_dir: { text: "视频直接放在媒体库目录下,不写剧集和季的文件" },
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -72,6 +72,14 @@ export const OrganizeSettings = z.object({
|
||||
allowCrossVolumeMove: z.boolean(),
|
||||
/** 整理 / 文件管理识别时视频文件的最小体积(字节),低于它的视为样片跳过;请求(向导)里给的值优先 */
|
||||
minSizeBytes: z.number().int().nonnegative(),
|
||||
/** SCR-1:整理入库成功后自动刮削(往媒体库目录写 NFO 与图片);缺省关(同 MP 目录「刮削」缺省) */
|
||||
scrapeAfterImport: z.boolean(),
|
||||
/** SCR-1:目标文件已存在时 missing = 只补缺失(MP「仅缺失」)/ overwrite = 全部覆盖 */
|
||||
scrapeMode: z.enum(["missing", "overwrite"]),
|
||||
/** SCR-1:下载海报、背景图、季海报 */
|
||||
scrapeImages: z.boolean(),
|
||||
/** SCR-1:下载每集剧照(`<视频名>-thumb.jpg`) */
|
||||
scrapeEpisodeThumbs: z.boolean(),
|
||||
});
|
||||
export type OrganizeSettings = z.infer<typeof OrganizeSettings>;
|
||||
|
||||
@@ -618,7 +626,15 @@ export const DEFAULT_SETTINGS: GlobalSettings = {
|
||||
globalImageCache: false,
|
||||
globalImageCacheDays: 7,
|
||||
},
|
||||
organize: { defaultMode: "copy", allowCrossVolumeMove: true, minSizeBytes: 50 * 1024 * 1024 },
|
||||
organize: {
|
||||
defaultMode: "copy",
|
||||
allowCrossVolumeMove: true,
|
||||
minSizeBytes: 50 * 1024 * 1024,
|
||||
scrapeAfterImport: false,
|
||||
scrapeMode: "missing",
|
||||
scrapeImages: true,
|
||||
scrapeEpisodeThumbs: true,
|
||||
},
|
||||
retention: { seedRetentionDays: 7, logRetentionDays: 180 },
|
||||
pi: {
|
||||
defaultUpstreamId: null,
|
||||
@@ -927,17 +943,22 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [
|
||||
{
|
||||
id: "library",
|
||||
label: "媒体库与目录",
|
||||
description: "媒体库目录、默认整理方式、命名模板",
|
||||
description: "媒体库目录、默认整理方式、刮削、命名模板",
|
||||
legacyTabs: [{ tab: "library-roots", anchor: "roots" }],
|
||||
anchors: [
|
||||
{ id: "roots", label: "媒体库目录" },
|
||||
{ id: "organize", label: "整理方式" },
|
||||
{ id: "scrape", label: "刮削" },
|
||||
{ id: "naming", label: "命名模板" },
|
||||
],
|
||||
fields: [
|
||||
"organize.defaultMode",
|
||||
"organize.allowCrossVolumeMove",
|
||||
"organize.minSizeBytes",
|
||||
"organize.scrapeAfterImport",
|
||||
"organize.scrapeMode",
|
||||
"organize.scrapeImages",
|
||||
"organize.scrapeEpisodeThumbs",
|
||||
"naming.movie",
|
||||
"naming.tv",
|
||||
"naming.variety",
|
||||
|
||||
@@ -63,6 +63,7 @@ const ROWS: Record<string, { row: Obj; dbOnly?: string[]; apiOnly?: string[] }>
|
||||
LibraryRoot: { row: C.LibraryRoot },
|
||||
MediaAsset: { row: C.MediaAsset },
|
||||
MediaEntity: { row: C.MediaEntity },
|
||||
MediaScrape: { row: C.MediaScrape },
|
||||
MediaServerInstance: { row: C.MediaServerInstance },
|
||||
Observation: { row: C.Observation },
|
||||
OrganizationFileResult: { row: C.OrganizationFileResult },
|
||||
@@ -209,6 +210,9 @@ const ENUMS: Record<string, z.ZodEnum> = {
|
||||
MediaAssetStatus: C.AssetStatus,
|
||||
MediaEntityKind: C.MediaKind,
|
||||
MediaEntityStatus: C.MediaEntityStatus,
|
||||
MediaScrapeMode: C.ScrapeMode,
|
||||
MediaScrapeStatus: C.ScrapeStatus,
|
||||
MediaScrapeTrigger: C.ScrapeTrigger,
|
||||
MediaServerInstanceKind: C.MediaServerKind,
|
||||
MediaServerInstanceHealthStatus: C.InstanceHealth,
|
||||
ObservationStatus: C.ObservationStatus,
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { execDdl } from "../src/migration-helpers.ts";
|
||||
|
||||
// SCR-1 刮削:每部作品最近一次刮削(往媒体库目录写 NFO 与图片)的状态与逐文件结果。
|
||||
// 一部作品一行,重新刮削覆盖;历史看审计(organize.scrape)与引擎运行记录。
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`create table if not exists media_scrape (
|
||||
media_entity_id uuid primary key references media_entity (id) on delete cascade,
|
||||
status text not null default 'queued' check (status in ('queued', 'running', 'succeeded', 'partial', 'failed')),
|
||||
trigger text not null default 'manual' check (trigger in ('auto', 'manual')),
|
||||
mode text not null default 'missing' check (mode in ('missing', 'overwrite')),
|
||||
record_id uuid references organization_record (id) on delete set null,
|
||||
engine_run_id text,
|
||||
requested_at timestamptz not null default now(),
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
written integer not null default 0,
|
||||
skipped integer not null default 0,
|
||||
failed integer not null default 0,
|
||||
reason text,
|
||||
files jsonb not null default '[]'::jsonb,
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now()
|
||||
);
|
||||
drop trigger if exists media_scrape_touch_updated_at on media_scrape;
|
||||
create trigger media_scrape_touch_updated_at before update on media_scrape
|
||||
for each row execute function umt_touch_updated_at();`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(db, "drop table if exists media_scrape;");
|
||||
}
|
||||
@@ -159,6 +159,9 @@ export type MediaAssetKind = "video" | "bluray_dir" | "subtitle" | "audio" | "nf
|
||||
export type MediaAssetStatus = "active" | "replaced" | "retired" | "missing";
|
||||
export type MediaEntityKind = "movie" | "series" | "variety" | "collection" | "other";
|
||||
export type MediaEntityStatus = "provisional" | "active" | "merged" | "archived";
|
||||
export type MediaScrapeStatus = "queued" | "running" | "succeeded" | "partial" | "failed";
|
||||
export type MediaScrapeTrigger = "auto" | "manual";
|
||||
export type MediaScrapeMode = "missing" | "overwrite";
|
||||
export type MediaServerInstanceKind = "emby" | "jellyfin";
|
||||
export type MediaServerInstanceHealthStatus = "unknown" | "healthy" | "degraded" | "offline";
|
||||
export type MetadataEnrichmentStatus = "suggested" | "applied" | "failed";
|
||||
@@ -915,6 +918,25 @@ export interface MediaEntityTable {
|
||||
naming_override: Json | null;
|
||||
}
|
||||
|
||||
export interface MediaScrapeTable {
|
||||
media_entity_id: string;
|
||||
status: Generated<MediaScrapeStatus>;
|
||||
trigger: Generated<MediaScrapeTrigger>;
|
||||
mode: Generated<MediaScrapeMode>;
|
||||
record_id: string | null;
|
||||
engine_run_id: string | null;
|
||||
requested_at: GeneratedTimestamptz;
|
||||
started_at: Timestamptz | null;
|
||||
finished_at: Timestamptz | null;
|
||||
written: Generated<number>;
|
||||
skipped: Generated<number>;
|
||||
failed: Generated<number>;
|
||||
reason: string | null;
|
||||
files: Generated<Json>;
|
||||
created_at: GeneratedTimestamptz;
|
||||
updated_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface MediaServerInstanceTable {
|
||||
id: Generated<string>;
|
||||
name: string;
|
||||
@@ -1698,6 +1720,7 @@ export interface DB {
|
||||
mcp_server_config: McpServerConfigTable;
|
||||
media_asset: MediaAssetTable;
|
||||
media_entity: MediaEntityTable;
|
||||
media_scrape: MediaScrapeTable;
|
||||
media_server_instance: MediaServerInstanceTable;
|
||||
metadata_enrichment: MetadataEnrichmentTable;
|
||||
metadata_field: MetadataFieldTable;
|
||||
@@ -1858,6 +1881,9 @@ export type MediaAssetUpdate = Updateable<MediaAssetTable>;
|
||||
export type MediaEntityRow = Selectable<MediaEntityTable>;
|
||||
export type NewMediaEntity = Insertable<MediaEntityTable>;
|
||||
export type MediaEntityUpdate = Updateable<MediaEntityTable>;
|
||||
export type MediaScrapeRow = Selectable<MediaScrapeTable>;
|
||||
export type NewMediaScrape = Insertable<MediaScrapeTable>;
|
||||
export type MediaScrapeUpdate = Updateable<MediaScrapeTable>;
|
||||
export type MediaServerInstanceRow = Selectable<MediaServerInstanceTable>;
|
||||
export type NewMediaServerInstance = Insertable<MediaServerInstanceTable>;
|
||||
export type MediaServerInstanceUpdate = Updateable<MediaServerInstanceTable>;
|
||||
|
||||
@@ -26,6 +26,7 @@ const EXPECTED_TABLES = [
|
||||
"path_mapping",
|
||||
// 内容 / 身份
|
||||
"media_entity",
|
||||
"media_scrape",
|
||||
"content_unit",
|
||||
"title_alias",
|
||||
"external_identity",
|
||||
@@ -153,6 +154,8 @@ const NON_ENTITY_TABLES = [
|
||||
"kv_cache",
|
||||
// 站点请求计数:(site_id, hour) 主键
|
||||
"site_request_count",
|
||||
// 作品最近一次刮削:media_entity_id 主键(SCR-1)
|
||||
"media_scrape",
|
||||
];
|
||||
|
||||
describe.skipIf(skip)("schema migrations (real PG)", () => {
|
||||
|
||||
Reference in new issue
Block a user