feat(scrape): 任意目录就地刮削,按路径识别作品,每次刮削一条记录(SCR-2)
- POST /files/scrape 改为对目录 / 视频文件刮削:已登记用作品,否则按路径识别(本地 + TMDB), 认不准返回候选,选定后再提交;未登记直接用 TMDB 字段,addToLibrary 顺手建作品 - 作品详情重新刮削、入库后自动刮削改为对作品目录刮削,同一实现 - 迁移 0058_scrape_record 替代 media_scrape;GET /files/scrape-records - 与 MP ScrapingChain 的文件布局差分(7 个场景,3 处登记偏离) Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
f60254e265
commit
87bf33b42b
29 files changed
+2272
-653
No files matched your search
@@ -0,0 +1,72 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { execDdl } from "../src/migration-helpers.ts";
|
||||
|
||||
// SCR-2 刮削照 MP 简化:刮削对象从"作品"改为"目录 / 视频文件"(不要求先登记),每次刮削一条记录:
|
||||
// 目录(登记目录 id + 相对路径)、识别结果(本地作品 / TMDB 条目)、写了哪些文件、跳过 / 失败原因。
|
||||
// 替代 SCR-1 的 media_scrape(一部作品一行);作品详情取该作品最近一条记录。
|
||||
|
||||
const MEDIA_SCRAPE = `create table if not exists media_scrape (
|
||||
media_entity_id uuid primary key references media_entity (id) on delete cascade,
|
||||
status text not null default 'queued' check (status in ('queued', 'running', 'succeeded', 'partial', 'failed')),
|
||||
trigger text not null default 'manual' check (trigger in ('auto', 'manual')),
|
||||
mode text not null default 'missing' check (mode in ('missing', 'overwrite')),
|
||||
record_id uuid references organization_record (id) on delete set null,
|
||||
engine_run_id text,
|
||||
requested_at timestamptz not null default now(),
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
written integer not null default 0,
|
||||
skipped integer not null default 0,
|
||||
failed integer not null default 0,
|
||||
reason text,
|
||||
files jsonb not null default '[]'::jsonb,
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now()
|
||||
);
|
||||
drop trigger if exists media_scrape_touch_updated_at on media_scrape;
|
||||
create trigger media_scrape_touch_updated_at before update on media_scrape
|
||||
for each row execute function umt_touch_updated_at();`;
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`create table if not exists scrape_record (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
root_id text not null,
|
||||
root_path text not null,
|
||||
library_root_id uuid references library_root (id) on delete set null,
|
||||
path text not null default '',
|
||||
media_entity_id uuid references media_entity (id) on delete set null,
|
||||
match jsonb,
|
||||
status text not null default 'queued' check (status in ('queued', 'running', 'succeeded', 'partial', 'failed')),
|
||||
trigger text not null default 'manual' check (trigger in ('auto', 'manual')),
|
||||
mode text not null default 'missing' check (mode in ('missing', 'overwrite')),
|
||||
organization_record_id uuid references organization_record (id) on delete set null,
|
||||
engine_run_id text,
|
||||
actor_type text,
|
||||
actor_id text,
|
||||
requested_at timestamptz not null default now(),
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
written integer not null default 0,
|
||||
skipped integer not null default 0,
|
||||
failed integer not null default 0,
|
||||
reason text,
|
||||
files jsonb not null default '[]'::jsonb,
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now()
|
||||
);
|
||||
create index if not exists scrape_record_entity_idx on scrape_record (media_entity_id, requested_at desc);
|
||||
create index if not exists scrape_record_path_idx on scrape_record (root_id, path, requested_at desc);
|
||||
drop trigger if exists scrape_record_touch_updated_at on scrape_record;
|
||||
create trigger scrape_record_touch_updated_at before update on scrape_record
|
||||
for each row execute function umt_touch_updated_at();
|
||||
drop table if exists media_scrape;`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(db, `drop table if exists scrape_record; ${MEDIA_SCRAPE}`);
|
||||
}
|
||||
+34
-26
@@ -159,9 +159,6 @@ export type MediaAssetKind = "video" | "bluray_dir" | "subtitle" | "audio" | "nf
|
||||
export type MediaAssetStatus = "active" | "replaced" | "retired" | "missing";
|
||||
export type MediaEntityKind = "movie" | "series" | "variety" | "collection" | "other";
|
||||
export type MediaEntityStatus = "provisional" | "active" | "merged" | "archived";
|
||||
export type MediaScrapeStatus = "queued" | "running" | "succeeded" | "partial" | "failed";
|
||||
export type MediaScrapeTrigger = "auto" | "manual";
|
||||
export type MediaScrapeMode = "missing" | "overwrite";
|
||||
export type MediaServerInstanceKind = "emby" | "jellyfin";
|
||||
export type MediaServerInstanceHealthStatus = "unknown" | "healthy" | "degraded" | "offline";
|
||||
export type MetadataEnrichmentStatus = "suggested" | "applied" | "failed";
|
||||
@@ -226,6 +223,9 @@ export type ScheduledJobKind = "task" | "agent_check";
|
||||
export type ScheduledJobLastStatus = "ok" | "failed" | "skipped" | "running";
|
||||
export type ScheduledJobRunTrigger = "schedule" | "manual" | "skip";
|
||||
export type ScheduledJobRunStatus = "ok" | "failed" | "skipped" | "running";
|
||||
export type ScrapeRecordStatus = "queued" | "running" | "succeeded" | "partial" | "failed";
|
||||
export type ScrapeRecordTrigger = "auto" | "manual";
|
||||
export type ScrapeRecordMode = "missing" | "overwrite";
|
||||
export type SecretPurpose =
|
||||
| "site_cookie"
|
||||
| "site_passkey"
|
||||
@@ -918,25 +918,6 @@ export interface MediaEntityTable {
|
||||
naming_override: Json | null;
|
||||
}
|
||||
|
||||
export interface MediaScrapeTable {
|
||||
media_entity_id: string;
|
||||
status: Generated<MediaScrapeStatus>;
|
||||
trigger: Generated<MediaScrapeTrigger>;
|
||||
mode: Generated<MediaScrapeMode>;
|
||||
record_id: string | null;
|
||||
engine_run_id: string | null;
|
||||
requested_at: GeneratedTimestamptz;
|
||||
started_at: Timestamptz | null;
|
||||
finished_at: Timestamptz | null;
|
||||
written: Generated<number>;
|
||||
skipped: Generated<number>;
|
||||
failed: Generated<number>;
|
||||
reason: string | null;
|
||||
files: Generated<Json>;
|
||||
created_at: GeneratedTimestamptz;
|
||||
updated_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface MediaServerInstanceTable {
|
||||
id: Generated<string>;
|
||||
name: string;
|
||||
@@ -1345,6 +1326,33 @@ export interface ScheduledJobRunTable {
|
||||
created_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface ScrapeRecordTable {
|
||||
id: Generated<string>;
|
||||
root_id: string;
|
||||
root_path: string;
|
||||
library_root_id: string | null;
|
||||
path: Generated<string>;
|
||||
media_entity_id: string | null;
|
||||
match: Json | null;
|
||||
status: Generated<ScrapeRecordStatus>;
|
||||
trigger: Generated<ScrapeRecordTrigger>;
|
||||
mode: Generated<ScrapeRecordMode>;
|
||||
organization_record_id: string | null;
|
||||
engine_run_id: string | null;
|
||||
actor_type: string | null;
|
||||
actor_id: string | null;
|
||||
requested_at: GeneratedTimestamptz;
|
||||
started_at: Timestamptz | null;
|
||||
finished_at: Timestamptz | null;
|
||||
written: Generated<number>;
|
||||
skipped: Generated<number>;
|
||||
failed: Generated<number>;
|
||||
reason: string | null;
|
||||
files: Generated<Json>;
|
||||
created_at: GeneratedTimestamptz;
|
||||
updated_at: GeneratedTimestamptz;
|
||||
}
|
||||
|
||||
export interface SecretTable {
|
||||
id: Generated<string>;
|
||||
name: string;
|
||||
@@ -1720,7 +1728,6 @@ export interface DB {
|
||||
mcp_server_config: McpServerConfigTable;
|
||||
media_asset: MediaAssetTable;
|
||||
media_entity: MediaEntityTable;
|
||||
media_scrape: MediaScrapeTable;
|
||||
media_server_instance: MediaServerInstanceTable;
|
||||
metadata_enrichment: MetadataEnrichmentTable;
|
||||
metadata_field: MetadataFieldTable;
|
||||
@@ -1744,6 +1751,7 @@ export interface DB {
|
||||
rule_group: RuleGroupTable;
|
||||
scheduled_job: ScheduledJobTable;
|
||||
scheduled_job_run: ScheduledJobRunTable;
|
||||
scrape_record: ScrapeRecordTable;
|
||||
secret: SecretTable;
|
||||
server_mapping: ServerMappingTable;
|
||||
settings: SettingsTable;
|
||||
@@ -1881,9 +1889,6 @@ export type MediaAssetUpdate = Updateable<MediaAssetTable>;
|
||||
export type MediaEntityRow = Selectable<MediaEntityTable>;
|
||||
export type NewMediaEntity = Insertable<MediaEntityTable>;
|
||||
export type MediaEntityUpdate = Updateable<MediaEntityTable>;
|
||||
export type MediaScrapeRow = Selectable<MediaScrapeTable>;
|
||||
export type NewMediaScrape = Insertable<MediaScrapeTable>;
|
||||
export type MediaScrapeUpdate = Updateable<MediaScrapeTable>;
|
||||
export type MediaServerInstanceRow = Selectable<MediaServerInstanceTable>;
|
||||
export type NewMediaServerInstance = Insertable<MediaServerInstanceTable>;
|
||||
export type MediaServerInstanceUpdate = Updateable<MediaServerInstanceTable>;
|
||||
@@ -1953,6 +1958,9 @@ export type ScheduledJobUpdate = Updateable<ScheduledJobTable>;
|
||||
export type ScheduledJobRunRow = Selectable<ScheduledJobRunTable>;
|
||||
export type NewScheduledJobRun = Insertable<ScheduledJobRunTable>;
|
||||
export type ScheduledJobRunUpdate = Updateable<ScheduledJobRunTable>;
|
||||
export type ScrapeRecordRow = Selectable<ScrapeRecordTable>;
|
||||
export type NewScrapeRecord = Insertable<ScrapeRecordTable>;
|
||||
export type ScrapeRecordUpdate = Updateable<ScrapeRecordTable>;
|
||||
export type SecretRow = Selectable<SecretTable>;
|
||||
export type NewSecret = Insertable<SecretTable>;
|
||||
export type SecretUpdate = Updateable<SecretTable>;
|
||||
|
||||
@@ -26,7 +26,6 @@ const EXPECTED_TABLES = [
|
||||
"path_mapping",
|
||||
// 内容 / 身份
|
||||
"media_entity",
|
||||
"media_scrape",
|
||||
"content_unit",
|
||||
"title_alias",
|
||||
"external_identity",
|
||||
@@ -124,6 +123,8 @@ const EXPECTED_TABLES = [
|
||||
"kv_cache",
|
||||
// 站点请求计数(0046,按小时聚合)
|
||||
"site_request_count",
|
||||
// SCR-2 刮削记录(0058,替代 0057 media_scrape)
|
||||
"scrape_record",
|
||||
].sort();
|
||||
|
||||
/** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */
|
||||
@@ -154,8 +155,6 @@ const NON_ENTITY_TABLES = [
|
||||
"kv_cache",
|
||||
// 站点请求计数:(site_id, hour) 主键
|
||||
"site_request_count",
|
||||
// 作品最近一次刮削:media_entity_id 主键(SCR-1)
|
||||
"media_scrape",
|
||||
];
|
||||
|
||||
describe.skipIf(skip)("schema migrations (real PG)", () => {
|
||||
|
||||
Reference in new issue
Block a user