feat: v0.3.8–v0.3.14 工作区汇总(依赖升级、整理来源、手动入库、原地入库等)

生产 v0.3.8~v0.3.13 均由 8013196 加工作区修改构建部署,此前未提交;本提交补齐源码,
并含未部署的 v0.3.14 原地入库。中间各版本源码状态未保存,无法逐版拆分,详见
docs/plan/17-full-upgrade.md 与 docs/plan/18-organize-release.md。

- v0.3.8 全面升级:Deno workspace 依赖、Router 8 / Vite 8、camofox 1.18.0 / Node 24、
  PostgreSQL 18.6、备份客户端 18.6、演示 qB 5.2.4
- v0.3.9 媒体整理:分类规则、整理来源与目录快照(0059)、文件管理整理入口
- v0.3.10 手动下载与入库:独立手动下载、详情页资源下载、TMDB / Local 元数据兜底
  (metadata.resolve_organize)、正常多版本 / 一次性洗版(0060)
- v0.3.11 搜索详情与桌面黑边修复
- v0.3.12 列表分页、搜索历史、首页概况
- v0.3.13 统一整理弹窗与队列
- v0.3.14 原地入库(0061,未部署)

同批审计修复:
- 刮削入队:确认原记录存在后才删重放产生的新记录;写回 run 关联失败时保持 queued,
  由处理器补关联(对齐 MP 先登记、只在入队失败时记失败)
- 识别升级链只在结果更确定时采用,不再用 no_match 顶替原有歧义候选
- bluray_dir 资产以原盘目录本身为刮削根(MP 同做法)
- metadata.resolve_organize 登记到入库排障 Skill 与能力地图
- db/src/types.ts 按迁移重新生成;contracts 整理动作枚举顺序与数据库对齐
- 测试:原地入库用例改为直接查记录表;domain 测试模板补 version
- 前端:通知铃铛弹层与用户菜单统一样式;站点卡片共用卡头、底栏贴底、分页间距

依赖升级证据只提交摘要文件,registry-snapshot.json 移至 .local/(SHA-256 记于文档)。

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed 2026-10-02 15:47:17 +08:00
1 parent 8013196e3f
commit ceb66f8c77
265 files changed
+17751 -2483

No files matched your search

@@ -21,6 +21,7 @@ import {
} from "@umt/domain";
import { localIdentityCandidate } from "@umt/mapping";
import type { Principal } from "../../http/auth.ts";
import { successfulOrganizeCounts } from "../acquisition-organize/services/organize-history.ts";
import { EXTERNAL_INVOKE_TIMEOUT_MS, sharedInvokeLimiter } from "../agent/invoke-limits.ts";
import { actorOfP, errMessage, FilesError, type FmDeps, resolveIn } from "./common.ts";
import { kindOfName } from "./list.ts";
@@ -218,6 +219,10 @@ export async function identifyCore(
order by f.source_path, r.created_at desc`.execute(db)
: { rows: [] };
const recordsBy = new Map(recordRows.rows.map((r) => [r.source_path, r]));
const successfulCounts = await successfulOrganizeCounts(
db,
videos.map((v) => v.abs),
);
const localCache = new Map<string, IdentityCandidate[]>();
const remoteCache = new Map<string, { list: IdentityCandidate[]; warning: string | null; pending?: boolean }>();
const entityKinds = new Map<string, string>();
@@ -371,6 +376,7 @@ export async function identifyCore(
count: rec?.n ?? 0,
lastRecordId: rec?.last_id ?? null,
lastStatus: (rec?.last_status ?? null) as OrganizationStatus | null,
successfulCount: successfulCounts.get(v.abs) ?? 0,
};
const parsed0 = parseReleasePath(segments(v.rel));
const parsed: ParsedRelease = {
+23 -1
View File
@@ -10,7 +10,9 @@ import { SourcesRuntime } from "../sources-identity/runtime.ts";
import type { ServerContext, ServerModule } from "../types.ts";
import { defaultOptions, type FmDeps, type FmOptions } from "./common.ts";
import { IDENTIFY_REMOTE_TASK, runIdentifyRemote } from "./identify.ts";
import { registerOrganizeSourceRoutes } from "./routes/organize-sources.ts";
import { registerFileRoutes } from "./routes.ts";
import { queueDueSourceScans, scanOrganizeSource } from "./source-monitor.ts";
import { purgeTrash } from "./trash.ts";
/**
@@ -52,9 +54,18 @@ export function createFileManagerModule(overrides: FmOverrides = {}): ServerModu
},
registerRoutes(app, ctx) {
registerFileRoutes(app, lazy(ctx));
registerOrganizeSourceRoutes(app, lazy(ctx));
},
registerTasks(registry, ctx) {
registry.register(
defineTask({
name: "scanOrganizeSource",
version: 1,
lane: "maintenance",
inputSchema: z.object({ sourceId: z.string().uuid() }),
retryPolicy: { maximumAttempts: 3, initialInterval: "5s" },
handler: async (t) => t.step.run("scan", () => scanOrganizeSource(lazy(ctx), t.input)),
}),
defineTask({
name: "purgeTrash",
version: 1,
@@ -84,7 +95,18 @@ export function createFileManagerModule(overrides: FmOverrides = {}): ServerModu
},
start(ctx) {
const d = lazy(ctx);
if (!d.options.backgroundLoops || d.options.purgeIntervalMs <= 0) return;
if (!d.options.backgroundLoops) return;
registerScheduledJob(ctx, {
key: "organize.sources",
defaults: {
name: "整理来源扫描",
description: "检查来源目录的新文件",
group: "maintain",
schedule: { type: "interval", everyMs: 60_000 },
},
run: async (c) => queueDueSourceScans(d, c.slot),
});
if (d.options.purgeIntervalMs <= 0) return;
registerScheduledJob(ctx, {
key: "trash.purge",
defaults: {
@@ -0,0 +1,112 @@
import { randomUUID } from "node:crypto";
import { constants } from "node:fs";
import { access } from "node:fs/promises";
import type { OrganizeSourceConfig } from "@umt/contracts";
import { sql } from "@umt/db";
import { SETTINGS_CHANGED_EVENT } from "../auth-settings/provided.ts";
import { FilesError, type FmDeps, jsonb, resolveIn } from "./common.ts";
import { isWritableTarget } from "./list.ts";
export async function sourceConfigs(d: FmDeps): Promise<OrganizeSourceConfig[]> {
const org = await d.ctx.settings.get("organize");
return (org as { sources?: OrganizeSourceConfig[] }).sources ?? [];
}
export async function sourceConfig(d: FmDeps, id: string): Promise<OrganizeSourceConfig> {
const found = (await sourceConfigs(d)).find((s) => s.id === id);
if (!found) throw new FilesError("not_found", "整理来源不存在");
return found;
}
const iso = (v: Date | string | null | undefined) => (v ? new Date(v).toISOString() : null);
export async function listOrganizeSources(d: FmDeps) {
const rows = await d.ctx.db.selectFrom("organize_source_snapshot").selectAll().execute();
return {
items: (await sourceConfigs(d)).map((s) => {
const row = rows.find((r) => r.source_id === s.id);
const files = (row?.snapshot as { files?: Record<string, unknown> } | undefined)?.files ?? {};
return {
...s,
baselineAt: iso(row?.baseline_at),
lastScanAt: iso(row?.last_scan_at),
lastError: row?.last_error ?? null,
fileCount: Object.keys(files).length,
};
}),
};
}
async function editSources(
d: FmDeps,
edit: (items: OrganizeSourceConfig[]) => OrganizeSourceConfig[],
resetId?: string,
lockId = resetId,
) {
const defaults = await d.ctx.settings.get("organize");
await d.ctx.db.transaction().execute(async (trx) => {
if (lockId) await sql`select pg_advisory_xact_lock(hashtext(${`organize-source:${lockId}`}))`.execute(trx);
// Serialize even the first insert, where no settings row exists to lock.
await sql`select pg_advisory_xact_lock(hashtext('organize.sources'))`.execute(trx);
const row = await trx
.selectFrom("settings")
.select("value")
.where("key", "=", "organize")
.forUpdate()
.executeTakeFirst();
const org = (row?.value ?? defaults) as typeof defaults & { sources?: OrganizeSourceConfig[] };
await trx
.insertInto("settings")
.values({ key: "organize", value: jsonb({ ...org, sources: edit(org.sources ?? []) }), updated_at: new Date() })
.onConflict((oc) =>
oc
.column("key")
.doUpdateSet({ value: jsonb({ ...org, sources: edit(org.sources ?? []) }), updated_at: new Date() }),
)
.execute();
if (resetId) await trx.deleteFrom("organize_source_snapshot").where("source_id", "=", resetId).execute();
});
const settings = d.ctx.settings as typeof d.ctx.settings & { invalidate?: () => void };
settings.invalidate?.();
await d.ctx.events.publish(SETTINGS_CHANGED_EVENT, { keys: ["organize"] });
}
export async function saveOrganizeSource(d: FmDeps, id: string, input: Omit<OrganizeSourceConfig, "id">) {
const source = await resolveIn(d, input.source.root, input.source.path);
if (source.isSymlink || !source.lst?.isDirectory()) throw new FilesError("bad_request", "来源必须为可访问的目录");
const target = await resolveIn(d, input.target.root, input.target.path);
if (
target.isSymlink ||
!target.lst?.isDirectory() ||
!(await isWritableTarget(target.def, target.rootReal)) ||
!(await access(target.abs, constants.W_OK).then(
() => true,
() => false,
))
)
throw new FilesError("not_writable", "目标必须为可写的纳管媒体库目录");
if (input.trigger === "qb_tag") {
if (!input.downloaderInstanceId || !input.tag?.trim())
throw new FilesError("bad_request", "标签触发需要下载器和标签");
const instance = await d.ctx.db
.selectFrom("downloader_instance")
.select("id")
.where("id", "=", input.downloaderInstanceId)
.executeTakeFirst();
if (!instance || (source.def.instanceId && source.def.instanceId !== instance.id))
throw new FilesError("bad_request", "来源和下载器不匹配");
}
const previous = (await sourceConfigs(d)).find((s) => s.id === id);
const reset = !previous || previous.source.root !== input.source.root || previous.source.path !== input.source.path;
const config = { ...input, id };
await editSources(d, (items) => [...items.filter((s) => s.id !== id), config], reset ? id : undefined, id);
if (reset)
await d.ctx.engine.enqueue(
"scanOrganizeSource",
{ sourceId: id },
{ idempotencyKey: `source-baseline:${id}:${randomUUID()}`, traceId: randomUUID() },
);
return (await listOrganizeSources(d)).items.find((s) => s.id === id)!;
}
export async function deleteOrganizeSource(d: FmDeps, id: string) {
const deleted = (await sourceConfigs(d)).some((s) => s.id === id);
await editSources(d, (items) => items.filter((s) => s.id !== id), id);
return { deleted };
}
@@ -29,7 +29,7 @@ type Req = {
sourcePaths: string[];
mediaEntityId: string;
contentUnitId?: string | undefined;
action: "copy" | "move";
action: "copy" | "move" | "in_place";
targetLibraryRootId?: string | undefined;
};
@@ -45,7 +45,7 @@ interface Frozen {
interface Group {
entity: string;
unit: string | null;
action: "copy" | "move";
action: "copy" | "move" | "in_place";
targetDir: string | null;
targetLibraryRootId: string | null;
slotId: string | null;
@@ -67,7 +67,7 @@ function filesOf(plan: FrozenPlan | undefined): GroupFile[] {
const heldByExisting = new Set(
plan.organize.conflicts.filter((c) => c.kind === "target_exists" && c.blocking).map((c) => c.src),
);
return plan.organize.files.map((f) => {
const active = plan.organize.files.map((f) => {
const existing: GroupFile["existing"] =
f.action === "replace"
? "replace"
@@ -88,16 +88,36 @@ function filesOf(plan: FrozenPlan | undefined): GroupFile[] {
skipReason: f.skipReason ?? null,
};
});
return [
...active,
...(plan.skippedSuccessful ?? []).map((f) => ({
source: f.src,
target: f.dst ?? null,
kind: f.kind,
size: f.size,
action: "skip" as const,
existing: null,
skipReason: "successful_record",
})),
];
}
/** 被排除的文件(G2:too_small → below_min_size) */
function excludedOf(plan: FrozenPlan | undefined): NonNullable<FilesOrganizeGroup["excluded"]> {
return (plan?.organize.excluded ?? []).map((x) => ({
source: x.src,
kind: x.kind,
size: x.reason === "too_small" && x.detail && /^\d+$/.test(x.detail) ? Number(x.detail) : null,
reason: x.reason === "too_small" ? "below_min_size" : x.reason,
}));
return [
...(plan?.organize.excluded ?? []).map((x) => ({
source: x.src,
kind: x.kind,
size: x.reason === "too_small" && x.detail && /^\d+$/.test(x.detail) ? Number(x.detail) : null,
reason: x.reason === "too_small" ? "below_min_size" : x.reason,
})),
...(plan?.skippedSuccessful ?? []).map((f) => ({
source: f.src,
kind: f.kind,
size: f.size,
reason: "successful_record",
})),
];
}
/** 目标目录分段(相对库根;G4):剧集末段为季目录;bound = 沿用已有路径绑定(本次不新建) */
@@ -105,7 +125,8 @@ function segmentsOf(plan: FrozenPlan | undefined, overrideDir: boolean): FilesOr
const dir = plan?.targetDirs[0];
if (!plan || !dir) return [];
const parts = relative(plan.rootPath, dir).split("/").filter(Boolean);
if (overrideDir) return parts.map((text) => ({ text, role: "dir" as const, bound: false }));
if (overrideDir || plan.organize.operation === "in_place")
return parts.map((text) => ({ text, role: "dir" as const, bound: false }));
const newEntity = plan.newBindings.some((b) => b.scope === "entity");
const newSeason = plan.newBindings.some((b) => b.scope === "season");
const rootIdx = plan.mediaType === "tv" ? parts.length - 2 : parts.length - 1;
@@ -152,11 +173,30 @@ export async function organizeFiles(
const actorType = principal?.kind === "api_key" ? "api_key" : "user";
const actor = actorOfP(principal);
// G1:外部候选 → 本地作品(就地建,幂等);每项解析出 mediaEntityId
const created = new Map<string, { entityId: string; created: boolean; title: string; externalId: string }>();
// 先核验整批人工集号,避免无效输入创建外部作品或冻结计划。数字范围由请求 schema 校验。
const localIds = [...new Set(body.items.flatMap((it) => (it.mediaEntityId ? [it.mediaEntityId] : [])))];
const localKinds = new Map(
localIds.length
? (await db.selectFrom("media_entity").select(["id", "kind"]).where("id", "in", localIds).execute()).map((e) => [
e.id,
e.kind,
])
: [],
);
for (const it of body.items) {
if (!!it.mediaEntityId === !!it.external)
throw new FilesError("bad_request", `整理项 ${it.path || "/"} 需要且只能指定 mediaEntityId 或 external 之一`);
const kind = it.external?.kind ?? localKinds.get(it.mediaEntityId!);
if (!it.episodes?.length || kind === undefined || kind === "movie") continue;
if (!episodeRange(it.episodes))
throw new FilesError("bad_request", `${it.path}:集号 ${it.episodes.join(",")} 不连续,只支持连续区间`);
const r = await resolveIn(d, it.root, it.path);
if (r.lst?.isDirectory()) throw new FilesError("bad_request", `${r.rel}:目录项不能指定集号,请展开后逐个视频指定`);
}
// G1:外部候选 → 本地作品(就地建,幂等);每项解析出 mediaEntityId
const created = new Map<string, { entityId: string; created: boolean; title: string; externalId: string }>();
for (const it of body.items) {
if (!it.external) continue;
const k = externalKey(it.external);
if (created.has(k)) continue;
@@ -228,7 +268,13 @@ export async function organizeFiles(
// 目标目录覆写:writable 库根下已存在的目录
let targetDir: string | null = null;
let targetLib: string | null = body.targetLibraryRootId ?? null;
if (it.targetDir) {
if (action === "in_place") {
// 本次保留实际库根与路径,不使用来源配置的搬运目标。
if (!r.def.libraryRootId)
throw new FilesError("bad_request", "原地入库需要文件位于已登记的媒体库目录;请先在设置中登记该目录");
targetLib = r.def.libraryRootId;
}
if (action !== "in_place" && it.targetDir) {
const t = await resolveIn(d, it.targetDir.root, it.targetDir.path);
if (!t.lst?.isDirectory() || t.isSymlink || !t.def.libraryRootId)
throw new FilesError("bad_request", `目标目录必须是媒体库目录下已存在的目录:${it.targetDir.path || "/"}`);
@@ -244,9 +290,9 @@ export async function organizeFiles(
if (it.season !== undefined) ov.season = it.season;
if (it.episodes?.length) {
const range = episodeRange(it.episodes);
if (!range) warnings.push(`${r.rel}:集号 ${it.episodes.join(",")} 不连续,未应用(只支持连续区间)`);
else if (isDir) warnings.push(`${r.rel}:目录项不能指定集号(会套到其下所有视频),未应用`);
else ov.episodes = range;
if (!range) throw new FilesError("bad_request", `${r.rel}:集号不连续,只支持连续区间`);
if (isDir) throw new FilesError("bad_request", `${r.rel}:目录项不能指定集号,请展开后逐个视频指定`);
ov.episodes = range;
}
}
}
@@ -339,6 +385,7 @@ export async function organizeFiles(
const hasOverrides = Object.keys(g.overrides).length > 0;
const reqHash = hashJson({
key: requestKey(req),
...(body.sourceConfigId && g.action !== "in_place" ? { sourceConfigId: body.sourceConfigId } : {}),
...(hasOverrides ? { overrides: g.overrides } : {}),
...(g.targetDir ? { targetDir: g.targetDir } : {}),
...(g.slotId ? { slotId: g.slotId } : {}),
@@ -346,6 +393,8 @@ export async function organizeFiles(
...(minSizeBytes !== undefined ? { minSizeBytes } : {}),
...(onExisting !== "conflict" ? { onExisting } : {}),
...(namingMode !== "template" ? { naming: namingMode } : {}),
...(body.skipSuccessfulRecords ? { skipSuccessfulRecords: true } : {}),
...(body.scrapeAfterImport !== undefined ? { scrapeAfterImport: body.scrapeAfterImport } : {}),
});
const ext = fromExternal.get(g.entity);
const base = {
@@ -363,6 +412,7 @@ export async function organizeFiles(
};
const importReq = (dryRun: boolean, taskId?: string) => ({
...req,
...(body.sourceConfigId && g.action !== "in_place" ? { sourceConfigId: body.sourceConfigId } : {}),
...(hasOverrides ? { fileOverrides: g.overrides } : {}),
...(g.targetDir ? { targetDir: g.targetDir } : {}),
...(g.slotId ? { slotId: g.slotId } : {}),
@@ -371,6 +421,8 @@ export async function organizeFiles(
// 向导里显式选择"替换"即为人工授权;替换走 replace_with_verify(新文件核验通过后旧文件进回收站)
...(onExisting === "replace_with_verify" ? { grantReplace: true } : {}),
namingMode,
skipSuccessfulRecords: body.skipSuccessfulRecords,
scrapeAfterImport: body.scrapeAfterImport,
trigger: "manual" as const,
actorType: actorType as "api_key" | "user",
actorId: actor,
@@ -421,6 +473,20 @@ export async function organizeFiles(
const nowPairs = pairsOf(again).map(({ src, dst }) => ({ src, dst }));
if (hashJson(nowPairs) !== hashJson(frozen.pairs))
throw new FilesError("plan_stale", "目标目录或绑定已变化,与预览不一致。重新预览");
if (again.plan?.skippedSuccessful?.length && !again.plan.organize.files.length) {
await retirePlan(d, planId);
out.push({
...base,
...planExtras(again.plan, !!g.targetDir),
planId,
files: filesOf(again.plan),
truncated: false,
recordId: null,
taskId: null,
error: null,
});
continue;
}
if (again.status === "failed") throw new FilesError("bad_request", `计划不可执行:${again.reason ?? "blocked"}`);
if (g.action === "move") {
@@ -116,7 +116,7 @@ export function registerFileRoutes(app: ModuleApp, d: FmDeps) {
mount(app, F.scrape, async (c, { body }) =>
idem(d, c, "scrape", body, () => scrapeFiles(d, c.get("principal"), body)),
);
mount(app, F.scrapeRecords, async (_c, { query }) => scrapeRecords(d, query.ids));
mount(app, F.scrapeRecords, async (_c, { query }) => scrapeRecords(d, query.ids, query.organizationRecordId));
mount(app, F.probe, async (_c, { body }) => probeFiles(d, body));
mount(app, F.records, async (_c, { query }) => findRecords(d, query));
mount(app, F.download, async (c, { query }) => downloadFile(d, c.get("principal"), query), true);
@@ -0,0 +1,24 @@
import { Api } from "@umt/contracts";
import { ApiException } from "../../../http/errors.ts";
import { endpoint } from "../../auth-settings/endpoint.ts";
import type { ModuleApp } from "../../types.ts";
import { type FmDeps, fsErrorToFiles } from "../common.ts";
import { deleteOrganizeSource, listOrganizeSources, saveOrganizeSource } from "../organize-sources.ts";
import { scanStock } from "../source-monitor.ts";
async function guarded<T>(fn: () => Promise<T>): Promise<T> {
try {
return await fn();
} catch (error) {
const converted = fsErrorToFiles(error);
if (converted) throw new ApiException(converted.code, `${converted.reason}: ${converted.message}`);
throw error;
}
}
export function registerOrganizeSourceRoutes(app: ModuleApp, d: FmDeps) {
const defs = Api.organizeSources;
endpoint(app, defs.list, () => guarded(() => listOrganizeSources(d)));
endpoint(app, defs.save, ({ params, body }) => guarded(() => saveOrganizeSource(d, params.id, body)));
endpoint(app, defs.remove, ({ params }) => guarded(() => deleteOrganizeSource(d, params.id)));
endpoint(app, defs.stock, ({ params, body }) => guarded(() => scanStock(d, params.id, body)));
}
+18 -3
View File
@@ -204,8 +204,23 @@ export async function scrapeFiles(
};
}
export async function scrapeRecords(d: FmDeps, idsParam: string) {
const ids = [...new Set(idsParam.split(",").map((s) => s.trim()))].filter((s) => /^[0-9a-f-]{36}$/i.test(s));
if (!ids.length || ids.length > 100) throw new FilesError("bad_request", "ids 需要 1–100 个刮削记录 id");
export async function scrapeRecords(d: FmDeps, idsParam?: string, organizationRecordId?: string) {
let ids =
idsParam === undefined
? []
: [...new Set(idsParam.split(",").map((s) => s.trim()))].filter((s) => /^[0-9a-f-]{36}$/i.test(s));
if (idsParam !== undefined && (!ids.length || ids.length > 100))
throw new FilesError("bad_request", "ids 需要 1–100 个刮削记录 id");
if (!idsParam && !organizationRecordId) throw new FilesError("bad_request", "需要 ids 或 organizationRecordId");
if (organizationRecordId) {
let query = d.ctx.db
.selectFrom("scrape_record")
.select("id")
.where("organization_record_id", "=", organizationRecordId);
if (idsParam !== undefined) query = query.where("id", "in", ids);
const matched = await query.orderBy("requested_at").orderBy("id").execute();
const allowed = new Set(matched.map((r) => r.id));
ids = idsParam === undefined ? matched.map((r) => r.id) : ids.filter((id) => allowed.has(id));
}
return { records: await loadScrapeRecords(d.ao, ids) };
}
@@ -0,0 +1,412 @@
import { lstat, opendir, realpath } from "node:fs/promises";
import { basename, join } from "node:path";
import { isDownloadComplete } from "@umt/adapters";
import type { FileRef, OrganizeSourceConfig } from "@umt/contracts";
import { sql } from "@umt/db";
import { hostPathOf, loadInstances, upsertDecision } from "../acquisition-organize/services/common.ts";
import { planImport } from "../acquisition-organize/services/organize.ts";
import { downloadTasksUnder } from "../acquisition-organize/services/takeover.ts";
import { entityFromCandidate } from "./candidate.ts";
import {
FilesError,
type FmDeps,
hashJson,
isTempName,
jsonb,
recheck,
relOf,
resolveIn,
rootDef,
within,
} from "./common.ts";
import { candidateKey, identifyCore } from "./identify.ts";
import { kindOfName } from "./list.ts";
import { sourceConfig, sourceConfigs } from "./organize-sources.ts";
export interface SnapshotFile {
size: number;
mtimeMs: number;
accepted: boolean;
}
interface Snapshot {
files: Record<string, SnapshotFile>;
}
export const SOURCE_SCAN_LIMIT = 5000;
/** A changed file becomes eligible only after two observations of the same size and mtime. */
export function advanceSnapshot(previous: Snapshot | null, observed: Record<string, Omit<SnapshotFile, "accepted">>) {
const files: Snapshot["files"] = {};
const eligible: string[] = [];
for (const [path, fact] of Object.entries(observed)) {
const old = previous?.files[path];
const same = old?.size === fact.size && old.mtimeMs === fact.mtimeMs;
const accepted = previous === null || (same && old.accepted);
files[path] = { ...fact, accepted };
if (same && !accepted) eligible.push(path);
}
return { snapshot: { files }, eligible };
}
export async function walkSource(d: FmDeps, config: OrganizeSourceConfig, path = config.source.path) {
if (!within(path, config.source.path) && config.source.path !== "")
throw new FilesError("path_escape", "扫描路径必须在来源目录内");
const start = await resolveIn(d, config.source.root, path);
if (start.isSymlink || !start.lst?.isDirectory()) throw new FilesError("bad_request", "来源必须为目录");
const files: Record<string, Omit<SnapshotFile, "accepted">> = {};
const stack = [start.abs];
let seen = 0;
let truncated = false;
while (stack.length && !truncated) {
const directory = await opendir(stack.pop()!);
for await (const entry of directory) {
if (++seen > SOURCE_SCAN_LIMIT) {
truncated = true;
break;
}
if (entry.name.startsWith(".") || isTempName(entry.name) || entry.isSymbolicLink()) continue;
const abs = join(directory.path, entry.name);
// Re-resolve parents before traversing to keep directory replacement races inside the root guard.
const resolved = await resolveIn(d, config.source.root, relOf(start.rootReal, abs));
if (resolved.isSymlink) continue;
if (resolved.lst?.isDirectory()) stack.push(resolved.abs);
else if (resolved.lst?.isFile() && kindOfName(entry.name) === "video") {
const fact = await lstat(resolved.abs);
if (!fact.isFile() || fact.isSymbolicLink() || fact.size <= 0) continue;
files[relOf(start.rootReal, resolved.abs)] = { size: Number(fact.size), mtimeMs: Number(fact.mtimeMs) };
}
}
}
return { files, truncated, rootReal: start.rootReal };
}
/** Query unfiltered QB state once: an untagged download may never have a DB task row. */
async function activeDownloaderPaths(d: FmDeps, config: OrganizeSourceConfig): Promise<string[]> {
const source = await rootDef(d.ctx.db, config.source.root);
const instanceIds = [...new Set([config.downloaderInstanceId, source.instanceId].filter((id): id is string => !!id))];
if (!instanceIds.length) return [];
const topology = await loadInstances(d.ctx.db, { enabledOnly: false });
const paths: string[] = [];
for (const instanceId of instanceIds) {
const instance = topology.find((t) => t.inst.id === instanceId);
if (!instance) throw new FilesError("upstream", "来源下载器不存在,无法确认文件是否下载完成");
try {
const torrents = await (await d.ao.ports.downloader(instance.inst)).list();
for (const torrent of torrents) {
if (isDownloadComplete({ progress: torrent.progress, amountLeft: torrent.amountLeft, state: torrent.rawState }))
continue;
const content = torrent.contentPath || join(torrent.savePath, torrent.name);
const host = hostPathOf(instance, content);
if (!host) throw new Error("下载器内容路径未映射,无法确认源文件是否下载完成");
paths.push(await realpath(host).catch(() => host));
}
} catch (error) {
throw new FilesError(
"upstream",
`无法查询来源下载器,暂停本次整理:${error instanceof Error ? error.message : String(error)}`,
);
}
}
return paths;
}
async function organizedPaths(
d: FmDeps,
paths: string[],
scan: { files: Record<string, Omit<SnapshotFile, "accepted">>; rootReal: string },
) {
if (!paths.length) return new Set<string>();
const rows = await d.ctx.db
.selectFrom("organization_file_result as f")
.innerJoin("organization_record as r", "r.id", "f.organization_record_id")
.select([
"f.source_path",
"f.source_size_bytes",
"f.status as file_status",
"f.skip_reason",
"r.status as record_status",
"r.plan",
])
.where("f.source_path", "in", paths)
.where("r.status", "in", ["running", "queued", "succeeded", "partial"])
.execute();
const done = new Set<string>();
for (const row of rows) {
if (["running", "queued"].includes(row.record_status)) {
done.add(row.source_path);
continue;
}
if (row.file_status !== "succeeded" && !(row.file_status === "skipped" && row.skip_reason === "already_in_library"))
continue;
const observed = scan.files[relOf(scan.rootReal, row.source_path)];
if (!observed) continue;
const recorded = (row.plan as { sourceFacts?: Record<string, { size: number; mtimeMs: number }> })?.sourceFacts?.[
row.source_path
];
if (
recorded
? recorded.size === observed.size && recorded.mtimeMs === observed.mtimeMs
: row.source_size_bytes === null || Number(row.source_size_bytes) === observed.size
)
done.add(row.source_path);
}
return done;
}
export async function scanStock(d: FmDeps, id: string, input: { path?: string } = {}) {
const config = await sourceConfig(d, id);
const liveActive = await activeDownloaderPaths(d, config);
const scan = await walkSource(d, config, input.path ?? config.source.path);
const paths = Object.keys(scan.files);
const done = await organizedPaths(
d,
paths.map((p) => join(scan.rootReal, p)),
scan,
);
const items: FileRef[] = [];
let skipped = 0;
let unavailable = 0;
let truncated = scan.truncated;
const stockLimit = Math.min(200, d.options.identifyMaxVideos);
const known = await downloadTasksUnder(
d.ctx.db,
paths.map((p) => join(scan.rootReal, p)),
);
const topology = await loadInstances(d.ctx.db, { enabledOnly: false });
const active = known
.filter((t) => !["completed", "seeding", "removed"].includes(t.status))
.flatMap((t) => {
const instance = topology.find((i) => i.inst.id === t.instance_id);
const host = instance && t.content_path ? hostPathOf(instance, t.content_path) : null;
return host ? [host] : [];
});
active.push(...liveActive);
for (const path of paths) {
const abs = join(scan.rootReal, path);
if (done.has(abs)) {
skipped++;
continue;
}
if (active.some((p) => within(abs, p))) {
unavailable++;
continue;
}
if (items.length >= stockLimit) {
truncated = true;
continue;
}
items.push({ root: config.source.root as FileRef["root"], path });
}
return { items, skipped, unavailable, truncated };
}
export async function scanOrganizeSource(d: FmDeps, input: { sourceId: string }) {
const config = await sourceConfig(d, input.sourceId).catch(() => null);
if (!config) return { baseline: false, eligible: 0, queued: 0 };
// Cross-worker lock covers observations and planning. A duplicate scan never counts as a second stable observation.
return d.ctx.db.transaction().execute(async (trx) => {
const lock = await sql<{
locked: boolean;
}>`select pg_try_advisory_xact_lock(hashtext(${`organize-source:${config.id}`})) as locked`.execute(trx);
if (!lock.rows[0]?.locked) return { baseline: false, eligible: 0, queued: 0 };
const stored = await trx.selectFrom("settings").select("value").where("key", "=", "organize").executeTakeFirst();
const freshConfig = (
(stored?.value as { sources?: OrganizeSourceConfig[] })?.sources ?? (await sourceConfigs(d))
).find((s) => s.id === config.id);
if (!freshConfig || hashJson(freshConfig) !== hashJson(config)) return { baseline: false, eligible: 0, queued: 0 };
const row = await trx
.selectFrom("organize_source_snapshot")
.selectAll()
.where("source_id", "=", config.id)
.executeTakeFirst();
const now = new Date();
try {
const liveActive = await activeDownloaderPaths(d, config);
const scan = await walkSource(d, config);
if (scan.truncated) throw new FilesError("too_large", "来源超过单次扫描上限,请选择更小的目录");
const baseline = !row?.baseline_at;
const advance = advanceSnapshot(baseline ? null : (row!.snapshot as unknown as Snapshot), scan.files);
let queued = 0;
const automation = await d.ctx.settings.get("automation");
if (
!baseline &&
config.enabled &&
config.trigger === "snapshot" &&
automation.enabled &&
!automation.readOnlyReconcile
) {
const done = await organizedPaths(
d,
advance.eligible.map((p) => join(scan.rootReal, p)),
scan,
);
for (const path of advance.eligible) {
const abs = join(scan.rootReal, path);
if (done.has(abs)) {
advance.snapshot.files[path]!.accepted = true;
continue;
}
if (liveActive.some((p) => within(abs, p))) continue;
// Any known QB task stays under QB's completion/tag policy, including unmanaged and untagged tasks.
const known = await downloadTasksUnder(d.ctx.db, [abs]);
if (known.length) continue;
const identified = await identifyCore(
d,
{
items: [{ root: config.source.root as FileRef["root"], path }],
hints: config.mediaType === "all" ? {} : { mediaKind: config.mediaType === "tv" ? "series" : "movie" },
},
{ mode: "remote", actorId: "system:source-monitor" },
);
const key = identified.autoPicks.get(`${config.source.root}\u0000${path}`);
const candidate = identified.result.items[0]?.candidates.find((c) => candidateKey(c) === key);
if (!candidate) {
const dedupeKey = `source_identity:${config.id}:${hashJson(path)}`;
const existing = await trx
.selectFrom("decision")
.select("id")
.where("dedupe_key", "=", dedupeKey)
.where("status", "in", ["open", "investigating", "awaiting_user"])
.executeTakeFirst();
if (!existing)
await upsertDecision(d.ao, trx, {
kind: "identity_ambiguous",
title: `整理来源待识别:${basename(path)}`,
subjectType: "organize_source",
subjectId: config.id,
dedupeKey,
evidence: { sourceConfigId: config.id, root: config.source.root, path },
options: [{ key: "dismiss", label: "忽略" }],
});
advance.snapshot.files[path]!.accepted = true;
continue;
}
const entity =
candidate.mediaEntityId ??
(
await entityFromCandidate(d, undefined, {
provider: "tmdb",
externalId: candidate.externalId!,
kind: candidate.kind === "movie" ? "movie" : "tv",
})
).mediaEntityId;
const organizeSettings = await d.ctx.settings.get("organize");
const fingerprint = hashJson({
config,
classificationYaml: organizeSettings.classificationYaml,
path,
...scan.files[path],
});
const result = await planImport(d.ao, {
sourcePaths: [abs],
pathGuard: (p) => recheck(p, scan.rootReal),
mediaEntityId: entity,
action: config.action,
trigger: "auto",
actorType: "system",
sourceConfigId: config.id,
idempotencyKey: `source:${fingerprint}`,
});
if (result.recordId && ["running", "queued"].includes(result.status)) {
await d.ctx.engine.enqueue(
"verifyAndImport",
{ recordId: result.recordId },
{ idempotencyKey: `verifyAndImport:source:${fingerprint}`, traceId: config.id },
);
queued++;
}
if (result.status === "failed" && result.reason !== "classification_unavailable") {
const dedupeKey = `source_blocked:${config.id}:${hashJson(path)}`;
const existing = await trx
.selectFrom("decision")
.select("id")
.where("dedupe_key", "=", dedupeKey)
.where("status", "in", ["open", "investigating", "awaiting_user"])
.executeTakeFirst();
if (!existing)
await upsertDecision(d.ao, trx, {
kind: "other",
title: `来源整理未完成:${basename(path)}`,
summary: `原因:${result.reason ?? "整理计划失败"}。可从整理来源的存量整理向导处理这个文件。`,
subjectType: "organize_source",
subjectId: config.id,
dedupeKey,
evidence: {
sourceConfigId: config.id,
root: config.source.root,
path,
recordId: result.recordId,
reason: result.reason,
fingerprint,
},
options: [{ key: "dismiss", label: "忽略" }],
});
}
advance.snapshot.files[path]!.accepted = result.reason !== "classification_unavailable";
}
}
// A source edit/delete while scanning invalidates the observation instead of restoring the old baseline.
const settings = await trx
.selectFrom("settings")
.select("value")
.where("key", "=", "organize")
.executeTakeFirst();
const current = (
(settings?.value as { sources?: OrganizeSourceConfig[] })?.sources ?? (await sourceConfigs(d))
).find((s) => s.id === config.id);
if (!current || hashJson(current.source) !== hashJson(config.source))
return { baseline: false, eligible: 0, queued };
await trx
.insertInto("organize_source_snapshot")
.values({
source_id: config.id,
snapshot: jsonb(advance.snapshot),
baseline_at: row?.baseline_at ?? now,
last_scan_at: now,
last_error: null,
})
.onConflict((oc) =>
oc.column("source_id").doUpdateSet({
snapshot: jsonb(advance.snapshot),
baseline_at: row?.baseline_at ?? now,
last_scan_at: now,
last_error: null,
}),
)
.execute();
return { baseline, eligible: advance.eligible.length, queued };
} catch (error) {
await trx
.insertInto("organize_source_snapshot")
.values({
source_id: config.id,
snapshot: jsonb({ files: {} }),
baseline_at: null,
last_scan_at: now,
last_error: error instanceof Error ? error.message : String(error),
})
.onConflict((oc) =>
oc
.column("source_id")
.doUpdateSet({ last_scan_at: now, last_error: error instanceof Error ? error.message : String(error) }),
)
.execute();
return { baseline: false, eligible: 0, queued: 0 };
}
});
}
export async function queueDueSourceScans(d: FmDeps, slot: string | number) {
const rows = await d.ctx.db.selectFrom("organize_source_snapshot").select(["source_id", "last_scan_at"]).execute();
let queued = 0;
for (const config of await sourceConfigs(d)) {
if (!config.enabled || config.trigger !== "snapshot") continue;
const last = rows.find((r) => r.source_id === config.id)?.last_scan_at;
if (last && Date.now() - new Date(last).getTime() < config.intervalSec * 1000) continue;
await d.ctx.engine.enqueue(
"scanOrganizeSource",
{ sourceId: config.id },
{ idempotencyKey: `source-scan:${config.id}:${slot}`, traceId: config.id },
);
queued++;
}
return { summary: `已入队 ${queued} 个来源扫描` };
}
+31 -8
View File
@@ -76,13 +76,23 @@ export async function recordSources(d: FmDeps, recordId: string): Promise<FilesR
const db = d.ctx.db;
const rec = await db
.selectFrom("organization_record")
.select(["id", "media_entity_id", "action", "status", "source_path", "source_path_view"])
.select([
"id",
"media_entity_id",
"action",
"status",
"source_path",
"source_path_view",
"target_path",
"target_library_root_id",
"plan",
])
.where("id", "=", recordId)
.executeTakeFirst();
if (!rec) throw new FilesError("not_found", "整理记录不存在");
const mains = await db
.selectFrom("organization_file_result")
.select(["source_path"])
.select(["source_path", "target_path", "target_library_root_id"])
.where("organization_record_id", "=", rec.id)
.where("role", "=", "main")
.orderBy("ordinal")
@@ -95,13 +105,20 @@ export async function recordSources(d: FmDeps, recordId: string): Promise<FilesR
? { root: def.id as FileRootId, path: abs === def.hostPath ? "" : abs.slice(def.hostPath.length + 1) }
: null;
};
// 多个主视频都在记录的源目录下、且源目录不是登记目录本身 → 给目录(向导按目录展开);否则逐个主视频
let sources = mains.map((m) => m.source_path);
const dir = rec.source_path;
// Successful moves have removed their inputs: reorganize the existing library files.
const moved = rec.action === "move" && rec.status === "succeeded";
const targetOf = (rootId: string | null, path: string | null) => {
const root = defs.find((d) => d.libraryRootId === rootId);
return root && path && within(join(root.hostPath, path), root.hostPath) ? join(root.hostPath, path) : null;
};
let sources = moved
? mains.map((m) => targetOf(m.target_library_root_id, m.target_path)).filter((p): p is string => !!p)
: mains.map((m) => m.source_path);
const dir = moved ? targetOf(rec.target_library_root_id, rec.target_path) : rec.source_path;
if (
dir &&
mains.length > 1 &&
mains.every((m) => within(m.source_path, dir)) &&
sources.length > 1 &&
sources.every((p) => within(p, dir)) &&
!defs.some((x) => x.hostPath === dir.replace(/\/+$/, ""))
)
sources = [dir];
@@ -109,7 +126,7 @@ export async function recordSources(d: FmDeps, recordId: string): Promise<FilesR
const items: FilesRecordSourcesResult["items"] = [];
const unresolved: string[] = [];
for (const abs of [...new Set(sources)]) {
const ref = rec.source_path_view === "local" ? toRef(abs) : null;
const ref = moved || rec.source_path_view === "local" ? toRef(abs) : null;
const exists = ref
? await lstat(abs).then(
() => true,
@@ -126,5 +143,11 @@ export async function recordSources(d: FmDeps, recordId: string): Promise<FilesR
status: rec.status as OrganizationStatus,
items,
unresolved,
...(rec.plan &&
typeof rec.plan === "object" &&
"sourceConfigId" in rec.plan &&
typeof rec.plan.sourceConfigId === "string"
? { sourceConfigId: rec.plan.sourceConfigId }
: {}),
};
}