fix(organize): resume original imports after episode corrections
This commit is contained in:
1 parent
679389f854
commit
e874247e7c
17 files changed
+702
-26
No files matched your search
@@ -4,6 +4,7 @@ import { evaluateAcquisition } from "./services/acquisition.ts";
|
||||
import { type AoDeps, automationBlock, ensureTaskRow, markTask } from "./services/common.ts";
|
||||
import { cleanupDownloadCache, submitDownload } from "./services/downloads.ts";
|
||||
import { executeRecord, planImport } from "./services/organize.ts";
|
||||
import { resumeEpisodeOrganize } from "./services/organize-episode-recovery.ts";
|
||||
import { reconcileLibrary } from "./services/reconcile.ts";
|
||||
import { runScrapeRecord, SCRAPE_TASK } from "./services/scrape.ts";
|
||||
import { errMessage } from "./util.ts";
|
||||
@@ -115,9 +116,31 @@ export function registerAoTasks(registry: TaskRegistry, d: AoDeps) {
|
||||
name: "verifyAndImport",
|
||||
version: 1,
|
||||
lane: "file",
|
||||
inputSchema: z.object({ downloadTaskId: id.optional(), recordId: id.optional(), taskId: id.optional() }),
|
||||
inputSchema: z.object({
|
||||
downloadTaskId: id.optional(),
|
||||
recordId: id.optional(),
|
||||
taskId: id.optional(),
|
||||
mediaEntityId: id.optional(),
|
||||
}),
|
||||
retryPolicy: FILE_RETRY,
|
||||
handler: async (t) => {
|
||||
if (t.input.mediaEntityId && !t.input.recordId && !t.input.downloadTaskId) {
|
||||
const taskId = await taskRow(
|
||||
d,
|
||||
"verifyAndImport",
|
||||
t.input.taskId,
|
||||
`episodes:${t.input.mediaEntityId}:${t.runId}`,
|
||||
t.input,
|
||||
t.runId,
|
||||
{ type: "media_entity", id: t.input.mediaEntityId },
|
||||
);
|
||||
return tracked(
|
||||
d,
|
||||
taskId,
|
||||
() => t.step.run("resume-episodes", () => resumeEpisodeOrganize(d, t.input.mediaEntityId!)),
|
||||
(r) => (r.waiting ? "waiting" : "succeeded"),
|
||||
);
|
||||
}
|
||||
const key = t.input.recordId ? `record:${t.input.recordId}` : `import:${t.input.downloadTaskId ?? t.runId}`;
|
||||
const taskId = await taskRow(
|
||||
d,
|
||||
|
||||
@@ -17,6 +17,7 @@ import { registerOrganizationRoutes } from "./routes/organization.ts";
|
||||
import { registerRuleRoutes } from "./routes/rules.ts";
|
||||
import { type AoDeps, automationBlock, enqueueSweep, enqueueTracked, timeSlot } from "./services/common.ts";
|
||||
import { reportVolumeStates, SWEEP_SLOT_MS, sweepReservedPlans, syncDownloads } from "./services/downloads.ts";
|
||||
import { sweepEpisodeOrganize } from "./services/organize-episode-recovery.ts";
|
||||
import { recoverStalledRecords } from "./services/organize-records.ts";
|
||||
import { sweepDueSlots, sweepMetadataRefresh, sweepSubscriptionSearch } from "./services/subscriptions.ts";
|
||||
import { organizeDecisionHook } from "./services/takeover.ts";
|
||||
@@ -91,6 +92,10 @@ export function createAcquisitionOrganizeModule(overrides: AoOverrides = {}): Se
|
||||
registerAoCapabilities(registry, () => lazy(ctx));
|
||||
},
|
||||
outboxRoutes: () => ({
|
||||
[EventType.mediaEpisodeMappingChanged]: (ev) => {
|
||||
const p = ev.payload as EventPayload<"media.episode_mapping_changed">;
|
||||
return { task: "verifyAndImport", input: { mediaEntityId: p.mediaEntityId } };
|
||||
},
|
||||
[EventType.releaseResolved]: (ev) => {
|
||||
const p = ev.payload as EventPayload<"release.resolved">;
|
||||
return {
|
||||
@@ -167,10 +172,12 @@ export function createAcquisitionOrganizeModule(overrides: AoOverrides = {}): Se
|
||||
const submitted = await sweepReservedPlans(d);
|
||||
const imports = await sweepPendingImports(d);
|
||||
const rec = await recoverStalledRecords(d);
|
||||
const episodes = await sweepEpisodeOrganize(d);
|
||||
return (
|
||||
`补提交 ${submitted},补整理 ${imports}` +
|
||||
(rec.requeued ? `,重排整理 ${rec.requeued}` : "") +
|
||||
(rec.failed ? `,整理停止 ${rec.failed}` : "") +
|
||||
(episodes ? `,季集恢复 ${episodes}` : "") +
|
||||
(rec.removed ? `,移除下载任务 ${rec.removed}` : "")
|
||||
);
|
||||
}),
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import { sql } from "@umt/db";
|
||||
import { parseReleasePath } from "@umt/domain";
|
||||
import { wordsOption } from "../../sources-identity/service/identify-words-cache.ts";
|
||||
import { errMessage, hashJson } from "../util.ts";
|
||||
import { type AoDeps, automationBlock, enqueueSweep, timeSlot } from "./common.ts";
|
||||
import { SWEEP_SLOT_MS } from "./downloads.ts";
|
||||
import { planFileEpisodes, segmentsWithin } from "./organize-plan.ts";
|
||||
import { organizeRequestFromRecord, retryOrganizeRecord } from "./organize-retry.ts";
|
||||
import type { FrozenPlan } from "./organize-types.ts";
|
||||
|
||||
/** 自动事实恢复只尝试仍未被后继记录接手的来源,失败后继留给原记录就地处理。 */
|
||||
function pendingEpisodeRecoverySql() {
|
||||
return sql<boolean>`r.status in ('failed', 'partial') and r.attention_cleared_at is null
|
||||
and r.plan -> 'unresolvedEpisodes' is not null and r.plan -> 'request' is not null
|
||||
and not exists (select 1 from organization_record next
|
||||
where (r.download_task_id is not null and next.download_task_id = r.download_task_id and next.created_at > r.created_at)
|
||||
or next.plan ->> 'retryOfRecordId' = r.id::text)`;
|
||||
}
|
||||
|
||||
/** 事实更新后只恢复季集待确认的整理;执行异常仍走原有文件账本恢复。 */
|
||||
export async function resumeEpisodeOrganize(d: AoDeps, entityId: string) {
|
||||
const result = { resumed: 0, waiting: 0 };
|
||||
if (await automationBlock(d)) return result;
|
||||
const entity = await d.ctx.db.selectFrom("media_entity").selectAll().where("id", "=", entityId).executeTakeFirst();
|
||||
if (!entity || (entity.kind !== "series" && entity.kind !== "variety")) return result;
|
||||
const rows = await d.ctx.db
|
||||
.selectFrom("organization_record as r")
|
||||
.selectAll("r")
|
||||
.where("r.media_entity_id", "=", entityId)
|
||||
.where(pendingEpisodeRecoverySql())
|
||||
.orderBy("r.created_at")
|
||||
.execute();
|
||||
const words = await wordsOption(d.ctx.db);
|
||||
for (const record of rows) {
|
||||
const plan = record.plan as unknown as FrozenPlan;
|
||||
const request = organizeRequestFromRecord(record);
|
||||
// 临时季集结构是本次专属约定,不按作品规则改变它;旧记录缺原请求时交给人工。
|
||||
if (!request || plan.episodeStructure?.temporary) continue;
|
||||
const pending = Object.entries(plan.unresolvedEpisodes ?? {}).filter(([, r]) => r.method === "unresolved");
|
||||
if (!pending.length) continue;
|
||||
const download = record.download_task_id
|
||||
? await d.ctx.db
|
||||
.selectFrom("download_task")
|
||||
.selectAll()
|
||||
.where("id", "=", record.download_task_id)
|
||||
.executeTakeFirst()
|
||||
: undefined;
|
||||
const release = download?.source_release_id
|
||||
? await d.ctx.db
|
||||
.selectFrom("source_release")
|
||||
.selectAll()
|
||||
.where("id", "=", download.source_release_id)
|
||||
.executeTakeFirst()
|
||||
: undefined;
|
||||
const episodes = await planFileEpisodes(d, entity, release, download, null);
|
||||
const resolved = pending.map(
|
||||
([path]) =>
|
||||
episodes.map(parseReleasePath(segmentsWithin(path, plan.sourceRoot), words), episodes.opts).resolution,
|
||||
);
|
||||
if (resolved.some((r) => !r.canonical || r.method === "unresolved" || r.method === "unverified" || r.special)) {
|
||||
result.waiting++;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const retry = await retryOrganizeRecord(d, record.id, { type: "system", id: null }, "episode-mapping", {
|
||||
mediaEntityId: entityId,
|
||||
skipSuccessfulRecords: true,
|
||||
sourcePaths: pending.map(([path]) => path),
|
||||
idempotencyKey: `organize-episodes:${record.id}:${hashJson(resolved)}`,
|
||||
});
|
||||
if (retry.status === "running" || retry.status === "queued") result.resumed++;
|
||||
else result.waiting++;
|
||||
} catch (error) {
|
||||
result.waiting++;
|
||||
d.ctx.logger.warn({ recordId: record.id, error: errMessage(error) }, "季集映射已更新,整理仍需处理");
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** 复用补整理巡检:暂停期间已处理的映射在恢复自动化后仍有唤醒机会。 */
|
||||
export async function sweepEpisodeOrganize(d: AoDeps): Promise<number> {
|
||||
if (await automationBlock(d)) return 0;
|
||||
const entities = await d.ctx.db
|
||||
.selectFrom("organization_record as r")
|
||||
.select("r.media_entity_id")
|
||||
.distinct()
|
||||
.where("r.media_entity_id", "is not", null)
|
||||
.where(pendingEpisodeRecoverySql())
|
||||
.execute();
|
||||
let queued = 0;
|
||||
for (const entity of entities) {
|
||||
if (!entity.media_entity_id) continue;
|
||||
const run = await enqueueSweep(
|
||||
d,
|
||||
{
|
||||
kind: "verifyAndImport",
|
||||
idempotencyKey: `episode-organize:${entity.media_entity_id}`,
|
||||
input: { mediaEntityId: entity.media_entity_id },
|
||||
subjectType: "media_entity",
|
||||
subjectId: entity.media_entity_id,
|
||||
},
|
||||
timeSlot(d.ports.now(), SWEEP_SLOT_MS),
|
||||
);
|
||||
if (run) queued++;
|
||||
}
|
||||
return queued;
|
||||
}
|
||||
@@ -41,6 +41,7 @@ import {
|
||||
type FileEpisodeOptions,
|
||||
publishedAtOf,
|
||||
settleEpisodeDecision,
|
||||
settleOrganizeEpisodeDecisions,
|
||||
} from "../../sources-identity/service/episode-resolution.ts";
|
||||
import type { Layout } from "../../sources-identity/service/episode-structure.ts";
|
||||
import { wordsOption } from "../../sources-identity/service/identify-words-cache.ts";
|
||||
@@ -279,7 +280,7 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
|
||||
let sourceRoot: string | null = null;
|
||||
const topo = dt ? (await loadInstances(db, { enabledOnly: false, ids: [dt.instance_id] }))[0] : undefined;
|
||||
let qbFiles: QbFileLike[] = [];
|
||||
if (dt) {
|
||||
if (dt && !(req.retryOfRecordId && req.sourcePaths?.length)) {
|
||||
if (!topo || !dt.save_path) return fail("source_path_unknown");
|
||||
const base = hostPathOf(topo, dt.save_path);
|
||||
if (!base) return fail("path_mapping_missing", { savePath: dt.save_path });
|
||||
@@ -352,7 +353,12 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
|
||||
files.push({ path: p, size: st.facts.size, device: st.facts.dev });
|
||||
}
|
||||
}
|
||||
sourceRoot = req.sourcePaths.length === 1 ? req.sourcePaths[0]!.replace(/\/+$/, "") : null;
|
||||
sourceRoot =
|
||||
dt?.content_path && topo
|
||||
? hostPathOf(topo, dt.content_path)
|
||||
: req.sourcePaths.length === 1
|
||||
? req.sourcePaths[0]!.replace(/\/+$/, "")
|
||||
: null;
|
||||
if (sourceRoot && files.length === 1 && files[0]!.path === sourceRoot) {
|
||||
sourceRoot = dirname(sourceRoot);
|
||||
}
|
||||
@@ -748,6 +754,7 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
|
||||
const raw = parsed;
|
||||
const r = episodes.map(raw, episodes.opts);
|
||||
resolution = r.resolution;
|
||||
episodes.samples.push({ path: f.path, resolution: r.resolution, parsed: raw });
|
||||
if (r.resolution.method === "unresolved" || r.resolution.special) {
|
||||
const unresolved = r.resolution.method === "unresolved";
|
||||
mappedOverrides[f.path] = {
|
||||
@@ -756,7 +763,6 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
|
||||
: { reason: "special_extra", detail: `rule:${r.resolution.special}` },
|
||||
};
|
||||
unresolvedEpisodes[f.path] = episodeFacts(r.resolution);
|
||||
if (unresolved) episodes.unresolved.push({ path: f.path, resolution: r.resolution, parsed: raw });
|
||||
continue;
|
||||
}
|
||||
parsed = r.parsed;
|
||||
@@ -1183,15 +1189,25 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise<PlanRes
|
||||
await emitRecord(d, trx, rec.id);
|
||||
return { recordId: rec.id, status: rec.status, reason };
|
||||
});
|
||||
// 文件集数待确认:并入关联资源的 episode_mapping 待处理(待处理按资源归集;没有关联资源只记整理失败原因)
|
||||
// 文件是独立样本;资源标题已解决也不能覆盖真正失败的文件季集。
|
||||
// 临时结构下的待确认是相对指定结构的,不并进作品的集数待确认
|
||||
const pending = episodes?.unresolved[0];
|
||||
if (pending && episodes?.release && !episodes.toEntity) {
|
||||
await settleEpisodeDecision(d.ctx, db, {
|
||||
sourceReleaseId: episodes.release.id,
|
||||
if (result.recordId && episodes?.samples.some((f) => f.resolution.method === "unresolved") && !episodes.toEntity) {
|
||||
const releaseResolution = episodes.release?.episode_resolution as EpisodeResolution | null;
|
||||
if (episodes.release && releaseResolution?.method === "unresolved") {
|
||||
await settleEpisodeDecision(d.ctx, db, {
|
||||
sourceReleaseId: episodes.release.id,
|
||||
entityId: entity.id,
|
||||
resolution: releaseResolution,
|
||||
parsed: parsedOf(episodes.release),
|
||||
});
|
||||
}
|
||||
await settleOrganizeEpisodeDecisions(d.ctx, db, {
|
||||
recordId: result.recordId,
|
||||
entityId: entity.id,
|
||||
resolution: pending.resolution,
|
||||
parsed: pending.parsed,
|
||||
files: episodes.samples.map((file) => ({
|
||||
...file,
|
||||
publishedAt: episodes.release?.source_published_at?.toISOString() ?? null,
|
||||
})),
|
||||
});
|
||||
}
|
||||
return result;
|
||||
@@ -1204,14 +1220,14 @@ interface FileEpisodes {
|
||||
toEntity?: (span: EpisodeSpan) => EpisodeSpan | null;
|
||||
/** 下载关联的资源(已绑定本作品) */
|
||||
release: SourceReleaseRow | null;
|
||||
unresolved: { path: string; resolution: EpisodeResolution; parsed: ParsedRelease }[];
|
||||
samples: { path: string; resolution: EpisodeResolution; parsed: ParsedRelease }[];
|
||||
}
|
||||
|
||||
/**
|
||||
* 整理计划的集数映射装配:作品季表 + 规则加载一次;下载关联的资源(下载任务直接关联,或经下载 / 导入计划)
|
||||
* 提供发布时间,原始季集与文件相同时直接沿用它的规范季集
|
||||
*/
|
||||
async function planFileEpisodes(
|
||||
export async function planFileEpisodes(
|
||||
d: AoDeps,
|
||||
entity: MediaEntityRow,
|
||||
rel: SourceReleaseRow | undefined,
|
||||
@@ -1239,7 +1255,7 @@ async function planFileEpisodes(
|
||||
toEntity: s.toEntity,
|
||||
opts: linked ? { publishedAt: publishedAtOf(linked) } : {},
|
||||
release: own,
|
||||
unresolved: [],
|
||||
samples: [],
|
||||
};
|
||||
}
|
||||
const map = await createEntityEpisodeMapper(db, entity.id, entity.kind);
|
||||
@@ -1251,7 +1267,7 @@ async function planFileEpisodes(
|
||||
...(res?.method ? { release: res } : {}),
|
||||
},
|
||||
release: own,
|
||||
unresolved: [],
|
||||
samples: [],
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -30,13 +30,39 @@ export function organizeRequestFromRecord(
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 按原请求重新规划;不复用旧冻结计划,来源路径重新经过当前目录边界。 */
|
||||
interface RetryOptions {
|
||||
mediaEntityId?: string;
|
||||
idempotencyKey?: string;
|
||||
skipSuccessfulRecords?: boolean;
|
||||
sourcePaths?: string[];
|
||||
}
|
||||
|
||||
/** 同一原记录的重新规划串行;事件、巡检和人工同时到达也不能重复冻结文件计划。 */
|
||||
export async function retryOrganizeRecord(
|
||||
d: AoDeps,
|
||||
recordId: string,
|
||||
actor: DecisionHookActor,
|
||||
via: string,
|
||||
overrides: { mediaEntityId?: string; idempotencyKey?: string; skipSuccessfulRecords?: boolean } = {},
|
||||
overrides: RetryOptions = {},
|
||||
) {
|
||||
return d.ctx.db.connection().execute(async (connection) => {
|
||||
const key = `organize-retry:${recordId}`;
|
||||
await sql`select pg_advisory_lock(hashtextextended(${key}, 0))`.execute(connection);
|
||||
try {
|
||||
return await retryLocked({ ...d, ctx: { ...d.ctx, db: connection } }, recordId, actor, via, overrides);
|
||||
} finally {
|
||||
await sql`select pg_advisory_unlock(hashtextextended(${key}, 0))`.execute(connection);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** 按原请求重新规划;不复用旧冻结计划,来源路径重新经过当前目录边界。 */
|
||||
async function retryLocked(
|
||||
d: AoDeps,
|
||||
recordId: string,
|
||||
actor: DecisionHookActor,
|
||||
via: string,
|
||||
overrides: RetryOptions,
|
||||
) {
|
||||
const record = await d.ctx.db
|
||||
.selectFrom("organization_record")
|
||||
@@ -56,6 +82,17 @@ export async function retryOrganizeRecord(
|
||||
}
|
||||
const request = organizeRequestFromRecord(record);
|
||||
if (!request) throw new Error("整理记录没有可恢复的来源,重新选择来源后整理");
|
||||
if (overrides.sourcePaths) {
|
||||
const files = await d.ctx.db
|
||||
.selectFrom("organization_file_result")
|
||||
.select("source_path")
|
||||
.where("organization_record_id", "=", recordId)
|
||||
.execute();
|
||||
const original = new Set(files.map((f) => f.source_path));
|
||||
if (!overrides.sourcePaths.length || overrides.sourcePaths.some((p) => !original.has(p)))
|
||||
throw new Error("恢复只能选择原整理记录中的文件");
|
||||
request.sourcePaths = [...new Set(overrides.sourcePaths)];
|
||||
}
|
||||
if (overrides.skipSuccessfulRecords && request.sourcePaths?.length) {
|
||||
const done = await d.ctx.db
|
||||
.selectFrom("organization_file_result")
|
||||
|
||||
@@ -540,7 +540,7 @@ export async function checkRecovery(ctx: ServerContext, actions: RecoveryAction[
|
||||
: "pending",
|
||||
);
|
||||
} else if (a.capabilityId === "episode_mapping.apply_rule") {
|
||||
// 事项内资源都已不是待确认(写进作品识别词后的重算会把事项自动关闭)即视为恢复
|
||||
// 资源与真实整理文件都须解决;仅资源列表为空不能代表文件映射已恢复。
|
||||
const id = String(a.input.decisionId ?? "");
|
||||
const d = id
|
||||
? await ctx.db.selectFrom("decision").select(["status", "evidence"]).where("id", "=", id).executeTakeFirst()
|
||||
@@ -554,8 +554,18 @@ export async function checkRecovery(ctx: ServerContext, actions: RecoveryAction[
|
||||
.execute()
|
||||
: [];
|
||||
const unresolved = rows.filter((r) => r.method === "unresolved").length;
|
||||
facts[`decision:${id}`] = d ? { status: d.status, releases: ids.length, unresolved } : null;
|
||||
states.push(!d ? "failed" : d.status === "resolved" || unresolved === 0 ? "recovered" : "pending");
|
||||
const fileSamples =
|
||||
((d?.evidence ?? {}) as { fileSamples?: { resolution?: { method?: string } }[] }).fileSamples ?? [];
|
||||
const unresolvedFiles = fileSamples.filter((f) => f.resolution?.method === "unresolved").length;
|
||||
facts[`decision:${id}`] = d ? { status: d.status, releases: ids.length, unresolved, unresolvedFiles } : null;
|
||||
states.push(
|
||||
!d
|
||||
? "failed"
|
||||
: unresolvedFiles === 0 &&
|
||||
(d.status === "resolved" || (ids.length > 0 && rows.length === ids.length && unresolved === 0))
|
||||
? "recovered"
|
||||
: "pending",
|
||||
);
|
||||
}
|
||||
}
|
||||
const state = states.includes("failed") ? "failed" : states.every((x) => x === "recovered") ? "recovered" : "pending";
|
||||
|
||||
@@ -302,6 +302,8 @@ export async function simulateEpisodeRule(
|
||||
}
|
||||
if (regressions.length)
|
||||
failures.push(`会让 ${regressions.length} 条已对上的资源变成待确认(如「${regressions[0]!.title}」)`);
|
||||
if (fileRegressions.length)
|
||||
failures.push(`会让 ${fileRegressions.length} 个已对上的文件变成待确认(如「${fileRegressions[0]!.file.path}」)`);
|
||||
const summary: EpisodeSimulationSummary = {
|
||||
samples: ids.size + files.length,
|
||||
resolved: resolved.length + resolvedFiles.length,
|
||||
|
||||
@@ -460,7 +460,16 @@ export async function settleOrganizeEpisodeDecisions(
|
||||
const previous = existing.find((d) => d.dedupe_key === key);
|
||||
const ev = previous?.evidence as EpisodeDecisionEvidence | undefined;
|
||||
const group = groups.get(key);
|
||||
const files = [...episodeFileSamples(ev).filter((f) => f.recordId !== input.recordId), ...(group?.files ?? [])];
|
||||
// 同批已确定的文件作为模拟护栏,不能用修正坏文件的规则破坏它们。
|
||||
// 其它范围的未决文件由其自己的事项处理,不要求一条规则解决整批问题。
|
||||
const context = group
|
||||
? input.files.filter((f) => f.resolution.method !== "unresolved").map((f) => ({ ...f, recordId: input.recordId }))
|
||||
: [];
|
||||
const files = [
|
||||
...episodeFileSamples(ev).filter((f) => f.recordId !== input.recordId),
|
||||
...(group?.files ?? []),
|
||||
...context,
|
||||
];
|
||||
const ids = ev?.sourceReleaseIds ?? [];
|
||||
if (!ids.length && !files.some((f) => f.resolution.method === "unresolved")) {
|
||||
await closeDecisions(
|
||||
@@ -826,6 +835,7 @@ export async function reresolveEntityEpisodes(
|
||||
}
|
||||
}
|
||||
await reresolveFileSamples(ctx, exec, entityId, entity.kind, table, rules);
|
||||
await ctx.outbox.emit(exec, "media.episode_mapping_changed", { mediaEntityId: entityId });
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,406 @@
|
||||
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
executeRecord,
|
||||
type FrozenPlan,
|
||||
planImport,
|
||||
} from "../../src/modules/acquisition-organize/services/organize.ts";
|
||||
import {
|
||||
resumeEpisodeOrganize,
|
||||
sweepEpisodeOrganize,
|
||||
} from "../../src/modules/acquisition-organize/services/organize-episode-recovery.ts";
|
||||
import { checkRecovery } from "../../src/modules/agent/investigate.ts";
|
||||
import { reresolveEntityEpisodes } from "../../src/modules/sources-identity/service/episode-resolution.ts";
|
||||
import { createFmHarness, type FmHarness, pgAvailable, seedDownload, seedEntity } from "../file-manager/harness.ts";
|
||||
import { setWordsText } from "../support/identify-words.ts";
|
||||
import { seedRelease } from "./harness.ts";
|
||||
|
||||
describe.skipIf(!(await pgAvailable()))("季集事实补齐后继续原整理", () => {
|
||||
let h: FmHarness;
|
||||
beforeAll(async () => {
|
||||
h = await createFmHarness("episode_resume", {}, { runTransfers: false });
|
||||
});
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
});
|
||||
|
||||
async function show(title: string) {
|
||||
const id = await seedEntity(h.db, { title, kind: "series", year: 2024 });
|
||||
await h.db
|
||||
.updateTable("media_entity")
|
||||
.set({ metadata: JSON.stringify({ airStatus: { value: "Ended", source: "tmdb" } }) })
|
||||
.where("id", "=", id)
|
||||
.execute();
|
||||
await h.db
|
||||
.insertInto("content_unit")
|
||||
.values({
|
||||
media_entity_id: id,
|
||||
kind: "season",
|
||||
season_number: 1,
|
||||
episode_total: 12,
|
||||
status: "confirmed",
|
||||
source: "tmdb",
|
||||
})
|
||||
.execute();
|
||||
return id;
|
||||
}
|
||||
|
||||
it("按路径整理无需订阅也保留全部文件样本,修正规则后恢复,重放不重复排队", async () => {
|
||||
const entity = await show("Episode Resume");
|
||||
const paths = ["Episode.Resume.S02E01.1080p.mkv", "Episode.Resume.S02E02.1080p.mkv"].map((name) =>
|
||||
join(h.dl, name),
|
||||
);
|
||||
for (const path of paths) await writeFile(path, path.padEnd(128, "x"));
|
||||
const original = await planImport(h.module.deps().ao, {
|
||||
sourcePaths: paths,
|
||||
mediaEntityId: entity,
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
action: "copy",
|
||||
targetDir: join(h.lib, "preferred"),
|
||||
namingMode: "keep",
|
||||
scrapeAfterImport: false,
|
||||
minSizeBytes: 0,
|
||||
});
|
||||
expect(original.status).toBe("failed");
|
||||
const decision = await h.db
|
||||
.selectFrom("decision")
|
||||
.selectAll()
|
||||
.where("kind", "=", "episode_mapping")
|
||||
.where("subject_id", "=", entity)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect((decision.evidence as { fileSamples: unknown[] }).fileSamples).toHaveLength(2);
|
||||
expect(
|
||||
(
|
||||
await checkRecovery(h.module.deps().ao.ctx, [
|
||||
{ capabilityId: "episode_mapping.apply_rule", input: { decisionId: decision.id }, status: "succeeded" },
|
||||
])
|
||||
).state,
|
||||
).toBe("pending");
|
||||
expect(await resumeEpisodeOrganize(h.module.deps().ao, entity)).toEqual({ resumed: 0, waiting: 1 });
|
||||
await setWordsText(h.db, entity, "S02 >> S01 EP\n");
|
||||
await reresolveEntityEpisodes(h.module.deps().ao.ctx, h.db, entity);
|
||||
expect(
|
||||
await h.db.selectFrom("outbox_event").select("id").where("type", "=", "media.episode_mapping_changed").execute(),
|
||||
).not.toHaveLength(0);
|
||||
const concurrent = await Promise.all([
|
||||
resumeEpisodeOrganize(h.module.deps().ao, entity),
|
||||
resumeEpisodeOrganize(h.module.deps().ao, entity),
|
||||
]);
|
||||
expect(concurrent.every((r) => r.waiting === 0)).toBe(true);
|
||||
expect(
|
||||
await h.db
|
||||
.selectFrom("organization_record")
|
||||
.select("id")
|
||||
.where("media_entity_id", "=", entity)
|
||||
.where("id", "!=", original.recordId!)
|
||||
.execute(),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
h.engine.enqueued.filter(
|
||||
(e) =>
|
||||
e.task === "verifyAndImport" &&
|
||||
(e.input as { recordId?: string }).recordId &&
|
||||
(e.input as { recordId?: string }).recordId !== original.recordId,
|
||||
),
|
||||
).toHaveLength(1);
|
||||
const next = await h.db
|
||||
.selectFrom("organization_record")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", entity)
|
||||
.where("id", "!=", original.recordId!)
|
||||
.executeTakeFirstOrThrow();
|
||||
const plan = next.plan as unknown as FrozenPlan;
|
||||
expect(plan.request).toMatchObject({
|
||||
action: "copy",
|
||||
targetDir: join(h.lib, "preferred"),
|
||||
namingMode: "keep",
|
||||
scrapeAfterImport: false,
|
||||
});
|
||||
expect(plan.retryOfRecordId).toBe(original.recordId);
|
||||
expect(await resumeEpisodeOrganize(h.module.deps().ao, entity)).toEqual({ resumed: 0, waiting: 0 });
|
||||
expect((await executeRecord(h.module.deps().ao, next.id)).status).toBe("succeeded");
|
||||
const future = await planImport(h.module.deps().ao, {
|
||||
sourcePaths: [paths[0]!],
|
||||
mediaEntityId: entity,
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
dryRun: true,
|
||||
minSizeBytes: 0,
|
||||
});
|
||||
expect(Object.values(future.plan!.facts)[0]!.episodeResolution).toMatchObject({
|
||||
method: "rule",
|
||||
canonical: { season: 1, episodes: { start: 1, end: 1 } },
|
||||
});
|
||||
});
|
||||
|
||||
it("部分移动仅继续剩余文件,已入库内容保持原样", async () => {
|
||||
const entity = await show("Partial Resume");
|
||||
const first = join(h.dl, "Partial.Resume.S01E01.1080p.mkv");
|
||||
const second = join(h.dl, "Partial.Resume.S02E02.1080p.mkv");
|
||||
await writeFile(first, "first".padEnd(128, "x"));
|
||||
await writeFile(second, "second".padEnd(128, "y"));
|
||||
const original = await planImport(h.module.deps().ao, {
|
||||
sourcePaths: [first, second],
|
||||
mediaEntityId: entity,
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
action: "move",
|
||||
scrapeAfterImport: false,
|
||||
minSizeBytes: 0,
|
||||
});
|
||||
expect((await executeRecord(h.module.deps().ao, original.recordId!)).status).toBe("partial");
|
||||
const decision = await h.db
|
||||
.selectFrom("decision")
|
||||
.select("evidence")
|
||||
.where("kind", "=", "episode_mapping")
|
||||
.where("subject_id", "=", entity)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect((decision.evidence as { fileSamples: { path: string }[] }).fileSamples.map((f) => f.path).sort()).toEqual(
|
||||
[first, second].sort(),
|
||||
);
|
||||
await expect(access(first)).rejects.toThrow();
|
||||
const record = await h.db
|
||||
.selectFrom("organization_record")
|
||||
.select("plan")
|
||||
.where("id", "=", original.recordId!)
|
||||
.executeTakeFirstOrThrow();
|
||||
const target = (record.plan as unknown as FrozenPlan).organize.files.find((f) => f.src === first)!.dst;
|
||||
const content = await readFile(target, "utf8");
|
||||
await setWordsText(h.db, entity, "S02 >> S01 EP\n");
|
||||
expect(await resumeEpisodeOrganize(h.module.deps().ao, entity)).toEqual({ resumed: 1, waiting: 0 });
|
||||
const next = await h.db
|
||||
.selectFrom("organization_record")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", entity)
|
||||
.where("id", "!=", original.recordId!)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect((next.plan as unknown as FrozenPlan).request?.sourcePaths).toEqual([second]);
|
||||
expect((await executeRecord(h.module.deps().ao, next.id)).status).toBe("succeeded");
|
||||
expect(await readFile(target, "utf8")).toBe(content);
|
||||
expect(
|
||||
await h.db
|
||||
.selectFrom("media_asset")
|
||||
.select("id")
|
||||
.where("media_entity_id", "=", entity)
|
||||
.where("status", "=", "active")
|
||||
.execute(),
|
||||
).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("下载任务部分移动后仍可恢复,不重新检查已经移走的源文件", async () => {
|
||||
const entity = await show("Download Resume");
|
||||
const name = "Download.Resume";
|
||||
const dir = join(h.dl, name);
|
||||
await mkdir(dir);
|
||||
const paths = ["Download.Resume.S01E01.1080p.mkv", "Download.Resume.S02E02.1080p.mkv"].map((f) => join(dir, f));
|
||||
for (const path of paths) await writeFile(path, path.padEnd(128, "x"));
|
||||
const dt = await seedDownload(h.db, h, { name, status: "completed" });
|
||||
const site = await h.db
|
||||
.insertInto("site")
|
||||
.values({ key: "resume", name: "恢复测试站", adapter: "nexusphp", base_url: "https://pt.example.test" })
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow();
|
||||
const release = await seedRelease(h.db, {
|
||||
siteId: site.id,
|
||||
entityId: entity,
|
||||
title: name,
|
||||
size: 256,
|
||||
hash: "a".repeat(40),
|
||||
hr: false,
|
||||
});
|
||||
await h.db
|
||||
.updateTable("download_task")
|
||||
.set({
|
||||
media_entity_id: entity,
|
||||
source_release_id: release,
|
||||
file_list: JSON.stringify(paths.map((p) => ({ path: p.slice(h.dl.length + 1), size: 128 }))),
|
||||
})
|
||||
.where("id", "=", dt)
|
||||
.execute();
|
||||
const original = await planImport(h.module.deps().ao, {
|
||||
downloadTaskId: dt,
|
||||
mediaEntityId: entity,
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
action: "move",
|
||||
scrapeAfterImport: false,
|
||||
minSizeBytes: 0,
|
||||
});
|
||||
expect((await executeRecord(h.module.deps().ao, original.recordId!)).status).toBe("partial");
|
||||
await expect(access(paths[0]!)).rejects.toThrow();
|
||||
await setWordsText(h.db, entity, "S02 >> S01 EP\n");
|
||||
expect(await resumeEpisodeOrganize(h.module.deps().ao, entity)).toEqual({ resumed: 1, waiting: 0 });
|
||||
const next = await h.db
|
||||
.selectFrom("organization_record")
|
||||
.selectAll()
|
||||
.where("download_task_id", "=", dt)
|
||||
.where("id", "!=", original.recordId!)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect((next.plan as unknown as FrozenPlan).request).toMatchObject({
|
||||
downloadTaskId: dt,
|
||||
sourcePaths: [paths[1]!],
|
||||
});
|
||||
expect((await executeRecord(h.module.deps().ao, next.id)).status).toBe("succeeded");
|
||||
expect(
|
||||
await h.db
|
||||
.selectFrom("media_asset")
|
||||
.select("id")
|
||||
.where("media_entity_id", "=", entity)
|
||||
.where("status", "=", "active")
|
||||
.execute(),
|
||||
).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("补齐在播季表后沿用原资源发布时间,恢复正确的绝对集号换算", async () => {
|
||||
const entity = await show("Dated Resume");
|
||||
const name = "Dated.Resume.S02E13.1080p.mkv";
|
||||
const path = join(h.dl, name);
|
||||
await writeFile(path, "dated".padEnd(128, "x"));
|
||||
const dt = await seedDownload(h.db, h, { name, status: "completed" });
|
||||
const site = await h.db
|
||||
.insertInto("site")
|
||||
.values({
|
||||
key: "dated-resume",
|
||||
name: "发布时间测试站",
|
||||
adapter: "nexusphp",
|
||||
base_url: "https://dated.example.test",
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow();
|
||||
const release = await seedRelease(h.db, {
|
||||
siteId: site.id,
|
||||
entityId: entity,
|
||||
title: name,
|
||||
size: 128,
|
||||
hash: "b".repeat(40),
|
||||
});
|
||||
await h.db
|
||||
.updateTable("source_release")
|
||||
.set({ source_published_at: new Date("2026-01-08T16:30:00Z") })
|
||||
.where("id", "=", release)
|
||||
.execute();
|
||||
await h.db
|
||||
.updateTable("download_task")
|
||||
.set({
|
||||
media_entity_id: entity,
|
||||
source_release_id: release,
|
||||
file_list: JSON.stringify([{ path: name, size: 128 }]),
|
||||
})
|
||||
.where("id", "=", dt)
|
||||
.execute();
|
||||
const original = await planImport(h.module.deps().ao, {
|
||||
downloadTaskId: dt,
|
||||
mediaEntityId: entity,
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
action: "copy",
|
||||
scrapeAfterImport: false,
|
||||
minSizeBytes: 0,
|
||||
});
|
||||
expect(original.status).toBe("failed");
|
||||
await h.db
|
||||
.updateTable("media_entity")
|
||||
.set({ metadata: JSON.stringify({ airStatus: "Returning Series" }) })
|
||||
.where("id", "=", entity)
|
||||
.execute();
|
||||
const season = await h.db
|
||||
.insertInto("content_unit")
|
||||
.values({
|
||||
media_entity_id: entity,
|
||||
kind: "season",
|
||||
season_number: 2,
|
||||
episode_total: 12,
|
||||
status: "confirmed",
|
||||
source: "tmdb",
|
||||
})
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow();
|
||||
await h.db
|
||||
.insertInto("content_unit")
|
||||
.values({
|
||||
media_entity_id: entity,
|
||||
kind: "episode",
|
||||
parent_unit_id: season.id,
|
||||
season_number: 2,
|
||||
episode_number: 1,
|
||||
air_date: "2026-01-08",
|
||||
status: "confirmed",
|
||||
source: "tmdb",
|
||||
})
|
||||
.execute();
|
||||
expect(await resumeEpisodeOrganize(h.module.deps().ao, entity)).toEqual({ resumed: 1, waiting: 0 });
|
||||
const next = await h.db
|
||||
.selectFrom("organization_record")
|
||||
.selectAll()
|
||||
.where("download_task_id", "=", dt)
|
||||
.where("id", "!=", original.recordId!)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(Object.values((next.plan as unknown as FrozenPlan).facts)[0]!.episodeResolution).toMatchObject({
|
||||
method: "absolute_in_season",
|
||||
canonical: { season: 2, episodes: { start: 1, end: 1 } },
|
||||
});
|
||||
expect((await executeRecord(h.module.deps().ao, next.id)).status).toBe("succeeded");
|
||||
});
|
||||
|
||||
it("暂停期间修正规则,启用后只恢复原目录的未解决文件", async () => {
|
||||
const entity = await show("Directory Resume");
|
||||
const dir = join(h.dl, "Directory.Resume");
|
||||
await mkdir(dir);
|
||||
const first = join(dir, "Directory.Resume.S01E01.1080p.mkv");
|
||||
const second = join(dir, "Directory.Resume.S02E02.1080p.mkv");
|
||||
for (const path of [first, second]) await writeFile(path, path.padEnd(128, "x"));
|
||||
const original = await planImport(h.module.deps().ao, {
|
||||
sourcePaths: [dir],
|
||||
mediaEntityId: entity,
|
||||
trigger: "manual",
|
||||
actorType: "user",
|
||||
action: "copy",
|
||||
scrapeAfterImport: false,
|
||||
minSizeBytes: 0,
|
||||
});
|
||||
expect((await executeRecord(h.module.deps().ao, original.recordId!)).status).toBe("partial");
|
||||
const added = join(dir, "Directory.Resume.S01E03.1080p.mkv");
|
||||
await writeFile(added, "later".padEnd(128, "x"));
|
||||
await h.db
|
||||
.insertInto("settings")
|
||||
.values({ key: "automation", value: JSON.stringify({ enabled: false }) })
|
||||
.onConflict((oc) => oc.column("key").doUpdateSet({ value: JSON.stringify({ enabled: false }) }))
|
||||
.execute();
|
||||
await setWordsText(h.db, entity, "S02 >> S01 EP\n");
|
||||
await reresolveEntityEpisodes(h.module.deps().ao.ctx, h.db, entity);
|
||||
expect(await resumeEpisodeOrganize(h.module.deps().ao, entity)).toEqual({ resumed: 0, waiting: 0 });
|
||||
expect(await sweepEpisodeOrganize(h.module.deps().ao)).toBe(0);
|
||||
await h.db
|
||||
.updateTable("settings")
|
||||
.set({ value: JSON.stringify({ enabled: true }) })
|
||||
.where("key", "=", "automation")
|
||||
.execute();
|
||||
expect(await sweepEpisodeOrganize(h.module.deps().ao)).toBe(1);
|
||||
expect(
|
||||
h.engine.enqueued.some(
|
||||
(e) => e.task === "verifyAndImport" && (e.input as { mediaEntityId?: string }).mediaEntityId === entity,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(await resumeEpisodeOrganize(h.module.deps().ao, entity)).toEqual({ resumed: 1, waiting: 0 });
|
||||
const next = await h.db
|
||||
.selectFrom("organization_record")
|
||||
.selectAll()
|
||||
.where("media_entity_id", "=", entity)
|
||||
.where("id", "!=", original.recordId!)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect((next.plan as unknown as FrozenPlan).request?.sourcePaths).toEqual([second]);
|
||||
expect((await executeRecord(h.module.deps().ao, next.id)).status).toBe("succeeded");
|
||||
expect(
|
||||
await h.db
|
||||
.selectFrom("media_asset")
|
||||
.select("id")
|
||||
.where("media_entity_id", "=", entity)
|
||||
.where("status", "=", "active")
|
||||
.execute(),
|
||||
).toHaveLength(2);
|
||||
expect(await readFile(added, "utf8")).toBe("later".padEnd(128, "x"));
|
||||
expect(await sweepEpisodeOrganize(h.module.deps().ao)).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -264,6 +264,25 @@ describe.skipIf(!pg)("整理真实文件的季集映射闭环", () => {
|
||||
).rejects.toThrow("已对上的资源");
|
||||
});
|
||||
|
||||
it("首次混合整理保留已对上的文件,纯文件偏移也要通过回归保护", async () => {
|
||||
const entityId = await series();
|
||||
await settleOrganizeEpisodeDecisions(h.ctx, h.db, {
|
||||
recordId: randomUUID(),
|
||||
entityId,
|
||||
files: [await file(entityId, 1), await file(entityId, 40)],
|
||||
});
|
||||
const d = (await pending(entityId))[0]!;
|
||||
expect(episodeFileSamples(d.evidence)).toHaveLength(2);
|
||||
const simulation = await simulateForAgent(h.db, { decisionId: d.id, line: "S01 @ANi >> S02 EP-30" }, null);
|
||||
expect(simulation.summary).toMatchObject({ samples: 2, resolved: 1, regressions: 1, ok: false });
|
||||
await expect(
|
||||
applyAgentEpisodeRule(h.ctx, { decisionId: d.id, line: simulation.line, reasons: ["同批文件"] }, ACTOR, null),
|
||||
).rejects.toThrow("已对上的文件");
|
||||
expect(
|
||||
await h.db.selectFrom("identify_words").select("id").where("media_entity_id", "=", entityId).execute(),
|
||||
).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("保存部分有效的映射保留未决文件与已解决样本,不误关闭事项", async () => {
|
||||
const entityId = await series();
|
||||
const files = [await file(entityId, 40), await file(entityId, 60)];
|
||||
|
||||
@@ -105,14 +105,14 @@ export const BUDGET_CARD: CardSpec = {
|
||||
export const INVESTIGATE_CARD: CardSpec = {
|
||||
anchor: "investigate",
|
||||
title: "自动处理",
|
||||
description: "整理时识别不到作品就叫智能体识别;下载失败、集数对不上时自动发起一次调查,受下面的上限约束。",
|
||||
description: "整理时作品或季集认不出,智能体调查一次;其他下载问题按自动调查设置处理。",
|
||||
fields: [
|
||||
{
|
||||
name: "agent.autoIdentifyOrganize",
|
||||
label: "整理识别不到作品时叫智能体识别",
|
||||
label: "整理识别不到作品或季集时,交给智能体",
|
||||
kind: "switch",
|
||||
wide: true,
|
||||
hint: "认出来自动重新整理;认不出的留在整理记录「需要处理」里",
|
||||
hint: "认出来自动重新整理;季集规则有持续授权才自动写入,否则留下建议;拿不准的交给你确认",
|
||||
},
|
||||
{ name: "agent.autoInvestigate", label: "自动调查", kind: "switch", wide: true },
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user