feat(identity): lazy TMDB lookup and bounded identity pool (DEC-1)
RSS/list releases that match no subscription or local work no longer hit
TMDB or open a pending item; they go to the identity pool as `unmatched`.
TMDB is queried only on a local alias hit, a published new-title rule whose
prefilter passes, manual/Agent retry, search results, or when the new
setting identity.eagerTmdbLookup (default off) is on. Mirrors MoviePilot:
torrents are recognised only when matched to a subscription or searched,
and recognition failures are logged, not alerted.
- Pending items only for related releases (ambiguous after lookup, or
related to a local work but unresolved); same source + normalized title
merge into one decision ("来源「X」的 N 条资源待识别"), resolve/dismiss
applies to the whole group, members detach as they get resolved.
- Pool governance: unmatched reviews capped (daily, 3x), title_alias
insert trigger wakes similar entries, stale unmatched entries archived
after identity.poolArchiveDays (30); review idempotency keys include the
due time.
- Migration 0047_identity_pool: reason/outcome `unmatched`,
identity_candidate.normalized_title (+trgm index), wake trigger, and
cleanup of open identity_unresolved items with no related work
(dismissed, resolution=moved_to_pool).
- Inbox: group by type + work/source with collapse, hide info-only items
by default (minSeverity=warning), badge counts warning+ only; pool sheet
gets manual TMDB/local-work assign and dismiss; settings card for the
new identity settings.
- Runbook docs/runbook/identity.md.
Requires rebuild/restart (migration runs on start).
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
524af64bec
commit
94fd48bff2
24 files changed
+1326
-153
No files matched your search
@@ -126,8 +126,11 @@ export type IdentityRevisionCreate = z.infer<typeof IdentityRevisionCreate>;
|
||||
|
||||
export const IdentityCandidateStatus = z.enum(["pending", "resolved", "dismissed", "archived"]);
|
||||
export type IdentityCandidateStatus = z.infer<typeof IdentityCandidateStatus>;
|
||||
/** proposed = 证据足够但自动化关闭/只读,待确认 */
|
||||
export const IdentityCandidateReason = z.enum(["ambiguous", "no_match", "proposed", "lookup_failed"]);
|
||||
/**
|
||||
* proposed = 证据足够但自动化关闭/只读,待确认;
|
||||
* unmatched = 惰性识别:本地召回无结果且没有触发 TMDB 查询(不相关的 RSS / 列表资源),只进池、不开待处理
|
||||
*/
|
||||
export const IdentityCandidateReason = z.enum(["ambiguous", "no_match", "proposed", "lookup_failed", "unmatched"]);
|
||||
export type IdentityCandidateReason = z.infer<typeof IdentityCandidateReason>;
|
||||
|
||||
/** 待识别池里的一个候选(只存 JSON,不建正式作品) */
|
||||
@@ -151,6 +154,8 @@ export const IdentityCandidate = EntityBase.extend({
|
||||
reason: IdentityCandidateReason,
|
||||
/** 解析出的检索条件(标题/年份/季集) */
|
||||
query: JsonObject,
|
||||
/** 归一化标题(与 title_alias.normalized 同口径):新增别名时按它唤醒复查,待处理按"来源 + 它"合并 */
|
||||
normalizedTitle: z.string().nullable(),
|
||||
candidates: z.array(PoolCandidate),
|
||||
evidenceVersion: z.string(),
|
||||
attempts: z.number().int().nonnegative(),
|
||||
@@ -171,6 +176,7 @@ export const ResolutionOutcome = z.enum([
|
||||
"lookup_failed",
|
||||
"manual",
|
||||
"skipped_locked",
|
||||
"unmatched",
|
||||
]);
|
||||
export type ResolutionOutcome = z.infer<typeof ResolutionOutcome>;
|
||||
|
||||
|
||||
@@ -357,6 +357,19 @@ export const BudgetSettings = z.object({
|
||||
});
|
||||
export type BudgetSettings = z.infer<typeof BudgetSettings>;
|
||||
|
||||
/**
|
||||
* 身份识别(DEC-1 惰性识别,对照 MP:RSS / 列表资源只在匹配订阅或用户搜索时识别)。
|
||||
* eagerTmdbLookup = true 恢复旧行为:每条新观察到的资源都查 TMDB。每条资源约 1–2 次搜索 + 命中时 1 次详情
|
||||
* (识别缓存命中则 0 次);一个每小时 100 条的 RSS 约为每天 2400–7200 次 TMDB 请求,多数是站点独有的短剧 / 综艺,查不到。
|
||||
* 关闭(缺省)时只在命中订阅 / 本地作品别名、已发布的新作品收录规则前置条件通过、用户手动识别或搜索时查询。
|
||||
*/
|
||||
export const IdentitySettings = z.object({
|
||||
eagerTmdbLookup: z.boolean(),
|
||||
/** 待识别池里 unmatched 条目多少天没有新证据就归档(仍可按状态筛出、可手动指定) */
|
||||
poolArchiveDays: z.number().int().min(1).max(3650),
|
||||
});
|
||||
export type IdentitySettings = z.infer<typeof IdentitySettings>;
|
||||
|
||||
export const GlobalSettings = z.object({
|
||||
automation: AutomationSettings,
|
||||
cache: CacheSettings,
|
||||
@@ -368,6 +381,7 @@ export const GlobalSettings = z.object({
|
||||
metadata: MetadataSettings,
|
||||
sites: SitesSettings,
|
||||
naming: NamingSettings,
|
||||
identity: IdentitySettings,
|
||||
});
|
||||
export type GlobalSettings = z.infer<typeof GlobalSettings>;
|
||||
|
||||
@@ -438,6 +452,7 @@ export const DEFAULT_SETTINGS: GlobalSettings = {
|
||||
userDataMaxSeedingPages: 10,
|
||||
},
|
||||
naming: { ...DEFAULT_NAMING_TEMPLATES, lockFileNameForBound: false },
|
||||
identity: { eagerTmdbLookup: false, poolArchiveDays: 30 },
|
||||
};
|
||||
|
||||
/** PATCH /settings:按分组部分更新;服务端与现值合并后用 GlobalSettings 整体再校验 */
|
||||
@@ -486,6 +501,7 @@ export const GlobalSettingsUpdate = z.object({
|
||||
.optional(),
|
||||
sites: SitesSettings.partial().optional(),
|
||||
naming: NamingSettings.partial().optional(),
|
||||
identity: IdentitySettings.partial().optional(),
|
||||
});
|
||||
export type GlobalSettingsUpdate = z.infer<typeof GlobalSettingsUpdate>;
|
||||
|
||||
@@ -510,6 +526,7 @@ export function mergeSettings(current: GlobalSettings, patch: GlobalSettingsUpda
|
||||
metadata: { ...current.metadata, ...metadata },
|
||||
sites: { ...(current.sites ?? DEFAULT_SETTINGS.sites), ...patch.sites },
|
||||
naming: { ...(current.naming ?? DEFAULT_SETTINGS.naming), ...patch.naming },
|
||||
identity: { ...(current.identity ?? DEFAULT_SETTINGS.identity), ...patch.identity },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -666,11 +683,12 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [
|
||||
{
|
||||
id: "metadata",
|
||||
label: "元数据",
|
||||
description: "TMDB API Key 与地址、TMDB 缓存、图片代理",
|
||||
description: "TMDB API Key 与地址、TMDB 缓存、资源识别、图片代理",
|
||||
legacyTabs: [],
|
||||
anchors: [
|
||||
{ id: "tmdb", label: "TMDB" },
|
||||
{ id: "tmdb-cache", label: "TMDB 缓存" },
|
||||
{ id: "identity", label: "资源识别" },
|
||||
{ id: "images", label: "图片代理与缓存" },
|
||||
],
|
||||
fields: [
|
||||
@@ -679,6 +697,8 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [
|
||||
"cache.tmdbTtlHours",
|
||||
"cache.tmdbMaxSize",
|
||||
"cache.emptyResultTtlSec",
|
||||
"identity.eagerTmdbLookup",
|
||||
"identity.poolArchiveDays",
|
||||
"cache.globalImageCache",
|
||||
"cache.globalImageCacheDays",
|
||||
],
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import { execDdl, oneOf } from "../src/migration-helpers.ts";
|
||||
|
||||
// DEC-1 惰性识别 + 待识别池治理(REQ §4.2–4.3、§7.5、§13;对照 MP:RSS 资源只在匹配订阅 / 用户搜索时识别,识别失败不告警):
|
||||
// - identity_candidate.reason 新增 unmatched:本地召回无结果、也不属于任何需要 TMDB 查询的触发(订阅别名 / 新作品收录规则 /
|
||||
// 手动 / 搜索),只进池、不查 TMDB、不开待处理;
|
||||
// - resolution_attempt.outcome 同步新增 unmatched;
|
||||
// - identity_candidate.normalized_title:归一化标题(与 title_alias.normalized 同一口径),也是待处理合并分组键
|
||||
// (来源 + 归一化标题)的一部分;title_alias 插入触发器按它唤醒相近的池条目复查(新证据到来即重试);
|
||||
// - 清理存量:未关闭的 identity_unresolved 事项里,资源未关联作品、池内无候选、标题也不命中任何本地别名的,
|
||||
// 事项关闭为 dismissed(resolution.resolution = 'moved_to_pool'),池条目改为 unmatched 并解除事项关联。
|
||||
|
||||
const OLD_REASONS = ["ambiguous", "no_match", "proposed", "lookup_failed"] as const;
|
||||
const OLD_OUTCOMES = [
|
||||
"auto",
|
||||
"proposed",
|
||||
"ambiguous",
|
||||
"no_match",
|
||||
"lookup_failed",
|
||||
"manual",
|
||||
"skipped_locked",
|
||||
] as const;
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
alter table identity_candidate drop constraint identity_candidate_reason_check;
|
||||
alter table identity_candidate add constraint identity_candidate_reason_check
|
||||
${oneOf("reason", [...OLD_REASONS, "unmatched"])};
|
||||
alter table resolution_attempt drop constraint resolution_attempt_outcome_check;
|
||||
alter table resolution_attempt add constraint resolution_attempt_outcome_check
|
||||
${oneOf("outcome", [...OLD_OUTCOMES, "unmatched"])};
|
||||
|
||||
alter table identity_candidate add column normalized_title text;
|
||||
update identity_candidate set normalized_title = nullif(query->>'normalized', '');
|
||||
create index identity_candidate_normalized_trgm_idx on identity_candidate
|
||||
using gin (normalized_title gin_trgm_ops) where status = 'pending';
|
||||
|
||||
-- 新证据唤醒:任何路径新增别名(建档 / 手动加别名 / 合并)后,归一化标题相同或足够相近(与本地召回同阈值 0.45)的
|
||||
-- unmatched / no_match 池条目立即到期复查(复查仍是本地召回优先,命中别名才查 TMDB)
|
||||
create function identity_pool_wake_on_alias() returns trigger language plpgsql as $$
|
||||
begin
|
||||
if new.normalized is null or new.normalized = '' then return null; end if;
|
||||
update identity_candidate set next_review_at = now()
|
||||
where status = 'pending' and reason in ('unmatched', 'no_match') and normalized_title is not null
|
||||
and (next_review_at is null or next_review_at > now())
|
||||
and (normalized_title = new.normalized
|
||||
or (length(new.normalized) >= 3 and normalized_title % new.normalized
|
||||
and similarity(normalized_title, new.normalized) >= 0.45));
|
||||
return null;
|
||||
end $$;
|
||||
create trigger title_alias_wake_identity_pool after insert on title_alias
|
||||
for each row execute function identity_pool_wake_on_alias();
|
||||
|
||||
with moved as (
|
||||
select p.id as pool_id, d.id as decision_id
|
||||
from identity_candidate p
|
||||
join source_release r on r.id = p.source_release_id
|
||||
join decision d on d.id = p.decision_id or d.dedupe_key = 'identity:' || p.source_release_id::text
|
||||
where d.kind = 'identity_unresolved'
|
||||
and d.status in ('open', 'investigating', 'awaiting_user')
|
||||
and p.status = 'pending'
|
||||
and r.media_entity_id is null
|
||||
and jsonb_array_length(p.candidates) = 0
|
||||
and not exists (
|
||||
select 1 from title_alias a
|
||||
where p.normalized_title is not null
|
||||
and (a.normalized = p.normalized_title
|
||||
or (length(p.normalized_title) >= 3 and similarity(a.normalized, p.normalized_title) >= 0.45)))
|
||||
), closed as (
|
||||
update decision set status = 'dismissed',
|
||||
resolution = jsonb_build_object('resolution', 'moved_to_pool', 'by', 'migration:0047_identity_pool'),
|
||||
resolved_by = 'system:identity-pool', resolved_at = now()
|
||||
where id in (select decision_id from moved)
|
||||
returning id
|
||||
)
|
||||
update identity_candidate set reason = 'unmatched', decision_id = null, next_review_at = null
|
||||
where id in (select pool_id from moved);
|
||||
`,
|
||||
);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await execDdl(
|
||||
db,
|
||||
`
|
||||
update identity_candidate set reason = 'no_match' where reason = 'unmatched';
|
||||
update resolution_attempt set outcome = 'no_match' where outcome = 'unmatched';
|
||||
drop trigger if exists title_alias_wake_identity_pool on title_alias;
|
||||
drop function if exists identity_pool_wake_on_alias();
|
||||
drop index if exists identity_candidate_normalized_trgm_idx;
|
||||
alter table identity_candidate drop column normalized_title;
|
||||
alter table identity_candidate drop constraint identity_candidate_reason_check;
|
||||
alter table identity_candidate add constraint identity_candidate_reason_check ${oneOf("reason", OLD_REASONS)};
|
||||
alter table resolution_attempt drop constraint resolution_attempt_outcome_check;
|
||||
alter table resolution_attempt add constraint resolution_attempt_outcome_check ${oneOf("outcome", OLD_OUTCOMES)};
|
||||
`,
|
||||
);
|
||||
}
|
||||
@@ -139,7 +139,7 @@ export type FeedSourceHealthStatus = "unknown" | "healthy" | "degraded" | "faili
|
||||
export type FieldOverrideSource = "tmdb" | "user" | "agent" | "nfo" | "site" | "import";
|
||||
export type FieldOverrideActorType = "user" | "agent" | "system" | "api_key";
|
||||
export type IdentityCandidateStatus = "pending" | "resolved" | "dismissed" | "archived";
|
||||
export type IdentityCandidateReason = "ambiguous" | "no_match" | "proposed" | "lookup_failed";
|
||||
export type IdentityCandidateReason = "ambiguous" | "no_match" | "proposed" | "lookup_failed" | "unmatched";
|
||||
export type IdentityRedirectActorType = "user" | "agent" | "system" | "api_key";
|
||||
export type IdentityRevisionAction =
|
||||
| "created"
|
||||
@@ -210,7 +210,8 @@ export type ResolutionAttemptOutcome =
|
||||
| "no_match"
|
||||
| "lookup_failed"
|
||||
| "manual"
|
||||
| "skipped_locked";
|
||||
| "skipped_locked"
|
||||
| "unmatched";
|
||||
export type ResolutionAttemptActorType = "user" | "agent" | "system" | "api_key";
|
||||
export type RuleGroupKind = "acquisition" | "upgrade" | "search";
|
||||
export type RuleGroupSource = "manual" | "chat" | "agent";
|
||||
@@ -787,6 +788,7 @@ export interface IdentityCandidateTable {
|
||||
status: Generated<IdentityCandidateStatus>;
|
||||
reason: IdentityCandidateReason;
|
||||
query: Generated<Json>;
|
||||
normalized_title: string | null;
|
||||
candidates: Generated<Json>;
|
||||
evidence_version: string;
|
||||
attempts: Generated<number>;
|
||||
|
||||
Reference in new issue
Block a user