perf(db): 标题模糊匹配改用 % 运算符走 trigram 索引,巡检回填加部分索引,删除无用索引
- 别名 / 资源标题模糊匹配原来写 similarity(x, q) >= t,函数调用用不上 trigram 索引;改为 x % q and similarity(x, q) >= t(% 缺省阈值 0.3 不高于各处阈值,结果相同)。source_release 标题搜索 在部署规模(8440 行)从约 75 ms 降到 0.7 ms - 调度器每 15 秒的巡检回填原来整扫 scheduled_job_run:迁移 0089 加部分索引 scheduled_job_run_harvest_idx - 迁移 0089 删除没有任何查询会用到的 content_unit_air_date_idx;镜像带 pg_stat_statements 时建扩展, 没有或权限不够时跳过,不让迁移失败 - gen:types 跳过扩展自带的视图(public.pg_stat_statements) Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
1 parent
155f30417c
commit
9fffee16c8
9 files changed
+69
-8
No files matched your search
@@ -240,7 +240,8 @@ export async function identifyCore(
|
|||||||
const rows = await sql<{ media_entity_id: string }>`
|
const rows = await sql<{ media_entity_id: string }>`
|
||||||
select distinct a.media_entity_id from title_alias a join media_entity e on e.id = a.media_entity_id
|
select distinct a.media_entity_id from title_alias a join media_entity e on e.id = a.media_entity_id
|
||||||
where e.status not in ('merged', 'archived')
|
where e.status not in ('merged', 'archived')
|
||||||
and (a.normalized = ${q} or (length(${q}) >= 3 and similarity(a.normalized, ${q}) >= ${TRGM}))
|
and (a.normalized = ${q}
|
||||||
|
${[...q].length >= 3 ? sql`or (a.normalized % ${q} and similarity(a.normalized, ${q}) >= ${TRGM})` : sql``})
|
||||||
limit 20`.execute(db);
|
limit 20`.execute(db);
|
||||||
for (const r of rows.rows) ids.add(r.media_entity_id);
|
for (const r of rows.rows) ids.add(r.media_entity_id);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -289,7 +289,10 @@ export class Scheduler {
|
|||||||
if (manual || status !== job.last_status || status === "failed") this.publish(job.id, status);
|
if (manual || status !== job.last_status || status === "failed") this.publish(job.id, status);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 巡检入队的 agent_run 结束后:回填终态与摘要;配置了 reportAsDecision 的写一条 info 待处理 */
|
/**
|
||||||
|
* 巡检入队的 agent_run 结束后:回填终态与摘要;配置了 reportAsDecision 的写一条 info 待处理。
|
||||||
|
* 条件与部分索引 scheduled_job_run_harvest_idx(迁移 0089)一致,每 15 秒一次不再全表扫运行历史。
|
||||||
|
*/
|
||||||
async harvest(): Promise<number> {
|
async harvest(): Promise<number> {
|
||||||
const rows = await sql<{
|
const rows = await sql<{
|
||||||
id: string;
|
id: string;
|
||||||
@@ -304,7 +307,7 @@ export class Scheduler {
|
|||||||
from scheduled_job_run r
|
from scheduled_job_run r
|
||||||
join scheduled_job j on j.id = r.job_id
|
join scheduled_job j on j.id = r.job_id
|
||||||
join agent_run a on a.id = r.agent_run_id
|
join agent_run a on a.id = r.agent_run_id
|
||||||
where r.agent_status is null and a.status = any(${TERMINAL_AGENT})
|
where r.agent_status is null and r.agent_run_id is not null and a.status = any(${TERMINAL_AGENT})
|
||||||
limit 50`.execute(this.db);
|
limit 50`.execute(this.db);
|
||||||
for (const r of rows.rows) {
|
for (const r of rows.rows) {
|
||||||
const summary = (r.summary ?? "").trim() || `智能体运行结束:${r.status}`;
|
const summary = (r.summary ?? "").trim() || `智能体运行结束:${r.status}`;
|
||||||
|
|||||||
@@ -47,7 +47,8 @@ export async function searchLocalMedia(
|
|||||||
a.alias as matched
|
a.alias as matched
|
||||||
from title_alias a join media_entity e on e.id = a.media_entity_id
|
from title_alias a join media_entity e on e.id = a.media_entity_id
|
||||||
where e.status <> 'merged'
|
where e.status <> 'merged'
|
||||||
and (a.normalized = ${n} or a.normalized like ${`${n}%`} or similarity(a.normalized, ${n}) >= 0.3)
|
and (a.normalized = ${n} or a.normalized like ${`${n}%`}
|
||||||
|
or (a.normalized % ${n} and similarity(a.normalized, ${n}) >= 0.3))
|
||||||
order by e.id, score desc
|
order by e.id, score desc
|
||||||
limit 200`.execute(ctx.db);
|
limit 200`.execute(ctx.db);
|
||||||
rows.push(...r.rows);
|
rows.push(...r.rows);
|
||||||
|
|||||||
@@ -39,7 +39,8 @@ export function registerSourcesCapabilities(registry: CapabilityRegistry, rt: ()
|
|||||||
select distinct on (e.id) e.*, greatest(similarity(a.normalized, ${n}), case when a.normalized = ${n} then 1 else 0 end) as score,
|
select distinct on (e.id) e.*, greatest(similarity(a.normalized, ${n}), case when a.normalized = ${n} then 1 else 0 end) as score,
|
||||||
a.alias as matched
|
a.alias as matched
|
||||||
from title_alias a join media_entity e on e.id = a.media_entity_id
|
from title_alias a join media_entity e on e.id = a.media_entity_id
|
||||||
where e.status <> 'merged' and (a.normalized = ${n} or similarity(a.normalized, ${n}) >= 0.3)
|
where e.status <> 'merged'
|
||||||
|
and (a.normalized = ${n} or (a.normalized % ${n} and similarity(a.normalized, ${n}) >= 0.3))
|
||||||
${input.kind ? sql`and e.kind = ${input.kind}` : sql``}
|
${input.kind ? sql`and e.kind = ${input.kind}` : sql``}
|
||||||
${input.year ? sql`and (e.year is null or abs(e.year - ${input.year}) <= 1)` : sql``}
|
${input.year ? sql`and (e.year is null or abs(e.year - ${input.year}) <= 1)` : sql``}
|
||||||
order by e.id, score desc`.execute(db);
|
order by e.id, score desc`.execute(db);
|
||||||
|
|||||||
@@ -389,7 +389,7 @@ export function identityRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
|||||||
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
|
if (query.mediaEntityId) q = q.where("media_entity_id", "=", query.mediaEntityId);
|
||||||
if (query.q) {
|
if (query.q) {
|
||||||
const n = normalizeTitle(query.q);
|
const n = normalizeTitle(query.q);
|
||||||
q = q.where(sql<boolean>`(normalized = ${n} or similarity(normalized, ${n}) >= 0.3)`);
|
q = q.where(sql<boolean>`(normalized = ${n} or (normalized % ${n} and similarity(normalized, ${n}) >= 0.3))`);
|
||||||
}
|
}
|
||||||
const rows = await q
|
const rows = await q
|
||||||
.orderBy("created_at")
|
.orderBy("created_at")
|
||||||
|
|||||||
@@ -852,7 +852,11 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
|||||||
if (query.resolutionStatus) q = q.where("resolution_status", "=", query.resolutionStatus);
|
if (query.resolutionStatus) q = q.where("resolution_status", "=", query.resolutionStatus);
|
||||||
if (query.q)
|
if (query.q)
|
||||||
q = q.where((eb) =>
|
q = q.where((eb) =>
|
||||||
eb.or([eb("title", "ilike", `%${query.q}%`), sql<boolean>`similarity(title, ${query.q}) >= 0.3`]),
|
eb.or([
|
||||||
|
eb("title", "ilike", `%${query.q}%`),
|
||||||
|
// `%` 让 source_release_title_trgm_idx 可用(两个分支都走索引,BitmapOr)
|
||||||
|
sql<boolean>`(title % ${query.q} and similarity(title, ${query.q}) >= 0.3)`,
|
||||||
|
]),
|
||||||
);
|
);
|
||||||
const rows = await q
|
const rows = await q
|
||||||
.orderBy("first_seen_at", "desc")
|
.orderBy("first_seen_at", "desc")
|
||||||
|
|||||||
@@ -164,9 +164,11 @@ async function localCandidates(
|
|||||||
for (const r of row) if (r.media_entity_id) ids.add(r.media_entity_id);
|
for (const r of row) if (r.media_entity_id) ids.add(r.media_entity_id);
|
||||||
}
|
}
|
||||||
for (const q of queryTitles(parsed)) {
|
for (const q of queryTitles(parsed)) {
|
||||||
|
// `%` 让 title_alias_normalized_trgm_idx 可用(阈值 0.3 ≤ TRGM_THRESHOLD,结果与只写 similarity 相同)
|
||||||
const rows = await sql<{ media_entity_id: string }>`
|
const rows = await sql<{ media_entity_id: string }>`
|
||||||
select distinct media_entity_id from title_alias
|
select distinct media_entity_id from title_alias
|
||||||
where normalized = ${q} or (length(${q}) >= 3 and similarity(normalized, ${q}) >= ${TRGM_THRESHOLD})
|
where normalized = ${q}
|
||||||
|
${[...q].length >= 3 ? sql`or (normalized % ${q} and similarity(normalized, ${q}) >= ${TRGM_THRESHOLD})` : sql``}
|
||||||
limit 20`.execute(exec);
|
limit 20`.execute(exec);
|
||||||
for (const r of rows.rows) ids.add(r.media_entity_id);
|
for (const r of rows.rows) ids.add(r.media_entity_id);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
import type { Kysely } from "kysely";
|
||||||
|
import { execDdl } from "../src/migration-helpers.ts";
|
||||||
|
|
||||||
|
// 数据库性能与可观测(docs/runbook/storage.md「数据库性能」):
|
||||||
|
// - scheduled_job_run_harvest_idx:调度器每 15 秒回填巡检结果(scheduler.harvest),原来每次全表扫运行历史;
|
||||||
|
// 部分索引只含「有 agent_run 且尚未回填」的行,通常为 0~几行;
|
||||||
|
// - 删除 content_unit_air_date_idx:没有任何查询按 air_date 过滤或排序(订阅日历在内存里按日期筛),
|
||||||
|
// 按作品取单元走 content_unit_entity_numbering_idx;
|
||||||
|
// - pg_stat_statements:镜像自带时才建(库级共享对象,放 public;down 不卸载)。要真正采集还需要
|
||||||
|
// postgres 以 shared_preload_libraries=pg_stat_statements 启动(docker-compose.yml),没有预加载时只是查询会报错,
|
||||||
|
// 状态页按「未启用」处理。没有该扩展或权限不够时跳过,不让迁移失败。
|
||||||
|
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型
|
||||||
|
export async function up(db: Kysely<any>): Promise<void> {
|
||||||
|
await execDdl(
|
||||||
|
db,
|
||||||
|
`
|
||||||
|
create index if not exists scheduled_job_run_harvest_idx on scheduled_job_run (agent_run_id)
|
||||||
|
where agent_status is null and agent_run_id is not null;
|
||||||
|
drop index if exists content_unit_air_date_idx;
|
||||||
|
do $$
|
||||||
|
begin
|
||||||
|
if exists (select 1 from pg_available_extensions where name = 'pg_stat_statements') then
|
||||||
|
begin
|
||||||
|
create extension if not exists pg_stat_statements with schema public;
|
||||||
|
exception when others then
|
||||||
|
raise notice 'pg_stat_statements 未安装:%', sqlerrm;
|
||||||
|
end;
|
||||||
|
end if;
|
||||||
|
end $$;
|
||||||
|
`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: 同上
|
||||||
|
export async function down(db: Kysely<any>): Promise<void> {
|
||||||
|
await execDdl(
|
||||||
|
db,
|
||||||
|
`
|
||||||
|
drop index if exists scheduled_job_run_harvest_idx;
|
||||||
|
create index if not exists content_unit_air_date_idx on content_unit (media_entity_id, air_date) where air_date is not null;
|
||||||
|
-- 扩展为库级共享对象,down 不卸载
|
||||||
|
`,
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -51,6 +51,10 @@ export async function generateTypesSource(db: Kysely<any>): Promise<string> {
|
|||||||
select table_name, column_name, data_type, udt_name, is_nullable, column_default
|
select table_name, column_name, data_type, udt_name, is_nullable, column_default
|
||||||
from information_schema.columns
|
from information_schema.columns
|
||||||
where table_schema = current_schema() and table_name not like 'kysely_%'
|
where table_schema = current_schema() and table_name not like 'kysely_%'
|
||||||
|
-- 扩展自带的视图(如 public.pg_stat_statements,迁移 0089)不属于应用表
|
||||||
|
and not exists (select 1 from pg_depend d join pg_class r on r.oid = d.objid
|
||||||
|
where d.classid = 'pg_class'::regclass and d.deptype = 'e'
|
||||||
|
and r.relname = table_name and r.relnamespace = current_schema()::regnamespace)
|
||||||
order by table_name, ordinal_position`.execute(db);
|
order by table_name, ordinal_position`.execute(db);
|
||||||
const checks = await sql<{ table_name: string; def: string }>`
|
const checks = await sql<{ table_name: string; def: string }>`
|
||||||
select c.relname as table_name, pg_get_constraintdef(k.oid) as def
|
select c.relname as table_name, pg_get_constraintdef(k.oid) as def
|
||||||
|
|||||||
Reference in new issue
Block a user