merge: feat/y-meta-cache
Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
rootandClaude Opus 5.5 committed
commit
6d37a337c9
51 files changed
+2407
-200
No files matched your search
@@ -0,0 +1,126 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import type { ScrapeFileResult, ScrapeMatch } from "@umt/contracts";
|
||||
import { createTempTree, type TempTree } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { queueScrape, runScrapeRecord } from "../../src/modules/acquisition-organize/services/scrape.ts";
|
||||
import { createHarness, type Harness, pgAvailable, seedBase } from "./harness.ts";
|
||||
|
||||
// TMDB 标志 → clearlogo.png:电影写影片目录、剧集写剧集根目录;覆盖方式与其它图片一致
|
||||
// (更新:没变不重写、本工具写过且没被改过的按新图覆盖、被人改过的跳过;只补缺失;全部覆盖)。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const at = new Date().toISOString();
|
||||
const f = (value: unknown) => ({ value, source: "tmdb", language: null, at, locked: false });
|
||||
const enc = (s: string) => new TextEncoder().encode(s);
|
||||
|
||||
describe.skipIf(!hasPg)("刮削:TMDB 标志写 clearlogo.png", () => {
|
||||
let h: Harness;
|
||||
let lib: TempTree;
|
||||
let rootId = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
lib = await createTempTree(
|
||||
{
|
||||
"Movies/星际穿越 (2014)/Interstellar.2014.mkv": 100,
|
||||
"TV/人生切割术 (2022)/Season 1/Severance.S01E01.mkv": 100,
|
||||
},
|
||||
{ sentinel: true, prefix: "umt-scrlogo-" },
|
||||
);
|
||||
h = await createHarness("scrape_logo", {
|
||||
// 内容 = 路径,方便看出写的是哪张
|
||||
scrapeImage: async (p) => ({ content: enc(p), mime: p.endsWith(".png") ? "image/png" : "image/jpeg" }),
|
||||
});
|
||||
rootId = (await seedBase(h.db, { libraryPath: lib.root, cachePath: lib.root, qbUrl: "http://127.0.0.1:9" })).rootId;
|
||||
}, 60_000);
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await lib?.cleanup();
|
||||
});
|
||||
|
||||
async function entity(kind: "movie" | "series", title: string, meta: object) {
|
||||
return (
|
||||
await h.db
|
||||
.insertInto("media_entity")
|
||||
.values({ kind, title, year: 2020, status: "active", metadata: JSON.stringify(meta) })
|
||||
.returning("id")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
}
|
||||
const setMeta = (id: string, meta: object) =>
|
||||
h.db
|
||||
.updateTable("media_entity")
|
||||
.set({ metadata: JSON.stringify(meta) })
|
||||
.where("id", "=", id)
|
||||
.execute();
|
||||
async function scrape(
|
||||
entityId: string,
|
||||
kind: "movie" | "tv",
|
||||
path: string,
|
||||
mode?: "missing" | "overwrite" | "update",
|
||||
) {
|
||||
const match: ScrapeMatch = {
|
||||
mediaEntityId: entityId,
|
||||
provider: null,
|
||||
externalId: null,
|
||||
kind,
|
||||
title: "x",
|
||||
year: null,
|
||||
source: "library",
|
||||
};
|
||||
const q = await queueScrape(h.d, {
|
||||
rootId: `lib:${rootId}`,
|
||||
rootPath: lib.root,
|
||||
libraryRootId: rootId,
|
||||
path,
|
||||
match,
|
||||
mode,
|
||||
trigger: "manual",
|
||||
});
|
||||
return runScrapeRecord(h.d, q.recordId);
|
||||
}
|
||||
const logo = (files: ScrapeFileResult[]) => files.find((x) => x.role === "clearlogo");
|
||||
const MOVIE_DIR = "Movies/星际穿越 (2014)";
|
||||
const LOGO = `${MOVIE_DIR}/clearlogo.png`;
|
||||
|
||||
it("电影:写在影片目录 clearlogo.png;更新模式内容没变不重写,换了标志按新图覆盖", async () => {
|
||||
const movie = await entity("movie", "星际穿越", { title: f("星际穿越"), logoPath: f("/logo-a.png") });
|
||||
const r = await scrape(movie, "movie", MOVIE_DIR);
|
||||
expect(logo(r.files)).toMatchObject({
|
||||
path: LOGO,
|
||||
action: "written",
|
||||
hash: expect.stringMatching(/^[0-9a-f]{64}$/),
|
||||
});
|
||||
expect(await readFile(lib.path(LOGO), "utf8")).toBe("/logo-a.png");
|
||||
const same = await scrape(movie, "movie", MOVIE_DIR, "update");
|
||||
expect(logo(same.files)).toMatchObject({ action: "skipped", reason: "scrape_unchanged" });
|
||||
await setMeta(movie, { title: f("星际穿越"), logoPath: f("/logo-b.png") });
|
||||
const upd = await scrape(movie, "movie", MOVIE_DIR, "update");
|
||||
expect(logo(upd.files)).toMatchObject({ action: "written" });
|
||||
expect(await readFile(lib.path(LOGO), "utf8")).toBe("/logo-b.png");
|
||||
|
||||
// 被人换掉的标志:更新跳过、只补缺失跳过、全部覆盖改写
|
||||
await writeFile(lib.path(LOGO), "我自己的标志");
|
||||
await setMeta(movie, { title: f("星际穿越"), logoPath: f("/logo-c.png") });
|
||||
expect(logo((await scrape(movie, "movie", MOVIE_DIR, "update")).files)).toMatchObject({
|
||||
action: "skipped",
|
||||
reason: "scrape_user_modified",
|
||||
});
|
||||
expect(logo((await scrape(movie, "movie", MOVIE_DIR, "missing")).files)).toMatchObject({
|
||||
action: "skipped",
|
||||
reason: "scrape_exists",
|
||||
});
|
||||
expect(await readFile(lib.path(LOGO), "utf8")).toBe("我自己的标志");
|
||||
expect(logo((await scrape(movie, "movie", MOVIE_DIR, "overwrite")).files)).toMatchObject({ action: "written" });
|
||||
expect(await readFile(lib.path(LOGO), "utf8")).toBe("/logo-c.png");
|
||||
});
|
||||
|
||||
it("剧集:写在剧集根目录,季目录不写;没有标志不写", async () => {
|
||||
const show = await entity("series", "人生切割术", { title: f("人生切割术"), logoPath: f("/sev.png") });
|
||||
const r = await scrape(show, "tv", "TV/人生切割术 (2022)");
|
||||
expect(logo(r.files)).toMatchObject({ path: "TV/人生切割术 (2022)/clearlogo.png", action: "written" });
|
||||
expect(r.files.filter((x) => x.role === "clearlogo")).toHaveLength(1);
|
||||
await setMeta(show, { title: f("人生切割术") });
|
||||
const none = await scrape(show, "tv", "TV/人生切割术 (2022)", "overwrite");
|
||||
expect(logo(none.files)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
Vendored
+13
-5
@@ -67,7 +67,10 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
await (await new SourcesRuntime(h.ctx).tmdb())!.details("movie", 872585);
|
||||
const calls = tmdb.calls.length;
|
||||
const rows = (
|
||||
await sql<{ key: string; days: number }>`select key, extract(epoch from expires_at - now()) / 86400 as days
|
||||
await sql<{
|
||||
key: string;
|
||||
days: number;
|
||||
}>`select key, extract(epoch from coalesce(fresh_until, expires_at) - now()) / 86400 as days
|
||||
from kv_cache where region = 'tmdb.request'`.execute(h.db)
|
||||
).rows;
|
||||
const detail = rows.find((x) => x.key.startsWith("/movie/872585?"))!;
|
||||
@@ -80,7 +83,7 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
const days = async (like: string) =>
|
||||
Number(
|
||||
(
|
||||
await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d
|
||||
await sql<{ d: number }>`select extract(epoch from coalesce(fresh_until, expires_at) - now()) / 86400 as d
|
||||
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
||||
).rows[0]!.d,
|
||||
);
|
||||
@@ -96,7 +99,7 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
expect(empty).toBeLessThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("追踪中的作品:详情缓存命中时续期到 now + 详情 TTL;未追踪的不动", async () => {
|
||||
it("关注中的作品:详情缓存命中时续期到 now + 详情 TTL 并标 pinned;未关注的不动", async () => {
|
||||
const ent = (
|
||||
await sql<{
|
||||
id: string;
|
||||
@@ -111,19 +114,24 @@ describe.skipIf(!hasPg)("C2-A 缓存(真实 PG)", () => {
|
||||
const client = (await new SourcesRuntime(h.ctx).tmdb())!;
|
||||
await client.details("movie", 872585);
|
||||
await client.details("movie", 693134);
|
||||
await sql`update kv_cache set expires_at = now() + interval '1 day'
|
||||
await sql`update kv_cache set fresh_until = now() + interval '1 day', expires_at = now() + interval '2 days',
|
||||
pinned = false
|
||||
where region = 'tmdb.request' and (key like '/movie/872585?%' or key like '/movie/693134?%')`.execute(h.db);
|
||||
await client.details("movie", 872585);
|
||||
await client.details("movie", 693134);
|
||||
const left = async (like: string) =>
|
||||
Number(
|
||||
(
|
||||
await sql<{ d: number }>`select extract(epoch from expires_at - now()) / 86400 as d
|
||||
await sql<{ d: number }>`select extract(epoch from fresh_until - now()) / 86400 as d
|
||||
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
||||
).rows[0]!.d,
|
||||
);
|
||||
expect(await left("/movie/872585?%")).toBeGreaterThan(19.9);
|
||||
expect(await left("/movie/693134?%")).toBeLessThanOrEqual(1);
|
||||
const pinned = await sql<{ k: string; p: boolean }>`select key as k, pinned as p from kv_cache
|
||||
where region = 'tmdb.request' and (key like '/movie/872585?%' or key like '/movie/693134?%')`.execute(h.db);
|
||||
expect(pinned.rows.filter((r) => r.p).every((r) => r.k.startsWith("/movie/872585?"))).toBe(true);
|
||||
expect(pinned.rows.some((r) => r.p)).toBe(true);
|
||||
});
|
||||
|
||||
it("GET /cache/tmdb 列出条目(键 / 过期 / 大小)与上限;POST /cache/tmdb/clear 清空", async () => {
|
||||
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
import { metadataCacheFor, PgCache, sql } from "@umt/db";
|
||||
import { createMockTmdb, type MockTmdb } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { createCacheModule } from "../../src/modules/cache/index.ts";
|
||||
import { SourcesRuntime } from "../../src/modules/sources-identity/runtime.ts";
|
||||
import { metaRequestCache } from "../../src/modules/sources-identity/service/meta-subjects.ts";
|
||||
import { metaCachePolicy, regionHitStats, resetTrackedTmdbRefs } from "../../src/shared-cache.ts";
|
||||
import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts";
|
||||
import { DEV_KEY, json } from "../helpers.ts";
|
||||
|
||||
// 统一元数据缓存接到 TMDB / 豆瓣 / bgm:策略表、过期先用旧值后台刷新、同请求合并、失败不缓存、
|
||||
// 关注中作品 pinned 不被淘汰、按 region 统计命中(GET /cache/regions)。
|
||||
|
||||
const hasPg = await pgAvailable();
|
||||
const KEY = "0123456789abcdef0123456789abcdef";
|
||||
|
||||
const until = async (cond: () => boolean | Promise<boolean>, ms = 5000) => {
|
||||
const end = Date.now() + ms;
|
||||
while (!(await cond())) {
|
||||
if (Date.now() > end) throw new Error("等待超时");
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
}
|
||||
};
|
||||
|
||||
describe("策略表(来源 × 资源类型)", () => {
|
||||
it("缺省:搜索 7 天(空 1 天)、详情 30 天(空 1 天)、列表 6 小时(空 1 小时)、图片 90 天", async () => {
|
||||
const { DEFAULT_STORAGE_SETTINGS } = await import("@umt/contracts");
|
||||
const t = DEFAULT_STORAGE_SETTINGS.cacheTtl;
|
||||
const day = 86_400;
|
||||
for (const s of ["tmdb", "douban", "bangumi"] as const) {
|
||||
expect(metaCachePolicy(t, s, "search")).toEqual({ ttl: 7 * day, emptyTtl: day });
|
||||
expect(metaCachePolicy(t, s, "detail")).toEqual({ ttl: 30 * day, emptyTtl: day });
|
||||
expect(metaCachePolicy(t, s, "list")).toEqual({ ttl: 6 * 3600, emptyTtl: 3600 });
|
||||
}
|
||||
expect(metaCachePolicy(t, "tmdb", "image")).toEqual({ ttl: 90 * day, emptyTtl: day });
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!hasPg)("统一元数据缓存(真实 PG)", () => {
|
||||
let h: Harness;
|
||||
let admin: ReturnType<typeof bearer>;
|
||||
let tmdb: MockTmdb;
|
||||
|
||||
beforeAll(async () => {
|
||||
h = await createHarness({ extraModules: [createCacheModule()] });
|
||||
admin = bearer(h.app, DEV_KEY);
|
||||
tmdb = await createMockTmdb({ apiKey: KEY });
|
||||
const r = await admin("PATCH", "/settings", {
|
||||
metadata: { tmdbApiKey: KEY, tmdbBaseUrl: `${tmdb.url}/3` },
|
||||
cache: { tmdbMaxSize: 16 },
|
||||
});
|
||||
expect(r.status).toBe(200);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await tmdb?.close();
|
||||
await h?.close();
|
||||
});
|
||||
|
||||
const client = async () => (await new SourcesRuntime(h.ctx).tmdb())!;
|
||||
const row = async (like: string) =>
|
||||
(
|
||||
await sql<{ key: string; fresh: number; pinned: boolean }>`select key,
|
||||
extract(epoch from fresh_until - now()) / 86400 as fresh, pinned
|
||||
from kv_cache where region = 'tmdb.request' and key like ${like}`.execute(h.db)
|
||||
).rows[0];
|
||||
|
||||
it("TMDB 详情过期在宽限期内:先返回旧值,后台只刷一次;刷新后是新值", async () => {
|
||||
const c = await client();
|
||||
await c.details("movie", 872585);
|
||||
const before = tmdb.callsTo("/3/movie/872585").length;
|
||||
// 改成「已过新鲜期、还在宽限期」并在值里做个记号(热度 12345)
|
||||
await sql`update kv_cache set fresh_until = now() - interval '1 second', expires_at = now() + interval '1 day',
|
||||
value = jsonb_set(value, '{popularity}', '12345')
|
||||
where region = 'tmdb.request' and key like '/movie/872585?%'`.execute(h.db);
|
||||
const keys = (
|
||||
await sql<{
|
||||
key: string;
|
||||
}>`select key from kv_cache where region = 'tmdb.request' and key like '/movie/872585?%'`.execute(h.db)
|
||||
).rows.map((r) => r.key);
|
||||
expect(keys.length).toBeGreaterThan(0);
|
||||
// 回源慢一点,让两个调用方都在刷新完成前读到
|
||||
tmdb.fault("/3/movie/872585", { delayMs: 300, times: 10 });
|
||||
const [a, b] = await Promise.all([c.details("movie", 872585), c.details("movie", 872585)]);
|
||||
expect(a.popularity).toBe(12345);
|
||||
expect(b.popularity).toBe(12345);
|
||||
// 后台刷新:两个调用方同时拿到旧值,每个过期键只回源一次;刷新后新鲜且是新值
|
||||
await until(async () => {
|
||||
const r = await sql<{ n: number }>`select count(*)::int as n from kv_cache where region = 'tmdb.request'
|
||||
and key like '/movie/872585?%' and fresh_until > now() + interval '29 days'`.execute(h.db);
|
||||
return r.rows[0]!.n === keys.length;
|
||||
});
|
||||
expect(tmdb.callsTo("/3/movie/872585").length - before).toBe(keys.length);
|
||||
tmdb.clearFaults();
|
||||
expect((await c.details("movie", 872585)).popularity).not.toBe(12345);
|
||||
});
|
||||
|
||||
it("同请求合并:并发同一详情只回源一次;失败(5xx)不缓存", async () => {
|
||||
const c = await client();
|
||||
const n0 = tmdb.callsTo("/3/tv/95396/season/1").length;
|
||||
await Promise.all(Array.from({ length: 5 }, () => c.getJson("/tv/95396/season/1", { language: "zh-CN" })));
|
||||
expect(tmdb.callsTo("/3/tv/95396/season/1").length - n0).toBe(1);
|
||||
// 5xx 重试用尽后抛错(客户端缺省重试 3 次,退避约 3.5 秒),不写缓存
|
||||
tmdb.fault("/3/tv/100088/season/1", { status: 503, times: 10 });
|
||||
await expect(c.getJson("/tv/100088/season/1", { language: "zh-CN" })).rejects.toThrow();
|
||||
expect(await row("/tv/100088/season/1?%")).toBeUndefined();
|
||||
tmdb.clearFaults();
|
||||
});
|
||||
|
||||
it("关注中的作品:条目 pinned,条数上限淘汰时不被清掉", async () => {
|
||||
const ent = (
|
||||
await sql<{
|
||||
id: string;
|
||||
}>`insert into media_entity (kind, title) values ('series', '人生切割术') returning id`.execute(h.db)
|
||||
).rows[0]!.id;
|
||||
await sql`insert into external_identity (media_entity_id, provider, entity_type, external_id, status)
|
||||
values (${ent}, 'tmdb', 'tv', '95396', 'accepted')`.execute(h.db);
|
||||
await sql`insert into subscription (media_entity_id, media_kind, status) values (${ent}, 'series', 'active')`.execute(
|
||||
h.db,
|
||||
);
|
||||
resetTrackedTmdbRefs(h.ctx);
|
||||
await sql`delete from kv_cache where region = 'tmdb.request'`.execute(h.db);
|
||||
const c = await client();
|
||||
await c.getJson("/tv/95396", { language: "zh-CN" });
|
||||
expect((await row("/tv/95396?%"))?.pinned).toBe(true);
|
||||
// 上限 16:再写一批别的请求,关注作品的条目留着
|
||||
for (let i = 0; i < 24; i++) await c.searchRaw("movie", `不存在${i}`);
|
||||
expect(await row("/tv/95396?%")).toBeDefined();
|
||||
const n = await sql<{ n: number }>`select count(*)::int as n from kv_cache where region = 'tmdb.request'`.execute(
|
||||
h.db,
|
||||
);
|
||||
expect(n.rows[0]!.n).toBeLessThanOrEqual(16);
|
||||
});
|
||||
|
||||
it("豆瓣 / bgm 请求缓存按策略表:搜索 7 天、列表 6 小时、空结果按查无 / 列表空;失败不写", async () => {
|
||||
const cache = metaRequestCache(h.ctx);
|
||||
await cache.cached({ source: "douban", key: "GET s1", kind: "search", empty: () => false }, async () => [1]);
|
||||
await cache.cached({ source: "douban", key: "GET s0", kind: "search", empty: () => true }, async () => []);
|
||||
await cache.cached({ source: "bangumi", key: "GET l1", kind: "list", empty: () => false }, async () => ({
|
||||
data: [1],
|
||||
}));
|
||||
await expect(
|
||||
cache.cached({ source: "bangumi", key: "GET bad", kind: "list", empty: () => false }, async () => {
|
||||
throw new Error("HTTP 403");
|
||||
}),
|
||||
).rejects.toThrow("403");
|
||||
const ttl = async (region: string, key: string) =>
|
||||
Number(
|
||||
(
|
||||
await sql<{ s: number }>`select extract(epoch from fresh_until - now()) as s from kv_cache
|
||||
where region = ${region} and key = ${key}`.execute(h.db)
|
||||
).rows[0]?.s ?? -1,
|
||||
);
|
||||
expect(await ttl("douban.request", "GET s1")).toBeGreaterThan(7 * 86400 - 60);
|
||||
expect(await ttl("douban.request", "GET s0")).toBeLessThanOrEqual(86400);
|
||||
expect(await ttl("bangumi.request", "GET l1")).toBeLessThanOrEqual(6 * 3600);
|
||||
expect(await ttl("bangumi.request", "GET l1")).toBeGreaterThan(6 * 3600 - 60);
|
||||
expect(await ttl("bangumi.request", "GET bad")).toBe(-1);
|
||||
});
|
||||
|
||||
it("命中统计:各 region 计 L1 / L2 / 旧值 / 未命中 / 失败 / 后台刷新;写入 cache_stat 后接口合计并给命中率", async () => {
|
||||
const mc = metadataCacheFor(h.db);
|
||||
const pending = mc.pendingStats().get("tmdb.request");
|
||||
expect(pending?.misses).toBeGreaterThan(0);
|
||||
expect(pending?.staleHits).toBeGreaterThan(0);
|
||||
expect(pending?.refreshes).toBeGreaterThan(0);
|
||||
expect(pending?.failures).toBeGreaterThan(0);
|
||||
await mc.flushStats();
|
||||
expect(mc.pendingStats().size).toBe(0);
|
||||
// 再来一次 L2 命中(未写入 cache_stat 的也算进接口)
|
||||
await (await client()).getJson("/tv/95396", { language: "zh-CN" });
|
||||
const stats = await regionHitStats(h.ctx);
|
||||
expect(stats.get("tmdb.request")!.l2Hits).toBeGreaterThan(0);
|
||||
const list = await json(await admin("GET", "/cache/regions"));
|
||||
const tmdbRow = list.data.items.find((i: { region: string }) => i.region === "tmdb.request");
|
||||
expect(tmdbRow.hits.hitRate).toBeGreaterThan(0);
|
||||
expect(tmdbRow.hits.hitRate).toBeLessThan(1);
|
||||
expect(tmdbRow.hits).toMatchObject({ misses: expect.any(Number), l2Hits: expect.any(Number) });
|
||||
// 非元数据 region 没有命中统计
|
||||
await new PgCache(h.db).set("site.detail", "x", 1, { ttl: 60 });
|
||||
const again = await json(await admin("GET", "/cache/regions"));
|
||||
expect(again.data.items.find((i: { region: string }) => i.region === "site.detail").hits).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -40,6 +40,19 @@ const DEVIATIONS: { id: string; apply: (files: string[], scenario: string) => st
|
||||
? fs
|
||||
: fs.map((f) => (f === "poster.jpg" || f === "fanart.jpg" ? `${LOOSE.replace(/\.mkv$/, "")}-${f}` : f)),
|
||||
},
|
||||
// 4. 作品级标志 clearlogo.png(TMDB logos;MP 只在接 fanart.tv 时写 logo,我们不接 fanart.tv):
|
||||
// 写在作品海报旁边——电影影片目录 / 剧集根目录 clearlogo.png,库根的电影 <视频名>-clearlogo.png;季目录不写
|
||||
{
|
||||
id: "tmdb_clearlogo",
|
||||
apply: (fs) => [
|
||||
...fs,
|
||||
...fs.flatMap((f) => {
|
||||
const m = /^(.*?)(\/|^|-)poster\.jpg$/.exec(f);
|
||||
if (!m || /(^|\/)Season \d+$/.test(m[1]!) || /(^|\/)season(\d+|-specials)$/.test(m[1]!)) return [];
|
||||
return [`${m[1]}${m[2]}clearlogo.png`];
|
||||
}),
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const detail = (entityType: "movie" | "tv", externalId: string): TmdbRichDetail =>
|
||||
@@ -69,6 +82,7 @@ const detail = (entityType: "movie" | "tv", externalId: string): TmdbRichDetail
|
||||
originCountry: [],
|
||||
posterPath: "/poster.jpg",
|
||||
backdropPath: "/backdrop.jpg",
|
||||
logoPath: "/logo.png",
|
||||
cast: [],
|
||||
crew: [],
|
||||
externalIds: { imdb: null, tvdb: null, wikidata: null },
|
||||
@@ -124,7 +138,10 @@ describe.skipIf(skip)("刮削文件布局与 MP 差分", () => {
|
||||
"scrlayout",
|
||||
{
|
||||
ao: {
|
||||
scrapeImage: async () => ({ content: new Uint8Array([0xff, 0xd8, 0xff]), mime: "image/jpeg" }),
|
||||
scrapeImage: async (p) =>
|
||||
p.endsWith(".png")
|
||||
? { content: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1]), mime: "image/png" }
|
||||
: { content: new Uint8Array([0xff, 0xd8, 0xff]), mime: "image/jpeg" },
|
||||
scrapeTmdbDetail: async (type, id) => detail(type, id),
|
||||
scrapeTmdbSeason: async (_tv, n) => (n === 1 ? season1 : null),
|
||||
},
|
||||
|
||||
@@ -218,13 +218,13 @@ describe.skipIf(!up)("发现页接口", () => {
|
||||
expect(meta.filter((m) => m.includes("movie_showing"))).toHaveLength(3);
|
||||
expect(meta.at(-1)).toContain("start=20&count=20");
|
||||
|
||||
// 空结果:写入但用空结果 TTL(缺省 1 小时),正常 6 小时
|
||||
// 空结果:写入但用空结果 TTL(缺省 1 小时),正常 6 小时(看新鲜期;过期后还有 50% 宽限期先用旧值)
|
||||
doubanReply = () => json({ subject_collection_items: [], total: 0 });
|
||||
await req("/discover/douban?list=tv_korean");
|
||||
const rows = await cache.items(META_REQUEST_REGIONS.douban);
|
||||
const ttlOf = (part: string) => {
|
||||
const row = rows.find((x) => x.key.includes(part))!;
|
||||
return Math.round((new Date(row.expiresAt!).getTime() - Date.now()) / 60_000);
|
||||
return Math.round((new Date(row.freshUntil).getTime() - Date.now()) / 60_000);
|
||||
};
|
||||
expect(ttlOf("tv_korean")).toBeLessThanOrEqual(60);
|
||||
expect(ttlOf("tv_korean")).toBeGreaterThan(55);
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createTmdbClient } from "@umt/adapters";
|
||||
import { sql } from "@umt/db";
|
||||
import { createMockTmdb, type MockTmdb, SPLIT_COUR_FIXTURE, TMDB_FIXTURES } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { tmdbSharedCache } from "../../src/modules/sources-identity/runtime.ts";
|
||||
import { prewarmMetadata } from "../../src/modules/sources-identity/service/metadata-prewarm.ts";
|
||||
import { enqueueWatchedRefresh } from "../../src/modules/sources-identity/service/watched.ts";
|
||||
import { tmdbCachePolicy } from "../../src/shared-cache.ts";
|
||||
import { createHarness, type Harness, pgAvailable } from "./harness.ts";
|
||||
|
||||
// 关注作品预热:进入关注时入队 prewarmMetadata(幂等键按作品 + 日期);执行后详情(含标志)、各季、选中的剧集组
|
||||
// 在 TMDB 请求缓存里且 pinned,海报 / 背景 / 标志进图片缓存;再跑一次全部命中,不发请求。
|
||||
|
||||
const skip = !(await pgAvailable());
|
||||
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1]);
|
||||
const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 1]);
|
||||
|
||||
describe.skipIf(skip)("关注作品预热", () => {
|
||||
let h: Harness;
|
||||
let tmdb: MockTmdb;
|
||||
let entityId: string;
|
||||
const images: string[] = [];
|
||||
const dataDir = mkdtempSync(join(tmpdir(), "umt-prewarm-"));
|
||||
|
||||
beforeAll(async () => {
|
||||
tmdb = await createMockTmdb({
|
||||
fixtures: [
|
||||
...TMDB_FIXTURES,
|
||||
{
|
||||
...SPLIT_COUR_FIXTURE,
|
||||
rich: { ...SPLIT_COUR_FIXTURE.rich, backdrop_path: "/split-bg.jpg" },
|
||||
logos: [
|
||||
{ file_path: "/split-en.png", iso_639_1: "en" },
|
||||
{ file_path: "/split-ja.png", iso_639_1: "ja" },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
let policy: Awaited<ReturnType<typeof tmdbCachePolicy>> | null = null;
|
||||
h = await createHarness("prewarm", {
|
||||
// 和服务端一样走统一元数据缓存(PG 请求缓存)
|
||||
tmdb: () =>
|
||||
createTmdbClient({
|
||||
apiKey: "test-tmdb-key",
|
||||
baseUrl: `${tmdb.url}/3`,
|
||||
retries: 0,
|
||||
sharedCache: tmdbSharedCache(h.ctx, policy!),
|
||||
}),
|
||||
// 图片上游(image.tmdb.org)
|
||||
fetch: async (input) => {
|
||||
const url = String(input);
|
||||
images.push(url);
|
||||
return new Response(url.endsWith(".png") ? PNG : JPEG);
|
||||
},
|
||||
});
|
||||
policy = await tmdbCachePolicy(h.ctx);
|
||||
(h.ctx.config as { DATA_DIR?: string }).DATA_DIR = dataDir;
|
||||
entityId = (
|
||||
await sql<{ id: string }>`insert into media_entity (kind, title, episode_group_id, episode_group_name)
|
||||
values ('series', '分割番', 'grp-production', '制作季') returning id`.execute(h.db)
|
||||
).rows[0]!.id;
|
||||
await sql`insert into external_identity (media_entity_id, provider, entity_type, external_id, status)
|
||||
values (${entityId}, 'tmdb', 'tv', ${String(SPLIT_COUR_FIXTURE.id)}, 'accepted')`.execute(h.db);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await tmdb?.close();
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("进入关注(建订阅)时入队预热,幂等键按作品 + 日期;同一天再进入不重复排", async () => {
|
||||
await sql`insert into subscription (media_entity_id, media_kind, status) values (${entityId}, 'series', 'active')`.execute(
|
||||
h.db,
|
||||
);
|
||||
await enqueueWatchedRefresh(h.ctx, entityId, "subscription");
|
||||
await enqueueWatchedRefresh(h.ctx, entityId, "download");
|
||||
const jobs = h.engine.enqueued.filter((e) => e.task === "prewarmMetadata");
|
||||
expect(jobs).toHaveLength(1);
|
||||
expect(jobs[0]).toMatchObject({
|
||||
input: { mediaEntityId: entityId },
|
||||
key: `prewarmMetadata:${entityId}:${new Date().toISOString().slice(0, 10)}`,
|
||||
});
|
||||
});
|
||||
|
||||
it("执行后:详情 / 各季 / 剧集组 / 标志在缓存里且 pinned;海报、背景、标志进图片缓存", async () => {
|
||||
const r = await prewarmMetadata(h.rt, entityId);
|
||||
expect(r).toMatchObject({ status: "warmed", episodeGroup: true, images: 3, errors: [] });
|
||||
expect(r.seasons).toBe(SPLIT_COUR_FIXTURE.seasons!.length);
|
||||
const rows = (
|
||||
await sql<{ key: string; pinned: boolean }>`select key, pinned from kv_cache where region = 'tmdb.request'
|
||||
order by key`.execute(h.db)
|
||||
).rows;
|
||||
const has = (re: RegExp) => rows.filter((x) => re.test(x.key));
|
||||
const id = SPLIT_COUR_FIXTURE.id;
|
||||
expect(has(new RegExp(`^/tv/${id}\\?.*images`)).length).toBeGreaterThan(0);
|
||||
for (const s of SPLIT_COUR_FIXTURE.seasons!)
|
||||
expect(has(new RegExp(`^/tv/${id}/season/${s.season_number}\\?`)).length).toBeGreaterThan(0);
|
||||
expect(has(/^\/tv\/episode_group\/grp-production\?/)).toHaveLength(1);
|
||||
// 原语言是日语:另取了日语标志
|
||||
expect(has(new RegExp(`^/tv/${id}/images\\?include_image_language=ja`))).toHaveLength(1);
|
||||
expect(rows.every((x) => x.pinned)).toBe(true);
|
||||
for (const f of ["split.jpg", "split-bg.jpg", "split-ja.png"])
|
||||
expect(existsSync(join(dataDir, "cache/images/t/p/original", f))).toBe(true);
|
||||
expect(images.every((u) => u.startsWith("https://image.tmdb.org/t/p/original/"))).toBe(true);
|
||||
});
|
||||
|
||||
it("再预热一次全部命中:不发 TMDB 请求、不再下载图片", async () => {
|
||||
const calls = tmdb.calls.length;
|
||||
const fetched = images.length;
|
||||
const r = await prewarmMetadata(h.rt, entityId);
|
||||
expect(r.status).toBe("warmed");
|
||||
expect(tmdb.calls.length).toBe(calls);
|
||||
expect(images.length).toBe(fetched);
|
||||
});
|
||||
|
||||
it("没有 TMDB 身份的作品跳过", async () => {
|
||||
const other = (
|
||||
await sql<{ id: string }>`insert into media_entity (kind, title) values ('movie', '本地') returning id`.execute(
|
||||
h.db,
|
||||
)
|
||||
).rows[0]!.id;
|
||||
expect(await prewarmMetadata(h.rt, other)).toMatchObject({ status: "skipped", reason: "no_tmdb_identity" });
|
||||
});
|
||||
});
|
||||
@@ -69,12 +69,13 @@ describe.skipIf(!up)("M-Team 快照与任务注册", () => {
|
||||
api.setFailMode(null);
|
||||
});
|
||||
|
||||
it("模块注册 7 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => {
|
||||
it("模块注册 8 个任务与 3 条 outbox 路由;任务处理器经 step.run 调服务", async () => {
|
||||
const reg = new TaskRegistry();
|
||||
h.module.registerTasks(reg, h.ctx);
|
||||
expect(reg.names()).toEqual([
|
||||
"enrichPoolEntry",
|
||||
"ingestFeed",
|
||||
"prewarmMetadata",
|
||||
"refreshMetadata",
|
||||
"refreshSiteAccount",
|
||||
"resolveObservation",
|
||||
|
||||
Reference in new issue
Block a user