feat(sites): on-demand site detail page content (INT-2)
- SiteReleaseDetail.page: description, MediaInfo/BDInfo, screenshots,
IMDb / Douban / TMDB ids, file list, tags, uploader, plus a parse probe
(labels found) for layout-change diagnosis. Generic NexusPHP rules
(field label -> next sibling) starting from the MovieClaw detail
selectors; site overrides under umt.detail.page (HHClub: div labels).
M-Team builds the same structure from descr BBCode + mediainfo.
- Fetched only on demand (drawer open without cache, "复核详情",
release.inspect); content in kv_cache region site.detail for the
site detail cache period (>= 24 h); within the window the site is not
requested again. External ids persist in source_release.raw._detail,
which list snapshots do not overwrite.
- Screenshots only as proxy paths /api/v1/img/site-detail/{rid}/{i}:
public-address check on every hop, no site credentials, magic-byte
image check, 8 MB cap. Poster proxy shares the same helper.
- Identity: site ids count as explicit-id evidence; when TMDB is
queried, a TMDB id goes straight to details and an IMDb id to
/find (no title search on a hit). Douban only for local recall.
- Release drawer "站点详情页" section; fixtures and tests for HHClub div
layout, rowhead layout, BBCode, cache/proxy/identity end to end.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
6e9e8efec2
commit
04cffcd1f1
30 files changed
+1133
-64
No files matched your search
@@ -1,4 +1,3 @@
|
||||
import { assertPublicUrl } from "@umt/adapters";
|
||||
import { isCapabilityError } from "@umt/agent";
|
||||
import {
|
||||
EnrichApplyInput,
|
||||
@@ -14,6 +13,7 @@ import type { ModuleApp } from "../../types.ts";
|
||||
import { route } from "../http.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { applyEnrichment, domainAllowed, hostOf } from "../service/enrich.ts";
|
||||
import { fetchPublicImage, imageResponse } from "../service/image-proxy.ts";
|
||||
|
||||
// META-1 元数据补全 API:先看建议(POST /identity-pool/{id}/enrich,经能力注册中心调用 metadata.enrich,
|
||||
// 授权 / 审计与 Agent 同一路径),再应用(POST /identity-pool/{id}/enrich/apply,只接受控制台管理员会话);
|
||||
@@ -22,19 +22,6 @@ import { applyEnrichment, domainAllowed, hostOf } from "../service/enrich.ts";
|
||||
const T_ID = ["identity"];
|
||||
const POSTER_MAX_BYTES = 5 * 1024 * 1024;
|
||||
|
||||
function imageMime(b: Uint8Array): string | null {
|
||||
if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
|
||||
if (b.length >= 8 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47) return "image/png";
|
||||
if (
|
||||
b.length >= 12 &&
|
||||
String.fromCharCode(...b.slice(0, 4)) === "RIFF" &&
|
||||
String.fromCharCode(...b.slice(8, 12)) === "WEBP"
|
||||
)
|
||||
return "image/webp";
|
||||
if (b.length >= 6 && String.fromCharCode(...b.slice(0, 4)) === "GIF8") return "image/gif";
|
||||
return null;
|
||||
}
|
||||
|
||||
export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void {
|
||||
route(
|
||||
app,
|
||||
@@ -119,44 +106,8 @@ export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void {
|
||||
const srcHost = hostOf(s.data.fields.posterUrl?.sourceUrl ?? "");
|
||||
if (!srcHost || !domainAllowed(srcHost, identity.enrichDomains ?? []))
|
||||
return c.json(errorBody("forbidden", "海报出处不在补全白名单里"), 403);
|
||||
try {
|
||||
const doFetch = r.overrides.fetch ?? fetch;
|
||||
let target = await assertPublicUrl(url);
|
||||
let res: Response | null = null;
|
||||
for (let hop = 0; hop <= 3; hop++) {
|
||||
res = await doFetch(target.href, {
|
||||
redirect: "manual",
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
headers: { accept: "image/avif,image/webp,image/*;q=0.8" },
|
||||
});
|
||||
const loc = res.status >= 300 && res.status < 400 ? res.headers.get("location") : null;
|
||||
if (!loc) break;
|
||||
await res.body?.cancel().catch(() => {});
|
||||
if (hop === 3) throw new Error("重定向次数过多");
|
||||
target = await assertPublicUrl(new URL(loc, target).href);
|
||||
}
|
||||
if (res?.status !== 200) {
|
||||
await res?.body?.cancel().catch(() => {});
|
||||
return c.json(errorBody("upstream_unavailable", `海报返回 HTTP ${res?.status ?? "?"}`), 502);
|
||||
}
|
||||
const len = Number(res.headers.get("content-length") ?? 0);
|
||||
if (len > POSTER_MAX_BYTES) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
return c.json(errorBody("upstream_unavailable", "海报超过 5 MB"), 502);
|
||||
}
|
||||
const buf = new Uint8Array(await res.arrayBuffer());
|
||||
const mime = buf.byteLength <= POSTER_MAX_BYTES ? imageMime(buf) : null;
|
||||
if (!mime) return c.json(errorBody("upstream_unavailable", "海报地址返回的不是图片"), 502);
|
||||
return new Response(buf as BodyInit, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"content-type": mime,
|
||||
"cache-control": `public, max-age=${86400 * 7}`,
|
||||
"x-content-type-options": "nosniff",
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
return c.json(errorBody("upstream_unavailable", `拉取海报失败:${(e as Error).message}`), 502);
|
||||
}
|
||||
const got = await fetchPublicImage(r.overrides.fetch ?? fetch, url, { maxBytes: POSTER_MAX_BYTES, label: "海报" });
|
||||
if (!got.ok) return c.json(errorBody("upstream_unavailable", got.message), 502);
|
||||
return imageResponse(got, 86400 * 7);
|
||||
});
|
||||
}
|
||||
@@ -1,14 +1,19 @@
|
||||
import { Id, WorkbenchReleaseDetail, WorkbenchReleasePage, WorkbenchReleaseQuery } from "@umt/contracts";
|
||||
import { z } from "zod";
|
||||
import { requireScope } from "../../../http/auth.ts";
|
||||
import { errorBody } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
import { route } from "../http.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { detailScreenshotUrl } from "../service/detail-page.ts";
|
||||
import { fetchPublicImage, imageResponse } from "../service/image-proxy.ts";
|
||||
import { listSiteReleases, releaseDetail } from "../service/workbench.ts";
|
||||
|
||||
// 站点工作台(整理视图)数据 API。库存叠加(已有资产/Slot/下载)不在这里:条目带 entity.id,
|
||||
// 前端按作品 ID 二次查询 W2-C 的媒体库端点。
|
||||
|
||||
const T = ["site-workbench"];
|
||||
const SHOT_MAX_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
route(
|
||||
@@ -39,4 +44,19 @@ export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
},
|
||||
async ({ params, query }) => releaseDetail(rt(), params.id, params.rid, query.refresh === "true"),
|
||||
);
|
||||
|
||||
// 详情页截图代理:不进 OpenAPI(二进制响应);只取该资源详情缓存里的第 index 张图,不接受任意 URL,不带站点凭据
|
||||
app.get("/img/site-detail/:rid/:index", requireScope("read"), async (c) => {
|
||||
const rid = c.req.param("rid");
|
||||
const index = Number(c.req.param("index"));
|
||||
if (!/^[0-9a-f-]{36}$/i.test(rid) || !Number.isInteger(index) || index < 0 || index > 99) {
|
||||
return c.json(errorBody("bad_request", "截图地址无效"), 400);
|
||||
}
|
||||
const r = rt();
|
||||
const url = await detailScreenshotUrl(r, rid, index);
|
||||
if (!url) return c.json(errorBody("not_found", "截图不存在或详情缓存已过期"), 404);
|
||||
const got = await fetchPublicImage(r.overrides.fetch ?? fetch, url, { maxBytes: SHOT_MAX_BYTES, label: "截图" });
|
||||
if (!got.ok) return c.json(errorBody("upstream_unavailable", got.message), 502);
|
||||
return imageResponse(got, 86_400);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import type { SiteDetailPage } from "@umt/adapters";
|
||||
import type { SiteReleasePage } from "@umt/contracts";
|
||||
import { sql } from "@umt/db";
|
||||
import { sharedCache } from "../../../shared-cache.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import type { Exec } from "./common.ts";
|
||||
|
||||
// 站点详情页扩展内容(简介 / MediaInfo / 截图 / 外部 ID / 文件 / 标签 / 上传者):
|
||||
// - 只在按需抓详情时得到(工作台打开详情、release.inspect 复核),放 PG kv_cache region site.detail,
|
||||
// key = source_release.id,TTL = 站点详情缓存期(至少 24 小时);不进 source_release 列,过期即丢。
|
||||
// - 外部 ID(IMDb / 豆瓣 / TMDB)另写 source_release.raw._detail(列表快照 upsert 不覆盖它),供身份解析用。
|
||||
// - 截图对外只给代理路径 /api/v1/img/site-detail/{releaseId}/{index},原始地址留在服务端。
|
||||
|
||||
export const DETAIL_PAGE_REGION = "site.detail";
|
||||
const MIN_TTL_SEC = 86_400;
|
||||
const MAX_ENTRIES = 5_000;
|
||||
|
||||
interface StoredPage extends SiteDetailPage {
|
||||
fetchedAt: string;
|
||||
}
|
||||
|
||||
export interface DetailIds {
|
||||
imdbId?: string;
|
||||
doubanId?: string;
|
||||
tmdb?: { type: "movie" | "tv"; id: string };
|
||||
}
|
||||
|
||||
export async function saveDetailPage(
|
||||
rt: SourcesRuntime,
|
||||
exec: Exec,
|
||||
srId: string,
|
||||
page: SiteDetailPage,
|
||||
cacheSec: number,
|
||||
): Promise<void> {
|
||||
const stored: StoredPage = { ...page, fetchedAt: rt.now().toISOString() };
|
||||
await sharedCache(rt.ctx)
|
||||
.set(DETAIL_PAGE_REGION, srId, stored, { ttl: Math.max(cacheSec, MIN_TTL_SEC), maxsize: MAX_ENTRIES })
|
||||
.catch((e) => rt.ctx.logger.warn({ err: { message: (e as Error).message } }, "详情页缓存写入失败"));
|
||||
const ids: DetailIds = {
|
||||
...(page.imdbId ? { imdbId: page.imdbId } : {}),
|
||||
...(page.doubanId ? { doubanId: page.doubanId } : {}),
|
||||
...(page.tmdb ? { tmdb: page.tmdb } : {}),
|
||||
};
|
||||
await sql`update source_release set raw = raw || jsonb_build_object('_detail', ${JSON.stringify(ids)}::jsonb)
|
||||
where id = ${srId}`.execute(exec);
|
||||
}
|
||||
|
||||
async function loadStored(rt: SourcesRuntime, srId: string): Promise<StoredPage | null> {
|
||||
try {
|
||||
const v = (await sharedCache(rt.ctx).get(DETAIL_PAGE_REGION, srId)) as StoredPage | null;
|
||||
return v && typeof v === "object" && typeof v.fetchedAt === "string" ? v : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function hasDetailPage(rt: SourcesRuntime, srId: string): Promise<boolean> {
|
||||
return (await loadStored(rt, srId)) !== null;
|
||||
}
|
||||
|
||||
/** 缓存里的截图原始地址(图片代理用) */
|
||||
export async function detailScreenshotUrl(rt: SourcesRuntime, srId: string, index: number): Promise<string | null> {
|
||||
return (await loadStored(rt, srId))?.screenshots[index] ?? null;
|
||||
}
|
||||
|
||||
export async function loadDetailPage(rt: SourcesRuntime, srId: string): Promise<SiteReleasePage | null> {
|
||||
const p = await loadStored(rt, srId);
|
||||
if (!p) return null;
|
||||
return {
|
||||
fetchedAt: p.fetchedAt,
|
||||
descriptionText: p.descriptionText,
|
||||
mediaInfo: p.mediaInfo,
|
||||
screenshots: p.screenshots.map((u, i) => {
|
||||
let host = "";
|
||||
try {
|
||||
host = new URL(u).host;
|
||||
} catch {
|
||||
/* 已在解析时校验 */
|
||||
}
|
||||
return { url: `/api/v1/img/site-detail/${srId}/${i}`, host };
|
||||
}),
|
||||
imdbId: p.imdbId,
|
||||
doubanId: p.doubanId,
|
||||
tmdb: p.tmdb,
|
||||
files: p.files,
|
||||
tags: p.tags,
|
||||
uploader: p.uploader,
|
||||
};
|
||||
}
|
||||
|
||||
/** source_release.raw 里站点给出的外部 ID(列表 _site + 详情 _detail) */
|
||||
export function siteExplicitIds(
|
||||
raw: unknown,
|
||||
): { provider: "imdb" | "douban" | "tmdb"; id: string; type?: "movie" | "tv" }[] {
|
||||
const r = (raw ?? {}) as { _site?: { imdbId?: unknown; doubanId?: unknown }; _detail?: DetailIds };
|
||||
const out: { provider: "imdb" | "douban" | "tmdb"; id: string; type?: "movie" | "tv" }[] = [];
|
||||
const add = (provider: "imdb" | "douban" | "tmdb", id: unknown, type?: "movie" | "tv") => {
|
||||
if (typeof id !== "string" || !id || out.some((x) => x.provider === provider && x.id === id)) return;
|
||||
out.push({ provider, id, ...(type ? { type } : {}) });
|
||||
};
|
||||
add("imdb", r._detail?.imdbId);
|
||||
add("imdb", r._site?.imdbId);
|
||||
add("douban", r._detail?.doubanId);
|
||||
add("douban", r._site?.doubanId);
|
||||
if (r._detail?.tmdb) add("tmdb", r._detail.tmdb.id, r._detail.tmdb.type);
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { assertPublicUrl } from "@umt/adapters";
|
||||
|
||||
// 外站图片代理(补全海报、站点详情截图):只取公网地址(每跳重定向都复核),不带任何凭据,
|
||||
// 按文件头认图片类型(不信任上游 content-type,不放行 SVG),限制大小。
|
||||
|
||||
export type ImageFetchResult =
|
||||
| { ok: true; bytes: Uint8Array; mime: string }
|
||||
| { ok: false; status: 404 | 502; message: string };
|
||||
|
||||
export function imageMime(b: Uint8Array): string | null {
|
||||
if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
|
||||
if (b.length >= 8 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47) return "image/png";
|
||||
if (
|
||||
b.length >= 12 &&
|
||||
String.fromCharCode(...b.slice(0, 4)) === "RIFF" &&
|
||||
String.fromCharCode(...b.slice(8, 12)) === "WEBP"
|
||||
)
|
||||
return "image/webp";
|
||||
if (b.length >= 6 && String.fromCharCode(...b.slice(0, 4)) === "GIF8") return "image/gif";
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function fetchPublicImage(
|
||||
doFetch: typeof fetch,
|
||||
url: string,
|
||||
opts: { maxBytes: number; label: string; referer?: string },
|
||||
): Promise<ImageFetchResult> {
|
||||
const { maxBytes, label } = opts;
|
||||
try {
|
||||
let target = await assertPublicUrl(url);
|
||||
let res: Response | null = null;
|
||||
for (let hop = 0; hop <= 3; hop++) {
|
||||
res = await doFetch(target.href, {
|
||||
redirect: "manual",
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
headers: {
|
||||
accept: "image/avif,image/webp,image/*;q=0.8",
|
||||
...(opts.referer ? { referer: opts.referer } : {}),
|
||||
},
|
||||
});
|
||||
const loc = res.status >= 300 && res.status < 400 ? res.headers.get("location") : null;
|
||||
if (!loc) break;
|
||||
await res.body?.cancel().catch(() => {});
|
||||
if (hop === 3) throw new Error("重定向次数过多");
|
||||
target = await assertPublicUrl(new URL(loc, target).href);
|
||||
}
|
||||
if (res?.status !== 200) {
|
||||
await res?.body?.cancel().catch(() => {});
|
||||
return { ok: false, status: 502, message: `${label}返回 HTTP ${res?.status ?? "?"}` };
|
||||
}
|
||||
const len = Number(res.headers.get("content-length") ?? 0);
|
||||
const mb = Math.round(maxBytes / 1024 / 1024);
|
||||
if (len > maxBytes) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
return { ok: false, status: 502, message: `${label}超过 ${mb} MB` };
|
||||
}
|
||||
const buf = new Uint8Array(await res.arrayBuffer());
|
||||
const mime = buf.byteLength <= maxBytes ? imageMime(buf) : null;
|
||||
if (!mime) return { ok: false, status: 502, message: `${label}地址返回的不是图片` };
|
||||
return { ok: true, bytes: buf, mime };
|
||||
} catch (e) {
|
||||
return { ok: false, status: 502, message: `拉取${label}失败:${(e as Error).message}` };
|
||||
}
|
||||
}
|
||||
|
||||
export function imageResponse(r: { bytes: Uint8Array; mime: string }, maxAgeSec: number): Response {
|
||||
return new Response(r.bytes as BodyInit, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"content-type": r.mime,
|
||||
"cache-control": `private, max-age=${maxAgeSec}`,
|
||||
"x-content-type-options": "nosniff",
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
import { jsonb } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { type Actor, closeDecisions, type Exec, SYSTEM } from "./common.ts";
|
||||
import { siteExplicitIds } from "./detail-page.ts";
|
||||
import { maybeAutoEnrich } from "./enrich.ts";
|
||||
import {
|
||||
addAliasIfAbsent,
|
||||
@@ -113,6 +114,7 @@ export function tmdbToCandidate(m: MetadataCandidate): DomainCandidate {
|
||||
...(m.year !== null ? { year: m.year } : {}),
|
||||
aliases,
|
||||
...(seasonYears?.length ? { seasonYears } : {}),
|
||||
...(m.externalIds?.imdb ? { linkedExternalIds: [{ provider: "imdb", id: m.externalIds.imdb }] } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -137,8 +139,7 @@ async function localCandidates(
|
||||
release: SourceReleaseRow,
|
||||
): Promise<DomainCandidate[]> {
|
||||
const ids = new Set<string>();
|
||||
const site = ((release.raw as Record<string, unknown>)?._site ?? {}) as { imdbId?: string };
|
||||
const explicit = [...parsed.explicitIds, ...(site.imdbId ? [{ provider: "imdb" as const, id: site.imdbId }] : [])];
|
||||
const explicit = [...parsed.explicitIds, ...siteExplicitIds(release.raw)];
|
||||
for (const x of explicit) {
|
||||
const row = await exec
|
||||
.selectFrom("external_identity")
|
||||
@@ -190,10 +191,14 @@ async function tmdbCandidates(
|
||||
rt: SourcesRuntime,
|
||||
parsed: ParsedRelease,
|
||||
useCache: boolean,
|
||||
siteIds: ReturnType<typeof siteExplicitIds> = [],
|
||||
): Promise<{ remote: MetadataCandidate[]; cacheHit: boolean; meta: RecognizeMeta | null }> {
|
||||
const tmdb = await rt.tmdb();
|
||||
if (!tmdb) return { remote: [], cacheHit: false, meta: null };
|
||||
const meta = recognizeMetaOf(parsed);
|
||||
// 站点详情页给出的 TMDB / IMDb ID:按 ID 直取(TMDB 详情 / find),命中就不再按标题搜索
|
||||
const byId = await tmdbByExplicitIds(tmdb, siteIds);
|
||||
if (byId.length) return { remote: byId, cacheHit: false, meta };
|
||||
if (meta && useCache) {
|
||||
const hit = await loadRecognizeCache(rt, meta);
|
||||
if (hit) {
|
||||
@@ -214,6 +219,25 @@ async function tmdbCandidates(
|
||||
return { remote: [...seen.values()].slice(0, 20), cacheHit: false, meta };
|
||||
}
|
||||
|
||||
async function tmdbByExplicitIds(
|
||||
tmdb: NonNullable<Awaited<ReturnType<SourcesRuntime["tmdb"]>>>,
|
||||
ids: ReturnType<typeof siteExplicitIds>,
|
||||
): Promise<MetadataCandidate[]> {
|
||||
const tmdbId = ids.find((x) => x.provider === "tmdb" && x.type);
|
||||
if (tmdbId?.type) {
|
||||
try {
|
||||
return [await tmdb.details(tmdbId.type, tmdbId.id)];
|
||||
} catch (e) {
|
||||
if (!(e instanceof TmdbError) || e.kind !== "notfound") throw e;
|
||||
}
|
||||
}
|
||||
const imdb = ids.find((x) => x.provider === "imdb");
|
||||
if (!imdb) return [];
|
||||
const found = await tmdb.findByImdb(imdb.id);
|
||||
for (const m of found) m.externalIds = { ...m.externalIds, imdb: imdb.id };
|
||||
return found;
|
||||
}
|
||||
|
||||
async function loadRelease(rt: SourcesRuntime, input: ResolveInput) {
|
||||
const { db } = rt.ctx;
|
||||
const observationId = input.observationId ?? null;
|
||||
@@ -277,6 +301,12 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P
|
||||
}
|
||||
const parsed = parseRelease(release.title, release.subtitle ? { subtitle: release.subtitle } : {});
|
||||
const cols = toParsedColumns(parsed);
|
||||
// 站点给出的外部 ID(列表 / 详情页的 IMDb、豆瓣、TMDB 链接)作为显式 ID 证据参与打分
|
||||
const siteIds = siteExplicitIds(release.raw);
|
||||
for (const x of siteIds) {
|
||||
if (!parsed.explicitIds.some((e) => e.provider === x.provider))
|
||||
parsed.explicitIds.push({ provider: x.provider, id: x.id });
|
||||
}
|
||||
const trigger: ResolveTrigger = input.trigger ?? (input.force ? "review" : "observed");
|
||||
if (!parsed.title || parsed.auxiliaryOnly) {
|
||||
return toPool(rt, release, observationId, parsed, null, [], "unmatched", "unparseable_title", null, {
|
||||
@@ -297,7 +327,7 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P
|
||||
let lookupError: string | null = null;
|
||||
let recognize: { cacheHit: boolean; meta: RecognizeMeta | null } = { cacheHit: false, meta: null };
|
||||
try {
|
||||
const r = await tmdbCandidates(rt, parsed, !input.force);
|
||||
const r = await tmdbCandidates(rt, parsed, !input.force, siteIds);
|
||||
remote = r.remote;
|
||||
recognize = r;
|
||||
} catch (e) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { SiteError, type SitePage, type SiteRelease } from "@umt/adapters";
|
||||
import { type SiteDetailPage, SiteError, type SitePage, type SiteRelease } from "@umt/adapters";
|
||||
import type { SiteSnapshotKind } from "@umt/contracts";
|
||||
import { type SiteRow, sql } from "@umt/db";
|
||||
import { parseRelease } from "@umt/domain";
|
||||
@@ -7,6 +7,7 @@ import { jsonb } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { enqueueRelogin } from "./account.ts";
|
||||
import { type Exec, openDecision } from "./common.ts";
|
||||
import { hasDetailPage, saveDetailPage } from "./detail-page.ts";
|
||||
|
||||
// 站点快照(周期列表)、补漏(Feed 疑似缺口,有页数预算)、实时补查(工作台搜索)。
|
||||
// 所有请求经站点共享限流器;结果按 site+站点资源 ID upsert 到 source_release,并记录 site_snapshot 覆盖范围。
|
||||
@@ -215,14 +216,22 @@ export async function refreshReleaseDetail(rt: SourcesRuntime, srId: string): Pr
|
||||
if (!rel) return "not_found";
|
||||
const site = await db.selectFrom("site").selectAll().where("id", "=", rel.site_id).executeTakeFirst();
|
||||
if (!site?.enabled) return "site_disabled";
|
||||
// 详情缓存:站点访问默认的缓存期内抓过就不再请求站点(列表快照会持续更新做种 / 促销)
|
||||
// 详情缓存:站点访问默认的缓存期内抓过、且详情页扩展内容还在缓存里,就不再请求站点(列表快照会持续更新做种 / 促销)
|
||||
const { detailCacheSec } = await rt.siteAccess(site.key);
|
||||
if (rel.detail_fetched_at && rt.now().getTime() - rel.detail_fetched_at.getTime() < detailCacheSec * 1000) {
|
||||
if (
|
||||
rel.detail_fetched_at &&
|
||||
rt.now().getTime() - rel.detail_fetched_at.getTime() < detailCacheSec * 1000 &&
|
||||
(await hasDetailPage(rt, srId))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
let detail: SiteRelease;
|
||||
let page: SiteDetailPage | undefined;
|
||||
try {
|
||||
detail = mergeDetailOverList(await rt.siteAdapter(site).detail(rel.site_release_id), rel);
|
||||
const d = await rt.siteAdapter(site).detail(rel.site_release_id);
|
||||
page = d.page;
|
||||
const { page: _p, descriptionText: _d, ...plain } = d;
|
||||
detail = mergeDetailOverList(plain, rel);
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError) {
|
||||
if (e.isAuthError) await db.transaction().execute((trx) => onAuthError(rt, trx, site, e));
|
||||
@@ -247,6 +256,7 @@ export async function refreshReleaseDetail(rt: SourcesRuntime, srId: string): Pr
|
||||
})
|
||||
.where("id", "=", srId)
|
||||
.execute();
|
||||
if (page) await saveDetailPage(rt, trx, srId, page, detailCacheSec);
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import { normalizeTitle } from "@umt/domain";
|
||||
import { ApiException } from "../../../http/errors.ts";
|
||||
import { decodeCursor, page, toApi } from "../rows.ts";
|
||||
import { type SourcesRuntime, siteSearchConcurrency } from "../runtime.ts";
|
||||
import { loadDetailPage } from "./detail-page.ts";
|
||||
import { refreshReleaseDetail, snapshotSite } from "./snapshot.ts";
|
||||
|
||||
// 站点工作台(整理视图):本地快照优先;本地无结果 / 快照过期 / 用户要求时对已开启站点实时补查,
|
||||
@@ -224,5 +225,17 @@ export async function releaseDetail(
|
||||
recheckBeforeDownload: true,
|
||||
},
|
||||
refreshError,
|
||||
page: await loadDetailPage(rt, releaseId),
|
||||
detailSupported: await detailSupported(rt, siteId),
|
||||
};
|
||||
}
|
||||
|
||||
async function detailSupported(rt: SourcesRuntime, siteId: string): Promise<boolean> {
|
||||
const site = await rt.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst();
|
||||
if (!site?.enabled) return false;
|
||||
try {
|
||||
return rt.siteAdapter(site).capabilities.detail;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
import {
|
||||
AUDIENCES_FIXTURE_RELEASES,
|
||||
createMockNexusPhpSite,
|
||||
createMockTmdb,
|
||||
type MockNexusPhpSite,
|
||||
type MockTmdb,
|
||||
} from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { DETAIL_PAGE_REGION } from "../../src/modules/sources-identity/service/detail-page.ts";
|
||||
import { resolveRelease } from "../../src/modules/sources-identity/service/resolve.ts";
|
||||
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
|
||||
import { sharedCache } from "../../src/shared-cache.ts";
|
||||
import { createHarness, type Harness, json, pgAvailable, seedSite } from "./harness.ts";
|
||||
|
||||
const up = await pgAvailable();
|
||||
|
||||
const SHOT = "http://93.184.216.34/shots/1.png";
|
||||
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3, 4]);
|
||||
|
||||
describe.skipIf(!up)("站点详情页扩展内容:按需抓 + 24 小时缓存 + 截图代理 + 外部 ID 进身份解析", () => {
|
||||
let h: Harness;
|
||||
let mock: MockNexusPhpSite;
|
||||
let tmdb: MockTmdb;
|
||||
let siteId: string;
|
||||
let relId: string;
|
||||
const imageFetches: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
const releases = AUDIENCES_FIXTURE_RELEASES.map((r) =>
|
||||
r.id === "812345"
|
||||
? {
|
||||
...r,
|
||||
descriptionHtml: `◎片名 沙丘2<br>◎IMDb <a href="https://www.imdb.com/title/tt15239678/">tt15239678</a><br>
|
||||
<img src="${SHOT}"><img src="pic/smilies/2.gif">
|
||||
<div class="codemain">General<br>Complete name : Dune.mkv<br>Format : Matroska<br>File size : 78 GiB<br>Duration : 2 h 46 min<br>Video<br>Format : HEVC</div>`,
|
||||
}
|
||||
: r,
|
||||
);
|
||||
mock = await createMockNexusPhpSite({ releases });
|
||||
tmdb = await createMockTmdb();
|
||||
h = await createHarness("detailpage", {
|
||||
tmdbUrl: tmdb.url,
|
||||
fetch: (input, init) => {
|
||||
const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
|
||||
if (url.startsWith("http://93.184.216.34/")) {
|
||||
imageFetches.push(url);
|
||||
return Promise.resolve(new Response(PNG, { status: 200, headers: { "content-type": "image/png" } }));
|
||||
}
|
||||
return fetch(input, init);
|
||||
},
|
||||
});
|
||||
siteId = (await seedSite(h, { baseUrl: `${mock.baseUrl}/`, cookie: mock.cookie })).site.id;
|
||||
await snapshotSite(h.rt, { siteId, kind: "manual" });
|
||||
relId = (
|
||||
await h.db
|
||||
.selectFrom("source_release")
|
||||
.select(["id"])
|
||||
.where("site_release_id", "=", "812345")
|
||||
.executeTakeFirstOrThrow()
|
||||
).id;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await h?.close();
|
||||
await mock?.close();
|
||||
await tmdb?.close();
|
||||
});
|
||||
|
||||
const detailRequests = () => mock.requests.filter((r) => r.path.startsWith("/details.php")).length;
|
||||
|
||||
it("未抓过:page 为 null,标明站点支持详情页;不刷新就不请求站点", async () => {
|
||||
const before = detailRequests();
|
||||
const d = await json(await h.req(`/sites/${siteId}/releases/${relId}`));
|
||||
expect(d.status).toBe(200);
|
||||
expect(d.body.data).toMatchObject({ page: null, detailSupported: true });
|
||||
expect(detailRequests()).toBe(before);
|
||||
});
|
||||
|
||||
it("refresh 抓一次:简介 / MediaInfo / 截图(代理路径)/ IMDb;外部 ID 写 raw._detail;缓存期内不再请求", async () => {
|
||||
const before = detailRequests();
|
||||
const d = await json(await h.req(`/sites/${siteId}/releases/${relId}?refresh=true`));
|
||||
expect(d.body.data.refreshError).toBeNull();
|
||||
expect(detailRequests()).toBe(before + 1);
|
||||
const page = d.body.data.page;
|
||||
expect(page).toMatchObject({
|
||||
imdbId: "tt15239678",
|
||||
screenshots: [{ url: `/api/v1/img/site-detail/${relId}/0`, host: "93.184.216.34" }],
|
||||
});
|
||||
expect(page.mediaInfo).toMatch(/^General\nComplete name : Dune.mkv/);
|
||||
expect(page.descriptionText).toContain("◎片名 沙丘2");
|
||||
expect(page.descriptionText).not.toContain("Complete name");
|
||||
const row = await h.db
|
||||
.selectFrom("source_release")
|
||||
.select(["raw"])
|
||||
.where("id", "=", relId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect((row.raw as { _detail?: unknown })._detail).toEqual({ imdbId: "tt15239678" });
|
||||
|
||||
const again = await json(await h.req(`/sites/${siteId}/releases/${relId}?refresh=true`));
|
||||
expect(again.body.data.page.imdbId).toBe("tt15239678");
|
||||
expect(detailRequests()).toBe(before + 1);
|
||||
|
||||
// 列表快照覆盖 raw._site 后,详情给出的外部 ID 仍在
|
||||
await snapshotSite(h.rt, { siteId, kind: "manual" });
|
||||
const row2 = await h.db
|
||||
.selectFrom("source_release")
|
||||
.select(["raw"])
|
||||
.where("id", "=", relId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect((row2.raw as { _detail?: unknown })._detail).toEqual({ imdbId: "tt15239678" });
|
||||
});
|
||||
|
||||
it("截图代理:只取缓存里的第 index 张,按文件头认图片;越界 404", async () => {
|
||||
const res = await h.req(`/img/site-detail/${relId}/0`);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("content-type")).toBe("image/png");
|
||||
expect(new Uint8Array(await res.arrayBuffer())).toEqual(PNG);
|
||||
expect(imageFetches).toEqual([SHOT]);
|
||||
expect((await h.req(`/img/site-detail/${relId}/5`)).status).toBe(404);
|
||||
expect((await h.req(`/img/site-detail/not-an-id/0`)).status).toBe(400);
|
||||
});
|
||||
|
||||
it("缓存丢了(kv_cache 过期)再刷新会重新抓一次", async () => {
|
||||
await sharedCache(h.ctx).delete(DETAIL_PAGE_REGION, relId);
|
||||
const before = detailRequests();
|
||||
const d = await json(await h.req(`/sites/${siteId}/releases/${relId}`));
|
||||
expect(d.body.data.page).toBeNull();
|
||||
await h.req(`/sites/${siteId}/releases/${relId}?refresh=true`);
|
||||
expect(detailRequests()).toBe(before + 1);
|
||||
});
|
||||
|
||||
it("身份解析:详情页的 IMDb ID 走 TMDB find,不按标题搜索,显式 ID 证据", async () => {
|
||||
const searches = () => tmdb.callsTo(/\/3\/search\//).length;
|
||||
const s0 = searches();
|
||||
const res = await resolveRelease(h.rt, { sourceReleaseId: relId, force: true, trigger: "manual" });
|
||||
expect(tmdb.callsTo(/\/3\/find\/tt15239678/).length).toBeGreaterThan(0);
|
||||
expect(searches()).toBe(s0);
|
||||
expect(res.outcome).toBe("auto");
|
||||
const ev = await h.db
|
||||
.selectFrom("source_release")
|
||||
.select(["resolution_evidence", "media_entity_id"])
|
||||
.where("id", "=", relId)
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(ev.media_entity_id).toBe(res.mediaEntityId);
|
||||
const ext = await h.db
|
||||
.selectFrom("external_identity")
|
||||
.select(["provider", "external_id"])
|
||||
.where("media_entity_id", "=", res.mediaEntityId!)
|
||||
.where("provider", "=", "tmdb")
|
||||
.executeTakeFirstOrThrow();
|
||||
expect(ext.external_id).toBe("693134");
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { InventoryEntry, WorkbenchRelease } from "@umt/contracts";
|
||||
import { Bot, Download, RefreshCw } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { Link } from "react-router";
|
||||
import { toast } from "sonner";
|
||||
import { useEntityDetail } from "~/api/inventory.ts";
|
||||
@@ -17,6 +17,7 @@ import { formatBytes, formatDateTime } from "~/lib/utils.ts";
|
||||
import { DownloadDialog } from "./download-dialog.tsx";
|
||||
import { categoryLabel, hrOf, ORIGIN_LABEL, relTime } from "./release-fields.ts";
|
||||
import { InventoryOverlay, ReleaseBadges } from "./release-list.tsx";
|
||||
import { ReleasePageContent } from "./release-page-section.tsx";
|
||||
|
||||
const DECISION_LABEL = {
|
||||
accept: { label: "符合,可收录", tone: "success" },
|
||||
@@ -125,6 +126,16 @@ export function ReleaseDetailSheet({
|
||||
const [dl, setDl] = useState(false);
|
||||
const r = detail.data?.release ?? release;
|
||||
|
||||
// 按需抓详情页:打开抽屉时没有缓存的详情页扩展内容就抓一次(服务端缓存 24 小时,同一资源本次会话只自动抓一次)
|
||||
const autoTried = useRef(new Set<string>());
|
||||
const needPage = !!detail.data && !detail.data.page && detail.data.detailSupported;
|
||||
const rid = release?.id;
|
||||
useEffect(() => {
|
||||
if (!needPage || !rid || autoTried.current.has(rid) || refresh.isPending) return;
|
||||
autoTried.current.add(rid);
|
||||
refresh.mutate({ siteId, rid });
|
||||
}, [needPage, rid, siteId, refresh]);
|
||||
|
||||
return (
|
||||
<DetailSheet
|
||||
open={!!release}
|
||||
@@ -216,6 +227,21 @@ export function ReleaseDetailSheet({
|
||||
) : null}
|
||||
</DetailSection>
|
||||
|
||||
<DetailSection title="站点详情页">
|
||||
{detail.data?.page ? (
|
||||
<ReleasePageContent page={detail.data.page} />
|
||||
) : refresh.isPending || detail.isPending ? (
|
||||
<LoadingState rows={2} />
|
||||
) : detail.data && !detail.data.detailSupported ? (
|
||||
<p className="text-caption text-muted-foreground">这个站点不支持抓详情页。</p>
|
||||
) : (
|
||||
<p className="text-caption text-muted-foreground">
|
||||
还没有详情页内容{detail.data?.refreshError ? `(${detail.data.refreshError})` : ""}
|
||||
。点「复核详情」抓取。
|
||||
</p>
|
||||
)}
|
||||
</DetailSection>
|
||||
|
||||
<DetailSection title="本地库存与候选理由">
|
||||
<LocalInventory release={r} entry={entry} />
|
||||
</DetailSection>
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
import type { SiteReleasePage } from "@umt/contracts";
|
||||
import { ExternalLink } from "lucide-react";
|
||||
import { StatusBadge } from "~/components/console/status-badge.tsx";
|
||||
import { formatBytes, formatDateTime } from "~/lib/utils.ts";
|
||||
|
||||
/** 详情页扩展内容里的外部站点链接 */
|
||||
export function externalLinks(p: Pick<SiteReleasePage, "imdbId" | "doubanId" | "tmdb">) {
|
||||
const out: { label: string; href: string }[] = [];
|
||||
if (p.tmdb) out.push({ label: `TMDB ${p.tmdb.id}`, href: `https://www.themoviedb.org/${p.tmdb.type}/${p.tmdb.id}` });
|
||||
if (p.imdbId) out.push({ label: `IMDb ${p.imdbId}`, href: `https://www.imdb.com/title/${p.imdbId}/` });
|
||||
if (p.doubanId) out.push({ label: `豆瓣 ${p.doubanId}`, href: `https://movie.douban.com/subject/${p.doubanId}/` });
|
||||
return out;
|
||||
}
|
||||
|
||||
export function ReleasePageContent({ page }: { page: SiteReleasePage }) {
|
||||
const links = externalLinks(page);
|
||||
const empty =
|
||||
!links.length &&
|
||||
!page.uploader &&
|
||||
!page.tags.length &&
|
||||
!page.descriptionText &&
|
||||
!page.mediaInfo &&
|
||||
!page.screenshots.length &&
|
||||
!page.files.length;
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
{links.length || page.uploader ? (
|
||||
<div className="flex flex-wrap items-center gap-x-3 gap-y-1 text-caption">
|
||||
{links.map((l) => (
|
||||
<a
|
||||
key={l.href}
|
||||
href={l.href}
|
||||
target="_blank"
|
||||
rel="noreferrer noopener"
|
||||
className="inline-flex items-center gap-1 text-foreground underline underline-offset-4"
|
||||
>
|
||||
{l.label}
|
||||
<ExternalLink className="size-3" />
|
||||
</a>
|
||||
))}
|
||||
{page.uploader ? <span className="text-muted-foreground">上传者 {page.uploader}</span> : null}
|
||||
</div>
|
||||
) : null}
|
||||
{page.tags.length ? (
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{page.tags.map((t) => (
|
||||
<StatusBadge key={t}>{t}</StatusBadge>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
{page.descriptionText ? (
|
||||
<p className="max-h-72 overflow-auto text-caption break-words whitespace-pre-wrap text-foreground">
|
||||
{page.descriptionText}
|
||||
</p>
|
||||
) : null}
|
||||
{page.mediaInfo ? (
|
||||
<details className="rounded-lg border border-border/70 px-3 py-2">
|
||||
<summary className="cursor-pointer text-caption text-foreground">MediaInfo</summary>
|
||||
<pre className="mt-2 max-h-80 overflow-auto font-mono text-meta whitespace-pre text-muted-foreground">
|
||||
{page.mediaInfo}
|
||||
</pre>
|
||||
</details>
|
||||
) : null}
|
||||
{page.screenshots.length ? (
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
{page.screenshots.map((s, i) => (
|
||||
<a key={s.url} href={s.url} target="_blank" rel="noreferrer noopener" title={s.host}>
|
||||
<img
|
||||
src={s.url}
|
||||
alt={`截图 ${i + 1}`}
|
||||
loading="lazy"
|
||||
className="aspect-video w-full rounded-md border border-border/70 bg-muted object-cover"
|
||||
/>
|
||||
</a>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
{page.files.length ? (
|
||||
<details className="rounded-lg border border-border/70 px-3 py-2">
|
||||
<summary className="cursor-pointer text-caption text-foreground">文件清单({page.files.length})</summary>
|
||||
<ul className="mt-2 max-h-60 space-y-1 overflow-auto text-meta">
|
||||
{page.files.map((f, i) => (
|
||||
<li key={`${i}-${f.name}`} className="flex justify-between gap-3">
|
||||
<span className="min-w-0 font-mono break-all text-foreground">{f.name}</span>
|
||||
<span className="shrink-0 text-muted-foreground">
|
||||
{f.sizeBytes === null ? "" : formatBytes(f.sizeBytes, 1)}
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
) : null}
|
||||
{empty ? <p className="text-caption text-muted-foreground">详情页上没有解析到简介、截图或外部链接。</p> : null}
|
||||
<p className="text-meta text-muted-foreground">
|
||||
抓取于 {formatDateTime(page.fetchedAt)},缓存 24 小时;截图经服务端代理加载。
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user