feat(sites): on-demand site detail page content (INT-2)
- SiteReleaseDetail.page: description, MediaInfo/BDInfo, screenshots,
IMDb / Douban / TMDB ids, file list, tags, uploader, plus a parse probe
(labels found) for layout-change diagnosis. Generic NexusPHP rules
(field label -> next sibling) starting from the MovieClaw detail
selectors; site overrides under umt.detail.page (HHClub: div labels).
M-Team builds the same structure from descr BBCode + mediainfo.
- Fetched only on demand (drawer open without cache, "复核详情",
release.inspect); content in kv_cache region site.detail for the
site detail cache period (>= 24 h); within the window the site is not
requested again. External ids persist in source_release.raw._detail,
which list snapshots do not overwrite.
- Screenshots only as proxy paths /api/v1/img/site-detail/{rid}/{i}:
public-address check on every hop, no site credentials, magic-byte
image check, 8 MB cap. Poster proxy shares the same helper.
- Identity: site ids count as explicit-id evidence; when TMDB is
queried, a TMDB id goes straight to details and an IMDb id to
/find (no title search on a hit). Douban only for local recall.
- Release drawer "站点详情页" section; fixtures and tests for HHClub div
layout, rowhead layout, BBCode, cache/proxy/identity end to end.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
6e9e8efec2
commit
04cffcd1f1
30 files changed
+1133
-64
No files matched your search
@@ -1,4 +1,3 @@
|
|||||||
import { assertPublicUrl } from "@umt/adapters";
|
|
||||||
import { isCapabilityError } from "@umt/agent";
|
import { isCapabilityError } from "@umt/agent";
|
||||||
import {
|
import {
|
||||||
EnrichApplyInput,
|
EnrichApplyInput,
|
||||||
@@ -14,6 +13,7 @@ import type { ModuleApp } from "../../types.ts";
|
|||||||
import { route } from "../http.ts";
|
import { route } from "../http.ts";
|
||||||
import type { SourcesRuntime } from "../runtime.ts";
|
import type { SourcesRuntime } from "../runtime.ts";
|
||||||
import { applyEnrichment, domainAllowed, hostOf } from "../service/enrich.ts";
|
import { applyEnrichment, domainAllowed, hostOf } from "../service/enrich.ts";
|
||||||
|
import { fetchPublicImage, imageResponse } from "../service/image-proxy.ts";
|
||||||
|
|
||||||
// META-1 元数据补全 API:先看建议(POST /identity-pool/{id}/enrich,经能力注册中心调用 metadata.enrich,
|
// META-1 元数据补全 API:先看建议(POST /identity-pool/{id}/enrich,经能力注册中心调用 metadata.enrich,
|
||||||
// 授权 / 审计与 Agent 同一路径),再应用(POST /identity-pool/{id}/enrich/apply,只接受控制台管理员会话);
|
// 授权 / 审计与 Agent 同一路径),再应用(POST /identity-pool/{id}/enrich/apply,只接受控制台管理员会话);
|
||||||
@@ -22,19 +22,6 @@ import { applyEnrichment, domainAllowed, hostOf } from "../service/enrich.ts";
|
|||||||
const T_ID = ["identity"];
|
const T_ID = ["identity"];
|
||||||
const POSTER_MAX_BYTES = 5 * 1024 * 1024;
|
const POSTER_MAX_BYTES = 5 * 1024 * 1024;
|
||||||
|
|
||||||
function imageMime(b: Uint8Array): string | null {
|
|
||||||
if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
|
|
||||||
if (b.length >= 8 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47) return "image/png";
|
|
||||||
if (
|
|
||||||
b.length >= 12 &&
|
|
||||||
String.fromCharCode(...b.slice(0, 4)) === "RIFF" &&
|
|
||||||
String.fromCharCode(...b.slice(8, 12)) === "WEBP"
|
|
||||||
)
|
|
||||||
return "image/webp";
|
|
||||||
if (b.length >= 6 && String.fromCharCode(...b.slice(0, 4)) === "GIF8") return "image/gif";
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void {
|
export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void {
|
||||||
route(
|
route(
|
||||||
app,
|
app,
|
||||||
@@ -119,44 +106,8 @@ export function enrichRoutes(app: ModuleApp, rt: () => SourcesRuntime): void {
|
|||||||
const srcHost = hostOf(s.data.fields.posterUrl?.sourceUrl ?? "");
|
const srcHost = hostOf(s.data.fields.posterUrl?.sourceUrl ?? "");
|
||||||
if (!srcHost || !domainAllowed(srcHost, identity.enrichDomains ?? []))
|
if (!srcHost || !domainAllowed(srcHost, identity.enrichDomains ?? []))
|
||||||
return c.json(errorBody("forbidden", "海报出处不在补全白名单里"), 403);
|
return c.json(errorBody("forbidden", "海报出处不在补全白名单里"), 403);
|
||||||
try {
|
const got = await fetchPublicImage(r.overrides.fetch ?? fetch, url, { maxBytes: POSTER_MAX_BYTES, label: "海报" });
|
||||||
const doFetch = r.overrides.fetch ?? fetch;
|
if (!got.ok) return c.json(errorBody("upstream_unavailable", got.message), 502);
|
||||||
let target = await assertPublicUrl(url);
|
return imageResponse(got, 86400 * 7);
|
||||||
let res: Response | null = null;
|
|
||||||
for (let hop = 0; hop <= 3; hop++) {
|
|
||||||
res = await doFetch(target.href, {
|
|
||||||
redirect: "manual",
|
|
||||||
signal: AbortSignal.timeout(15_000),
|
|
||||||
headers: { accept: "image/avif,image/webp,image/*;q=0.8" },
|
|
||||||
});
|
|
||||||
const loc = res.status >= 300 && res.status < 400 ? res.headers.get("location") : null;
|
|
||||||
if (!loc) break;
|
|
||||||
await res.body?.cancel().catch(() => {});
|
|
||||||
if (hop === 3) throw new Error("重定向次数过多");
|
|
||||||
target = await assertPublicUrl(new URL(loc, target).href);
|
|
||||||
}
|
|
||||||
if (res?.status !== 200) {
|
|
||||||
await res?.body?.cancel().catch(() => {});
|
|
||||||
return c.json(errorBody("upstream_unavailable", `海报返回 HTTP ${res?.status ?? "?"}`), 502);
|
|
||||||
}
|
|
||||||
const len = Number(res.headers.get("content-length") ?? 0);
|
|
||||||
if (len > POSTER_MAX_BYTES) {
|
|
||||||
await res.body?.cancel().catch(() => {});
|
|
||||||
return c.json(errorBody("upstream_unavailable", "海报超过 5 MB"), 502);
|
|
||||||
}
|
|
||||||
const buf = new Uint8Array(await res.arrayBuffer());
|
|
||||||
const mime = buf.byteLength <= POSTER_MAX_BYTES ? imageMime(buf) : null;
|
|
||||||
if (!mime) return c.json(errorBody("upstream_unavailable", "海报地址返回的不是图片"), 502);
|
|
||||||
return new Response(buf as BodyInit, {
|
|
||||||
status: 200,
|
|
||||||
headers: {
|
|
||||||
"content-type": mime,
|
|
||||||
"cache-control": `public, max-age=${86400 * 7}`,
|
|
||||||
"x-content-type-options": "nosniff",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} catch (e) {
|
|
||||||
return c.json(errorBody("upstream_unavailable", `拉取海报失败：${(e as Error).message}`), 502);
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -1,14 +1,19 @@
|
|||||||
import { Id, WorkbenchReleaseDetail, WorkbenchReleasePage, WorkbenchReleaseQuery } from "@umt/contracts";
|
import { Id, WorkbenchReleaseDetail, WorkbenchReleasePage, WorkbenchReleaseQuery } from "@umt/contracts";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
import { requireScope } from "../../../http/auth.ts";
|
||||||
|
import { errorBody } from "../../../http/errors.ts";
|
||||||
import type { ModuleApp } from "../../types.ts";
|
import type { ModuleApp } from "../../types.ts";
|
||||||
import { route } from "../http.ts";
|
import { route } from "../http.ts";
|
||||||
import type { SourcesRuntime } from "../runtime.ts";
|
import type { SourcesRuntime } from "../runtime.ts";
|
||||||
|
import { detailScreenshotUrl } from "../service/detail-page.ts";
|
||||||
|
import { fetchPublicImage, imageResponse } from "../service/image-proxy.ts";
|
||||||
import { listSiteReleases, releaseDetail } from "../service/workbench.ts";
|
import { listSiteReleases, releaseDetail } from "../service/workbench.ts";
|
||||||
|
|
||||||
// 站点工作台(整理视图)数据 API。库存叠加(已有资产/Slot/下载)不在这里:条目带 entity.id,
|
// 站点工作台(整理视图)数据 API。库存叠加(已有资产/Slot/下载)不在这里:条目带 entity.id,
|
||||||
// 前端按作品 ID 二次查询 W2-C 的媒体库端点。
|
// 前端按作品 ID 二次查询 W2-C 的媒体库端点。
|
||||||
|
|
||||||
const T = ["site-workbench"];
|
const T = ["site-workbench"];
|
||||||
|
const SHOT_MAX_BYTES = 8 * 1024 * 1024;
|
||||||
|
|
||||||
export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||||
route(
|
route(
|
||||||
@@ -39,4 +44,19 @@ export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
|||||||
},
|
},
|
||||||
async ({ params, query }) => releaseDetail(rt(), params.id, params.rid, query.refresh === "true"),
|
async ({ params, query }) => releaseDetail(rt(), params.id, params.rid, query.refresh === "true"),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// 详情页截图代理:不进 OpenAPI(二进制响应);只取该资源详情缓存里的第 index 张图,不接受任意 URL,不带站点凭据
|
||||||
|
app.get("/img/site-detail/:rid/:index", requireScope("read"), async (c) => {
|
||||||
|
const rid = c.req.param("rid");
|
||||||
|
const index = Number(c.req.param("index"));
|
||||||
|
if (!/^[0-9a-f-]{36}$/i.test(rid) || !Number.isInteger(index) || index < 0 || index > 99) {
|
||||||
|
return c.json(errorBody("bad_request", "截图地址无效"), 400);
|
||||||
|
}
|
||||||
|
const r = rt();
|
||||||
|
const url = await detailScreenshotUrl(r, rid, index);
|
||||||
|
if (!url) return c.json(errorBody("not_found", "截图不存在或详情缓存已过期"), 404);
|
||||||
|
const got = await fetchPublicImage(r.overrides.fetch ?? fetch, url, { maxBytes: SHOT_MAX_BYTES, label: "截图" });
|
||||||
|
if (!got.ok) return c.json(errorBody("upstream_unavailable", got.message), 502);
|
||||||
|
return imageResponse(got, 86_400);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
import type { SiteDetailPage } from "@umt/adapters";
|
||||||
|
import type { SiteReleasePage } from "@umt/contracts";
|
||||||
|
import { sql } from "@umt/db";
|
||||||
|
import { sharedCache } from "../../../shared-cache.ts";
|
||||||
|
import type { SourcesRuntime } from "../runtime.ts";
|
||||||
|
import type { Exec } from "./common.ts";
|
||||||
|
|
||||||
|
// 站点详情页扩展内容(简介 / MediaInfo / 截图 / 外部 ID / 文件 / 标签 / 上传者):
|
||||||
|
// - 只在按需抓详情时得到(工作台打开详情、release.inspect 复核),放 PG kv_cache region site.detail,
|
||||||
|
// key = source_release.id,TTL = 站点详情缓存期(至少 24 小时);不进 source_release 列,过期即丢。
|
||||||
|
// - 外部 ID(IMDb / 豆瓣 / TMDB)另写 source_release.raw._detail(列表快照 upsert 不覆盖它),供身份解析用。
|
||||||
|
// - 截图对外只给代理路径 /api/v1/img/site-detail/{releaseId}/{index},原始地址留在服务端。
|
||||||
|
|
||||||
|
export const DETAIL_PAGE_REGION = "site.detail";
|
||||||
|
const MIN_TTL_SEC = 86_400;
|
||||||
|
const MAX_ENTRIES = 5_000;
|
||||||
|
|
||||||
|
interface StoredPage extends SiteDetailPage {
|
||||||
|
fetchedAt: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DetailIds {
|
||||||
|
imdbId?: string;
|
||||||
|
doubanId?: string;
|
||||||
|
tmdb?: { type: "movie" | "tv"; id: string };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function saveDetailPage(
|
||||||
|
rt: SourcesRuntime,
|
||||||
|
exec: Exec,
|
||||||
|
srId: string,
|
||||||
|
page: SiteDetailPage,
|
||||||
|
cacheSec: number,
|
||||||
|
): Promise<void> {
|
||||||
|
const stored: StoredPage = { ...page, fetchedAt: rt.now().toISOString() };
|
||||||
|
await sharedCache(rt.ctx)
|
||||||
|
.set(DETAIL_PAGE_REGION, srId, stored, { ttl: Math.max(cacheSec, MIN_TTL_SEC), maxsize: MAX_ENTRIES })
|
||||||
|
.catch((e) => rt.ctx.logger.warn({ err: { message: (e as Error).message } }, "详情页缓存写入失败"));
|
||||||
|
const ids: DetailIds = {
|
||||||
|
...(page.imdbId ? { imdbId: page.imdbId } : {}),
|
||||||
|
...(page.doubanId ? { doubanId: page.doubanId } : {}),
|
||||||
|
...(page.tmdb ? { tmdb: page.tmdb } : {}),
|
||||||
|
};
|
||||||
|
await sql`update source_release set raw = raw || jsonb_build_object('_detail', ${JSON.stringify(ids)}::jsonb)
|
||||||
|
where id = ${srId}`.execute(exec);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadStored(rt: SourcesRuntime, srId: string): Promise<StoredPage | null> {
|
||||||
|
try {
|
||||||
|
const v = (await sharedCache(rt.ctx).get(DETAIL_PAGE_REGION, srId)) as StoredPage | null;
|
||||||
|
return v && typeof v === "object" && typeof v.fetchedAt === "string" ? v : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function hasDetailPage(rt: SourcesRuntime, srId: string): Promise<boolean> {
|
||||||
|
return (await loadStored(rt, srId)) !== null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 缓存里的截图原始地址(图片代理用) */
|
||||||
|
export async function detailScreenshotUrl(rt: SourcesRuntime, srId: string, index: number): Promise<string | null> {
|
||||||
|
return (await loadStored(rt, srId))?.screenshots[index] ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function loadDetailPage(rt: SourcesRuntime, srId: string): Promise<SiteReleasePage | null> {
|
||||||
|
const p = await loadStored(rt, srId);
|
||||||
|
if (!p) return null;
|
||||||
|
return {
|
||||||
|
fetchedAt: p.fetchedAt,
|
||||||
|
descriptionText: p.descriptionText,
|
||||||
|
mediaInfo: p.mediaInfo,
|
||||||
|
screenshots: p.screenshots.map((u, i) => {
|
||||||
|
let host = "";
|
||||||
|
try {
|
||||||
|
host = new URL(u).host;
|
||||||
|
} catch {
|
||||||
|
/* 已在解析时校验 */
|
||||||
|
}
|
||||||
|
return { url: `/api/v1/img/site-detail/${srId}/${i}`, host };
|
||||||
|
}),
|
||||||
|
imdbId: p.imdbId,
|
||||||
|
doubanId: p.doubanId,
|
||||||
|
tmdb: p.tmdb,
|
||||||
|
files: p.files,
|
||||||
|
tags: p.tags,
|
||||||
|
uploader: p.uploader,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** source_release.raw 里站点给出的外部 ID(列表 _site + 详情 _detail) */
|
||||||
|
export function siteExplicitIds(
|
||||||
|
raw: unknown,
|
||||||
|
): { provider: "imdb" | "douban" | "tmdb"; id: string; type?: "movie" | "tv" }[] {
|
||||||
|
const r = (raw ?? {}) as { _site?: { imdbId?: unknown; doubanId?: unknown }; _detail?: DetailIds };
|
||||||
|
const out: { provider: "imdb" | "douban" | "tmdb"; id: string; type?: "movie" | "tv" }[] = [];
|
||||||
|
const add = (provider: "imdb" | "douban" | "tmdb", id: unknown, type?: "movie" | "tv") => {
|
||||||
|
if (typeof id !== "string" || !id || out.some((x) => x.provider === provider && x.id === id)) return;
|
||||||
|
out.push({ provider, id, ...(type ? { type } : {}) });
|
||||||
|
};
|
||||||
|
add("imdb", r._detail?.imdbId);
|
||||||
|
add("imdb", r._site?.imdbId);
|
||||||
|
add("douban", r._detail?.doubanId);
|
||||||
|
add("douban", r._site?.doubanId);
|
||||||
|
if (r._detail?.tmdb) add("tmdb", r._detail.tmdb.id, r._detail.tmdb.type);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import { assertPublicUrl } from "@umt/adapters";
|
||||||
|
|
||||||
|
// 外站图片代理(补全海报、站点详情截图):只取公网地址(每跳重定向都复核),不带任何凭据,
|
||||||
|
// 按文件头认图片类型(不信任上游 content-type,不放行 SVG),限制大小。
|
||||||
|
|
||||||
|
export type ImageFetchResult =
|
||||||
|
| { ok: true; bytes: Uint8Array; mime: string }
|
||||||
|
| { ok: false; status: 404 | 502; message: string };
|
||||||
|
|
||||||
|
export function imageMime(b: Uint8Array): string | null {
|
||||||
|
if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
|
||||||
|
if (b.length >= 8 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47) return "image/png";
|
||||||
|
if (
|
||||||
|
b.length >= 12 &&
|
||||||
|
String.fromCharCode(...b.slice(0, 4)) === "RIFF" &&
|
||||||
|
String.fromCharCode(...b.slice(8, 12)) === "WEBP"
|
||||||
|
)
|
||||||
|
return "image/webp";
|
||||||
|
if (b.length >= 6 && String.fromCharCode(...b.slice(0, 4)) === "GIF8") return "image/gif";
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchPublicImage(
|
||||||
|
doFetch: typeof fetch,
|
||||||
|
url: string,
|
||||||
|
opts: { maxBytes: number; label: string; referer?: string },
|
||||||
|
): Promise<ImageFetchResult> {
|
||||||
|
const { maxBytes, label } = opts;
|
||||||
|
try {
|
||||||
|
let target = await assertPublicUrl(url);
|
||||||
|
let res: Response | null = null;
|
||||||
|
for (let hop = 0; hop <= 3; hop++) {
|
||||||
|
res = await doFetch(target.href, {
|
||||||
|
redirect: "manual",
|
||||||
|
signal: AbortSignal.timeout(15_000),
|
||||||
|
headers: {
|
||||||
|
accept: "image/avif,image/webp,image/*;q=0.8",
|
||||||
|
...(opts.referer ? { referer: opts.referer } : {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const loc = res.status >= 300 && res.status < 400 ? res.headers.get("location") : null;
|
||||||
|
if (!loc) break;
|
||||||
|
await res.body?.cancel().catch(() => {});
|
||||||
|
if (hop === 3) throw new Error("重定向次数过多");
|
||||||
|
target = await assertPublicUrl(new URL(loc, target).href);
|
||||||
|
}
|
||||||
|
if (res?.status !== 200) {
|
||||||
|
await res?.body?.cancel().catch(() => {});
|
||||||
|
return { ok: false, status: 502, message: `${label}返回 HTTP ${res?.status ?? "?"}` };
|
||||||
|
}
|
||||||
|
const len = Number(res.headers.get("content-length") ?? 0);
|
||||||
|
const mb = Math.round(maxBytes / 1024 / 1024);
|
||||||
|
if (len > maxBytes) {
|
||||||
|
await res.body?.cancel().catch(() => {});
|
||||||
|
return { ok: false, status: 502, message: `${label}超过 ${mb} MB` };
|
||||||
|
}
|
||||||
|
const buf = new Uint8Array(await res.arrayBuffer());
|
||||||
|
const mime = buf.byteLength <= maxBytes ? imageMime(buf) : null;
|
||||||
|
if (!mime) return { ok: false, status: 502, message: `${label}地址返回的不是图片` };
|
||||||
|
return { ok: true, bytes: buf, mime };
|
||||||
|
} catch (e) {
|
||||||
|
return { ok: false, status: 502, message: `拉取${label}失败：${(e as Error).message}` };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function imageResponse(r: { bytes: Uint8Array; mime: string }, maxAgeSec: number): Response {
|
||||||
|
return new Response(r.bytes as BodyInit, {
|
||||||
|
status: 200,
|
||||||
|
headers: {
|
||||||
|
"content-type": r.mime,
|
||||||
|
"cache-control": `private, max-age=${maxAgeSec}`,
|
||||||
|
"x-content-type-options": "nosniff",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -23,6 +23,7 @@ import {
|
|||||||
import { jsonb } from "../rows.ts";
|
import { jsonb } from "../rows.ts";
|
||||||
import type { SourcesRuntime } from "../runtime.ts";
|
import type { SourcesRuntime } from "../runtime.ts";
|
||||||
import { type Actor, closeDecisions, type Exec, SYSTEM } from "./common.ts";
|
import { type Actor, closeDecisions, type Exec, SYSTEM } from "./common.ts";
|
||||||
|
import { siteExplicitIds } from "./detail-page.ts";
|
||||||
import { maybeAutoEnrich } from "./enrich.ts";
|
import { maybeAutoEnrich } from "./enrich.ts";
|
||||||
import {
|
import {
|
||||||
addAliasIfAbsent,
|
addAliasIfAbsent,
|
||||||
@@ -113,6 +114,7 @@ export function tmdbToCandidate(m: MetadataCandidate): DomainCandidate {
|
|||||||
...(m.year !== null ? { year: m.year } : {}),
|
...(m.year !== null ? { year: m.year } : {}),
|
||||||
aliases,
|
aliases,
|
||||||
...(seasonYears?.length ? { seasonYears } : {}),
|
...(seasonYears?.length ? { seasonYears } : {}),
|
||||||
|
...(m.externalIds?.imdb ? { linkedExternalIds: [{ provider: "imdb", id: m.externalIds.imdb }] } : {}),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -137,8 +139,7 @@ async function localCandidates(
|
|||||||
release: SourceReleaseRow,
|
release: SourceReleaseRow,
|
||||||
): Promise<DomainCandidate[]> {
|
): Promise<DomainCandidate[]> {
|
||||||
const ids = new Set<string>();
|
const ids = new Set<string>();
|
||||||
const site = ((release.raw as Record<string, unknown>)?._site ?? {}) as { imdbId?: string };
|
const explicit = [...parsed.explicitIds, ...siteExplicitIds(release.raw)];
|
||||||
const explicit = [...parsed.explicitIds, ...(site.imdbId ? [{ provider: "imdb" as const, id: site.imdbId }] : [])];
|
|
||||||
for (const x of explicit) {
|
for (const x of explicit) {
|
||||||
const row = await exec
|
const row = await exec
|
||||||
.selectFrom("external_identity")
|
.selectFrom("external_identity")
|
||||||
@@ -190,10 +191,14 @@ async function tmdbCandidates(
|
|||||||
rt: SourcesRuntime,
|
rt: SourcesRuntime,
|
||||||
parsed: ParsedRelease,
|
parsed: ParsedRelease,
|
||||||
useCache: boolean,
|
useCache: boolean,
|
||||||
|
siteIds: ReturnType<typeof siteExplicitIds> = [],
|
||||||
): Promise<{ remote: MetadataCandidate[]; cacheHit: boolean; meta: RecognizeMeta | null }> {
|
): Promise<{ remote: MetadataCandidate[]; cacheHit: boolean; meta: RecognizeMeta | null }> {
|
||||||
const tmdb = await rt.tmdb();
|
const tmdb = await rt.tmdb();
|
||||||
if (!tmdb) return { remote: [], cacheHit: false, meta: null };
|
if (!tmdb) return { remote: [], cacheHit: false, meta: null };
|
||||||
const meta = recognizeMetaOf(parsed);
|
const meta = recognizeMetaOf(parsed);
|
||||||
|
// 站点详情页给出的 TMDB / IMDb ID:按 ID 直取(TMDB 详情 / find),命中就不再按标题搜索
|
||||||
|
const byId = await tmdbByExplicitIds(tmdb, siteIds);
|
||||||
|
if (byId.length) return { remote: byId, cacheHit: false, meta };
|
||||||
if (meta && useCache) {
|
if (meta && useCache) {
|
||||||
const hit = await loadRecognizeCache(rt, meta);
|
const hit = await loadRecognizeCache(rt, meta);
|
||||||
if (hit) {
|
if (hit) {
|
||||||
@@ -214,6 +219,25 @@ async function tmdbCandidates(
|
|||||||
return { remote: [...seen.values()].slice(0, 20), cacheHit: false, meta };
|
return { remote: [...seen.values()].slice(0, 20), cacheHit: false, meta };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function tmdbByExplicitIds(
|
||||||
|
tmdb: NonNullable<Awaited<ReturnType<SourcesRuntime["tmdb"]>>>,
|
||||||
|
ids: ReturnType<typeof siteExplicitIds>,
|
||||||
|
): Promise<MetadataCandidate[]> {
|
||||||
|
const tmdbId = ids.find((x) => x.provider === "tmdb" && x.type);
|
||||||
|
if (tmdbId?.type) {
|
||||||
|
try {
|
||||||
|
return [await tmdb.details(tmdbId.type, tmdbId.id)];
|
||||||
|
} catch (e) {
|
||||||
|
if (!(e instanceof TmdbError) || e.kind !== "notfound") throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const imdb = ids.find((x) => x.provider === "imdb");
|
||||||
|
if (!imdb) return [];
|
||||||
|
const found = await tmdb.findByImdb(imdb.id);
|
||||||
|
for (const m of found) m.externalIds = { ...m.externalIds, imdb: imdb.id };
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
async function loadRelease(rt: SourcesRuntime, input: ResolveInput) {
|
async function loadRelease(rt: SourcesRuntime, input: ResolveInput) {
|
||||||
const { db } = rt.ctx;
|
const { db } = rt.ctx;
|
||||||
const observationId = input.observationId ?? null;
|
const observationId = input.observationId ?? null;
|
||||||
@@ -277,6 +301,12 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P
|
|||||||
}
|
}
|
||||||
const parsed = parseRelease(release.title, release.subtitle ? { subtitle: release.subtitle } : {});
|
const parsed = parseRelease(release.title, release.subtitle ? { subtitle: release.subtitle } : {});
|
||||||
const cols = toParsedColumns(parsed);
|
const cols = toParsedColumns(parsed);
|
||||||
|
// 站点给出的外部 ID(列表 / 详情页的 IMDb、豆瓣、TMDB 链接)作为显式 ID 证据参与打分
|
||||||
|
const siteIds = siteExplicitIds(release.raw);
|
||||||
|
for (const x of siteIds) {
|
||||||
|
if (!parsed.explicitIds.some((e) => e.provider === x.provider))
|
||||||
|
parsed.explicitIds.push({ provider: x.provider, id: x.id });
|
||||||
|
}
|
||||||
const trigger: ResolveTrigger = input.trigger ?? (input.force ? "review" : "observed");
|
const trigger: ResolveTrigger = input.trigger ?? (input.force ? "review" : "observed");
|
||||||
if (!parsed.title || parsed.auxiliaryOnly) {
|
if (!parsed.title || parsed.auxiliaryOnly) {
|
||||||
return toPool(rt, release, observationId, parsed, null, [], "unmatched", "unparseable_title", null, {
|
return toPool(rt, release, observationId, parsed, null, [], "unmatched", "unparseable_title", null, {
|
||||||
@@ -297,7 +327,7 @@ export async function resolveRelease(rt: SourcesRuntime, input: ResolveInput): P
|
|||||||
let lookupError: string | null = null;
|
let lookupError: string | null = null;
|
||||||
let recognize: { cacheHit: boolean; meta: RecognizeMeta | null } = { cacheHit: false, meta: null };
|
let recognize: { cacheHit: boolean; meta: RecognizeMeta | null } = { cacheHit: false, meta: null };
|
||||||
try {
|
try {
|
||||||
const r = await tmdbCandidates(rt, parsed, !input.force);
|
const r = await tmdbCandidates(rt, parsed, !input.force, siteIds);
|
||||||
remote = r.remote;
|
remote = r.remote;
|
||||||
recognize = r;
|
recognize = r;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { SiteError, type SitePage, type SiteRelease } from "@umt/adapters";
|
import { type SiteDetailPage, SiteError, type SitePage, type SiteRelease } from "@umt/adapters";
|
||||||
import type { SiteSnapshotKind } from "@umt/contracts";
|
import type { SiteSnapshotKind } from "@umt/contracts";
|
||||||
import { type SiteRow, sql } from "@umt/db";
|
import { type SiteRow, sql } from "@umt/db";
|
||||||
import { parseRelease } from "@umt/domain";
|
import { parseRelease } from "@umt/domain";
|
||||||
@@ -7,6 +7,7 @@ import { jsonb } from "../rows.ts";
|
|||||||
import type { SourcesRuntime } from "../runtime.ts";
|
import type { SourcesRuntime } from "../runtime.ts";
|
||||||
import { enqueueRelogin } from "./account.ts";
|
import { enqueueRelogin } from "./account.ts";
|
||||||
import { type Exec, openDecision } from "./common.ts";
|
import { type Exec, openDecision } from "./common.ts";
|
||||||
|
import { hasDetailPage, saveDetailPage } from "./detail-page.ts";
|
||||||
|
|
||||||
// 站点快照(周期列表)、补漏(Feed 疑似缺口,有页数预算)、实时补查(工作台搜索)。
|
// 站点快照(周期列表)、补漏(Feed 疑似缺口,有页数预算)、实时补查(工作台搜索)。
|
||||||
// 所有请求经站点共享限流器;结果按 site+站点资源 ID upsert 到 source_release,并记录 site_snapshot 覆盖范围。
|
// 所有请求经站点共享限流器;结果按 site+站点资源 ID upsert 到 source_release,并记录 site_snapshot 覆盖范围。
|
||||||
@@ -215,14 +216,22 @@ export async function refreshReleaseDetail(rt: SourcesRuntime, srId: string): Pr
|
|||||||
if (!rel) return "not_found";
|
if (!rel) return "not_found";
|
||||||
const site = await db.selectFrom("site").selectAll().where("id", "=", rel.site_id).executeTakeFirst();
|
const site = await db.selectFrom("site").selectAll().where("id", "=", rel.site_id).executeTakeFirst();
|
||||||
if (!site?.enabled) return "site_disabled";
|
if (!site?.enabled) return "site_disabled";
|
||||||
// 详情缓存:站点访问默认的缓存期内抓过就不再请求站点(列表快照会持续更新做种 / 促销)
|
// 详情缓存:站点访问默认的缓存期内抓过、且详情页扩展内容还在缓存里,就不再请求站点(列表快照会持续更新做种 / 促销)
|
||||||
const { detailCacheSec } = await rt.siteAccess(site.key);
|
const { detailCacheSec } = await rt.siteAccess(site.key);
|
||||||
if (rel.detail_fetched_at && rt.now().getTime() - rel.detail_fetched_at.getTime() < detailCacheSec * 1000) {
|
if (
|
||||||
|
rel.detail_fetched_at &&
|
||||||
|
rt.now().getTime() - rel.detail_fetched_at.getTime() < detailCacheSec * 1000 &&
|
||||||
|
(await hasDetailPage(rt, srId))
|
||||||
|
) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
let detail: SiteRelease;
|
let detail: SiteRelease;
|
||||||
|
let page: SiteDetailPage | undefined;
|
||||||
try {
|
try {
|
||||||
detail = mergeDetailOverList(await rt.siteAdapter(site).detail(rel.site_release_id), rel);
|
const d = await rt.siteAdapter(site).detail(rel.site_release_id);
|
||||||
|
page = d.page;
|
||||||
|
const { page: _p, descriptionText: _d, ...plain } = d;
|
||||||
|
detail = mergeDetailOverList(plain, rel);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (e instanceof SiteError) {
|
if (e instanceof SiteError) {
|
||||||
if (e.isAuthError) await db.transaction().execute((trx) => onAuthError(rt, trx, site, e));
|
if (e.isAuthError) await db.transaction().execute((trx) => onAuthError(rt, trx, site, e));
|
||||||
@@ -247,6 +256,7 @@ export async function refreshReleaseDetail(rt: SourcesRuntime, srId: string): Pr
|
|||||||
})
|
})
|
||||||
.where("id", "=", srId)
|
.where("id", "=", srId)
|
||||||
.execute();
|
.execute();
|
||||||
|
if (page) await saveDetailPage(rt, trx, srId, page, detailCacheSec);
|
||||||
});
|
});
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import { normalizeTitle } from "@umt/domain";
|
|||||||
import { ApiException } from "../../../http/errors.ts";
|
import { ApiException } from "../../../http/errors.ts";
|
||||||
import { decodeCursor, page, toApi } from "../rows.ts";
|
import { decodeCursor, page, toApi } from "../rows.ts";
|
||||||
import { type SourcesRuntime, siteSearchConcurrency } from "../runtime.ts";
|
import { type SourcesRuntime, siteSearchConcurrency } from "../runtime.ts";
|
||||||
|
import { loadDetailPage } from "./detail-page.ts";
|
||||||
import { refreshReleaseDetail, snapshotSite } from "./snapshot.ts";
|
import { refreshReleaseDetail, snapshotSite } from "./snapshot.ts";
|
||||||
|
|
||||||
// 站点工作台(整理视图):本地快照优先;本地无结果 / 快照过期 / 用户要求时对已开启站点实时补查,
|
// 站点工作台(整理视图):本地快照优先;本地无结果 / 快照过期 / 用户要求时对已开启站点实时补查,
|
||||||
@@ -224,5 +225,17 @@ export async function releaseDetail(
|
|||||||
recheckBeforeDownload: true,
|
recheckBeforeDownload: true,
|
||||||
},
|
},
|
||||||
refreshError,
|
refreshError,
|
||||||
|
page: await loadDetailPage(rt, releaseId),
|
||||||
|
detailSupported: await detailSupported(rt, siteId),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function detailSupported(rt: SourcesRuntime, siteId: string): Promise<boolean> {
|
||||||
|
const site = await rt.ctx.db.selectFrom("site").selectAll().where("id", "=", siteId).executeTakeFirst();
|
||||||
|
if (!site?.enabled) return false;
|
||||||
|
try {
|
||||||
|
return rt.siteAdapter(site).capabilities.detail;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
import {
|
||||||
|
AUDIENCES_FIXTURE_RELEASES,
|
||||||
|
createMockNexusPhpSite,
|
||||||
|
createMockTmdb,
|
||||||
|
type MockNexusPhpSite,
|
||||||
|
type MockTmdb,
|
||||||
|
} from "@umt/testkit";
|
||||||
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||||
|
import { DETAIL_PAGE_REGION } from "../../src/modules/sources-identity/service/detail-page.ts";
|
||||||
|
import { resolveRelease } from "../../src/modules/sources-identity/service/resolve.ts";
|
||||||
|
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
|
||||||
|
import { sharedCache } from "../../src/shared-cache.ts";
|
||||||
|
import { createHarness, type Harness, json, pgAvailable, seedSite } from "./harness.ts";
|
||||||
|
|
||||||
|
const up = await pgAvailable();
|
||||||
|
|
||||||
|
const SHOT = "http://93.184.216.34/shots/1.png";
|
||||||
|
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3, 4]);
|
||||||
|
|
||||||
|
describe.skipIf(!up)("站点详情页扩展内容:按需抓 + 24 小时缓存 + 截图代理 + 外部 ID 进身份解析", () => {
|
||||||
|
let h: Harness;
|
||||||
|
let mock: MockNexusPhpSite;
|
||||||
|
let tmdb: MockTmdb;
|
||||||
|
let siteId: string;
|
||||||
|
let relId: string;
|
||||||
|
const imageFetches: string[] = [];
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
const releases = AUDIENCES_FIXTURE_RELEASES.map((r) =>
|
||||||
|
r.id === "812345"
|
||||||
|
? {
|
||||||
|
...r,
|
||||||
|
descriptionHtml: `◎片名 沙丘2<br>◎IMDb <a href="https://www.imdb.com/title/tt15239678/">tt15239678</a><br>
|
||||||
|
<img src="${SHOT}"><img src="pic/smilies/2.gif">
|
||||||
|
<div class="codemain">General<br>Complete name : Dune.mkv<br>Format : Matroska<br>File size : 78 GiB<br>Duration : 2 h 46 min<br>Video<br>Format : HEVC</div>`,
|
||||||
|
}
|
||||||
|
: r,
|
||||||
|
);
|
||||||
|
mock = await createMockNexusPhpSite({ releases });
|
||||||
|
tmdb = await createMockTmdb();
|
||||||
|
h = await createHarness("detailpage", {
|
||||||
|
tmdbUrl: tmdb.url,
|
||||||
|
fetch: (input, init) => {
|
||||||
|
const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
|
||||||
|
if (url.startsWith("http://93.184.216.34/")) {
|
||||||
|
imageFetches.push(url);
|
||||||
|
return Promise.resolve(new Response(PNG, { status: 200, headers: { "content-type": "image/png" } }));
|
||||||
|
}
|
||||||
|
return fetch(input, init);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
siteId = (await seedSite(h, { baseUrl: `${mock.baseUrl}/`, cookie: mock.cookie })).site.id;
|
||||||
|
await snapshotSite(h.rt, { siteId, kind: "manual" });
|
||||||
|
relId = (
|
||||||
|
await h.db
|
||||||
|
.selectFrom("source_release")
|
||||||
|
.select(["id"])
|
||||||
|
.where("site_release_id", "=", "812345")
|
||||||
|
.executeTakeFirstOrThrow()
|
||||||
|
).id;
|
||||||
|
});
|
||||||
|
afterAll(async () => {
|
||||||
|
await h?.close();
|
||||||
|
await mock?.close();
|
||||||
|
await tmdb?.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
const detailRequests = () => mock.requests.filter((r) => r.path.startsWith("/details.php")).length;
|
||||||
|
|
||||||
|
it("未抓过:page 为 null,标明站点支持详情页;不刷新就不请求站点", async () => {
|
||||||
|
const before = detailRequests();
|
||||||
|
const d = await json(await h.req(`/sites/${siteId}/releases/${relId}`));
|
||||||
|
expect(d.status).toBe(200);
|
||||||
|
expect(d.body.data).toMatchObject({ page: null, detailSupported: true });
|
||||||
|
expect(detailRequests()).toBe(before);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refresh 抓一次:简介 / MediaInfo / 截图(代理路径)/ IMDb;外部 ID 写 raw._detail;缓存期内不再请求", async () => {
|
||||||
|
const before = detailRequests();
|
||||||
|
const d = await json(await h.req(`/sites/${siteId}/releases/${relId}?refresh=true`));
|
||||||
|
expect(d.body.data.refreshError).toBeNull();
|
||||||
|
expect(detailRequests()).toBe(before + 1);
|
||||||
|
const page = d.body.data.page;
|
||||||
|
expect(page).toMatchObject({
|
||||||
|
imdbId: "tt15239678",
|
||||||
|
screenshots: [{ url: `/api/v1/img/site-detail/${relId}/0`, host: "93.184.216.34" }],
|
||||||
|
});
|
||||||
|
expect(page.mediaInfo).toMatch(/^General\nComplete name : Dune.mkv/);
|
||||||
|
expect(page.descriptionText).toContain("◎片名 沙丘2");
|
||||||
|
expect(page.descriptionText).not.toContain("Complete name");
|
||||||
|
const row = await h.db
|
||||||
|
.selectFrom("source_release")
|
||||||
|
.select(["raw"])
|
||||||
|
.where("id", "=", relId)
|
||||||
|
.executeTakeFirstOrThrow();
|
||||||
|
expect((row.raw as { _detail?: unknown })._detail).toEqual({ imdbId: "tt15239678" });
|
||||||
|
|
||||||
|
const again = await json(await h.req(`/sites/${siteId}/releases/${relId}?refresh=true`));
|
||||||
|
expect(again.body.data.page.imdbId).toBe("tt15239678");
|
||||||
|
expect(detailRequests()).toBe(before + 1);
|
||||||
|
|
||||||
|
// 列表快照覆盖 raw._site 后,详情给出的外部 ID 仍在
|
||||||
|
await snapshotSite(h.rt, { siteId, kind: "manual" });
|
||||||
|
const row2 = await h.db
|
||||||
|
.selectFrom("source_release")
|
||||||
|
.select(["raw"])
|
||||||
|
.where("id", "=", relId)
|
||||||
|
.executeTakeFirstOrThrow();
|
||||||
|
expect((row2.raw as { _detail?: unknown })._detail).toEqual({ imdbId: "tt15239678" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("截图代理:只取缓存里的第 index 张,按文件头认图片;越界 404", async () => {
|
||||||
|
const res = await h.req(`/img/site-detail/${relId}/0`);
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
expect(res.headers.get("content-type")).toBe("image/png");
|
||||||
|
expect(new Uint8Array(await res.arrayBuffer())).toEqual(PNG);
|
||||||
|
expect(imageFetches).toEqual([SHOT]);
|
||||||
|
expect((await h.req(`/img/site-detail/${relId}/5`)).status).toBe(404);
|
||||||
|
expect((await h.req(`/img/site-detail/not-an-id/0`)).status).toBe(400);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("缓存丢了(kv_cache 过期)再刷新会重新抓一次", async () => {
|
||||||
|
await sharedCache(h.ctx).delete(DETAIL_PAGE_REGION, relId);
|
||||||
|
const before = detailRequests();
|
||||||
|
const d = await json(await h.req(`/sites/${siteId}/releases/${relId}`));
|
||||||
|
expect(d.body.data.page).toBeNull();
|
||||||
|
await h.req(`/sites/${siteId}/releases/${relId}?refresh=true`);
|
||||||
|
expect(detailRequests()).toBe(before + 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("身份解析:详情页的 IMDb ID 走 TMDB find,不按标题搜索,显式 ID 证据", async () => {
|
||||||
|
const searches = () => tmdb.callsTo(/\/3\/search\//).length;
|
||||||
|
const s0 = searches();
|
||||||
|
const res = await resolveRelease(h.rt, { sourceReleaseId: relId, force: true, trigger: "manual" });
|
||||||
|
expect(tmdb.callsTo(/\/3\/find\/tt15239678/).length).toBeGreaterThan(0);
|
||||||
|
expect(searches()).toBe(s0);
|
||||||
|
expect(res.outcome).toBe("auto");
|
||||||
|
const ev = await h.db
|
||||||
|
.selectFrom("source_release")
|
||||||
|
.select(["resolution_evidence", "media_entity_id"])
|
||||||
|
.where("id", "=", relId)
|
||||||
|
.executeTakeFirstOrThrow();
|
||||||
|
expect(ev.media_entity_id).toBe(res.mediaEntityId);
|
||||||
|
const ext = await h.db
|
||||||
|
.selectFrom("external_identity")
|
||||||
|
.select(["provider", "external_id"])
|
||||||
|
.where("media_entity_id", "=", res.mediaEntityId!)
|
||||||
|
.where("provider", "=", "tmdb")
|
||||||
|
.executeTakeFirstOrThrow();
|
||||||
|
expect(ext.external_id).toBe("693134");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
import type { InventoryEntry, WorkbenchRelease } from "@umt/contracts";
|
import type { InventoryEntry, WorkbenchRelease } from "@umt/contracts";
|
||||||
import { Bot, Download, RefreshCw } from "lucide-react";
|
import { Bot, Download, RefreshCw } from "lucide-react";
|
||||||
import { useState } from "react";
|
import { useEffect, useRef, useState } from "react";
|
||||||
import { Link } from "react-router";
|
import { Link } from "react-router";
|
||||||
import { toast } from "sonner";
|
import { toast } from "sonner";
|
||||||
import { useEntityDetail } from "~/api/inventory.ts";
|
import { useEntityDetail } from "~/api/inventory.ts";
|
||||||
@@ -17,6 +17,7 @@ import { formatBytes, formatDateTime } from "~/lib/utils.ts";
|
|||||||
import { DownloadDialog } from "./download-dialog.tsx";
|
import { DownloadDialog } from "./download-dialog.tsx";
|
||||||
import { categoryLabel, hrOf, ORIGIN_LABEL, relTime } from "./release-fields.ts";
|
import { categoryLabel, hrOf, ORIGIN_LABEL, relTime } from "./release-fields.ts";
|
||||||
import { InventoryOverlay, ReleaseBadges } from "./release-list.tsx";
|
import { InventoryOverlay, ReleaseBadges } from "./release-list.tsx";
|
||||||
|
import { ReleasePageContent } from "./release-page-section.tsx";
|
||||||
|
|
||||||
const DECISION_LABEL = {
|
const DECISION_LABEL = {
|
||||||
accept: { label: "符合，可收录", tone: "success" },
|
accept: { label: "符合，可收录", tone: "success" },
|
||||||
@@ -125,6 +126,16 @@ export function ReleaseDetailSheet({
|
|||||||
const [dl, setDl] = useState(false);
|
const [dl, setDl] = useState(false);
|
||||||
const r = detail.data?.release ?? release;
|
const r = detail.data?.release ?? release;
|
||||||
|
|
||||||
|
// 按需抓详情页:打开抽屉时没有缓存的详情页扩展内容就抓一次(服务端缓存 24 小时,同一资源本次会话只自动抓一次)
|
||||||
|
const autoTried = useRef(new Set<string>());
|
||||||
|
const needPage = !!detail.data && !detail.data.page && detail.data.detailSupported;
|
||||||
|
const rid = release?.id;
|
||||||
|
useEffect(() => {
|
||||||
|
if (!needPage || !rid || autoTried.current.has(rid) || refresh.isPending) return;
|
||||||
|
autoTried.current.add(rid);
|
||||||
|
refresh.mutate({ siteId, rid });
|
||||||
|
}, [needPage, rid, siteId, refresh]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<DetailSheet
|
<DetailSheet
|
||||||
open={!!release}
|
open={!!release}
|
||||||
@@ -216,6 +227,21 @@ export function ReleaseDetailSheet({
|
|||||||
) : null}
|
) : null}
|
||||||
</DetailSection>
|
</DetailSection>
|
||||||
|
|
||||||
|
<DetailSection title="站点详情页">
|
||||||
|
{detail.data?.page ? (
|
||||||
|
<ReleasePageContent page={detail.data.page} />
|
||||||
|
) : refresh.isPending || detail.isPending ? (
|
||||||
|
<LoadingState rows={2} />
|
||||||
|
) : detail.data && !detail.data.detailSupported ? (
|
||||||
|
<p className="text-caption text-muted-foreground">这个站点不支持抓详情页。</p>
|
||||||
|
) : (
|
||||||
|
<p className="text-caption text-muted-foreground">
|
||||||
|
还没有详情页内容{detail.data?.refreshError ? `（${detail.data.refreshError}）` : ""}
|
||||||
|
。点「复核详情」抓取。
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</DetailSection>
|
||||||
|
|
||||||
<DetailSection title="本地库存与候选理由">
|
<DetailSection title="本地库存与候选理由">
|
||||||
<LocalInventory release={r} entry={entry} />
|
<LocalInventory release={r} entry={entry} />
|
||||||
</DetailSection>
|
</DetailSection>
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
import type { SiteReleasePage } from "@umt/contracts";
|
||||||
|
import { ExternalLink } from "lucide-react";
|
||||||
|
import { StatusBadge } from "~/components/console/status-badge.tsx";
|
||||||
|
import { formatBytes, formatDateTime } from "~/lib/utils.ts";
|
||||||
|
|
||||||
|
/** 详情页扩展内容里的外部站点链接 */
|
||||||
|
export function externalLinks(p: Pick<SiteReleasePage, "imdbId" | "doubanId" | "tmdb">) {
|
||||||
|
const out: { label: string; href: string }[] = [];
|
||||||
|
if (p.tmdb) out.push({ label: `TMDB ${p.tmdb.id}`, href: `https://www.themoviedb.org/${p.tmdb.type}/${p.tmdb.id}` });
|
||||||
|
if (p.imdbId) out.push({ label: `IMDb ${p.imdbId}`, href: `https://www.imdb.com/title/${p.imdbId}/` });
|
||||||
|
if (p.doubanId) out.push({ label: `豆瓣 ${p.doubanId}`, href: `https://movie.douban.com/subject/${p.doubanId}/` });
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ReleasePageContent({ page }: { page: SiteReleasePage }) {
|
||||||
|
const links = externalLinks(page);
|
||||||
|
const empty =
|
||||||
|
!links.length &&
|
||||||
|
!page.uploader &&
|
||||||
|
!page.tags.length &&
|
||||||
|
!page.descriptionText &&
|
||||||
|
!page.mediaInfo &&
|
||||||
|
!page.screenshots.length &&
|
||||||
|
!page.files.length;
|
||||||
|
return (
|
||||||
|
<div className="space-y-3">
|
||||||
|
{links.length || page.uploader ? (
|
||||||
|
<div className="flex flex-wrap items-center gap-x-3 gap-y-1 text-caption">
|
||||||
|
{links.map((l) => (
|
||||||
|
<a
|
||||||
|
key={l.href}
|
||||||
|
href={l.href}
|
||||||
|
target="_blank"
|
||||||
|
rel="noreferrer noopener"
|
||||||
|
className="inline-flex items-center gap-1 text-foreground underline underline-offset-4"
|
||||||
|
>
|
||||||
|
{l.label}
|
||||||
|
<ExternalLink className="size-3" />
|
||||||
|
</a>
|
||||||
|
))}
|
||||||
|
{page.uploader ? <span className="text-muted-foreground">上传者 {page.uploader}</span> : null}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
{page.tags.length ? (
|
||||||
|
<div className="flex flex-wrap gap-1.5">
|
||||||
|
{page.tags.map((t) => (
|
||||||
|
<StatusBadge key={t}>{t}</StatusBadge>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
{page.descriptionText ? (
|
||||||
|
<p className="max-h-72 overflow-auto text-caption break-words whitespace-pre-wrap text-foreground">
|
||||||
|
{page.descriptionText}
|
||||||
|
</p>
|
||||||
|
) : null}
|
||||||
|
{page.mediaInfo ? (
|
||||||
|
<details className="rounded-lg border border-border/70 px-3 py-2">
|
||||||
|
<summary className="cursor-pointer text-caption text-foreground">MediaInfo</summary>
|
||||||
|
<pre className="mt-2 max-h-80 overflow-auto font-mono text-meta whitespace-pre text-muted-foreground">
|
||||||
|
{page.mediaInfo}
|
||||||
|
</pre>
|
||||||
|
</details>
|
||||||
|
) : null}
|
||||||
|
{page.screenshots.length ? (
|
||||||
|
<div className="grid grid-cols-2 gap-2">
|
||||||
|
{page.screenshots.map((s, i) => (
|
||||||
|
<a key={s.url} href={s.url} target="_blank" rel="noreferrer noopener" title={s.host}>
|
||||||
|
<img
|
||||||
|
src={s.url}
|
||||||
|
alt={`截图 ${i + 1}`}
|
||||||
|
loading="lazy"
|
||||||
|
className="aspect-video w-full rounded-md border border-border/70 bg-muted object-cover"
|
||||||
|
/>
|
||||||
|
</a>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
{page.files.length ? (
|
||||||
|
<details className="rounded-lg border border-border/70 px-3 py-2">
|
||||||
|
<summary className="cursor-pointer text-caption text-foreground">文件清单（{page.files.length}）</summary>
|
||||||
|
<ul className="mt-2 max-h-60 space-y-1 overflow-auto text-meta">
|
||||||
|
{page.files.map((f, i) => (
|
||||||
|
<li key={`${i}-${f.name}`} className="flex justify-between gap-3">
|
||||||
|
<span className="min-w-0 font-mono break-all text-foreground">{f.name}</span>
|
||||||
|
<span className="shrink-0 text-muted-foreground">
|
||||||
|
{f.sizeBytes === null ? "" : formatBytes(f.sizeBytes, 1)}
|
||||||
|
</span>
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
</details>
|
||||||
|
) : null}
|
||||||
|
{empty ? <p className="text-caption text-muted-foreground">详情页上没有解析到简介、截图或外部链接。</p> : null}
|
||||||
|
<p className="text-meta text-muted-foreground">
|
||||||
|
抓取于 {formatDateTime(page.fetchedAt)}，缓存 24 小时；截图经服务端代理加载。
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -164,5 +164,5 @@ NFO / 图片投影、身份巡检与挂载根因聚合、HTTP 失败自动切浏
|
|||||||
- **短剧等 TMDB 缺失作品的自助补全(META-1,用户 2026-09-29)**:能力 `metadata.enrich`(propose):Brave 搜索 + 受限抓取(平台页/百科类)→ 模型抽字段(标题/别名/年份/平台/集数/简介/海报 URL)带来源与证据等级 → 写本地临时实体与字段级元数据(source=web/agent,可锁可改,不冒充 TMDB);触发:池条目"补全"、订阅/规则命中但 TMDB 无结果、Agent 委托;按归一化标题缓存、每日预算;后续 TMDB 收录由身份巡检补映射;不改路径不下载。DEC-1 之后实施。
|
- **短剧等 TMDB 缺失作品的自助补全(META-1,用户 2026-09-29)**:能力 `metadata.enrich`(propose):Brave 搜索 + 受限抓取(平台页/百科类)→ 模型抽字段(标题/别名/年份/平台/集数/简介/海报 URL)带来源与证据等级 → 写本地临时实体与字段级元数据(source=web/agent,可锁可改,不冒充 TMDB);触发:池条目"补全"、订阅/规则命中但 TMDB 无结果、Agent 委托;按归一化标题缓存、每日预算;后续 TMDB 收录由身份巡检补映射;不改路径不下载。DEC-1 之后实施。
|
||||||
- **Agent 快捷面板与会话排版(AGUI-2,用户 2026-09-29)**:抽屉内「当前/历史」Tab,历史列表可搜索筛选并在抽屉内打开(可续跑),"在页面打开"次要;会话页:阅读列宽、同回合分组、工具卡默认一行折叠、步骤清单改右侧栏/顶部折叠条不再 sticky 遮挡、预算在右栏底、长汇报折叠、输入区固定底部。
|
- **Agent 快捷面板与会话排版(AGUI-2,用户 2026-09-29)**:抽屉内「当前/历史」Tab,历史列表可搜索筛选并在抽屉内打开(可续跑),"在页面打开"次要;会话页:阅读列宽、同回合分组、工具卡默认一行折叠、步骤清单改右侧栏/顶部折叠条不再 sticky 遮挡、预算在右栏底、长汇报折叠、输入区固定底部。
|
||||||
- **凭据统一注入浏览器(BR-2,用户 2026-09-29)**:打开站点浏览器会话时服务端把账号 Cookie(手填或自动登录得到的)注入 camofox profile,截图/桌面模式无需再登录;交还/关闭/定时从浏览器回写 Cookie 到凭据存储,HTTP 抓取/RSS/快照共用;账号显示登录态来源与时间;Cookie 不进日志/模型。
|
- **凭据统一注入浏览器(BR-2,用户 2026-09-29)**:打开站点浏览器会话时服务端把账号 Cookie(手填或自动登录得到的)注入 camofox profile,截图/桌面模式无需再登录;交还/关闭/定时从浏览器回写 Cookie 到凭据存储,HTTP 抓取/RSS/快照共用;账号显示登录态来源与时间;Cookie 不进日志/模型。
|
||||||
- **站点详情页抓取(用户 2026-09-29,憨憨 details.php)**:`SiteReleaseDetail` 扩展简介/MediaInfo/截图(经图片代理)/IMDb·豆瓣·TMDB ID/文件清单/标签/上传者;只按需抓(点开详情或评估缺字段),kv_cache 24 小时;详情抽屉展示;IMDb/豆瓣 ID 作为强证据进身份解析(TMDB find)。
|
- **站点详情页抓取(用户 2026-09-29,憨憨 details.php)**【INT-2 已做,见 runbook/sites.md「详情页扩展内容」】:`SiteReleaseDetail` 扩展简介/MediaInfo/截图(经图片代理)/IMDb·豆瓣·TMDB ID/文件清单/标签/上传者;只按需抓(点开详情或评估缺字段),kv_cache 24 小时;详情抽屉展示;IMDb/豆瓣 ID 作为强证据进身份解析(TMDB find)。
|
||||||
- **META-1 后续**:真机 4 条短剧在白名单站点均无结果——扩大抓取白名单(短剧平台/百科类/豆瓣搜索快照/Bing 摘要模式),必要时只用搜索摘要抽字段不抓页;记录命中率。
|
- **META-1 后续**:真机 4 条短剧在白名单站点均无结果——扩大抓取白名单(短剧平台/百科类/豆瓣搜索快照/Bing 摘要模式),必要时只用搜索摘要抽字段不抓页;记录命中率。
|
||||||
@@ -91,6 +91,16 @@
|
|||||||
|
|
||||||
每从站点限流器拿到一个令牌(列表、搜索、详情、下载、登录、用户数据、原站视图导航)记一次请求,写 `site_request_count`(迁移 `0046`,按小时聚合,多进程共用,保留 7 天)。`GET /sites/{id}/request-stats`(read)返回最近 24 小时总数、按小时明细、快照实际周期与合并后的访问默认;站点页概览「近 24 小时请求」与资源页列表头显示。计数只在升级后开始累计。
|
每从站点限流器拿到一个令牌(列表、搜索、详情、下载、登录、用户数据、原站视图导航)记一次请求,写 `site_request_count`(迁移 `0046`,按小时聚合,多进程共用,保留 7 天)。`GET /sites/{id}/request-stats`(read)返回最近 24 小时总数、按小时明细、快照实际周期与合并后的访问默认;站点页概览「近 24 小时请求」与资源页列表头显示。计数只在升级后开始累计。
|
||||||
|
|
||||||
|
## 详情页扩展内容(INT-2)
|
||||||
|
|
||||||
|
资源详情抽屉的「站点详情页」一节:简介、MediaInfo / BDInfo、截图、IMDb / 豆瓣 / TMDB 链接、文件清单、标签、上传者。
|
||||||
|
|
||||||
|
- 只按需抓:打开抽屉时如果没有缓存,前端自动调一次 `GET /sites/{id}/releases/{rid}?refresh=true`(同一资源本次会话只自动一次),或点「复核详情」;`release.inspect` 的 `refreshDetail` 同一路径。列表快照、搜索不抓详情。
|
||||||
|
- 缓存:内容放 PG `kv_cache` region `site.detail`,key = `source_release.id`,TTL 取站点详情缓存期(至少 24 小时)。缓存期内 `detail_fetched_at` 新鲜且缓存还在就不请求站点;kv_cache 被清掉(过期 / 重启丢 UNLOGGED 表)后下次刷新会重新抓一次。
|
||||||
|
- 外部 ID 另写 `source_release.raw._detail`(`imdbId / doubanId / tmdb{type,id}`),列表快照 upsert 不覆盖它。身份解析把它和列表给的 `raw._site.imdbId / doubanId` 一起当显式 ID:本地召回查已接受的外部身份;需要查 TMDB 时,有 TMDB ID 直接取详情,否则 IMDb ID 走 `/find/{imdb}?external_source=imdb_id`,命中就不再按标题搜索;TMDB 不支持豆瓣 ID,豆瓣只用于本地召回。是否查 TMDB 仍按惰性识别规则(DEC-1),外部 ID 本身不触发查询。
|
||||||
|
- 截图:响应里只有代理路径 `/api/v1/img/site-detail/{releaseId}/{index}`,服务端从缓存取原始地址,公网校验(每跳重定向复核)、不带站点凭据、按文件头认图片(不放行 SVG)、≤ 8 MB,浏览器缓存 24 小时。站内附件如果需要登录才能看会显示为裂图,不会为截图消耗站点请求。
|
||||||
|
- 解析:`packages/adapters/src/sites/spider/detail-page.ts`。简介取 MovieClaw `detail_description_css`(缺省 `#kdescr`);MediaInfo 取定义 `umt.detail.page.mediaInfo` → 「MediaInfo」字段标签 → 简介里第一个像 MediaInfo 的引用 / 代码块;文件清单取 `umt.detail.page.files` → MovieClaw `detail_file_list_css` → 「文件列表」字段 → `#filelist` 表格,不额外请求 `viewfilelist.php`。字段标签除 `td.rowhead` 外可在 `umt.detail.page.labelSelectors` 加(憨憨:`div.font-bold.leading-6`、`span.font-bold`)。`page.probe` 记录页面上找到的字段标签,只存缓存不展示,站点改版时用来排查。馒头从 API 的 `descr`(BBCode)与 `mediainfo` 取同样的结构。
|
||||||
|
|
||||||
## Agent 能力(W10-pre2)
|
## Agent 能力(W10-pre2)
|
||||||
|
|
||||||
处理器在 `apps/server/src/modules/sources-identity/capabilities-sites.ts`,调用上面同一组 service。能力 ID 只允许小写 + 下划线(`CapabilityId` 正则),所以规格里的 `site.healthCheck` / `site.refreshLogin` / `site.userData` 落为下划线形式。
|
处理器在 `apps/server/src/modules/sources-identity/capabilities-sites.ts`,调用上面同一组 service。能力 ID 只允许小写 + 下划线(`CapabilityId` 正则),所以规格里的 `site.healthCheck` / `site.refreshLogin` / `site.userData` 落为下划线形式。
|
||||||
|
|||||||
@@ -258,6 +258,9 @@
|
|||||||
"imdb": "a[href*='imdb.com']::attr(href)",
|
"imdb": "a[href*='imdb.com']::attr(href)",
|
||||||
"douban": "a[href*='douban.com']::attr(href)",
|
"douban": "a[href*='douban.com']::attr(href)",
|
||||||
"fileList": "td.rowfollow > ul > li::text"
|
"fileList": "td.rowfollow > ul > li::text"
|
||||||
|
},
|
||||||
|
"page": {
|
||||||
|
"labelSelectors": ["div.font-bold.leading-6", "span.font-bold"]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"vipLevels": ["贵宾", "VIP"],
|
"vipLevels": ["贵宾", "VIP"],
|
||||||
|
|||||||
@@ -141,6 +141,16 @@ const UmtNexusSchema = z.strictObject({
|
|||||||
fileList: z.string().optional(),
|
fileList: z.string().optional(),
|
||||||
})
|
})
|
||||||
.optional(),
|
.optional(),
|
||||||
|
page: z
|
||||||
|
.strictObject({
|
||||||
|
labelSelectors: z.array(z.string()).optional(),
|
||||||
|
mediaInfo: z.string().optional(),
|
||||||
|
screenshots: z.string().optional(),
|
||||||
|
tags: z.string().optional(),
|
||||||
|
uploader: z.string().optional(),
|
||||||
|
files: z.string().optional(),
|
||||||
|
})
|
||||||
|
.optional(),
|
||||||
})
|
})
|
||||||
.optional(),
|
.optional(),
|
||||||
downloadPath: z.string().min(1),
|
downloadPath: z.string().min(1),
|
||||||
|
|||||||
@@ -0,0 +1,108 @@
|
|||||||
|
/**
|
||||||
|
* 详情页扩展内容的通用文本规则(NexusPHP HTML 与 M-Team BBCode 共用):MediaInfo 识别、截图过滤、TMDB 链接。
|
||||||
|
*/
|
||||||
|
import type { SiteDetailPage } from "./types.ts";
|
||||||
|
import { extractDoubanId, extractImdbId } from "./util.ts";
|
||||||
|
|
||||||
|
/** 表情 / 图标 / 旗帜 / 头像等小图,不算截图 */
|
||||||
|
const NOT_SCREENSHOT =
|
||||||
|
/(smilies|smiley|emoji|\/icons?\/|pic\/(?:trans|cattrans|flag)|\/flags?\/|avatar|\.svg(?:$|\?))/i;
|
||||||
|
|
||||||
|
export const MAX_SCREENSHOTS = 20;
|
||||||
|
export const MAX_FILES = 500;
|
||||||
|
|
||||||
|
/** MediaInfo / BDInfo 文本:有概览段落 + 视频段落,且至少 5 行「键 : 值」 */
|
||||||
|
export function looksLikeMediaInfo(text: string): boolean {
|
||||||
|
if (!/(^|\n)\s*(General|概览|常规|DISC INFO|Disc Title|Complete name)\b/i.test(text)) return false;
|
||||||
|
if (!/(^|\n)\s*(Video|视频)\b/i.test(text)) return false;
|
||||||
|
return (text.match(/^[^\n:：]{2,40}[ \t]*[:：][ \t]*\S/gm) ?? []).length >= 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function extractTmdb(text: string | null | undefined): SiteDetailPage["tmdb"] {
|
||||||
|
const m = text ? /themoviedb\.org\/(movie|tv)\/(\d+)/.exec(text) : null;
|
||||||
|
return m ? { type: m[1] as "movie" | "tv", id: m[2]! } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 图片地址转绝对 http(s) 地址并过滤非截图;无效返回 null */
|
||||||
|
export function screenshotUrl(src: string | null | undefined, base: string): string | null {
|
||||||
|
if (!src || src.startsWith("data:")) return null;
|
||||||
|
let u: URL;
|
||||||
|
try {
|
||||||
|
u = new URL(src.trim(), base);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (u.protocol !== "http:" && u.protocol !== "https:") return null;
|
||||||
|
if (NOT_SCREENSHOT.test(u.pathname)) return null;
|
||||||
|
return u.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 多行文本规整:去 \r、行尾空白,连续空行压成一行 */
|
||||||
|
export function tidyText(text: string): string {
|
||||||
|
return text
|
||||||
|
.replace(/\r/g, "")
|
||||||
|
.replace(/ /g, " ")
|
||||||
|
.split("\n")
|
||||||
|
.map((l) => l.replace(/\s+$/, ""))
|
||||||
|
.join("\n")
|
||||||
|
.replace(/\n{3,}/g, "\n\n")
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function emptyDetailPage(): SiteDetailPage {
|
||||||
|
return {
|
||||||
|
descriptionText: null,
|
||||||
|
mediaInfo: null,
|
||||||
|
screenshots: [],
|
||||||
|
imdbId: null,
|
||||||
|
doubanId: null,
|
||||||
|
tmdb: null,
|
||||||
|
files: [],
|
||||||
|
tags: [],
|
||||||
|
uploader: null,
|
||||||
|
probe: { labels: [], hasDescription: false, imageCount: 0 },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* M-Team 详情(BBCode 简介 + 独立 mediainfo 字段)→ 扩展内容。
|
||||||
|
* 截图取 [img] 标签;简介去掉 BBCode 标记与 MediaInfo 引用块。
|
||||||
|
*/
|
||||||
|
export function pageFromBbcode(
|
||||||
|
descr: string | null | undefined,
|
||||||
|
base: string,
|
||||||
|
extra: { mediaInfo?: string | null; imdbId?: string | null; doubanId?: string | null; tags?: string[] } = {},
|
||||||
|
): SiteDetailPage {
|
||||||
|
const page = emptyDetailPage();
|
||||||
|
const src = descr ?? "";
|
||||||
|
const imgs: string[] = [];
|
||||||
|
for (const m of src.matchAll(/\[img(?:=([^\]]+))?\]([^[]*)\[\/img\]|\[img=([^\]]+)\]/gi)) {
|
||||||
|
const u = screenshotUrl((m[2] || m[1] || m[3] || "").trim(), base);
|
||||||
|
if (u && !imgs.includes(u)) imgs.push(u);
|
||||||
|
}
|
||||||
|
page.probe.imageCount = imgs.length;
|
||||||
|
page.screenshots = imgs.slice(0, MAX_SCREENSHOTS);
|
||||||
|
let mediaInfo = extra.mediaInfo?.trim() || null;
|
||||||
|
let body = src.replace(/\[img(?:=[^\]]+)?\][^[]*\[\/img\]|\[img=[^\]]+\]/gi, "");
|
||||||
|
// [quote] / [code] / [mediainfo] 块里的 MediaInfo
|
||||||
|
body = body.replace(
|
||||||
|
/\[(quote|code|mediainfo|box|hide|spoiler)(=[^\]]*)?\]([\s\S]*?)\[\/\1\]/gi,
|
||||||
|
(all, _t, _a, inner) => {
|
||||||
|
const text = tidyText(String(inner).replace(/\[\/?[a-z*]+(?:=[^\]]*)?\]/gi, ""));
|
||||||
|
if (looksLikeMediaInfo(text)) {
|
||||||
|
mediaInfo ??= text;
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
return all;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const text = tidyText(body.replace(/\[\/?[a-z*]+(?:=[^\]]*)?\]/gi, ""));
|
||||||
|
page.probe.hasDescription = !!src.trim();
|
||||||
|
page.descriptionText = text ? text.slice(0, 20_000) : null;
|
||||||
|
page.mediaInfo = mediaInfo ? mediaInfo.slice(0, 50_000) : null;
|
||||||
|
page.imdbId = extra.imdbId ?? extractImdbId(src) ?? null;
|
||||||
|
page.doubanId = extra.doubanId ?? extractDoubanId(src) ?? null;
|
||||||
|
page.tmdb = extractTmdb(src);
|
||||||
|
page.tags = [...new Set(extra.tags ?? [])].slice(0, 20);
|
||||||
|
return page;
|
||||||
|
}
|
||||||
@@ -90,6 +90,8 @@ export {
|
|||||||
type SiteCapabilities,
|
type SiteCapabilities,
|
||||||
type SiteCategory,
|
type SiteCategory,
|
||||||
type SiteCredentialKind,
|
type SiteCredentialKind,
|
||||||
|
type SiteDetailFile,
|
||||||
|
type SiteDetailPage,
|
||||||
SiteError,
|
SiteError,
|
||||||
type SiteErrorKind,
|
type SiteErrorKind,
|
||||||
type SiteHealth,
|
type SiteHealth,
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
*/
|
*/
|
||||||
import { redactUrl } from "../../rss/redact.ts";
|
import { redactUrl } from "../../rss/redact.ts";
|
||||||
import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
|
import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
|
||||||
|
import { pageFromBbcode } from "../detail-text.ts";
|
||||||
import { readBody, siteFetch } from "../http.ts";
|
import { readBody, siteFetch } from "../http.ts";
|
||||||
import { healthFromError } from "../spider/adapter.ts";
|
import { healthFromError } from "../spider/adapter.ts";
|
||||||
import { promotionFromFactors } from "../spider/map.ts";
|
import { promotionFromFactors } from "../spider/map.ts";
|
||||||
@@ -154,6 +155,12 @@ export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps
|
|||||||
const [row] = parseMteamResult([data], def.domain);
|
const [row] = parseMteamResult([data], def.domain);
|
||||||
const r: SiteReleaseDetail = mteamTorrentToRelease(row!, data, def, http.now().toISOString());
|
const r: SiteReleaseDetail = mteamTorrentToRelease(row!, data, def, http.now().toISOString());
|
||||||
if (data.descr) r.descriptionText = String(data.descr).slice(0, 20_000);
|
if (data.descr) r.descriptionText = String(data.descr).slice(0, 20_000);
|
||||||
|
r.page = pageFromBbcode(data.descr, def.domain, {
|
||||||
|
mediaInfo: typeof data.mediainfo === "string" ? data.mediainfo : null,
|
||||||
|
imdbId: r.imdbId ?? null,
|
||||||
|
doubanId: r.doubanId ?? null,
|
||||||
|
tags: r.tags,
|
||||||
|
});
|
||||||
return r;
|
return r;
|
||||||
},
|
},
|
||||||
|
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ export interface MteamTorrentJson {
|
|||||||
labels?: string | number | null;
|
labels?: string | number | null;
|
||||||
labelsNew?: string[] | null;
|
labelsNew?: string[] | null;
|
||||||
descr?: string | null;
|
descr?: string | null;
|
||||||
|
mediainfo?: string | null;
|
||||||
status?: {
|
status?: {
|
||||||
seeders?: string | number | null;
|
seeders?: string | number | null;
|
||||||
leechers?: string | number | null;
|
leechers?: string | number | null;
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
import { HHANCLUB_DETAIL_PAGE_FULL } from "@umt/testkit";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { hhanclubDefinition } from "../definitions/index.ts";
|
||||||
|
import { looksLikeMediaInfo, pageFromBbcode } from "../detail-text.ts";
|
||||||
|
import { getSiteDefinition } from "../registry.ts";
|
||||||
|
import { parseDetailPage } from "./detail.ts";
|
||||||
|
import type { IndexerDefinition } from "./schema.ts";
|
||||||
|
|
||||||
|
const OBSERVED = "2026-09-29T00:00:00.000Z";
|
||||||
|
|
||||||
|
describe("详情页扩展内容(HHClub div 模板)", () => {
|
||||||
|
const { kind, detail } = parseDetailPage(HHANCLUB_DETAIL_PAGE_FULL, hhanclubDefinition, "91002", OBSERVED);
|
||||||
|
const page = detail!.page!;
|
||||||
|
|
||||||
|
it("外部 ID、上传者、标签", () => {
|
||||||
|
expect(kind).toBe("ok");
|
||||||
|
expect(page).toMatchObject({
|
||||||
|
imdbId: "tt14452776",
|
||||||
|
doubanId: "36000001",
|
||||||
|
tmdb: { type: "tv", id: "136315" },
|
||||||
|
uploader: "uploader01",
|
||||||
|
tags: ["官方", "中字", "完结"],
|
||||||
|
});
|
||||||
|
expect(detail!.imdbId).toBe("tt14452776");
|
||||||
|
expect(detail!.tags).toEqual(["官方", "中字", "完结"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("MediaInfo 从简介的代码块取出,简介里不再重复", () => {
|
||||||
|
expect(page.mediaInfo).toMatch(/^General\nUnique ID : 1234\n/);
|
||||||
|
expect(page.mediaInfo).toContain("Format : HEVC");
|
||||||
|
expect(page.descriptionText).toContain("◎译 名 示例剧集");
|
||||||
|
expect(page.descriptionText).not.toContain("Unique ID");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("截图:相对地址补全、优先 data-src、过滤表情与占位图", () => {
|
||||||
|
expect(page.screenshots).toEqual([
|
||||||
|
"https://hhanclub.net/attachments/202609/poster.jpg",
|
||||||
|
"https://img.example-host.com/i/shot1.png",
|
||||||
|
"https://img.example-host.com/i/shot2.jpg",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("文件清单:「名称 (大小)」", () => {
|
||||||
|
expect(page.files).toEqual([
|
||||||
|
{ name: "Example.Show.S01E01.mkv", sizeBytes: Math.round(1.04 * 1024 ** 3) },
|
||||||
|
{ name: "Example.Show.S01E02.mkv", sizeBytes: Math.round(1.02 * 1024 ** 3) },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("诊断:记录页面上的字段标签", () => {
|
||||||
|
expect(page.probe.labels).toEqual(expect.arrayContaining(["标题", "副标题", "标签", "上传者", "文件列表"]));
|
||||||
|
expect(page.probe.hasDescription).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("详情页扩展内容(标准 NexusPHP rowhead 表格)", () => {
|
||||||
|
const def = getSiteDefinition("audiences") as IndexerDefinition;
|
||||||
|
const html = `<html><body><a href="logout.php">退出</a><h1 id="top">Some.Movie.2024.1080p.BluRay.x264-GRP</h1>
|
||||||
|
<table>
|
||||||
|
<tr><td class="rowhead">副标题</td><td class="rowfollow">某电影</td></tr>
|
||||||
|
<tr><td class="rowhead">基本信息</td><td class="rowfollow"><b>大小：</b>8.00 GB <b>类型:</b> 电影</td></tr>
|
||||||
|
<tr><td class="rowhead">标签</td><td class="rowfollow"><span>中字</span><span>DIY</span></td></tr>
|
||||||
|
<tr><td class="rowhead">上传者</td><td class="rowfollow"><i>匿名</i></td></tr>
|
||||||
|
<tr><td class="rowhead">简介</td><td class="rowfollow"><div id="kdescr">一段简介<br>第二行
|
||||||
|
<div class="codemain">DISC INFO:<br>Disc Title: X<br>Size: 1<br>Protection: AACS<br>Playlist: 00800.MPLS<br>Length: 2:00:00<br>Video: MPEG-4 AVC<br>Total Bitrate: 30 Mbps</div>
|
||||||
|
<img src="https://img.example-host.com/a.png"></div></td></tr>
|
||||||
|
</table>
|
||||||
|
<table id="filelist"><tr><td>Some.Movie.mkv</td><td>8.00 GB</td></tr></table></body></html>`;
|
||||||
|
|
||||||
|
it("rowhead 标签、BDInfo、#filelist 表格", () => {
|
||||||
|
const { detail } = parseDetailPage(html, def, "1", OBSERVED);
|
||||||
|
const page = detail!.page!;
|
||||||
|
expect(page.tags).toEqual(["中字", "DIY"]);
|
||||||
|
expect(page.uploader).toBe("匿名");
|
||||||
|
expect(page.mediaInfo).toContain("Disc Title: X");
|
||||||
|
expect(page.descriptionText).toBe("一段简介\n第二行");
|
||||||
|
expect(page.screenshots).toEqual(["https://img.example-host.com/a.png"]);
|
||||||
|
expect(page.files).toEqual([{ name: "Some.Movie.mkv", sizeBytes: 8 * 1024 ** 3 }]);
|
||||||
|
expect(page.imdbId).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("M-Team BBCode 简介", () => {
|
||||||
|
it("截图取 [img],MediaInfo 取引用块,外部 ID 从正文取", () => {
|
||||||
|
const descr = `[img]https://img.example-host.com/p.jpg[/img]
|
||||||
|
◎简介 这是简介 https://www.imdb.com/title/tt15239678/
|
||||||
|
[quote]General
|
||||||
|
Unique ID : 1
|
||||||
|
Complete name : a.mkv
|
||||||
|
Format : Matroska
|
||||||
|
File size : 1 GiB
|
||||||
|
|
||||||
|
Video
|
||||||
|
Format : AVC[/quote]
|
||||||
|
[img=https://img.example-host.com/s.png]`;
|
||||||
|
const page = pageFromBbcode(descr, "https://kp.m-team.cc/", { tags: ["中字"] });
|
||||||
|
expect(page.screenshots).toEqual(["https://img.example-host.com/p.jpg", "https://img.example-host.com/s.png"]);
|
||||||
|
expect(page.mediaInfo).toMatch(/^General\n/);
|
||||||
|
expect(page.descriptionText).toBe("◎简介 这是简介 https://www.imdb.com/title/tt15239678/");
|
||||||
|
expect(page.imdbId).toBe("tt15239678");
|
||||||
|
expect(page.tags).toEqual(["中字"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("MediaInfo 识别不误判普通简介", () => {
|
||||||
|
expect(looksLikeMediaInfo("General\n一段文字\nVideo")).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
/**
|
||||||
|
* 详情页扩展内容(简介 / MediaInfo / 截图 / IMDb·豆瓣·TMDB / 文件清单 / 标签 / 上传者)—— 本项目扩展。
|
||||||
|
* 以 MovieClaw detail_*_css(description / imdb / douban / file_list)为起点,其余按 NexusPHP details.php 公开结构
|
||||||
|
* 做通用规则:字段标签(td.rowhead 或站点配置的 div 标签)的下一个兄弟元素是值。站点特化写在定义 `umt.detail.page`。
|
||||||
|
* 文件清单只取页面上已有的内容,不为它额外请求 viewfilelist.php(请求克制)。
|
||||||
|
*/
|
||||||
|
import type { CheerioAPI } from "cheerio";
|
||||||
|
import {
|
||||||
|
emptyDetailPage,
|
||||||
|
extractTmdb,
|
||||||
|
looksLikeMediaInfo,
|
||||||
|
MAX_FILES,
|
||||||
|
MAX_SCREENSHOTS,
|
||||||
|
screenshotUrl,
|
||||||
|
tidyText,
|
||||||
|
} from "../detail-text.ts";
|
||||||
|
import type { SiteDetailFile, SiteDetailPage } from "../types.ts";
|
||||||
|
import { extractDoubanId, extractImdbId } from "../util.ts";
|
||||||
|
import { mcSelect } from "./mc-selector.ts";
|
||||||
|
import type { IndexerDefinition } from "./schema.ts";
|
||||||
|
import { parseSizeMp } from "./spider.ts";
|
||||||
|
|
||||||
|
type DetailConf = NonNullable<IndexerDefinition["umt"]["detail"]>;
|
||||||
|
type El = ReturnType<CheerioAPI>;
|
||||||
|
|
||||||
|
const LABELS = {
|
||||||
|
description: ["简介", "Description"],
|
||||||
|
mediaInfo: ["MediaInfo", "媒体信息", "BDInfo"],
|
||||||
|
tags: ["标签", "Tags"],
|
||||||
|
uploader: ["上传者", "发布者", "发布人", "上传人", "Uploader"],
|
||||||
|
files: ["文件列表", "文件清单", "File list", "Files"],
|
||||||
|
};
|
||||||
|
|
||||||
|
/** 简介里可能装 MediaInfo 的块 */
|
||||||
|
const MI_BLOCKS = "fieldset, div.codemain, pre, code, blockquote, .spoiler-content, .hide, div.nexus-media-info-raw";
|
||||||
|
|
||||||
|
const normLabel = (t: string) =>
|
||||||
|
t
|
||||||
|
.replace(/\s+/g, "")
|
||||||
|
.replace(/[:：]+$/, "")
|
||||||
|
.toLowerCase();
|
||||||
|
|
||||||
|
/** 保留换行的文本:<br> 与块级元素换行 */
|
||||||
|
function blockText($: CheerioAPI, el: El): string {
|
||||||
|
const c = el.clone();
|
||||||
|
c.find("script, style").remove();
|
||||||
|
c.find("br").replaceWith("\n");
|
||||||
|
c.find("div, p, li, tr, pre, fieldset, blockquote, h1, h2, h3, h4, h5, h6").each((_, n) => {
|
||||||
|
$(n).append("\n");
|
||||||
|
});
|
||||||
|
return tidyText(c.text());
|
||||||
|
}
|
||||||
|
|
||||||
|
const SIZE_TAIL = /^(.*?)\s*[(（[]\s*([\d.,]+\s*[KMGTP]?i?B)\s*[)）\]]\s*$/i;
|
||||||
|
const HAS_EXT = /\.[A-Za-z0-9]{2,5}$/;
|
||||||
|
|
||||||
|
function fileFromText(text: string): SiteDetailFile | null {
|
||||||
|
const t = text.replace(/\s+/g, " ").trim();
|
||||||
|
if (!t) return null;
|
||||||
|
const m = SIZE_TAIL.exec(t);
|
||||||
|
if (m?.[1]) return { name: m[1], sizeBytes: parseSizeMp(m[2]!) || null };
|
||||||
|
return HAS_EXT.test(t) ? { name: t, sizeBytes: null } : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function filesFrom($: CheerioAPI, rows: El): SiteDetailFile[] {
|
||||||
|
const out: SiteDetailFile[] = [];
|
||||||
|
rows.each((_, n) => {
|
||||||
|
if (out.length >= MAX_FILES) return;
|
||||||
|
const row = $(n);
|
||||||
|
const cells = row.children("td");
|
||||||
|
if (cells.length >= 2) {
|
||||||
|
const name = cells.first().text().replace(/\s+/g, " ").trim();
|
||||||
|
const size = cells.eq(1).text().trim();
|
||||||
|
if (name && HAS_EXT.test(name) && /\d/.test(size)) out.push({ name, sizeBytes: parseSizeMp(size) || null });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const f = fileFromText(row.text());
|
||||||
|
if (f) out.push(f);
|
||||||
|
});
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string): SiteDetailPage {
|
||||||
|
const page = emptyDetailPage();
|
||||||
|
const p = conf.page ?? {};
|
||||||
|
|
||||||
|
// 字段标签 → 值元素
|
||||||
|
const pairs: { label: string; value: El }[] = [];
|
||||||
|
const addLabels = (sel: string, valueFor: (e: El) => El) => {
|
||||||
|
$(sel).each((_, n) => {
|
||||||
|
const e = $(n);
|
||||||
|
const label = normLabel(e.text());
|
||||||
|
if (!label || label.length > 16) return;
|
||||||
|
const value = valueFor(e);
|
||||||
|
if (value.length) pairs.push({ label, value });
|
||||||
|
});
|
||||||
|
};
|
||||||
|
addLabels(conf.rowLabel, (e) => e.nextAll(conf.rowValue).first());
|
||||||
|
for (const sel of p.labelSelectors ?? []) addLabels(sel, (e) => e.next());
|
||||||
|
const byLabel = (names: string[]): El | null =>
|
||||||
|
pairs.find((x) => names.some((n) => x.label.startsWith(normLabel(n))))?.value ?? null;
|
||||||
|
page.probe.labels = [...new Set(pairs.map((x) => x.label))].slice(0, 40);
|
||||||
|
|
||||||
|
// 简介容器
|
||||||
|
const descSel = conf.fields?.description ?? conf.description;
|
||||||
|
let desc: El | null = descSel ? $(descSel).first() : null;
|
||||||
|
if (!desc?.length) desc = byLabel(LABELS.description);
|
||||||
|
page.probe.hasDescription = !!desc?.length;
|
||||||
|
|
||||||
|
// MediaInfo:站点选择器 → 字段标签 → 简介里的引用 / 代码块(取最内层文本,整块从简介里去掉)
|
||||||
|
let mediaInfoEl: El | null = null;
|
||||||
|
let miBlockIndex = -1;
|
||||||
|
if (p.mediaInfo) {
|
||||||
|
const el = $(p.mediaInfo).first();
|
||||||
|
if (el.length) mediaInfoEl = el;
|
||||||
|
}
|
||||||
|
mediaInfoEl ??= byLabel(LABELS.mediaInfo);
|
||||||
|
const blocks = desc?.length ? desc.find(MI_BLOCKS) : null;
|
||||||
|
if (!mediaInfoEl && blocks) {
|
||||||
|
blocks.each((i, n) => {
|
||||||
|
if (miBlockIndex >= 0) return;
|
||||||
|
if (looksLikeMediaInfo(blockText($, $(n)))) miBlockIndex = i;
|
||||||
|
});
|
||||||
|
if (miBlockIndex >= 0) {
|
||||||
|
const outer = blocks.eq(miBlockIndex);
|
||||||
|
const inner = outer
|
||||||
|
.find(MI_BLOCKS)
|
||||||
|
.filter((_, n) => looksLikeMediaInfo(blockText($, $(n))))
|
||||||
|
.last();
|
||||||
|
mediaInfoEl = inner.length ? inner : outer;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (mediaInfoEl) {
|
||||||
|
const t = blockText($, mediaInfoEl);
|
||||||
|
if (t) page.mediaInfo = t.slice(0, 50_000);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 截图
|
||||||
|
const imgs: string[] = [];
|
||||||
|
const imgEls = p.screenshots ? $(p.screenshots) : desc?.length ? desc.find("img") : null;
|
||||||
|
imgEls?.each((_, n) => {
|
||||||
|
const e = $(n);
|
||||||
|
const u = screenshotUrl(e.attr("data-src") ?? e.attr("data-original") ?? e.attr("src"), base);
|
||||||
|
if (u && !imgs.includes(u)) imgs.push(u);
|
||||||
|
});
|
||||||
|
page.probe.imageCount = imgs.length;
|
||||||
|
page.screenshots = imgs.slice(0, MAX_SCREENSHOTS);
|
||||||
|
|
||||||
|
// 简介纯文本:去掉 MediaInfo 块与图片
|
||||||
|
if (desc?.length) {
|
||||||
|
const c = desc.clone();
|
||||||
|
if (miBlockIndex >= 0) c.find(MI_BLOCKS).eq(miBlockIndex).remove();
|
||||||
|
c.find("img").remove();
|
||||||
|
const t = blockText($, c);
|
||||||
|
if (t) page.descriptionText = t.slice(0, 20_000);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 外部 ID:页面上的链接优先,其次简介正文
|
||||||
|
const descHtml = desc?.html() ?? "";
|
||||||
|
const f = conf.fields ?? {};
|
||||||
|
page.imdbId =
|
||||||
|
extractImdbId(mcSelect($, null, f.imdb ?? "a[href*='imdb.com']::attr(href)")) ?? extractImdbId(descHtml) ?? null;
|
||||||
|
page.doubanId =
|
||||||
|
extractDoubanId(mcSelect($, null, f.douban ?? "a[href*='douban.com']::attr(href)")) ??
|
||||||
|
extractDoubanId(descHtml) ??
|
||||||
|
null;
|
||||||
|
page.tmdb = extractTmdb($("a[href*='themoviedb.org']").first().attr("href")) ?? extractTmdb(descHtml);
|
||||||
|
|
||||||
|
// 文件清单:站点选择器 → MovieClaw file_list(去伪元素)→ 字段标签 → #filelist 表格
|
||||||
|
const fileSel = p.files ?? f.fileList?.replace(/::text$/, "");
|
||||||
|
let files = fileSel ? filesFrom($, $(fileSel)) : [];
|
||||||
|
if (!files.length) {
|
||||||
|
const v = byLabel(LABELS.files);
|
||||||
|
if (v) files = filesFrom($, v.find("li, tr"));
|
||||||
|
}
|
||||||
|
if (!files.length) files = filesFrom($, $("#filelist tr"));
|
||||||
|
page.files = files;
|
||||||
|
|
||||||
|
// 标签
|
||||||
|
const tagEl = p.tags ? $(p.tags) : byLabel(LABELS.tags);
|
||||||
|
if (tagEl?.length) {
|
||||||
|
const parts = tagEl.find("span, a").length
|
||||||
|
? tagEl
|
||||||
|
.find("span, a")
|
||||||
|
.map((_, n) => $(n).text().trim())
|
||||||
|
.get()
|
||||||
|
: tagEl.text().split(/[\s,，|/]+/);
|
||||||
|
page.tags = [...new Set(parts.map((t) => t.trim()).filter((t) => t && t.length <= 20))].slice(0, 20);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 上传者
|
||||||
|
const upEl = p.uploader ? $(p.uploader).first() : byLabel(LABELS.uploader);
|
||||||
|
if (upEl?.length) {
|
||||||
|
const a = upEl.is("a") ? upEl : upEl.find("a[href*='userdetails.php']").first();
|
||||||
|
const name = (a.length ? a.text() : upEl.text()).replace(/\s+/g, " ").trim();
|
||||||
|
if (name) page.uploader = name.slice(0, 64);
|
||||||
|
}
|
||||||
|
if (!page.uploader) {
|
||||||
|
for (const x of pairs) {
|
||||||
|
const m = /(?:上传者|发布者|发布人|Uploader)\s*[:：]\s*(\S+)/i.exec(x.value.text());
|
||||||
|
if (m?.[1]) {
|
||||||
|
page.uploader = m[1].slice(0, 64);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return page;
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@
|
|||||||
import type { CheerioAPI } from "cheerio";
|
import type { CheerioAPI } from "cheerio";
|
||||||
import type { SiteReleaseDetail } from "../types.ts";
|
import type { SiteReleaseDetail } from "../types.ts";
|
||||||
import { extractDoubanId, extractImdbId, parseCount, parseSiteTime } from "../util.ts";
|
import { extractDoubanId, extractImdbId, parseCount, parseSiteTime } from "../util.ts";
|
||||||
|
import { extractDetailPage } from "./detail-page.ts";
|
||||||
import { categoryName, promotionFromFactors } from "./map.ts";
|
import { categoryName, promotionFromFactors } from "./map.ts";
|
||||||
import { mcSelect } from "./mc-selector.ts";
|
import { mcSelect } from "./mc-selector.ts";
|
||||||
import type { IndexerDefinition } from "./schema.ts";
|
import type { IndexerDefinition } from "./schema.ts";
|
||||||
@@ -115,5 +116,10 @@ export function parseDetailPage(
|
|||||||
if (imdb) detail.imdbId = imdb;
|
if (imdb) detail.imdbId = imdb;
|
||||||
const douban = extractDoubanId(mcSelect($, null, f.douban)) ?? extractDoubanId(descHtml);
|
const douban = extractDoubanId(mcSelect($, null, f.douban)) ?? extractDoubanId(descHtml);
|
||||||
if (douban) detail.doubanId = douban;
|
if (douban) detail.doubanId = douban;
|
||||||
|
const page = extractDetailPage($, conf, def.domain);
|
||||||
|
page.imdbId ??= detail.imdbId ?? null;
|
||||||
|
page.doubanId ??= detail.doubanId ?? null;
|
||||||
|
if (page.tags.length) detail.tags = page.tags;
|
||||||
|
detail.page = page;
|
||||||
return { kind: "ok", detail };
|
return { kind: "ok", detail };
|
||||||
}
|
}
|
||||||
@@ -198,6 +198,20 @@ export interface UmtSiteExtension {
|
|||||||
fields?: Partial<
|
fields?: Partial<
|
||||||
Record<"title" | "subtitle" | "size" | "description" | "download" | "imdb" | "douban" | "fileList", string>
|
Record<"title" | "subtitle" | "size" | "description" | "download" | "imdb" | "douban" | "fileList", string>
|
||||||
>;
|
>;
|
||||||
|
/**
|
||||||
|
* 详情页扩展内容(简介 / MediaInfo / 截图 / 文件 / 标签 / 上传者)的站点特化,均可省略(走通用规则,见 detail-page.ts)。
|
||||||
|
* 选择器为普通 CSS。
|
||||||
|
*/
|
||||||
|
page?: {
|
||||||
|
/** 除 rowLabel 外的字段标签元素(div 版详情页:标签元素的下一个兄弟元素是值) */
|
||||||
|
labelSelectors?: string[];
|
||||||
|
mediaInfo?: string;
|
||||||
|
screenshots?: string;
|
||||||
|
tags?: string;
|
||||||
|
uploader?: string;
|
||||||
|
/** 文件清单行(每行一个文件,文本形如「名称 (大小)」或两列表格) */
|
||||||
|
files?: string;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
/** 下载路径(按资源 ID 拼接) */
|
/** 下载路径(按资源 ID 拼接) */
|
||||||
downloadPath: string;
|
downloadPath: string;
|
||||||
|
|||||||
@@ -60,8 +60,34 @@ export interface SiteRelease {
|
|||||||
observedAt: string;
|
observedAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 详情页文件清单一项;大小未知为 null */
|
||||||
|
export interface SiteDetailFile {
|
||||||
|
name: string;
|
||||||
|
sizeBytes: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 详情页扩展内容(只在按需抓详情时得到,服务端放 kv_cache,不进 source_release 列)。
|
||||||
|
* 截图是站点页面上的原始图片地址,展示时经服务端图片代理,不让浏览器直连外站。
|
||||||
|
*/
|
||||||
|
export interface SiteDetailPage {
|
||||||
|
/** 简介纯文本(已去掉 MediaInfo 块) */
|
||||||
|
descriptionText: string | null;
|
||||||
|
mediaInfo: string | null;
|
||||||
|
screenshots: string[];
|
||||||
|
imdbId: string | null;
|
||||||
|
doubanId: string | null;
|
||||||
|
tmdb: { type: "movie" | "tv"; id: string } | null;
|
||||||
|
files: SiteDetailFile[];
|
||||||
|
tags: string[];
|
||||||
|
uploader: string | null;
|
||||||
|
/** 解析诊断:页面上找到的字段标签、是否有简介容器、图片数(用于站点改版排查,不展示) */
|
||||||
|
probe: { labels: string[]; hasDescription: boolean; imageCount: number };
|
||||||
|
}
|
||||||
|
|
||||||
export interface SiteReleaseDetail extends SiteRelease {
|
export interface SiteReleaseDetail extends SiteRelease {
|
||||||
descriptionText?: string;
|
descriptionText?: string;
|
||||||
|
page?: SiteDetailPage;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SiteListFilters {
|
export interface SiteListFilters {
|
||||||
|
|||||||
@@ -348,6 +348,16 @@ export class TmdbClient {
|
|||||||
return [...m, ...t].sort((a, b) => (b.popularity ?? 0) - (a.popularity ?? 0));
|
return [...m, ...t].sort((a, b) => (b.popularity ?? 0) - (a.popularity ?? 0));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** /find/{imdb_id}?external_source=imdb_id:IMDb ID → 电影 / 剧集候选(站点详情页给出的 IMDb 链接走这里) */
|
||||||
|
async findByImdb(imdbId: string): Promise<MetadataCandidate[]> {
|
||||||
|
if (!/^tt\d{5,10}$/.test(imdbId)) return [];
|
||||||
|
const lang = this.langs[0]!;
|
||||||
|
const p = await this.get<Json>(`/find/${imdbId}`, { external_source: "imdb_id", language: lang });
|
||||||
|
const movies = (Array.isArray(p.movie_results) ? p.movie_results : []) as Json[];
|
||||||
|
const tvs = (Array.isArray(p.tv_results) ? p.tv_results : []) as Json[];
|
||||||
|
return [...movies.map((r) => this.fromBrief("movie", r, lang)), ...tvs.map((r) => this.fromBrief("tv", r, lang))];
|
||||||
|
}
|
||||||
|
|
||||||
/** 详情:多语言标题、别名(alternative_titles)、外部 ID、季列表(剧集)。不存在抛 notfound。 */
|
/** 详情:多语言标题、别名(alternative_titles)、外部 ID、季列表(剧集)。不存在抛 notfound。 */
|
||||||
async details(type: MediaEntityType, id: string | number): Promise<MetadataCandidate> {
|
async details(type: MediaEntityType, id: string | number): Promise<MetadataCandidate> {
|
||||||
const [first, ...rest] = await Promise.all(
|
const [first, ...rest] = await Promise.all(
|
||||||
|
|||||||
@@ -636,6 +636,24 @@ export const WorkbenchReleasePage = z.object({
|
|||||||
});
|
});
|
||||||
export type WorkbenchReleasePage = z.infer<typeof WorkbenchReleasePage>;
|
export type WorkbenchReleasePage = z.infer<typeof WorkbenchReleasePage>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 站点详情页扩展内容(按需抓详情时得到,服务端缓存 24 小时;过期后需再次复核详情)。
|
||||||
|
* 截图地址是服务端图片代理路径(/api/v1/img/site-detail/{releaseId}/{index}),不直连外站。
|
||||||
|
*/
|
||||||
|
export const SiteReleasePage = z.object({
|
||||||
|
fetchedAt: Timestamp,
|
||||||
|
descriptionText: z.string().nullable(),
|
||||||
|
mediaInfo: z.string().nullable(),
|
||||||
|
screenshots: z.array(z.object({ url: z.string(), host: z.string() })),
|
||||||
|
imdbId: z.string().nullable(),
|
||||||
|
doubanId: z.string().nullable(),
|
||||||
|
tmdb: z.object({ type: z.enum(["movie", "tv"]), id: z.string() }).nullable(),
|
||||||
|
files: z.array(z.object({ name: z.string(), sizeBytes: z.number().int().nullable() })),
|
||||||
|
tags: z.array(z.string()),
|
||||||
|
uploader: z.string().nullable(),
|
||||||
|
});
|
||||||
|
export type SiteReleasePage = z.infer<typeof SiteReleasePage>;
|
||||||
|
|
||||||
export const WorkbenchReleaseDetail = z.object({
|
export const WorkbenchReleaseDetail = z.object({
|
||||||
release: WorkbenchRelease,
|
release: WorkbenchRelease,
|
||||||
observations: z.array(
|
observations: z.array(
|
||||||
@@ -657,5 +675,9 @@ export const WorkbenchReleaseDetail = z.object({
|
|||||||
}),
|
}),
|
||||||
/** 刷新详情失败时保留既有数据并给出原因 */
|
/** 刷新详情失败时保留既有数据并给出原因 */
|
||||||
refreshError: z.string().nullable(),
|
refreshError: z.string().nullable(),
|
||||||
|
/** 详情页扩展内容;没抓过、缓存已过期或站点不支持详情页时为 null */
|
||||||
|
page: SiteReleasePage.nullable(),
|
||||||
|
/** 站点能否抓详情页(false 时前端不自动抓) */
|
||||||
|
detailSupported: z.boolean(),
|
||||||
});
|
});
|
||||||
export type WorkbenchReleaseDetail = z.infer<typeof WorkbenchReleaseDetail>;
|
export type WorkbenchReleaseDetail = z.infer<typeof WorkbenchReleaseDetail>;
|
||||||
@@ -80,3 +80,31 @@ export const HHANCLUB_LIST_PAGE_2026 = `<!DOCTYPE html><html><head><title>HHCLUB
|
|||||||
</div>
|
</div>
|
||||||
<div class="pagination"><a href="?page=0">1-100</a><a href="?page=1">101-200</a></div>
|
<div class="pagination"><a href="?page=0">1-100</a><a href="?page=1">101-200</a></div>
|
||||||
</body></html>`;
|
</body></html>`;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 详情页扩展内容(简介 / MediaInfo / 截图 / 外部链接 / 标签 / 上传者 / 文件清单)fixture:
|
||||||
|
* 结构按 HHClub div 模板(「标签」div + 相邻值 div)与 NexusPHP #kdescr 简介编写,数值均为示例。
|
||||||
|
*/
|
||||||
|
export const HHANCLUB_DETAIL_PAGE_FULL = `<!DOCTYPE html><html><head><title>HHCLUB :: 种子详情</title></head><body>
|
||||||
|
<div id="user-info-panel"><a href="userdetails.php?id=7" class="VIP_Name">bob</a> <a href="logout.php">退出</a></div>
|
||||||
|
<div class="grid">
|
||||||
|
<div class="font-bold leading-6">标题</div><div class="font-bold leading-6">Example.Show.S01.2026.2160p.WEB-DL.H265.AAC-HHWEB</div>
|
||||||
|
<div class="font-bold leading-6">副标题</div><div class="font-bold leading-6">示例剧集 | 全12集 | 4K</div>
|
||||||
|
<div><span class="font-bold">大小：</span><span>12.50 GB</span></div>
|
||||||
|
<div class="font-bold leading-6">标签</div><div class="font-bold leading-6"><span class="tag">官方</span><span class="tag">中字</span><span class="tag">完结</span></div>
|
||||||
|
<div class="font-bold leading-6">上传者</div><div class="font-bold leading-6"><a href="userdetails.php?id=321">uploader01</a></div>
|
||||||
|
<div><a href="download.php?id=91002">下载种子</a></div>
|
||||||
|
<div class="font-bold leading-6">简介</div>
|
||||||
|
<div id="kdescr">
|
||||||
|
<img src="attachments/202609/poster.jpg" alt="poster"><br>
|
||||||
|
◎译 名 示例剧集<br>◎年 代 2026<br>
|
||||||
|
◎IMDb链接 <a href="https://www.imdb.com/title/tt14452776/">https://www.imdb.com/title/tt14452776/</a><br>
|
||||||
|
◎豆瓣链接 <a href="https://movie.douban.com/subject/36000001/">https://movie.douban.com/subject/36000001/</a><br>
|
||||||
|
◎TMDB <a href="https://www.themoviedb.org/tv/136315">themoviedb</a><br>
|
||||||
|
<img src="pic/smilies/1.gif">
|
||||||
|
<fieldset><legend>MediaInfo</legend><div class="codemain">General<br>Unique ID : 1234<br>Complete name : Example.Show.S01E01.mkv<br>Format : Matroska<br>File size : 1.04 GiB<br>Duration : 45 min<br><br>Video<br>ID : 1<br>Format : HEVC<br>Width : 3 840 pixels</div></fieldset>
|
||||||
|
<img src="https://img.example-host.com/i/shot1.png"><img data-src="https://img.example-host.com/i/shot2.jpg" src="pic/trans.gif">
|
||||||
|
</div>
|
||||||
|
<div class="font-bold leading-6">文件列表</div>
|
||||||
|
<div class="font-bold leading-6"><ul><li>Example.Show.S01E01.mkv (1.04 GB)</li><li>Example.Show.S01E02.mkv (1.02 GB)</li></ul></div>
|
||||||
|
</div></body></html>`;
|
||||||
@@ -144,7 +144,7 @@ function detailBody(r: MockNexusRelease): string {
|
|||||||
<tr><td class="rowhead">副标题</td><td class="rowfollow" align="left">${esc(r.subtitle ?? "")}</td></tr>
|
<tr><td class="rowhead">副标题</td><td class="rowfollow" align="left">${esc(r.subtitle ?? "")}</td></tr>
|
||||||
<tr><td class="rowhead">基本信息</td><td class="rowfollow" align="left"><b>大小：</b>${esc(r.sizeText)} <b>类型:</b> ${esc(r.catName)} 发布于 <span title="${r.uploadedAt}">${esc(r.relativeTime ?? r.uploadedAt)}</span></td></tr>
|
<tr><td class="rowhead">基本信息</td><td class="rowfollow" align="left"><b>大小：</b>${esc(r.sizeText)} <b>类型:</b> ${esc(r.catName)} 发布于 <span title="${r.uploadedAt}">${esc(r.relativeTime ?? r.uploadedAt)}</span></td></tr>
|
||||||
<tr><td class="rowhead">同伴</td><td class="rowfollow" align="left">${r.seeders.replace(/,/g, "")}个做种者 | ${r.leechers.replace(/,/g, "")}个下载者</td></tr>
|
<tr><td class="rowhead">同伴</td><td class="rowfollow" align="left">${r.seeders.replace(/,/g, "")}个做种者 | ${r.leechers.replace(/,/g, "")}个下载者</td></tr>
|
||||||
<tr><td class="rowhead">简介</td><td class="rowfollow" align="left"><div id="kdescr">${esc(r.description ?? "暂无简介")}</div></td></tr>
|
<tr><td class="rowhead">简介</td><td class="rowfollow" align="left"><div id="kdescr">${r.descriptionHtml ?? esc(r.description ?? "暂无简介")}</div></td></tr>
|
||||||
</table>`;
|
</table>`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ export interface MockNexusRelease {
|
|||||||
hr?: boolean;
|
hr?: boolean;
|
||||||
uploader?: string;
|
uploader?: string;
|
||||||
description?: string;
|
description?: string;
|
||||||
|
/** 简介原样 HTML(优先于 description;用于详情页扩展内容测试) */
|
||||||
|
descriptionHtml?: string;
|
||||||
/** 下载返回磁力而不是 .torrent */
|
/** 下载返回磁力而不是 .torrent */
|
||||||
magnet?: string;
|
magnet?: string;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ const notFound: MockReply = {
|
|||||||
json: { success: false, status_code: 34, status_message: "The resource you requested could not be found." },
|
json: { success: false, status_code: 34, status_message: "The resource you requested could not be found." },
|
||||||
};
|
};
|
||||||
|
|
||||||
/** mock TMDB v3:/3/search/{movie,tv}、/3/{movie,tv}/:id(append_to_response)、/3/tv/:id/season/:n */
|
/** mock TMDB v3:/3/search/{movie,tv}、/3/find/:imdbId、/3/{movie,tv}/:id(append_to_response)、/3/tv/:id/season/:n */
|
||||||
export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTmdb> {
|
export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTmdb> {
|
||||||
const fx = opts.fixtures ?? TMDB_FIXTURES;
|
const fx = opts.fixtures ?? TMDB_FIXTURES;
|
||||||
const apiKey = opts.apiKey ?? "test-tmdb-key";
|
const apiKey = opts.apiKey ?? "test-tmdb-key";
|
||||||
@@ -84,6 +84,23 @@ export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTm
|
|||||||
return { json: { page: 1, results, total_pages: 1, total_results: results.length } };
|
return { json: { page: 1, results, total_pages: 1, total_results: results.length } };
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
[
|
||||||
|
/^\/3\/find\/(tt\d+)$/,
|
||||||
|
(m, q) => {
|
||||||
|
if (q.external_source !== "imdb_id") return { json: { movie_results: [], tv_results: [] } };
|
||||||
|
const hit = fx.filter((f) => f.external_ids.imdb_id === m[1]);
|
||||||
|
return {
|
||||||
|
json: {
|
||||||
|
movie_results: hit
|
||||||
|
.filter((f) => f.type === "movie")
|
||||||
|
.map((f) => ({ ...brief(f, q.language), media_type: "movie" })),
|
||||||
|
tv_results: hit
|
||||||
|
.filter((f) => f.type === "tv")
|
||||||
|
.map((f) => ({ ...brief(f, q.language), media_type: "tv" })),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
],
|
||||||
[
|
[
|
||||||
/^\/3\/tv\/(\d+)\/season\/(\d+)$/,
|
/^\/3\/tv\/(\d+)\/season\/(\d+)$/,
|
||||||
(m, q) => {
|
(m, q) => {
|
||||||
|
|||||||
@@ -26,6 +26,11 @@
|
|||||||
"userDataIntervalSec": 21600,
|
"userDataIntervalSec": 21600,
|
||||||
"searchConcurrency": 1,
|
"searchConcurrency": 1,
|
||||||
"detailCacheSec": 86400
|
"detailCacheSec": 86400
|
||||||
|
},
|
||||||
|
"detail": {
|
||||||
|
"page": {
|
||||||
|
"labelSelectors": ["div.font-bold.leading-6", "span.font-bold"]
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"torrents": {
|
"torrents": {
|
||||||
|
|||||||
Reference in new issue
Block a user