feat(sites): on-demand site detail page content (INT-2)
- SiteReleaseDetail.page: description, MediaInfo/BDInfo, screenshots,
IMDb / Douban / TMDB ids, file list, tags, uploader, plus a parse probe
(labels found) for layout-change diagnosis. Generic NexusPHP rules
(field label -> next sibling) starting from the MovieClaw detail
selectors; site overrides under umt.detail.page (HHClub: div labels).
M-Team builds the same structure from descr BBCode + mediainfo.
- Fetched only on demand (drawer open without cache, "复核详情",
release.inspect); content in kv_cache region site.detail for the
site detail cache period (>= 24 h); within the window the site is not
requested again. External ids persist in source_release.raw._detail,
which list snapshots do not overwrite.
- Screenshots only as proxy paths /api/v1/img/site-detail/{rid}/{i}:
public-address check on every hop, no site credentials, magic-byte
image check, 8 MB cap. Poster proxy shares the same helper.
- Identity: site ids count as explicit-id evidence; when TMDB is
queried, a TMDB id goes straight to details and an IMDb id to
/find (no title search on a hit). Douban only for local recall.
- Release drawer "站点详情页" section; fixtures and tests for HHClub div
layout, rowhead layout, BBCode, cache/proxy/identity end to end.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
6e9e8efec2
commit
04cffcd1f1
30 files changed
+1133
-64
No files matched your search
@@ -258,6 +258,9 @@
|
||||
"imdb": "a[href*='imdb.com']::attr(href)",
|
||||
"douban": "a[href*='douban.com']::attr(href)",
|
||||
"fileList": "td.rowfollow > ul > li::text"
|
||||
},
|
||||
"page": {
|
||||
"labelSelectors": ["div.font-bold.leading-6", "span.font-bold"]
|
||||
}
|
||||
},
|
||||
"vipLevels": ["贵宾", "VIP"],
|
||||
|
||||
@@ -141,6 +141,16 @@ const UmtNexusSchema = z.strictObject({
|
||||
fileList: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
page: z
|
||||
.strictObject({
|
||||
labelSelectors: z.array(z.string()).optional(),
|
||||
mediaInfo: z.string().optional(),
|
||||
screenshots: z.string().optional(),
|
||||
tags: z.string().optional(),
|
||||
uploader: z.string().optional(),
|
||||
files: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
})
|
||||
.optional(),
|
||||
downloadPath: z.string().min(1),
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* 详情页扩展内容的通用文本规则(NexusPHP HTML 与 M-Team BBCode 共用):MediaInfo 识别、截图过滤、TMDB 链接。
|
||||
*/
|
||||
import type { SiteDetailPage } from "./types.ts";
|
||||
import { extractDoubanId, extractImdbId } from "./util.ts";
|
||||
|
||||
/** 表情 / 图标 / 旗帜 / 头像等小图,不算截图 */
|
||||
const NOT_SCREENSHOT =
|
||||
/(smilies|smiley|emoji|\/icons?\/|pic\/(?:trans|cattrans|flag)|\/flags?\/|avatar|\.svg(?:$|\?))/i;
|
||||
|
||||
export const MAX_SCREENSHOTS = 20;
|
||||
export const MAX_FILES = 500;
|
||||
|
||||
/** MediaInfo / BDInfo 文本:有概览段落 + 视频段落,且至少 5 行「键 : 值」 */
|
||||
export function looksLikeMediaInfo(text: string): boolean {
|
||||
if (!/(^|\n)\s*(General|概览|常规|DISC INFO|Disc Title|Complete name)\b/i.test(text)) return false;
|
||||
if (!/(^|\n)\s*(Video|视频)\b/i.test(text)) return false;
|
||||
return (text.match(/^[^\n::]{2,40}[ \t]*[::][ \t]*\S/gm) ?? []).length >= 5;
|
||||
}
|
||||
|
||||
export function extractTmdb(text: string | null | undefined): SiteDetailPage["tmdb"] {
|
||||
const m = text ? /themoviedb\.org\/(movie|tv)\/(\d+)/.exec(text) : null;
|
||||
return m ? { type: m[1] as "movie" | "tv", id: m[2]! } : null;
|
||||
}
|
||||
|
||||
/** 图片地址转绝对 http(s) 地址并过滤非截图;无效返回 null */
|
||||
export function screenshotUrl(src: string | null | undefined, base: string): string | null {
|
||||
if (!src || src.startsWith("data:")) return null;
|
||||
let u: URL;
|
||||
try {
|
||||
u = new URL(src.trim(), base);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (u.protocol !== "http:" && u.protocol !== "https:") return null;
|
||||
if (NOT_SCREENSHOT.test(u.pathname)) return null;
|
||||
return u.toString();
|
||||
}
|
||||
|
||||
/** 多行文本规整:去 \r、行尾空白,连续空行压成一行 */
|
||||
export function tidyText(text: string): string {
|
||||
return text
|
||||
.replace(/\r/g, "")
|
||||
.replace(/ /g, " ")
|
||||
.split("\n")
|
||||
.map((l) => l.replace(/\s+$/, ""))
|
||||
.join("\n")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function emptyDetailPage(): SiteDetailPage {
|
||||
return {
|
||||
descriptionText: null,
|
||||
mediaInfo: null,
|
||||
screenshots: [],
|
||||
imdbId: null,
|
||||
doubanId: null,
|
||||
tmdb: null,
|
||||
files: [],
|
||||
tags: [],
|
||||
uploader: null,
|
||||
probe: { labels: [], hasDescription: false, imageCount: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* M-Team 详情(BBCode 简介 + 独立 mediainfo 字段)→ 扩展内容。
|
||||
* 截图取 [img] 标签;简介去掉 BBCode 标记与 MediaInfo 引用块。
|
||||
*/
|
||||
export function pageFromBbcode(
|
||||
descr: string | null | undefined,
|
||||
base: string,
|
||||
extra: { mediaInfo?: string | null; imdbId?: string | null; doubanId?: string | null; tags?: string[] } = {},
|
||||
): SiteDetailPage {
|
||||
const page = emptyDetailPage();
|
||||
const src = descr ?? "";
|
||||
const imgs: string[] = [];
|
||||
for (const m of src.matchAll(/\[img(?:=([^\]]+))?\]([^[]*)\[\/img\]|\[img=([^\]]+)\]/gi)) {
|
||||
const u = screenshotUrl((m[2] || m[1] || m[3] || "").trim(), base);
|
||||
if (u && !imgs.includes(u)) imgs.push(u);
|
||||
}
|
||||
page.probe.imageCount = imgs.length;
|
||||
page.screenshots = imgs.slice(0, MAX_SCREENSHOTS);
|
||||
let mediaInfo = extra.mediaInfo?.trim() || null;
|
||||
let body = src.replace(/\[img(?:=[^\]]+)?\][^[]*\[\/img\]|\[img=[^\]]+\]/gi, "");
|
||||
// [quote] / [code] / [mediainfo] 块里的 MediaInfo
|
||||
body = body.replace(
|
||||
/\[(quote|code|mediainfo|box|hide|spoiler)(=[^\]]*)?\]([\s\S]*?)\[\/\1\]/gi,
|
||||
(all, _t, _a, inner) => {
|
||||
const text = tidyText(String(inner).replace(/\[\/?[a-z*]+(?:=[^\]]*)?\]/gi, ""));
|
||||
if (looksLikeMediaInfo(text)) {
|
||||
mediaInfo ??= text;
|
||||
return "";
|
||||
}
|
||||
return all;
|
||||
},
|
||||
);
|
||||
const text = tidyText(body.replace(/\[\/?[a-z*]+(?:=[^\]]*)?\]/gi, ""));
|
||||
page.probe.hasDescription = !!src.trim();
|
||||
page.descriptionText = text ? text.slice(0, 20_000) : null;
|
||||
page.mediaInfo = mediaInfo ? mediaInfo.slice(0, 50_000) : null;
|
||||
page.imdbId = extra.imdbId ?? extractImdbId(src) ?? null;
|
||||
page.doubanId = extra.doubanId ?? extractDoubanId(src) ?? null;
|
||||
page.tmdb = extractTmdb(src);
|
||||
page.tags = [...new Set(extra.tags ?? [])].slice(0, 20);
|
||||
return page;
|
||||
}
|
||||
@@ -90,6 +90,8 @@ export {
|
||||
type SiteCapabilities,
|
||||
type SiteCategory,
|
||||
type SiteCredentialKind,
|
||||
type SiteDetailFile,
|
||||
type SiteDetailPage,
|
||||
SiteError,
|
||||
type SiteErrorKind,
|
||||
type SiteHealth,
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
*/
|
||||
import { redactUrl } from "../../rss/redact.ts";
|
||||
import { buildHttpContext, type SiteAdapterDeps } from "../deps.ts";
|
||||
import { pageFromBbcode } from "../detail-text.ts";
|
||||
import { readBody, siteFetch } from "../http.ts";
|
||||
import { healthFromError } from "../spider/adapter.ts";
|
||||
import { promotionFromFactors } from "../spider/map.ts";
|
||||
@@ -154,6 +155,12 @@ export function createMteamAdapter(input: MteamDefinition, deps: SiteAdapterDeps
|
||||
const [row] = parseMteamResult([data], def.domain);
|
||||
const r: SiteReleaseDetail = mteamTorrentToRelease(row!, data, def, http.now().toISOString());
|
||||
if (data.descr) r.descriptionText = String(data.descr).slice(0, 20_000);
|
||||
r.page = pageFromBbcode(data.descr, def.domain, {
|
||||
mediaInfo: typeof data.mediainfo === "string" ? data.mediainfo : null,
|
||||
imdbId: r.imdbId ?? null,
|
||||
doubanId: r.doubanId ?? null,
|
||||
tags: r.tags,
|
||||
});
|
||||
return r;
|
||||
},
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ export interface MteamTorrentJson {
|
||||
labels?: string | number | null;
|
||||
labelsNew?: string[] | null;
|
||||
descr?: string | null;
|
||||
mediainfo?: string | null;
|
||||
status?: {
|
||||
seeders?: string | number | null;
|
||||
leechers?: string | number | null;
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import { HHANCLUB_DETAIL_PAGE_FULL } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { hhanclubDefinition } from "../definitions/index.ts";
|
||||
import { looksLikeMediaInfo, pageFromBbcode } from "../detail-text.ts";
|
||||
import { getSiteDefinition } from "../registry.ts";
|
||||
import { parseDetailPage } from "./detail.ts";
|
||||
import type { IndexerDefinition } from "./schema.ts";
|
||||
|
||||
const OBSERVED = "2026-09-29T00:00:00.000Z";
|
||||
|
||||
describe("详情页扩展内容(HHClub div 模板)", () => {
|
||||
const { kind, detail } = parseDetailPage(HHANCLUB_DETAIL_PAGE_FULL, hhanclubDefinition, "91002", OBSERVED);
|
||||
const page = detail!.page!;
|
||||
|
||||
it("外部 ID、上传者、标签", () => {
|
||||
expect(kind).toBe("ok");
|
||||
expect(page).toMatchObject({
|
||||
imdbId: "tt14452776",
|
||||
doubanId: "36000001",
|
||||
tmdb: { type: "tv", id: "136315" },
|
||||
uploader: "uploader01",
|
||||
tags: ["官方", "中字", "完结"],
|
||||
});
|
||||
expect(detail!.imdbId).toBe("tt14452776");
|
||||
expect(detail!.tags).toEqual(["官方", "中字", "完结"]);
|
||||
});
|
||||
|
||||
it("MediaInfo 从简介的代码块取出,简介里不再重复", () => {
|
||||
expect(page.mediaInfo).toMatch(/^General\nUnique ID : 1234\n/);
|
||||
expect(page.mediaInfo).toContain("Format : HEVC");
|
||||
expect(page.descriptionText).toContain("◎译 名 示例剧集");
|
||||
expect(page.descriptionText).not.toContain("Unique ID");
|
||||
});
|
||||
|
||||
it("截图:相对地址补全、优先 data-src、过滤表情与占位图", () => {
|
||||
expect(page.screenshots).toEqual([
|
||||
"https://hhanclub.net/attachments/202609/poster.jpg",
|
||||
"https://img.example-host.com/i/shot1.png",
|
||||
"https://img.example-host.com/i/shot2.jpg",
|
||||
]);
|
||||
});
|
||||
|
||||
it("文件清单:「名称 (大小)」", () => {
|
||||
expect(page.files).toEqual([
|
||||
{ name: "Example.Show.S01E01.mkv", sizeBytes: Math.round(1.04 * 1024 ** 3) },
|
||||
{ name: "Example.Show.S01E02.mkv", sizeBytes: Math.round(1.02 * 1024 ** 3) },
|
||||
]);
|
||||
});
|
||||
|
||||
it("诊断:记录页面上的字段标签", () => {
|
||||
expect(page.probe.labels).toEqual(expect.arrayContaining(["标题", "副标题", "标签", "上传者", "文件列表"]));
|
||||
expect(page.probe.hasDescription).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("详情页扩展内容(标准 NexusPHP rowhead 表格)", () => {
|
||||
const def = getSiteDefinition("audiences") as IndexerDefinition;
|
||||
const html = `<html><body><a href="logout.php">退出</a><h1 id="top">Some.Movie.2024.1080p.BluRay.x264-GRP</h1>
|
||||
<table>
|
||||
<tr><td class="rowhead">副标题</td><td class="rowfollow">某电影</td></tr>
|
||||
<tr><td class="rowhead">基本信息</td><td class="rowfollow"><b>大小:</b>8.00 GB <b>类型:</b> 电影</td></tr>
|
||||
<tr><td class="rowhead">标签</td><td class="rowfollow"><span>中字</span><span>DIY</span></td></tr>
|
||||
<tr><td class="rowhead">上传者</td><td class="rowfollow"><i>匿名</i></td></tr>
|
||||
<tr><td class="rowhead">简介</td><td class="rowfollow"><div id="kdescr">一段简介<br>第二行
|
||||
<div class="codemain">DISC INFO:<br>Disc Title: X<br>Size: 1<br>Protection: AACS<br>Playlist: 00800.MPLS<br>Length: 2:00:00<br>Video: MPEG-4 AVC<br>Total Bitrate: 30 Mbps</div>
|
||||
<img src="https://img.example-host.com/a.png"></div></td></tr>
|
||||
</table>
|
||||
<table id="filelist"><tr><td>Some.Movie.mkv</td><td>8.00 GB</td></tr></table></body></html>`;
|
||||
|
||||
it("rowhead 标签、BDInfo、#filelist 表格", () => {
|
||||
const { detail } = parseDetailPage(html, def, "1", OBSERVED);
|
||||
const page = detail!.page!;
|
||||
expect(page.tags).toEqual(["中字", "DIY"]);
|
||||
expect(page.uploader).toBe("匿名");
|
||||
expect(page.mediaInfo).toContain("Disc Title: X");
|
||||
expect(page.descriptionText).toBe("一段简介\n第二行");
|
||||
expect(page.screenshots).toEqual(["https://img.example-host.com/a.png"]);
|
||||
expect(page.files).toEqual([{ name: "Some.Movie.mkv", sizeBytes: 8 * 1024 ** 3 }]);
|
||||
expect(page.imdbId).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("M-Team BBCode 简介", () => {
|
||||
it("截图取 [img],MediaInfo 取引用块,外部 ID 从正文取", () => {
|
||||
const descr = `[img]https://img.example-host.com/p.jpg[/img]
|
||||
◎简介 这是简介 https://www.imdb.com/title/tt15239678/
|
||||
[quote]General
|
||||
Unique ID : 1
|
||||
Complete name : a.mkv
|
||||
Format : Matroska
|
||||
File size : 1 GiB
|
||||
|
||||
Video
|
||||
Format : AVC[/quote]
|
||||
[img=https://img.example-host.com/s.png]`;
|
||||
const page = pageFromBbcode(descr, "https://kp.m-team.cc/", { tags: ["中字"] });
|
||||
expect(page.screenshots).toEqual(["https://img.example-host.com/p.jpg", "https://img.example-host.com/s.png"]);
|
||||
expect(page.mediaInfo).toMatch(/^General\n/);
|
||||
expect(page.descriptionText).toBe("◎简介 这是简介 https://www.imdb.com/title/tt15239678/");
|
||||
expect(page.imdbId).toBe("tt15239678");
|
||||
expect(page.tags).toEqual(["中字"]);
|
||||
});
|
||||
|
||||
it("MediaInfo 识别不误判普通简介", () => {
|
||||
expect(looksLikeMediaInfo("General\n一段文字\nVideo")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,208 @@
|
||||
/**
|
||||
* 详情页扩展内容(简介 / MediaInfo / 截图 / IMDb·豆瓣·TMDB / 文件清单 / 标签 / 上传者)—— 本项目扩展。
|
||||
* 以 MovieClaw detail_*_css(description / imdb / douban / file_list)为起点,其余按 NexusPHP details.php 公开结构
|
||||
* 做通用规则:字段标签(td.rowhead 或站点配置的 div 标签)的下一个兄弟元素是值。站点特化写在定义 `umt.detail.page`。
|
||||
* 文件清单只取页面上已有的内容,不为它额外请求 viewfilelist.php(请求克制)。
|
||||
*/
|
||||
import type { CheerioAPI } from "cheerio";
|
||||
import {
|
||||
emptyDetailPage,
|
||||
extractTmdb,
|
||||
looksLikeMediaInfo,
|
||||
MAX_FILES,
|
||||
MAX_SCREENSHOTS,
|
||||
screenshotUrl,
|
||||
tidyText,
|
||||
} from "../detail-text.ts";
|
||||
import type { SiteDetailFile, SiteDetailPage } from "../types.ts";
|
||||
import { extractDoubanId, extractImdbId } from "../util.ts";
|
||||
import { mcSelect } from "./mc-selector.ts";
|
||||
import type { IndexerDefinition } from "./schema.ts";
|
||||
import { parseSizeMp } from "./spider.ts";
|
||||
|
||||
type DetailConf = NonNullable<IndexerDefinition["umt"]["detail"]>;
|
||||
type El = ReturnType<CheerioAPI>;
|
||||
|
||||
const LABELS = {
|
||||
description: ["简介", "Description"],
|
||||
mediaInfo: ["MediaInfo", "媒体信息", "BDInfo"],
|
||||
tags: ["标签", "Tags"],
|
||||
uploader: ["上传者", "发布者", "发布人", "上传人", "Uploader"],
|
||||
files: ["文件列表", "文件清单", "File list", "Files"],
|
||||
};
|
||||
|
||||
/** 简介里可能装 MediaInfo 的块 */
|
||||
const MI_BLOCKS = "fieldset, div.codemain, pre, code, blockquote, .spoiler-content, .hide, div.nexus-media-info-raw";
|
||||
|
||||
const normLabel = (t: string) =>
|
||||
t
|
||||
.replace(/\s+/g, "")
|
||||
.replace(/[::]+$/, "")
|
||||
.toLowerCase();
|
||||
|
||||
/** 保留换行的文本:<br> 与块级元素换行 */
|
||||
function blockText($: CheerioAPI, el: El): string {
|
||||
const c = el.clone();
|
||||
c.find("script, style").remove();
|
||||
c.find("br").replaceWith("\n");
|
||||
c.find("div, p, li, tr, pre, fieldset, blockquote, h1, h2, h3, h4, h5, h6").each((_, n) => {
|
||||
$(n).append("\n");
|
||||
});
|
||||
return tidyText(c.text());
|
||||
}
|
||||
|
||||
const SIZE_TAIL = /^(.*?)\s*[(([]\s*([\d.,]+\s*[KMGTP]?i?B)\s*[))\]]\s*$/i;
|
||||
const HAS_EXT = /\.[A-Za-z0-9]{2,5}$/;
|
||||
|
||||
function fileFromText(text: string): SiteDetailFile | null {
|
||||
const t = text.replace(/\s+/g, " ").trim();
|
||||
if (!t) return null;
|
||||
const m = SIZE_TAIL.exec(t);
|
||||
if (m?.[1]) return { name: m[1], sizeBytes: parseSizeMp(m[2]!) || null };
|
||||
return HAS_EXT.test(t) ? { name: t, sizeBytes: null } : null;
|
||||
}
|
||||
|
||||
function filesFrom($: CheerioAPI, rows: El): SiteDetailFile[] {
|
||||
const out: SiteDetailFile[] = [];
|
||||
rows.each((_, n) => {
|
||||
if (out.length >= MAX_FILES) return;
|
||||
const row = $(n);
|
||||
const cells = row.children("td");
|
||||
if (cells.length >= 2) {
|
||||
const name = cells.first().text().replace(/\s+/g, " ").trim();
|
||||
const size = cells.eq(1).text().trim();
|
||||
if (name && HAS_EXT.test(name) && /\d/.test(size)) out.push({ name, sizeBytes: parseSizeMp(size) || null });
|
||||
return;
|
||||
}
|
||||
const f = fileFromText(row.text());
|
||||
if (f) out.push(f);
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string): SiteDetailPage {
|
||||
const page = emptyDetailPage();
|
||||
const p = conf.page ?? {};
|
||||
|
||||
// 字段标签 → 值元素
|
||||
const pairs: { label: string; value: El }[] = [];
|
||||
const addLabels = (sel: string, valueFor: (e: El) => El) => {
|
||||
$(sel).each((_, n) => {
|
||||
const e = $(n);
|
||||
const label = normLabel(e.text());
|
||||
if (!label || label.length > 16) return;
|
||||
const value = valueFor(e);
|
||||
if (value.length) pairs.push({ label, value });
|
||||
});
|
||||
};
|
||||
addLabels(conf.rowLabel, (e) => e.nextAll(conf.rowValue).first());
|
||||
for (const sel of p.labelSelectors ?? []) addLabels(sel, (e) => e.next());
|
||||
const byLabel = (names: string[]): El | null =>
|
||||
pairs.find((x) => names.some((n) => x.label.startsWith(normLabel(n))))?.value ?? null;
|
||||
page.probe.labels = [...new Set(pairs.map((x) => x.label))].slice(0, 40);
|
||||
|
||||
// 简介容器
|
||||
const descSel = conf.fields?.description ?? conf.description;
|
||||
let desc: El | null = descSel ? $(descSel).first() : null;
|
||||
if (!desc?.length) desc = byLabel(LABELS.description);
|
||||
page.probe.hasDescription = !!desc?.length;
|
||||
|
||||
// MediaInfo:站点选择器 → 字段标签 → 简介里的引用 / 代码块(取最内层文本,整块从简介里去掉)
|
||||
let mediaInfoEl: El | null = null;
|
||||
let miBlockIndex = -1;
|
||||
if (p.mediaInfo) {
|
||||
const el = $(p.mediaInfo).first();
|
||||
if (el.length) mediaInfoEl = el;
|
||||
}
|
||||
mediaInfoEl ??= byLabel(LABELS.mediaInfo);
|
||||
const blocks = desc?.length ? desc.find(MI_BLOCKS) : null;
|
||||
if (!mediaInfoEl && blocks) {
|
||||
blocks.each((i, n) => {
|
||||
if (miBlockIndex >= 0) return;
|
||||
if (looksLikeMediaInfo(blockText($, $(n)))) miBlockIndex = i;
|
||||
});
|
||||
if (miBlockIndex >= 0) {
|
||||
const outer = blocks.eq(miBlockIndex);
|
||||
const inner = outer
|
||||
.find(MI_BLOCKS)
|
||||
.filter((_, n) => looksLikeMediaInfo(blockText($, $(n))))
|
||||
.last();
|
||||
mediaInfoEl = inner.length ? inner : outer;
|
||||
}
|
||||
}
|
||||
if (mediaInfoEl) {
|
||||
const t = blockText($, mediaInfoEl);
|
||||
if (t) page.mediaInfo = t.slice(0, 50_000);
|
||||
}
|
||||
|
||||
// 截图
|
||||
const imgs: string[] = [];
|
||||
const imgEls = p.screenshots ? $(p.screenshots) : desc?.length ? desc.find("img") : null;
|
||||
imgEls?.each((_, n) => {
|
||||
const e = $(n);
|
||||
const u = screenshotUrl(e.attr("data-src") ?? e.attr("data-original") ?? e.attr("src"), base);
|
||||
if (u && !imgs.includes(u)) imgs.push(u);
|
||||
});
|
||||
page.probe.imageCount = imgs.length;
|
||||
page.screenshots = imgs.slice(0, MAX_SCREENSHOTS);
|
||||
|
||||
// 简介纯文本:去掉 MediaInfo 块与图片
|
||||
if (desc?.length) {
|
||||
const c = desc.clone();
|
||||
if (miBlockIndex >= 0) c.find(MI_BLOCKS).eq(miBlockIndex).remove();
|
||||
c.find("img").remove();
|
||||
const t = blockText($, c);
|
||||
if (t) page.descriptionText = t.slice(0, 20_000);
|
||||
}
|
||||
|
||||
// 外部 ID:页面上的链接优先,其次简介正文
|
||||
const descHtml = desc?.html() ?? "";
|
||||
const f = conf.fields ?? {};
|
||||
page.imdbId =
|
||||
extractImdbId(mcSelect($, null, f.imdb ?? "a[href*='imdb.com']::attr(href)")) ?? extractImdbId(descHtml) ?? null;
|
||||
page.doubanId =
|
||||
extractDoubanId(mcSelect($, null, f.douban ?? "a[href*='douban.com']::attr(href)")) ??
|
||||
extractDoubanId(descHtml) ??
|
||||
null;
|
||||
page.tmdb = extractTmdb($("a[href*='themoviedb.org']").first().attr("href")) ?? extractTmdb(descHtml);
|
||||
|
||||
// 文件清单:站点选择器 → MovieClaw file_list(去伪元素)→ 字段标签 → #filelist 表格
|
||||
const fileSel = p.files ?? f.fileList?.replace(/::text$/, "");
|
||||
let files = fileSel ? filesFrom($, $(fileSel)) : [];
|
||||
if (!files.length) {
|
||||
const v = byLabel(LABELS.files);
|
||||
if (v) files = filesFrom($, v.find("li, tr"));
|
||||
}
|
||||
if (!files.length) files = filesFrom($, $("#filelist tr"));
|
||||
page.files = files;
|
||||
|
||||
// 标签
|
||||
const tagEl = p.tags ? $(p.tags) : byLabel(LABELS.tags);
|
||||
if (tagEl?.length) {
|
||||
const parts = tagEl.find("span, a").length
|
||||
? tagEl
|
||||
.find("span, a")
|
||||
.map((_, n) => $(n).text().trim())
|
||||
.get()
|
||||
: tagEl.text().split(/[\s,,|/]+/);
|
||||
page.tags = [...new Set(parts.map((t) => t.trim()).filter((t) => t && t.length <= 20))].slice(0, 20);
|
||||
}
|
||||
|
||||
// 上传者
|
||||
const upEl = p.uploader ? $(p.uploader).first() : byLabel(LABELS.uploader);
|
||||
if (upEl?.length) {
|
||||
const a = upEl.is("a") ? upEl : upEl.find("a[href*='userdetails.php']").first();
|
||||
const name = (a.length ? a.text() : upEl.text()).replace(/\s+/g, " ").trim();
|
||||
if (name) page.uploader = name.slice(0, 64);
|
||||
}
|
||||
if (!page.uploader) {
|
||||
for (const x of pairs) {
|
||||
const m = /(?:上传者|发布者|发布人|Uploader)\s*[::]\s*(\S+)/i.exec(x.value.text());
|
||||
if (m?.[1]) {
|
||||
page.uploader = m[1].slice(0, 64);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return page;
|
||||
}
|
||||
@@ -6,6 +6,7 @@
|
||||
import type { CheerioAPI } from "cheerio";
|
||||
import type { SiteReleaseDetail } from "../types.ts";
|
||||
import { extractDoubanId, extractImdbId, parseCount, parseSiteTime } from "../util.ts";
|
||||
import { extractDetailPage } from "./detail-page.ts";
|
||||
import { categoryName, promotionFromFactors } from "./map.ts";
|
||||
import { mcSelect } from "./mc-selector.ts";
|
||||
import type { IndexerDefinition } from "./schema.ts";
|
||||
@@ -115,5 +116,10 @@ export function parseDetailPage(
|
||||
if (imdb) detail.imdbId = imdb;
|
||||
const douban = extractDoubanId(mcSelect($, null, f.douban)) ?? extractDoubanId(descHtml);
|
||||
if (douban) detail.doubanId = douban;
|
||||
const page = extractDetailPage($, conf, def.domain);
|
||||
page.imdbId ??= detail.imdbId ?? null;
|
||||
page.doubanId ??= detail.doubanId ?? null;
|
||||
if (page.tags.length) detail.tags = page.tags;
|
||||
detail.page = page;
|
||||
return { kind: "ok", detail };
|
||||
}
|
||||
@@ -198,6 +198,20 @@ export interface UmtSiteExtension {
|
||||
fields?: Partial<
|
||||
Record<"title" | "subtitle" | "size" | "description" | "download" | "imdb" | "douban" | "fileList", string>
|
||||
>;
|
||||
/**
|
||||
* 详情页扩展内容(简介 / MediaInfo / 截图 / 文件 / 标签 / 上传者)的站点特化,均可省略(走通用规则,见 detail-page.ts)。
|
||||
* 选择器为普通 CSS。
|
||||
*/
|
||||
page?: {
|
||||
/** 除 rowLabel 外的字段标签元素(div 版详情页:标签元素的下一个兄弟元素是值) */
|
||||
labelSelectors?: string[];
|
||||
mediaInfo?: string;
|
||||
screenshots?: string;
|
||||
tags?: string;
|
||||
uploader?: string;
|
||||
/** 文件清单行(每行一个文件,文本形如「名称 (大小)」或两列表格) */
|
||||
files?: string;
|
||||
};
|
||||
};
|
||||
/** 下载路径(按资源 ID 拼接) */
|
||||
downloadPath: string;
|
||||
|
||||
@@ -60,8 +60,34 @@ export interface SiteRelease {
|
||||
observedAt: string;
|
||||
}
|
||||
|
||||
/** 详情页文件清单一项;大小未知为 null */
|
||||
export interface SiteDetailFile {
|
||||
name: string;
|
||||
sizeBytes: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 详情页扩展内容(只在按需抓详情时得到,服务端放 kv_cache,不进 source_release 列)。
|
||||
* 截图是站点页面上的原始图片地址,展示时经服务端图片代理,不让浏览器直连外站。
|
||||
*/
|
||||
export interface SiteDetailPage {
|
||||
/** 简介纯文本(已去掉 MediaInfo 块) */
|
||||
descriptionText: string | null;
|
||||
mediaInfo: string | null;
|
||||
screenshots: string[];
|
||||
imdbId: string | null;
|
||||
doubanId: string | null;
|
||||
tmdb: { type: "movie" | "tv"; id: string } | null;
|
||||
files: SiteDetailFile[];
|
||||
tags: string[];
|
||||
uploader: string | null;
|
||||
/** 解析诊断:页面上找到的字段标签、是否有简介容器、图片数(用于站点改版排查,不展示) */
|
||||
probe: { labels: string[]; hasDescription: boolean; imageCount: number };
|
||||
}
|
||||
|
||||
export interface SiteReleaseDetail extends SiteRelease {
|
||||
descriptionText?: string;
|
||||
page?: SiteDetailPage;
|
||||
}
|
||||
|
||||
export interface SiteListFilters {
|
||||
|
||||
@@ -348,6 +348,16 @@ export class TmdbClient {
|
||||
return [...m, ...t].sort((a, b) => (b.popularity ?? 0) - (a.popularity ?? 0));
|
||||
}
|
||||
|
||||
/** /find/{imdb_id}?external_source=imdb_id:IMDb ID → 电影 / 剧集候选(站点详情页给出的 IMDb 链接走这里) */
|
||||
async findByImdb(imdbId: string): Promise<MetadataCandidate[]> {
|
||||
if (!/^tt\d{5,10}$/.test(imdbId)) return [];
|
||||
const lang = this.langs[0]!;
|
||||
const p = await this.get<Json>(`/find/${imdbId}`, { external_source: "imdb_id", language: lang });
|
||||
const movies = (Array.isArray(p.movie_results) ? p.movie_results : []) as Json[];
|
||||
const tvs = (Array.isArray(p.tv_results) ? p.tv_results : []) as Json[];
|
||||
return [...movies.map((r) => this.fromBrief("movie", r, lang)), ...tvs.map((r) => this.fromBrief("tv", r, lang))];
|
||||
}
|
||||
|
||||
/** 详情:多语言标题、别名(alternative_titles)、外部 ID、季列表(剧集)。不存在抛 notfound。 */
|
||||
async details(type: MediaEntityType, id: string | number): Promise<MetadataCandidate> {
|
||||
const [first, ...rest] = await Promise.all(
|
||||
|
||||
@@ -636,6 +636,24 @@ export const WorkbenchReleasePage = z.object({
|
||||
});
|
||||
export type WorkbenchReleasePage = z.infer<typeof WorkbenchReleasePage>;
|
||||
|
||||
/**
|
||||
* 站点详情页扩展内容(按需抓详情时得到,服务端缓存 24 小时;过期后需再次复核详情)。
|
||||
* 截图地址是服务端图片代理路径(/api/v1/img/site-detail/{releaseId}/{index}),不直连外站。
|
||||
*/
|
||||
export const SiteReleasePage = z.object({
|
||||
fetchedAt: Timestamp,
|
||||
descriptionText: z.string().nullable(),
|
||||
mediaInfo: z.string().nullable(),
|
||||
screenshots: z.array(z.object({ url: z.string(), host: z.string() })),
|
||||
imdbId: z.string().nullable(),
|
||||
doubanId: z.string().nullable(),
|
||||
tmdb: z.object({ type: z.enum(["movie", "tv"]), id: z.string() }).nullable(),
|
||||
files: z.array(z.object({ name: z.string(), sizeBytes: z.number().int().nullable() })),
|
||||
tags: z.array(z.string()),
|
||||
uploader: z.string().nullable(),
|
||||
});
|
||||
export type SiteReleasePage = z.infer<typeof SiteReleasePage>;
|
||||
|
||||
export const WorkbenchReleaseDetail = z.object({
|
||||
release: WorkbenchRelease,
|
||||
observations: z.array(
|
||||
@@ -657,5 +675,9 @@ export const WorkbenchReleaseDetail = z.object({
|
||||
}),
|
||||
/** 刷新详情失败时保留既有数据并给出原因 */
|
||||
refreshError: z.string().nullable(),
|
||||
/** 详情页扩展内容;没抓过、缓存已过期或站点不支持详情页时为 null */
|
||||
page: SiteReleasePage.nullable(),
|
||||
/** 站点能否抓详情页(false 时前端不自动抓) */
|
||||
detailSupported: z.boolean(),
|
||||
});
|
||||
export type WorkbenchReleaseDetail = z.infer<typeof WorkbenchReleaseDetail>;
|
||||
@@ -80,3 +80,31 @@ export const HHANCLUB_LIST_PAGE_2026 = `<!DOCTYPE html><html><head><title>HHCLUB
|
||||
</div>
|
||||
<div class="pagination"><a href="?page=0">1-100</a><a href="?page=1">101-200</a></div>
|
||||
</body></html>`;
|
||||
|
||||
/**
|
||||
* 详情页扩展内容(简介 / MediaInfo / 截图 / 外部链接 / 标签 / 上传者 / 文件清单)fixture:
|
||||
* 结构按 HHClub div 模板(「标签」div + 相邻值 div)与 NexusPHP #kdescr 简介编写,数值均为示例。
|
||||
*/
|
||||
export const HHANCLUB_DETAIL_PAGE_FULL = `<!DOCTYPE html><html><head><title>HHCLUB :: 种子详情</title></head><body>
|
||||
<div id="user-info-panel"><a href="userdetails.php?id=7" class="VIP_Name">bob</a> <a href="logout.php">退出</a></div>
|
||||
<div class="grid">
|
||||
<div class="font-bold leading-6">标题</div><div class="font-bold leading-6">Example.Show.S01.2026.2160p.WEB-DL.H265.AAC-HHWEB</div>
|
||||
<div class="font-bold leading-6">副标题</div><div class="font-bold leading-6">示例剧集 | 全12集 | 4K</div>
|
||||
<div><span class="font-bold">大小:</span><span>12.50 GB</span></div>
|
||||
<div class="font-bold leading-6">标签</div><div class="font-bold leading-6"><span class="tag">官方</span><span class="tag">中字</span><span class="tag">完结</span></div>
|
||||
<div class="font-bold leading-6">上传者</div><div class="font-bold leading-6"><a href="userdetails.php?id=321">uploader01</a></div>
|
||||
<div><a href="download.php?id=91002">下载种子</a></div>
|
||||
<div class="font-bold leading-6">简介</div>
|
||||
<div id="kdescr">
|
||||
<img src="attachments/202609/poster.jpg" alt="poster"><br>
|
||||
◎译 名 示例剧集<br>◎年 代 2026<br>
|
||||
◎IMDb链接 <a href="https://www.imdb.com/title/tt14452776/">https://www.imdb.com/title/tt14452776/</a><br>
|
||||
◎豆瓣链接 <a href="https://movie.douban.com/subject/36000001/">https://movie.douban.com/subject/36000001/</a><br>
|
||||
◎TMDB <a href="https://www.themoviedb.org/tv/136315">themoviedb</a><br>
|
||||
<img src="pic/smilies/1.gif">
|
||||
<fieldset><legend>MediaInfo</legend><div class="codemain">General<br>Unique ID : 1234<br>Complete name : Example.Show.S01E01.mkv<br>Format : Matroska<br>File size : 1.04 GiB<br>Duration : 45 min<br><br>Video<br>ID : 1<br>Format : HEVC<br>Width : 3 840 pixels</div></fieldset>
|
||||
<img src="https://img.example-host.com/i/shot1.png"><img data-src="https://img.example-host.com/i/shot2.jpg" src="pic/trans.gif">
|
||||
</div>
|
||||
<div class="font-bold leading-6">文件列表</div>
|
||||
<div class="font-bold leading-6"><ul><li>Example.Show.S01E01.mkv (1.04 GB)</li><li>Example.Show.S01E02.mkv (1.02 GB)</li></ul></div>
|
||||
</div></body></html>`;
|
||||
@@ -144,7 +144,7 @@ function detailBody(r: MockNexusRelease): string {
|
||||
<tr><td class="rowhead">副标题</td><td class="rowfollow" align="left">${esc(r.subtitle ?? "")}</td></tr>
|
||||
<tr><td class="rowhead">基本信息</td><td class="rowfollow" align="left"><b>大小:</b>${esc(r.sizeText)} <b>类型:</b> ${esc(r.catName)} 发布于 <span title="${r.uploadedAt}">${esc(r.relativeTime ?? r.uploadedAt)}</span></td></tr>
|
||||
<tr><td class="rowhead">同伴</td><td class="rowfollow" align="left">${r.seeders.replace(/,/g, "")}个做种者 | ${r.leechers.replace(/,/g, "")}个下载者</td></tr>
|
||||
<tr><td class="rowhead">简介</td><td class="rowfollow" align="left"><div id="kdescr">${esc(r.description ?? "暂无简介")}</div></td></tr>
|
||||
<tr><td class="rowhead">简介</td><td class="rowfollow" align="left"><div id="kdescr">${r.descriptionHtml ?? esc(r.description ?? "暂无简介")}</div></td></tr>
|
||||
</table>`;
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,8 @@ export interface MockNexusRelease {
|
||||
hr?: boolean;
|
||||
uploader?: string;
|
||||
description?: string;
|
||||
/** 简介原样 HTML(优先于 description;用于详情页扩展内容测试) */
|
||||
descriptionHtml?: string;
|
||||
/** 下载返回磁力而不是 .torrent */
|
||||
magnet?: string;
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ const notFound: MockReply = {
|
||||
json: { success: false, status_code: 34, status_message: "The resource you requested could not be found." },
|
||||
};
|
||||
|
||||
/** mock TMDB v3:/3/search/{movie,tv}、/3/{movie,tv}/:id(append_to_response)、/3/tv/:id/season/:n */
|
||||
/** mock TMDB v3:/3/search/{movie,tv}、/3/find/:imdbId、/3/{movie,tv}/:id(append_to_response)、/3/tv/:id/season/:n */
|
||||
export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTmdb> {
|
||||
const fx = opts.fixtures ?? TMDB_FIXTURES;
|
||||
const apiKey = opts.apiKey ?? "test-tmdb-key";
|
||||
@@ -84,6 +84,23 @@ export async function createMockTmdb(opts: MockTmdbOptions = {}): Promise<MockTm
|
||||
return { json: { page: 1, results, total_pages: 1, total_results: results.length } };
|
||||
},
|
||||
],
|
||||
[
|
||||
/^\/3\/find\/(tt\d+)$/,
|
||||
(m, q) => {
|
||||
if (q.external_source !== "imdb_id") return { json: { movie_results: [], tv_results: [] } };
|
||||
const hit = fx.filter((f) => f.external_ids.imdb_id === m[1]);
|
||||
return {
|
||||
json: {
|
||||
movie_results: hit
|
||||
.filter((f) => f.type === "movie")
|
||||
.map((f) => ({ ...brief(f, q.language), media_type: "movie" })),
|
||||
tv_results: hit
|
||||
.filter((f) => f.type === "tv")
|
||||
.map((f) => ({ ...brief(f, q.language), media_type: "tv" })),
|
||||
},
|
||||
};
|
||||
},
|
||||
],
|
||||
[
|
||||
/^\/3\/tv\/(\d+)\/season\/(\d+)$/,
|
||||
(m, q) => {
|
||||
|
||||
Reference in new issue
Block a user