fix(sites): normalize HHClub RSS items, complete detail page parsing (HH-2)
- RSS: split NexusPHP "[category]name[subtitle][size][uploader]" titles in the site adapter layer (category id/name/media type, clean title, subtitle, size); snapshot fields win on merge; lazily renormalize old RSS rows and re-emit release.observed; unknown list fields show an em dash - Detail: HHClub section layout (douban info block as description, screenshots section, IMDb/TMDB from MediaInfo lines, upload time field, on-demand viewfilelist.php once); site CDN screenshots get a Referer - Site request count now includes site-bound RSS fetches - Origin view shows "自动化" when idle and "Agent" only after recent automation actions (lastAutomationAt); agent step titles no longer get squeezed to a sliver by long notes Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
4a212ed19e
commit
cabd772243
32 files changed
+816
-65
No files matched your search
@@ -25,10 +25,13 @@ type El = ReturnType<CheerioAPI>;
|
||||
|
||||
const LABELS = {
|
||||
description: ["简介", "Description"],
|
||||
/** 没有简介容器时的次选:豆瓣 / IMDb 信息块(憨憨把豆瓣资料拆成独立一节) */
|
||||
descriptionFallback: ["豆瓣", "豆瓣信息", "影片信息", "IMDb信息", "IMDb"],
|
||||
screenshots: ["截图", "Screenshots", "Screenshot"],
|
||||
mediaInfo: ["MediaInfo", "媒体信息", "BDInfo"],
|
||||
tags: ["标签", "Tags"],
|
||||
uploader: ["上传者", "发布者", "发布人", "上传人", "Uploader"],
|
||||
files: ["文件列表", "文件清单", "File list", "Files"],
|
||||
files: ["文件列表", "文件清单", "种子文件", "File list", "Files"],
|
||||
};
|
||||
|
||||
/** 简介里可能装 MediaInfo 的块 */
|
||||
@@ -62,6 +65,19 @@ function fileFromText(text: string): SiteDetailFile | null {
|
||||
return HAS_EXT.test(t) ? { name: t, sizeBytes: null } : null;
|
||||
}
|
||||
|
||||
/** 文本里的 IMDb ID:imdb.com/title 链接或「IMDB : tt…」一行;不接受孤立的 tt 串(避免误中哈希) */
|
||||
export function imdbFromText(text: string): string | null {
|
||||
const m =
|
||||
/imdb\.com\/title\/(tt\d{7,8})\b/i.exec(text) ??
|
||||
/\bIMDb\s*(?:ID|链接|編號|编号)?\s*[::]?\s*(tt\d{7,8})\b/i.exec(text);
|
||||
return m?.[1] ?? null;
|
||||
}
|
||||
|
||||
/** NexusPHP viewfilelist.php 返回的文件表(按需 AJAX 取到的 HTML 片段) */
|
||||
export function filesFromFileListHtml($: CheerioAPI): SiteDetailFile[] {
|
||||
return filesFrom($, $("tr"));
|
||||
}
|
||||
|
||||
function filesFrom($: CheerioAPI, rows: El): SiteDetailFile[] {
|
||||
const out: SiteDetailFile[] = [];
|
||||
rows.each((_, n) => {
|
||||
@@ -70,7 +86,7 @@ function filesFrom($: CheerioAPI, rows: El): SiteDetailFile[] {
|
||||
const cells = row.children("td");
|
||||
if (cells.length >= 2) {
|
||||
const name = cells.first().text().replace(/\s+/g, " ").trim();
|
||||
const size = cells.eq(1).text().trim();
|
||||
const size = cells.eq(1).text().replace(/\s+/g, " ").trim();
|
||||
if (name && HAS_EXT.test(name) && /\d/.test(size)) out.push({ name, sizeBytes: parseSizeMp(size) || null });
|
||||
return;
|
||||
}
|
||||
@@ -106,6 +122,7 @@ export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string)
|
||||
let desc: El | null = descSel ? $(descSel).first() : null;
|
||||
if (!desc?.length) desc = byLabel(LABELS.description);
|
||||
page.probe.hasDescription = !!desc?.length;
|
||||
const infoBlock = desc?.length ? null : byLabel(LABELS.descriptionFallback);
|
||||
|
||||
// MediaInfo:站点选择器 → 字段标签 → 简介里的引用 / 代码块(取最内层文本,整块从简介里去掉)
|
||||
let mediaInfoEl: El | null = null;
|
||||
@@ -137,12 +154,19 @@ export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string)
|
||||
|
||||
// 截图
|
||||
const imgs: string[] = [];
|
||||
const imgEls = p.screenshots ? $(p.screenshots) : desc?.length ? desc.find("img") : null;
|
||||
imgEls?.each((_, n) => {
|
||||
const e = $(n);
|
||||
const u = screenshotUrl(e.attr("data-src") ?? e.attr("data-original") ?? e.attr("src"), base);
|
||||
if (u && !imgs.includes(u)) imgs.push(u);
|
||||
});
|
||||
const pushImgs = (els: El | null) =>
|
||||
els?.each((_, n) => {
|
||||
const e = $(n);
|
||||
const u = screenshotUrl(e.attr("data-src") ?? e.attr("data-original") ?? e.attr("src"), base);
|
||||
if (u && !imgs.includes(u)) imgs.push(u);
|
||||
});
|
||||
if (p.screenshots) pushImgs($(p.screenshots));
|
||||
else {
|
||||
// 独立的「截图」一节(憨憨)优先;没有再取简介里的图
|
||||
const shotBlock = byLabel(LABELS.screenshots);
|
||||
if (shotBlock) pushImgs(shotBlock.is("img") ? shotBlock : shotBlock.find("img"));
|
||||
if (!imgs.length && desc?.length) pushImgs(desc.find("img"));
|
||||
}
|
||||
page.probe.imageCount = imgs.length;
|
||||
page.screenshots = imgs.slice(0, MAX_SCREENSHOTS);
|
||||
|
||||
@@ -153,18 +177,36 @@ export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string)
|
||||
c.find("img").remove();
|
||||
const t = blockText($, c);
|
||||
if (t) page.descriptionText = t.slice(0, 20_000);
|
||||
} else if (infoBlock) {
|
||||
const c = infoBlock.clone();
|
||||
c.find("img").remove();
|
||||
// 块里的「标签」span 与值之间补上冒号 / 换行,读起来是「片名:xxx」一行一项
|
||||
c.find(p.labelSelectors?.join(", ") || "span.font-bold").each((_, n) => {
|
||||
const e = $(n);
|
||||
const t = e.text().trim();
|
||||
if (t && t.length <= 16 && !/[::]$/.test(t)) e.text(`${t}:`);
|
||||
e.before("\n");
|
||||
});
|
||||
const t = blockText($, c)
|
||||
.replace(/[ \t]*\n[ \t]*/g, "\n")
|
||||
.replace(/:\s*\n(?=\S)/g, ":");
|
||||
if (t) page.descriptionText = t.slice(0, 20_000);
|
||||
}
|
||||
|
||||
// 外部 ID:页面上的链接优先,其次简介正文
|
||||
const descHtml = desc?.html() ?? "";
|
||||
// 外部 ID:页面上的链接优先,其次简介 / 信息块正文,再次 MediaInfo 里的 IMDB / TMDB 行(憨憨的 MediaInfo 带这两行)
|
||||
const descHtml = `${desc?.html() ?? ""}\n${infoBlock?.html() ?? ""}`;
|
||||
const idText = `${descHtml}\n${page.mediaInfo ?? ""}`;
|
||||
const f = conf.fields ?? {};
|
||||
page.imdbId =
|
||||
extractImdbId(mcSelect($, null, f.imdb ?? "a[href*='imdb.com']::attr(href)")) ?? extractImdbId(descHtml) ?? null;
|
||||
extractImdbId(mcSelect($, null, f.imdb ?? "a[href*='imdb.com']::attr(href)")) ??
|
||||
imdbFromText(idText) ??
|
||||
extractImdbId(descHtml) ??
|
||||
null;
|
||||
page.doubanId =
|
||||
extractDoubanId(mcSelect($, null, f.douban ?? "a[href*='douban.com']::attr(href)")) ??
|
||||
extractDoubanId(descHtml) ??
|
||||
extractDoubanId(idText) ??
|
||||
null;
|
||||
page.tmdb = extractTmdb($("a[href*='themoviedb.org']").first().attr("href")) ?? extractTmdb(descHtml);
|
||||
page.tmdb = extractTmdb($("a[href*='themoviedb.org']").first().attr("href")) ?? extractTmdb(idText);
|
||||
|
||||
// 文件清单:站点选择器 → MovieClaw file_list(去伪元素)→ 字段标签 → #filelist 表格
|
||||
const fileSel = p.files ?? f.fileList?.replace(/::text$/, "");
|
||||
|
||||
Reference in new issue
Block a user