fix(adapters): 观众详情页按真实页面适配(DET-2)
- 促销:详情 h1 是 font.free / font.halfdown 等 + 剩余时间 span[title],新增 umt.detail.promo 覆盖列表促销字段(以前一律判成无促销并覆盖列表的免费) - 标题去掉 h1 里的「剩余时间：」;发布时间不再取标题区的促销到期时间 - 做种 / 下载 / 完成数、种子 Hash:umt.detail.fields.seeders / leechers / snatched / infoHash - 副标题去掉评分徽标;标签只取最内层;上传者取 span.torrent-uploader-wrap,标签精确匹配(避开「发布者带宽」) - 简介:去掉站点声明(page.descriptionStrip)与 MediaInfo 控件(codetop / show / hide),源码换行不再算一行 - 截图排除豆瓣海报与站点横幅 - 列表副标题改取 span.torrent-subtitle-text(以前前面粘着标签文字) - 脱敏真实样本:详情 2 页进 differential/detail.test.ts,列表 1 页进 MP 差分 Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
322ecb2f8c
commit
92830c3ef3
17 files changed
+1301
-69
No files matched your search
@@ -140,7 +140,7 @@
|
||||
}
|
||||
},
|
||||
"description": {
|
||||
"selector": "span.torrent-subtitle-line"
|
||||
"selector": "span.torrent-subtitle-text"
|
||||
},
|
||||
"category": {
|
||||
"selector": "a[href*=\"?cat=\"]",
|
||||
@@ -168,7 +168,7 @@
|
||||
}
|
||||
},
|
||||
"umt": {
|
||||
"version": "audiences@0.2.0-unverified",
|
||||
"version": "audiences@0.3.0",
|
||||
"kind": "nexusphp",
|
||||
"displayName": "观众",
|
||||
"timezone": "+08:00",
|
||||
@@ -225,7 +225,55 @@
|
||||
},
|
||||
"description": "#kdescr",
|
||||
"headerScope": "h1#top",
|
||||
"notFoundMarkers": ["没有该ID的种子", "No torrent with ID"]
|
||||
"notFoundMarkers": ["没有该ID的种子", "No torrent with ID"],
|
||||
"fields": {
|
||||
"seeders": "#seeders",
|
||||
"leechers": "#leechers",
|
||||
"snatched": "a[title='完成'][href*='viewsnatches.php']",
|
||||
"infoHash": "#copyTorrentHashLink::attr(data-hash)"
|
||||
},
|
||||
"promo": {
|
||||
"downloadvolumefactor": {
|
||||
"case": {
|
||||
"font.free": 0,
|
||||
"font.twoupfree": 0,
|
||||
"font.halfdown": 0.5,
|
||||
"font.twouphalfdown": 0.5,
|
||||
"font.thirtypercent": 0.3,
|
||||
"img.pro_free": 0,
|
||||
"img.pro_free2up": 0,
|
||||
"img.pro_50pctdown": 0.5,
|
||||
"img.pro_50pctdown2up": 0.5,
|
||||
"img.pro_30pctdown": 0.3,
|
||||
"*": 1
|
||||
}
|
||||
},
|
||||
"uploadvolumefactor": {
|
||||
"case": {
|
||||
"font.twoup": 2,
|
||||
"font.twoupfree": 2,
|
||||
"font.twouphalfdown": 2,
|
||||
"img.pro_50pctdown2up": 2,
|
||||
"img.pro_free2up": 2,
|
||||
"img.pro_2up": 2,
|
||||
"*": 1
|
||||
}
|
||||
},
|
||||
"freedate": {
|
||||
"selector": "img[class^=\"pro_\"], span[title]",
|
||||
"attribute": "title",
|
||||
"filters": [
|
||||
{
|
||||
"name": "re_search",
|
||||
"args": ["\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}", 0]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"page": {
|
||||
"uploader": "span.torrent-uploader-wrap",
|
||||
"descriptionStrip": "#ad_torrentdetail"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -139,8 +139,13 @@ const UmtNexusSchema = z.strictObject({
|
||||
imdb: z.string().optional(),
|
||||
douban: z.string().optional(),
|
||||
fileList: z.string().optional(),
|
||||
seeders: z.string().optional(),
|
||||
leechers: z.string().optional(),
|
||||
snatched: z.string().optional(),
|
||||
infoHash: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
promo: z.record(z.string(), SpiderFieldSelectorSchema).optional(),
|
||||
page: z
|
||||
.strictObject({
|
||||
labelSelectors: z.array(z.string()).optional(),
|
||||
@@ -150,6 +155,7 @@ const UmtNexusSchema = z.strictObject({
|
||||
uploader: z.string().optional(),
|
||||
files: z.string().optional(),
|
||||
fileListPath: z.string().optional(),
|
||||
descriptionStrip: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
})
|
||||
|
||||
@@ -8,6 +8,9 @@ import { extractDoubanId, extractImdbId } from "./util.ts";
|
||||
const NOT_SCREENSHOT =
|
||||
/(smilies|smiley|emoji|\/icons?\/|pic\/(?:trans|cattrans|flag)|\/flags?\/|avatar|\.svg(?:$|\?))/i;
|
||||
|
||||
/** 按完整地址排除:豆瓣海报(含站点的豆瓣图片代理 `?https://img…doubanio.com/…poster…`)与站点横幅装饰图 */
|
||||
const NOT_SCREENSHOT_URL = /(doubanio\.com\/view\/photo\/[a-z_]*poster|\/banners?\/)/i;
|
||||
|
||||
export const MAX_SCREENSHOTS = 20;
|
||||
export const MAX_FILES = 500;
|
||||
|
||||
@@ -36,7 +39,7 @@ export function screenshotUrl(src: string | null | undefined, base: string): str
|
||||
return null;
|
||||
}
|
||||
if (u.protocol !== "http:" && u.protocol !== "https:") return null;
|
||||
if (NOT_SCREENSHOT.test(u.pathname)) return null;
|
||||
if (NOT_SCREENSHOT.test(u.pathname) || NOT_SCREENSHOT_URL.test(u.href)) return null;
|
||||
return u.toString();
|
||||
}
|
||||
|
||||
|
||||
@@ -43,17 +43,35 @@ const normLabel = (t: string) =>
|
||||
.replace(/[:：]+$/, "")
|
||||
.toLowerCase();
|
||||
|
||||
/** 保留换行的文本:<br> 与块级元素换行 */
|
||||
/** 换行占位:先把 <br> / 块级换行记成它,再去掉 HTML 源码里的换行(浏览器按空白处理),最后还原 */
|
||||
const BR = "\u0000";
|
||||
|
||||
/**
|
||||
* 保留换行的文本:<br> 与块级元素换行。源码换行不算换行(观众的简介是 `<br />\r\n`,
|
||||
* 两者都算会让每行之间多一空行);<pre> 里的换行保留。
|
||||
*/
|
||||
function blockText($: CheerioAPI, el: El): string {
|
||||
const c = el.clone();
|
||||
c.find("script, style").remove();
|
||||
c.find("br").replaceWith("\n");
|
||||
c.find("div, p, li, tr, pre, fieldset, blockquote, h1, h2, h3, h4, h5, h6").each((_, n) => {
|
||||
$(n).append("\n");
|
||||
c.find("pre, textarea").each((_, n) => {
|
||||
$(n).text($(n).text().replace(/\r?\n/g, BR));
|
||||
});
|
||||
return tidyText(c.text());
|
||||
c.find("br").replaceWith(BR);
|
||||
c.find("div, p, li, tr, pre, fieldset, blockquote, h1, h2, h3, h4, h5, h6").each((_, n) => {
|
||||
$(n).append(BR);
|
||||
});
|
||||
const text = c
|
||||
.text()
|
||||
.replace(/\r?\n/g, " ")
|
||||
// `<br />\r\n下一行` 的源码换行变成了行首一个空格
|
||||
.replace(new RegExp(`${BR} `, "g"), BR)
|
||||
.replaceAll(BR, "\n");
|
||||
return tidyText(text);
|
||||
}
|
||||
|
||||
/** NexusPHP MediaInfo 控件的外壳:标题栏 div.codetop、摘要表 div.show、原文 div.hide(观众三者相邻) */
|
||||
const MI_WIDGET = "div.codetop, div.show, div.hide";
|
||||
|
||||
const SIZE_TAIL = /^(.*?)\s*[(（[]\s*([\d.,]+\s*[KMGTP]?i?B)\s*[)）\]]\s*$/i;
|
||||
const HAS_EXT = /\.[A-Za-z0-9]{2,5}$/;
|
||||
|
||||
@@ -113,8 +131,9 @@ export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string)
|
||||
};
|
||||
addLabels(conf.rowLabel, (e) => e.nextAll(conf.rowValue).first());
|
||||
for (const sel of p.labelSelectors ?? []) addLabels(sel, (e) => e.next());
|
||||
const byLabel = (names: string[]): El | null =>
|
||||
pairs.find((x) => names.some((n) => x.label.startsWith(normLabel(n))))?.value ?? null;
|
||||
const byLabel = (names: string[], exact = false): El | null =>
|
||||
pairs.find((x) => names.some((n) => (exact ? x.label === normLabel(n) : x.label.startsWith(normLabel(n)))))
|
||||
?.value ?? null;
|
||||
page.probe.labels = [...new Set(pairs.map((x) => x.label))].slice(0, 40);
|
||||
|
||||
// 简介容器
|
||||
@@ -170,10 +189,30 @@ export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string)
|
||||
page.probe.imageCount = imgs.length;
|
||||
page.screenshots = imgs.slice(0, MAX_SCREENSHOTS);
|
||||
|
||||
// 简介纯文本:去掉 MediaInfo 块与图片
|
||||
// 简介纯文本:去掉站点固定内容、MediaInfo 块(连同控件标题栏 / 摘要表)与图片
|
||||
if (desc?.length) {
|
||||
const c = desc.clone();
|
||||
if (miBlockIndex >= 0) c.find(MI_BLOCKS).eq(miBlockIndex).remove();
|
||||
if (miBlockIndex >= 0) {
|
||||
const isMi = (n: Parameters<typeof $>[0]) => looksLikeMediaInfo(blockText($, $(n)));
|
||||
// 同一份 MediaInfo 常出现两次(摘要 div.show 与原文 div.hide):另外去掉最内层的 MediaInfo 块,
|
||||
// 在控件外壳(div.codetop / div.show / div.hide)里时连同相邻外壳一起去掉
|
||||
const extra = c
|
||||
.find(MI_BLOCKS)
|
||||
.filter((_, n) => isMi(n) && !$(n).find(MI_BLOCKS).toArray().some(isMi))
|
||||
.map((_, n) => {
|
||||
const e = $(n);
|
||||
const shell = e.parent().is(MI_WIDGET) ? e.parent() : e;
|
||||
const out = [shell.get(0)!];
|
||||
if (!shell.is(MI_WIDGET)) return out;
|
||||
for (let x = shell.prev(); x.is(MI_WIDGET) && out.length < 3; x = x.prev()) out.push(x.get(0)!);
|
||||
for (let x = shell.next(); x.is(MI_WIDGET) && out.length < 4; x = x.next()) out.push(x.get(0)!);
|
||||
return out;
|
||||
})
|
||||
.get();
|
||||
c.find(MI_BLOCKS).eq(miBlockIndex).remove();
|
||||
for (const w of extra) $(w).remove();
|
||||
}
|
||||
if (p.descriptionStrip) c.find(p.descriptionStrip).remove();
|
||||
c.find("img").remove();
|
||||
const t = blockText($, c);
|
||||
if (t) page.descriptionText = t.slice(0, 20_000);
|
||||
@@ -221,17 +260,16 @@ export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string)
|
||||
// 标签
|
||||
const tagEl = p.tags ? $(p.tags) : byLabel(LABELS.tags);
|
||||
if (tagEl?.length) {
|
||||
const parts = tagEl.find("span, a").length
|
||||
? tagEl
|
||||
.find("span, a")
|
||||
.map((_, n) => $(n).text().trim())
|
||||
.get()
|
||||
: tagEl.text().split(/[\s,，|/]+/);
|
||||
// 只取最内层的 span / a(观众外面还包了一层 span.detail-tag-list,取外层会把全部标签连成一个)
|
||||
const leaves = tagEl.find("span, a").filter((_, n) => $(n).find("span, a").length === 0);
|
||||
const parts = leaves.length ? leaves.map((_, n) => $(n).text().trim()).get() : tagEl.text().split(/[\s,，|/]+/);
|
||||
page.tags = [...new Set(parts.map((t) => t.trim()).filter((t) => t && t.length <= 20))].slice(0, 20);
|
||||
}
|
||||
|
||||
// 上传者
|
||||
const upEl = p.uploader ? $(p.uploader).first() : byLabel(LABELS.uploader);
|
||||
// 上传者标签精确匹配:观众另有「发布者带宽」一行
|
||||
const upSel = p.uploader ? $(p.uploader).first() : null;
|
||||
const upEl = upSel?.length ? upSel : byLabel(LABELS.uploader, true);
|
||||
if (upEl?.length) {
|
||||
const a = upEl.is("a") ? upEl : upEl.find("a[href*='userdetails.php']").first();
|
||||
const name = (a.length ? a.text() : upEl.text()).replace(/\s+/g, " ").trim();
|
||||
|
||||
@@ -51,8 +51,15 @@ export function parseDetailPage(
|
||||
.text()
|
||||
)
|
||||
.replace(/\s+/g, " ")
|
||||
// 促销到期的「剩余时间：」是 h1 的直接文本节点(观众),不属于标题
|
||||
.replace(/\s*(?:剩余时间|剩餘時間|Remaining(?: time)?)\s*[:：]?\s*$/i, "")
|
||||
.trim();
|
||||
const subtitle = f.subtitle ? (mcSelect($, null, f.subtitle) ?? undefined) : row(conf.labels.subtitle)?.text().trim();
|
||||
const subtitleEl = f.subtitle ? null : row(conf.labels.subtitle)?.clone();
|
||||
// 副标题行右侧的豆瓣 / IMDb / 烂番茄等评分徽标(观众)不算副标题
|
||||
subtitleEl?.find("[title*='评分'], [title*='Rating'], img").remove();
|
||||
const subtitle = f.subtitle
|
||||
? (mcSelect($, null, f.subtitle) ?? undefined)
|
||||
: subtitleEl?.text().replace(/\s+/g, " ").trim();
|
||||
const basicText = row(conf.labels.basicInfo)?.text().replace(/\s+/g, " ") ?? "";
|
||||
const sizeText = f.size
|
||||
? (/([\d.,]+\s*[KMGTP]?i?B)/i.exec(mcSelect($, null, f.size) ?? "")?.[1] ?? null)
|
||||
@@ -64,6 +71,9 @@ export function parseDetailPage(
|
||||
const peersText = row(conf.labels.peers)?.text().replace(/\s+/g, " ") ?? "";
|
||||
const seeders = /(\d[\d,]*)\s*个?做种者|(\d[\d,]*)\s*seeder/i.exec(peersText);
|
||||
const leechers = /(\d[\d,]*)\s*个?下载者|(\d[\d,]*)\s*leecher/i.exec(peersText);
|
||||
// 站点把做种 / 下载 / 完成数放在其它位置时(观众:基本信息行右侧的 #seeders / #leechers / 完成链接)按选择器取
|
||||
const countOf = (sel: string | undefined) => (sel ? parseCount(mcSelect($, null, sel) ?? undefined) : null);
|
||||
const infoHash = mcSelect($, null, f.infoHash)?.toLowerCase();
|
||||
const descEl = $(conf.description).first();
|
||||
const descText = descEl.text().trim();
|
||||
const descHtml = descEl.html() ?? "";
|
||||
@@ -76,17 +86,22 @@ export function parseDetailPage(
|
||||
list: def.torrents.list,
|
||||
fields: Object.fromEntries(
|
||||
(["downloadvolumefactor", "uploadvolumefactor", "freedate", "hr"] as const)
|
||||
.filter((k) => fields[k])
|
||||
.map((k) => [k, fields[k]]),
|
||||
.map((k) => [k, conf.promo?.[k] ?? fields[k]] as const)
|
||||
.filter(([, v]) => v),
|
||||
),
|
||||
},
|
||||
browse: undefined,
|
||||
};
|
||||
const promo = new SiteSpider(promoDef).infoFrom($, $(conf.headerScope).first());
|
||||
// 发布时间:带完整时间的 span[title] → 「发布时间 / 上传时间 / 添加时间」字段的文本(憨憨是 div 标签 + 值)
|
||||
// 标题区(headerScope)里的 span[title] 是促销到期时间,不是发布时间
|
||||
const timeTitle =
|
||||
$("span[title]")
|
||||
.filter((_, el) => /\d{4}-\d{2}-\d{2} \d{2}:\d{2}/.test($(el).attr("title") ?? ""))
|
||||
.filter(
|
||||
(_, el) =>
|
||||
/\d{4}-\d{2}-\d{2} \d{2}:\d{2}/.test($(el).attr("title") ?? "") &&
|
||||
$(el).closest(conf.headerScope).length === 0,
|
||||
)
|
||||
.first()
|
||||
.attr("title") ??
|
||||
/(?:发布时间|上传时间|添加时间|Added)\s*[:：]?\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}(?::\d{2})?)/.exec(
|
||||
@@ -99,9 +114,9 @@ export function parseDetailPage(
|
||||
title,
|
||||
category: { id: catId, name: catText ?? categoryName(def, catId ?? undefined) },
|
||||
sizeBytes: sizeText ? parseSizeMp(sizeText) : null,
|
||||
seeders: seeders ? parseCount(seeders[1] ?? seeders[2]) : null,
|
||||
leechers: leechers ? parseCount(leechers[1] ?? leechers[2]) : null,
|
||||
snatched: null,
|
||||
seeders: seeders ? parseCount(seeders[1] ?? seeders[2]) : countOf(f.seeders),
|
||||
leechers: leechers ? parseCount(leechers[1] ?? leechers[2]) : countOf(f.leechers),
|
||||
snatched: countOf(f.snatched),
|
||||
promotion: promotionFromFactors(
|
||||
promo.downloadvolumefactor,
|
||||
promo.uploadvolumefactor,
|
||||
@@ -116,6 +131,7 @@ export function parseDetailPage(
|
||||
observedAt,
|
||||
};
|
||||
if (subtitle) detail.subtitle = subtitle;
|
||||
if (infoHash && /^[0-9a-f]{40}$/.test(infoHash)) detail.infoHash = infoHash;
|
||||
if (descText) detail.descriptionText = descText.slice(0, 20_000);
|
||||
const imdb = extractImdbId(mcSelect($, null, f.imdb)) ?? extractImdbId(descHtml) ?? extractImdbId(descText);
|
||||
if (imdb) detail.imdbId = imdb;
|
||||
|
||||
@@ -196,8 +196,27 @@ export interface UmtSiteExtension {
|
||||
* 用于 HHClub 这类 div 详情页。
|
||||
*/
|
||||
fields?: Partial<
|
||||
Record<"title" | "subtitle" | "size" | "description" | "download" | "imdb" | "douban" | "fileList", string>
|
||||
Record<
|
||||
| "title"
|
||||
| "subtitle"
|
||||
| "size"
|
||||
| "description"
|
||||
| "download"
|
||||
| "imdb"
|
||||
| "douban"
|
||||
| "fileList"
|
||||
| "seeders"
|
||||
| "leechers"
|
||||
| "snatched"
|
||||
| "infoHash",
|
||||
string
|
||||
>
|
||||
>;
|
||||
/**
|
||||
* 详情页促销 / H&R 字段(downloadvolumefactor / uploadvolumefactor / freedate / hr),覆盖同名列表字段。
|
||||
* 详情页头部的促销标记常与列表不同(列表 img.pro_free,详情 h1 里 font.free + 「剩余时间」span)。
|
||||
*/
|
||||
promo?: Record<string, SpiderFieldSelector>;
|
||||
/**
|
||||
* 详情页扩展内容(简介 / MediaInfo / 截图 / 文件 / 标签 / 上传者)的站点特化,均可省略(走通用规则,见 detail-page.ts)。
|
||||
* 选择器为普通 CSS。
|
||||
@@ -216,6 +235,8 @@ export interface UmtSiteExtension {
|
||||
* 每次详情刷新最多 1 次,失败不影响详情。
|
||||
*/
|
||||
fileListPath?: string;
|
||||
/** 从简介里去掉的站点固定内容(如站点声明),普通 CSS */
|
||||
descriptionStrip?: string;
|
||||
};
|
||||
};
|
||||
/** 下载路径(按资源 ID 拼接) */
|
||||
|
||||
Reference in new issue
Block a user