feat(adapters): RSS and PT site adapters ported from MoviePilot v3 + mocks (W1-3b)
- rss: port of MP RssHelper._parse_impl/get_rss_link semantics with safe XML (entities off, DTD/ENTITY rejected, size/item caps), ETag/Last-Modified, backoff + Retry-After, circuit breaker with manual reset, passkey redaction, overlap-window GUID+digest dedupe - sites/spider: TS port of MP SiteSpider (indexer schema, search URL builder, field DSL, filters, Jinja templates via nunjucks, login-page detection); audiences definition hand-written as JSON; raw category id preserved; shared per-site token bucket - sites/mteam: port of MP MTorrentSpider (params, result mapping, promo factors, genDlToken flow) without embedding the API key in enclosures; business code/message checks - testkit: mock RSS server, mock NexusPHP site, mock M-Team API - docs: W1-3b migration mapping appended to porting ledger Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
1 parent
ef2e504a94
commit
a6d049db81
47 files changed
+5414
No files matched your search
@@ -0,0 +1,110 @@
|
||||
/**
|
||||
* 详情页解析 —— 本项目扩展。MP 通用 SiteSpider 不解析详情页(MP 只在列表/搜索结果上工作),
|
||||
* 这里按 NexusPHP details.php 公开结构(h1#top + td.rowhead/td.rowfollow)实现,选择器在站点定义 `umt.detail` 中配置。
|
||||
* 促销/H&R 复用站点定义中 downloadvolumefactor / uploadvolumefactor / freedate / hr 字段(与列表同语义)。
|
||||
*/
|
||||
import type { CheerioAPI } from "cheerio";
|
||||
import type { SiteReleaseDetail } from "../types";
|
||||
import { extractDoubanId, extractImdbId, parseCount, parseSiteTime } from "../util";
|
||||
import { categoryName, promotionFromFactors } from "./map";
|
||||
import type { IndexerDefinition } from "./schema";
|
||||
import { loadHtml, parseSizeMp, SiteSpider } from "./spider";
|
||||
|
||||
export type DetailPageKind = "ok" | "login" | "not_found" | "unknown";
|
||||
|
||||
export function parseDetailPage(
|
||||
html: string,
|
||||
def: IndexerDefinition,
|
||||
id: string,
|
||||
observedAt: string,
|
||||
): { kind: DetailPageKind; detail: SiteReleaseDetail | null } {
|
||||
const conf = def.umt.detail;
|
||||
if (!conf) return { kind: "unknown", detail: null };
|
||||
const $ = loadHtml(html);
|
||||
if (SiteSpider.isLoginOrPermissionPage($)) return { kind: "login", detail: null };
|
||||
const bodyHead = $("body").text().replace(/\s+/g, " ").slice(0, 2000);
|
||||
if (conf.notFoundMarkers.some((m) => bodyHead.includes(m))) return { kind: "not_found", detail: null };
|
||||
const h1 = $(conf.title).first();
|
||||
if (h1.length === 0) return { kind: "unknown", detail: null };
|
||||
|
||||
const row = (labels: string[]) => {
|
||||
let found: ReturnType<CheerioAPI> | null = null;
|
||||
$(conf.rowLabel).each((_, el) => {
|
||||
const label = $(el).text().replace(/\s+/g, "");
|
||||
if (!found && labels.some((l) => label.startsWith(l.replace(/\s+/g, "")))) {
|
||||
found = $(el).nextAll(conf.rowValue).first();
|
||||
}
|
||||
});
|
||||
return found as ReturnType<CheerioAPI> | null;
|
||||
};
|
||||
|
||||
const title = h1
|
||||
.contents()
|
||||
.filter((_, n) => n.type === "text")
|
||||
.text()
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
const subtitle = row(conf.labels.subtitle)?.text().trim();
|
||||
const basicText = row(conf.labels.basicInfo)?.text().replace(/\s+/g, " ") ?? "";
|
||||
const sizeText = /(?:大小|Size)\s*[::]\s*([\d.,]+\s*[KMGTP]?i?B)/i.exec(basicText)?.[1] ?? null;
|
||||
const catText = /(?:类型|Type)\s*[::]\s*(\S+)/i.exec(basicText)?.[1] ?? null;
|
||||
const catId = catText
|
||||
? (Object.entries(def.umt.categoryNames ?? {}).find(([, name]) => name === catText)?.[0] ?? null)
|
||||
: null;
|
||||
const peersText = row(conf.labels.peers)?.text().replace(/\s+/g, " ") ?? "";
|
||||
const seeders = /(\d[\d,]*)\s*个?做种者|(\d[\d,]*)\s*seeder/i.exec(peersText);
|
||||
const leechers = /(\d[\d,]*)\s*个?下载者|(\d[\d,]*)\s*leecher/i.exec(peersText);
|
||||
const descEl = $(conf.description).first();
|
||||
const descText = descEl.text().trim();
|
||||
const descHtml = descEl.html() ?? "";
|
||||
|
||||
// 促销/HR:只用这四个字段在详情页头部取值
|
||||
const fields = def.torrents.fields;
|
||||
const promoDef: IndexerDefinition = {
|
||||
...def,
|
||||
torrents: {
|
||||
list: def.torrents.list,
|
||||
fields: Object.fromEntries(
|
||||
(["downloadvolumefactor", "uploadvolumefactor", "freedate", "hr"] as const)
|
||||
.filter((k) => fields[k])
|
||||
.map((k) => [k, fields[k]]),
|
||||
),
|
||||
},
|
||||
browse: undefined,
|
||||
};
|
||||
const promo = new SiteSpider(promoDef).infoFrom($, $(conf.headerScope).first());
|
||||
const timeTitle = $("span[title]")
|
||||
.filter((_, el) => /\d{4}-\d{2}-\d{2} \d{2}:\d{2}/.test($(el).attr("title") ?? ""))
|
||||
.first()
|
||||
.attr("title");
|
||||
|
||||
const detail: SiteReleaseDetail = {
|
||||
siteId: def.id,
|
||||
siteReleaseId: id,
|
||||
title,
|
||||
category: { id: catId, name: catText ?? categoryName(def, catId ?? undefined) },
|
||||
sizeBytes: sizeText ? parseSizeMp(sizeText) : null,
|
||||
seeders: seeders ? parseCount(seeders[1] ?? seeders[2]) : null,
|
||||
leechers: leechers ? parseCount(leechers[1] ?? leechers[2]) : null,
|
||||
snatched: null,
|
||||
promotion: promotionFromFactors(
|
||||
promo.downloadvolumefactor,
|
||||
promo.uploadvolumefactor,
|
||||
promo.freedate ? parseSiteTime(promo.freedate, def.umt.timezone) : null,
|
||||
),
|
||||
hr: promo.hit_and_run === undefined ? "unknown" : promo.hit_and_run,
|
||||
uploadedAt: parseSiteTime(timeTitle, def.umt.timezone),
|
||||
detailUrl: new URL(`${def.umt.detailPath}?id=${id}`, def.domain).toString(),
|
||||
downloadUrl: new URL(`${def.umt.downloadPath}?id=${id}`, def.domain).toString(),
|
||||
tags: [],
|
||||
raw: { basicInfoText: basicText.trim().slice(0, 500), peersText: peersText.trim(), ...promo },
|
||||
observedAt,
|
||||
};
|
||||
if (subtitle) detail.subtitle = subtitle;
|
||||
if (descText) detail.descriptionText = descText.slice(0, 20_000);
|
||||
const imdb = extractImdbId(descHtml) ?? extractImdbId(descText);
|
||||
if (imdb) detail.imdbId = imdb;
|
||||
const douban = extractDoubanId(descHtml);
|
||||
if (douban) detail.doubanId = douban;
|
||||
return { kind: "ok", detail };
|
||||
}
|
||||
Reference in new issue
Block a user