fix(sites): normalize HHClub RSS items, complete detail page parsing (HH-2)
- RSS: split NexusPHP "[category]name[subtitle][size][uploader]" titles in the site adapter layer (category id/name/media type, clean title, subtitle, size); snapshot fields win on merge; lazily renormalize old RSS rows and re-emit release.observed; unknown list fields show an em dash - Detail: HHClub section layout (douban info block as description, screenshots section, IMDb/TMDB from MediaInfo lines, upload time field, on-demand viewfilelist.php once); site CDN screenshots get a Referer - Site request count now includes site-bound RSS fetches - Origin view shows "自动化" when idle and "Agent" only after recent automation actions (lastAutomationAt); agent step titles no longer get squeezed to a sliver by long notes Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
4a212ed19e
commit
cabd772243
32 files changed
+810
-59
No files matched your search
@@ -96,7 +96,10 @@ describe.skipIf(!up)("e2e:登录失效 → 人工接管 → 交还 → Agent 重
|
||||
const fresh = await invoke("browser.snapshot", { siteId: h.siteId });
|
||||
expect(fresh).toMatchObject({ controller: "automation", loginRequired: false, waitingReason: null });
|
||||
expect(fresh.snapshot.text).toContain("种子列表");
|
||||
expect((await h.req(`/browser/sessions/${sid}`)).body.data.needsSnapshot).toBe(false);
|
||||
const afterAgent = (await h.req(`/browser/sessions/${sid}`)).body.data;
|
||||
expect(afterAgent.needsSnapshot).toBe(false);
|
||||
// 原站视图据此显示「Agent 正在操作」;人工动作不刷新它
|
||||
expect(Date.now() - Date.parse(afterAgent.lastAutomationAt)).toBeLessThan(60_000);
|
||||
const dl = await invoke("browser.download", { sessionId: sid, target: { ref: "e11" }, timeoutMs: 2_000 });
|
||||
expect(dl).toMatchObject({
|
||||
sourceReleaseId: h.releaseId,
|
||||
|
||||
@@ -147,6 +147,8 @@ export interface RecoveryState {
|
||||
downloads?: { sha256: string; artifactId: string; sourceReleaseId: string | null; planId: string | null }[];
|
||||
/** BR-2:打开时的凭据注入结果与最近一次回写检查 */
|
||||
credential?: BrowserCredentialState;
|
||||
/** 最近一次自动化(Agent 能力)动作成功的时间;原站视图据此区分「Agent 正在操作」与「无人操作」 */
|
||||
lastAutomationAt?: string;
|
||||
}
|
||||
|
||||
/** 有活动的会话每隔多久从浏览器读回一次 Cookie */
|
||||
@@ -641,6 +643,7 @@ export class BrowserService {
|
||||
needsSnapshot: rs.needsSnapshot === true,
|
||||
waitingReason: row.controller === "waiting" || row.controller === "human" ? (rs.waitingReason ?? null) : null,
|
||||
credential: rs.credential ?? null,
|
||||
lastAutomationAt: rs.lastAutomationAt ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1193,7 +1196,10 @@ export class BrowserService {
|
||||
await this.persist(id, loaded.provider).catch(() => {});
|
||||
throw fromBrowserError(e);
|
||||
}
|
||||
if (opts.persist !== false) await this.persist(id, loaded.provider, opts.patch?.(out) ?? {});
|
||||
const automation: RecoveryState = actor.kind === "automation" ? { lastAutomationAt: this.now().toISOString() } : {};
|
||||
if (opts.persist !== false)
|
||||
await this.persist(id, loaded.provider, { ...(opts.patch?.(out) ?? {}), ...automation });
|
||||
else if (automation.lastAutomationAt) await this.patchRecovery(await this.sessionRow(id), automation);
|
||||
this.maybeSyncCookies(row);
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import { errorBody } from "../../../http/errors.ts";
|
||||
import type { ModuleApp } from "../../types.ts";
|
||||
import { route } from "../http.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { detailScreenshotUrl } from "../service/detail-page.ts";
|
||||
import { detailScreenshotRequest } from "../service/detail-page.ts";
|
||||
import { fetchPublicImage, imageResponse } from "../service/image-proxy.ts";
|
||||
import { listSiteReleases, releaseDetail } from "../service/workbench.ts";
|
||||
|
||||
@@ -53,9 +53,13 @@ export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
return c.json(errorBody("bad_request", "截图地址无效"), 400);
|
||||
}
|
||||
const r = rt();
|
||||
const url = await detailScreenshotUrl(r, rid, index);
|
||||
if (!url) return c.json(errorBody("not_found", "截图不存在或详情缓存已过期"), 404);
|
||||
const got = await fetchPublicImage(r.overrides.fetch ?? fetch, url, { maxBytes: SHOT_MAX_BYTES, label: "截图" });
|
||||
const req = await detailScreenshotRequest(r, rid, index);
|
||||
if (!req) return c.json(errorBody("not_found", "截图不存在或详情缓存已过期"), 404);
|
||||
const got = await fetchPublicImage(r.overrides.fetch ?? fetch, req.url, {
|
||||
maxBytes: SHOT_MAX_BYTES,
|
||||
label: "截图",
|
||||
...(req.referer ? { referer: req.referer } : {}),
|
||||
});
|
||||
if (!got.ok) return c.json(errorBody("upstream_unavailable", got.message), 502);
|
||||
return imageResponse(got, 86_400);
|
||||
});
|
||||
|
||||
@@ -63,6 +63,38 @@ export async function detailScreenshotUrl(rt: SourcesRuntime, srId: string, inde
|
||||
return (await loadStored(rt, srId))?.screenshots[index] ?? null;
|
||||
}
|
||||
|
||||
/** 根域(去 www.、取最后两级):pic.hhanclub.net 与 hhanclub.net 视为同站 */
|
||||
function rootHost(u: string): string | null {
|
||||
try {
|
||||
const host = new URL(u).hostname.toLowerCase().replace(/^www\./, "");
|
||||
return /^[\d.]+$/.test(host) || host.includes(":") ? host : host.split(".").slice(-2).join(".");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 截图代理请求:原始地址 + 站点自家图床(同根域,如 pic.hhanclub.net / proxy-cover.hhanclub.net)要带的 Referer。
|
||||
* 站点 CDN 常做防盗链,只认本站 Referer;外部图床不带 Referer。任何情况下都不带 Cookie。
|
||||
*/
|
||||
export async function detailScreenshotRequest(
|
||||
rt: SourcesRuntime,
|
||||
srId: string,
|
||||
index: number,
|
||||
): Promise<{ url: string; referer?: string } | null> {
|
||||
const url = await detailScreenshotUrl(rt, srId, index);
|
||||
if (!url) return null;
|
||||
const site = await rt.ctx.db
|
||||
.selectFrom("source_release as r")
|
||||
.innerJoin("site as s", "s.id", "r.site_id")
|
||||
.select(["s.base_url"])
|
||||
.where("r.id", "=", srId)
|
||||
.executeTakeFirst();
|
||||
const siteRoot = site ? rootHost(site.base_url) : null;
|
||||
if (siteRoot && rootHost(url) === siteRoot) return { url, referer: `${new URL(site!.base_url).origin}/` };
|
||||
return { url };
|
||||
}
|
||||
|
||||
export async function loadDetailPage(rt: SourcesRuntime, srId: string): Promise<SiteReleasePage | null> {
|
||||
const p = await loadStored(rt, srId);
|
||||
if (!p) return null;
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
type FeedFetchState,
|
||||
type FeedItem,
|
||||
getSiteDefinition,
|
||||
normalizeSiteFeedItem,
|
||||
} from "@umt/adapters";
|
||||
import type { FeedHealth } from "@umt/contracts";
|
||||
import { type FeedSourceRow, type SiteRow, sql } from "@umt/db";
|
||||
@@ -157,6 +158,9 @@ export async function ingestFeed(
|
||||
|
||||
const url = await secrets.resolve(feed.url_secret_ref);
|
||||
const result = await rt.rss.fetchFeed(url, stateFromRow(feed), { ignoreBackoff: !!opts.manual });
|
||||
// 挂站点的 Feed:每次真正发出的 RSS 请求也记进站点请求计数(熔断 / 退避跳过的不算)
|
||||
if (site && result.kind !== "skipped") rt.countSiteRequest(site.id);
|
||||
if (site) await renormalizeFeedReleases(rt, site).catch(() => {});
|
||||
const now = rt.now();
|
||||
const auth = isAuthFailure(result);
|
||||
const health: FeedHealth = auth ? "failing" : healthOf(result);
|
||||
@@ -175,7 +179,7 @@ export async function ingestFeed(
|
||||
if (result.kind === "ok") {
|
||||
const applied = applyFeedItems(window, result.feed.items);
|
||||
window = applied.cursor;
|
||||
const counts = await writeItems(rt, trx, feed, site?.id ?? null, applied.newItems, applied.updatedItems);
|
||||
const counts = await writeItems(rt, trx, feed, site ?? null, applied.newItems, applied.updatedItems);
|
||||
const unchanged = result.feed.items
|
||||
.filter((i) => !applied.newItems.includes(i) && !applied.updatedItems.includes(i))
|
||||
.map((i) => i.guid);
|
||||
@@ -312,16 +316,36 @@ function observationRaw(item: FeedItem) {
|
||||
};
|
||||
}
|
||||
|
||||
/** 规范化后的 RSS 条目字段(写进 source_release.raw.rss,v 为规范化版本) */
|
||||
export const RSS_NORMALIZE_VERSION = 1;
|
||||
|
||||
function feedFields(siteKey: string, item: Pick<FeedItem, "title" | "size" | "categories">) {
|
||||
const n = normalizeSiteFeedItem(siteKey, item.title);
|
||||
// 分类:标题前缀 → RSS <category>(可能是名称)
|
||||
const catName = n.category.name ?? item.categories[0] ?? null;
|
||||
return {
|
||||
title: n.title,
|
||||
subtitle: n.subtitle,
|
||||
categoryRaw: n.category.id ?? catName,
|
||||
categoryName: catName,
|
||||
mediaType: n.mediaType,
|
||||
// enclosure length 是精确字节数,标题尾部的大小是四舍五入后的显示值
|
||||
sizeBytes: item.size > 0 ? item.size : n.sizeBytes,
|
||||
uploader: n.uploader,
|
||||
};
|
||||
}
|
||||
|
||||
async function writeItems(
|
||||
rt: SourcesRuntime,
|
||||
trx: Exec,
|
||||
feed: FeedSourceRow,
|
||||
siteId: string | null,
|
||||
site: Pick<SiteRow, "id" | "key"> | null,
|
||||
newItems: FeedItem[],
|
||||
updatedItems: FeedItem[],
|
||||
): Promise<{ newReleases: number }> {
|
||||
let newReleases = 0;
|
||||
const updated = new Set(updatedItems);
|
||||
const siteId = site?.id ?? null;
|
||||
for (const item of [...newItems, ...updatedItems]) {
|
||||
const raw = observationRaw(item);
|
||||
const obs = (
|
||||
@@ -337,7 +361,7 @@ async function writeItems(
|
||||
returning id, source_release_id, (select content_digest from prev) as prev_digest`.execute(trx)
|
||||
).rows[0]!;
|
||||
|
||||
if (!siteId) {
|
||||
if (!site || !siteId) {
|
||||
// 无站点的通用 Feed:只留观察,无法按站点资源聚合
|
||||
await trx
|
||||
.updateTable("observation")
|
||||
@@ -349,20 +373,36 @@ async function writeItems(
|
||||
const srid =
|
||||
extractSiteReleaseId(item) ?? `guid:${createHash("sha1").update(item.guid).digest("hex").slice(0, 20)}`;
|
||||
const contentChanged = updated.has(item) || (obs.prev_digest !== null && obs.prev_digest !== item.rawDigest);
|
||||
const sizeBytes = item.size > 0 ? item.size : null;
|
||||
const f = feedFields(site.key, item);
|
||||
const rssRaw = {
|
||||
link: item.link,
|
||||
categories: item.categories,
|
||||
rawTitle: item.title,
|
||||
categoryName: f.categoryName,
|
||||
mediaType: f.mediaType,
|
||||
...(f.uploader ? { uploader: f.uploader } : {}),
|
||||
v: RSS_NORMALIZE_VERSION,
|
||||
};
|
||||
// 与列表快照按站点资源 ID 合并:快照见过(raw._site)的行,标题 / 副标题 / 分类 / 做种 / 促销 / H&R 以快照为准,
|
||||
// RSS 只补空缺;快照没见过的行,内容变化时用 RSS 的新值
|
||||
const rel = (
|
||||
await sql<{ id: string; inserted: boolean; prev_title: string | null; resolution_locked: boolean }>`
|
||||
with prev as (select title from source_release where site_id = ${siteId} and site_release_id = ${srid})
|
||||
insert into source_release (site_id, site_release_id, title, category_raw, size_bytes, source_published_at, raw)
|
||||
values (${siteId}, ${srid}, ${item.title}, ${item.categories[0] ?? null}, ${sizeBytes}, ${item.publishedAt ?? null},
|
||||
${jsonb({ rss: { link: item.link, categories: item.categories } })}::jsonb)
|
||||
insert into source_release (site_id, site_release_id, title, subtitle, category_raw, size_bytes, source_published_at, raw)
|
||||
values (${siteId}, ${srid}, ${f.title}, ${f.subtitle}, ${f.categoryRaw}, ${f.sizeBytes}, ${item.publishedAt ?? null},
|
||||
${jsonb({ rss: rssRaw })}::jsonb)
|
||||
on conflict (site_id, site_release_id) do update set
|
||||
last_seen_at = now(),
|
||||
title = case when ${contentChanged} then excluded.title else source_release.title end,
|
||||
title = case when source_release.raw ? '_site' then source_release.title
|
||||
when ${contentChanged} then excluded.title else source_release.title end,
|
||||
subtitle = case when source_release.raw ? '_site' then coalesce(source_release.subtitle, excluded.subtitle)
|
||||
when ${contentChanged} then coalesce(excluded.subtitle, source_release.subtitle)
|
||||
else coalesce(source_release.subtitle, excluded.subtitle) end,
|
||||
size_bytes = case when ${contentChanged} then coalesce(excluded.size_bytes, source_release.size_bytes)
|
||||
else coalesce(source_release.size_bytes, excluded.size_bytes) end,
|
||||
source_published_at = coalesce(source_release.source_published_at, excluded.source_published_at),
|
||||
category_raw = coalesce(source_release.category_raw, excluded.category_raw)
|
||||
category_raw = coalesce(source_release.category_raw, excluded.category_raw),
|
||||
raw = source_release.raw || excluded.raw
|
||||
returning id, (xmax = 0) as inserted, (select title from prev) as prev_title, resolution_locked`.execute(trx)
|
||||
).rows[0]!;
|
||||
await trx
|
||||
@@ -379,7 +419,7 @@ async function writeItems(
|
||||
{ sourceReleaseId: rel.id, observationId: obs.id, siteId, isNew: true },
|
||||
{ idempotencyKey: `release.observed:${rel.id}` },
|
||||
);
|
||||
} else if (contentChanged && rel.prev_title !== item.title && !rel.resolution_locked) {
|
||||
} else if (contentChanged && rel.prev_title !== f.title && !rel.resolution_locked) {
|
||||
// 标题修正:按新标题重新识别(证据版本变化)
|
||||
await rt.ctx.outbox.emit(
|
||||
trx,
|
||||
@@ -391,3 +431,50 @@ async function writeItems(
|
||||
}
|
||||
return { newReleases };
|
||||
}
|
||||
|
||||
/**
|
||||
* 旧 RSS 行补规范化:规范化上线前写进来的站点 RSS 行(raw.rss 没有 v,标题还是 `[分类]…[大小]` 整串),
|
||||
* 在该站 Feed 下次抓取时按原标题重算一次(每次最多 500 行,不请求站点)。快照见过的行不动。
|
||||
* 标题变了且未锁定的,重新发 release.observed 让身份解析按干净标题再识别。
|
||||
*/
|
||||
export async function renormalizeFeedReleases(rt: SourcesRuntime, site: Pick<SiteRow, "id" | "key">): Promise<number> {
|
||||
const rows = await rt.ctx.db
|
||||
.selectFrom("source_release")
|
||||
.select(["id", "title", "subtitle", "category_raw", "size_bytes", "raw", "resolution_locked"])
|
||||
.where("site_id", "=", site.id)
|
||||
.where(sql<boolean>`raw ? 'rss'`)
|
||||
.where(sql<boolean>`not (raw ? '_site')`)
|
||||
.where(sql<boolean>`coalesce((raw->'rss'->>'v')::int, 0) < ${RSS_NORMALIZE_VERSION}`)
|
||||
.limit(500)
|
||||
.execute();
|
||||
if (!rows.length) return 0;
|
||||
await rt.ctx.db.transaction().execute(async (trx) => {
|
||||
for (const r of rows) {
|
||||
const rss = ((r.raw ?? {}) as { rss?: Record<string, unknown> }).rss ?? {};
|
||||
const rawTitle = typeof rss.rawTitle === "string" ? rss.rawTitle : r.title;
|
||||
const categories = Array.isArray(rss.categories) ? (rss.categories as string[]) : [];
|
||||
const f = feedFields(site.key, { title: rawTitle, size: Number(r.size_bytes ?? 0), categories });
|
||||
const nextRss = {
|
||||
...rss,
|
||||
rawTitle,
|
||||
categoryName: f.categoryName,
|
||||
mediaType: f.mediaType,
|
||||
...(f.uploader ? { uploader: f.uploader } : {}),
|
||||
v: RSS_NORMALIZE_VERSION,
|
||||
};
|
||||
await sql`update source_release set title = ${f.title}, subtitle = coalesce(subtitle, ${f.subtitle}),
|
||||
category_raw = coalesce(${f.categoryRaw}, category_raw), size_bytes = coalesce(size_bytes, ${f.sizeBytes}),
|
||||
raw = raw || jsonb_build_object('rss', ${JSON.stringify(nextRss)}::jsonb)
|
||||
where id = ${r.id}`.execute(trx);
|
||||
if (f.title !== r.title && !r.resolution_locked) {
|
||||
await rt.ctx.outbox.emit(
|
||||
trx,
|
||||
"release.observed",
|
||||
{ sourceReleaseId: r.id, observationId: null, siteId: site.id, isNew: false },
|
||||
{ idempotencyKey: `release.observed:${r.id}:rss-norm-${RSS_NORMALIZE_VERSION}` },
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
return rows.length;
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
import type { SiteRelease } from "@umt/adapters";
|
||||
import { createMockRssServer, type MockFeedItem, type MockRssServer } from "@umt/testkit";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { ingestFeed } from "../../src/modules/sources-identity/service/feeds.ts";
|
||||
import { upsertSiteRelease } from "../../src/modules/sources-identity/service/snapshot.ts";
|
||||
import { listSiteReleases } from "../../src/modules/sources-identity/service/workbench.ts";
|
||||
import { createHarness, type Harness, pgAvailable, seedFeed, seedSite } from "./harness.ts";
|
||||
|
||||
// HH-2:憨憨 RSS 条目规范化(`[分类]种子名[副标题][大小]`)、与列表快照合并、旧行补规范化、RSS 请求计数
|
||||
const up = await pgAvailable();
|
||||
const BASE = "https://hhanclub.example";
|
||||
|
||||
const item = (id: string, title: string, pubDate: string, extra: Partial<MockFeedItem> = {}): MockFeedItem => ({
|
||||
guid: `hh-${id}`,
|
||||
title,
|
||||
link: `${BASE}/details.php?id=${id}`,
|
||||
enclosureUrl: `${BASE}/download.php?id=${id}&passkey=PK`,
|
||||
size: 731525231,
|
||||
pubDate: new Date(pubDate),
|
||||
categories: [],
|
||||
...extra,
|
||||
});
|
||||
|
||||
const snapshotRelease = (id: string, title: string, uploadedAt: string): SiteRelease => ({
|
||||
siteId: "hhanclub",
|
||||
siteReleaseId: id,
|
||||
title,
|
||||
subtitle: "快照副标题",
|
||||
category: { id: "402", name: "电视剧" },
|
||||
sizeBytes: 1024,
|
||||
seeders: 22,
|
||||
leechers: 66,
|
||||
snatched: 5,
|
||||
promotion: { type: "free", downloadFactor: 0, uploadFactor: 1, expiresAt: null },
|
||||
hr: false,
|
||||
uploadedAt,
|
||||
detailUrl: `${BASE}/details.php?id=${id}`,
|
||||
tags: [],
|
||||
raw: {},
|
||||
observedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
describe.skipIf(!up)("站点 RSS 条目规范化(HH-2)", () => {
|
||||
let h: Harness;
|
||||
let siteId: string;
|
||||
const servers: MockRssServer[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
h = await createHarness("rssnorm");
|
||||
siteId = (await seedSite(h, { key: "hhanclub", baseUrl: `${BASE}/`, cookie: "c=1" })).site.id;
|
||||
});
|
||||
afterAll(async () => {
|
||||
for (const s of servers) await s.close();
|
||||
await h?.close();
|
||||
});
|
||||
|
||||
const feed = async (items: MockFeedItem[]) => {
|
||||
const s = await createMockRssServer({ items, passkey: "PK" });
|
||||
servers.push(s);
|
||||
return { server: s, row: await seedFeed(h, { siteId, url: s.feedUrl, name: "憨憨 RSS" }) };
|
||||
};
|
||||
const rel = (id: string) =>
|
||||
h.db
|
||||
.selectFrom("source_release")
|
||||
.selectAll()
|
||||
.where("site_id", "=", siteId)
|
||||
.where("site_release_id", "=", id)
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
it("拆出分类 / 副标题 / 干净标题;大小取 enclosure 精确值;原始标题留在 raw.rss;请求计入站点计数", async () => {
|
||||
const { row } = await feed([
|
||||
item(
|
||||
"219390",
|
||||
"[音乐]Travis - L.A. Times (2024) - WEB-DL - 24bit ALAC-HHWEB[Travis - L.A. Times | ALAC | 苹果音乐][697.64 MB]",
|
||||
"2026-09-29T08:51:01Z",
|
||||
),
|
||||
item("219391", "[纪录片]Some Doc 2026 2160p WEB-DL-HHWEB[新][某纪录片][1.5 GB]", "2026-09-29T08:10:00Z", {
|
||||
size: 0,
|
||||
}),
|
||||
]);
|
||||
expect((await ingestFeed(h.rt, row.id)).status).toBe("ok");
|
||||
const r = await rel("219390");
|
||||
expect(r).toMatchObject({
|
||||
title: "Travis - L.A. Times (2024) - WEB-DL - 24bit ALAC-HHWEB",
|
||||
subtitle: "Travis - L.A. Times | ALAC | 苹果音乐",
|
||||
category_raw: "410",
|
||||
});
|
||||
expect(Number(r.size_bytes)).toBe(731525231);
|
||||
expect((r.raw as { rss: Record<string, unknown> }).rss).toMatchObject({
|
||||
categoryName: "音乐",
|
||||
mediaType: "music",
|
||||
rawTitle: expect.stringContaining("[697.64 MB]"),
|
||||
v: 1,
|
||||
});
|
||||
const doc = await rel("219391");
|
||||
expect(doc).toMatchObject({ title: "Some Doc 2026 2160p WEB-DL-HHWEB", subtitle: "某纪录片", category_raw: "404" });
|
||||
expect(Number(doc.size_bytes)).toBe(Math.round(1.5 * 1024 ** 3));
|
||||
|
||||
// 计数是异步写的,等一下
|
||||
let n = 0;
|
||||
for (let i = 0; i < 50 && n === 0; i++) {
|
||||
n = await h.rt.siteRequestCount(siteId);
|
||||
if (!n) await new Promise((res) => setTimeout(res, 20));
|
||||
}
|
||||
expect(n).toBe(1);
|
||||
});
|
||||
|
||||
it("与列表快照按站点资源 ID 合并:快照补做种 / 促销 / H&R,之后 RSS 更新不再覆盖快照标题", async () => {
|
||||
const { server, row } = await feed([
|
||||
item("219380", "[电视剧]Show S01 2026 1080p WEB-DL-HHWEB[示例剧 | 第1集][1.2 GB]", "2026-09-29T07:00:00Z"),
|
||||
]);
|
||||
await ingestFeed(h.rt, row.id);
|
||||
await h.db
|
||||
.transaction()
|
||||
.execute((trx) =>
|
||||
upsertSiteRelease(
|
||||
h.rt,
|
||||
trx,
|
||||
{ id: siteId },
|
||||
snapshotRelease("219380", "Show S01 2026 1080p WEB-DL-HHWEB", "2026-09-29T07:00:00Z"),
|
||||
),
|
||||
);
|
||||
let r = await rel("219380");
|
||||
expect(r).toMatchObject({ seeders: 22, leechers: 66, subtitle: "快照副标题", category_raw: "402" });
|
||||
expect((r.raw as { _site: { hr: boolean } })._site.hr).toBe(false);
|
||||
expect((r.raw as { rss: { categoryName: string } }).rss.categoryName).toBe("电视剧");
|
||||
|
||||
server.updateItem("hh-219380", { title: "[电视剧]Show S01 2026 1080p WEB-DL-HHWEB[示例剧 | 第1-2集][2.4 GB]" });
|
||||
await ingestFeed(h.rt, row.id, { manual: true });
|
||||
r = await rel("219380");
|
||||
expect(r.title).toBe("Show S01 2026 1080p WEB-DL-HHWEB");
|
||||
expect(r.seeders).toBe(22);
|
||||
});
|
||||
|
||||
it("规范化前写进来的旧 RSS 行:下次抓取时按原标题补规范化并重新触发识别", async () => {
|
||||
const dirty = "[音乐]Old Album (2020) - WEB-DL - 16bit ALAC-HHWEB[Old Album | ALAC][120.5 MB]";
|
||||
const old = await h.db
|
||||
.insertInto("source_release")
|
||||
.values({
|
||||
site_id: siteId,
|
||||
site_release_id: "219001",
|
||||
title: dirty,
|
||||
size_bytes: 126353408,
|
||||
raw: JSON.stringify({ rss: { link: `${BASE}/details.php?id=219001`, categories: [] } }),
|
||||
})
|
||||
.returning(["id"])
|
||||
.executeTakeFirstOrThrow();
|
||||
const { row } = await feed([
|
||||
item("219002", "[电影]New Movie 2026 1080p-HHWEB[新片][4 GB]", "2026-09-29T09:00:00Z"),
|
||||
]);
|
||||
await ingestFeed(h.rt, row.id);
|
||||
const r = await rel("219001");
|
||||
expect(r).toMatchObject({
|
||||
title: "Old Album (2020) - WEB-DL - 16bit ALAC-HHWEB",
|
||||
subtitle: "Old Album | ALAC",
|
||||
category_raw: "410",
|
||||
});
|
||||
expect((r.raw as { rss: { rawTitle: string } }).rss.rawTitle).toBe(dirty);
|
||||
const events = (await h.outbox("release.observed")).filter((e) => e.payload.sourceReleaseId === old.id);
|
||||
expect(events.map((e) => e.payload.isNew)).toEqual([false]);
|
||||
});
|
||||
|
||||
it("资源列表按发布时间排序:缺做种 / H&R 的 RSS 条目不会因此排到前面", async () => {
|
||||
await h.db
|
||||
.transaction()
|
||||
.execute((trx) =>
|
||||
upsertSiteRelease(
|
||||
h.rt,
|
||||
trx,
|
||||
{ id: siteId },
|
||||
snapshotRelease("219500", "Newest Snapshot 2026-HHWEB", "2026-09-29T10:00:00Z"),
|
||||
),
|
||||
);
|
||||
const list = await listSiteReleases(h.rt, siteId, { limit: 50, live: "false" });
|
||||
const ids = list.items.map((i) => i.siteReleaseId);
|
||||
expect(ids[0]).toBe("219500");
|
||||
expect(ids.indexOf("219390")).toBeLessThan(ids.indexOf("219391"));
|
||||
const times = list.items.map((i) => Date.parse(i.sourcePublishedAt ?? i.firstSeenAt));
|
||||
expect([...times].sort((a, b) => b - a)).toEqual(times);
|
||||
});
|
||||
});
|
||||
@@ -28,7 +28,7 @@ export function planTodoItems(steps: AgentPlanStep[]): TodoItem[] {
|
||||
id: s.id,
|
||||
title: s.title,
|
||||
status: STEP_STATUS[s.status],
|
||||
...(s.note ? { detail: <span className="block max-w-48 truncate">{s.note}</span> } : {}),
|
||||
...(s.note ? { detail: <span title={s.note}>{s.note}</span> } : {}),
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
// 改造:SVG 状态图标保留几何(虚线圈=待办、弧线=进行中、勾=完成、叉=取消),pathLength 弹簧动画换成
|
||||
// CSS 过渡/animate-spin(motion-reduce 关闭);完成色 emerald → success,取消 rose → destructive,且都配 sr-only 文字;
|
||||
// 表头完成数去掉滚字;折叠用 Disclosure;maxHeight 改成 Tailwind 类;新增 `error` 状态(工具失败)、
|
||||
// `skipped` 状态(Agent 步骤清单里跳过的步骤)与 `detail` 槽。
|
||||
// `skipped` 状态(Agent 步骤清单里跳过的步骤)与 `detail` 槽(可压缩,标题至少留一半宽度)。
|
||||
import { ChevronDown, ListTodo } from "lucide-react";
|
||||
import { type ReactNode, useEffect, useId, useRef, useState } from "react";
|
||||
import { cn } from "~/lib/utils.ts";
|
||||
@@ -162,8 +162,10 @@ export function TodoList({
|
||||
<StatusIcon status={s} />
|
||||
<span className="sr-only">{LABEL[s]}:</span>
|
||||
<span
|
||||
title={typeof item.title === "string" ? item.title : undefined}
|
||||
className={cn(
|
||||
"min-w-0 flex-1 truncate text-caption leading-5",
|
||||
// 标题至少占一半宽度:备注(detail)再长也只能压缩自己,不能把标题挤成一条竖线
|
||||
"min-w-[50%] flex-1 truncate text-caption leading-5",
|
||||
s === "pending" && "text-muted-foreground",
|
||||
s === "in-progress" && "text-foreground",
|
||||
s === "completed" && "text-muted-foreground line-through decoration-foreground/30",
|
||||
@@ -174,7 +176,7 @@ export function TodoList({
|
||||
{item.title}
|
||||
</span>
|
||||
{item.detail ? (
|
||||
<span className="shrink-0 text-meta text-muted-foreground">{item.detail}</span>
|
||||
<span className="min-w-0 shrink truncate text-meta text-muted-foreground">{item.detail}</span>
|
||||
) : null}
|
||||
</li>
|
||||
);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { holderName, toViewport, waitingText } from "./coords.ts";
|
||||
import { agentActive, controllerLabel, holderName, toViewport, waitingText } from "./coords.ts";
|
||||
|
||||
describe("原站视图坐标换算", () => {
|
||||
const natural = { width: 1280, height: 800 };
|
||||
@@ -29,3 +29,18 @@ describe("原站视图坐标换算", () => {
|
||||
expect(holderName("admin_session:alice")).toBe("alice");
|
||||
});
|
||||
});
|
||||
|
||||
describe("当前控制文案", () => {
|
||||
const now = Date.parse("2026-09-29T10:00:00Z");
|
||||
it("自动化且没有最近动作 → 「自动化」;两分钟内有自动化动作 → 「Agent」", () => {
|
||||
expect(controllerLabel({ controller: "automation", lastAutomationAt: null }, now)).toBe("自动化");
|
||||
expect(controllerLabel({ controller: "automation" }, now)).toBe("自动化");
|
||||
expect(controllerLabel({ controller: "automation", lastAutomationAt: "2026-09-29T09:50:00Z" }, now)).toBe("自动化");
|
||||
expect(controllerLabel({ controller: "automation", lastAutomationAt: "2026-09-29T09:59:10Z" }, now)).toBe("Agent");
|
||||
expect(agentActive({ controller: "automation", lastAutomationAt: "2026-09-29T09:59:10Z" }, now)).toBe(true);
|
||||
});
|
||||
it("人工接管 / 等待处理不看自动化时间", () => {
|
||||
expect(controllerLabel({ controller: "human", lastAutomationAt: "2026-09-29T09:59:50Z" }, now)).toBe("人工接管");
|
||||
expect(controllerLabel({ controller: "waiting" }, now)).toBe("等待处理");
|
||||
});
|
||||
});
|
||||
@@ -17,12 +17,33 @@ export function toViewport(
|
||||
}
|
||||
|
||||
export const CONTROLLER_LABEL: Record<BrowserController, string> = {
|
||||
automation: "Agent",
|
||||
automation: "自动化",
|
||||
human: "人工接管",
|
||||
waiting: "等待处理",
|
||||
closed: "已关闭",
|
||||
};
|
||||
|
||||
/** 自动化动作在这段时间内发生过,算「Agent 正在操作」 */
|
||||
export const AGENT_ACTIVE_MS = 2 * 60_000;
|
||||
|
||||
/** Agent 是否正在用这个会话:控制者是自动化,且最近 AGENT_ACTIVE_MS 内有过自动化动作 */
|
||||
export function agentActive(
|
||||
s: { controller: BrowserController; lastAutomationAt?: string | null },
|
||||
now = Date.now(),
|
||||
): boolean {
|
||||
if (s.controller !== "automation" || !s.lastAutomationAt) return false;
|
||||
const t = new Date(s.lastAutomationAt).getTime();
|
||||
return Number.isFinite(t) && now - t < AGENT_ACTIVE_MS;
|
||||
}
|
||||
|
||||
/** 「当前控制」文案:无人操作时是「自动化」,Agent 正在操作才显示「Agent」 */
|
||||
export function controllerLabel(
|
||||
s: { controller: BrowserController; lastAutomationAt?: string | null },
|
||||
now = Date.now(),
|
||||
): string {
|
||||
return agentActive(s, now) ? "Agent" : CONTROLLER_LABEL[s.controller];
|
||||
}
|
||||
|
||||
const WAITING_TEXT: Record<string, string> = {
|
||||
captcha: "登录页需要验证码",
|
||||
challenge: "页面需要人工验证",
|
||||
|
||||
@@ -9,7 +9,7 @@ import { StatusBadge, type StatusTone } from "~/components/console/status-badge.
|
||||
import { Button } from "~/components/ui/button.tsx";
|
||||
import { apiData, errorMessage } from "~/lib/api.ts";
|
||||
import { qk } from "~/lib/query-keys.ts";
|
||||
import { CONTROLLER_LABEL, credentialBadge, holderName, waitingText } from "./coords.ts";
|
||||
import { agentActive, controllerLabel, credentialBadge, holderName, waitingText } from "./coords.ts";
|
||||
import { SiteOriginView } from "./site-origin-view.tsx";
|
||||
|
||||
// 原站视图桌面模式(docs/design/webtop-remote-browser.md §5.7):iframe 嵌浏览器桌面(Selkies),
|
||||
@@ -128,7 +128,7 @@ function DesktopView({
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<StatusBadge tone={conn.tone}>{conn.text}</StatusBadge>
|
||||
<StatusBadge tone={CONTROLLER_TONE[s.controller] ?? "neutral"}>
|
||||
当前控制：{CONTROLLER_LABEL[s.controller]}
|
||||
当前控制：{controllerLabel(s)}
|
||||
{human && leaseHolder ? `(${leaseHolder})` : ""}
|
||||
</StatusBadge>
|
||||
{waiting ? <StatusBadge tone="warning">{waiting}</StatusBadge> : null}
|
||||
@@ -197,7 +197,9 @@ function DesktopView({
|
||||
<p className="text-caption text-muted-foreground">
|
||||
{mine
|
||||
? "直接在画面里点击和输入。中文、日文用本机输入法输入；复制粘贴用 Ctrl+C / Ctrl+V。"
|
||||
: "只读观看，Agent 可能正在操作。需要自己点或输入时先接管。"}
|
||||
: agentActive(s)
|
||||
? "只读观看，Agent 正在操作。需要自己点或输入时先接管，Agent 会暂停。"
|
||||
: "只读观看，目前没有 Agent 在操作。需要自己点或输入时先接管。"}
|
||||
</p>
|
||||
|
||||
<div className="pg-slab relative min-w-0 overflow-hidden rounded-xl">
|
||||
|
||||
@@ -34,7 +34,7 @@ import { StatusBadge, type StatusTone } from "~/components/console/status-badge.
|
||||
import { Button } from "~/components/ui/button.tsx";
|
||||
import { Input } from "~/components/ui/input.tsx";
|
||||
import { errorMessage } from "~/lib/api.ts";
|
||||
import { CONTROLLER_LABEL, credentialBadge, holderName, toViewport, waitingText } from "./coords.ts";
|
||||
import { controllerLabel, credentialBadge, holderName, toViewport, waitingText } from "./coords.ts";
|
||||
|
||||
// 原站视图(REQ §4.7.1、§4.7.3):同一个 camofox 会话的定时截图 + 人工输入映射。
|
||||
// 人工接管前服务端先停掉该会话的自动化输入;交还后 Agent 必须重新读取页面。
|
||||
@@ -146,7 +146,7 @@ export function SiteOriginView({ site }: { site: SiteListItem }) {
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<StatusBadge tone={conn.tone}>{conn.text}</StatusBadge>
|
||||
<StatusBadge tone={CONTROLLER_TONE[s.controller] ?? "neutral"}>
|
||||
当前控制：{CONTROLLER_LABEL[s.controller]}
|
||||
当前控制：{controllerLabel(s)}
|
||||
{human && leaseHolder ? `(${leaseHolder})` : ""}
|
||||
</StatusBadge>
|
||||
{waiting ? <StatusBadge tone="warning">{waiting}</StatusBadge> : null}
|
||||
|
||||
@@ -390,9 +390,9 @@ function ReleaseRow({ r, onDownload }: { r: SearchRelease; onDownload: () => voi
|
||||
<span>{r.sizeBytes === null ? "大小未知" : formatBytes(r.sizeBytes, 1)}</span>
|
||||
<span className="inline-flex items-center gap-0.5" title="做种 / 下载">
|
||||
<ArrowUp className="h-3 w-3" />
|
||||
{r.seeders ?? "?"}
|
||||
{r.seeders ?? "—"}
|
||||
<ArrowDown className="ml-1 h-3 w-3" />
|
||||
{r.leechers ?? "?"}
|
||||
{r.leechers ?? "—"}
|
||||
</span>
|
||||
<span title="站点发布时间">发布 {relTime(r.uploadedAt)}</span>
|
||||
{tech ? <span className="break-all">{tech}</span> : null}
|
||||
|
||||
@@ -217,7 +217,7 @@ export function ReleaseDetailSheet({
|
||||
{ label: "促销截止", value: formatDateTime(detail.data.volatile.promotionExpiresAt) },
|
||||
{
|
||||
label: "做种 / 下载",
|
||||
value: `${r.seeders ?? "?"} / ${r.leechers ?? "?"}（${relTime(detail.data.volatile.peersObservedAt)}观察）`,
|
||||
value: `${r.seeders ?? "—"} / ${r.leechers ?? "—"}（${relTime(detail.data.volatile.peersObservedAt)}观察）`,
|
||||
},
|
||||
{ label: "详情抓取于", value: formatDateTime(detail.data.volatile.detailFetchedAt) },
|
||||
]}
|
||||
|
||||
@@ -16,9 +16,23 @@ export function hrOf(r: Pick<SourceRelease, "raw">): "yes" | "no" | "unknown" {
|
||||
return hr === true ? "yes" : hr === false ? "no" : "unknown";
|
||||
}
|
||||
|
||||
/** 站点 RSS 规范化后的字段(raw.rss):分类名、媒体大类 */
|
||||
function rssExtra(r: Pick<SourceRelease, "raw">): { categoryName?: unknown } {
|
||||
const s = (r.raw as { rss?: { categoryName?: unknown } }).rss;
|
||||
return s && typeof s === "object" ? s : {};
|
||||
}
|
||||
|
||||
/** 分类名:列表快照的站点分类名 → RSS 标题前缀里的分类名 → 站点分类 ID */
|
||||
export function categoryLabel(r: Pick<SourceRelease, "raw" | "categoryRaw">): string | null {
|
||||
const name = siteExtra(r).categoryName;
|
||||
return typeof name === "string" && name ? name : r.categoryRaw;
|
||||
if (typeof name === "string" && name) return name;
|
||||
const rss = rssExtra(r).categoryName;
|
||||
return typeof rss === "string" && rss ? rss : r.categoryRaw;
|
||||
}
|
||||
|
||||
/** 列表里的数字:未知显示「—」(未知 ≠ 0) */
|
||||
export function countText(n: number | null | undefined): string {
|
||||
return typeof n === "number" && Number.isFinite(n) ? String(n) : "—";
|
||||
}
|
||||
|
||||
const PROMO_LABEL: Record<string, string> = {
|
||||
|
||||
@@ -6,6 +6,7 @@ import { cn, formatBytes } from "~/lib/utils.ts";
|
||||
import {
|
||||
absTime,
|
||||
categoryLabel,
|
||||
countText,
|
||||
hrOf,
|
||||
ORIGIN_LABEL,
|
||||
promotionLabel,
|
||||
@@ -15,8 +16,11 @@ import {
|
||||
snatchedOf,
|
||||
} from "./release-fields.ts";
|
||||
|
||||
/** 促销 / H&R 徽章(列表与详情共用):促销带剩余时间,过期的写明 */
|
||||
export function ReleaseBadges({ r }: { r: WorkbenchRelease }) {
|
||||
/**
|
||||
* 促销 / H&R 徽章(列表与详情共用):促销带剩余时间,过期的写明。
|
||||
* H&R 未知只在 showUnknownHr 时标出(详情里);列表行不为未知字段挂徽章,避免 RSS 条目满屏「未知」。
|
||||
*/
|
||||
export function ReleaseBadges({ r, showUnknownHr = false }: { r: WorkbenchRelease; showUnknownHr?: boolean }) {
|
||||
const promo = promotionLabel(r);
|
||||
const hr = hrOf(r);
|
||||
const left = promo && !promo.expired ? remainingLabel(r.promotionExpiresAt) : null;
|
||||
@@ -31,7 +35,7 @@ export function ReleaseBadges({ r }: { r: WorkbenchRelease }) {
|
||||
</span>
|
||||
) : null}
|
||||
{hr === "yes" ? <StatusBadge tone="warning">H&R</StatusBadge> : null}
|
||||
{hr === "unknown" ? <StatusBadge>H&R 未知</StatusBadge> : null}
|
||||
{hr === "unknown" && showUnknownHr ? <StatusBadge>H&R 未知</StatusBadge> : null}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -159,19 +163,17 @@ export function ReleaseRow({
|
||||
{r.subtitle ?? ""}
|
||||
</span>
|
||||
<span className="flex flex-wrap items-center gap-x-3 gap-y-1 tabular-nums sm:shrink-0 sm:flex-nowrap sm:gap-x-4">
|
||||
<span className="sm:w-14 sm:truncate" title={r.categoryRaw ? `站点分类 ${r.categoryRaw}` : undefined}>
|
||||
{cat ?? "分类未知"}
|
||||
</span>
|
||||
<span className="sm:w-16 sm:text-right">
|
||||
{r.sizeBytes === null ? "大小未知" : formatBytes(r.sizeBytes, 1)}
|
||||
<span className="sm:w-14 sm:truncate" title={r.categoryRaw ? `站点分类 ${r.categoryRaw}` : "分类未知"}>
|
||||
{cat ?? "—"}
|
||||
</span>
|
||||
<span className="sm:w-16 sm:text-right">{r.sizeBytes === null ? "—" : formatBytes(r.sizeBytes, 1)}</span>
|
||||
<span className="inline-flex items-center gap-0.5 sm:w-32" title="做种 / 下载 / 完成">
|
||||
<ArrowUp className="h-3 w-3" aria-label="做种" />
|
||||
<span className={cn(r.seeders === 0 && "text-warning")}>{r.seeders ?? "?"}</span>
|
||||
<span className={cn(r.seeders === 0 && "text-warning")}>{countText(r.seeders)}</span>
|
||||
<ArrowDown className="ml-1.5 h-3 w-3" aria-label="下载" />
|
||||
{r.leechers ?? "?"}
|
||||
{countText(r.leechers)}
|
||||
<Check className="ml-1.5 h-3 w-3" aria-label="完成" />
|
||||
{snatched ?? "?"}
|
||||
{countText(snatched)}
|
||||
</span>
|
||||
<span className="sm:w-20 sm:text-right" title={timeTitle}>
|
||||
{relTime(r.sourcePublishedAt)}
|
||||
|
||||
@@ -291,6 +291,9 @@ pixelflux + labwc Wayland 桌面 + Selkies 2.0 Web 客户端)+ camofox 1.17.0 **
|
||||
原站视图 Tab 调 `GET /api/v1/browser/desktop/{sessionId}/_status` 决定模式:`available`(容器 3000 能连上且有访问凭据)
|
||||
且 `window.isSecureContext` → 桌面模式(iframe);否则截图模式,顶部写原因:"桌面模式不可用:…" 或 "桌面模式需要通过 HTTPS 访问控制台"。
|
||||
桌面模式顶栏:连接状态(已连接,只读 / 可操作)、当前控制者、接管 / 交还(沿用 `control/acquire|release`)、重新连接、截图模式、关闭会话。
|
||||
「当前控制」(HH-2,两种模式共用 `coords.ts controllerLabel`):`controller = automation` 且 2 分钟内有过自动化动作(Agent 能力经
|
||||
`BrowserService.run` 成功执行,写 `recovery_state.lastAutomationAt`,会话视图字段 `lastAutomationAt`)才显示「Agent」,说明写
|
||||
「Agent 正在操作」;否则显示「自动化」,说明写「目前没有 Agent 在操作」。人工接管显示「人工接管(用户名)」,人工动作不刷新该时间。
|
||||
接管后直接在画面里点和输入;中文、日文用**本机输入法**(Selkies 接收 composition 事件,按 Unicode 注入上屏文本);
|
||||
复制粘贴用 Ctrl+C / Ctrl+V。
|
||||
|
||||
|
||||
@@ -54,6 +54,10 @@
|
||||
|
||||
- **站点内添加**(站点详情「快照与 Feed」→ 新建 Feed,`components/sites/site-feed-dialog.tsx`):只填地址、用途(发现 = `generic` / 追踪 = `search` / 促销 = `promotion`,缺省发现)、可选备注(即名称)。地址输入时下方预览脱敏后的显示值;域名与站点不符给黄色提示但允许保存。保存后立即 `POST /feed-sources/{id}/fetch` 一次,对话框每 2 s 轮询 `GET /feed-sources/{id}`(最多 90 s),显示首抓条目数(游标窗口 `lastGuidSet` 长度)或 `lastError`。
|
||||
- **`POST /feed-sources` 带 `siteId` 时的推断**(`service/feeds.ts feedDefaultsForSite`):`siteAccountId` 缺省取该站启用账号,优先带 passkey 的,再取最早建的;`parserVersion` 记为 `rss@1;<站点定义 umt.version>`(解析仍走通用 RSS 解析器,版本只作追溯);`name` 缺省「站点名 RSS」;`pollIntervalSec` 缺省 1800(站点请求克制;不挂站点的 Feed 仍缺省 900)。地址主机键(去 `www.`、取最后两级,IP 原样)与站点 `baseUrl` / 定义 `domain` 都不一致时,响应附 `warnings[]`(只在创建响应出现)。`name` 现在可省,不挂站点时缺省为地址主机名。
|
||||
- **站点 RSS 条目规范化(HH-2,`packages/adapters/src/sites/feed-item.ts normalizeSiteFeedItem`)**:NexusPHP `torrentrss.php` 的标题是 `[分类]种子名[副标题][大小][上传者]`(憨憨实测 `[音乐]Travis - L.A. Times (2024) - WEB-DL - 24bit ALAC-HHWEB[Travis - L.A. Times | ALAC | …][697.64 MB]`)。挂站点的 Feed 入库时按站点定义拆开:前缀只认定义里登记过的分类名(`umt.categoryNames` / `category.*.desc`),换成站点分类 ID 写 `category_raw`(与列表快照一致,资源页分类筛选可用);尾部配平地剥最多 3 段方括号,只有出现大小段时才把它前面一段当副标题、后面一段当上传者;去掉「[新]」。大小优先用 enclosure `length`(精确字节),没有才用标题里的显示值。`source_release.raw.rss` 记 `rawTitle / categoryName / mediaType(movie·tv·music·null) / uploader / v`。不改标题解析器(`[分类]` 前缀引起的误识别归 ID-2)。
|
||||
- **与列表快照合并**:同站同资源 ID 一行。快照见过的行(`raw._site`)标题 / 副标题 / 分类 / 做种 / 促销 / H&R 以快照为准,RSS 只补空缺、更新 `raw.rss`;快照没见过的行,RSS 内容变化时用新值。快照后到时照常覆盖 RSS 值。
|
||||
- **旧行补规范化**:规范化上线前写入的 RSS 行(`raw.rss.v` 缺失)在该站任一 Feed 下次抓取时按原标题重算(每次最多 500 行,不请求站点);标题变了且未锁定的重新发 `release.observed`(幂等键 `…:rss-norm-1`)。
|
||||
- **资源列表展示**:列表按 `coalesce(发布时间, 首次发现)` 倒序,和字段是否齐全无关。RSS 独有的条目(憨憨的音乐区不在 `torrents.php` 快照里)没有做种 / 下载 / 完成、H&R,列表显示「—」,不挂「H&R 未知」徽章(详情抽屉的 H&R 字段仍写「未知」);分类名取快照分类名 → `raw.rss.categoryName` → 分类 ID。
|
||||
- **来源页任意 RSS**(`components/workbench/feed-form.tsx`):完整表单不变;新建时地址域名匹配已登记站点会自动选中该站和账号并提示「按地址域名关联到 X · 不关联」,手动改过站点后不再自动改。
|
||||
|
||||
## 定时任务
|
||||
@@ -89,7 +93,7 @@
|
||||
|
||||
站点定义可带 `umt.defaults`(`snapshotIntervalSec / snapshotMaxPages / healthCheckIntervalSec / userDataIntervalSec / searchConcurrency / detailCacheSec`),是按站点实测给的保守下限;运行时 `siteAccessFor(def, settings.sites)` 合并:周期、缓存取较长,页数取较小;`searchConcurrency`(缺省 1)让同站不同关键词的实时补查排队执行。目前只有憨憨配置了(快照 60 分钟 1 页、健康检测 6 小时、用户数据 6 小时、搜索并发 1、详情缓存 24 小时、限流 3 秒 1 个令牌突发 2)。
|
||||
|
||||
每从站点限流器拿到一个令牌(列表、搜索、详情、下载、登录、用户数据、原站视图导航)记一次请求,写 `site_request_count`(迁移 `0046`,按小时聚合,多进程共用,保留 7 天)。`GET /sites/{id}/request-stats`(read)返回最近 24 小时总数、按小时明细、快照实际周期与合并后的访问默认;站点页概览「近 24 小时请求」与资源页列表头显示。计数只在升级后开始累计。
|
||||
每从站点限流器拿到一个令牌(列表 / 快照、搜索、详情、详情的文件清单、下载、健康检测、登录、用户数据、原站视图导航)记一次请求;挂站点的 Feed 每次真正发出 RSS 请求(不含熔断 / 退避跳过)也记一次(HH-2 之前漏算)。写 `site_request_count`(迁移 `0046`,按小时聚合,多进程共用,重启不丢,保留 7 天)。`GET /sites/{id}/request-stats`(read)返回最近 24 小时总数、按小时明细、快照实际周期与合并后的访问默认;站点页概览「近 24 小时请求」与资源页列表头显示。计数从迁移 0046 上线后开始累计,之前的请求没有记录,无法回填;RSS 请求从 HH-2 起计入。
|
||||
|
||||
## 详情页扩展内容(INT-2)
|
||||
|
||||
@@ -99,6 +103,8 @@
|
||||
- 缓存:内容放 PG `kv_cache` region `site.detail`,key = `source_release.id`,TTL 取站点详情缓存期(至少 24 小时)。缓存期内 `detail_fetched_at` 新鲜且缓存还在就不请求站点;kv_cache 被清掉(过期 / 重启丢 UNLOGGED 表)后下次刷新会重新抓一次。
|
||||
- 外部 ID 另写 `source_release.raw._detail`(`imdbId / doubanId / tmdb{type,id}`),列表快照 upsert 不覆盖它。身份解析把它和列表给的 `raw._site.imdbId / doubanId` 一起当显式 ID:本地召回查已接受的外部身份;需要查 TMDB 时,有 TMDB ID 直接取详情,否则 IMDb ID 走 `/find/{imdb}?external_source=imdb_id`,命中就不再按标题搜索;TMDB 不支持豆瓣 ID,豆瓣只用于本地召回。是否查 TMDB 仍按惰性识别规则(DEC-1),外部 ID 本身不触发查询。
|
||||
- 截图:响应里只有代理路径 `/api/v1/img/site-detail/{releaseId}/{index}`,服务端从缓存取原始地址,公网校验(每跳重定向复核)、不带站点凭据、按文件头认图片(不放行 SVG)、≤ 8 MB,浏览器缓存 24 小时。站内附件如果需要登录才能看会显示为裂图,不会为截图消耗站点请求。
|
||||
- 憨憨详情页(HH-2 按真实页面 `probe.labels` 核对):没有 `#kdescr`,内容分节为「标签 div + 相邻值 div」:`mediainfo`、`海报`、`豆瓣`(内有 `span.font-bold` 小标签 片名 / 年代 / 类别 / 导演 …)、`截图`、`种子文件`(只有「查看列表」按钮)、`发布者`、`发布时间`。对应规则:简介没有容器时取「豆瓣 / 影片信息 / IMDb」信息块,小标签补冒号、一项一行;截图取「截图」一节的 img(`data-src` 优先,海报不算截图),没有再取简介里的图;IMDb 依次取页面链接 → 简介 / 信息块 / MediaInfo 里的 `imdb.com/title/tt…` 或 `IMDB : tt…` 行(只收 `tt` + 7~8 位,不收孤立 tt 串),TMDB 取链接或 `TMDB : tv/294990` 行,豆瓣取链接或 `douban.com/subject/…`;兜底出的 IMDb / 豆瓣也回填到 `SiteRelease.imdbId / doubanId`;发布时间没有 `span[title]` 时取「发布时间 / 上传时间」字段文本。文件清单:定义 `umt.detail.page.fileListPath`(憨憨 `viewfilelist.php?id={id}`)配置了且页面上没有内联清单时,同一次详情刷新里再请求 1 次(走同一限流器、计数),失败只是没有清单。fixture `HHANCLUB_DETAIL_PAGE_2026` / `HHANCLUB_FILELIST_FRAGMENT`:标签名称与顺序来自真实页面,嵌套按已验证的标题 / 副标题结构推定,部署后打开一条憨憨资源的详情抽屉核对一次。
|
||||
- 截图代理的 Referer:截图与站点同根域(憨憨图床 `pic.hhanclub.net`、`proxy-cover.hhanclub.net`)时带 `Referer: <站点地址>/`(防盗链),外部图床(如 pixhost)不带;任何情况都不带 Cookie。
|
||||
- 解析:`packages/adapters/src/sites/spider/detail-page.ts`。简介取 MovieClaw `detail_description_css`(缺省 `#kdescr`);MediaInfo 取定义 `umt.detail.page.mediaInfo` → 「MediaInfo」字段标签 → 简介里第一个像 MediaInfo 的引用 / 代码块;文件清单取 `umt.detail.page.files` → MovieClaw `detail_file_list_css` → 「文件列表」字段 → `#filelist` 表格,不额外请求 `viewfilelist.php`。字段标签除 `td.rowhead` 外可在 `umt.detail.page.labelSelectors` 加(憨憨:`div.font-bold.leading-6`、`span.font-bold`)。`page.probe` 记录页面上找到的字段标签,只存缓存不展示,站点改版时用来排查。馒头从 API 的 `descr`(BBCode)与 `mediainfo` 取同样的结构。
|
||||
|
||||
## Agent 能力(W10-pre2)
|
||||
|
||||
@@ -260,7 +260,8 @@
|
||||
"fileList": "td.rowfollow > ul > li::text"
|
||||
},
|
||||
"page": {
|
||||
"labelSelectors": ["div.font-bold.leading-6", "span.font-bold"]
|
||||
"labelSelectors": ["div.font-bold.leading-6", "span.font-bold"],
|
||||
"fileListPath": "viewfilelist.php?id={id}"
|
||||
}
|
||||
},
|
||||
"vipLevels": ["贵宾", "VIP"],
|
||||
|
||||
@@ -149,6 +149,7 @@ const UmtNexusSchema = z.strictObject({
|
||||
tags: z.string().optional(),
|
||||
uploader: z.string().optional(),
|
||||
files: z.string().optional(),
|
||||
fileListPath: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
})
|
||||
|
||||
@@ -18,9 +18,12 @@ export function looksLikeMediaInfo(text: string): boolean {
|
||||
return (text.match(/^[^\n:：]{2,40}[ \t]*[:：][ \t]*\S/gm) ?? []).length >= 5;
|
||||
}
|
||||
|
||||
/** TMDB:themoviedb.org 链接,或 MediaInfo / 简介里的「TMDB : tv/294990」一行 */
|
||||
export function extractTmdb(text: string | null | undefined): SiteDetailPage["tmdb"] {
|
||||
const m = text ? /themoviedb\.org\/(movie|tv)\/(\d+)/.exec(text) : null;
|
||||
return m ? { type: m[1] as "movie" | "tv", id: m[2]! } : null;
|
||||
const m = text
|
||||
? (/themoviedb\.org\/(movie|tv)\/(\d+)/.exec(text) ?? /\bTMDB\s*(?:ID)?\s*[:：]\s*(movie|tv)\/(\d+)/i.exec(text))
|
||||
: null;
|
||||
return m ? { type: m[1]!.toLowerCase() as "movie" | "tv", id: m[2]! } : null;
|
||||
}
|
||||
|
||||
/** 图片地址转绝对 http(s) 地址并过滤非截图;无效返回 null */
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeSiteFeedItem } from "./feed-item.ts";
|
||||
|
||||
// 标题取自 2026-09-29 憨憨 RSS 真实条目(公开种子名,无凭据)
|
||||
describe("站点 RSS 条目规范化(憨憨)", () => {
|
||||
it("[分类]种子名[副标题][大小] → 分类 / 标题 / 副标题 / 大小", () => {
|
||||
const r = normalizeSiteFeedItem(
|
||||
"hhanclub",
|
||||
"[音乐]Travis - L.A. Times (2024) - WEB-DL - 24bit ALAC-HHWEB[Travis - L.A. Times | ALAC | 苹果音乐 | 现场版专辑][697.64 MB]",
|
||||
);
|
||||
expect(r).toEqual({
|
||||
title: "Travis - L.A. Times (2024) - WEB-DL - 24bit ALAC-HHWEB",
|
||||
subtitle: "Travis - L.A. Times | ALAC | 苹果音乐 | 现场版专辑",
|
||||
category: { id: "410", name: "音乐" },
|
||||
mediaType: "music",
|
||||
sizeBytes: Math.round(697.64 * 1024 ** 2),
|
||||
uploader: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("电视剧:副标题里的 | 与 / 保留,GB 换算", () => {
|
||||
const r = normalizeSiteFeedItem(
|
||||
"hhanclub",
|
||||
"[电视剧]Spring of the Blade S01 2026 2160p YK WEB-DL H.265 DV HFR HQ AAC2.0-HHWEB[一瓯春 | 第20-24集 | 4K 高码 杜比视界 | 类型: 爱情/古装][26.37 GB]",
|
||||
);
|
||||
expect(r.title).toBe("Spring of the Blade S01 2026 2160p YK WEB-DL H.265 DV HFR HQ AAC2.0-HHWEB");
|
||||
expect(r.subtitle).toBe("一瓯春 | 第20-24集 | 4K 高码 杜比视界 | 类型: 爱情/古装");
|
||||
expect(r.category).toEqual({ id: "402", name: "电视剧" });
|
||||
expect(r.mediaType).toBe("tv");
|
||||
expect(r.sizeBytes).toBe(Math.round(26.37 * 1024 ** 3));
|
||||
});
|
||||
|
||||
it("副标题内嵌方括号、[新] 标记、不在任何媒体大类的分类(纪录片)", () => {
|
||||
const r = normalizeSiteFeedItem(
|
||||
"hhanclub",
|
||||
"[纪录片]Thirteen Talks S09 2016 2160p WEB-DL H.265 AAC2.0-HHWEB[新][十三邀 | 第14集 [内嵌中字]][3.6 GB]",
|
||||
);
|
||||
expect(r.title).toBe("Thirteen Talks S09 2016 2160p WEB-DL H.265 AAC2.0-HHWEB");
|
||||
expect(r.subtitle).toBe("十三邀 | 第14集 [内嵌中字]");
|
||||
expect(r.category).toEqual({ id: "404", name: "纪录片" });
|
||||
expect(r.mediaType).toBeNull();
|
||||
});
|
||||
|
||||
it("带上传者段:[副标题][大小][上传者]", () => {
|
||||
const r = normalizeSiteFeedItem("hhanclub", "[电影]Some Movie 2024 1080p WEB-DL-HHWEB[某电影][8.5 GB][uploader01]");
|
||||
expect(r).toMatchObject({
|
||||
title: "Some Movie 2024 1080p WEB-DL-HHWEB",
|
||||
subtitle: "某电影",
|
||||
uploader: "uploader01",
|
||||
});
|
||||
});
|
||||
|
||||
it("没有大小段时不拆尾部方括号;未登记的前缀不当分类", () => {
|
||||
const r = normalizeSiteFeedItem("hhanclub", "[BDMV]Some.Disc.2024[Extras]");
|
||||
expect(r).toMatchObject({ title: "[BDMV]Some.Disc.2024[Extras]", subtitle: null, sizeBytes: null });
|
||||
expect(r.category).toEqual({ id: null, name: null });
|
||||
});
|
||||
|
||||
it("未知站点只去 [新]", () => {
|
||||
const r = normalizeSiteFeedItem(null, "[电影]Foo 2024[新][foo][1 GB]");
|
||||
expect(r.category.id).toBeNull();
|
||||
expect(r.title).toBe("[电影]Foo 2024");
|
||||
expect(r.subtitle).toBe("foo");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* 站点 RSS 条目规范化 —— 本项目扩展(站点适配层)。
|
||||
* NexusPHP `torrentrss.php` 按用户勾选拼标题:`[分类]` + 种子名 + `[副标题]` + `[大小]` + `[上传者]`
|
||||
* (憨憨:`[音乐]Travis - L.A. Times (2024) - WEB-DL - 24bit ALAC-HHWEB[Travis - L.A. Times | ALAC][697.64 MB]`)。
|
||||
* 这里把它拆成与列表快照一致的字段:干净标题、副标题、站点分类(ID + 名称 + 媒体大类)、大小。
|
||||
* 只认站点定义里登记过的分类名作前缀;不改标题解析器(release-parser)。
|
||||
*/
|
||||
import { getSiteDefinition } from "./registry.ts";
|
||||
import { parseSizeMp } from "./spider/spider.ts";
|
||||
|
||||
export type FeedMediaType = "movie" | "tv" | "music";
|
||||
|
||||
export interface SiteFeedItemFields {
|
||||
/** 去掉分类前缀、副标题、大小、上传者、「[新]」后的种子名 */
|
||||
title: string;
|
||||
subtitle: string | null;
|
||||
/** 站点原始分类:ID 取站点定义里同名分类(与列表快照的 category_raw 一致);没有登记为 null */
|
||||
category: { id: string | null; name: string | null };
|
||||
/** 站点分类所属的媒体大类(定义 category.movie / tv / music);不属于任何一组为 null */
|
||||
mediaType: FeedMediaType | null;
|
||||
/** 标题尾部的大小(站点四舍五入后的显示值);enclosure 有 length 时调用方应优先用它 */
|
||||
sizeBytes: number | null;
|
||||
uploader: string | null;
|
||||
}
|
||||
|
||||
const NEW_MARK = /\s*[[(（【]\s*新\s*[\])）】]\s*/g;
|
||||
const SIZE = /^\s*([\d.,]+\s*[KMGTP]i?B)\s*$/i;
|
||||
|
||||
interface CategoryDefLike {
|
||||
category?: Partial<Record<FeedMediaType, { id?: unknown; desc?: unknown }[]>>;
|
||||
umt?: { categoryNames?: Record<string, string> };
|
||||
}
|
||||
|
||||
/** 站点定义里登记的「分类名 → ID」 */
|
||||
function categoryIndex(siteKey: string | null | undefined): Map<string, string> {
|
||||
const out = new Map<string, string>();
|
||||
const def = (siteKey ? getSiteDefinition(siteKey) : undefined) as CategoryDefLike | undefined;
|
||||
if (!def) return out;
|
||||
for (const [id, name] of Object.entries(def.umt?.categoryNames ?? {})) if (name) out.set(name, id);
|
||||
for (const k of ["movie", "tv", "music"] as const) {
|
||||
for (const c of def.category?.[k] ?? []) {
|
||||
if (typeof c.desc === "string" && c.desc && c.id != null && !out.has(c.desc)) out.set(c.desc, String(c.id));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** 站点分类 ID → 媒体大类(只在唯一一组里出现时成立) */
|
||||
export function siteCategoryMediaType(siteKey: string | null | undefined, id: string | null): FeedMediaType | null {
|
||||
if (!id || !siteKey) return null;
|
||||
const def = getSiteDefinition(siteKey) as CategoryDefLike | undefined;
|
||||
const hits = (["movie", "tv", "music"] as const).filter((k) =>
|
||||
(def?.category?.[k] ?? []).some((c) => c.id != null && String(c.id) === id),
|
||||
);
|
||||
return hits.length === 1 ? hits[0]! : null;
|
||||
}
|
||||
|
||||
/** 从尾部剥一个配平的 `[...]`;不以 `]` 结尾或不配平返回 null */
|
||||
function popTrailingGroup(s: string): { rest: string; inner: string } | null {
|
||||
const t = s.trimEnd();
|
||||
if (!t.endsWith("]")) return null;
|
||||
let depth = 0;
|
||||
for (let i = t.length - 1; i >= 0; i--) {
|
||||
const ch = t[i];
|
||||
if (ch === "]") depth++;
|
||||
else if (ch === "[") {
|
||||
depth--;
|
||||
if (depth === 0) return { rest: t.slice(0, i), inner: t.slice(i + 1, -1) };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 规范化站点 RSS 条目标题。没有站点定义时只去「[新]」,其余原样返回。
|
||||
* 只有标题尾部出现大小段(`[697.64 MB]`)时才把它前面的一段当副标题,避免误伤本身以 `[...]` 结尾的种子名。
|
||||
*/
|
||||
export function normalizeSiteFeedItem(siteKey: string | null | undefined, rawTitle: string): SiteFeedItemFields {
|
||||
const cats = categoryIndex(siteKey);
|
||||
let s = rawTitle.replace(/\s+/g, " ").trim();
|
||||
let category: SiteFeedItemFields["category"] = { id: null, name: null };
|
||||
|
||||
const prefix = /^\s*\[([^\][]{1,16})\]\s*/.exec(s);
|
||||
if (prefix && cats.has(prefix[1]!.trim())) {
|
||||
const name = prefix[1]!.trim();
|
||||
category = { id: cats.get(name) ?? null, name };
|
||||
s = s.slice(prefix[0].length);
|
||||
}
|
||||
|
||||
let sizeBytes: number | null = null;
|
||||
let subtitle: string | null = null;
|
||||
let uploader: string | null = null;
|
||||
const groups: string[] = [];
|
||||
let rest = s;
|
||||
// 最多看尾部 3 段:[副标题][大小][上传者]
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const g = popTrailingGroup(rest);
|
||||
if (!g) break;
|
||||
groups.unshift(g.inner);
|
||||
rest = g.rest;
|
||||
}
|
||||
const sizeIdx = groups.findIndex((g) => SIZE.test(g));
|
||||
if (sizeIdx >= 0) {
|
||||
const bytes = parseSizeMp(SIZE.exec(groups[sizeIdx]!)![1]!);
|
||||
sizeBytes = bytes > 0 ? bytes : null;
|
||||
const before = groups.slice(0, sizeIdx);
|
||||
const after = groups.slice(sizeIdx + 1);
|
||||
// 大小之前最后一段是副标题,再往前的属于种子名本身
|
||||
if (before.length) subtitle = before.pop()!.trim() || null;
|
||||
if (after.length === 1 && after[0]!.trim().length <= 32) uploader = after[0]!.trim() || null;
|
||||
s = `${rest}${before.map((b) => `[${b}]`).join("")}${after.length > 1 ? after.map((a) => `[${a}]`).join("") : ""}`;
|
||||
}
|
||||
|
||||
const title = s.replace(NEW_MARK, " ").replace(/\s+/g, " ").trim() || rawTitle.trim();
|
||||
return {
|
||||
title,
|
||||
subtitle: subtitle ? subtitle.replace(NEW_MARK, " ").trim() || null : null,
|
||||
category,
|
||||
mediaType: siteCategoryMediaType(siteKey, category.id),
|
||||
sizeBytes,
|
||||
uploader,
|
||||
};
|
||||
}
|
||||
@@ -16,6 +16,12 @@ export {
|
||||
SpiderFilterSchema,
|
||||
} from "./definitions/index.ts";
|
||||
export type { SiteAdapterDeps } from "./deps.ts";
|
||||
export {
|
||||
type FeedMediaType,
|
||||
normalizeSiteFeedItem,
|
||||
type SiteFeedItemFields,
|
||||
siteCategoryMediaType,
|
||||
} from "./feed-item.ts";
|
||||
export {
|
||||
classifyLoginError,
|
||||
loginFailureText,
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
} from "../types.ts";
|
||||
import { inspectTorrent } from "../util.ts";
|
||||
import { parseDetailPage } from "./detail.ts";
|
||||
import { filesFromFileListHtml } from "./detail-page.ts";
|
||||
import { spiderTorrentToRelease } from "./map.ts";
|
||||
import type { IndexerDefinition } from "./schema.ts";
|
||||
import { loadHtml, normalizeSelector, SiteSpider } from "./spider.ts";
|
||||
@@ -178,6 +179,17 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap
|
||||
if (r.kind === "login") throw new SiteError(def.id, "auth_expired", `详情 ${siteReleaseId}:返回登录页`);
|
||||
if (r.kind === "not_found") throw new SiteError(def.id, "not_found", `详情 ${siteReleaseId}:资源不存在`);
|
||||
if (!r.detail) throw new SiteError(def.id, "parse_error", `详情 ${siteReleaseId}:页面结构无法识别`);
|
||||
// 文件清单不在页面上(NexusPHP 点「查看列表」才 AJAX 取)→ 按需 1 次;失败只是没有文件清单
|
||||
const flPath = ext.detail.page?.fileListPath;
|
||||
if (flPath && r.detail.page && !r.detail.page.files.length) {
|
||||
try {
|
||||
const flUrl = new URL(flPath.replace("{id}", siteReleaseId), base).toString();
|
||||
const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts);
|
||||
r.detail.page.files = filesFromFileListHtml(loadHtml(frag));
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError && e.isAuthError) throw e;
|
||||
}
|
||||
}
|
||||
return r.detail;
|
||||
},
|
||||
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { HHANCLUB_DETAIL_PAGE_FULL } from "@umt/testkit";
|
||||
import { HHANCLUB_DETAIL_PAGE_2026, HHANCLUB_DETAIL_PAGE_FULL, HHANCLUB_FILELIST_FRAGMENT } from "@umt/testkit";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { hhanclubDefinition } from "../definitions/index.ts";
|
||||
import { looksLikeMediaInfo, pageFromBbcode } from "../detail-text.ts";
|
||||
import { createSiteRateLimiter } from "../rate-limit.ts";
|
||||
import { getSiteDefinition } from "../registry.ts";
|
||||
import { staticSecretResolver } from "../types.ts";
|
||||
import { createSpiderSiteAdapter } from "./adapter.ts";
|
||||
import { parseDetailPage } from "./detail.ts";
|
||||
import type { IndexerDefinition } from "./schema.ts";
|
||||
|
||||
@@ -105,3 +108,58 @@ Format : AVC[/quote]
|
||||
expect(looksLikeMediaInfo("General\n一段文字\nVideo")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("详情页扩展内容(憨憨 2026 真实结构:分节 div,无 #kdescr)", () => {
|
||||
const { kind, detail } = parseDetailPage(HHANCLUB_DETAIL_PAGE_2026, hhanclubDefinition, "91003", OBSERVED);
|
||||
const page = detail!.page!;
|
||||
|
||||
it("MediaInfo 取「mediainfo」一节;IMDb / TMDB 从 MediaInfo 行兜底,豆瓣取页面链接", () => {
|
||||
expect(kind).toBe("ok");
|
||||
expect(page.mediaInfo).toContain("Complete name");
|
||||
expect(page).toMatchObject({ imdbId: "tt45950001", tmdb: { type: "tv", id: "290001" }, doubanId: "36000002" });
|
||||
expect(detail!.imdbId).toBe("tt45950001");
|
||||
});
|
||||
|
||||
it("简介取豆瓣信息块,小标签补冒号、一项一行", () => {
|
||||
expect(page.descriptionText).toContain("片名：示例剧集");
|
||||
expect(page.descriptionText).toContain("类别：爱情 / 古装");
|
||||
expect(page.descriptionText).toContain("一段剧情简介");
|
||||
});
|
||||
|
||||
it("截图取「截图」一节(优先 data-src),不含海报与占位图", () => {
|
||||
expect(page.screenshots).toEqual([
|
||||
"https://pic.hhanclub.net/img/2026/09/29/shot1.png",
|
||||
"https://img3.pixhost.cc/images/1/shot2.jpg",
|
||||
]);
|
||||
});
|
||||
|
||||
it("上传者、标签、发布时间(字段文本兜底)", () => {
|
||||
expect(page.uploader).toBe("匿名");
|
||||
expect(page.tags).toEqual(["官方", "国语", "中字", "杜比视界"]);
|
||||
expect(detail!.uploadedAt).toBe("2026-09-29T08:42:57.000Z");
|
||||
expect(page.files).toEqual([]);
|
||||
});
|
||||
|
||||
it("文件清单不在页面上 → 按需请求 viewfilelist.php 一次", async () => {
|
||||
const paths: string[] = [];
|
||||
const fetch = (async (input: RequestInfo | URL) => {
|
||||
const u = new URL(String(input));
|
||||
paths.push(`${u.pathname}${u.search}`);
|
||||
const body = u.pathname === "/viewfilelist.php" ? HHANCLUB_FILELIST_FRAGMENT : HHANCLUB_DETAIL_PAGE_2026;
|
||||
return new Response(body, { status: 200, headers: { "content-type": "text/html; charset=utf-8" } });
|
||||
}) as typeof globalThis.fetch;
|
||||
const limiter = createSiteRateLimiter({ minIntervalMs: 0 });
|
||||
const a = createSpiderSiteAdapter(hhanclubDefinition, {
|
||||
fetch,
|
||||
limiter,
|
||||
secrets: staticSecretResolver({ hhanclub: { cookie: "c=1" } }),
|
||||
});
|
||||
const d = await a.detail("91003");
|
||||
expect(paths).toEqual(["/details.php?id=91003&hit=1", "/viewfilelist.php?id=91003"]);
|
||||
expect(limiter.stats.granted).toBe(2);
|
||||
expect(d.page!.files).toEqual([
|
||||
{ name: "Example.Show.S01E18.2026.2160p.mkv", sizeBytes: Math.round(5.04 * 1024 ** 3) },
|
||||
{ name: "Example.Show.S01E19.2026.2160p.mkv", sizeBytes: Math.round(4.98 * 1024 ** 3) },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -25,10 +25,13 @@ type El = ReturnType<CheerioAPI>;
|
||||
|
||||
const LABELS = {
|
||||
description: ["简介", "Description"],
|
||||
/** 没有简介容器时的次选:豆瓣 / IMDb 信息块(憨憨把豆瓣资料拆成独立一节) */
|
||||
descriptionFallback: ["豆瓣", "豆瓣信息", "影片信息", "IMDb信息", "IMDb"],
|
||||
screenshots: ["截图", "Screenshots", "Screenshot"],
|
||||
mediaInfo: ["MediaInfo", "媒体信息", "BDInfo"],
|
||||
tags: ["标签", "Tags"],
|
||||
uploader: ["上传者", "发布者", "发布人", "上传人", "Uploader"],
|
||||
files: ["文件列表", "文件清单", "File list", "Files"],
|
||||
files: ["文件列表", "文件清单", "种子文件", "File list", "Files"],
|
||||
};
|
||||
|
||||
/** 简介里可能装 MediaInfo 的块 */
|
||||
@@ -62,6 +65,19 @@ function fileFromText(text: string): SiteDetailFile | null {
|
||||
return HAS_EXT.test(t) ? { name: t, sizeBytes: null } : null;
|
||||
}
|
||||
|
||||
/** 文本里的 IMDb ID:imdb.com/title 链接或「IMDB : tt…」一行;不接受孤立的 tt 串(避免误中哈希) */
|
||||
export function imdbFromText(text: string): string | null {
|
||||
const m =
|
||||
/imdb\.com\/title\/(tt\d{7,8})\b/i.exec(text) ??
|
||||
/\bIMDb\s*(?:ID|链接|編號|编号)?\s*[:：]?\s*(tt\d{7,8})\b/i.exec(text);
|
||||
return m?.[1] ?? null;
|
||||
}
|
||||
|
||||
/** NexusPHP viewfilelist.php 返回的文件表(按需 AJAX 取到的 HTML 片段) */
|
||||
export function filesFromFileListHtml($: CheerioAPI): SiteDetailFile[] {
|
||||
return filesFrom($, $("tr"));
|
||||
}
|
||||
|
||||
function filesFrom($: CheerioAPI, rows: El): SiteDetailFile[] {
|
||||
const out: SiteDetailFile[] = [];
|
||||
rows.each((_, n) => {
|
||||
@@ -70,7 +86,7 @@ function filesFrom($: CheerioAPI, rows: El): SiteDetailFile[] {
|
||||
const cells = row.children("td");
|
||||
if (cells.length >= 2) {
|
||||
const name = cells.first().text().replace(/\s+/g, " ").trim();
|
||||
const size = cells.eq(1).text().trim();
|
||||
const size = cells.eq(1).text().replace(/\s+/g, " ").trim();
|
||||
if (name && HAS_EXT.test(name) && /\d/.test(size)) out.push({ name, sizeBytes: parseSizeMp(size) || null });
|
||||
return;
|
||||
}
|
||||
@@ -106,6 +122,7 @@ export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string)
|
||||
let desc: El | null = descSel ? $(descSel).first() : null;
|
||||
if (!desc?.length) desc = byLabel(LABELS.description);
|
||||
page.probe.hasDescription = !!desc?.length;
|
||||
const infoBlock = desc?.length ? null : byLabel(LABELS.descriptionFallback);
|
||||
|
||||
// MediaInfo:站点选择器 → 字段标签 → 简介里的引用 / 代码块(取最内层文本,整块从简介里去掉)
|
||||
let mediaInfoEl: El | null = null;
|
||||
@@ -137,12 +154,19 @@ export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string)
|
||||
|
||||
// 截图
|
||||
const imgs: string[] = [];
|
||||
const imgEls = p.screenshots ? $(p.screenshots) : desc?.length ? desc.find("img") : null;
|
||||
imgEls?.each((_, n) => {
|
||||
const pushImgs = (els: El | null) =>
|
||||
els?.each((_, n) => {
|
||||
const e = $(n);
|
||||
const u = screenshotUrl(e.attr("data-src") ?? e.attr("data-original") ?? e.attr("src"), base);
|
||||
if (u && !imgs.includes(u)) imgs.push(u);
|
||||
});
|
||||
if (p.screenshots) pushImgs($(p.screenshots));
|
||||
else {
|
||||
// 独立的「截图」一节(憨憨)优先;没有再取简介里的图
|
||||
const shotBlock = byLabel(LABELS.screenshots);
|
||||
if (shotBlock) pushImgs(shotBlock.is("img") ? shotBlock : shotBlock.find("img"));
|
||||
if (!imgs.length && desc?.length) pushImgs(desc.find("img"));
|
||||
}
|
||||
page.probe.imageCount = imgs.length;
|
||||
page.screenshots = imgs.slice(0, MAX_SCREENSHOTS);
|
||||
|
||||
@@ -153,18 +177,36 @@ export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string)
|
||||
c.find("img").remove();
|
||||
const t = blockText($, c);
|
||||
if (t) page.descriptionText = t.slice(0, 20_000);
|
||||
} else if (infoBlock) {
|
||||
const c = infoBlock.clone();
|
||||
c.find("img").remove();
|
||||
// 块里的「标签」span 与值之间补上冒号 / 换行,读起来是「片名:xxx」一行一项
|
||||
c.find(p.labelSelectors?.join(", ") || "span.font-bold").each((_, n) => {
|
||||
const e = $(n);
|
||||
const t = e.text().trim();
|
||||
if (t && t.length <= 16 && !/[:：]$/.test(t)) e.text(`${t}：`);
|
||||
e.before("\n");
|
||||
});
|
||||
const t = blockText($, c)
|
||||
.replace(/[ \t]*\n[ \t]*/g, "\n")
|
||||
.replace(/：\s*\n(?=\S)/g, "：");
|
||||
if (t) page.descriptionText = t.slice(0, 20_000);
|
||||
}
|
||||
|
||||
// 外部 ID:页面上的链接优先,其次简介正文
|
||||
const descHtml = desc?.html() ?? "";
|
||||
// 外部 ID:页面上的链接优先,其次简介 / 信息块正文,再次 MediaInfo 里的 IMDB / TMDB 行(憨憨的 MediaInfo 带这两行)
|
||||
const descHtml = `${desc?.html() ?? ""}\n${infoBlock?.html() ?? ""}`;
|
||||
const idText = `${descHtml}\n${page.mediaInfo ?? ""}`;
|
||||
const f = conf.fields ?? {};
|
||||
page.imdbId =
|
||||
extractImdbId(mcSelect($, null, f.imdb ?? "a[href*='imdb.com']::attr(href)")) ?? extractImdbId(descHtml) ?? null;
|
||||
extractImdbId(mcSelect($, null, f.imdb ?? "a[href*='imdb.com']::attr(href)")) ??
|
||||
imdbFromText(idText) ??
|
||||
extractImdbId(descHtml) ??
|
||||
null;
|
||||
page.doubanId =
|
||||
extractDoubanId(mcSelect($, null, f.douban ?? "a[href*='douban.com']::attr(href)")) ??
|
||||
extractDoubanId(descHtml) ??
|
||||
extractDoubanId(idText) ??
|
||||
null;
|
||||
page.tmdb = extractTmdb($("a[href*='themoviedb.org']").first().attr("href")) ?? extractTmdb(descHtml);
|
||||
page.tmdb = extractTmdb($("a[href*='themoviedb.org']").first().attr("href")) ?? extractTmdb(idText);
|
||||
|
||||
// 文件清单:站点选择器 → MovieClaw file_list(去伪元素)→ 字段标签 → #filelist 表格
|
||||
const fileSel = p.files ?? f.fileList?.replace(/::text$/, "");
|
||||
|
||||
@@ -83,10 +83,15 @@ export function parseDetailPage(
|
||||
browse: undefined,
|
||||
};
|
||||
const promo = new SiteSpider(promoDef).infoFrom($, $(conf.headerScope).first());
|
||||
const timeTitle = $("span[title]")
|
||||
// 发布时间:带完整时间的 span[title] → 「发布时间 / 上传时间 / 添加时间」字段的文本(憨憨是 div 标签 + 值)
|
||||
const timeTitle =
|
||||
$("span[title]")
|
||||
.filter((_, el) => /\d{4}-\d{2}-\d{2} \d{2}:\d{2}/.test($(el).attr("title") ?? ""))
|
||||
.first()
|
||||
.attr("title");
|
||||
.attr("title") ??
|
||||
/(?:发布时间|上传时间|添加时间|Added)\s*[:：]?\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}(?::\d{2})?)/.exec(
|
||||
$("body").text().replace(/\s+/g, " "),
|
||||
)?.[1];
|
||||
|
||||
const detail: SiteReleaseDetail = {
|
||||
siteId: def.id,
|
||||
@@ -119,6 +124,9 @@ export function parseDetailPage(
|
||||
const page = extractDetailPage($, conf, def.domain);
|
||||
page.imdbId ??= detail.imdbId ?? null;
|
||||
page.doubanId ??= detail.doubanId ?? null;
|
||||
// 详情页扩展内容里兜底出来的外部 ID(MediaInfo 的 IMDB 行、豆瓣信息块)也回填到资源本身
|
||||
if (!detail.imdbId && page.imdbId) detail.imdbId = page.imdbId;
|
||||
if (!detail.doubanId && page.doubanId) detail.doubanId = page.doubanId;
|
||||
if (page.tags.length) detail.tags = page.tags;
|
||||
detail.page = page;
|
||||
return { kind: "ok", detail };
|
||||
|
||||
@@ -211,6 +211,11 @@ export interface UmtSiteExtension {
|
||||
uploader?: string;
|
||||
/** 文件清单行(每行一个文件,文本形如「名称 (大小)」或两列表格) */
|
||||
files?: string;
|
||||
/**
|
||||
* 页面上没有内联文件清单时,按需再取一次的 AJAX 端点(`{id}` 替换为资源 ID,NexusPHP 为 `viewfilelist.php?id={id}`);
|
||||
* 每次详情刷新最多 1 次,失败不影响详情。
|
||||
*/
|
||||
fileListPath?: string;
|
||||
};
|
||||
};
|
||||
/** 下载路径(按资源 ID 拼接) */
|
||||
|
||||
@@ -337,6 +337,8 @@ export const BrowserSessionView = BrowserSession.extend({
|
||||
waitingReason: z.string().nullable(),
|
||||
/** 打开会话时的凭据注入与回写状态(BR-2);旧会话没有 */
|
||||
credential: BrowserCredentialState.nullable().optional(),
|
||||
/** 最近一次自动化(Agent 能力)动作成功的时间;从未有过为 null。原站视图据此显示「Agent」或「自动化」 */
|
||||
lastAutomationAt: Timestamp.nullable().optional(),
|
||||
});
|
||||
export type BrowserSessionView = z.infer<typeof BrowserSessionView>;
|
||||
|
||||
|
||||
@@ -108,3 +108,49 @@ export const HHANCLUB_DETAIL_PAGE_FULL = `<!DOCTYPE html><html><head><title>HHCL
|
||||
<div class="font-bold leading-6">文件列表</div>
|
||||
<div class="font-bold leading-6"><ul><li>Example.Show.S01E01.mkv (1.04 GB)</li><li>Example.Show.S01E02.mkv (1.02 GB)</li></ul></div>
|
||||
</div></body></html>`;
|
||||
|
||||
/**
|
||||
* HH-2:按 2026-09-29 真实详情页的解析诊断(`page.probe.labels`)重建的最小片段。字段标签名称与顺序来自真实页面
|
||||
* (下载 / 标题 / 副标题 / 标签 / 基本信息 / 字幕 / mediainfo / 海报 / 豆瓣 / 截图 / 种子文件 / 同伴 …,豆瓣块里是
|
||||
* span.font-bold 小标签 片名 / 年代 / 类别 / 导演 …);没有 #kdescr 简介容器;MediaInfo 头部与 IMDB / TMDB 两行
|
||||
* 摘自真实缓存(ID 已替换)。标签与值的 DOM 嵌套是按标题 / 副标题已验证的「标签 div + 相邻值 div」推定的。
|
||||
* 文件清单不在页面上,点「查看列表」才 AJAX 取 viewfilelist.php(见 HHANCLUB_FILELIST_FRAGMENT)。
|
||||
*/
|
||||
export const HHANCLUB_DETAIL_PAGE_2026 = `<!DOCTYPE html><html><head><title>HHCLUB :: 种子详情</title></head><body>
|
||||
<div id="user-info-panel"><a href="userdetails.php?id=7" class="VIP_Name">bob</a> <a href="logout.php">退出</a></div>
|
||||
<div class="grid grid-cols-[80px_1fr]">
|
||||
<div class="font-bold leading-6">下载</div><div class="font-bold leading-6"><a href="download.php?id=91003">Example.Show.S01E18.torrent</a></div>
|
||||
<div class="font-bold leading-6">标题</div><div class="font-bold leading-6">Example Show S01 2026 2160p YK WEB-DL H.265 DV HQ AAC2.0-HHWEB</div>
|
||||
<div class="font-bold leading-6">副标题</div><div class="font-bold leading-6">示例剧集 | 第18-24集 | 4K 高码 杜比视界</div>
|
||||
<div class="font-bold leading-6">标签</div><div class="font-bold leading-6"><span>官方</span><span>国语</span><span>中字</span><span>杜比视界</span></div>
|
||||
<div class="font-bold leading-6">基本信息</div>
|
||||
<div class="font-bold leading-6">
|
||||
<span class="font-bold">大小：</span><span>33.62 GB</span>
|
||||
<span class="font-bold">类型：</span><span>电视剧</span>
|
||||
<span class="font-bold">发布者：</span><span>匿名</span>
|
||||
<span class="font-bold">发布时间：</span><span>2026-09-29 16:42:57</span>
|
||||
</div>
|
||||
<div class="font-bold leading-6">mediainfo</div>
|
||||
<div class="font-bold leading-6"><div class="nexus-media-info-raw">文件名:Example.Show.S01E18.2026.2160p.YK.WEB-DL.H.265.DV.HQ.AAC2.0-HHWEB.mkv<br>体 积: 5.04 GiB<br><br>General<br>Unique ID : 1580976599<br>Complete name : Example.Show.S01E18.mkv<br>Format : Matroska<br>File size : 5.04 GiB<br>Duration : 43 min 55 s<br>IMDB : tt45950001<br>TMDB : tv/290001<br><br>Video<br>ID : 1<br>Format : HEVC</div></div>
|
||||
<div class="font-bold leading-6">海报</div><div class="font-bold leading-6"><img src="https://proxy-cover.hhanclub.net/douban/p0000000001.jpg" alt="poster"></div>
|
||||
<div class="font-bold leading-6">豆瓣</div>
|
||||
<div class="font-bold leading-6">
|
||||
<div><span class="font-bold">片名</span><span>示例剧集</span></div>
|
||||
<div><span class="font-bold">年代</span><span>2026</span></div>
|
||||
<div><span class="font-bold">类别</span><span>爱情 / 古装</span></div>
|
||||
<div><span class="font-bold">导演</span><span>某导演</span></div>
|
||||
<div>一段剧情简介……</div>
|
||||
<a href="https://movie.douban.com/subject/36000002/">豆瓣</a>
|
||||
</div>
|
||||
<div class="font-bold leading-6">截图</div>
|
||||
<div class="font-bold leading-6"><img src="https://pic.hhanclub.net/img/2026/09/29/shot1.png"><img data-src="https://img3.pixhost.cc/images/1/shot2.jpg" src="pic/trans.gif"></div>
|
||||
<div class="font-bold leading-6">种子文件</div><div class="font-bold leading-6"><a href="javascript: viewfilelist(91003)">[查看列表]</a></div>
|
||||
<div class="font-bold leading-6">同伴</div><div class="font-bold leading-6">22个做种者 | 66个下载者</div>
|
||||
</div></body></html>`;
|
||||
|
||||
/** NexusPHP viewfilelist.php 返回的文件表片段(标准结构,文件名 / 大小为示例) */
|
||||
export const HHANCLUB_FILELIST_FRAGMENT = `<table class="main" border="1" cellspacing="0" cellpadding="5">
|
||||
<tr><td class="colhead">路径</td><td class="colhead" align="center"><img class="size" src="pic/trans.gif" alt="size"></td></tr>
|
||||
<tr><td class="rowfollow">Example.Show.S01E18.2026.2160p.mkv</td><td class="rowfollow" align="right">5.04 GB</td></tr>
|
||||
<tr><td class="rowfollow">Example.Show.S01E19.2026.2160p.mkv</td><td class="rowfollow" align="right">4.98 GB</td></tr>
|
||||
</table>`;
|
||||
@@ -29,7 +29,8 @@
|
||||
},
|
||||
"detail": {
|
||||
"page": {
|
||||
"labelSelectors": ["div.font-bold.leading-6", "span.font-bold"]
|
||||
"labelSelectors": ["div.font-bold.leading-6", "span.font-bold"],
|
||||
"fileListPath": "viewfilelist.php?id={id}"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user