From cabd772243b4cb42088321bc6dcc0354e9f3a886 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 29 Sep 2026 17:30:25 +0800 Subject: [PATCH] fix(sites): normalize HHClub RSS items, complete detail page parsing (HH-2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - RSS: split NexusPHP "[category]name[subtitle][size][uploader]" titles in the site adapter layer (category id/name/media type, clean title, subtitle, size); snapshot fields win on merge; lazily renormalize old RSS rows and re-emit release.observed; unknown list fields show an em dash - Detail: HHClub section layout (douban info block as description, screenshots section, IMDb/TMDB from MediaInfo lines, upload time field, on-demand viewfilelist.php once); site CDN screenshots get a Referer - Site request count now includes site-bound RSS fetches - Origin view shows "自动化" when idle and "Agent" only after recent automation actions (lastAutomationAt); agent step titles no longer get squeezed to a sliver by long notes Co-Authored-By: Claude Opus 5.5 --- .../browser/__tests__/browser-e2e.test.ts | 5 +- apps/server/src/modules/browser/service.ts | 8 +- .../sources-identity/routes/workbench.ts | 12 +- .../sources-identity/service/detail-page.ts | 32 ++++ .../modules/sources-identity/service/feeds.ts | 107 ++++++++++- .../sources-identity/rss-normalize.test.ts | 181 ++++++++++++++++++ .../components/agent/agent-interaction.tsx | 2 +- apps/web/app/components/agentui/todo-list.tsx | 8 +- .../web/app/components/browser/coords.test.ts | 17 +- apps/web/app/components/browser/coords.ts | 23 ++- .../app/components/browser/desktop-view.tsx | 8 +- .../components/browser/site-origin-view.tsx | 4 +- .../app/components/search/release-results.tsx | 4 +- .../workbench/release-detail-sheet.tsx | 2 +- .../components/workbench/release-fields.ts | 16 +- .../app/components/workbench/release-list.tsx | 24 +-- docs/runbook/browser.md | 3 + docs/runbook/sites.md | 8 +- .../src/sites/definitions/hhanclub.json | 3 +- .../adapters/src/sites/definitions/schema.ts | 1 + packages/adapters/src/sites/detail-text.ts | 7 +- packages/adapters/src/sites/feed-item.test.ts | 65 +++++++ packages/adapters/src/sites/feed-item.ts | 123 ++++++++++++ packages/adapters/src/sites/index.ts | 6 + packages/adapters/src/sites/spider/adapter.ts | 12 ++ .../src/sites/spider/detail-page.test.ts | 60 +++++- .../adapters/src/sites/spider/detail-page.ts | 68 +++++-- packages/adapters/src/sites/spider/detail.ts | 16 +- packages/adapters/src/sites/spider/schema.ts | 5 + packages/contracts/src/browser.ts | 2 + .../sites/fixtures/definitions/hhanclub.ts | 46 +++++ .../mp-sync/movieclaw-overrides/hhanclub.json | 3 +- 32 files changed, 816 insertions(+), 65 deletions(-) create mode 100644 apps/server/test/sources-identity/rss-normalize.test.ts create mode 100644 packages/adapters/src/sites/feed-item.test.ts create mode 100644 packages/adapters/src/sites/feed-item.ts diff --git a/apps/server/src/modules/browser/__tests__/browser-e2e.test.ts b/apps/server/src/modules/browser/__tests__/browser-e2e.test.ts index 0cd4bce2..cbd8f66d 100644 --- a/apps/server/src/modules/browser/__tests__/browser-e2e.test.ts +++ b/apps/server/src/modules/browser/__tests__/browser-e2e.test.ts @@ -96,7 +96,10 @@ describe.skipIf(!up)("e2e:登录失效 → 人工接管 → 交还 → Agent 重 const fresh = await invoke("browser.snapshot", { siteId: h.siteId }); expect(fresh).toMatchObject({ controller: "automation", loginRequired: false, waitingReason: null }); expect(fresh.snapshot.text).toContain("种子列表"); - expect((await h.req(`/browser/sessions/${sid}`)).body.data.needsSnapshot).toBe(false); + const afterAgent = (await h.req(`/browser/sessions/${sid}`)).body.data; + expect(afterAgent.needsSnapshot).toBe(false); + // 原站视图据此显示「Agent 正在操作」;人工动作不刷新它 + expect(Date.now() - Date.parse(afterAgent.lastAutomationAt)).toBeLessThan(60_000); const dl = await invoke("browser.download", { sessionId: sid, target: { ref: "e11" }, timeoutMs: 2_000 }); expect(dl).toMatchObject({ sourceReleaseId: h.releaseId, diff --git a/apps/server/src/modules/browser/service.ts b/apps/server/src/modules/browser/service.ts index 26c1700a..0a957717 100644 --- a/apps/server/src/modules/browser/service.ts +++ b/apps/server/src/modules/browser/service.ts @@ -147,6 +147,8 @@ export interface RecoveryState { downloads?: { sha256: string; artifactId: string; sourceReleaseId: string | null; planId: string | null }[]; /** BR-2:打开时的凭据注入结果与最近一次回写检查 */ credential?: BrowserCredentialState; + /** 最近一次自动化(Agent 能力)动作成功的时间;原站视图据此区分「Agent 正在操作」与「无人操作」 */ + lastAutomationAt?: string; } /** 有活动的会话每隔多久从浏览器读回一次 Cookie */ @@ -641,6 +643,7 @@ export class BrowserService { needsSnapshot: rs.needsSnapshot === true, waitingReason: row.controller === "waiting" || row.controller === "human" ? (rs.waitingReason ?? null) : null, credential: rs.credential ?? null, + lastAutomationAt: rs.lastAutomationAt ?? null, }; } @@ -1193,7 +1196,10 @@ export class BrowserService { await this.persist(id, loaded.provider).catch(() => {}); throw fromBrowserError(e); } - if (opts.persist !== false) await this.persist(id, loaded.provider, opts.patch?.(out) ?? {}); + const automation: RecoveryState = actor.kind === "automation" ? { lastAutomationAt: this.now().toISOString() } : {}; + if (opts.persist !== false) + await this.persist(id, loaded.provider, { ...(opts.patch?.(out) ?? {}), ...automation }); + else if (automation.lastAutomationAt) await this.patchRecovery(await this.sessionRow(id), automation); this.maybeSyncCookies(row); return out; } diff --git a/apps/server/src/modules/sources-identity/routes/workbench.ts b/apps/server/src/modules/sources-identity/routes/workbench.ts index d9156514..623a84d3 100644 --- a/apps/server/src/modules/sources-identity/routes/workbench.ts +++ b/apps/server/src/modules/sources-identity/routes/workbench.ts @@ -5,7 +5,7 @@ import { errorBody } from "../../../http/errors.ts"; import type { ModuleApp } from "../../types.ts"; import { route } from "../http.ts"; import type { SourcesRuntime } from "../runtime.ts"; -import { detailScreenshotUrl } from "../service/detail-page.ts"; +import { detailScreenshotRequest } from "../service/detail-page.ts"; import { fetchPublicImage, imageResponse } from "../service/image-proxy.ts"; import { listSiteReleases, releaseDetail } from "../service/workbench.ts"; @@ -53,9 +53,13 @@ export function workbenchRoutes(app: ModuleApp, rt: () => SourcesRuntime) { return c.json(errorBody("bad_request", "截图地址无效"), 400); } const r = rt(); - const url = await detailScreenshotUrl(r, rid, index); - if (!url) return c.json(errorBody("not_found", "截图不存在或详情缓存已过期"), 404); - const got = await fetchPublicImage(r.overrides.fetch ?? fetch, url, { maxBytes: SHOT_MAX_BYTES, label: "截图" }); + const req = await detailScreenshotRequest(r, rid, index); + if (!req) return c.json(errorBody("not_found", "截图不存在或详情缓存已过期"), 404); + const got = await fetchPublicImage(r.overrides.fetch ?? fetch, req.url, { + maxBytes: SHOT_MAX_BYTES, + label: "截图", + ...(req.referer ? { referer: req.referer } : {}), + }); if (!got.ok) return c.json(errorBody("upstream_unavailable", got.message), 502); return imageResponse(got, 86_400); }); diff --git a/apps/server/src/modules/sources-identity/service/detail-page.ts b/apps/server/src/modules/sources-identity/service/detail-page.ts index dadeb58f..647583f9 100644 --- a/apps/server/src/modules/sources-identity/service/detail-page.ts +++ b/apps/server/src/modules/sources-identity/service/detail-page.ts @@ -63,6 +63,38 @@ export async function detailScreenshotUrl(rt: SourcesRuntime, srId: string, inde return (await loadStored(rt, srId))?.screenshots[index] ?? null; } +/** 根域(去 www.、取最后两级):pic.hhanclub.net 与 hhanclub.net 视为同站 */ +function rootHost(u: string): string | null { + try { + const host = new URL(u).hostname.toLowerCase().replace(/^www\./, ""); + return /^[\d.]+$/.test(host) || host.includes(":") ? host : host.split(".").slice(-2).join("."); + } catch { + return null; + } +} + +/** + * 截图代理请求:原始地址 + 站点自家图床(同根域,如 pic.hhanclub.net / proxy-cover.hhanclub.net)要带的 Referer。 + * 站点 CDN 常做防盗链,只认本站 Referer;外部图床不带 Referer。任何情况下都不带 Cookie。 + */ +export async function detailScreenshotRequest( + rt: SourcesRuntime, + srId: string, + index: number, +): Promise<{ url: string; referer?: string } | null> { + const url = await detailScreenshotUrl(rt, srId, index); + if (!url) return null; + const site = await rt.ctx.db + .selectFrom("source_release as r") + .innerJoin("site as s", "s.id", "r.site_id") + .select(["s.base_url"]) + .where("r.id", "=", srId) + .executeTakeFirst(); + const siteRoot = site ? rootHost(site.base_url) : null; + if (siteRoot && rootHost(url) === siteRoot) return { url, referer: `${new URL(site!.base_url).origin}/` }; + return { url }; +} + export async function loadDetailPage(rt: SourcesRuntime, srId: string): Promise { const p = await loadStored(rt, srId); if (!p) return null; diff --git a/apps/server/src/modules/sources-identity/service/feeds.ts b/apps/server/src/modules/sources-identity/service/feeds.ts index 2882a2f0..35e09b05 100644 --- a/apps/server/src/modules/sources-identity/service/feeds.ts +++ b/apps/server/src/modules/sources-identity/service/feeds.ts @@ -7,6 +7,7 @@ import { type FeedFetchState, type FeedItem, getSiteDefinition, + normalizeSiteFeedItem, } from "@umt/adapters"; import type { FeedHealth } from "@umt/contracts"; import { type FeedSourceRow, type SiteRow, sql } from "@umt/db"; @@ -157,6 +158,9 @@ export async function ingestFeed( const url = await secrets.resolve(feed.url_secret_ref); const result = await rt.rss.fetchFeed(url, stateFromRow(feed), { ignoreBackoff: !!opts.manual }); + // 挂站点的 Feed:每次真正发出的 RSS 请求也记进站点请求计数(熔断 / 退避跳过的不算) + if (site && result.kind !== "skipped") rt.countSiteRequest(site.id); + if (site) await renormalizeFeedReleases(rt, site).catch(() => {}); const now = rt.now(); const auth = isAuthFailure(result); const health: FeedHealth = auth ? "failing" : healthOf(result); @@ -175,7 +179,7 @@ export async function ingestFeed( if (result.kind === "ok") { const applied = applyFeedItems(window, result.feed.items); window = applied.cursor; - const counts = await writeItems(rt, trx, feed, site?.id ?? null, applied.newItems, applied.updatedItems); + const counts = await writeItems(rt, trx, feed, site ?? null, applied.newItems, applied.updatedItems); const unchanged = result.feed.items .filter((i) => !applied.newItems.includes(i) && !applied.updatedItems.includes(i)) .map((i) => i.guid); @@ -312,16 +316,36 @@ function observationRaw(item: FeedItem) { }; } +/** 规范化后的 RSS 条目字段(写进 source_release.raw.rss,v 为规范化版本) */ +export const RSS_NORMALIZE_VERSION = 1; + +function feedFields(siteKey: string, item: Pick) { + const n = normalizeSiteFeedItem(siteKey, item.title); + // 分类:标题前缀 → RSS (可能是名称) + const catName = n.category.name ?? item.categories[0] ?? null; + return { + title: n.title, + subtitle: n.subtitle, + categoryRaw: n.category.id ?? catName, + categoryName: catName, + mediaType: n.mediaType, + // enclosure length 是精确字节数,标题尾部的大小是四舍五入后的显示值 + sizeBytes: item.size > 0 ? item.size : n.sizeBytes, + uploader: n.uploader, + }; +} + async function writeItems( rt: SourcesRuntime, trx: Exec, feed: FeedSourceRow, - siteId: string | null, + site: Pick | null, newItems: FeedItem[], updatedItems: FeedItem[], ): Promise<{ newReleases: number }> { let newReleases = 0; const updated = new Set(updatedItems); + const siteId = site?.id ?? null; for (const item of [...newItems, ...updatedItems]) { const raw = observationRaw(item); const obs = ( @@ -337,7 +361,7 @@ async function writeItems( returning id, source_release_id, (select content_digest from prev) as prev_digest`.execute(trx) ).rows[0]!; - if (!siteId) { + if (!site || !siteId) { // 无站点的通用 Feed:只留观察,无法按站点资源聚合 await trx .updateTable("observation") @@ -349,20 +373,36 @@ async function writeItems( const srid = extractSiteReleaseId(item) ?? `guid:${createHash("sha1").update(item.guid).digest("hex").slice(0, 20)}`; const contentChanged = updated.has(item) || (obs.prev_digest !== null && obs.prev_digest !== item.rawDigest); - const sizeBytes = item.size > 0 ? item.size : null; + const f = feedFields(site.key, item); + const rssRaw = { + link: item.link, + categories: item.categories, + rawTitle: item.title, + categoryName: f.categoryName, + mediaType: f.mediaType, + ...(f.uploader ? { uploader: f.uploader } : {}), + v: RSS_NORMALIZE_VERSION, + }; + // 与列表快照按站点资源 ID 合并:快照见过(raw._site)的行,标题 / 副标题 / 分类 / 做种 / 促销 / H&R 以快照为准, + // RSS 只补空缺;快照没见过的行,内容变化时用 RSS 的新值 const rel = ( await sql<{ id: string; inserted: boolean; prev_title: string | null; resolution_locked: boolean }>` with prev as (select title from source_release where site_id = ${siteId} and site_release_id = ${srid}) - insert into source_release (site_id, site_release_id, title, category_raw, size_bytes, source_published_at, raw) - values (${siteId}, ${srid}, ${item.title}, ${item.categories[0] ?? null}, ${sizeBytes}, ${item.publishedAt ?? null}, - ${jsonb({ rss: { link: item.link, categories: item.categories } })}::jsonb) + insert into source_release (site_id, site_release_id, title, subtitle, category_raw, size_bytes, source_published_at, raw) + values (${siteId}, ${srid}, ${f.title}, ${f.subtitle}, ${f.categoryRaw}, ${f.sizeBytes}, ${item.publishedAt ?? null}, + ${jsonb({ rss: rssRaw })}::jsonb) on conflict (site_id, site_release_id) do update set last_seen_at = now(), - title = case when ${contentChanged} then excluded.title else source_release.title end, + title = case when source_release.raw ? '_site' then source_release.title + when ${contentChanged} then excluded.title else source_release.title end, + subtitle = case when source_release.raw ? '_site' then coalesce(source_release.subtitle, excluded.subtitle) + when ${contentChanged} then coalesce(excluded.subtitle, source_release.subtitle) + else coalesce(source_release.subtitle, excluded.subtitle) end, size_bytes = case when ${contentChanged} then coalesce(excluded.size_bytes, source_release.size_bytes) else coalesce(source_release.size_bytes, excluded.size_bytes) end, source_published_at = coalesce(source_release.source_published_at, excluded.source_published_at), - category_raw = coalesce(source_release.category_raw, excluded.category_raw) + category_raw = coalesce(source_release.category_raw, excluded.category_raw), + raw = source_release.raw || excluded.raw returning id, (xmax = 0) as inserted, (select title from prev) as prev_title, resolution_locked`.execute(trx) ).rows[0]!; await trx @@ -379,7 +419,7 @@ async function writeItems( { sourceReleaseId: rel.id, observationId: obs.id, siteId, isNew: true }, { idempotencyKey: `release.observed:${rel.id}` }, ); - } else if (contentChanged && rel.prev_title !== item.title && !rel.resolution_locked) { + } else if (contentChanged && rel.prev_title !== f.title && !rel.resolution_locked) { // 标题修正:按新标题重新识别(证据版本变化) await rt.ctx.outbox.emit( trx, @@ -391,3 +431,50 @@ async function writeItems( } return { newReleases }; } + +/** + * 旧 RSS 行补规范化:规范化上线前写进来的站点 RSS 行(raw.rss 没有 v,标题还是 `[分类]…[大小]` 整串), + * 在该站 Feed 下次抓取时按原标题重算一次(每次最多 500 行,不请求站点)。快照见过的行不动。 + * 标题变了且未锁定的,重新发 release.observed 让身份解析按干净标题再识别。 + */ +export async function renormalizeFeedReleases(rt: SourcesRuntime, site: Pick): Promise { + const rows = await rt.ctx.db + .selectFrom("source_release") + .select(["id", "title", "subtitle", "category_raw", "size_bytes", "raw", "resolution_locked"]) + .where("site_id", "=", site.id) + .where(sql`raw ? 'rss'`) + .where(sql`not (raw ? '_site')`) + .where(sql`coalesce((raw->'rss'->>'v')::int, 0) < ${RSS_NORMALIZE_VERSION}`) + .limit(500) + .execute(); + if (!rows.length) return 0; + await rt.ctx.db.transaction().execute(async (trx) => { + for (const r of rows) { + const rss = ((r.raw ?? {}) as { rss?: Record }).rss ?? {}; + const rawTitle = typeof rss.rawTitle === "string" ? rss.rawTitle : r.title; + const categories = Array.isArray(rss.categories) ? (rss.categories as string[]) : []; + const f = feedFields(site.key, { title: rawTitle, size: Number(r.size_bytes ?? 0), categories }); + const nextRss = { + ...rss, + rawTitle, + categoryName: f.categoryName, + mediaType: f.mediaType, + ...(f.uploader ? { uploader: f.uploader } : {}), + v: RSS_NORMALIZE_VERSION, + }; + await sql`update source_release set title = ${f.title}, subtitle = coalesce(subtitle, ${f.subtitle}), + category_raw = coalesce(${f.categoryRaw}, category_raw), size_bytes = coalesce(size_bytes, ${f.sizeBytes}), + raw = raw || jsonb_build_object('rss', ${JSON.stringify(nextRss)}::jsonb) + where id = ${r.id}`.execute(trx); + if (f.title !== r.title && !r.resolution_locked) { + await rt.ctx.outbox.emit( + trx, + "release.observed", + { sourceReleaseId: r.id, observationId: null, siteId: site.id, isNew: false }, + { idempotencyKey: `release.observed:${r.id}:rss-norm-${RSS_NORMALIZE_VERSION}` }, + ); + } + } + }); + return rows.length; +} diff --git a/apps/server/test/sources-identity/rss-normalize.test.ts b/apps/server/test/sources-identity/rss-normalize.test.ts new file mode 100644 index 00000000..6579d319 --- /dev/null +++ b/apps/server/test/sources-identity/rss-normalize.test.ts @@ -0,0 +1,181 @@ +import type { SiteRelease } from "@umt/adapters"; +import { createMockRssServer, type MockFeedItem, type MockRssServer } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { ingestFeed } from "../../src/modules/sources-identity/service/feeds.ts"; +import { upsertSiteRelease } from "../../src/modules/sources-identity/service/snapshot.ts"; +import { listSiteReleases } from "../../src/modules/sources-identity/service/workbench.ts"; +import { createHarness, type Harness, pgAvailable, seedFeed, seedSite } from "./harness.ts"; + +// HH-2:憨憨 RSS 条目规范化(`[分类]种子名[副标题][大小]`)、与列表快照合并、旧行补规范化、RSS 请求计数 +const up = await pgAvailable(); +const BASE = "https://hhanclub.example"; + +const item = (id: string, title: string, pubDate: string, extra: Partial = {}): MockFeedItem => ({ + guid: `hh-${id}`, + title, + link: `${BASE}/details.php?id=${id}`, + enclosureUrl: `${BASE}/download.php?id=${id}&passkey=PK`, + size: 731525231, + pubDate: new Date(pubDate), + categories: [], + ...extra, +}); + +const snapshotRelease = (id: string, title: string, uploadedAt: string): SiteRelease => ({ + siteId: "hhanclub", + siteReleaseId: id, + title, + subtitle: "快照副标题", + category: { id: "402", name: "电视剧" }, + sizeBytes: 1024, + seeders: 22, + leechers: 66, + snatched: 5, + promotion: { type: "free", downloadFactor: 0, uploadFactor: 1, expiresAt: null }, + hr: false, + uploadedAt, + detailUrl: `${BASE}/details.php?id=${id}`, + tags: [], + raw: {}, + observedAt: new Date().toISOString(), +}); + +describe.skipIf(!up)("站点 RSS 条目规范化(HH-2)", () => { + let h: Harness; + let siteId: string; + const servers: MockRssServer[] = []; + + beforeAll(async () => { + h = await createHarness("rssnorm"); + siteId = (await seedSite(h, { key: "hhanclub", baseUrl: `${BASE}/`, cookie: "c=1" })).site.id; + }); + afterAll(async () => { + for (const s of servers) await s.close(); + await h?.close(); + }); + + const feed = async (items: MockFeedItem[]) => { + const s = await createMockRssServer({ items, passkey: "PK" }); + servers.push(s); + return { server: s, row: await seedFeed(h, { siteId, url: s.feedUrl, name: "憨憨 RSS" }) }; + }; + const rel = (id: string) => + h.db + .selectFrom("source_release") + .selectAll() + .where("site_id", "=", siteId) + .where("site_release_id", "=", id) + .executeTakeFirstOrThrow(); + + it("拆出分类 / 副标题 / 干净标题;大小取 enclosure 精确值;原始标题留在 raw.rss;请求计入站点计数", async () => { + const { row } = await feed([ + item( + "219390", + "[音乐]Travis - L.A. Times (2024) - WEB-DL - 24bit ALAC-HHWEB[Travis - L.A. Times | ALAC | 苹果音乐][697.64 MB]", + "2026-09-29T08:51:01Z", + ), + item("219391", "[纪录片]Some Doc 2026 2160p WEB-DL-HHWEB[新][某纪录片][1.5 GB]", "2026-09-29T08:10:00Z", { + size: 0, + }), + ]); + expect((await ingestFeed(h.rt, row.id)).status).toBe("ok"); + const r = await rel("219390"); + expect(r).toMatchObject({ + title: "Travis - L.A. Times (2024) - WEB-DL - 24bit ALAC-HHWEB", + subtitle: "Travis - L.A. Times | ALAC | 苹果音乐", + category_raw: "410", + }); + expect(Number(r.size_bytes)).toBe(731525231); + expect((r.raw as { rss: Record }).rss).toMatchObject({ + categoryName: "音乐", + mediaType: "music", + rawTitle: expect.stringContaining("[697.64 MB]"), + v: 1, + }); + const doc = await rel("219391"); + expect(doc).toMatchObject({ title: "Some Doc 2026 2160p WEB-DL-HHWEB", subtitle: "某纪录片", category_raw: "404" }); + expect(Number(doc.size_bytes)).toBe(Math.round(1.5 * 1024 ** 3)); + + // 计数是异步写的,等一下 + let n = 0; + for (let i = 0; i < 50 && n === 0; i++) { + n = await h.rt.siteRequestCount(siteId); + if (!n) await new Promise((res) => setTimeout(res, 20)); + } + expect(n).toBe(1); + }); + + it("与列表快照按站点资源 ID 合并:快照补做种 / 促销 / H&R,之后 RSS 更新不再覆盖快照标题", async () => { + const { server, row } = await feed([ + item("219380", "[电视剧]Show S01 2026 1080p WEB-DL-HHWEB[示例剧 | 第1集][1.2 GB]", "2026-09-29T07:00:00Z"), + ]); + await ingestFeed(h.rt, row.id); + await h.db + .transaction() + .execute((trx) => + upsertSiteRelease( + h.rt, + trx, + { id: siteId }, + snapshotRelease("219380", "Show S01 2026 1080p WEB-DL-HHWEB", "2026-09-29T07:00:00Z"), + ), + ); + let r = await rel("219380"); + expect(r).toMatchObject({ seeders: 22, leechers: 66, subtitle: "快照副标题", category_raw: "402" }); + expect((r.raw as { _site: { hr: boolean } })._site.hr).toBe(false); + expect((r.raw as { rss: { categoryName: string } }).rss.categoryName).toBe("电视剧"); + + server.updateItem("hh-219380", { title: "[电视剧]Show S01 2026 1080p WEB-DL-HHWEB[示例剧 | 第1-2集][2.4 GB]" }); + await ingestFeed(h.rt, row.id, { manual: true }); + r = await rel("219380"); + expect(r.title).toBe("Show S01 2026 1080p WEB-DL-HHWEB"); + expect(r.seeders).toBe(22); + }); + + it("规范化前写进来的旧 RSS 行:下次抓取时按原标题补规范化并重新触发识别", async () => { + const dirty = "[音乐]Old Album (2020) - WEB-DL - 16bit ALAC-HHWEB[Old Album | ALAC][120.5 MB]"; + const old = await h.db + .insertInto("source_release") + .values({ + site_id: siteId, + site_release_id: "219001", + title: dirty, + size_bytes: 126353408, + raw: JSON.stringify({ rss: { link: `${BASE}/details.php?id=219001`, categories: [] } }), + }) + .returning(["id"]) + .executeTakeFirstOrThrow(); + const { row } = await feed([ + item("219002", "[电影]New Movie 2026 1080p-HHWEB[新片][4 GB]", "2026-09-29T09:00:00Z"), + ]); + await ingestFeed(h.rt, row.id); + const r = await rel("219001"); + expect(r).toMatchObject({ + title: "Old Album (2020) - WEB-DL - 16bit ALAC-HHWEB", + subtitle: "Old Album | ALAC", + category_raw: "410", + }); + expect((r.raw as { rss: { rawTitle: string } }).rss.rawTitle).toBe(dirty); + const events = (await h.outbox("release.observed")).filter((e) => e.payload.sourceReleaseId === old.id); + expect(events.map((e) => e.payload.isNew)).toEqual([false]); + }); + + it("资源列表按发布时间排序:缺做种 / H&R 的 RSS 条目不会因此排到前面", async () => { + await h.db + .transaction() + .execute((trx) => + upsertSiteRelease( + h.rt, + trx, + { id: siteId }, + snapshotRelease("219500", "Newest Snapshot 2026-HHWEB", "2026-09-29T10:00:00Z"), + ), + ); + const list = await listSiteReleases(h.rt, siteId, { limit: 50, live: "false" }); + const ids = list.items.map((i) => i.siteReleaseId); + expect(ids[0]).toBe("219500"); + expect(ids.indexOf("219390")).toBeLessThan(ids.indexOf("219391")); + const times = list.items.map((i) => Date.parse(i.sourcePublishedAt ?? i.firstSeenAt)); + expect([...times].sort((a, b) => b - a)).toEqual(times); + }); +}); diff --git a/apps/web/app/components/agent/agent-interaction.tsx b/apps/web/app/components/agent/agent-interaction.tsx index b181ab28..4d0ded93 100644 --- a/apps/web/app/components/agent/agent-interaction.tsx +++ b/apps/web/app/components/agent/agent-interaction.tsx @@ -28,7 +28,7 @@ export function planTodoItems(steps: AgentPlanStep[]): TodoItem[] { id: s.id, title: s.title, status: STEP_STATUS[s.status], - ...(s.note ? { detail: {s.note} } : {}), + ...(s.note ? { detail: {s.note} } : {}), })); } diff --git a/apps/web/app/components/agentui/todo-list.tsx b/apps/web/app/components/agentui/todo-list.tsx index b0d32448..f81dafc1 100644 --- a/apps/web/app/components/agentui/todo-list.tsx +++ b/apps/web/app/components/agentui/todo-list.tsx @@ -3,7 +3,7 @@ // 改造:SVG 状态图标保留几何(虚线圈=待办、弧线=进行中、勾=完成、叉=取消),pathLength 弹簧动画换成 // CSS 过渡/animate-spin(motion-reduce 关闭);完成色 emerald → success,取消 rose → destructive,且都配 sr-only 文字; // 表头完成数去掉滚字;折叠用 Disclosure;maxHeight 改成 Tailwind 类;新增 `error` 状态(工具失败)、 -// `skipped` 状态(Agent 步骤清单里跳过的步骤)与 `detail` 槽。 +// `skipped` 状态(Agent 步骤清单里跳过的步骤)与 `detail` 槽(可压缩,标题至少留一半宽度)。 import { ChevronDown, ListTodo } from "lucide-react"; import { type ReactNode, useEffect, useId, useRef, useState } from "react"; import { cn } from "~/lib/utils.ts"; @@ -162,8 +162,10 @@ export function TodoList({ {LABEL[s]}: {item.detail ? ( - {item.detail} + {item.detail} ) : null} ); diff --git a/apps/web/app/components/browser/coords.test.ts b/apps/web/app/components/browser/coords.test.ts index ef5ab875..539f9a30 100644 --- a/apps/web/app/components/browser/coords.test.ts +++ b/apps/web/app/components/browser/coords.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { holderName, toViewport, waitingText } from "./coords.ts"; +import { agentActive, controllerLabel, holderName, toViewport, waitingText } from "./coords.ts"; describe("原站视图坐标换算", () => { const natural = { width: 1280, height: 800 }; @@ -29,3 +29,18 @@ describe("原站视图坐标换算", () => { expect(holderName("admin_session:alice")).toBe("alice"); }); }); + +describe("当前控制文案", () => { + const now = Date.parse("2026-09-29T10:00:00Z"); + it("自动化且没有最近动作 → 「自动化」;两分钟内有自动化动作 → 「Agent」", () => { + expect(controllerLabel({ controller: "automation", lastAutomationAt: null }, now)).toBe("自动化"); + expect(controllerLabel({ controller: "automation" }, now)).toBe("自动化"); + expect(controllerLabel({ controller: "automation", lastAutomationAt: "2026-09-29T09:50:00Z" }, now)).toBe("自动化"); + expect(controllerLabel({ controller: "automation", lastAutomationAt: "2026-09-29T09:59:10Z" }, now)).toBe("Agent"); + expect(agentActive({ controller: "automation", lastAutomationAt: "2026-09-29T09:59:10Z" }, now)).toBe(true); + }); + it("人工接管 / 等待处理不看自动化时间", () => { + expect(controllerLabel({ controller: "human", lastAutomationAt: "2026-09-29T09:59:50Z" }, now)).toBe("人工接管"); + expect(controllerLabel({ controller: "waiting" }, now)).toBe("等待处理"); + }); +}); diff --git a/apps/web/app/components/browser/coords.ts b/apps/web/app/components/browser/coords.ts index 582dae94..7b01e0a9 100644 --- a/apps/web/app/components/browser/coords.ts +++ b/apps/web/app/components/browser/coords.ts @@ -17,12 +17,33 @@ export function toViewport( } export const CONTROLLER_LABEL: Record = { - automation: "Agent", + automation: "自动化", human: "人工接管", waiting: "等待处理", closed: "已关闭", }; +/** 自动化动作在这段时间内发生过,算「Agent 正在操作」 */ +export const AGENT_ACTIVE_MS = 2 * 60_000; + +/** Agent 是否正在用这个会话:控制者是自动化,且最近 AGENT_ACTIVE_MS 内有过自动化动作 */ +export function agentActive( + s: { controller: BrowserController; lastAutomationAt?: string | null }, + now = Date.now(), +): boolean { + if (s.controller !== "automation" || !s.lastAutomationAt) return false; + const t = new Date(s.lastAutomationAt).getTime(); + return Number.isFinite(t) && now - t < AGENT_ACTIVE_MS; +} + +/** 「当前控制」文案:无人操作时是「自动化」,Agent 正在操作才显示「Agent」 */ +export function controllerLabel( + s: { controller: BrowserController; lastAutomationAt?: string | null }, + now = Date.now(), +): string { + return agentActive(s, now) ? "Agent" : CONTROLLER_LABEL[s.controller]; +} + const WAITING_TEXT: Record = { captcha: "登录页需要验证码", challenge: "页面需要人工验证", diff --git a/apps/web/app/components/browser/desktop-view.tsx b/apps/web/app/components/browser/desktop-view.tsx index 2ec8d63e..0deb9b33 100644 --- a/apps/web/app/components/browser/desktop-view.tsx +++ b/apps/web/app/components/browser/desktop-view.tsx @@ -9,7 +9,7 @@ import { StatusBadge, type StatusTone } from "~/components/console/status-badge. import { Button } from "~/components/ui/button.tsx"; import { apiData, errorMessage } from "~/lib/api.ts"; import { qk } from "~/lib/query-keys.ts"; -import { CONTROLLER_LABEL, credentialBadge, holderName, waitingText } from "./coords.ts"; +import { agentActive, controllerLabel, credentialBadge, holderName, waitingText } from "./coords.ts"; import { SiteOriginView } from "./site-origin-view.tsx"; // 原站视图桌面模式(docs/design/webtop-remote-browser.md §5.7):iframe 嵌浏览器桌面(Selkies), @@ -128,7 +128,7 @@ function DesktopView({
{conn.text} - 当前控制：{CONTROLLER_LABEL[s.controller]} + 当前控制：{controllerLabel(s)} {human && leaseHolder ? `(${leaseHolder})` : ""} {waiting ? {waiting} : null} @@ -197,7 +197,9 @@ function DesktopView({

{mine ? "直接在画面里点击和输入。中文、日文用本机输入法输入；复制粘贴用 Ctrl+C / Ctrl+V。" - : "只读观看，Agent 可能正在操作。需要自己点或输入时先接管。"} + : agentActive(s) + ? "只读观看，Agent 正在操作。需要自己点或输入时先接管，Agent 会暂停。" + : "只读观看，目前没有 Agent 在操作。需要自己点或输入时先接管。"}

diff --git a/apps/web/app/components/browser/site-origin-view.tsx b/apps/web/app/components/browser/site-origin-view.tsx index 7d632422..4f853d5c 100644 --- a/apps/web/app/components/browser/site-origin-view.tsx +++ b/apps/web/app/components/browser/site-origin-view.tsx @@ -34,7 +34,7 @@ import { StatusBadge, type StatusTone } from "~/components/console/status-badge. import { Button } from "~/components/ui/button.tsx"; import { Input } from "~/components/ui/input.tsx"; import { errorMessage } from "~/lib/api.ts"; -import { CONTROLLER_LABEL, credentialBadge, holderName, toViewport, waitingText } from "./coords.ts"; +import { controllerLabel, credentialBadge, holderName, toViewport, waitingText } from "./coords.ts"; // 原站视图(REQ §4.7.1、§4.7.3):同一个 camofox 会话的定时截图 + 人工输入映射。 // 人工接管前服务端先停掉该会话的自动化输入;交还后 Agent 必须重新读取页面。 @@ -146,7 +146,7 @@ export function SiteOriginView({ site }: { site: SiteListItem }) {
{conn.text} - 当前控制：{CONTROLLER_LABEL[s.controller]} + 当前控制：{controllerLabel(s)} {human && leaseHolder ? `(${leaseHolder})` : ""} {waiting ? {waiting} : null} diff --git a/apps/web/app/components/search/release-results.tsx b/apps/web/app/components/search/release-results.tsx index 989b761d..6e77d665 100644 --- a/apps/web/app/components/search/release-results.tsx +++ b/apps/web/app/components/search/release-results.tsx @@ -390,9 +390,9 @@ function ReleaseRow({ r, onDownload }: { r: SearchRelease; onDownload: () => voi {r.sizeBytes === null ? "大小未知" : formatBytes(r.sizeBytes, 1)} - {r.seeders ?? "?"} + {r.seeders ?? "—"} - {r.leechers ?? "?"} + {r.leechers ?? "—"} 发布 {relTime(r.uploadedAt)} {tech ? {tech} : null} diff --git a/apps/web/app/components/workbench/release-detail-sheet.tsx b/apps/web/app/components/workbench/release-detail-sheet.tsx index 4e384974..4bbfeb14 100644 --- a/apps/web/app/components/workbench/release-detail-sheet.tsx +++ b/apps/web/app/components/workbench/release-detail-sheet.tsx @@ -217,7 +217,7 @@ export function ReleaseDetailSheet({ { label: "促销截止", value: formatDateTime(detail.data.volatile.promotionExpiresAt) }, { label: "做种 / 下载", - value: `${r.seeders ?? "?"} / ${r.leechers ?? "?"}（${relTime(detail.data.volatile.peersObservedAt)}观察）`, + value: `${r.seeders ?? "—"} / ${r.leechers ?? "—"}（${relTime(detail.data.volatile.peersObservedAt)}观察）`, }, { label: "详情抓取于", value: formatDateTime(detail.data.volatile.detailFetchedAt) }, ]} diff --git a/apps/web/app/components/workbench/release-fields.ts b/apps/web/app/components/workbench/release-fields.ts index 6cbfd7c7..e520fb3f 100644 --- a/apps/web/app/components/workbench/release-fields.ts +++ b/apps/web/app/components/workbench/release-fields.ts @@ -16,9 +16,23 @@ export function hrOf(r: Pick): "yes" | "no" | "unknown" { return hr === true ? "yes" : hr === false ? "no" : "unknown"; } +/** 站点 RSS 规范化后的字段(raw.rss):分类名、媒体大类 */ +function rssExtra(r: Pick): { categoryName?: unknown } { + const s = (r.raw as { rss?: { categoryName?: unknown } }).rss; + return s && typeof s === "object" ? s : {}; +} + +/** 分类名:列表快照的站点分类名 → RSS 标题前缀里的分类名 → 站点分类 ID */ export function categoryLabel(r: Pick): string | null { const name = siteExtra(r).categoryName; - return typeof name === "string" && name ? name : r.categoryRaw; + if (typeof name === "string" && name) return name; + const rss = rssExtra(r).categoryName; + return typeof rss === "string" && rss ? rss : r.categoryRaw; +} + +/** 列表里的数字:未知显示「—」(未知 ≠ 0) */ +export function countText(n: number | null | undefined): string { + return typeof n === "number" && Number.isFinite(n) ? String(n) : "—"; } const PROMO_LABEL: Record = { diff --git a/apps/web/app/components/workbench/release-list.tsx b/apps/web/app/components/workbench/release-list.tsx index 809a0334..29594f5f 100644 --- a/apps/web/app/components/workbench/release-list.tsx +++ b/apps/web/app/components/workbench/release-list.tsx @@ -6,6 +6,7 @@ import { cn, formatBytes } from "~/lib/utils.ts"; import { absTime, categoryLabel, + countText, hrOf, ORIGIN_LABEL, promotionLabel, @@ -15,8 +16,11 @@ import { snatchedOf, } from "./release-fields.ts"; -/** 促销 / H&R 徽章(列表与详情共用):促销带剩余时间,过期的写明 */ -export function ReleaseBadges({ r }: { r: WorkbenchRelease }) { +/** + * 促销 / H&R 徽章(列表与详情共用):促销带剩余时间,过期的写明。 + * H&R 未知只在 showUnknownHr 时标出(详情里);列表行不为未知字段挂徽章,避免 RSS 条目满屏「未知」。 + */ +export function ReleaseBadges({ r, showUnknownHr = false }: { r: WorkbenchRelease; showUnknownHr?: boolean }) { const promo = promotionLabel(r); const hr = hrOf(r); const left = promo && !promo.expired ? remainingLabel(r.promotionExpiresAt) : null; @@ -31,7 +35,7 @@ export function ReleaseBadges({ r }: { r: WorkbenchRelease }) { ) : null} {hr === "yes" ? H&R : null} - {hr === "unknown" ? H&R 未知 : null} + {hr === "unknown" && showUnknownHr ? H&R 未知 : null} ); } @@ -159,19 +163,17 @@ export function ReleaseRow({ {r.subtitle ?? ""} - - {cat ?? "分类未知"} - - - {r.sizeBytes === null ? "大小未知" : formatBytes(r.sizeBytes, 1)} + + {cat ?? "—"} + {r.sizeBytes === null ? "—" : formatBytes(r.sizeBytes, 1)} - {r.seeders ?? "?"} + {countText(r.seeders)} - {r.leechers ?? "?"} + {countText(r.leechers)} - {snatched ?? "?"} + {countText(snatched)} {relTime(r.sourcePublishedAt)} diff --git a/docs/runbook/browser.md b/docs/runbook/browser.md index 2ca084b6..80d58258 100644 --- a/docs/runbook/browser.md +++ b/docs/runbook/browser.md @@ -291,6 +291,9 @@ pixelflux + labwc Wayland 桌面 + Selkies 2.0 Web 客户端)+ camofox 1.17.0 ** 原站视图 Tab 调 `GET /api/v1/browser/desktop/{sessionId}/_status` 决定模式:`available`(容器 3000 能连上且有访问凭据) 且 `window.isSecureContext` → 桌面模式(iframe);否则截图模式,顶部写原因:"桌面模式不可用:…" 或 "桌面模式需要通过 HTTPS 访问控制台"。 桌面模式顶栏:连接状态(已连接,只读 / 可操作)、当前控制者、接管 / 交还(沿用 `control/acquire|release`)、重新连接、截图模式、关闭会话。 +「当前控制」(HH-2,两种模式共用 `coords.ts controllerLabel`):`controller = automation` 且 2 分钟内有过自动化动作(Agent 能力经 +`BrowserService.run` 成功执行,写 `recovery_state.lastAutomationAt`,会话视图字段 `lastAutomationAt`)才显示「Agent」,说明写 +「Agent 正在操作」;否则显示「自动化」,说明写「目前没有 Agent 在操作」。人工接管显示「人工接管(用户名)」,人工动作不刷新该时间。 接管后直接在画面里点和输入;中文、日文用**本机输入法**(Selkies 接收 composition 事件,按 Unicode 注入上屏文本); 复制粘贴用 Ctrl+C / Ctrl+V。 diff --git a/docs/runbook/sites.md b/docs/runbook/sites.md index 09df0ba6..e2523045 100644 --- a/docs/runbook/sites.md +++ b/docs/runbook/sites.md @@ -54,6 +54,10 @@ - **站点内添加**(站点详情「快照与 Feed」→ 新建 Feed,`components/sites/site-feed-dialog.tsx`):只填地址、用途(发现 = `generic` / 追踪 = `search` / 促销 = `promotion`,缺省发现)、可选备注(即名称)。地址输入时下方预览脱敏后的显示值;域名与站点不符给黄色提示但允许保存。保存后立即 `POST /feed-sources/{id}/fetch` 一次,对话框每 2 s 轮询 `GET /feed-sources/{id}`(最多 90 s),显示首抓条目数(游标窗口 `lastGuidSet` 长度)或 `lastError`。 - **`POST /feed-sources` 带 `siteId` 时的推断**(`service/feeds.ts feedDefaultsForSite`):`siteAccountId` 缺省取该站启用账号,优先带 passkey 的,再取最早建的;`parserVersion` 记为 `rss@1;<站点定义 umt.version>`(解析仍走通用 RSS 解析器,版本只作追溯);`name` 缺省「站点名 RSS」;`pollIntervalSec` 缺省 1800(站点请求克制;不挂站点的 Feed 仍缺省 900)。地址主机键(去 `www.`、取最后两级,IP 原样)与站点 `baseUrl` / 定义 `domain` 都不一致时,响应附 `warnings[]`(只在创建响应出现)。`name` 现在可省,不挂站点时缺省为地址主机名。 +- **站点 RSS 条目规范化(HH-2,`packages/adapters/src/sites/feed-item.ts normalizeSiteFeedItem`)**:NexusPHP `torrentrss.php` 的标题是 `[分类]种子名[副标题][大小][上传者]`(憨憨实测 `[音乐]Travis - L.A. Times (2024) - WEB-DL - 24bit ALAC-HHWEB[Travis - L.A. Times | ALAC | …][697.64 MB]`)。挂站点的 Feed 入库时按站点定义拆开:前缀只认定义里登记过的分类名(`umt.categoryNames` / `category.*.desc`),换成站点分类 ID 写 `category_raw`(与列表快照一致,资源页分类筛选可用);尾部配平地剥最多 3 段方括号,只有出现大小段时才把它前面一段当副标题、后面一段当上传者;去掉「[新]」。大小优先用 enclosure `length`(精确字节),没有才用标题里的显示值。`source_release.raw.rss` 记 `rawTitle / categoryName / mediaType(movie·tv·music·null) / uploader / v`。不改标题解析器(`[分类]` 前缀引起的误识别归 ID-2)。 +- **与列表快照合并**:同站同资源 ID 一行。快照见过的行(`raw._site`)标题 / 副标题 / 分类 / 做种 / 促销 / H&R 以快照为准,RSS 只补空缺、更新 `raw.rss`;快照没见过的行,RSS 内容变化时用新值。快照后到时照常覆盖 RSS 值。 +- **旧行补规范化**:规范化上线前写入的 RSS 行(`raw.rss.v` 缺失)在该站任一 Feed 下次抓取时按原标题重算(每次最多 500 行,不请求站点);标题变了且未锁定的重新发 `release.observed`(幂等键 `…:rss-norm-1`)。 +- **资源列表展示**:列表按 `coalesce(发布时间, 首次发现)` 倒序,和字段是否齐全无关。RSS 独有的条目(憨憨的音乐区不在 `torrents.php` 快照里)没有做种 / 下载 / 完成、H&R,列表显示「—」,不挂「H&R 未知」徽章(详情抽屉的 H&R 字段仍写「未知」);分类名取快照分类名 → `raw.rss.categoryName` → 分类 ID。 - **来源页任意 RSS**(`components/workbench/feed-form.tsx`):完整表单不变;新建时地址域名匹配已登记站点会自动选中该站和账号并提示「按地址域名关联到 X · 不关联」,手动改过站点后不再自动改。 ## 定时任务 @@ -89,7 +93,7 @@ 站点定义可带 `umt.defaults`(`snapshotIntervalSec / snapshotMaxPages / healthCheckIntervalSec / userDataIntervalSec / searchConcurrency / detailCacheSec`),是按站点实测给的保守下限;运行时 `siteAccessFor(def, settings.sites)` 合并:周期、缓存取较长,页数取较小;`searchConcurrency`(缺省 1)让同站不同关键词的实时补查排队执行。目前只有憨憨配置了(快照 60 分钟 1 页、健康检测 6 小时、用户数据 6 小时、搜索并发 1、详情缓存 24 小时、限流 3 秒 1 个令牌突发 2)。 -每从站点限流器拿到一个令牌(列表、搜索、详情、下载、登录、用户数据、原站视图导航)记一次请求,写 `site_request_count`(迁移 `0046`,按小时聚合,多进程共用,保留 7 天)。`GET /sites/{id}/request-stats`(read)返回最近 24 小时总数、按小时明细、快照实际周期与合并后的访问默认;站点页概览「近 24 小时请求」与资源页列表头显示。计数只在升级后开始累计。 +每从站点限流器拿到一个令牌(列表 / 快照、搜索、详情、详情的文件清单、下载、健康检测、登录、用户数据、原站视图导航)记一次请求;挂站点的 Feed 每次真正发出 RSS 请求(不含熔断 / 退避跳过)也记一次(HH-2 之前漏算)。写 `site_request_count`(迁移 `0046`,按小时聚合,多进程共用,重启不丢,保留 7 天)。`GET /sites/{id}/request-stats`(read)返回最近 24 小时总数、按小时明细、快照实际周期与合并后的访问默认;站点页概览「近 24 小时请求」与资源页列表头显示。计数从迁移 0046 上线后开始累计,之前的请求没有记录,无法回填;RSS 请求从 HH-2 起计入。 ## 详情页扩展内容(INT-2) @@ -99,6 +103,8 @@ - 缓存:内容放 PG `kv_cache` region `site.detail`,key = `source_release.id`,TTL 取站点详情缓存期(至少 24 小时)。缓存期内 `detail_fetched_at` 新鲜且缓存还在就不请求站点;kv_cache 被清掉(过期 / 重启丢 UNLOGGED 表)后下次刷新会重新抓一次。 - 外部 ID 另写 `source_release.raw._detail`(`imdbId / doubanId / tmdb{type,id}`),列表快照 upsert 不覆盖它。身份解析把它和列表给的 `raw._site.imdbId / doubanId` 一起当显式 ID:本地召回查已接受的外部身份;需要查 TMDB 时,有 TMDB ID 直接取详情,否则 IMDb ID 走 `/find/{imdb}?external_source=imdb_id`,命中就不再按标题搜索;TMDB 不支持豆瓣 ID,豆瓣只用于本地召回。是否查 TMDB 仍按惰性识别规则(DEC-1),外部 ID 本身不触发查询。 - 截图:响应里只有代理路径 `/api/v1/img/site-detail/{releaseId}/{index}`,服务端从缓存取原始地址,公网校验(每跳重定向复核)、不带站点凭据、按文件头认图片(不放行 SVG)、≤ 8 MB,浏览器缓存 24 小时。站内附件如果需要登录才能看会显示为裂图,不会为截图消耗站点请求。 +- 憨憨详情页(HH-2 按真实页面 `probe.labels` 核对):没有 `#kdescr`,内容分节为「标签 div + 相邻值 div」:`mediainfo`、`海报`、`豆瓣`(内有 `span.font-bold` 小标签 片名 / 年代 / 类别 / 导演 …)、`截图`、`种子文件`(只有「查看列表」按钮)、`发布者`、`发布时间`。对应规则:简介没有容器时取「豆瓣 / 影片信息 / IMDb」信息块,小标签补冒号、一项一行;截图取「截图」一节的 img(`data-src` 优先,海报不算截图),没有再取简介里的图;IMDb 依次取页面链接 → 简介 / 信息块 / MediaInfo 里的 `imdb.com/title/tt…` 或 `IMDB : tt…` 行(只收 `tt` + 7~8 位,不收孤立 tt 串),TMDB 取链接或 `TMDB : tv/294990` 行,豆瓣取链接或 `douban.com/subject/…`;兜底出的 IMDb / 豆瓣也回填到 `SiteRelease.imdbId / doubanId`;发布时间没有 `span[title]` 时取「发布时间 / 上传时间」字段文本。文件清单:定义 `umt.detail.page.fileListPath`(憨憨 `viewfilelist.php?id={id}`)配置了且页面上没有内联清单时,同一次详情刷新里再请求 1 次(走同一限流器、计数),失败只是没有清单。fixture `HHANCLUB_DETAIL_PAGE_2026` / `HHANCLUB_FILELIST_FRAGMENT`:标签名称与顺序来自真实页面,嵌套按已验证的标题 / 副标题结构推定,部署后打开一条憨憨资源的详情抽屉核对一次。 +- 截图代理的 Referer:截图与站点同根域(憨憨图床 `pic.hhanclub.net`、`proxy-cover.hhanclub.net`)时带 `Referer: <站点地址>/`(防盗链),外部图床(如 pixhost)不带;任何情况都不带 Cookie。 - 解析:`packages/adapters/src/sites/spider/detail-page.ts`。简介取 MovieClaw `detail_description_css`(缺省 `#kdescr`);MediaInfo 取定义 `umt.detail.page.mediaInfo` → 「MediaInfo」字段标签 → 简介里第一个像 MediaInfo 的引用 / 代码块;文件清单取 `umt.detail.page.files` → MovieClaw `detail_file_list_css` → 「文件列表」字段 → `#filelist` 表格,不额外请求 `viewfilelist.php`。字段标签除 `td.rowhead` 外可在 `umt.detail.page.labelSelectors` 加(憨憨:`div.font-bold.leading-6`、`span.font-bold`)。`page.probe` 记录页面上找到的字段标签,只存缓存不展示,站点改版时用来排查。馒头从 API 的 `descr`(BBCode)与 `mediainfo` 取同样的结构。 ## Agent 能力(W10-pre2) diff --git a/packages/adapters/src/sites/definitions/hhanclub.json b/packages/adapters/src/sites/definitions/hhanclub.json index 303500e9..4c46f915 100644 --- a/packages/adapters/src/sites/definitions/hhanclub.json +++ b/packages/adapters/src/sites/definitions/hhanclub.json @@ -260,7 +260,8 @@ "fileList": "td.rowfollow > ul > li::text" }, "page": { - "labelSelectors": ["div.font-bold.leading-6", "span.font-bold"] + "labelSelectors": ["div.font-bold.leading-6", "span.font-bold"], + "fileListPath": "viewfilelist.php?id={id}" } }, "vipLevels": ["贵宾", "VIP"], diff --git a/packages/adapters/src/sites/definitions/schema.ts b/packages/adapters/src/sites/definitions/schema.ts index 081fe19b..83fe8ae2 100644 --- a/packages/adapters/src/sites/definitions/schema.ts +++ b/packages/adapters/src/sites/definitions/schema.ts @@ -149,6 +149,7 @@ const UmtNexusSchema = z.strictObject({ tags: z.string().optional(), uploader: z.string().optional(), files: z.string().optional(), + fileListPath: z.string().optional(), }) .optional(), }) diff --git a/packages/adapters/src/sites/detail-text.ts b/packages/adapters/src/sites/detail-text.ts index d7318537..ea569a30 100644 --- a/packages/adapters/src/sites/detail-text.ts +++ b/packages/adapters/src/sites/detail-text.ts @@ -18,9 +18,12 @@ export function looksLikeMediaInfo(text: string): boolean { return (text.match(/^[^\n:：]{2,40}[ \t]*[:：][ \t]*\S/gm) ?? []).length >= 5; } +/** TMDB:themoviedb.org 链接,或 MediaInfo / 简介里的「TMDB : tv/294990」一行 */ export function extractTmdb(text: string | null | undefined): SiteDetailPage["tmdb"] { - const m = text ? /themoviedb\.org\/(movie|tv)\/(\d+)/.exec(text) : null; - return m ? { type: m[1] as "movie" | "tv", id: m[2]! } : null; + const m = text + ? (/themoviedb\.org\/(movie|tv)\/(\d+)/.exec(text) ?? /\bTMDB\s*(?:ID)?\s*[:：]\s*(movie|tv)\/(\d+)/i.exec(text)) + : null; + return m ? { type: m[1]!.toLowerCase() as "movie" | "tv", id: m[2]! } : null; } /** 图片地址转绝对 http(s) 地址并过滤非截图;无效返回 null */ diff --git a/packages/adapters/src/sites/feed-item.test.ts b/packages/adapters/src/sites/feed-item.test.ts new file mode 100644 index 00000000..29e1f072 --- /dev/null +++ b/packages/adapters/src/sites/feed-item.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; +import { normalizeSiteFeedItem } from "./feed-item.ts"; + +// 标题取自 2026-09-29 憨憨 RSS 真实条目(公开种子名,无凭据) +describe("站点 RSS 条目规范化(憨憨)", () => { + it("[分类]种子名[副标题][大小] → 分类 / 标题 / 副标题 / 大小", () => { + const r = normalizeSiteFeedItem( + "hhanclub", + "[音乐]Travis - L.A. Times (2024) - WEB-DL - 24bit ALAC-HHWEB[Travis - L.A. Times | ALAC | 苹果音乐 | 现场版专辑][697.64 MB]", + ); + expect(r).toEqual({ + title: "Travis - L.A. Times (2024) - WEB-DL - 24bit ALAC-HHWEB", + subtitle: "Travis - L.A. Times | ALAC | 苹果音乐 | 现场版专辑", + category: { id: "410", name: "音乐" }, + mediaType: "music", + sizeBytes: Math.round(697.64 * 1024 ** 2), + uploader: null, + }); + }); + + it("电视剧:副标题里的 | 与 / 保留,GB 换算", () => { + const r = normalizeSiteFeedItem( + "hhanclub", + "[电视剧]Spring of the Blade S01 2026 2160p YK WEB-DL H.265 DV HFR HQ AAC2.0-HHWEB[一瓯春 | 第20-24集 | 4K 高码 杜比视界 | 类型: 爱情/古装][26.37 GB]", + ); + expect(r.title).toBe("Spring of the Blade S01 2026 2160p YK WEB-DL H.265 DV HFR HQ AAC2.0-HHWEB"); + expect(r.subtitle).toBe("一瓯春 | 第20-24集 | 4K 高码 杜比视界 | 类型: 爱情/古装"); + expect(r.category).toEqual({ id: "402", name: "电视剧" }); + expect(r.mediaType).toBe("tv"); + expect(r.sizeBytes).toBe(Math.round(26.37 * 1024 ** 3)); + }); + + it("副标题内嵌方括号、[新] 标记、不在任何媒体大类的分类(纪录片)", () => { + const r = normalizeSiteFeedItem( + "hhanclub", + "[纪录片]Thirteen Talks S09 2016 2160p WEB-DL H.265 AAC2.0-HHWEB[新][十三邀 | 第14集 [内嵌中字]][3.6 GB]", + ); + expect(r.title).toBe("Thirteen Talks S09 2016 2160p WEB-DL H.265 AAC2.0-HHWEB"); + expect(r.subtitle).toBe("十三邀 | 第14集 [内嵌中字]"); + expect(r.category).toEqual({ id: "404", name: "纪录片" }); + expect(r.mediaType).toBeNull(); + }); + + it("带上传者段:[副标题][大小][上传者]", () => { + const r = normalizeSiteFeedItem("hhanclub", "[电影]Some Movie 2024 1080p WEB-DL-HHWEB[某电影][8.5 GB][uploader01]"); + expect(r).toMatchObject({ + title: "Some Movie 2024 1080p WEB-DL-HHWEB", + subtitle: "某电影", + uploader: "uploader01", + }); + }); + + it("没有大小段时不拆尾部方括号;未登记的前缀不当分类", () => { + const r = normalizeSiteFeedItem("hhanclub", "[BDMV]Some.Disc.2024[Extras]"); + expect(r).toMatchObject({ title: "[BDMV]Some.Disc.2024[Extras]", subtitle: null, sizeBytes: null }); + expect(r.category).toEqual({ id: null, name: null }); + }); + + it("未知站点只去 [新]", () => { + const r = normalizeSiteFeedItem(null, "[电影]Foo 2024[新][foo][1 GB]"); + expect(r.category.id).toBeNull(); + expect(r.title).toBe("[电影]Foo 2024"); + expect(r.subtitle).toBe("foo"); + }); +}); diff --git a/packages/adapters/src/sites/feed-item.ts b/packages/adapters/src/sites/feed-item.ts new file mode 100644 index 00000000..d184628f --- /dev/null +++ b/packages/adapters/src/sites/feed-item.ts @@ -0,0 +1,123 @@ +/** + * 站点 RSS 条目规范化 —— 本项目扩展(站点适配层)。 + * NexusPHP `torrentrss.php` 按用户勾选拼标题:`[分类]` + 种子名 + `[副标题]` + `[大小]` + `[上传者]` + * (憨憨:`[音乐]Travis - L.A. Times (2024) - WEB-DL - 24bit ALAC-HHWEB[Travis - L.A. Times | ALAC][697.64 MB]`)。 + * 这里把它拆成与列表快照一致的字段:干净标题、副标题、站点分类(ID + 名称 + 媒体大类)、大小。 + * 只认站点定义里登记过的分类名作前缀;不改标题解析器(release-parser)。 + */ +import { getSiteDefinition } from "./registry.ts"; +import { parseSizeMp } from "./spider/spider.ts"; + +export type FeedMediaType = "movie" | "tv" | "music"; + +export interface SiteFeedItemFields { + /** 去掉分类前缀、副标题、大小、上传者、「[新]」后的种子名 */ + title: string; + subtitle: string | null; + /** 站点原始分类:ID 取站点定义里同名分类(与列表快照的 category_raw 一致);没有登记为 null */ + category: { id: string | null; name: string | null }; + /** 站点分类所属的媒体大类(定义 category.movie / tv / music);不属于任何一组为 null */ + mediaType: FeedMediaType | null; + /** 标题尾部的大小(站点四舍五入后的显示值);enclosure 有 length 时调用方应优先用它 */ + sizeBytes: number | null; + uploader: string | null; +} + +const NEW_MARK = /\s*[[(（【]\s*新\s*[\])）】]\s*/g; +const SIZE = /^\s*([\d.,]+\s*[KMGTP]i?B)\s*$/i; + +interface CategoryDefLike { + category?: Partial>; + umt?: { categoryNames?: Record }; +} + +/** 站点定义里登记的「分类名 → ID」 */ +function categoryIndex(siteKey: string | null | undefined): Map { + const out = new Map(); + const def = (siteKey ? getSiteDefinition(siteKey) : undefined) as CategoryDefLike | undefined; + if (!def) return out; + for (const [id, name] of Object.entries(def.umt?.categoryNames ?? {})) if (name) out.set(name, id); + for (const k of ["movie", "tv", "music"] as const) { + for (const c of def.category?.[k] ?? []) { + if (typeof c.desc === "string" && c.desc && c.id != null && !out.has(c.desc)) out.set(c.desc, String(c.id)); + } + } + return out; +} + +/** 站点分类 ID → 媒体大类(只在唯一一组里出现时成立) */ +export function siteCategoryMediaType(siteKey: string | null | undefined, id: string | null): FeedMediaType | null { + if (!id || !siteKey) return null; + const def = getSiteDefinition(siteKey) as CategoryDefLike | undefined; + const hits = (["movie", "tv", "music"] as const).filter((k) => + (def?.category?.[k] ?? []).some((c) => c.id != null && String(c.id) === id), + ); + return hits.length === 1 ? hits[0]! : null; +} + +/** 从尾部剥一个配平的 `[...]`;不以 `]` 结尾或不配平返回 null */ +function popTrailingGroup(s: string): { rest: string; inner: string } | null { + const t = s.trimEnd(); + if (!t.endsWith("]")) return null; + let depth = 0; + for (let i = t.length - 1; i >= 0; i--) { + const ch = t[i]; + if (ch === "]") depth++; + else if (ch === "[") { + depth--; + if (depth === 0) return { rest: t.slice(0, i), inner: t.slice(i + 1, -1) }; + } + } + return null; +} + +/** + * 规范化站点 RSS 条目标题。没有站点定义时只去「[新]」,其余原样返回。 + * 只有标题尾部出现大小段(`[697.64 MB]`)时才把它前面的一段当副标题,避免误伤本身以 `[...]` 结尾的种子名。 + */ +export function normalizeSiteFeedItem(siteKey: string | null | undefined, rawTitle: string): SiteFeedItemFields { + const cats = categoryIndex(siteKey); + let s = rawTitle.replace(/\s+/g, " ").trim(); + let category: SiteFeedItemFields["category"] = { id: null, name: null }; + + const prefix = /^\s*\[([^\][]{1,16})\]\s*/.exec(s); + if (prefix && cats.has(prefix[1]!.trim())) { + const name = prefix[1]!.trim(); + category = { id: cats.get(name) ?? null, name }; + s = s.slice(prefix[0].length); + } + + let sizeBytes: number | null = null; + let subtitle: string | null = null; + let uploader: string | null = null; + const groups: string[] = []; + let rest = s; + // 最多看尾部 3 段:[副标题][大小][上传者] + for (let i = 0; i < 3; i++) { + const g = popTrailingGroup(rest); + if (!g) break; + groups.unshift(g.inner); + rest = g.rest; + } + const sizeIdx = groups.findIndex((g) => SIZE.test(g)); + if (sizeIdx >= 0) { + const bytes = parseSizeMp(SIZE.exec(groups[sizeIdx]!)![1]!); + sizeBytes = bytes > 0 ? bytes : null; + const before = groups.slice(0, sizeIdx); + const after = groups.slice(sizeIdx + 1); + // 大小之前最后一段是副标题,再往前的属于种子名本身 + if (before.length) subtitle = before.pop()!.trim() || null; + if (after.length === 1 && after[0]!.trim().length <= 32) uploader = after[0]!.trim() || null; + s = `${rest}${before.map((b) => `[${b}]`).join("")}${after.length > 1 ? after.map((a) => `[${a}]`).join("") : ""}`; + } + + const title = s.replace(NEW_MARK, " ").replace(/\s+/g, " ").trim() || rawTitle.trim(); + return { + title, + subtitle: subtitle ? subtitle.replace(NEW_MARK, " ").trim() || null : null, + category, + mediaType: siteCategoryMediaType(siteKey, category.id), + sizeBytes, + uploader, + }; +} diff --git a/packages/adapters/src/sites/index.ts b/packages/adapters/src/sites/index.ts index 64e13ddc..03be54dc 100644 --- a/packages/adapters/src/sites/index.ts +++ b/packages/adapters/src/sites/index.ts @@ -16,6 +16,12 @@ export { SpiderFilterSchema, } from "./definitions/index.ts"; export type { SiteAdapterDeps } from "./deps.ts"; +export { + type FeedMediaType, + normalizeSiteFeedItem, + type SiteFeedItemFields, + siteCategoryMediaType, +} from "./feed-item.ts"; export { classifyLoginError, loginFailureText, diff --git a/packages/adapters/src/sites/spider/adapter.ts b/packages/adapters/src/sites/spider/adapter.ts index 32116d8e..8d7de10a 100644 --- a/packages/adapters/src/sites/spider/adapter.ts +++ b/packages/adapters/src/sites/spider/adapter.ts @@ -26,6 +26,7 @@ import { } from "../types.ts"; import { inspectTorrent } from "../util.ts"; import { parseDetailPage } from "./detail.ts"; +import { filesFromFileListHtml } from "./detail-page.ts"; import { spiderTorrentToRelease } from "./map.ts"; import type { IndexerDefinition } from "./schema.ts"; import { loadHtml, normalizeSelector, SiteSpider } from "./spider.ts"; @@ -178,6 +179,17 @@ export function createSpiderSiteAdapter(input: IndexerDefinition, deps: SiteAdap if (r.kind === "login") throw new SiteError(def.id, "auth_expired", `详情 ${siteReleaseId}:返回登录页`); if (r.kind === "not_found") throw new SiteError(def.id, "not_found", `详情 ${siteReleaseId}:资源不存在`); if (!r.detail) throw new SiteError(def.id, "parse_error", `详情 ${siteReleaseId}:页面结构无法识别`); + // 文件清单不在页面上(NexusPHP 点「查看列表」才 AJAX 取)→ 按需 1 次;失败只是没有文件清单 + const flPath = ext.detail.page?.fileListPath; + if (flPath && r.detail.page && !r.detail.page.files.length) { + try { + const flUrl = new URL(flPath.replace("{id}", siteReleaseId), base).toString(); + const frag = await fetchPage(flUrl, `文件清单 ${siteReleaseId}`, opts); + r.detail.page.files = filesFromFileListHtml(loadHtml(frag)); + } catch (e) { + if (e instanceof SiteError && e.isAuthError) throw e; + } + } return r.detail; }, diff --git a/packages/adapters/src/sites/spider/detail-page.test.ts b/packages/adapters/src/sites/spider/detail-page.test.ts index 257afa5d..26a78b52 100644 --- a/packages/adapters/src/sites/spider/detail-page.test.ts +++ b/packages/adapters/src/sites/spider/detail-page.test.ts @@ -1,8 +1,11 @@ -import { HHANCLUB_DETAIL_PAGE_FULL } from "@umt/testkit"; +import { HHANCLUB_DETAIL_PAGE_2026, HHANCLUB_DETAIL_PAGE_FULL, HHANCLUB_FILELIST_FRAGMENT } from "@umt/testkit"; import { describe, expect, it } from "vitest"; import { hhanclubDefinition } from "../definitions/index.ts"; import { looksLikeMediaInfo, pageFromBbcode } from "../detail-text.ts"; +import { createSiteRateLimiter } from "../rate-limit.ts"; import { getSiteDefinition } from "../registry.ts"; +import { staticSecretResolver } from "../types.ts"; +import { createSpiderSiteAdapter } from "./adapter.ts"; import { parseDetailPage } from "./detail.ts"; import type { IndexerDefinition } from "./schema.ts"; @@ -105,3 +108,58 @@ Format : AVC[/quote] expect(looksLikeMediaInfo("General\n一段文字\nVideo")).toBe(false); }); }); + +describe("详情页扩展内容(憨憨 2026 真实结构:分节 div,无 #kdescr)", () => { + const { kind, detail } = parseDetailPage(HHANCLUB_DETAIL_PAGE_2026, hhanclubDefinition, "91003", OBSERVED); + const page = detail!.page!; + + it("MediaInfo 取「mediainfo」一节;IMDb / TMDB 从 MediaInfo 行兜底,豆瓣取页面链接", () => { + expect(kind).toBe("ok"); + expect(page.mediaInfo).toContain("Complete name"); + expect(page).toMatchObject({ imdbId: "tt45950001", tmdb: { type: "tv", id: "290001" }, doubanId: "36000002" }); + expect(detail!.imdbId).toBe("tt45950001"); + }); + + it("简介取豆瓣信息块,小标签补冒号、一项一行", () => { + expect(page.descriptionText).toContain("片名：示例剧集"); + expect(page.descriptionText).toContain("类别：爱情 / 古装"); + expect(page.descriptionText).toContain("一段剧情简介"); + }); + + it("截图取「截图」一节(优先 data-src),不含海报与占位图", () => { + expect(page.screenshots).toEqual([ + "https://pic.hhanclub.net/img/2026/09/29/shot1.png", + "https://img3.pixhost.cc/images/1/shot2.jpg", + ]); + }); + + it("上传者、标签、发布时间(字段文本兜底)", () => { + expect(page.uploader).toBe("匿名"); + expect(page.tags).toEqual(["官方", "国语", "中字", "杜比视界"]); + expect(detail!.uploadedAt).toBe("2026-09-29T08:42:57.000Z"); + expect(page.files).toEqual([]); + }); + + it("文件清单不在页面上 → 按需请求 viewfilelist.php 一次", async () => { + const paths: string[] = []; + const fetch = (async (input: RequestInfo | URL) => { + const u = new URL(String(input)); + paths.push(`${u.pathname}${u.search}`); + const body = u.pathname === "/viewfilelist.php" ? HHANCLUB_FILELIST_FRAGMENT : HHANCLUB_DETAIL_PAGE_2026; + return new Response(body, { status: 200, headers: { "content-type": "text/html; charset=utf-8" } }); + }) as typeof globalThis.fetch; + const limiter = createSiteRateLimiter({ minIntervalMs: 0 }); + const a = createSpiderSiteAdapter(hhanclubDefinition, { + fetch, + limiter, + secrets: staticSecretResolver({ hhanclub: { cookie: "c=1" } }), + }); + const d = await a.detail("91003"); + expect(paths).toEqual(["/details.php?id=91003&hit=1", "/viewfilelist.php?id=91003"]); + expect(limiter.stats.granted).toBe(2); + expect(d.page!.files).toEqual([ + { name: "Example.Show.S01E18.2026.2160p.mkv", sizeBytes: Math.round(5.04 * 1024 ** 3) }, + { name: "Example.Show.S01E19.2026.2160p.mkv", sizeBytes: Math.round(4.98 * 1024 ** 3) }, + ]); + }); +}); diff --git a/packages/adapters/src/sites/spider/detail-page.ts b/packages/adapters/src/sites/spider/detail-page.ts index dfa14487..2f2cca81 100644 --- a/packages/adapters/src/sites/spider/detail-page.ts +++ b/packages/adapters/src/sites/spider/detail-page.ts @@ -25,10 +25,13 @@ type El = ReturnType; const LABELS = { description: ["简介", "Description"], + /** 没有简介容器时的次选:豆瓣 / IMDb 信息块(憨憨把豆瓣资料拆成独立一节) */ + descriptionFallback: ["豆瓣", "豆瓣信息", "影片信息", "IMDb信息", "IMDb"], + screenshots: ["截图", "Screenshots", "Screenshot"], mediaInfo: ["MediaInfo", "媒体信息", "BDInfo"], tags: ["标签", "Tags"], uploader: ["上传者", "发布者", "发布人", "上传人", "Uploader"], - files: ["文件列表", "文件清单", "File list", "Files"], + files: ["文件列表", "文件清单", "种子文件", "File list", "Files"], }; /** 简介里可能装 MediaInfo 的块 */ @@ -62,6 +65,19 @@ function fileFromText(text: string): SiteDetailFile | null { return HAS_EXT.test(t) ? { name: t, sizeBytes: null } : null; } +/** 文本里的 IMDb ID:imdb.com/title 链接或「IMDB : tt…」一行;不接受孤立的 tt 串(避免误中哈希) */ +export function imdbFromText(text: string): string | null { + const m = + /imdb\.com\/title\/(tt\d{7,8})\b/i.exec(text) ?? + /\bIMDb\s*(?:ID|链接|編號|编号)?\s*[:：]?\s*(tt\d{7,8})\b/i.exec(text); + return m?.[1] ?? null; +} + +/** NexusPHP viewfilelist.php 返回的文件表(按需 AJAX 取到的 HTML 片段) */ +export function filesFromFileListHtml($: CheerioAPI): SiteDetailFile[] { + return filesFrom($, $("tr")); +} + function filesFrom($: CheerioAPI, rows: El): SiteDetailFile[] { const out: SiteDetailFile[] = []; rows.each((_, n) => { @@ -70,7 +86,7 @@ function filesFrom($: CheerioAPI, rows: El): SiteDetailFile[] { const cells = row.children("td"); if (cells.length >= 2) { const name = cells.first().text().replace(/\s+/g, " ").trim(); - const size = cells.eq(1).text().trim(); + const size = cells.eq(1).text().replace(/\s+/g, " ").trim(); if (name && HAS_EXT.test(name) && /\d/.test(size)) out.push({ name, sizeBytes: parseSizeMp(size) || null }); return; } @@ -106,6 +122,7 @@ export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string) let desc: El | null = descSel ? $(descSel).first() : null; if (!desc?.length) desc = byLabel(LABELS.description); page.probe.hasDescription = !!desc?.length; + const infoBlock = desc?.length ? null : byLabel(LABELS.descriptionFallback); // MediaInfo:站点选择器 → 字段标签 → 简介里的引用 / 代码块(取最内层文本,整块从简介里去掉) let mediaInfoEl: El | null = null; @@ -137,12 +154,19 @@ export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string) // 截图 const imgs: string[] = []; - const imgEls = p.screenshots ? $(p.screenshots) : desc?.length ? desc.find("img") : null; - imgEls?.each((_, n) => { - const e = $(n); - const u = screenshotUrl(e.attr("data-src") ?? e.attr("data-original") ?? e.attr("src"), base); - if (u && !imgs.includes(u)) imgs.push(u); - }); + const pushImgs = (els: El | null) => + els?.each((_, n) => { + const e = $(n); + const u = screenshotUrl(e.attr("data-src") ?? e.attr("data-original") ?? e.attr("src"), base); + if (u && !imgs.includes(u)) imgs.push(u); + }); + if (p.screenshots) pushImgs($(p.screenshots)); + else { + // 独立的「截图」一节(憨憨)优先;没有再取简介里的图 + const shotBlock = byLabel(LABELS.screenshots); + if (shotBlock) pushImgs(shotBlock.is("img") ? shotBlock : shotBlock.find("img")); + if (!imgs.length && desc?.length) pushImgs(desc.find("img")); + } page.probe.imageCount = imgs.length; page.screenshots = imgs.slice(0, MAX_SCREENSHOTS); @@ -153,18 +177,36 @@ export function extractDetailPage($: CheerioAPI, conf: DetailConf, base: string) c.find("img").remove(); const t = blockText($, c); if (t) page.descriptionText = t.slice(0, 20_000); + } else if (infoBlock) { + const c = infoBlock.clone(); + c.find("img").remove(); + // 块里的「标签」span 与值之间补上冒号 / 换行,读起来是「片名:xxx」一行一项 + c.find(p.labelSelectors?.join(", ") || "span.font-bold").each((_, n) => { + const e = $(n); + const t = e.text().trim(); + if (t && t.length <= 16 && !/[:：]$/.test(t)) e.text(`${t}：`); + e.before("\n"); + }); + const t = blockText($, c) + .replace(/[ \t]*\n[ \t]*/g, "\n") + .replace(/：\s*\n(?=\S)/g, "："); + if (t) page.descriptionText = t.slice(0, 20_000); } - // 外部 ID:页面上的链接优先,其次简介正文 - const descHtml = desc?.html() ?? ""; + // 外部 ID:页面上的链接优先,其次简介 / 信息块正文,再次 MediaInfo 里的 IMDB / TMDB 行(憨憨的 MediaInfo 带这两行) + const descHtml = `${desc?.html() ?? ""}\n${infoBlock?.html() ?? ""}`; + const idText = `${descHtml}\n${page.mediaInfo ?? ""}`; const f = conf.fields ?? {}; page.imdbId = - extractImdbId(mcSelect($, null, f.imdb ?? "a[href*='imdb.com']::attr(href)")) ?? extractImdbId(descHtml) ?? null; + extractImdbId(mcSelect($, null, f.imdb ?? "a[href*='imdb.com']::attr(href)")) ?? + imdbFromText(idText) ?? + extractImdbId(descHtml) ?? + null; page.doubanId = extractDoubanId(mcSelect($, null, f.douban ?? "a[href*='douban.com']::attr(href)")) ?? - extractDoubanId(descHtml) ?? + extractDoubanId(idText) ?? null; - page.tmdb = extractTmdb($("a[href*='themoviedb.org']").first().attr("href")) ?? extractTmdb(descHtml); + page.tmdb = extractTmdb($("a[href*='themoviedb.org']").first().attr("href")) ?? extractTmdb(idText); // 文件清单:站点选择器 → MovieClaw file_list(去伪元素)→ 字段标签 → #filelist 表格 const fileSel = p.files ?? f.fileList?.replace(/::text$/, ""); diff --git a/packages/adapters/src/sites/spider/detail.ts b/packages/adapters/src/sites/spider/detail.ts index 4613e760..ce417a5c 100644 --- a/packages/adapters/src/sites/spider/detail.ts +++ b/packages/adapters/src/sites/spider/detail.ts @@ -83,10 +83,15 @@ export function parseDetailPage( browse: undefined, }; const promo = new SiteSpider(promoDef).infoFrom($, $(conf.headerScope).first()); - const timeTitle = $("span[title]") - .filter((_, el) => /\d{4}-\d{2}-\d{2} \d{2}:\d{2}/.test($(el).attr("title") ?? "")) - .first() - .attr("title"); + // 发布时间:带完整时间的 span[title] → 「发布时间 / 上传时间 / 添加时间」字段的文本(憨憨是 div 标签 + 值) + const timeTitle = + $("span[title]") + .filter((_, el) => /\d{4}-\d{2}-\d{2} \d{2}:\d{2}/.test($(el).attr("title") ?? "")) + .first() + .attr("title") ?? + /(?:发布时间|上传时间|添加时间|Added)\s*[:：]?\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}(?::\d{2})?)/.exec( + $("body").text().replace(/\s+/g, " "), + )?.[1]; const detail: SiteReleaseDetail = { siteId: def.id, @@ -119,6 +124,9 @@ export function parseDetailPage( const page = extractDetailPage($, conf, def.domain); page.imdbId ??= detail.imdbId ?? null; page.doubanId ??= detail.doubanId ?? null; + // 详情页扩展内容里兜底出来的外部 ID(MediaInfo 的 IMDB 行、豆瓣信息块)也回填到资源本身 + if (!detail.imdbId && page.imdbId) detail.imdbId = page.imdbId; + if (!detail.doubanId && page.doubanId) detail.doubanId = page.doubanId; if (page.tags.length) detail.tags = page.tags; detail.page = page; return { kind: "ok", detail }; diff --git a/packages/adapters/src/sites/spider/schema.ts b/packages/adapters/src/sites/spider/schema.ts index bbc13714..4a7d00b8 100644 --- a/packages/adapters/src/sites/spider/schema.ts +++ b/packages/adapters/src/sites/spider/schema.ts @@ -211,6 +211,11 @@ export interface UmtSiteExtension { uploader?: string; /** 文件清单行(每行一个文件,文本形如「名称 (大小)」或两列表格) */ files?: string; + /** + * 页面上没有内联文件清单时,按需再取一次的 AJAX 端点(`{id}` 替换为资源 ID,NexusPHP 为 `viewfilelist.php?id={id}`); + * 每次详情刷新最多 1 次,失败不影响详情。 + */ + fileListPath?: string; }; }; /** 下载路径(按资源 ID 拼接) */ diff --git a/packages/contracts/src/browser.ts b/packages/contracts/src/browser.ts index c37b9705..c8897351 100644 --- a/packages/contracts/src/browser.ts +++ b/packages/contracts/src/browser.ts @@ -337,6 +337,8 @@ export const BrowserSessionView = BrowserSession.extend({ waitingReason: z.string().nullable(), /** 打开会话时的凭据注入与回写状态(BR-2);旧会话没有 */ credential: BrowserCredentialState.nullable().optional(), + /** 最近一次自动化(Agent 能力)动作成功的时间;从未有过为 null。原站视图据此显示「Agent」或「自动化」 */ + lastAutomationAt: Timestamp.nullable().optional(), }); export type BrowserSessionView = z.infer; diff --git a/packages/testkit/src/sites/fixtures/definitions/hhanclub.ts b/packages/testkit/src/sites/fixtures/definitions/hhanclub.ts index e6f32a6e..a4812cb7 100644 --- a/packages/testkit/src/sites/fixtures/definitions/hhanclub.ts +++ b/packages/testkit/src/sites/fixtures/definitions/hhanclub.ts @@ -108,3 +108,49 @@ export const HHANCLUB_DETAIL_PAGE_FULL = `HHCL <div class="font-bold leading-6">文件列表</div> <div class="font-bold leading-6"><ul><li>Example.Show.S01E01.mkv (1.04 GB)</li><li>Example.Show.S01E02.mkv (1.02 GB)</li></ul></div> </div></body></html>`; + +/** + * HH-2:按 2026-09-29 真实详情页的解析诊断(`page.probe.labels`)重建的最小片段。字段标签名称与顺序来自真实页面 + * (下载 / 标题 / 副标题 / 标签 / 基本信息 / 字幕 / mediainfo / 海报 / 豆瓣 / 截图 / 种子文件 / 同伴 …,豆瓣块里是 + * span.font-bold 小标签 片名 / 年代 / 类别 / 导演 …);没有 #kdescr 简介容器;MediaInfo 头部与 IMDB / TMDB 两行 + * 摘自真实缓存(ID 已替换)。标签与值的 DOM 嵌套是按标题 / 副标题已验证的「标签 div + 相邻值 div」推定的。 + * 文件清单不在页面上,点「查看列表」才 AJAX 取 viewfilelist.php(见 HHANCLUB_FILELIST_FRAGMENT)。 + */ +export const HHANCLUB_DETAIL_PAGE_2026 = `<!DOCTYPE html><html><head><title>HHCLUB :: 种子详情 + +
+
下载
Example.Show.S01E18.torrent
+
标题
Example Show S01 2026 2160p YK WEB-DL H.265 DV HQ AAC2.0-HHWEB
+
副标题
示例剧集 | 第18-24集 | 4K 高码 杜比视界
+
标签
官方国语中字杜比视界
+
基本信息
+
+ 大小：33.62 GB + 类型：电视剧 + 发布者：匿名 + 发布时间：2026-09-29 16:42:57 +
+
mediainfo
+
文件名:Example.Show.S01E18.2026.2160p.YK.WEB-DL.H.265.DV.HQ.AAC2.0-HHWEB.mkv
体　积: 5.04 GiB

General
Unique ID : 1580976599
Complete name : Example.Show.S01E18.mkv
Format : Matroska
File size : 5.04 GiB
Duration : 43 min 55 s
IMDB : tt45950001
TMDB : tv/290001

Video
ID : 1
Format : HEVC
+
海报
poster
+
豆瓣
+
+
片名示例剧集
+
年代2026
+
类别爱情 / 古装
+
导演某导演
+
一段剧情简介……
+ 豆瓣 +
+
截图
+
+
种子文件
[查看列表]
+
同伴
22个做种者 | 66个下载者
+
`; + +/** NexusPHP viewfilelist.php 返回的文件表片段(标准结构,文件名 / 大小为示例) */ +export const HHANCLUB_FILELIST_FRAGMENT = ` + + + +
路径size
Example.Show.S01E18.2026.2160p.mkv5.04 GB
Example.Show.S01E19.2026.2160p.mkv4.98 GB
`; diff --git a/scripts/mp-sync/movieclaw-overrides/hhanclub.json b/scripts/mp-sync/movieclaw-overrides/hhanclub.json index d8994c9f..fde51b8b 100644 --- a/scripts/mp-sync/movieclaw-overrides/hhanclub.json +++ b/scripts/mp-sync/movieclaw-overrides/hhanclub.json @@ -29,7 +29,8 @@ }, "detail": { "page": { - "labelSelectors": ["div.font-bold.leading-6", "span.font-bold"] + "labelSelectors": ["div.font-bold.leading-6", "span.font-bold"], + "fileListPath": "viewfilelist.php?id={id}" } } },