diff --git a/apps/server/src/modules/sources-identity/runtime.ts b/apps/server/src/modules/sources-identity/runtime.ts index 40d6d8f6..2e66eed9 100644 --- a/apps/server/src/modules/sources-identity/runtime.ts +++ b/apps/server/src/modules/sources-identity/runtime.ts @@ -14,7 +14,7 @@ import { type TmdbSharedCache, } from "@umt/adapters"; import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts"; -import { pgTokenBucketFor, type SiteRow } from "@umt/db"; +import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db"; import { CACHE_REGIONS, sharedCache, type TmdbCachePolicy, tmdbCachePolicy } from "../../shared-cache.ts"; import type { ServerContext } from "../types.ts"; @@ -170,7 +170,7 @@ export class SourcesRuntime { if (hit && hit.sig === sig) return hit.adapter; const lo = limiterOptions(def, site.rate_limit, defaults.defaultRateLimit); // 跨进程:api / worker / worker:agent 对同一站点共用 PG 令牌桶(key = site:);进程内先串行排队 - const limiter = + const inner = this.overrides.sharedRateLimit === false ? createSiteRateLimiter(lo) : createSharedSiteRateLimiter({ @@ -183,6 +183,7 @@ export class SourcesRuntime { "站点共享限流不可用，本次退回进程内限流", ), }); + const limiter = countingLimiter(inner, () => this.countSiteRequest(site.id)); const cfg = (site.config ?? {}) as { apiBaseUrl?: string }; const baseUrl = def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url; @@ -197,6 +198,41 @@ export class SourcesRuntime { return adapter; } + /** + * 站点请求计数(site_request_count,按小时聚合):每拿到一个令牌记一次。写失败只记日志,不影响请求; + * 顺手删 7 天前的行(每进程每小时最多一次)。 + */ + private lastCountPrune = 0; + countSiteRequest(siteId: string): void { + const hour = new Date(Math.floor(this.now().getTime() / 3_600_000) * 3_600_000); + const db = this.ctx.db; + void sql`insert into site_request_count (site_id, hour, count) values (${siteId}, ${hour}, 1) + on conflict (site_id, hour) do update set count = site_request_count.count + 1` + .execute(db) + .then(async () => { + if (Date.now() - this.lastCountPrune < 3_600_000) return; + this.lastCountPrune = Date.now(); + await sql`delete from site_request_count where hour < ${new Date(hour.getTime() - 7 * 86_400_000)}`.execute(db); + }) + .catch((err) => + this.ctx.logger.warn({ siteId, err: { message: (err as Error).message } }, "站点请求计数写入失败"), + ); + } + + /** 最近 hours 小时(含当前小时)该站点的请求次数 */ + async siteRequestCount(siteId: string, hours = 24): Promise { + const hour = Math.floor(this.now().getTime() / 3_600_000) * 3_600_000; + const since = new Date(hour - (hours - 1) * 3_600_000); + const r = await sql<{ n: number }>`select coalesce(sum(count), 0)::int as n from site_request_count + where site_id = ${siteId} and hour >= ${since}`.execute(this.ctx.db); + return Number(r.rows[0]?.n ?? 0); + } + + /** 站点访问默认:站点定义 umt.defaults 与 settings.sites 取更保守的一方 */ + async siteAccess(siteKey: string): Promise { + return siteAccessFor(getSiteDefinition(siteKey) ?? null, await this.siteDefaults()); + } + /** 已建立的站点限流器(测试与诊断用) */ limiterOf(siteId: string): SiteRateLimiter | null { return this.adapters.get(siteId)?.limiter ?? null; @@ -260,6 +296,63 @@ function sameUrl(a: string, b: string): boolean { return a.replace(/\/+$/, "") === b.replace(/\/+$/, ""); } +/** 站点访问默认(已合并定义与设置) */ +export interface SiteAccess { + /** 站点行没设快照周期时用 */ + snapshotIntervalSec: number; + snapshotMaxPages: number; + healthCheckIntervalSec: number; + detailCacheSec: number; + searchConcurrency: number; +} + +type DefinitionDefaults = { + snapshotIntervalSec?: number; + snapshotMaxPages?: number; + healthCheckIntervalSec?: number; + detailCacheSec?: number; + searchConcurrency?: number; +}; + +/** 周期 / 缓存取较长,页数 / 并发取较小(定义值是按站点实测给的保守下限) */ +export function siteAccessFor(def: SiteDefinition | null, s: SitesSettings): SiteAccess { + const d = ((def?.umt as { defaults?: DefinitionDefaults } | undefined)?.defaults ?? {}) as DefinitionDefaults; + const max = (a: number | undefined, b: number) => (a === undefined ? b : Math.max(a, b)); + const min = (a: number | undefined, b: number) => (a === undefined ? b : Math.min(a, b)); + return { + snapshotIntervalSec: max(d.snapshotIntervalSec, s.defaultSnapshotIntervalSec), + snapshotMaxPages: min(d.snapshotMaxPages, s.defaultSnapshotMaxPages), + healthCheckIntervalSec: max(d.healthCheckIntervalSec, s.defaultHealthCheckIntervalSec), + detailCacheSec: max(d.detailCacheSec, s.defaultDetailCacheSec), + searchConcurrency: d.searchConcurrency ?? 1, + }; +} + +/** 同站实时搜索并发(只看站点定义,同步取值;缺省 1) */ +export function siteSearchConcurrency(siteKey: string): number { + const def = getSiteDefinition(siteKey); + return ((def?.umt as { defaults?: DefinitionDefaults } | undefined)?.defaults?.searchConcurrency ?? 1) as number; +} + +/** 包一层限流器:每发放一个令牌回调一次(计数) */ +function countingLimiter(inner: SiteRateLimiter, onGrant: () => void): SiteRateLimiter { + return { + get stats() { + return inner.stats; + }, + async acquire(signal) { + await inner.acquire(signal); + onGrant(); + }, + tryAcquire() { + const ok = inner.tryAcquire(); + if (ok) onGrant(); + return ok; + }, + penalize: (ms) => inner.penalize(ms), + }; +} + type RateLimitLike = { requestsPerMinute?: number; minIntervalMs?: number; concurrency?: number }; /** diff --git a/apps/server/src/modules/sources-identity/scheduler.ts b/apps/server/src/modules/sources-identity/scheduler.ts index 29ef66b8..a7335501 100644 --- a/apps/server/src/modules/sources-identity/scheduler.ts +++ b/apps/server/src/modules/sources-identity/scheduler.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; +import { getSiteDefinition } from "@umt/adapters"; import { sql } from "@umt/db"; -import type { SourcesRuntime } from "./runtime.ts"; +import { type SourcesRuntime, siteAccessFor } from "./runtime.ts"; import { archiveStalePool } from "./service/identity-decisions.ts"; // worker 后台调度:按 Feed 轮询策略发 feed.due(outbox → ingestFeed)、按站点快照周期入队 snapshotSite、 @@ -67,24 +68,32 @@ export async function tickFeeds(rt: SourcesRuntime): Promise { export async function tickSnapshots(rt: SourcesRuntime): Promise { const { ctx } = rt; const now = rt.now(); - // 站点未设周期时用全局默认(settings.sites.defaultSnapshotIntervalSec) - const fallback = (await rt.siteDefaults()).defaultSnapshotIntervalSec; - const sites = await sql<{ id: string; snapshot_interval_sec: number }>` - select id, coalesce(snapshot_interval_sec, ${fallback}::int) as snapshot_interval_sec from site - where enabled and snapshot_enabled and deleted_at is null - and (last_snapshot_at is null - or last_snapshot_at + make_interval(secs => coalesce(snapshot_interval_sec, ${fallback}::int)) <= ${now})`.execute( - ctx.db, - ); - for (const s of sites.rows) { - const slot = Math.floor(now.getTime() / 1000 / s.snapshot_interval_sec); + // 站点未设周期时用站点定义与全局默认里更长的一个(settings.sites.defaultSnapshotIntervalSec / umt.defaults) + const defaults = await rt.siteDefaults(); + const rows = await ctx.db + .selectFrom("site") + .select(["id", "key", "snapshot_interval_sec", "last_snapshot_at"]) + .where("enabled", "=", true) + .where("snapshot_enabled", "=", true) + .where("deleted_at", "is", null) + .execute(); + const due: Array<{ id: string; interval: number }> = []; + for (const r of rows) { + const interval = + r.snapshot_interval_sec ?? siteAccessFor(getSiteDefinition(r.key) ?? null, defaults).snapshotIntervalSec; + if (!r.last_snapshot_at || r.last_snapshot_at.getTime() + interval * 1000 <= now.getTime()) { + due.push({ id: r.id, interval }); + } + } + for (const s of due) { + const slot = Math.floor(now.getTime() / 1000 / s.interval); await ctx.engine.enqueue( "snapshotSite", { siteId: s.id, kind: "periodic" }, { idempotencyKey: `snapshotSite:${s.id}:${slot}`, traceId: randomUUID() }, ); } - return sites.rows.length; + return due.length; } /** diff --git a/apps/server/src/modules/sources-identity/service/account.ts b/apps/server/src/modules/sources-identity/service/account.ts index 60224c9c..973b8fef 100644 --- a/apps/server/src/modules/sources-identity/service/account.ts +++ b/apps/server/src/modules/sources-identity/service/account.ts @@ -901,7 +901,7 @@ export async function tickSiteUserData(rt: SourcesRuntime, slot: string): Promis return rows.length; } -/** sites.healthCheck:启用站点逐个检测(同步,走站点限流) */ +/** sites.healthCheck:启用站点逐个检测(同步,走站点限流);按站点访问默认的间隔跳过刚检测过的站点 */ export async function tickSiteHealth(rt: SourcesRuntime): Promise { const sites = await rt.ctx.db .selectFrom("site") @@ -911,7 +911,11 @@ export async function tickSiteHealth(rt: SourcesRuntime): Promise { .where("adapter", "!=", "generic_rss") .execute(); let n = 0; + const now = rt.now().getTime(); for (const s of sites) { + // 站点访问默认的最短间隔内检测过(手动或定时)就跳过;留 2 分钟余量,免得和定时任务周期错开一轮 + const { healthCheckIntervalSec } = await rt.siteAccess(s.key); + if (s.last_health_at && now - s.last_health_at.getTime() < healthCheckIntervalSec * 1000 - 120_000) continue; try { await checkSiteHealth(rt, s); n++; diff --git a/apps/server/src/modules/sources-identity/service/snapshot.ts b/apps/server/src/modules/sources-identity/service/snapshot.ts index 42854bce..ab3c5046 100644 --- a/apps/server/src/modules/sources-identity/service/snapshot.ts +++ b/apps/server/src/modules/sources-identity/service/snapshot.ts @@ -119,7 +119,11 @@ export async function snapshotSite(rt: SourcesRuntime, input: SnapshotInput): Pr return { status: "skipped", reason: (e as Error).message, ...none }; } const cfgPages = Number((site.config as { snapshotPages?: number })?.snapshotPages) || 1; - const maxPages = Math.min(20, Math.max(1, input.maxPages ?? cfgPages)); + let maxPages = Math.min(20, Math.max(1, input.maxPages ?? cfgPages)); + // 定时 / 手动快照受站点访问默认的页数上限约束(补漏有自己的页数预算,实时补查只抓 1 页) + if (input.kind === "periodic" || input.kind === "manual") { + maxPages = Math.min(maxPages, (await rt.siteAccess(site.key)).snapshotMaxPages); + } const snap = await db .insertInto("site_snapshot") .values({ @@ -211,9 +215,14 @@ export async function refreshReleaseDetail(rt: SourcesRuntime, srId: string): Pr if (!rel) return "not_found"; const site = await db.selectFrom("site").selectAll().where("id", "=", rel.site_id).executeTakeFirst(); if (!site?.enabled) return "site_disabled"; + // 详情缓存:站点访问默认的缓存期内抓过就不再请求站点(列表快照会持续更新做种 / 促销) + const { detailCacheSec } = await rt.siteAccess(site.key); + if (rel.detail_fetched_at && rt.now().getTime() - rel.detail_fetched_at.getTime() < detailCacheSec * 1000) { + return null; + } let detail: SiteRelease; try { - detail = await rt.siteAdapter(site).detail(rel.site_release_id); + detail = mergeDetailOverList(await rt.siteAdapter(site).detail(rel.site_release_id), rel); } catch (e) { if (e instanceof SiteError) { if (e.isAuthError) await db.transaction().execute((trx) => onAuthError(rt, trx, site, e)); @@ -226,7 +235,47 @@ export async function refreshReleaseDetail(rt: SourcesRuntime, srId: string): Pr } await db.transaction().execute(async (trx) => { await upsertSiteRelease(rt, trx, site, detail); - await trx.updateTable("source_release").set({ detail_fetched_at: rt.now() }).where("id", "=", srId).execute(); + const keepPromotion = detail.promotion.type === "unknown"; + await trx + .updateTable("source_release") + .set({ + detail_fetched_at: rt.now(), + // 详情页没解析出促销时保留列表快照的促销,不用"未知"覆盖 + ...(keepPromotion + ? { promotion: jsonb(rel.promotion ?? {}), promotion_expires_at: rel.promotion_expires_at } + : {}), + }) + .where("id", "=", srId) + .execute(); }); return null; } + +/** + * 详情页缺的字段沿用列表快照已有的值:有些站点(如 HHClub 的 div 详情页)详情里没有做种 / 完成数 / 分类名, + * upsert 时 seeders / leechers 与 raw._site 会被整体覆盖,不能用 null 冲掉列表数据。 + */ +export function mergeDetailOverList( + detail: SiteRelease, + rel: { + seeders: number | null; + leechers: number | null; + category_raw: string | null; + tags_raw: string[] | null; + raw: unknown; + }, +): SiteRelease { + const prev = ((rel.raw ?? {}) as { _site?: Record })._site ?? {}; + return { + ...detail, + seeders: detail.seeders ?? rel.seeders, + leechers: detail.leechers ?? rel.leechers, + snatched: detail.snatched ?? (typeof prev.snatched === "number" ? prev.snatched : null), + category: { + id: detail.category.id ?? rel.category_raw, + name: detail.category.name ?? (typeof prev.categoryName === "string" ? prev.categoryName : null), + }, + hr: detail.hr === "unknown" && typeof prev.hr === "boolean" ? prev.hr : detail.hr, + tags: detail.tags.length ? detail.tags : (rel.tags_raw ?? []), + }; +} diff --git a/apps/server/src/modules/sources-identity/service/workbench.ts b/apps/server/src/modules/sources-identity/service/workbench.ts index b89e3e8e..c26a7c88 100644 --- a/apps/server/src/modules/sources-identity/service/workbench.ts +++ b/apps/server/src/modules/sources-identity/service/workbench.ts @@ -9,7 +9,7 @@ import { type SiteRow, sql } from "@umt/db"; import { normalizeTitle } from "@umt/domain"; import { ApiException } from "../../../http/errors.ts"; import { decodeCursor, page, toApi } from "../rows.ts"; -import type { SourcesRuntime } from "../runtime.ts"; +import { type SourcesRuntime, siteSearchConcurrency } from "../runtime.ts"; import { refreshReleaseDetail, snapshotSite } from "./snapshot.ts"; // 站点工作台(整理视图):本地快照优先;本地无结果 / 快照过期 / 用户要求时对已开启站点实时补查, @@ -20,13 +20,15 @@ const LIVE_FAIL_BACKOFF_MS = 60_000; interface LiveState { inflight: Map>; failedUntil: Map; + /** 按站点串行的实时搜索(站点访问默认 searchConcurrency = 1 时) */ + siteChain: Map>; } const liveStates = new WeakMap(); function liveState(rt: SourcesRuntime): LiveState { let s = liveStates.get(rt); if (!s) { - s = { inflight: new Map(), failedUntil: new Map() }; + s = { inflight: new Map(), failedUntil: new Map(), siteChain: new Map() }; liveStates.set(rt, s); } return s; @@ -44,14 +46,22 @@ export async function liveSearch( if (failed && failed.until > rt.now().getTime()) return { error: `${failed.error}（退避中）`, coalesced: false }; const running = st.inflight.get(key); if (running) return { ...(await running), coalesced: true }; + const searchConcurrency = siteSearchConcurrency(site.key); + const prev = searchConcurrency <= 1 ? st.siteChain.get(site.id) : undefined; const p = (async () => { + // 同站不同关键词的实时搜索排队执行(不并发打站点) + if (prev) await prev.catch(() => {}); const r = await snapshotSite(rt, { siteId: site.id, kind: "live_search", query: q, maxPages: 1 }); const error = r.status === "failed" || r.status === "skipped" ? (r.error ?? r.reason ?? "failed") : null; if (error) st.failedUntil.set(key, { until: rt.now().getTime() + LIVE_FAIL_BACKOFF_MS, error }); else st.failedUntil.delete(key); return { error }; - })().finally(() => st.inflight.delete(key)); + })().finally(() => { + st.inflight.delete(key); + if (st.siteChain.get(site.id) === p) st.siteChain.delete(site.id); + }); st.inflight.set(key, p); + if (searchConcurrency <= 1) st.siteChain.set(site.id, p); return { ...(await p), coalesced: false }; } diff --git a/apps/server/test/sources-identity/api.test.ts b/apps/server/test/sources-identity/api.test.ts index 50dbb7b9..ddb7e262 100644 --- a/apps/server/test/sources-identity/api.test.ts +++ b/apps/server/test/sources-identity/api.test.ts @@ -175,6 +175,15 @@ describe.skipIf(!up)("来源管理与站点工作台 API", () => { .where("key", "=", `site:${siteId}`) .executeTakeFirst(); expect(bucket).toMatchObject({ capacity: 5, refill_per_sec: 200 }); + // 手动快照要 5 页也只抓访问默认的页数上限(settings.sites.defaultSnapshotMaxPages 缺省 1) + expect((await snapshotSite(h.rt, { siteId, kind: "manual", maxPages: 5 })).pages).toEqual([1]); + // 每拿到一个令牌计一次请求,按小时聚合;站点页读最近 24 小时 + await new Promise((r) => setTimeout(r, 50)); + const stats = await json(await h.req(`/sites/${siteId}/request-stats`)); + expect(stats.status).toBe(200); + expect(stats.body.data.last24h).toBeGreaterThanOrEqual(4); + expect(stats.body.data.hourly).toHaveLength(24); + expect(stats.body.data.access).toMatchObject({ snapshotMaxPages: 1, healthCheckIntervalSec: 21600 }); }); it("工作台列表:快照优先,标注来源与更新时间;筛选与分页", async () => { @@ -219,6 +228,13 @@ describe.skipIf(!up)("来源管理与站点工作台 API", () => { expect(d.body.data.refreshError).toBeNull(); expect(d.body.data.volatile.recheckBeforeDownload).toBe(true); expect(d.body.data.volatile.detailFetchedAt).not.toBeNull(); + // 详情缓存期内(settings.sites.defaultDetailCacheSec 缺省 24 小时)再次刷新不请求站点 + const before = mock.requests.length; + const cached = await json(await h.req(`/sites/${siteId}/releases/${rel.id}?refresh=true`)); + expect(cached.body.data.refreshError).toBeNull(); + expect(mock.requests.length).toBe(before); + // 缓存过期后才重新抓 + await h.db.updateTable("source_release").set({ detail_fetched_at: null }).where("id", "=", rel.id).execute(); mock.setLoggedIn(false); const d2 = await json(await h.req(`/sites/${siteId}/releases/${rel.id}?refresh=true`)); expect(d2.status).toBe(200); diff --git a/apps/server/test/sources-identity/detail-merge.test.ts b/apps/server/test/sources-identity/detail-merge.test.ts new file mode 100644 index 00000000..75732aca --- /dev/null +++ b/apps/server/test/sources-identity/detail-merge.test.ts @@ -0,0 +1,48 @@ +import type { SiteRelease } from "@umt/adapters"; +import { describe, expect, it } from "vitest"; +import { mergeDetailOverList } from "../../src/modules/sources-identity/service/snapshot.ts"; + +// HHClub 2026-09-29 实测:div 详情页没有做种 / 完成数 / 分类名,刷新详情后这些列被 null 冲掉。 +describe("详情刷新沿用列表字段", () => { + const detail: SiteRelease = { + siteId: "hhanclub", + siteReleaseId: "91001", + title: "示例短剧 1080p WEB-DL H265 AAC-HHWEB", + category: { id: "412", name: null }, + sizeBytes: 194385019, + seeders: null, + leechers: null, + snatched: null, + promotion: { type: "none", downloadFactor: 1, uploadFactor: 1, expiresAt: null }, + hr: "unknown", + uploadedAt: "2026-09-29T06:07:03.000Z", + detailUrl: "https://hhanclub.net/details.php?id=91001", + downloadUrl: "https://hhanclub.net/download.php?id=91001", + tags: [], + raw: {}, + observedAt: "2026-09-29T06:19:28.000Z", + }; + const rel = { + seeders: 1, + leechers: 58, + category_raw: "412", + tags_raw: ["官方"], + raw: { _site: { hr: false, snatched: 0, categoryName: "短剧" } }, + }; + + it("详情缺的做种 / 下载 / 完成 / 分类名 / H&R / 标签取列表快照值", () => { + expect(mergeDetailOverList(detail, rel)).toMatchObject({ + seeders: 1, + leechers: 58, + snatched: 0, + category: { id: "412", name: "短剧" }, + hr: false, + tags: ["官方"], + }); + }); + + it("详情有值时以详情为准", () => { + const m = mergeDetailOverList({ ...detail, seeders: 9, hr: true, category: { id: "402", name: "电视剧" } }, rel); + expect(m).toMatchObject({ seeders: 9, hr: true, category: { id: "402", name: "电视剧" } }); + }); +}); diff --git a/apps/web/app/api/sites.ts b/apps/web/app/api/sites.ts index 4a6db6ea..e448799c 100644 --- a/apps/web/app/api/sites.ts +++ b/apps/web/app/api/sites.ts @@ -12,6 +12,7 @@ import { SiteHealthResult, SiteListItem, SiteRefreshLoginAccepted, + SiteRequestStats, SiteSnapshot, type SiteUpdate, SiteUserDataPoint, @@ -33,6 +34,7 @@ export const siteKeys = { one: (id: string) => ["sites", "one", id] as const, accounts: (siteId?: string) => ["sites", "accounts", siteId ?? "all"] as const, snapshots: (siteId: string) => ["sites", "snapshots", siteId] as const, + requestStats: (siteId: string) => ["sites", "request-stats", siteId] as const, definitions: () => ["sites", "definitions"] as const, userData: (siteId: string, accountId?: string, days = 30) => ["sites", "user-data", siteId, accountId ?? "first", days] as const, @@ -88,6 +90,28 @@ export function useSiteSnapshots(siteId: string | undefined, limit = 20) { }); } +/** 最近 24 小时对该站的请求次数与生效的访问默认(每分钟刷新) */ +export function useSiteRequestStats(siteId: string | undefined) { + return useQuery({ + queryKey: siteKeys.requestStats(siteId ?? ""), + enabled: !!siteId, + refetchInterval: 60 * TTL_SECOND, + queryFn: ({ signal }) => + apiParsed(SiteRequestStats, "/api/v1/sites/:id/request-stats", { pathParams: { id: siteId ?? "" }, signal }), + }); +} + +/** 访问默认的一句话说明:"快照每 60 分钟 1 页 · 健康检测每 6 小时 · 详情缓存 24 小时" */ +export function accessSummary(s: SiteRequestStats): string { + const dur = (sec: number) => (sec % 3600 === 0 ? `${sec / 3600} 小时` : `${Math.round(sec / 60)} 分钟`); + const parts = [ + `快照每 ${dur(s.snapshotIntervalSec)} ${s.access.snapshotMaxPages} 页`, + `健康检测每 ${dur(s.access.healthCheckIntervalSec)}`, + ]; + if (s.access.detailCacheSec > 0) parts.push(`详情缓存 ${dur(s.access.detailCacheSec)}`); + return parts.join(" · "); +} + /** 启用 / 停用站点 */ export function useToggleSite() { const qc = useQueryClient(); diff --git a/apps/web/app/components/settings/sites.tsx b/apps/web/app/components/settings/sites.tsx index 5d39258d..1ce896f8 100644 --- a/apps/web/app/components/settings/sites.tsx +++ b/apps/web/app/components/settings/sites.tsx @@ -27,6 +27,24 @@ const DEFAULTS_CARD: CardSpec = { kind: "number", hint: "站点开了定时快照但没设周期时使用；5–10080", }, + { + name: "sites.defaultSnapshotMaxPages", + label: "快照页数", + kind: "number", + hint: "定时和手动快照最多抓几页列表；站点定义更少时按站点；1–20", + }, + { + name: "sites.defaultHealthCheckIntervalSec", + label: "健康检测间隔（小时）", + kind: "number", + hint: "同一站点两次定时检测至少隔这么久；站点定义更长时按站点", + }, + { + name: "sites.defaultDetailCacheSec", + label: "详情缓存（小时）", + kind: "number", + hint: "期内不重复抓同一条资源详情；0 为不缓存", + }, { name: "sites.userDataMaxSeedingPages", label: "做种列表页数上限", @@ -54,6 +72,9 @@ const DEFAULTS_CARD: CardSpec = { "sites.defaultRateLimit.concurrency": n(s.sites.defaultRateLimit.concurrency), "sites.defaultSnapshotIntervalSec": n(Math.round(s.sites.defaultSnapshotIntervalSec / 60)), "sites.userDataMaxSeedingPages": n(s.sites.userDataMaxSeedingPages), + "sites.defaultSnapshotMaxPages": n(s.sites.defaultSnapshotMaxPages), + "sites.defaultHealthCheckIntervalSec": n(s.sites.defaultHealthCheckIntervalSec / 3600), + "sites.defaultDetailCacheSec": n(s.sites.defaultDetailCacheSec / 3600), "sites.userAgent": s.sites.userAgent, "sites.proxyUrl": s.sites.proxyUrl ?? "", }), @@ -61,6 +82,8 @@ const DEFAULTS_CARD: CardSpec = { const min = num(v["sites.defaultSnapshotIntervalSec"]); const rpm = num(v["sites.defaultRateLimit.requestsPerMinute"]); const concurrency = num(v["sites.defaultRateLimit.concurrency"]); + const healthH = num(v["sites.defaultHealthCheckIntervalSec"]); + const detailH = num(v["sites.defaultDetailCacheSec"]); return { sites: { defaultRateLimit: { @@ -70,6 +93,9 @@ const DEFAULTS_CARD: CardSpec = { }, defaultSnapshotIntervalSec: min === undefined ? undefined : Math.round(min * 60), userDataMaxSeedingPages: num(v["sites.userDataMaxSeedingPages"]), + defaultSnapshotMaxPages: num(v["sites.defaultSnapshotMaxPages"]), + defaultHealthCheckIntervalSec: healthH === undefined ? undefined : Math.round(healthH * 3600), + defaultDetailCacheSec: detailH === undefined ? undefined : Math.round(detailH * 3600), userAgent: str(v["sites.userAgent"]) || undefined, proxyUrl: str(v["sites.proxyUrl"]) || null, }, diff --git a/apps/web/app/components/sites/site-overview.tsx b/apps/web/app/components/sites/site-overview.tsx index 02d74f9d..94e8f9c7 100644 --- a/apps/web/app/components/sites/site-overview.tsx +++ b/apps/web/app/components/sites/site-overview.tsx @@ -1,5 +1,5 @@ import type { SiteListItem } from "@umt/contracts"; -import { useSiteUserData } from "~/api/sites.ts"; +import { accessSummary, useSiteRequestStats, useSiteUserData } from "~/api/sites.ts"; import { DetailSection } from "~/components/console/detail-sheet.tsx"; import { ErrorState, LoadingState } from "~/components/console/empty-state.tsx"; import { KvList } from "~/components/console/kv-list.tsx"; @@ -26,6 +26,7 @@ export function SiteOverview({ site }: { site: SiteListItem }) { const acc = primaryAccount(site.accounts); const u = dAcc?.userData ?? null; const history = useSiteUserData(site.id, dAcc?.id, 30); + const stats = useSiteRequestStats(site.id); const st = ACCOUNT_OVERALL[overallStatus(site)]; const h = healthLine(site.lastHealth, site.lastHealthAt); @@ -104,6 +105,17 @@ export function SiteOverview({ site }: { site: SiteListItem }) { : "未开启", }, { label: "限流", value: rateLimitText(site.rateLimit) }, + { + label: "近 24 小时请求", + value: stats.data ? ( + + {stats.data.last24h} 次 + {accessSummary(stats.data)} + + ) : ( + "—" + ), + }, { label: "代理", value: site.proxy ? "使用" : "不使用" }, ]} /> diff --git a/apps/web/app/components/workbench/release-fields.ts b/apps/web/app/components/workbench/release-fields.ts index 08e7e845..6cbfd7c7 100644 --- a/apps/web/app/components/workbench/release-fields.ts +++ b/apps/web/app/components/workbench/release-fields.ts @@ -41,6 +41,97 @@ export function promotionLabel(r: Pick) => r.promotion.kind === "free" || r.promotion.kind === "2xfree"; +/** 完成数(站点 raw._site.snatched);没有时 null */ +export function snatchedOf(r: Pick): number | null { + const v = siteExtra(r).snatched; + return typeof v === "number" && Number.isFinite(v) ? v : null; +} + +/** 促销剩余时间:"剩 3 小时";没有截止时间或已过期返回 null */ +export function remainingLabel(iso: string | null | undefined, now = Date.now()): string | null { + if (!iso) return null; + const left = new Date(iso).getTime() - now; + if (!Number.isFinite(left) || left <= 0) return null; + if (left < 3_600_000) return `剩 ${Math.max(1, Math.round(left / 60_000))} 分钟`; + if (left < 2 * 86_400_000) return `剩 ${Math.round(left / 3_600_000)} 小时`; + return `剩 ${Math.round(left / 86_400_000)} 天`; +} + +/** 绝对时间(本地时区,精确到分钟),用在悬停提示 */ +export function absTime(iso: string | null | undefined): string { + if (!iso) return ""; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return iso; + const p = (n: number) => String(n).padStart(2, "0"); + return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}`; +} + +type ParsedLike = { + resolution?: string; + source?: string; + videoCodec?: string; + hdr?: string[]; + releaseGroup?: { name?: string }; + subtitles?: string[]; + season?: number; + seasonEnd?: number; + seasonInferred?: boolean; + episodes?: { start: number; end: number }; + totalEpisodes?: number; + isSeasonPack?: boolean; +}; + +export type SpecTagKind = "episode" | "resolution" | "source" | "codec" | "hdr" | "group" | "lang" | "site"; +export interface SpecTag { + key: string; + label: string; + kind: SpecTagKind; +} + +const pad2 = (n: number) => String(n).padStart(2, "0"); + +/** + * 资源规格标签:季集 / 分辨率 / 来源 / 编码 / HDR / 发布组 / 国语·中字,再加站点自带标签(tagsRaw)。 + * 都来自标题解析结果与标题、副标题原文,不猜;同名去重。 + */ +export function releaseSpecTags(r: Pick): SpecTag[] { + const p = (r.parsed ?? {}) as ParsedLike; + const out: SpecTag[] = []; + const seen = new Set(); + const add = (kind: SpecTagKind, label: string | undefined | null) => { + const l = label?.trim(); + if (!l || seen.has(l.toLowerCase())) return; + seen.add(l.toLowerCase()); + out.push({ key: `${kind}:${l}`, label: l, kind }); + }; + // 季号是推定的(标题里没写)就不显示季,只显示集 + const season = + p.season !== undefined && !p.seasonInferred + ? `S${pad2(p.season)}${p.seasonEnd && p.seasonEnd !== p.season ? `-S${pad2(p.seasonEnd)}` : ""}` + : ""; + const episode = p.episodes + ? `E${pad2(p.episodes.start)}${p.episodes.end !== p.episodes.start ? `-E${pad2(p.episodes.end)}` : ""}` + : ""; + const total = !episode && p.totalEpisodes ? `全 ${p.totalEpisodes} 集` : ""; + add("episode", [`${season}${episode}`, total].filter(Boolean).join(" ")); + add("resolution", p.resolution === "2160p" ? "4K" : p.resolution); + add("source", p.source); + add("codec", p.videoCodec); + for (const h of p.hdr ?? []) add("hdr", h); + add("group", p.releaseGroup?.name); + const text = `${r.title} ${r.subtitle ?? ""}`; + if (/国语|國語|国配|普通话|中配/.test(text)) add("lang", "国语"); + if (/粤语|粵語/.test(text)) add("lang", "粤语"); + if ( + /中字|中文字幕|简中|繁中|简繁|簡繁|双语字幕|雙語字幕|内嵌中|内封中/.test(text) || + (p.subtitles ?? []).some((l) => l.startsWith("zh")) + ) { + add("lang", "中字"); + } + for (const t of r.tagsRaw ?? []) add("site", t); + return out; +} + /** 相对时间(中文);未来时间显示"x 后" */ export function relTime(iso: string | null | undefined, now = Date.now()): string { if (!iso) return "—"; diff --git a/apps/web/app/components/workbench/release-list.tsx b/apps/web/app/components/workbench/release-list.tsx index d2b43a64..809a0334 100644 --- a/apps/web/app/components/workbench/release-list.tsx +++ b/apps/web/app/components/workbench/release-list.tsx @@ -1,21 +1,34 @@ import type { InventoryEntry, WorkbenchRelease } from "@umt/contracts"; -import { ArrowDown, ArrowUp, Clock } from "lucide-react"; +import { ArrowDown, ArrowUp, Check } from "lucide-react"; import { inventoryFlags } from "~/api/inventory.ts"; import { StatusBadge } from "~/components/console/status-badge.tsx"; import { cn, formatBytes } from "~/lib/utils.ts"; -import { categoryLabel, hrOf, ORIGIN_LABEL, promotionLabel, relTime } from "./release-fields.ts"; +import { + absTime, + categoryLabel, + hrOf, + ORIGIN_LABEL, + promotionLabel, + releaseSpecTags, + relTime, + remainingLabel, + snatchedOf, +} from "./release-fields.ts"; -/** 促销 / H&R 徽章(列表与详情共用) */ +/** 促销 / H&R 徽章(列表与详情共用):促销带剩余时间,过期的写明 */ export function ReleaseBadges({ r }: { r: WorkbenchRelease }) { const promo = promotionLabel(r); const hr = hrOf(r); + const left = promo && !promo.expired ? remainingLabel(r.promotionExpiresAt) : null; return ( <> {promo ? ( - - {promo.label} - {promo.expired ? "（已过期）" : r.promotionExpiresAt ? ` · ${relTime(r.promotionExpiresAt)}截止` : ""} - + + + {promo.label} + {promo.expired ? "（已过期）" : left ? ` · ${left}` : ""} + + ) : null} {hr === "yes" ? H&R : null} {hr === "unknown" ? H&R 未知 : null} @@ -23,6 +36,40 @@ export function ReleaseBadges({ r }: { r: WorkbenchRelease }) { ); } +/** 规格标签(季集 / 分辨率 / 来源 / 编码 / HDR / 发布组 / 国语·中字 / 站点标签):中性样式,不占状态色 */ +export function SpecTags({ r }: { r: Pick }) { + const tags = releaseSpecTags(r); + if (!tags.length) return null; + return ( + <> + {tags.map((t) => ( + + {t.label} + + ))} + + ); +} + +/** 没有关联作品时的说明(明确写出,不猜) */ +export function unresolvedLabel(release: Pick): string { + return release.resolutionStatus === "ambiguous" + ? "识别有歧义" + : release.resolutionStatus === "unidentifiable" + ? "无法识别" + : release.resolutionStatus === "rejected" + ? "已排除" + : "未识别到本地作品"; +} + /** 库存叠加:只来自本地身份/Slot/任务;未识别的资源明确写"未识别",不猜 */ export function InventoryOverlay({ release, @@ -33,17 +80,7 @@ export function InventoryOverlay({ entry: InventoryEntry | undefined; loading: boolean; }) { - if (!release.entity) { - const label = - release.resolutionStatus === "ambiguous" - ? "识别有歧义" - : release.resolutionStatus === "unidentifiable" - ? "无法识别" - : release.resolutionStatus === "rejected" - ? "已排除" - : "未识别到本地作品"; - return {label}; - } + if (!release.entity) return {unresolvedLabel(release)}; const flags = inventoryFlags(entry, release.resolutionStatus, release.id); return ( @@ -68,6 +105,11 @@ export function InventoryOverlay({ ); } +/** + * 资源列表行,两层: + * 主行 = 标题 + 规格标签 + 促销 / H&R;副行 = 副标题(单行省略)+ 分类 / 大小 / 做种·下载·完成 / 上传时间 + 库存叠加。 + * 宽屏副行的数字列定宽右对齐,上下行对得齐;窄屏整行折成卡片,数字换到副标题下面。 + */ export function ReleaseRow({ r, entry, @@ -86,6 +128,11 @@ export function ReleaseRow({ }) { const stale = staleBefore !== null && new Date(r.dataUpdatedAt).getTime() < staleBefore; const cat = categoryLabel(r); + const snatched = snatchedOf(r); + const timeTitle = [ + r.sourcePublishedAt ? `上传 ${absTime(r.sourcePublishedAt)}` : "上传时间未知", + `${ORIGIN_LABEL[r.origin]}数据 ${relTime(r.dataUpdatedAt)}确认`, + ].join("\n"); return (
    • ); } + +/** 副行末尾的库存叠加:只在识别到作品(或识别有歧义)时占一行,未识别的不重复刷"无法识别" */ +function InventoryLine({ + release, + entry, + loading, +}: { + release: WorkbenchRelease; + entry: InventoryEntry | undefined; + loading: boolean; +}) { + if (!release.entity && release.resolutionStatus !== "ambiguous") return null; + return ( +
    + +
    + ); +} diff --git a/apps/web/app/routes/console/sites.$siteId.tsx b/apps/web/app/routes/console/sites.$siteId.tsx index 031b2989..31dd52e9 100644 --- a/apps/web/app/routes/console/sites.$siteId.tsx +++ b/apps/web/app/routes/console/sites.$siteId.tsx @@ -17,7 +17,7 @@ import { Link, useNavigate, useParams, useSearchParams } from "react-router"; import { toast } from "sonner"; import { useInventory } from "~/api/inventory.ts"; import { type ReleaseFilters, useSiteReleases } from "~/api/releases.ts"; -import { snapshotStale, useSites, useSnapshotSite } from "~/api/sites.ts"; +import { accessSummary, snapshotStale, useSiteRequestStats, useSites, useSnapshotSite } from "~/api/sites.ts"; import { openAgentDrawer } from "~/components/agent/agent-store.ts"; import { SiteOriginPanel } from "~/components/browser/desktop-view.tsx"; import { DetailSection } from "~/components/console/detail-sheet.tsx"; @@ -157,6 +157,7 @@ export default function SiteWorkbenchPage() { const staleBefore = refreshFailed ? now + 1 : s ? now - (s.snapshotIntervalSec ?? 3600) * 2000 : null; const selected = all.find((r) => r.id === openId) ?? null; const freeCount = all.filter(isFree).length; + const reqStats = useSiteRequestStats(tab === "resources" ? siteId : undefined); const reset = () => { setQInput(""); @@ -292,10 +293,14 @@ export default function SiteWorkbenchPage() { variant="outline" size="sm" disabled={!s?.enabled || snap.isPending} - title="手动快照（受站点限流）" + title={ + reqStats.data + ? `抓取站点列表前 ${reqStats.data.access.snapshotMaxPages} 页（受站点限流）` + : "抓取站点列表（受站点限流）" + } onClick={() => snap.mutate( - { id: siteId, maxPages: 2 }, + { id: siteId }, { onSuccess: () => toast.success("快照已排队"), onError: (e) => toast.error(errorMessage(e)), @@ -389,6 +394,11 @@ export default function SiteWorkbenchPage() { 已加载 {all.length} 条{localActive ? `，本地筛选后 ${rows.length} 条` : ""}     {freeCount ? 其中免费 {freeCount} : null} + {reqStats.data ? ( + + 近 24 小时请求站点 {reqStats.data.last24h} 次 + + ) : null} {releases.isFetching && !releases.isFetchingNextPage ? 刷新中… : null}
      diff --git a/packages/adapters/src/sites/README.md b/packages/adapters/src/sites/README.md index 133b53b3..dca7df87 100644 --- a/packages/adapters/src/sites/README.md +++ b/packages/adapters/src/sites/README.md @@ -88,14 +88,33 @@ PT 站点适配器,按 MoviePilot v3(提交 e6820a8)迁移:通用配置驱动 sp | 状态 | 站点 | | --- | --- | -| 启用(主力,仍须真实账号核验) | audiences 观众、hhanclub 憨憨、chdbits 彩虹岛、mteam 馒头 | +| 启用,**已实测 2026-09-29**(真实账号只读核对) | hhanclub 憨憨 | +| 启用(主力,仍须真实账号核验) | audiences 观众、chdbits 彩虹岛、mteam 馒头 | | `enabled:false`,**未实测** | agsvpt、hdarea、hddolby、hdfans、hdhome、hdsky、hdtime、hdvideo、keepfrds、nicept、ourbits、piggo、pter、pthome、pttime、soulvoice、ssd、tjupt、ttg | 未实测站点只经 schema 校验,选择器是 MovieClaw 作者在 2026-08/09 的实测值,未经本项目核验。TTG 需要自定义适配器(中文分类、`/t/` 路径、做种/下载同列),通用 spider 结果不可靠;OurBits 的 `cat[]` 分类参数已由定义表达;hddolby 在 MovieClaw 中按 HTML 站点配置,MP 则用 API spider(`HDDolby`),两条路径待账号确认后二选一。 ### 待真实站点核验(W1-3c 新增) -1. HHClub:`div.torrent-table-sub-info` 卡片结构、`?cat[]=` 分类链接(若 URL 编码为 `cat%5B%5D=` 则 `\d+` 会取错,需改 querystring 过滤)、`promotion-tag-*` 新旧类名、`div.torrent-info-hr`、详情页 Tailwind 选择器、分页 0 起。 +1. HHClub(2026-09-29 已实测,见下节):仍未见到的只有促销(`promotion-tag-*`)与 H&R(`div.torrent-info-hr`)命中的实例——当次第 1 页 100 条全部无促销、无 H&R,类名与 Jackett 现行定义一致,暂不改。 + +### HHClub 实测记录(2026-09-29) + +真实账号只读核对,对站点共发 1 次详情请求(列表、用户数据、健康检测用添加站点时产品自己抓的结果;下载令牌只核对列表里的下载链接形态,没有下载)。 + +| 项 | 结果 | +| --- | --- | +| 列表 `div.torrent-table-sub-info` 卡片、分类 `?cat[]=` 链接、大小 / 做种 / 下载 / 完成 | 正确,全为数字;分类 401–405/412 原始 ID 与名称都在 | +| 标题 | **有误,已修**:标题链接文本末尾带新种标记「[新]」(100 条里 58 条;详情页标题没有),`title` 加 `replace` 过滤 | +| 副标题 | 正确,纯文本,无 HTML 残片 | +| 发布时间 | 正确:`span[title]` 绝对时间按 `+08:00` 转 UTC,相对时间只进 `date_elapsed` | +| 详情页 | 标题 / 副标题取到;没有做种、完成、分类名、基本信息行,**刷新详情会用 null 冲掉列表值,已修**(服务端合并,见 `mergeDetailOverList`) | +| 用户数据(`NexusHhanclub`) | 正确;等级「贵宾」之前 `vip` 为 null,**已加 `umt.vipLevels`** | +| 健康检测 / 登录标志 `a[href='logout.php']` | 正确 | +| 下载链接 | 列表给 `download.php?id=…&passkey=…`,入库时 passkey 脱敏 | +| 标签(官方 / 中字 / 国语等) | 列表没有可用的标签选择器;界面按标题解析与副标题原文生成规格标签 | + +访问默认(`umt.defaults`,与 `settings.sites` 取更保守的一方):快照每 60 分钟只抓第 1 页、健康检测 6 小时、用户数据 6 小时、实时搜索并发 1、详情缓存 24 小时;限流 3 秒 1 个令牌、突发 2。fixture:`@umt/testkit` 的 `HHANCLUB_LIST_PAGE_2026`(脱敏最小片段)。 2. CHDBits:域名 `ptchdbits.co`、`font.subtitle`、`td.embedded > span[title]` 促销截止、`a[href='logout.php']`。 3. 专用 spider 的 JSON 字段均按 MP 读取的键编写 fixture,真实响应待核;HDDolby `hr` 字段语义、`page_number` 起点;TorrentLeech 时区;Yema `tagList` 类型;直链缺失时的回退地址(`TODO(verify)`)。 4. 用户信息解析器 fixture 均为最小自编 HTML/JSON;`normalizeDatetime` 不处理相对时间(MP 用 dateparser)。 diff --git a/packages/adapters/src/sites/definitions/definitions.test.ts b/packages/adapters/src/sites/definitions/definitions.test.ts index 6c9e2dc0..0bd34206 100644 --- a/packages/adapters/src/sites/definitions/definitions.test.ts +++ b/packages/adapters/src/sites/definitions/definitions.test.ts @@ -1,4 +1,10 @@ -import { CHDBITS_LIST_PAGE, CHDBITS_LOGIN_PAGE, HHANCLUB_DETAIL_PAGE, HHANCLUB_LIST_PAGE } from "@umt/testkit"; +import { + CHDBITS_LIST_PAGE, + CHDBITS_LOGIN_PAGE, + HHANCLUB_DETAIL_PAGE, + HHANCLUB_LIST_PAGE, + HHANCLUB_LIST_PAGE_2026, +} from "@umt/testkit"; import { describe, expect, it } from "vitest"; import { getSiteDefinition, listSiteDefinitions } from "../registry.ts"; import { parseDetailPage } from "../spider/detail.ts"; @@ -102,6 +108,44 @@ describe("HHClub 定义(div 卡片模板)", () => { }); }); + it("2026-09-29 实测结构:去掉新种标记「[新]」,短剧 412、完成数 0、无促销、时间按 +08:00、下载链接 passkey 脱敏", () => { + const [a, b] = parse(HHANCLUB_LIST_PAGE_2026); + expect(a).toMatchObject({ + siteReleaseId: "91001", + title: "示例短剧 1080p WEB-DL H265 AAC-HHWEB", + subtitle: "示例短剧 | 全30集 | 1080p", + category: { id: "412", name: "短剧" }, + sizeBytes: Math.round(185.38 * 1024 ** 2), + seeders: 1, + leechers: 58, + snatched: 0, + promotion: { type: "none", downloadFactor: 1, uploadFactor: 1, expiresAt: null }, + hr: false, + uploadedAt: "2026-09-29T06:07:03.000Z", + detailUrl: "https://hhanclub.net/details.php?id=91001&hit=1", + }); + expect(a!.downloadUrl).not.toContain("0000000000000000"); + expect(b).toMatchObject({ + title: "Example Show S01 2026 2160p WEB-DL H.264 AAC-HHWEB", + category: { id: "402", name: "电视剧" }, + snatched: 187, + uploadedAt: "2026-09-29T04:03:28.000Z", + }); + }); + + it("站点访问默认:快照只抓 1 页、健康检测 6 小时、详情缓存 24 小时、搜索并发 1;3 秒 1 个令牌、突发 2", () => { + expect(hhanclubDefinition.umt.defaults).toEqual({ + snapshotIntervalSec: 3600, + snapshotMaxPages: 1, + healthCheckIntervalSec: 21600, + userDataIntervalSec: 21600, + searchConcurrency: 1, + detailCacheSec: 86400, + }); + expect(hhanclubDefinition.umt.rateLimit).toEqual({ minIntervalMs: 3000, burst: 2 }); + expect(hhanclubDefinition.umt.vipLevels).toContain("贵宾"); + }); + it("浏览从 page=0 起(MovieClaw page_offset -1);mtype=tv 选 402/403/407/412", () => { expect(new SiteSpider(hhanclubDefinition, { page: 0 }).getSearchUrl()).toBe( "https://hhanclub.net/torrents.php?page=0", diff --git a/packages/adapters/src/sites/definitions/hhanclub.json b/packages/adapters/src/sites/definitions/hhanclub.json index 5567acaa..a135c144 100644 --- a/packages/adapters/src/sites/definitions/hhanclub.json +++ b/packages/adapters/src/sites/definitions/hhanclub.json @@ -86,7 +86,17 @@ "attribute": "title" }, "title": { - "text": "{% if fields['title_optional'] %}{{ fields['title_optional'] }}{% else %}{{ fields['title_default'] }}{% endif %}" + "text": "{% if fields['title_optional'] %}{{ fields['title_optional'] }}{% else %}{{ fields['title_default'] }}{% endif %}", + "filters": [ + { + "name": "replace", + "args": ["[新]", ""] + }, + { + "name": "replace", + "args": ["(新)", ""] + } + ] }, "details": { "selector": "div.torrent-title a[href*='details.php']", @@ -175,13 +185,13 @@ } }, "umt": { - "version": "hhanclub@mc-v0.27.0", + "version": "hhanclub@mc-v0.27.0+umt.1", "kind": "nexusphp", "displayName": "憨憨", "timezone": "+08:00", "rateLimit": { "minIntervalMs": 3000, - "burst": 1 + "burst": 2 }, "capabilities": { "list": true, @@ -249,6 +259,15 @@ "douban": "a[href*='douban.com']::attr(href)", "fileList": "td.rowfollow > ul > li::text" } + }, + "vipLevels": ["贵宾", "VIP"], + "defaults": { + "snapshotIntervalSec": 3600, + "snapshotMaxPages": 1, + "healthCheckIntervalSec": 21600, + "userDataIntervalSec": 21600, + "searchConcurrency": 1, + "detailCacheSec": 86400 } } } diff --git a/packages/adapters/src/sites/definitions/schema.ts b/packages/adapters/src/sites/definitions/schema.ts index 06740ca3..97f172f8 100644 --- a/packages/adapters/src/sites/definitions/schema.ts +++ b/packages/adapters/src/sites/definitions/schema.ts @@ -74,6 +74,14 @@ const CapabilitiesSchema = z.strictObject({ download: z.boolean(), rss: z.boolean(), }); +const AccessDefaultsSchema = z.strictObject({ + snapshotIntervalSec: z.number().int().min(300).optional(), + snapshotMaxPages: z.number().int().min(1).max(10).optional(), + healthCheckIntervalSec: z.number().int().min(300).optional(), + userDataIntervalSec: z.number().int().min(300).optional(), + searchConcurrency: z.number().int().min(1).max(4).optional(), + detailCacheSec: z.number().int().min(0).optional(), +}); const RateLimitSchema = z.strictObject({ minIntervalMs: z.number().int().min(0), burst: z.number().int().min(1) }); const TimezoneSchema = z.string().regex(/^[+-]\d{2}:\d{2}$/, "时区须为 ±HH:MM 偏移"); const PairSchema = z.tuple([z.string(), z.string()]); @@ -87,6 +95,8 @@ const UmtNexusSchema = z.strictObject({ authSupported: z.array(z.enum(["cookie", "credential", "apikey"])).optional(), timezone: TimezoneSchema, rateLimit: RateLimitSchema, + /** 站点级访问默认(与 settings.sites 取更保守的一方) */ + defaults: AccessDefaultsSchema.optional(), capabilities: CapabilitiesSchema, categoryNames: z.record(z.string(), z.string()).optional(), loginSuccess: z.string().optional(), diff --git a/packages/adapters/src/sites/spider/schema.ts b/packages/adapters/src/sites/spider/schema.ts index feb39ee3..df9934d0 100644 --- a/packages/adapters/src/sites/spider/schema.ts +++ b/packages/adapters/src/sites/spider/schema.ts @@ -121,6 +121,21 @@ export interface IndexerDefinition { umt: UmtSiteExtension; } +export interface SiteAccessDefaults { + /** 定时快照周期(秒) */ + snapshotIntervalSec?: number; + /** 快照(定时 / 手动)最多抓几页列表 */ + snapshotMaxPages?: number; + /** 健康检测最短间隔(秒) */ + healthCheckIntervalSec?: number; + /** 用户数据刷新间隔(秒;定时任务 sites.user_data 的周期,仅作说明与校验) */ + userDataIntervalSec?: number; + /** 同站实时搜索并发 */ + searchConcurrency?: number; + /** 详情页结果缓存(秒):期内不重复抓详情 */ + detailCacheSec?: number; +} + export interface UmtSiteExtension { /** 定义版本,入 site_snapshot */ version: string; @@ -151,6 +166,11 @@ export interface UmtSiteExtension { /** 站点时间时区偏移,如 "+08:00";pubdate/freedate 按此转 UTC */ timezone: string; rateLimit: { minIntervalMs: number; burst: number }; + /** + * 站点级访问默认(本项目扩展):按站点实测给出的保守下限。运行时与 settings.sites 取更保守的一方 + * (周期 / 缓存取较长,页数 / 并发取较小);站点行自己设了快照周期时以站点行为准。 + */ + defaults?: SiteAccessDefaults; capabilities: { list: boolean; search: boolean; detail: boolean; download: boolean; rss: boolean }; /** 原始分类 ID → 站点分类名(列表只有 ID 时补名称;不做投影) */ categoryNames?: Record; diff --git a/packages/db/migrations/0046_site_request_count.ts b/packages/db/migrations/0046_site_request_count.ts new file mode 100644 index 00000000..5dbc29f5 --- /dev/null +++ b/packages/db/migrations/0046_site_request_count.ts @@ -0,0 +1,25 @@ +import type { Kysely } from "kysely"; +import { dropTables, execDdl } from "../src/migration-helpers.ts"; + +// 站点请求计数:每次从站点限流器拿到令牌(列表 / 搜索 / 详情 / 下载 / 登录 / 用户数据 / 原站视图)记一次, +// 按小时聚合,api / worker 多进程共用。站点页展示"最近 24 小时请求次数";保留 7 天,写入时顺手清理。 + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl( + db, + ` +create table site_request_count ( + site_id uuid not null references site(id) on delete cascade, + hour timestamptz not null, + count integer not null default 0 check (count >= 0), + primary key (site_id, hour) +); +`, + ); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl(db, dropTables(["site_request_count"])); +} diff --git a/packages/db/test/schema.test.ts b/packages/db/test/schema.test.ts index 54ac3cbd..c878f3f2 100644 --- a/packages/db/test/schema.test.ts +++ b/packages/db/test/schema.test.ts @@ -121,6 +121,8 @@ const EXPECTED_TABLES = [ "rate_limit_bucket", // C2-A PG 短期缓存(0042,UNLOGGED) "kv_cache", + // 站点请求计数(0046,按小时聚合) + "site_request_count", ].sort(); /** 不走 entityTable() 的表:追加型/引擎自有表,没有 uuid id + created_at/updated_at 三件套 */ @@ -149,6 +151,8 @@ const NON_ENTITY_TABLES = [ "rate_limit_bucket", // 缓存:(region, key) 主键 "kv_cache", + // 站点请求计数:(site_id, hour) 主键 + "site_request_count", ]; describe.skipIf(skip)("schema migrations (real PG)", () => { diff --git a/packages/testkit/src/sites/fixtures/definitions/hhanclub.ts b/packages/testkit/src/sites/fixtures/definitions/hhanclub.ts index 31e790b2..8ebe97f5 100644 --- a/packages/testkit/src/sites/fixtures/definitions/hhanclub.ts +++ b/packages/testkit/src/sites/fixtures/definitions/hhanclub.ts @@ -45,3 +45,38 @@ export const HHANCLUB_DETAIL_PAGE = `HHCLUB :: <div><a href="download.php?id=90002">下载种子</a></div> <div id="kdescr"><a href="https://www.imdb.com/title/tt15239678/">IMDb</a> <a href="https://movie.douban.com/subject/35575567/">豆瓣</a> 简介……</div> </div></body></html>`; + +/** + * 2026-09-29 真实账号核对后补的最小脱敏片段(名称、ID、数值均已替换;不含 passkey 与用户信息)。 + * 核对到的真实行为:列表标题链接文本末尾带新种标记「[新]」(详情页标题没有);短剧分类 412; + * 新种完成数为 0;无促销时没有 promotion-tag 元素;上传时间在 span[title] 里,文本是相对时间。 + * 下载链接带 passkey(这里用占位值)。 + */ +export const HHANCLUB_LIST_PAGE_2026 = `<!DOCTYPE html><html><head><title>HHCLUB :: 种子 + +
      +
      +
      短剧
      +
      示例短剧 1080p WEB-DL H265 AAC-HHWEB[新]
      +
      示例短剧 | 全30集 | 1080p
      +
      下载
      +
      5分钟
      +
      185.38 MB
      +
      1
      +
      58
      +
      0
      +
      +
      +
      电视剧
      +
      Example Show S01 2026 2160p WEB-DL H.264 AAC-HHWEB
      +
      示例剧集 | 第12集 | 4K | 类型: 奇幻 [内嵌中字]
      +
      下载
      +
      2时8分钟
      +
      1.46 GB
      +
      131
      +
      5
      +
      187
      +
      +
      + +`; diff --git a/scripts/mp-sync/movieclaw-overrides/hhanclub.json b/scripts/mp-sync/movieclaw-overrides/hhanclub.json index d9564c5e..eb663d80 100644 --- a/scripts/mp-sync/movieclaw-overrides/hhanclub.json +++ b/scripts/mp-sync/movieclaw-overrides/hhanclub.json @@ -1,5 +1,5 @@ { - "$comment": "HHClub 分类名取自 MovieClaw hhanclub.yaml 注释(导航栏分类表);REQ §4.5 要求综艺/体育/短剧保留原始分类。", + "$comment": "HHClub 分类名取自 MovieClaw hhanclub.yaml 注释(导航栏分类表);REQ §4.5 要求综艺/体育/短剧保留原始分类。2026-09-29 真实账号核对:列表标题链接里带站点的新种标记「[新]」(详情页标题没有),用 replace 过滤掉;等级「贵宾」为 VIP;访问默认按低频原则收紧(快照只抓第 1 页、健康检测 6 小时、详情缓存 24 小时、搜索并发 1)。", "umt": { "categoryNames": { "401": "电影", @@ -12,6 +12,37 @@ "410": "音乐", "411": "MV", "412": "短剧" + }, + "version": "hhanclub@mc-v0.27.0+umt.1", + "rateLimit": { + "minIntervalMs": 3000, + "burst": 2 + }, + "vipLevels": ["贵宾", "VIP"], + "defaults": { + "snapshotIntervalSec": 3600, + "snapshotMaxPages": 1, + "healthCheckIntervalSec": 21600, + "userDataIntervalSec": 21600, + "searchConcurrency": 1, + "detailCacheSec": 86400 + } + }, + "torrents": { + "fields": { + "title": { + "text": "{% if fields['title_optional'] %}{{ fields['title_optional'] }}{% else %}{{ fields['title_default'] }}{% endif %}", + "filters": [ + { + "name": "replace", + "args": ["[新]", ""] + }, + { + "name": "replace", + "args": ["(新)", ""] + } + ] + } } } }