fix(sites): HHClub parsing, resource list layout and request restraint
HHClub (憨憨) was checked read-only against a real account (one detail
request; list, user data and health came from the product's own fetch).
Parsing fixes:
- list titles carried the site's new-torrent marker "[新]" (58 of 100
rows; the detail page title has none); stripped with a replace filter
- refreshing a detail overwrote seeders / leechers / snatched / category
name / H&R with null because the HHClub div detail page lacks them;
detail values now fall back to the list snapshot (mergeDetailOverList),
and an unknown detail promotion keeps the list promotion
- level "贵宾" is VIP (umt.vipLevels)
- sanitized minimal fixture HHANCLUB_LIST_PAGE_2026 + tests; README marks
HHClub as verified 2026-09-29
Size, counts, category ids/names, +08:00 time and subtitle were correct.
Request restraint:
- site definitions may carry umt.defaults; merged with settings.sites
(longer intervals, fewer pages): new settings defaultSnapshotMaxPages 1,
defaultHealthCheckIntervalSec 6 h, defaultDetailCacheSec 24 h; default
burst 2. HHClub: snapshot 60 min page 1 only, health 6 h, search
concurrency 1, detail cache 24 h, 1 token / 3 s burst 2
- periodic / manual snapshots are capped to that page count; scheduled
health checks skip sites checked within the interval; detail refresh is
served from cache inside the window; live searches on one site queue
- every rate-limiter token is counted per site per hour
(site_request_count, migration 0046); GET /sites/{id}/request-stats;
site overview and resource list show requests in the last 24 hours
Resource list: two-layer rows. Main line: title + spec tags (episodes,
resolution, source, codec, HDR, group, 国语/中字, site tags) + promotion
with time left + H&R. Second line: subtitle (one line, ellipsis) and
fixed-width category / size / seed-leech-snatched / upload time
(relative, absolute on hover) columns, stale badge, inventory overlay;
folds into a card on phones.
Also aligns identity_candidate.normalized_title order in db types with
the migrations (schema drift test).
Takes effect after the stack is rebuilt (migration 0046).
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
20b63db254
commit
901b01d1a1
43 files changed
+1222
-1315
No files matched your search
@@ -19,6 +19,7 @@ import {
|
||||
SiteHealthResult,
|
||||
SiteRefreshLoginAccepted,
|
||||
SiteRefreshLoginInput,
|
||||
SiteRequestStats,
|
||||
SiteSnapshot,
|
||||
SiteUserDataPoint,
|
||||
SiteUserDataQuery,
|
||||
@@ -259,6 +260,49 @@ export function sourcesRoutes(app: ModuleApp, rt: () => SourcesRuntime) {
|
||||
return { items: toApiList(p.items), nextCursor: p.nextCursor };
|
||||
},
|
||||
);
|
||||
route(
|
||||
app,
|
||||
{
|
||||
method: "get",
|
||||
path: "/sites/{id}/request-stats",
|
||||
scope: "read",
|
||||
summary: "站点请求统计:最近 24 小时对该站发出的请求次数(按小时)与生效的访问默认",
|
||||
tags: T_SITES,
|
||||
params: IdParam,
|
||||
response: SiteRequestStats,
|
||||
},
|
||||
async ({ params }) => {
|
||||
const site = await db()
|
||||
.selectFrom("site")
|
||||
.select(["id", "key", "snapshot_interval_sec"])
|
||||
.where("id", "=", params.id)
|
||||
.where("deleted_at", "is", null)
|
||||
.executeTakeFirst();
|
||||
if (!site) throw new ApiException("not_found", "站点不存在");
|
||||
const now = rt().now().getTime();
|
||||
const cur = Math.floor(now / 3_600_000) * 3_600_000;
|
||||
const since = new Date(cur - 23 * 3_600_000);
|
||||
const rows = await db()
|
||||
.selectFrom("site_request_count")
|
||||
.select(["hour", "count"])
|
||||
.where("site_id", "=", site.id)
|
||||
.where("hour", ">=", since)
|
||||
.execute();
|
||||
const byHour = new Map(rows.map((r) => [r.hour.getTime(), r.count]));
|
||||
const hourly = Array.from({ length: 24 }, (_, i) => {
|
||||
const t = since.getTime() + i * 3_600_000;
|
||||
return { hour: new Date(t).toISOString(), count: byHour.get(t) ?? 0 };
|
||||
});
|
||||
const access = await rt().siteAccess(site.key);
|
||||
return {
|
||||
siteId: site.id,
|
||||
last24h: hourly.reduce((a, h) => a + h.count, 0),
|
||||
hourly,
|
||||
snapshotIntervalSec: site.snapshot_interval_sec ?? access.snapshotIntervalSec,
|
||||
access,
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// ── W9-A 站点中心:删除、健康检测、刷新登录、用户数据、凭证 ──
|
||||
route(
|
||||
|
||||
@@ -14,7 +14,7 @@ import {
|
||||
type TmdbSharedCache,
|
||||
} from "@umt/adapters";
|
||||
import { DEFAULT_SETTINGS, type SitesSettings } from "@umt/contracts";
|
||||
import { pgTokenBucketFor, type SiteRow } from "@umt/db";
|
||||
import { pgTokenBucketFor, type SiteRow, sql } from "@umt/db";
|
||||
import { CACHE_REGIONS, sharedCache, type TmdbCachePolicy, tmdbCachePolicy } from "../../shared-cache.ts";
|
||||
import type { ServerContext } from "../types.ts";
|
||||
|
||||
@@ -170,7 +170,7 @@ export class SourcesRuntime {
|
||||
if (hit && hit.sig === sig) return hit.adapter;
|
||||
const lo = limiterOptions(def, site.rate_limit, defaults.defaultRateLimit);
|
||||
// 跨进程:api / worker / worker:agent 对同一站点共用 PG 令牌桶(key = site:<id>);进程内先串行排队
|
||||
const limiter =
|
||||
const inner =
|
||||
this.overrides.sharedRateLimit === false
|
||||
? createSiteRateLimiter(lo)
|
||||
: createSharedSiteRateLimiter({
|
||||
@@ -183,6 +183,7 @@ export class SourcesRuntime {
|
||||
"站点共享限流不可用,本次退回进程内限流",
|
||||
),
|
||||
});
|
||||
const limiter = countingLimiter(inner, () => this.countSiteRequest(site.id));
|
||||
const cfg = (site.config ?? {}) as { apiBaseUrl?: string };
|
||||
const baseUrl =
|
||||
def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url;
|
||||
@@ -197,6 +198,41 @@ export class SourcesRuntime {
|
||||
return adapter;
|
||||
}
|
||||
|
||||
/**
|
||||
* 站点请求计数(site_request_count,按小时聚合):每拿到一个令牌记一次。写失败只记日志,不影响请求;
|
||||
* 顺手删 7 天前的行(每进程每小时最多一次)。
|
||||
*/
|
||||
private lastCountPrune = 0;
|
||||
countSiteRequest(siteId: string): void {
|
||||
const hour = new Date(Math.floor(this.now().getTime() / 3_600_000) * 3_600_000);
|
||||
const db = this.ctx.db;
|
||||
void sql`insert into site_request_count (site_id, hour, count) values (${siteId}, ${hour}, 1)
|
||||
on conflict (site_id, hour) do update set count = site_request_count.count + 1`
|
||||
.execute(db)
|
||||
.then(async () => {
|
||||
if (Date.now() - this.lastCountPrune < 3_600_000) return;
|
||||
this.lastCountPrune = Date.now();
|
||||
await sql`delete from site_request_count where hour < ${new Date(hour.getTime() - 7 * 86_400_000)}`.execute(db);
|
||||
})
|
||||
.catch((err) =>
|
||||
this.ctx.logger.warn({ siteId, err: { message: (err as Error).message } }, "站点请求计数写入失败"),
|
||||
);
|
||||
}
|
||||
|
||||
/** 最近 hours 小时(含当前小时)该站点的请求次数 */
|
||||
async siteRequestCount(siteId: string, hours = 24): Promise<number> {
|
||||
const hour = Math.floor(this.now().getTime() / 3_600_000) * 3_600_000;
|
||||
const since = new Date(hour - (hours - 1) * 3_600_000);
|
||||
const r = await sql<{ n: number }>`select coalesce(sum(count), 0)::int as n from site_request_count
|
||||
where site_id = ${siteId} and hour >= ${since}`.execute(this.ctx.db);
|
||||
return Number(r.rows[0]?.n ?? 0);
|
||||
}
|
||||
|
||||
/** 站点访问默认:站点定义 umt.defaults 与 settings.sites 取更保守的一方 */
|
||||
async siteAccess(siteKey: string): Promise<SiteAccess> {
|
||||
return siteAccessFor(getSiteDefinition(siteKey) ?? null, await this.siteDefaults());
|
||||
}
|
||||
|
||||
/** 已建立的站点限流器(测试与诊断用) */
|
||||
limiterOf(siteId: string): SiteRateLimiter | null {
|
||||
return this.adapters.get(siteId)?.limiter ?? null;
|
||||
@@ -260,6 +296,63 @@ function sameUrl(a: string, b: string): boolean {
|
||||
return a.replace(/\/+$/, "") === b.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
/** 站点访问默认(已合并定义与设置) */
|
||||
export interface SiteAccess {
|
||||
/** 站点行没设快照周期时用 */
|
||||
snapshotIntervalSec: number;
|
||||
snapshotMaxPages: number;
|
||||
healthCheckIntervalSec: number;
|
||||
detailCacheSec: number;
|
||||
searchConcurrency: number;
|
||||
}
|
||||
|
||||
type DefinitionDefaults = {
|
||||
snapshotIntervalSec?: number;
|
||||
snapshotMaxPages?: number;
|
||||
healthCheckIntervalSec?: number;
|
||||
detailCacheSec?: number;
|
||||
searchConcurrency?: number;
|
||||
};
|
||||
|
||||
/** 周期 / 缓存取较长,页数 / 并发取较小(定义值是按站点实测给的保守下限) */
|
||||
export function siteAccessFor(def: SiteDefinition | null, s: SitesSettings): SiteAccess {
|
||||
const d = ((def?.umt as { defaults?: DefinitionDefaults } | undefined)?.defaults ?? {}) as DefinitionDefaults;
|
||||
const max = (a: number | undefined, b: number) => (a === undefined ? b : Math.max(a, b));
|
||||
const min = (a: number | undefined, b: number) => (a === undefined ? b : Math.min(a, b));
|
||||
return {
|
||||
snapshotIntervalSec: max(d.snapshotIntervalSec, s.defaultSnapshotIntervalSec),
|
||||
snapshotMaxPages: min(d.snapshotMaxPages, s.defaultSnapshotMaxPages),
|
||||
healthCheckIntervalSec: max(d.healthCheckIntervalSec, s.defaultHealthCheckIntervalSec),
|
||||
detailCacheSec: max(d.detailCacheSec, s.defaultDetailCacheSec),
|
||||
searchConcurrency: d.searchConcurrency ?? 1,
|
||||
};
|
||||
}
|
||||
|
||||
/** 同站实时搜索并发(只看站点定义,同步取值;缺省 1) */
|
||||
export function siteSearchConcurrency(siteKey: string): number {
|
||||
const def = getSiteDefinition(siteKey);
|
||||
return ((def?.umt as { defaults?: DefinitionDefaults } | undefined)?.defaults?.searchConcurrency ?? 1) as number;
|
||||
}
|
||||
|
||||
/** 包一层限流器:每发放一个令牌回调一次(计数) */
|
||||
function countingLimiter(inner: SiteRateLimiter, onGrant: () => void): SiteRateLimiter {
|
||||
return {
|
||||
get stats() {
|
||||
return inner.stats;
|
||||
},
|
||||
async acquire(signal) {
|
||||
await inner.acquire(signal);
|
||||
onGrant();
|
||||
},
|
||||
tryAcquire() {
|
||||
const ok = inner.tryAcquire();
|
||||
if (ok) onGrant();
|
||||
return ok;
|
||||
},
|
||||
penalize: (ms) => inner.penalize(ms),
|
||||
};
|
||||
}
|
||||
|
||||
type RateLimitLike = { requestsPerMinute?: number; minIntervalMs?: number; concurrency?: number };
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { getSiteDefinition } from "@umt/adapters";
|
||||
import { sql } from "@umt/db";
|
||||
import type { SourcesRuntime } from "./runtime.ts";
|
||||
import { type SourcesRuntime, siteAccessFor } from "./runtime.ts";
|
||||
import { archiveStalePool } from "./service/identity-decisions.ts";
|
||||
|
||||
// worker 后台调度:按 Feed 轮询策略发 feed.due(outbox → ingestFeed)、按站点快照周期入队 snapshotSite、
|
||||
@@ -67,24 +68,32 @@ export async function tickFeeds(rt: SourcesRuntime): Promise<number> {
|
||||
export async function tickSnapshots(rt: SourcesRuntime): Promise<number> {
|
||||
const { ctx } = rt;
|
||||
const now = rt.now();
|
||||
// 站点未设周期时用全局默认(settings.sites.defaultSnapshotIntervalSec)
|
||||
const fallback = (await rt.siteDefaults()).defaultSnapshotIntervalSec;
|
||||
const sites = await sql<{ id: string; snapshot_interval_sec: number }>`
|
||||
select id, coalesce(snapshot_interval_sec, ${fallback}::int) as snapshot_interval_sec from site
|
||||
where enabled and snapshot_enabled and deleted_at is null
|
||||
and (last_snapshot_at is null
|
||||
or last_snapshot_at + make_interval(secs => coalesce(snapshot_interval_sec, ${fallback}::int)) <= ${now})`.execute(
|
||||
ctx.db,
|
||||
);
|
||||
for (const s of sites.rows) {
|
||||
const slot = Math.floor(now.getTime() / 1000 / s.snapshot_interval_sec);
|
||||
// 站点未设周期时用站点定义与全局默认里更长的一个(settings.sites.defaultSnapshotIntervalSec / umt.defaults)
|
||||
const defaults = await rt.siteDefaults();
|
||||
const rows = await ctx.db
|
||||
.selectFrom("site")
|
||||
.select(["id", "key", "snapshot_interval_sec", "last_snapshot_at"])
|
||||
.where("enabled", "=", true)
|
||||
.where("snapshot_enabled", "=", true)
|
||||
.where("deleted_at", "is", null)
|
||||
.execute();
|
||||
const due: Array<{ id: string; interval: number }> = [];
|
||||
for (const r of rows) {
|
||||
const interval =
|
||||
r.snapshot_interval_sec ?? siteAccessFor(getSiteDefinition(r.key) ?? null, defaults).snapshotIntervalSec;
|
||||
if (!r.last_snapshot_at || r.last_snapshot_at.getTime() + interval * 1000 <= now.getTime()) {
|
||||
due.push({ id: r.id, interval });
|
||||
}
|
||||
}
|
||||
for (const s of due) {
|
||||
const slot = Math.floor(now.getTime() / 1000 / s.interval);
|
||||
await ctx.engine.enqueue(
|
||||
"snapshotSite",
|
||||
{ siteId: s.id, kind: "periodic" },
|
||||
{ idempotencyKey: `snapshotSite:${s.id}:${slot}`, traceId: randomUUID() },
|
||||
);
|
||||
}
|
||||
return sites.rows.length;
|
||||
return due.length;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -889,7 +889,7 @@ export async function tickSiteUserData(rt: SourcesRuntime, slot: string): Promis
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
/** sites.healthCheck:启用站点逐个检测(同步,走站点限流) */
|
||||
/** sites.healthCheck:启用站点逐个检测(同步,走站点限流);按站点访问默认的间隔跳过刚检测过的站点 */
|
||||
export async function tickSiteHealth(rt: SourcesRuntime): Promise<number> {
|
||||
const sites = await rt.ctx.db
|
||||
.selectFrom("site")
|
||||
@@ -899,7 +899,11 @@ export async function tickSiteHealth(rt: SourcesRuntime): Promise<number> {
|
||||
.where("adapter", "!=", "generic_rss")
|
||||
.execute();
|
||||
let n = 0;
|
||||
const now = rt.now().getTime();
|
||||
for (const s of sites) {
|
||||
// 站点访问默认的最短间隔内检测过(手动或定时)就跳过;留 2 分钟余量,免得和定时任务周期错开一轮
|
||||
const { healthCheckIntervalSec } = await rt.siteAccess(s.key);
|
||||
if (s.last_health_at && now - s.last_health_at.getTime() < healthCheckIntervalSec * 1000 - 120_000) continue;
|
||||
try {
|
||||
await checkSiteHealth(rt, s);
|
||||
n++;
|
||||
|
||||
@@ -119,7 +119,11 @@ export async function snapshotSite(rt: SourcesRuntime, input: SnapshotInput): Pr
|
||||
return { status: "skipped", reason: (e as Error).message, ...none };
|
||||
}
|
||||
const cfgPages = Number((site.config as { snapshotPages?: number })?.snapshotPages) || 1;
|
||||
const maxPages = Math.min(20, Math.max(1, input.maxPages ?? cfgPages));
|
||||
let maxPages = Math.min(20, Math.max(1, input.maxPages ?? cfgPages));
|
||||
// 定时 / 手动快照受站点访问默认的页数上限约束(补漏有自己的页数预算,实时补查只抓 1 页)
|
||||
if (input.kind === "periodic" || input.kind === "manual") {
|
||||
maxPages = Math.min(maxPages, (await rt.siteAccess(site.key)).snapshotMaxPages);
|
||||
}
|
||||
const snap = await db
|
||||
.insertInto("site_snapshot")
|
||||
.values({
|
||||
@@ -211,9 +215,14 @@ export async function refreshReleaseDetail(rt: SourcesRuntime, srId: string): Pr
|
||||
if (!rel) return "not_found";
|
||||
const site = await db.selectFrom("site").selectAll().where("id", "=", rel.site_id).executeTakeFirst();
|
||||
if (!site?.enabled) return "site_disabled";
|
||||
// 详情缓存:站点访问默认的缓存期内抓过就不再请求站点(列表快照会持续更新做种 / 促销)
|
||||
const { detailCacheSec } = await rt.siteAccess(site.key);
|
||||
if (rel.detail_fetched_at && rt.now().getTime() - rel.detail_fetched_at.getTime() < detailCacheSec * 1000) {
|
||||
return null;
|
||||
}
|
||||
let detail: SiteRelease;
|
||||
try {
|
||||
detail = await rt.siteAdapter(site).detail(rel.site_release_id);
|
||||
detail = mergeDetailOverList(await rt.siteAdapter(site).detail(rel.site_release_id), rel);
|
||||
} catch (e) {
|
||||
if (e instanceof SiteError) {
|
||||
if (e.isAuthError) await db.transaction().execute((trx) => onAuthError(rt, trx, site, e));
|
||||
@@ -226,7 +235,47 @@ export async function refreshReleaseDetail(rt: SourcesRuntime, srId: string): Pr
|
||||
}
|
||||
await db.transaction().execute(async (trx) => {
|
||||
await upsertSiteRelease(rt, trx, site, detail);
|
||||
await trx.updateTable("source_release").set({ detail_fetched_at: rt.now() }).where("id", "=", srId).execute();
|
||||
const keepPromotion = detail.promotion.type === "unknown";
|
||||
await trx
|
||||
.updateTable("source_release")
|
||||
.set({
|
||||
detail_fetched_at: rt.now(),
|
||||
// 详情页没解析出促销时保留列表快照的促销,不用"未知"覆盖
|
||||
...(keepPromotion
|
||||
? { promotion: jsonb(rel.promotion ?? {}), promotion_expires_at: rel.promotion_expires_at }
|
||||
: {}),
|
||||
})
|
||||
.where("id", "=", srId)
|
||||
.execute();
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 详情页缺的字段沿用列表快照已有的值:有些站点(如 HHClub 的 div 详情页)详情里没有做种 / 完成数 / 分类名,
|
||||
* upsert 时 seeders / leechers 与 raw._site 会被整体覆盖,不能用 null 冲掉列表数据。
|
||||
*/
|
||||
export function mergeDetailOverList(
|
||||
detail: SiteRelease,
|
||||
rel: {
|
||||
seeders: number | null;
|
||||
leechers: number | null;
|
||||
category_raw: string | null;
|
||||
tags_raw: string[] | null;
|
||||
raw: unknown;
|
||||
},
|
||||
): SiteRelease {
|
||||
const prev = ((rel.raw ?? {}) as { _site?: Record<string, unknown> })._site ?? {};
|
||||
return {
|
||||
...detail,
|
||||
seeders: detail.seeders ?? rel.seeders,
|
||||
leechers: detail.leechers ?? rel.leechers,
|
||||
snatched: detail.snatched ?? (typeof prev.snatched === "number" ? prev.snatched : null),
|
||||
category: {
|
||||
id: detail.category.id ?? rel.category_raw,
|
||||
name: detail.category.name ?? (typeof prev.categoryName === "string" ? prev.categoryName : null),
|
||||
},
|
||||
hr: detail.hr === "unknown" && typeof prev.hr === "boolean" ? prev.hr : detail.hr,
|
||||
tags: detail.tags.length ? detail.tags : (rel.tags_raw ?? []),
|
||||
};
|
||||
}
|
||||
@@ -9,7 +9,7 @@ import { type SiteRow, sql } from "@umt/db";
|
||||
import { normalizeTitle } from "@umt/domain";
|
||||
import { ApiException } from "../../../http/errors.ts";
|
||||
import { decodeCursor, page, toApi } from "../rows.ts";
|
||||
import type { SourcesRuntime } from "../runtime.ts";
|
||||
import { type SourcesRuntime, siteSearchConcurrency } from "../runtime.ts";
|
||||
import { refreshReleaseDetail, snapshotSite } from "./snapshot.ts";
|
||||
|
||||
// 站点工作台(整理视图):本地快照优先;本地无结果 / 快照过期 / 用户要求时对已开启站点实时补查,
|
||||
@@ -20,13 +20,15 @@ const LIVE_FAIL_BACKOFF_MS = 60_000;
|
||||
interface LiveState {
|
||||
inflight: Map<string, Promise<{ error: string | null }>>;
|
||||
failedUntil: Map<string, { until: number; error: string }>;
|
||||
/** 按站点串行的实时搜索(站点访问默认 searchConcurrency = 1 时) */
|
||||
siteChain: Map<string, Promise<unknown>>;
|
||||
}
|
||||
|
||||
const liveStates = new WeakMap<SourcesRuntime, LiveState>();
|
||||
function liveState(rt: SourcesRuntime): LiveState {
|
||||
let s = liveStates.get(rt);
|
||||
if (!s) {
|
||||
s = { inflight: new Map(), failedUntil: new Map() };
|
||||
s = { inflight: new Map(), failedUntil: new Map(), siteChain: new Map() };
|
||||
liveStates.set(rt, s);
|
||||
}
|
||||
return s;
|
||||
@@ -44,14 +46,22 @@ export async function liveSearch(
|
||||
if (failed && failed.until > rt.now().getTime()) return { error: `${failed.error}(退避中)`, coalesced: false };
|
||||
const running = st.inflight.get(key);
|
||||
if (running) return { ...(await running), coalesced: true };
|
||||
const searchConcurrency = siteSearchConcurrency(site.key);
|
||||
const prev = searchConcurrency <= 1 ? st.siteChain.get(site.id) : undefined;
|
||||
const p = (async () => {
|
||||
// 同站不同关键词的实时搜索排队执行(不并发打站点)
|
||||
if (prev) await prev.catch(() => {});
|
||||
const r = await snapshotSite(rt, { siteId: site.id, kind: "live_search", query: q, maxPages: 1 });
|
||||
const error = r.status === "failed" || r.status === "skipped" ? (r.error ?? r.reason ?? "failed") : null;
|
||||
if (error) st.failedUntil.set(key, { until: rt.now().getTime() + LIVE_FAIL_BACKOFF_MS, error });
|
||||
else st.failedUntil.delete(key);
|
||||
return { error };
|
||||
})().finally(() => st.inflight.delete(key));
|
||||
})().finally(() => {
|
||||
st.inflight.delete(key);
|
||||
if (st.siteChain.get(site.id) === p) st.siteChain.delete(site.id);
|
||||
});
|
||||
st.inflight.set(key, p);
|
||||
if (searchConcurrency <= 1) st.siteChain.set(site.id, p);
|
||||
return { ...(await p), coalesced: false };
|
||||
}
|
||||
|
||||
|
||||
@@ -175,6 +175,15 @@ describe.skipIf(!up)("来源管理与站点工作台 API", () => {
|
||||
.where("key", "=", `site:${siteId}`)
|
||||
.executeTakeFirst();
|
||||
expect(bucket).toMatchObject({ capacity: 5, refill_per_sec: 200 });
|
||||
// 手动快照要 5 页也只抓访问默认的页数上限(settings.sites.defaultSnapshotMaxPages 缺省 1)
|
||||
expect((await snapshotSite(h.rt, { siteId, kind: "manual", maxPages: 5 })).pages).toEqual([1]);
|
||||
// 每拿到一个令牌计一次请求,按小时聚合;站点页读最近 24 小时
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
const stats = await json(await h.req(`/sites/${siteId}/request-stats`));
|
||||
expect(stats.status).toBe(200);
|
||||
expect(stats.body.data.last24h).toBeGreaterThanOrEqual(4);
|
||||
expect(stats.body.data.hourly).toHaveLength(24);
|
||||
expect(stats.body.data.access).toMatchObject({ snapshotMaxPages: 1, healthCheckIntervalSec: 21600 });
|
||||
});
|
||||
|
||||
it("工作台列表:快照优先,标注来源与更新时间;筛选与分页", async () => {
|
||||
@@ -219,6 +228,13 @@ describe.skipIf(!up)("来源管理与站点工作台 API", () => {
|
||||
expect(d.body.data.refreshError).toBeNull();
|
||||
expect(d.body.data.volatile.recheckBeforeDownload).toBe(true);
|
||||
expect(d.body.data.volatile.detailFetchedAt).not.toBeNull();
|
||||
// 详情缓存期内(settings.sites.defaultDetailCacheSec 缺省 24 小时)再次刷新不请求站点
|
||||
const before = mock.requests.length;
|
||||
const cached = await json(await h.req(`/sites/${siteId}/releases/${rel.id}?refresh=true`));
|
||||
expect(cached.body.data.refreshError).toBeNull();
|
||||
expect(mock.requests.length).toBe(before);
|
||||
// 缓存过期后才重新抓
|
||||
await h.db.updateTable("source_release").set({ detail_fetched_at: null }).where("id", "=", rel.id).execute();
|
||||
mock.setLoggedIn(false);
|
||||
const d2 = await json(await h.req(`/sites/${siteId}/releases/${rel.id}?refresh=true`));
|
||||
expect(d2.status).toBe(200);
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import type { SiteRelease } from "@umt/adapters";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { mergeDetailOverList } from "../../src/modules/sources-identity/service/snapshot.ts";
|
||||
|
||||
// HHClub 2026-09-29 实测:div 详情页没有做种 / 完成数 / 分类名,刷新详情后这些列被 null 冲掉。
|
||||
describe("详情刷新沿用列表字段", () => {
|
||||
const detail: SiteRelease = {
|
||||
siteId: "hhanclub",
|
||||
siteReleaseId: "91001",
|
||||
title: "示例短剧 1080p WEB-DL H265 AAC-HHWEB",
|
||||
category: { id: "412", name: null },
|
||||
sizeBytes: 194385019,
|
||||
seeders: null,
|
||||
leechers: null,
|
||||
snatched: null,
|
||||
promotion: { type: "none", downloadFactor: 1, uploadFactor: 1, expiresAt: null },
|
||||
hr: "unknown",
|
||||
uploadedAt: "2026-09-29T06:07:03.000Z",
|
||||
detailUrl: "https://hhanclub.net/details.php?id=91001",
|
||||
downloadUrl: "https://hhanclub.net/download.php?id=91001",
|
||||
tags: [],
|
||||
raw: {},
|
||||
observedAt: "2026-09-29T06:19:28.000Z",
|
||||
};
|
||||
const rel = {
|
||||
seeders: 1,
|
||||
leechers: 58,
|
||||
category_raw: "412",
|
||||
tags_raw: ["官方"],
|
||||
raw: { _site: { hr: false, snatched: 0, categoryName: "短剧" } },
|
||||
};
|
||||
|
||||
it("详情缺的做种 / 下载 / 完成 / 分类名 / H&R / 标签取列表快照值", () => {
|
||||
expect(mergeDetailOverList(detail, rel)).toMatchObject({
|
||||
seeders: 1,
|
||||
leechers: 58,
|
||||
snatched: 0,
|
||||
category: { id: "412", name: "短剧" },
|
||||
hr: false,
|
||||
tags: ["官方"],
|
||||
});
|
||||
});
|
||||
|
||||
it("详情有值时以详情为准", () => {
|
||||
const m = mergeDetailOverList({ ...detail, seeders: 9, hr: true, category: { id: "402", name: "电视剧" } }, rel);
|
||||
expect(m).toMatchObject({ seeders: 9, hr: true, category: { id: "402", name: "电视剧" } });
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user