fix(sources): 站点快照加总截止时间;定时任务兜住收尾写库错误
OurBits 一次定时快照卡在过验证 / 代抓的等待里 10 小时,任务一直停在运行中。 快照整体 5 分钟截止,到点按失败收尾;调度器执行的拒绝不再变成未处理错误。 Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Cxwgnn2pbzQ7rXv3rq7fei
This commit is contained in:
1 parent
47ca8886b1
commit
5c74666148
4 files changed
+102
-6
No files matched your search
@@ -13,6 +13,10 @@
|
|||||||
- 作品详情头图的「订阅」改回普通主按钮(静态高光在剧照上看得清),去掉实心按钮例外 `pill-solid`;发现页海报上的评分与状态角标改成烟色小玻璃。
|
- 作品详情头图的「订阅」改回普通主按钮(静态高光在剧照上看得清),去掉实心按钮例外 `pill-solid`;发现页海报上的评分与状态角标改成烟色小玻璃。
|
||||||
- 触屏判断改为通用方案(`<html data-input>`、`touch:` 变体、`useTouchInput()`):车机等把触屏报告成鼠标的设备也按触屏处理,悬停效果只给真鼠标。
|
- 触屏判断改为通用方案(`<html data-input>`、`touch:` 变体、`useTouchInput()`):车机等把触屏报告成鼠标的设备也按触屏处理,悬停效果只给真鼠标。
|
||||||
|
|
||||||
|
### 修复
|
||||||
|
- 站点快照加 5 分钟总截止时间:请求在过人机验证、浏览器代抓或限流排队时卡住,快照到点记为失败,后台任务不会一直停在运行中(之前 OurBits 的一次定时快照卡了 10 小时)。
|
||||||
|
- 定时任务在认领或收尾写库时出错(如关停时连接已断)不再成为未处理的错误,改为记一条日志。
|
||||||
|
|
||||||
## [0.3.37] - 2026-10-07
|
## [0.3.37] - 2026-10-07
|
||||||
|
|
||||||
### 调整
|
### 调整
|
||||||
|
|||||||
@@ -169,10 +169,15 @@ export class Scheduler {
|
|||||||
release = await lanes.enterMaintenance(job.run_requested_at !== null);
|
release = await lanes.enterMaintenance(job.run_requested_at !== null);
|
||||||
if (!release) continue;
|
if (!release) continue;
|
||||||
}
|
}
|
||||||
const p = this.execute(job, fn).finally(() => {
|
// execute 自己记录任务失败;这里兜住认领 / 收尾写库本身的错(比如关停时连接池已销毁),不留未处理的拒绝
|
||||||
release?.();
|
const p = this.execute(job, fn)
|
||||||
this.inflight.delete(job.id);
|
.catch((err) => {
|
||||||
});
|
if (!this.stopped) this.log.warn({ jobId: job.id, err: { message: errText(err) } }, "定时任务执行出错");
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
release?.();
|
||||||
|
this.inflight.delete(job.id);
|
||||||
|
});
|
||||||
this.inflight.set(job.id, p);
|
this.inflight.set(job.id, p);
|
||||||
started.push(job.id);
|
started.push(job.id);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -165,7 +165,27 @@ async function syncSnapshotFailureDecision(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function snapshotSite(rt: SourcesRuntime, input: SnapshotInput): Promise<SnapshotResult> {
|
/**
|
||||||
|
* 一次快照的总截止时间。站点请求里有几处等待不受单次请求超时约束(过人机验证、浏览器代抓、
|
||||||
|
* 共享限流排队),某次卡住会让任务一直停在运行中(2026-10-07 OurBits 卡了 10 小时);到点按失败收尾。
|
||||||
|
*/
|
||||||
|
export const SNAPSHOT_DEADLINE_MS = 5 * 60_000;
|
||||||
|
|
||||||
|
/** 到截止时间就不再等 p(p 自己收到 signal 会尽量提前结束;不配合的等待也不会卡住调用方) */
|
||||||
|
function untilDeadline<T>(p: Promise<T>, signal: AbortSignal, siteId: string, ms: number): Promise<T> {
|
||||||
|
return new Promise<T>((resolve, reject) => {
|
||||||
|
const onAbort = () => reject(new SiteError(siteId, "network", `快照 ${Math.ceil(ms / 1000)} 秒内没完成,已放弃`));
|
||||||
|
if (signal.aborted) return onAbort();
|
||||||
|
signal.addEventListener("abort", onAbort, { once: true });
|
||||||
|
p.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function snapshotSite(
|
||||||
|
rt: SourcesRuntime,
|
||||||
|
input: SnapshotInput,
|
||||||
|
opts: { deadlineMs?: number } = {},
|
||||||
|
): Promise<SnapshotResult> {
|
||||||
const { db } = rt.ctx;
|
const { db } = rt.ctx;
|
||||||
const none = { snapshotId: null, pages: [], itemCount: 0, newCount: 0, releaseIds: [] };
|
const none = { snapshotId: null, pages: [], itemCount: 0, newCount: 0, releaseIds: [] };
|
||||||
const site = await db.selectFrom("site").selectAll().where("id", "=", input.siteId).executeTakeFirst();
|
const site = await db.selectFrom("site").selectAll().where("id", "=", input.siteId).executeTakeFirst();
|
||||||
@@ -212,9 +232,14 @@ export async function snapshotSite(rt: SourcesRuntime, input: SnapshotInput): Pr
|
|||||||
const pages: number[] = [];
|
const pages: number[] = [];
|
||||||
const releases: SiteRelease[] = [];
|
const releases: SiteRelease[] = [];
|
||||||
let error: SiteError | Error | null = null;
|
let error: SiteError | Error | null = null;
|
||||||
|
const deadlineMs = opts.deadlineMs ?? SNAPSHOT_DEADLINE_MS;
|
||||||
|
const signal = AbortSignal.timeout(deadlineMs);
|
||||||
for (let p = 1; p <= maxPages; p++) {
|
for (let p = 1; p <= maxPages; p++) {
|
||||||
try {
|
try {
|
||||||
const res: SitePage = input.query ? await adapter.search(input.query, { page: p }) : await adapter.list(p);
|
const call = input.query
|
||||||
|
? adapter.search(input.query, { page: p }, { signal })
|
||||||
|
: adapter.list(p, undefined, { signal });
|
||||||
|
const res: SitePage = await untilDeadline(call, signal, site.id, deadlineMs);
|
||||||
pages.push(p);
|
pages.push(p);
|
||||||
releases.push(...res.items);
|
releases.push(...res.items);
|
||||||
if (res.items.length === 0 || (res.totalPages !== null && p >= res.totalPages)) break;
|
if (res.items.length === 0 || (res.totalPages !== null && p >= res.totalPages)) break;
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||||
|
import { snapshotSite } from "../../src/modules/sources-identity/service/snapshot.ts";
|
||||||
|
import { createHarness, type Harness, pgAvailable } from "./harness.ts";
|
||||||
|
|
||||||
|
// 快照总截止时间:站点请求卡住且不理会取消信号(过人机验证、浏览器代抓这类等待)时,
|
||||||
|
// 快照到点按失败收尾、写完 site_snapshot,任务不会一直停在运行中(2026-10-07 OurBits 卡了 10 小时)。
|
||||||
|
|
||||||
|
const up = await pgAvailable();
|
||||||
|
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: 响应体断言
|
||||||
|
const json = async (r: Response): Promise<any> => {
|
||||||
|
const body = await r.json();
|
||||||
|
if (!r.ok) throw new Error(`${r.status} ${JSON.stringify(body)}`);
|
||||||
|
return body.data;
|
||||||
|
};
|
||||||
|
|
||||||
|
describe.skipIf(!up)("快照截止时间", () => {
|
||||||
|
let h: Harness;
|
||||||
|
let siteId: string;
|
||||||
|
// 永不返回,也不理会 signal
|
||||||
|
const hang = (() => new Promise<Response>(() => {})) as typeof fetch;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
h = await createHarness("snapdeadline", { fetch: hang });
|
||||||
|
const s = await json(
|
||||||
|
await h.req("/sites", {
|
||||||
|
body: {
|
||||||
|
key: "audiences",
|
||||||
|
name: "观众",
|
||||||
|
adapter: "nexusphp",
|
||||||
|
baseUrl: "https://aud.test/",
|
||||||
|
rateLimit: { minIntervalMs: 0, concurrency: 10 },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
siteId = s.id;
|
||||||
|
await json(
|
||||||
|
await h.req(`/sites/${siteId}/credentials`, {
|
||||||
|
method: "PUT",
|
||||||
|
body: { kind: "cookie", cookie: "c_secure_uid=1; c_secure_pass=abc" },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
afterAll(async () => {
|
||||||
|
await h?.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("请求卡住 → 到点失败,快照记录收尾", async () => {
|
||||||
|
const t0 = Date.now();
|
||||||
|
const r = await snapshotSite(h.rt, { siteId, kind: "periodic" }, { deadlineMs: 300 });
|
||||||
|
expect(Date.now() - t0).toBeLessThan(5_000);
|
||||||
|
expect(r.status).toBe("failed");
|
||||||
|
expect(r.error).toContain("没完成");
|
||||||
|
const snap = await h.db
|
||||||
|
.selectFrom("site_snapshot")
|
||||||
|
.selectAll()
|
||||||
|
.where("id", "=", r.snapshotId!)
|
||||||
|
.executeTakeFirstOrThrow();
|
||||||
|
expect(snap.status).toBe("failed");
|
||||||
|
expect(snap.finished_at).not.toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in new issue
Block a user