From b4c9b1ead267bfa15c77dc423f99ddb3f7b9f4f8 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 28 Sep 2026 23:31:21 +0800 Subject: [PATCH] fix(system): retired heartbeat rows no longer alarm on status page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - GET /system/health splits processes into current instances (hosts with a beat in the last 10 min, newest per host+kind, not stopped) and retiredProcesses[]; 45s stale and level only consider current ones - heartbeat writer marks stopped_at on graceful exit (migration 0043), upsert clears it; pure api role now stops its writer on SIGTERM - scheduled job system.heartbeat_cleanup (maintain, 1h) deletes rows without a beat for 24h and samples older than 7 days - status page: per-role alarms ("io worker 45 秒没有心跳"), collapsible retired instances; glossary/runbook updated - bump all members to 0.2.2, CHANGELOG [0.2.2] Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 ++ apps/server/deno.json | 2 +- apps/server/src/config.ts | 19 ++ apps/server/src/main.ts | 5 + apps/server/src/modules/browser/routes.ts | 7 + apps/server/src/modules/browser/service.ts | 141 ++++++++++++++- apps/server/src/modules/system/health.ts | 112 ++++++++---- apps/server/src/modules/system/heartbeat.ts | 38 ++-- apps/server/src/modules/system/index.ts | 30 +++- apps/server/test/system/system.test.ts | 162 +++++++++++++++++- apps/web/app/routes/console/status.tsx | 102 ++++++++++- apps/web/package.json | 2 +- deno.json | 2 +- docs/design/glossary.md | 2 + docs/runbook/ops.md | 11 +- docs/runbook/schedules.md | 1 + packages/adapters/deno.json | 2 +- .../adapters/src/browser/camofox/client.ts | 41 ++++- packages/agent/deno.json | 2 +- packages/contracts/deno.json | 2 +- packages/contracts/src/api.ts | 25 +++ packages/contracts/src/browser.ts | 30 +++- packages/contracts/src/ops.ts | 7 +- packages/db/deno.json | 2 +- .../0043_process_heartbeat_stopped.ts | 16 ++ packages/db/src/index.ts | 2 +- packages/db/src/types.ts | 1 + packages/domain/deno.json | 2 +- packages/mapping/deno.json | 2 +- packages/tasks/deno.json | 2 +- packages/testkit/deno.json | 2 +- packages/testkit/src/browser/mock-camofox.ts | 54 ++++-- 32 files changed, 747 insertions(+), 91 deletions(-) create mode 100644 packages/db/migrations/0043_process_heartbeat_stopped.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c59e2d1..4abd97d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,16 @@ - 没有调查整个 qB 实例(含未纳管任务)的能力。 - AT18(恶意标题)、AT29(部分入库不清理)、AT42(未纳管占用)、AT73(多下载器同 hash)与流程版本升级没有测试。 +## [0.2.2] - 2026-09-28 + +### 修复 + +- 编译产物启动即 `ERR_MODULE_NOT_FOUND`:web 的 `shadcn` 开发依赖带进第二份 `@modelcontextprotocol/sdk@1.30.1`(zod@3),`deno compile --exclude-unused-npm` 嵌入的目录与裁剪后快照解析的路径不一致。去掉 `shadcn` 依赖,只把它的 `tailwind.css` 拷成 `apps/web/app/shadcn-tailwind.css`,MCP SDK 只剩一份。 +- 编译产物自检:`scripts/check-compiled-binary.sh` 用空环境跑 `--role preflight`,必须加载完整模块图并在配置校验处以退出码 2 结束;已接入 `deno task ci`(`check:binary`)与 Dockerfile 构建阶段。 +- 运行状态页把容器重建前留下的旧心跳行当成"N 个进程超过 45 秒没有心跳"报警。`GET /system/health` 现在把进程分成当前实例(最近 10 分钟有心跳的 host 上、同 host 同 kind 最新启动且未退出的进程)与已下线实例,45 秒 stale 判定与告警只看当前实例;已下线实例放在新字段 `retiredProcesses[]`,不计入 `level`,页面折叠显示。横幅按角色报警,如「io worker 45 秒没有心跳」。 +- 进程正常退出(SIGTERM / SIGINT)时把自己的心跳行标 `stopped_at`(迁移 0043),退出后立即不再当活进程;重启后 upsert 清回 null。纯 api 进程退出时也会停心跳写入器。 +- 心跳清理改为定时任务 `system.heartbeat_cleanup`(维护组,1 小时):删 24 小时没有心跳的进程行和 7 天前的采样;写入器不再自己裁剪。 + ## [0.2.0] - 2026-09-28 Agent 上游全覆盖与订阅登录、控制台信息架构重排、定时任务看板、全站文案第二轮审校;W9 与前端波补上站点中心、订阅重做、统一规则、命名模板、全局搜索、文件管理 v2 与统一任务页。基线见 `docs/plan/08-v0.2-closeout.md`(W8-A 与"W10 集成记录")、`docs/plan/09-visual-qa.md`、`docs/design/{console-ia,site-center,subscriptions,naming-template,file-manager-v2,pi-providers,agent-ui-reference}.md`、`docs/runbook/{schedules,agent,sites,rules}.md`。 diff --git a/apps/server/deno.json b/apps/server/deno.json index 1943301b..636efcd7 100644 --- a/apps/server/deno.json +++ b/apps/server/deno.json @@ -1,6 +1,6 @@ { "name": "@umt/server", - "version": "0.2.0", + "version": "0.2.2", "exports": { ".": "./src/main.ts" }, "tasks": { "dev": "deno run -A --watch --env-file=../../.env src/main.ts", diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index ef1133e3..0c1a86c3 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -1,6 +1,9 @@ import { join, resolve } from "node:path"; import { z } from "zod"; +/** compose 里 `${X:-}` 展开成空串:按未设置处理 */ +const emptyToUndef = (v: unknown) => (v === "" ? undefined : v); + const LogLevel = z.enum(["fatal", "error", "warn", "info", "debug", "trace", "silent"]); export const ConfigSchema = z.object({ @@ -66,6 +69,22 @@ export const ConfigSchema = z.object({ ORGANIZE_ON_EXISTING: z.enum(["conflict", "skip", "replace_with_verify"]).default("conflict"), /** 随行字幕中此语言加 .default 标记(对照 MP DEFAULT_SUB);不设 = 不加 */ ORGANIZE_DEFAULT_SUBTITLE_LANG: z.enum(["zh-cn", "zh-tw", "en", "ja", "ko"]).optional(), + // ── 浏览器服务(camofox-browser,docs/runbook/browser.md)── + /** + * 部署自带的浏览器服务地址(compose 缺省 http://camofox:9377;接自有实例时改成它的地址)。 + * 设置后启动时登记/更新名为 camofox-env 的浏览器服务;不设则只用设置页登记的。 + */ + BROWSER_PROVIDER_URL: z.preprocess(emptyToUndef, z.string().url().optional()), + /** 该服务的访问凭据(camofox CAMOFOX_ACCESS_KEY);只在进程内使用,不入库 */ + BROWSER_ACCESS_KEY: z.preprocess(emptyToUndef, z.string().min(1).optional()), + /** false = 停用 camofox-env(容器可照常运行,界面标"浏览器不可用") */ + BROWSER_ENABLED: z.preprocess( + emptyToUndef, + z + .enum(["true", "false", "1", "0"]) + .default("true") + .transform((v) => v === "true" || v === "1"), + ), }); export type Config = z.infer; diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index b9f2a083..4cc6008d 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -237,6 +237,11 @@ if (role === "api" || role === "all") { ac.abort(); await server.finished; }); + if (role === "api") { + // 纯 api 进程不走 worker 的模块 stop:这里停 system 模块的心跳写入器,把本进程的心跳行标 stopped_at + const sys = SERVER_MODULES.find((m) => m.name === "system"); + if (sys?.stop) stops.push(async () => await sys.stop?.()); + } } if (role === "worker" || role === "worker:agent" || role === "all") { const worker = await startWorker({ diff --git a/apps/server/src/modules/browser/routes.ts b/apps/server/src/modules/browser/routes.ts index fc95d62d..5575e1b6 100644 --- a/apps/server/src/modules/browser/routes.ts +++ b/apps/server/src/modules/browser/routes.ts @@ -85,6 +85,13 @@ export function registerBrowserRoutes(app: ModuleApp, svc: () => BrowserService) const B = Api.browser; route(app, B.providers, async ({ query }) => ({ items: await svc().listProviders(query.probe === "true") })); + route(app, B.createProvider, async ({ body }) => svc().createProvider(body)); + route(app, B.updateProvider, async ({ params, body }) => svc().updateProvider(params.id, body)); + route(app, B.probeProvider, async ({ params }) => { + const [v] = await svc().listProviders(true, params.id); + if (!v) throw new BrowserServiceError("not_found", "浏览器服务不存在"); + return v; + }); route(app, B.sessions, async ({ query }) => ({ items: await svc().list({ diff --git a/apps/server/src/modules/browser/service.ts b/apps/server/src/modules/browser/service.ts index 24b79f82..3059df14 100644 --- a/apps/server/src/modules/browser/service.ts +++ b/apps/server/src/modules/browser/service.ts @@ -114,8 +114,13 @@ export interface RecoveryState { downloads?: { sha256: string; artifactId: string; sourceReleaseId: string | null; planId: string | null }[]; } +/** 环境变量(BROWSER_PROVIDER_URL)登记的浏览器服务 */ +export const ENV_PROVIDER_NAME = "camofox-env"; +const ENV_MANAGED = "env"; + export class BrowserService { private readonly providers = new Map(); + private envSync: Promise | undefined; readonly now: () => Date; constructor( @@ -163,7 +168,11 @@ export class BrowserService { const { db, secrets } = this.ctx; return { resolve: async (req) => { - if (req.kind === "provider_access_key") return row.secret_ref ? secrets.resolve(row.secret_ref) : null; + if (req.kind === "provider_access_key") { + if (row.secret_ref) return secrets.resolve(row.secret_ref); + // 环境变量登记的服务:凭据只在进程内(BROWSER_ACCESS_KEY),不入库 + return obj(row.config).managedBy === ENV_MANAGED ? (this.ctx.config?.BROWSER_ACCESS_KEY ?? null) : null; + } let q = db .selectFrom("site_account") .selectAll() @@ -210,7 +219,126 @@ export class BrowserService { return provider; } + /** + * BROWSER_PROVIDER_URL / BROWSER_ACCESS_KEY / BROWSER_ENABLED → 名为 camofox-env 的浏览器服务(每个进程同步一次)。 + * 地址与启用由环境变量决定;config 里的其它键(verified、超时等)保留。没设地址时已有的 camofox-env 置为停用。 + */ + syncEnvProvider(): Promise { + this.envSync ??= this.doSyncEnvProvider().catch((e) => { + this.envSync = undefined; + this.ctx.logger.warn({ component: "browser", err: String(e) }, "登记环境变量浏览器服务失败"); + }); + return this.envSync; + } + + private async doSyncEnvProvider(): Promise { + const c = this.ctx.config; + const url = c?.BROWSER_PROVIDER_URL; + const enabled = !!url && c.BROWSER_ENABLED !== false; + const db = this.ctx.db; + const row = await db + .selectFrom("browser_provider") + .selectAll() + .where("name", "=", ENV_PROVIDER_NAME) + .executeTakeFirst(); + if (!url) { + if (row?.enabled) + await db.updateTable("browser_provider").set({ enabled: false }).where("id", "=", row.id).execute(); + return; + } + const config = JSON.stringify({ ...obj(row?.config), managedBy: ENV_MANAGED }); + if (row) + await db + .updateTable("browser_provider") + .set({ base_url: url, enabled, kind: "camofox", secret_ref: null, config }) + .where("id", "=", row.id) + .execute(); + else + await db + .insertInto("browser_provider") + .values({ name: ENV_PROVIDER_NAME, kind: "camofox", base_url: url, enabled, config }) + .onConflict((oc) => oc.column("name").doNothing()) + .execute(); + } + + /** 设置页登记浏览器服务:访问凭据加密存 secret;名称 camofox-env 保留给环境变量 */ + async createProvider(body: { + name: string; + baseUrl: string; + accessKey?: string | undefined; + enabled?: boolean | undefined; + }): Promise { + if (body.name === ENV_PROVIDER_NAME) + throw new BrowserServiceError("bad_request", `名称 ${ENV_PROVIDER_NAME} 保留给环境变量`); + const exists = await this.ctx.db + .selectFrom("browser_provider") + .select("id") + .where("name", "=", body.name) + .executeTakeFirst(); + if (exists) throw new BrowserServiceError("conflict", `浏览器服务「${body.name}」已存在`); + const secretRef = body.accessKey + ? await this.ctx.secrets.create({ purpose: "other", plaintext: body.accessKey, label: `browser:${body.name}` }) + : null; + const row = await this.ctx.db + .insertInto("browser_provider") + .values({ + name: body.name, + kind: "camofox", + base_url: body.baseUrl, + secret_ref: secretRef, + enabled: body.enabled ?? true, + }) + .returning("id") + .executeTakeFirstOrThrow(); + return this.providerView(row.id); + } + + /** 改地址 / 凭据 / 启用;accessKey = null 清除凭据。camofox-env 的地址与凭据归环境变量管,只能改启用 */ + async updateProvider( + id: string, + body: { baseUrl?: string | undefined; accessKey?: string | null | undefined; enabled?: boolean | undefined }, + ): Promise { + const row = await this.ctx.db.selectFrom("browser_provider").selectAll().where("id", "=", id).executeTakeFirst(); + if (!row) throw new BrowserServiceError("not_found", "浏览器服务不存在"); + if (obj(row.config).managedBy === ENV_MANAGED && (body.baseUrl !== undefined || body.accessKey !== undefined)) + throw new BrowserServiceError( + "conflict", + "该浏览器服务由环境变量 BROWSER_PROVIDER_URL / BROWSER_ACCESS_KEY 管理，改部署配置后重启", + ); + let secretRef = row.secret_ref; + if (body.accessKey === null) secretRef = null; + else if (body.accessKey) { + if (secretRef) await this.ctx.secrets.replace(secretRef, body.accessKey); + else + secretRef = await this.ctx.secrets.create({ + purpose: "other", + plaintext: body.accessKey, + label: `browser:${row.name}`, + }); + } + await this.ctx.db + .updateTable("browser_provider") + .set({ + ...(body.baseUrl !== undefined ? { base_url: body.baseUrl } : {}), + ...(body.enabled !== undefined ? { enabled: body.enabled } : {}), + secret_ref: secretRef, + }) + .where("id", "=", id) + .execute(); + if (row.secret_ref && secretRef === null) await this.ctx.secrets.remove(row.secret_ref).catch(() => {}); + // 凭据换了但 secret_ref 不变:签名不变,显式丢弃缓存的适配器 + this.providers.delete(id); + return this.providerView(id); + } + + private async providerView(id: string): Promise { + const v = (await this.listProviders(false, id))[0]; + if (!v) throw new BrowserServiceError("not_found", "浏览器服务不存在"); + return v; + } + async providerRow(id?: string): Promise { + await this.syncEnvProvider(); let q = this.ctx.db .selectFrom("browser_provider") .selectAll() @@ -223,12 +351,14 @@ export class BrowserService { return row; } - async listProviders(probe: boolean): Promise { - const rows = await this.ctx.db.selectFrom("browser_provider").selectAll().orderBy("created_at").execute(); + async listProviders(probe: boolean, onlyId?: string): Promise { + await this.syncEnvProvider(); + let rows = await this.ctx.db.selectFrom("browser_provider").selectAll().orderBy("created_at").execute(); + if (onlyId) rows = rows.filter((r) => r.id === onlyId); const out: BrowserProviderView[] = []; for (let row of rows) { let health: BrowserHealth | null = null; - if (probe && row.enabled && row.kind !== "none") { + if (probe && (row.enabled || onlyId) && row.kind !== "none") { const p = this.providerFor(row); health = await p.health(); let caps: BrowserCapabilities | null = null; @@ -255,6 +385,9 @@ export class BrowserService { kind: row.kind as "camofox" | "none", baseUrl: row.base_url, secretRef: row.secret_ref, + managedBy: obj(row.config).managedBy === ENV_MANAGED ? "env" : null, + hasAccessKey: + !!row.secret_ref || (obj(row.config).managedBy === ENV_MANAGED && !!this.ctx.config?.BROWSER_ACCESS_KEY), enabled: row.enabled, capabilities: Object.keys(caps).length ? (caps as unknown as BrowserCapabilities) : null, verifiedAt: iso(row.verified_at), diff --git a/apps/server/src/modules/system/health.ts b/apps/server/src/modules/system/health.ts index e6f116d4..800834fb 100644 --- a/apps/server/src/modules/system/health.ts +++ b/apps/server/src/modules/system/health.ts @@ -8,8 +8,8 @@ import type { SystemMount, SystemProcess, } from "@umt/contracts"; -import { type DB, type Kysely, sql } from "@umt/db"; -import { HEARTBEAT_STALE_MS } from "./heartbeat.ts"; +import { type DB, type Kysely, type Selectable, sql } from "@umt/db"; +import { CURRENT_HOST_WINDOW_MS, HEARTBEAT_ROW_RETENTION_MS, HEARTBEAT_STALE_MS } from "./heartbeat.ts"; // GET /system/health 各段的取数(W9-I)。每段独立 try/catch:一段失败只把该段标 unknown,不让整页 500。 // 慢 IO(statfs / 哨兵检查)都带超时:挂载卡死时只标 timeout,不拖住请求。 @@ -53,44 +53,84 @@ const num = (v: unknown) => (v == null ? null : Number(v)); // ── 进程 ── -export async function loadProcesses(db: Kysely, now: Date): Promise { +type HeartbeatRow = Selectable; + +function toProcess(r: HeartbeatRow, now: Date): SystemProcess { + const beat = toDate(r.last_beat_at); + const age = Math.max(0, now.getTime() - beat.getTime()); + const extra = (r.extra ?? {}) as Record; + const conc = (r.concurrency ?? {}) as { io?: number; agent?: number }; + const sse = extra.sseConnections; + return { + instanceId: r.instance_id, + processKind: r.process_kind, + pid: r.pid, + host: r.host, + version: r.version, + startedAt: toDate(r.started_at).toISOString(), + lastBeatAt: beat.toISOString(), + beatAgeSec: Math.round(age / 100) / 10, + stale: age > HEARTBEAT_STALE_MS, + stoppedAt: r.stopped_at ? toDate(r.stopped_at).toISOString() : null, + eventLoopLagMs: r.event_loop_lag_ms, + eventLoopLagMaxMs: r.event_loop_lag_max_ms, + concurrency: { + ...(typeof conc.io === "number" ? { io: conc.io } : {}), + ...(typeof conc.agent === "number" ? { agent: conc.agent } : {}), + }, + runningRuns: r.running_runs, + sseConnections: typeof sse === "number" ? sse : null, + extra, + }; +} + +/** + * 把心跳行分成当前实例与已下线实例: + * - 当前 host = 最近 10 分钟内有心跳且没标 stopped_at 的行所在 host(容器重建后 host 会变); + * - 当前实例 = 当前 host 上、没标 stopped_at、且是同 host 同 kind 里最新启动的那一行; + * - 其余(已正常退出、host 已不在、被同 host 新进程替换)都算已下线实例,不参与 stale 判定与告警。 + */ +export function splitProcesses( + ps: SystemProcess[], + now: Date, +): { processes: SystemProcess[]; retiredProcesses: SystemProcess[] } { + const alive = ps.filter((p) => !p.stoppedAt); + const hosts = new Set( + alive.filter((p) => now.getTime() - Date.parse(p.lastBeatAt) <= CURRENT_HOST_WINDOW_MS).map((p) => p.host), + ); + const newest = new Map(); + for (const p of alive) { + if (!hosts.has(p.host)) continue; + const k = `${p.host}\u0000${p.processKind}`; + const cur = newest.get(k); + if (!cur || Date.parse(p.startedAt) > Date.parse(cur.startedAt)) newest.set(k, p); + } + const current = new Set([...newest.values()].map((p) => p.instanceId)); + const processes = ps.filter((p) => current.has(p.instanceId)); + const retiredProcesses = ps + .filter((p) => !current.has(p.instanceId)) + .sort((a, b) => Date.parse(b.lastBeatAt) - Date.parse(a.lastBeatAt)); + return { processes, retiredProcesses }; +} + +export async function loadProcesses( + db: Kysely, + now: Date, +): Promise<{ processes: SystemProcess[]; retiredProcesses: SystemProcess[] }> { const rows = await db .selectFrom("process_heartbeat") .selectAll() - .where("last_beat_at", ">", new Date(now.getTime() - 24 * 3600_000)) + .where("last_beat_at", ">", new Date(now.getTime() - HEARTBEAT_ROW_RETENTION_MS)) .orderBy("process_kind") .orderBy("instance_id") .execute(); - return rows.map((r) => { - const beat = toDate(r.last_beat_at); - const age = Math.max(0, now.getTime() - beat.getTime()); - const extra = (r.extra ?? {}) as Record; - const conc = (r.concurrency ?? {}) as { io?: number; agent?: number }; - const sse = extra.sseConnections; - return { - instanceId: r.instance_id, - processKind: r.process_kind, - pid: r.pid, - host: r.host, - version: r.version, - startedAt: toDate(r.started_at).toISOString(), - lastBeatAt: beat.toISOString(), - beatAgeSec: Math.round(age / 100) / 10, - stale: age > HEARTBEAT_STALE_MS, - eventLoopLagMs: r.event_loop_lag_ms, - eventLoopLagMaxMs: r.event_loop_lag_max_ms, - concurrency: { - ...(typeof conc.io === "number" ? { io: conc.io } : {}), - ...(typeof conc.agent === "number" ? { agent: conc.agent } : {}), - }, - runningRuns: r.running_runs, - sseConnections: typeof sse === "number" ? sse : null, - extra, - }; - }); + return splitProcesses( + rows.map((r) => toProcess(r, now)), + now, + ); } -/** 进程段的档位:有 stale 或延迟 ≥ 5s 为 critical;延迟 ≥ 1s 为 warn;没有任何 worker 心跳为 warn */ +/** 进程段的档位(只看当前实例):有 stale 或延迟 ≥ 1s 为 warn;延迟 ≥ 5s 为 critical;没有任何 worker 心跳为 warn */ export function processLevel(ps: SystemProcess[]): HealthLevel { const live = ps.filter((p) => !p.stale); const levels: HealthLevel[] = []; @@ -344,8 +384,11 @@ export async function loadSchedules(db: Kysely, now: Date): Promise { const now = d.now?.() ?? new Date(); - const [processes, db, paths, engine, schedules] = await Promise.all([ - loadProcesses(d.db, now).catch(() => [] as SystemProcess[]), + const [{ processes, retiredProcesses }, db, paths, engine, schedules] = await Promise.all([ + loadProcesses(d.db, now).catch(() => ({ + processes: [] as SystemProcess[], + retiredProcesses: [] as SystemProcess[], + })), loadDb(d.db), loadDisksAndMounts(d).catch(() => ({ disks: [] as SystemDisk[], mounts: [] as SystemMount[] })), loadEngine(d.db, d.engineSchema, now).catch((): SystemHealth["engine"] => ({ @@ -388,6 +431,7 @@ export async function buildHealth(d: HealthDeps): Promise { uptimeSec: Math.max(0, Math.round((now.getTime() - d.self.startedAt.getTime()) / 1000)), }, processes, + retiredProcesses, sse: { connections: sse }, db, disks: paths.disks, diff --git a/apps/server/src/modules/system/heartbeat.ts b/apps/server/src/modules/system/heartbeat.ts index 207dd95c..db67f0ab 100644 --- a/apps/server/src/modules/system/heartbeat.ts +++ b/apps/server/src/modules/system/heartbeat.ts @@ -6,12 +6,17 @@ import serverDeno from "../../../deno.json" with { type: "json" }; // 进程心跳写入(W9-I):每个常驻进程每 15s upsert process_heartbeat 一行 + 追加一条采样。 // 写失败只记 debug(库暂时不可达不影响业务;读取方按 last_beat_at 判 stale)。 +// 正常退出(stop)时把本行标 stopped_at,读取方立即把它归入已下线实例;裁剪由定时任务 system.heartbeat_cleanup 负责。 export const HEARTBEAT_WRITE_MS = 15_000; /** 超过这个时长没有心跳即 stale(3 个写周期) */ export const HEARTBEAT_STALE_MS = 45_000; /** 采样保留天数 */ export const SAMPLE_RETENTION_DAYS = 7; +/** 进程行保留:超过这个时长没心跳的行删除 */ +export const HEARTBEAT_ROW_RETENTION_MS = 24 * 3600_000; +/** 当前实例判定窗口:最近这段时间内有心跳的 host 视为当前在跑的容器 */ +export const CURRENT_HOST_WINDOW_MS = 10 * 60_000; export const SERVER_VERSION: string = serverDeno.version; @@ -109,7 +114,7 @@ export async function writeHeartbeat( host = excluded.host, version = excluded.version, started_at = excluded.started_at, last_beat_at = excluded.last_beat_at, event_loop_lag_ms = excluded.event_loop_lag_ms, event_loop_lag_max_ms = excluded.event_loop_lag_max_ms, concurrency = excluded.concurrency, - running_runs = excluded.running_runs, extra = excluded.extra`.execute(db); + running_runs = excluded.running_runs, extra = excluded.extra, stopped_at = null`.execute(db); await sql` insert into process_heartbeat_sample (instance_id, process_kind, at, event_loop_lag_ms, event_loop_lag_max_ms, running_runs, sse_connections) @@ -117,10 +122,22 @@ export async function writeHeartbeat( ${s.sseConnections ?? null})`.execute(db); } -/** 裁剪过期采样与 7 天没心跳的进程行 */ -export async function pruneHeartbeats(db: Kysely, days = SAMPLE_RETENTION_DAYS): Promise { - await sql`delete from process_heartbeat_sample where at < now() - make_interval(days => ${days})`.execute(db); - await sql`delete from process_heartbeat where last_beat_at < now() - make_interval(days => ${days})`.execute(db); +/** 正常退出:把本实例的行标 stopped_at(之后读取方不再当活进程) */ +export async function markHeartbeatStopped(db: Kysely, instanceId: string, at = new Date()): Promise { + await sql`update process_heartbeat set stopped_at = ${at} where instance_id = ${instanceId}`.execute(db); +} + +/** 裁剪:超过 24 小时没心跳的进程行、超过 7 天的采样;返回删除行数 */ +export async function pruneHeartbeats( + db: Kysely, + o: { now?: Date; rowRetentionMs?: number; sampleDays?: number } = {}, +): Promise<{ rows: number; samples: number }> { + const now = o.now ?? new Date(); + const rowCutoff = new Date(now.getTime() - (o.rowRetentionMs ?? HEARTBEAT_ROW_RETENTION_MS)); + const sampleCutoff = new Date(now.getTime() - (o.sampleDays ?? SAMPLE_RETENTION_DAYS) * 24 * 3600_000); + const rows = await db.deleteFrom("process_heartbeat").where("last_beat_at", "<", rowCutoff).executeTakeFirst(); + const samples = await db.deleteFrom("process_heartbeat_sample").where("at", "<", sampleCutoff).executeTakeFirst(); + return { rows: Number(rows.numDeletedRows), samples: Number(samples.numDeletedRows) }; } export interface HeartbeatWriter { @@ -130,7 +147,7 @@ export interface HeartbeatWriter { stop(): Promise; } -/** 起一个周期写入器:立即写一次,之后每 intervalMs 写一次;每约 1 小时裁剪一次 */ +/** 起一个周期写入器:立即写一次,之后每 intervalMs 写一次;stop 时把本行标 stopped_at */ export function startHeartbeatWriter(o: HeartbeatWriterOptions): HeartbeatWriter { const host = o.host ?? safeHostname(); const pid = o.pid ?? Deno.pid; @@ -140,8 +157,6 @@ export function startHeartbeatWriter(o: HeartbeatWriterOptions): HeartbeatWriter const intervalMs = o.intervalMs ?? HEARTBEAT_WRITE_MS; const lag = o.lag ?? createLagSampler(); const log = o.logger.child({ component: "heartbeat", kind: o.kind }); - const pruneEvery = Math.max(1, Math.round(3_600_000 / intervalMs)); - let n = 0; let inflight: Promise | null = null; let stopped = false; @@ -165,8 +180,6 @@ export function startHeartbeatWriter(o: HeartbeatWriterOptions): HeartbeatWriter lagMaxMs: l.windowMaxMs, sample: { ...sample, extra: { ...memory(), lifetimeMaxLagMs: l.lifetimeMaxMs, ...(sample.extra ?? {}) } }, }); - if (n % pruneEvery === 0) await pruneHeartbeats(o.db); - n += 1; } catch (err) { log.debug({ err }, "心跳写入失败"); } @@ -194,6 +207,11 @@ export function startHeartbeatWriter(o: HeartbeatWriterOptions): HeartbeatWriter clearInterval(timer); lag.stop(); await inflight?.catch(() => {}); + try { + await markHeartbeatStopped(o.db, instanceId); + } catch (err) { + log.debug({ err }, "心跳退出标记失败"); + } }, }; } diff --git a/apps/server/src/modules/system/index.ts b/apps/server/src/modules/system/index.ts index d6851f1c..3e6a6364 100644 --- a/apps/server/src/modules/system/index.ts +++ b/apps/server/src/modules/system/index.ts @@ -4,9 +4,17 @@ import type { z } from "zod"; import { dataPaths } from "../../config.ts"; import { requireScope } from "../../http/auth.ts"; import { sseConnectionCount } from "../../http/sse.ts"; +import { registerScheduledJob } from "../schedules/registry.ts"; import type { ModuleApp, ServerContext, ServerModule } from "../types.ts"; import { buildHealth, type HealthDeps, loadHistory } from "./health.ts"; -import { type HeartbeatWriter, instanceIdOf, SERVER_VERSION, safeHostname, startHeartbeatWriter } from "./heartbeat.ts"; +import { + type HeartbeatWriter, + instanceIdOf, + pruneHeartbeats, + SERVER_VERSION, + safeHostname, + startHeartbeatWriter, +} from "./heartbeat.ts"; export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./heartbeat.ts"; @@ -15,6 +23,7 @@ export { HEARTBEAT_STALE_MS, HEARTBEAT_WRITE_MS, startHeartbeatWriter } from "./ * - GET /system/health:进程心跳 / 事件循环延迟 / SSE 连接 / DB / 磁盘 / 挂载根 / 引擎 / 定时任务,仅 admin; * - GET /system/health/history?hours=:心跳采样时间序列。 * 心跳写入:worker 进程在 worker/index.ts 里起写入器;纯 api 进程(role=api)在注册路由时起(role=all 由 worker 写 kind=all)。 + * 清理:定时任务 system.heartbeat_cleanup(维护组,1 小时)删 24 小时没心跳的进程行与 7 天前的采样。 */ export interface SystemModuleOptions { /** 引擎 schema(测试覆盖);缺省 openworkflow */ @@ -23,6 +32,8 @@ export interface SystemModuleOptions { healthDeps?: Partial>; /** 缺省 true;false 时 role=api 也不起心跳写入器(测试) */ apiHeartbeat?: boolean; + /** 定时任务 system.heartbeat_cleanup 的默认周期,默认 1 小时;<=0 不登记 */ + cleanupIntervalMs?: number; } type Infer = T extends z.ZodType ? z.infer : Record; @@ -93,6 +104,23 @@ export function createSystemModule(opts: SystemModuleOptions = {}): ServerModule route(app, Api.system.healthHistory, async ({ query }) => await loadHistory(ctx.db, query.hours)); }, registerTasks() {}, + start(ctx) { + const everyMs = opts.cleanupIntervalMs ?? 3600_000; + if (everyMs <= 0) return; + registerScheduledJob(ctx, { + key: "system.heartbeat_cleanup", + defaults: { + name: "进程心跳清理", + description: "删除 24 小时没有心跳的进程行和 7 天前的心跳采样", + group: "maintain", + schedule: { type: "interval", everyMs }, + }, + run: async ({ now }) => { + const r = await pruneHeartbeats(ctx.db, { now }); + return { summary: `删除 ${r.rows} 个进程行、${r.samples} 条采样` }; + }, + }); + }, async stop() { await writer?.stop(); writer = null; diff --git a/apps/server/test/system/system.test.ts b/apps/server/test/system/system.test.ts index 895085d5..087936d6 100644 --- a/apps/server/test/system/system.test.ts +++ b/apps/server/test/system/system.test.ts @@ -6,9 +6,17 @@ import { SystemHealth, SystemHealthHistory } from "@umt/contracts"; import { sql } from "@umt/db"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { inferTriggeredBy, triggeredByFromActor } from "../../src/modules/auth-settings/engine-business.ts"; -import { diskLevel, processLevel, worst } from "../../src/modules/system/health.ts"; -import { HEARTBEAT_STALE_MS, startHeartbeatWriter, writeHeartbeat } from "../../src/modules/system/heartbeat.ts"; +import { scheduleRegistry } from "../../src/modules/schedules/registry.ts"; +import { diskLevel, processLevel, splitProcesses, worst } from "../../src/modules/system/health.ts"; +import { + HEARTBEAT_STALE_MS, + markHeartbeatStopped, + pruneHeartbeats, + startHeartbeatWriter, + writeHeartbeat, +} from "../../src/modules/system/heartbeat.ts"; import { createSystemModule } from "../../src/modules/system/index.ts"; +import type { ServerContext } from "../../src/modules/types.ts"; import { bearer, createHarness, type Harness, pgAvailable } from "../auth-settings/harness.ts"; import { DEV_KEY, json } from "../helpers.ts"; @@ -39,6 +47,7 @@ describe("纯函数", () => { lastBeatAt: new Date().toISOString(), beatAgeSec: 0, stale: o.stale ?? false, + stoppedAt: null, eventLoopLagMs: 0, eventLoopLagMaxMs: o.lag ?? 0, concurrency: {}, @@ -53,6 +62,82 @@ describe("纯函数", () => { expect(processLevel([p("api"), p("worker", { lag: 6000 })])).toBe("critical"); }); + it("当前实例 / 已下线实例:host 10 分钟无心跳、已退出、被同 host 新进程替换的都算下线", () => { + const now = new Date("2026-09-28T12:00:00Z"); + const ago = (sec: number) => new Date(now.getTime() - sec * 1000).toISOString(); + const p = (id: string, kind: "api" | "worker" | "worker:agent", host: string, beatAgo: number, o = {}) => ({ + instanceId: id, + processKind: kind, + pid: 1, + host, + version: "t", + startedAt: ago(beatAgo + 600), + lastBeatAt: ago(beatAgo), + beatAgeSec: beatAgo, + stale: beatAgo > 45, + stoppedAt: null as string | null, + eventLoopLagMs: 0, + eventLoopLagMaxMs: 0, + concurrency: {}, + runningRuns: null, + sseConnections: null, + extra: {}, + ...o, + }); + const rows = [ + // 当前容器 3 个进程,心跳正常 + p("api:new-a:1", "api", "new-a", 5), + p("worker:new-w:1", "worker", "new-w", 5), + p("worker:agent:new-g:1", "worker:agent", "new-g", 5), + // 容器重建前的 3 行,最后心跳 40 分钟前 + p("api:old-a:1", "api", "old-a", 2400), + p("worker:old-w:1", "worker", "old-w", 2400), + p("worker:agent:old-g:1", "worker:agent", "old-g", 2400), + // 同容器进程重启:旧 pid 行 2 分钟没心跳、启动更早 + p("worker:new-w:0", "worker", "new-w", 120, { startedAt: ago(7200) }), + // 正常退出 + p("api:new-a:0", "api", "new-a", 20, { startedAt: ago(9000), stoppedAt: ago(20) }), + ]; + const r = splitProcesses(rows, now); + expect(r.processes.map((x) => x.instanceId).sort()).toEqual([ + "api:new-a:1", + "worker:agent:new-g:1", + "worker:new-w:1", + ]); + expect(r.retiredProcesses.map((x) => x.instanceId)).toEqual([ + "api:new-a:0", + "worker:new-w:0", + "api:old-a:1", + "worker:old-w:1", + "worker:agent:old-g:1", + ]); + expect(processLevel(r.processes)).toBe("ok"); + + // 当前容器里的 worker 停了心跳(1 分钟):仍是当前实例,stale 告警 + const hung = splitProcesses([p("api:a:1", "api", "a", 5), p("worker:w:1", "worker", "w", 60)], now); + expect(hung.processes).toHaveLength(2); + expect(processLevel(hung.processes)).toBe("warn"); + // 退出后立即不算活进程 + const gone = splitProcesses( + [p("api:a:1", "api", "a", 5), p("worker:w:1", "worker", "w", 3, { stoppedAt: ago(1) })], + now, + ); + expect(gone.processes.map((x) => x.instanceId)).toEqual(["api:a:1"]); + expect(gone.retiredProcesses.map((x) => x.instanceId)).toEqual(["worker:w:1"]); + }); + + it("system.heartbeat_cleanup:1 小时,维护组", async () => { + const ctx = {} as ServerContext; + await createSystemModule({ apiHeartbeat: false }).start?.(ctx); + expect(scheduleRegistry(ctx).get("system.heartbeat_cleanup")?.defaults).toMatchObject({ + group: "maintain", + schedule: { type: "interval", everyMs: 3_600_000 }, + }); + const off = {} as ServerContext; + await createSystemModule({ apiHeartbeat: false, cleanupIntervalMs: 0 }).start?.(off); + expect(scheduleRegistry(off).get("system.heartbeat_cleanup")).toBeUndefined(); + }); + it("触发来源推断", () => { expect(triggeredByFromActor("api_key:abc")).toBe("api_key"); expect(triggeredByFromActor("admin_session:u1")).toBe("user"); @@ -156,10 +241,15 @@ describe.skipIf(!hasPg)("W9-I 进程与健康(真实 PG)", () => { }); await w.beat(); await w.beat(); + expect( + (await h.db.selectFrom("process_heartbeat").select("stopped_at").executeTakeFirstOrThrow()).stopped_at, + ).toBeNull(); await w.stop(); expect(w.instanceId).toBe("worker:test-host:4242"); const rows = await h.db.selectFrom("process_heartbeat").selectAll().execute(); expect(rows).toHaveLength(1); + // 正常退出:标 stopped_at + expect(rows[0]!.stopped_at).toBeInstanceOf(Date); expect(rows[0]).toMatchObject({ process_kind: "worker", pid: 4242, @@ -190,8 +280,15 @@ describe.skipIf(!hasPg)("W9-I 进程与健康(真实 PG)", () => { expect(res.status).toBe(200); const d = SystemHealth.parse((await json(res)).data); const byId = new Map(d.processes.map((p) => [p.instanceId, p])); - expect(byId.get("worker:test-host:4242")).toMatchObject({ stale: false, runningRuns: 2, concurrency: { io: 4 } }); - expect(byId.get("api:old-host:1")).toMatchObject({ stale: true, sseConnections: 3 }); + // 已退出的 worker 立即归入已下线实例,不再当活进程 + expect(byId.has("worker:test-host:4242")).toBe(false); + expect(d.retiredProcesses.find((p) => p.instanceId === "worker:test-host:4242")).toMatchObject({ + runningRuns: 2, + concurrency: { io: 4 }, + stoppedAt: expect.any(String), + }); + // 60s 前的心跳,host 仍在 10 分钟窗口内 → 当前实例,stale + expect(byId.get("api:old-host:1")).toMatchObject({ stale: true, sseConnections: 3, stoppedAt: null }); // stale 进程的 SSE 不计入 expect(d.sse.connections).toBe(0); }); @@ -379,4 +476,61 @@ describe.skipIf(!hasPg)("W9-I 进程与健康(真实 PG)", () => { expect(detail.business).toMatchObject({ kind: "agent", refId: agentId }); expect(detail.stepAttempts).toEqual([]); }); + it("容器重建留下的旧心跳行不计入告警;24 小时的行由清理任务删除;重启后 upsert 清掉 stopped_at", async () => { + await sql`delete from process_heartbeat`.execute(h.db); + const now = Date.now(); + const beat = (id: string, kind: "api" | "worker" | "worker:agent", host: string, agoMs: number) => + writeHeartbeat(h.db, { + instanceId: id, + kind, + pid: 1, + host, + version: "0.2.0", + startedAt: new Date(now - agoMs - 3600_000), + lagMs: 0, + lagMaxMs: 0, + sample: {}, + at: new Date(now - agoMs), + }); + await beat("api:c-api:1", "api", "c-api", 2_000); + await beat("worker:c-io:1", "worker", "c-io", 2_000); + await beat("worker:agent:c-ag:1", "worker:agent", "c-ag", 2_000); + await beat("api:o-api:1", "api", "o-api", 40 * 60_000); + await beat("worker:o-io:1", "worker", "o-io", 40 * 60_000); + await beat("worker:agent:o-ag:1", "worker:agent", "o-ag", 40 * 60_000); + await beat("worker:ancient:1", "worker", "ancient", 25 * 3600_000); + + const d = SystemHealth.parse((await json(await admin("GET", "/system/health"))).data); + expect(d.processes.map((p) => p.instanceId).sort()).toEqual([ + "api:c-api:1", + "worker:agent:c-ag:1", + "worker:c-io:1", + ]); + expect(d.processes.every((p) => !p.stale)).toBe(true); + // 超过 24 小时的行不再返回 + expect(d.retiredProcesses.map((p) => p.instanceId).sort()).toEqual([ + "api:o-api:1", + "worker:agent:o-ag:1", + "worker:o-io:1", + ]); + + // 清理任务:删 24 小时没心跳的行 + const r = await pruneHeartbeats(h.db); + expect(r.rows).toBe(1); + const left = await h.db.selectFrom("process_heartbeat").select("instance_id").execute(); + expect(left.map((x) => x.instance_id)).not.toContain("worker:ancient:1"); + expect(left).toHaveLength(6); + + // 退出标记后再次心跳(同 instance_id 重启):stopped_at 清回 null + await markHeartbeatStopped(h.db, "worker:c-io:1"); + const d2 = SystemHealth.parse((await json(await admin("GET", "/system/health"))).data); + expect(d2.processes.map((p) => p.instanceId)).not.toContain("worker:c-io:1"); + await beat("worker:c-io:1", "worker", "c-io", 0); + const row = await h.db + .selectFrom("process_heartbeat") + .select("stopped_at") + .where("instance_id", "=", "worker:c-io:1") + .executeTakeFirstOrThrow(); + expect(row.stopped_at).toBeNull(); + }); }); diff --git a/apps/web/app/routes/console/status.tsx b/apps/web/app/routes/console/status.tsx index ef7ce32a..eaef8bbe 100644 --- a/apps/web/app/routes/console/status.tsx +++ b/apps/web/app/routes/console/status.tsx @@ -1,6 +1,17 @@ import { useQueryClient } from "@tanstack/react-query"; import type { HealthLevel, SystemHealth, SystemProcess } from "@umt/contracts"; -import { AlertTriangle, CircleCheck, Cpu, Database, HardDrive, ListTodo, RotateCw, Timer, XCircle } from "lucide-react"; +import { + AlertTriangle, + ChevronDown, + CircleCheck, + Cpu, + Database, + HardDrive, + ListTodo, + RotateCw, + Timer, + XCircle, +} from "lucide-react"; import { type ReactNode, useState } from "react"; import { Link } from "react-router"; import { useSystemHealth, useSystemHealthHistory } from "~/api/system.ts"; @@ -35,6 +46,14 @@ const KIND_LABEL: Record = { all: "单进程(all)", }; +/** 横幅告警里的进程称呼 */ +const KIND_NAME: Record = { + api: "API", + worker: "io worker", + "worker:agent": "Agent worker", + all: "单进程", +}; + const MOUNT_REASON: Record = { root_missing: "目录不存在", sentinel_missing: "哨兵文件缺失", @@ -74,8 +93,11 @@ function Card({ /** 顶部横幅:整体档位 + 需要看的几条 */ function problems(d: SystemHealth): string[] { const out: string[] = []; - const stale = d.processes.filter((p) => p.stale); - if (stale.length) out.push(`${stale.length} 个进程超过 45 秒没有心跳`); + // 只看当前实例;已下线实例不告警 + for (const p of d.processes.filter((x) => x.stale)) { + const line = `${KIND_NAME[p.processKind] ?? p.processKind} 45 秒没有心跳`; + if (!out.includes(line)) out.push(line); + } if (!d.processes.some((p) => !p.stale && p.processKind !== "api")) out.push("没有在线的 worker，任务不会被执行"); const lag = d.processes.filter((p) => !p.stale && p.eventLoopLagMaxMs >= 1000); if (lag.length) out.push(`${lag.length} 个进程事件循环延迟超过 1 秒`); @@ -136,7 +158,7 @@ function Processes({ d }: { d: SystemHealth }) { rows={d.processes} rowKey={(p) => p.instanceId} - empty={{ title: "24 小时内没有进程心跳", description: "进程启动后每 15 秒写一次心跳。" }} + empty={{ title: "没有在线的进程", description: "进程启动后每 15 秒写一次心跳。" }} columns={[ { key: "kind", @@ -236,6 +258,77 @@ function Processes({ d }: { d: SystemHealth }) { ); } +/** 已下线实例:已退出、容器已重建或被同容器新进程替换的旧心跳行,默认折叠,不参与告警 */ +function RetiredProcesses({ rows }: { rows: SystemProcess[] }) { + const [open, setOpen] = useState(false); + if (!rows.length) return null; + return ( +
+ + {open ? ( +
+ + rows={rows} + rowKey={(p) => p.instanceId} + empty={{ title: "没有已下线实例" }} + columns={[ + { + key: "id", + header: "实例", + cell: (p) => ( +
+

+ {KIND_LABEL[p.processKind] ?? p.processKind} + {p.stoppedAt ? 已退出 : null} +

+

+ {p.instanceId} +

+
+ ), + }, + { + key: "ver", + header: "版本", + hideBelow: "md", + cell: (p) => {p.version}, + }, + { + key: "start", + header: "启动", + hideBelow: "md", + cell: (p) => ( + + {formatRelative(p.startedAt)} + + ), + }, + { + key: "beat", + header: "最后心跳", + cell: (p) => ( + + {formatRelative(p.lastBeatAt)} + + ), + }, + ]} + /> +
+ ) : null} +
+ ); +} + function History() { const [hours, setHours] = useState(6); const h = useSystemHealthHistory(hours); @@ -397,6 +490,7 @@ export default function StatusPage() { SSE 连接合计 {d.sse.connections} +
diff --git a/apps/web/package.json b/apps/web/package.json index d245f220..dfebc580 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "@umt/web", - "version": "0.2.0", + "version": "0.2.2", "private": true, "type": "module", "scripts": { diff --git a/deno.json b/deno.json index 1156756b..92197d21 100644 --- a/deno.json +++ b/deno.json @@ -1,5 +1,5 @@ { - "version": "0.2.0", + "version": "0.2.2", "workspace": [ "./packages/adapters", "./packages/agent", diff --git a/docs/design/glossary.md b/docs/design/glossary.md index 38fb39a1..7a52a1a7 100644 --- a/docs/design/glossary.md +++ b/docs/design/glossary.md @@ -42,6 +42,8 @@ | agent.propose_options | 方案;动作叫"选定",都不要叫"退回" | 选项、提案 | | scope.proposeFirst | 先看方案再执行 | 只要方案、只给方案、plan_only 模式 | | 工作流引擎的一次执行 | 运行 | run | +| 正在写心跳的进程(process_heartbeat 当前行) | 当前实例;横幅按角色称呼:API / io worker / Agent worker / 单进程 | 活进程、在线实例 | +| 已退出、容器已重建或被同容器新进程替换的心跳行 | 已下线实例;正常退出的标"已退出" | 历史实例、僵尸进程、死进程 | ## 文案规则 diff --git a/docs/runbook/ops.md b/docs/runbook/ops.md index 70357728..cca2fc3c 100644 --- a/docs/runbook/ops.md +++ b/docs/runbook/ops.md @@ -13,7 +13,8 @@ - 每 15 秒 upsert `process_heartbeat` 一行,同时往 `process_heartbeat_sample` 追加一条采样。 - 行主键 `instance_id = ::`,容器重启覆盖同一行。 -- 写入方每小时裁剪一次:采样保留 7 天,7 天没心跳的进程行删除。 +- 正常退出(SIGTERM / SIGINT)时写入器把本行标 `stopped_at`,读取方立即归入已下线实例;重启后同一行的 upsert 把它清回 null。 +- 裁剪由定时任务 `system.heartbeat_cleanup`(维护组,1 小时)负责:24 小时没心跳的进程行删除,采样保留 7 天。 - 写库失败只记 debug 日志,不影响业务。 ## GET /system/health 字段 @@ -22,8 +23,10 @@ | --- | --- | | `level` | 各段最差档:`ok` / `warn` / `critical`。取不到数据的段是 `unknown`,不参与汇总 | | `api` | 处理本次请求的 API 进程:版本、启动时间、运行秒数 | -| `processes[]` | 最近 24 小时有心跳的进程 | -| `processes[].stale` | 超过 45 秒没心跳 | +| `processes[]` | 当前实例:最近 10 分钟内有心跳的 host 上、同 host 同 kind 最新启动且没标 `stopped_at` 的进程 | +| `processes[].stale` | 超过 45 秒没心跳。只有当前实例参与告警 | +| `processes[].stoppedAt` | 正常退出时间;非 null 即已下线 | +| `retiredProcesses[]` | 已下线实例:正常退出的、所在 host 10 分钟没有心跳(容器已重建)的、被同 host 新进程替换的。不计入 `level`,页面折叠显示 | | `processes[].eventLoopLagMs` | 最近一次采样的事件循环延迟(1 秒采一次) | | `processes[].eventLoopLagMaxMs` | 最近一个心跳周期(15 秒)内的最大延迟 | | `processes[].concurrency` | 引擎 worker 并发 `{io, agent}`;api 进程为 `{}` | @@ -48,7 +51,7 @@ ## 告警阈值建议 -- 进程 `stale`:warn。连续 2 分钟仍 stale 就去查容器状态和日志。 +- 当前实例 `stale`:warn。连续 2 分钟仍 stale 就去查容器状态和日志。容器整个没了 10 分钟后它的进程转入已下线实例,此时靠下一条"没有存活的 worker"兜底。 - 没有任何存活的 worker / all 进程:warn。任务不会被执行。 - 事件循环最大延迟 ≥ 1 秒记 warn,≥ 5 秒记 critical。延迟接近 15 秒时,引擎租约有丢失风险。 - DB 延迟 ≥ 100 ms 或连接数 ≥ 75% `max_connections` 记 warn;≥ 500 ms 或 ≥ 90% 记 critical。 diff --git a/docs/runbook/schedules.md b/docs/runbook/schedules.md index 1fd8af0e..73f1c872 100644 --- a/docs/runbook/schedules.md +++ b/docs/runbook/schedules.md @@ -30,6 +30,7 @@ | `trash.purge` | 回收站清理 | 维护 | 24 小时 | 入队 `purgeTrash` | | `pi.token_refresh` | 上游登录刷新 | 维护 | 10 分钟 | 入队 `refreshPiTokens`,幂等键 `refreshPiTokens:<时间槽>`;对 30 分钟内过期的订阅登录上游刷新 token | | `auth.cleanup` | 登录会话清理 | 维护 | 1 小时 | 直接执行 `cleanupAuthTables`:删 7 天前的登录尝试、30 天前过期或吊销的会话 | +| `system.heartbeat_cleanup` | 进程心跳清理 | 维护 | 1 小时 | 删 24 小时没有心跳的进程行、7 天前的心跳采样 | | `maintain.backup` | 备份 | 维护 | 每天 03:30(占位) | 未配置。备份目前由宿主机上的 `scripts/backup.sh` 执行 | | `check.pending` | 待处理巡检 | Agent | 6 小时，默认关闭 | 只读。有未关闭事项时才会运行 | | `check.downloads` | 下载诊断 | Agent | 2 小时，默认关闭 | 默认只读。打开 `allowRecover` 后，在授权范围内重试或重新校验 | diff --git a/packages/adapters/deno.json b/packages/adapters/deno.json index 25c16a15..192f250d 100644 --- a/packages/adapters/deno.json +++ b/packages/adapters/deno.json @@ -1,6 +1,6 @@ { "name": "@umt/adapters", - "version": "0.2.0", + "version": "0.2.2", "exports": { ".": "./src/index.ts", "./fsworker": "./src/fs/fsworker.ts" }, "tasks": { "test": "deno run -A npm:vitest run", diff --git a/packages/adapters/src/browser/camofox/client.ts b/packages/adapters/src/browser/camofox/client.ts index 825133ad..7fd9f66f 100644 --- a/packages/adapters/src/browser/camofox/client.ts +++ b/packages/adapters/src/browser/camofox/client.ts @@ -1,6 +1,7 @@ /** * camofox-browser 适配器(BrowserProvider 首个实现)。 - * 对照:jo-inc/camofox-browser master openapi.json(info.version 1.17.0)与 README,仅读文档,未连接真实实例。 + * 对照:jo-inc/camofox-browser v1.17.0(commit 389c996)openapi.json、README 与 server.js;2026-09-28 对真实容器核验, + * 结果见 docs/runbook/browser.md「真机核验」。 * * 上游模型与本项目合同的映射: * - camofox userId = 一个隔离 BrowserContext(cookies/storage),本项目用 profileKey(站点/账号)充当; @@ -9,10 +10,10 @@ * 认证:README/OpenAPI 为 `Authorization: Bearer `(设置后除 /health 外全部路由要求); * 凭据每次请求经 BrowserSecretResolver 解出,不缓存、不入日志、不进错误信息。 * - * 已实现(仅对 mock 与上游文档验证):health、capabilities、会话开/关/列举、标签页增删与对账、navigate(白名单/超时/重试)、 + * 已实现:health、capabilities、会话开/关/列举、标签页增删与对账、navigate(白名单/超时/重试)、 * 后退/前进/刷新、snapshot(脱敏)、screenshot、click/type/select/scroll/press、extract、waitForDownload、 * 内存版控制权租约与串行锁。会话/标签/租约的持久化与对账由服务层(apps/server modules/browser)以数据库为准。 - * 未核验的上游行为(坐标点击、无目标输入、downloads 字段形状)见 docs/runbook/browser.md。 + * 与文档不同的实测行为(截图直接返回 image/png、click 不接受坐标、焦点输入需 mode=keyboard、downloads 字段名)已按实测适配。 */ import type { BrowserCapabilities, @@ -718,6 +719,13 @@ export class CamofoxBrowserProvider implements BrowserProvider { await this.withProfile(s, async () => { this.assertActor(s, ref.as); const tab = this.tabOf(s, ref); + // 上游 click/type 只认 ref/selector(1.17.0 实测不接受 {x, y}):坐标先经 evaluate 标记该点元素,再按选择器操作, + // 这样仍走上游完整的鼠标事件序列与下载守卫,而不是页面内合成 click + if (typeof payload.x === "number" && typeof payload.y === "number") { + payload.selector = await this.pointSelector(s, tab, payload.x, payload.y); + delete payload.x; + delete payload.y; + } const path = `/tabs/${encodeURIComponent(tab.upstreamTabId)}/${action}`; const res = await this.request("POST", path, { body: { userId: s.session.profileKey, ...payload }, @@ -732,6 +740,22 @@ export class CamofoxBrowserProvider implements BrowserProvider { }); } + /** 视口坐标 → 唯一属性选择器(POST /tabs/{id}/evaluate + elementFromPoint);坐标处无元素报 bad_request */ + private async pointSelector(s: SessionState, tab: BrowserTab, x: number, y: number): Promise { + const nonce = this.newId().replace(/[^a-z0-9]/gi, "").slice(0, 16); + const expression = `(() => { const e = document.elementFromPoint(${x}, ${y}); if (!e) return false; + for (const o of document.querySelectorAll("[data-umt-pt]")) o.removeAttribute("data-umt-pt"); + e.setAttribute("data-umt-pt", "${nonce}"); return true; })()`; + const path = `/tabs/${encodeURIComponent(tab.upstreamTabId)}/evaluate`; + const res = await this.request("POST", path, { + body: { userId: s.session.profileKey, expression }, + timeoutMs: s.budget.timeoutMs, + }); + const out = await this.json(res, path); + if (out.result !== true) throw new BrowserError("bad_request", "该坐标处没有可操作的元素"); + return `[data-umt-pt="${nonce}"]`; + } + /** 点击可能跳转或改动页面:之后旧引用作废 */ async click(ref: TabRef, target: ElementTarget): Promise { await this.act( @@ -765,7 +789,8 @@ export class CamofoxBrowserProvider implements BrowserProvider { ref, "type", (s) => ({ - ...(target ? this.assertTarget(s, target) : {}), + // 无目标 = 输入到当前焦点:上游 mode=fill 必须带 ref/selector,焦点输入要 mode=keyboard(1.17.0 实测) + ...(target ? this.assertTarget(s, target) : { mode: "keyboard" }), text, ...(input.clear !== undefined ? { clear: input.clear } : {}), ...(input.submit !== undefined ? { submit: input.submit } : {}), @@ -875,7 +900,13 @@ export class CamofoxBrowserProvider implements BrowserProvider { if (s.seenDownloads.has(key)) continue; const status = str(d.status) ?? str(d.state); if (status && !["completed", "complete", "done", "finished"].includes(status)) continue; - const b64 = str(d.data) ?? str(d.base64) ?? str(d.content); + // 1.17.0 实测字段:{ id, url, suggestedFilename, mimeType, bytes, createdAt, failure, dataBase64 | dataSkipped | readError } + if (d.failure) { + s.seenDownloads.add(key); + throw new BrowserError("upstream", "浏览器下载失败"); + } + if (d.dataSkipped) throw new BrowserError("budget_exceeded", `下载产物超出字节预算(${opts.maxBytes})`); + const b64 = str(d.dataBase64) ?? str(d.data) ?? str(d.base64) ?? str(d.content); const size = int(d.size) ?? int(d.bytes) ?? null; if (size !== null && size > opts.maxBytes) throw new BrowserError("budget_exceeded", `下载产物超出字节预算(${opts.maxBytes})`); diff --git a/packages/agent/deno.json b/packages/agent/deno.json index 5cb6e729..49f4d0e4 100644 --- a/packages/agent/deno.json +++ b/packages/agent/deno.json @@ -1,6 +1,6 @@ { "name": "@umt/agent", - "version": "0.2.0", + "version": "0.2.2", "exports": { ".": "./src/index.ts" }, "tasks": { "test": "deno run -A npm:vitest run", diff --git a/packages/contracts/deno.json b/packages/contracts/deno.json index 13d64616..6833d433 100644 --- a/packages/contracts/deno.json +++ b/packages/contracts/deno.json @@ -1,6 +1,6 @@ { "name": "@umt/contracts", - "version": "0.2.0", + "version": "0.2.2", "exports": { ".": "./src/index.ts" }, "tasks": { "test": "deno run -A npm:vitest run", diff --git a/packages/contracts/src/api.ts b/packages/contracts/src/api.ts index d5efdc4e..9a647ad6 100644 --- a/packages/contracts/src/api.ts +++ b/packages/contracts/src/api.ts @@ -2167,6 +2167,31 @@ export const Api = { query: browser.BrowserProvidersQuery, response: z.object({ items: z.array(browser.BrowserProviderView) }), }), + createProvider: ep({ + method: "post", + path: "/browser/providers", + scope: "admin", + summary: "登记浏览器服务（camofox 地址、访问凭据存凭证库、启用）", + body: browser.BrowserProviderCreate, + response: browser.BrowserProviderView, + }), + updateProvider: ep({ + method: "patch", + path: "/browser/providers/{id}", + scope: "admin", + summary: "修改浏览器服务地址、访问凭据或启用（环境变量登记的只能改启用）", + params: browser.BrowserProviderParams, + body: browser.BrowserProviderUpdate, + response: browser.BrowserProviderView, + }), + probeProvider: ep({ + method: "post", + path: "/browser/providers/{id}/probe", + scope: "admin", + summary: "测试连通：探测该浏览器服务的 /health 与 /openapi.json 并写回能力", + params: browser.BrowserProviderParams, + response: browser.BrowserProviderView, + }), sessions: ep({ method: "get", path: "/browser/sessions", diff --git a/packages/contracts/src/browser.ts b/packages/contracts/src/browser.ts index 81322779..48dadebc 100644 --- a/packages/contracts/src/browser.ts +++ b/packages/contracts/src/browser.ts @@ -240,8 +240,12 @@ export const BrowserProviderView = z.object({ baseUrl: z.string().nullable(), /** 访问凭据的 secret 引用(明文永不出 API) */ secretRef: Id.nullable(), + /** env = 由 BROWSER_PROVIDER_URL / BROWSER_ACCESS_KEY 登记(地址与凭据只能改部署配置);null = 设置页登记 */ + managedBy: z.enum(["env"]).nullable(), + /** 是否配置了访问凭据(secret 或环境变量) */ + hasAccessKey: z.boolean(), enabled: z.boolean(), - /** 最近一次声明/探测到的能力;verified 只由人工核验写入 config.verified */ + /** 最近一次声明/探测到的能力;verified 只由人工核验写入(config.verified 或适配器内置的已核验版本表) */ capabilities: BrowserCapabilities.nullable(), verifiedAt: Timestamp.nullable(), healthStatus: InstanceHealth, @@ -251,6 +255,30 @@ export const BrowserProviderView = z.object({ }); export type BrowserProviderView = z.infer; +const HttpUrl = z + .string() + .url() + .refine((u) => /^https?:\/\//i.test(u), "只支持 http(s) 地址"); + +/** 设置页登记浏览器服务(camofox);accessKey 明文只在请求里出现一次,落库为 secret */ +export const BrowserProviderCreate = z.object({ + name: z.string().min(1).max(64), + baseUrl: HttpUrl, + accessKey: z.string().min(1).max(512).optional(), + enabled: z.boolean().optional(), +}); +export type BrowserProviderCreate = z.infer; + +/** 改地址 / 凭据 / 启用;accessKey = null 清除凭据 */ +export const BrowserProviderUpdate = z.object({ + baseUrl: HttpUrl.optional(), + accessKey: z.string().min(1).max(512).nullable().optional(), + enabled: z.boolean().optional(), +}); +export type BrowserProviderUpdate = z.infer; + +export const BrowserProviderParams = z.object({ id: Id }); + export const BrowserProvidersQuery = z.object({ /** true = 现场探测 /health 与 /openapi.json 并写回 */ probe: QueryBool, diff --git a/packages/contracts/src/ops.ts b/packages/contracts/src/ops.ts index 78a1bc6b..b8375f84 100644 --- a/packages/contracts/src/ops.ts +++ b/packages/contracts/src/ops.ts @@ -381,8 +381,10 @@ export const SystemProcess = z.object({ lastBeatAt: Timestamp, /** 距上次心跳秒数 */ beatAgeSec: z.number().nonnegative(), - /** 超过 45s 没心跳 */ + /** 超过 45s 没心跳(只对当前实例告警) */ stale: z.boolean(), + /** 进程正常退出时写入;非 null 即已下线 */ + stoppedAt: Timestamp.nullable(), /** 最近一次采样的事件循环延迟 */ eventLoopLagMs: NonNegInt, /** 最近一个心跳周期(15s)内的最大延迟 */ @@ -434,7 +436,10 @@ export const SystemHealth = z.object({ checkedAt: Timestamp, /** 处理本请求的 api 进程 */ api: z.object({ version: z.string(), startedAt: Timestamp, uptimeSec: z.number().nonnegative() }), + /** 当前实例:最近 10 分钟内有心跳的 host 上、同 host 同 kind 最新启动且未退出的进程;stale 与告警只看这里 */ processes: z.array(SystemProcess), + /** 已下线实例:已正常退出、所在 host 10 分钟没有心跳(容器已重建)、或被同 host 新进程替换;不计入 level */ + retiredProcesses: z.array(SystemProcess), sse: z.object({ connections: NonNegInt }), db: z.object({ level: HealthLevel, diff --git a/packages/db/deno.json b/packages/db/deno.json index 035edbf9..b5760dfb 100644 --- a/packages/db/deno.json +++ b/packages/db/deno.json @@ -1,6 +1,6 @@ { "name": "@umt/db", - "version": "0.2.0", + "version": "0.2.2", "exports": { ".": "./src/index.ts" }, "tasks": { "test": "deno run -A npm:vitest run", diff --git a/packages/db/migrations/0043_process_heartbeat_stopped.ts b/packages/db/migrations/0043_process_heartbeat_stopped.ts new file mode 100644 index 00000000..647f13b9 --- /dev/null +++ b/packages/db/migrations/0043_process_heartbeat_stopped.ts @@ -0,0 +1,16 @@ +import type { Kysely } from "kysely"; +import { execDdl } from "../src/migration-helpers.ts"; + +// 心跳残留修复:进程正常退出(SIGTERM / SIGINT)时把自己的行标 stopped_at,读取方立即把它当已下线实例, +// 不再等 45s 判 stale。重启后同一 instance_id 的 upsert 把 stopped_at 清回 null。 +// 超过 24 小时没心跳的行由定时任务 system.heartbeat_cleanup 删除。 + +// biome-ignore lint/suspicious/noExplicitAny: 迁移不依赖应用表类型 +export async function up(db: Kysely): Promise { + await execDdl(db, `alter table process_heartbeat add column stopped_at timestamptz;`); +} + +// biome-ignore lint/suspicious/noExplicitAny: 同上 +export async function down(db: Kysely): Promise { + await execDdl(db, `alter table process_heartbeat drop column if exists stopped_at;`); +} diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index b2642d60..2d2930ba 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,5 +1,5 @@ // 统一从本包取 Kysely 与 sql,避免各包各装一份 kysely -export { type Kysely, sql, type Transaction } from "kysely"; +export { type Kysely, type Selectable, sql, type Transaction } from "kysely"; export { type CachedOptions, cachedCall, diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 8515f3af..82677901 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -1152,6 +1152,7 @@ export interface ProcessHeartbeatTable { concurrency: Generated; running_runs: number | null; extra: Generated; + stopped_at: Timestamptz | null; } export interface ProcessHeartbeatSampleTable { diff --git a/packages/domain/deno.json b/packages/domain/deno.json index af0bc7f3..b627da71 100644 --- a/packages/domain/deno.json +++ b/packages/domain/deno.json @@ -1,6 +1,6 @@ { "name": "@umt/domain", - "version": "0.2.0", + "version": "0.2.2", "exports": { ".": "./src/index.ts" }, "tasks": { "test": "deno run -A npm:vitest run", diff --git a/packages/mapping/deno.json b/packages/mapping/deno.json index 6b0d7880..05d8d06b 100644 --- a/packages/mapping/deno.json +++ b/packages/mapping/deno.json @@ -1,6 +1,6 @@ { "name": "@umt/mapping", - "version": "0.2.0", + "version": "0.2.2", "exports": { ".": "./src/index.ts" }, "tasks": { "test": "deno run -A npm:vitest run", diff --git a/packages/tasks/deno.json b/packages/tasks/deno.json index 1c9c04a1..e76f7864 100644 --- a/packages/tasks/deno.json +++ b/packages/tasks/deno.json @@ -1,6 +1,6 @@ { "name": "@umt/tasks", - "version": "0.2.0", + "version": "0.2.2", "exports": { ".": "./src/index.ts" }, "tasks": { "test": "deno run -A npm:vitest run", diff --git a/packages/testkit/deno.json b/packages/testkit/deno.json index cd65156f..dbec17be 100644 --- a/packages/testkit/deno.json +++ b/packages/testkit/deno.json @@ -1,6 +1,6 @@ { "name": "@umt/testkit", - "version": "0.2.0", + "version": "0.2.2", "exports": { ".": "./src/index.ts" }, "tasks": { "test": "deno run -A npm:vitest run", diff --git a/packages/testkit/src/browser/mock-camofox.ts b/packages/testkit/src/browser/mock-camofox.ts index f0675021..268b0055 100644 --- a/packages/testkit/src/browser/mock-camofox.ts +++ b/packages/testkit/src/browser/mock-camofox.ts @@ -4,14 +4,15 @@ import { deflateSync } from "node:zlib"; import type { MockCall, MockFault, MockHttp, MockReply } from "../tmdb/mock-http.ts"; /** - * mock camofox-browser(形状按 jo-inc/camofox-browser openapi.json 1.17.0,取值虚构,未对真实实例核验)。 + * mock camofox-browser(形状按 jo-inc/camofox-browser v1.17.0,2026-09-28 与真实容器对照过请求/响应字段;取值虚构)。 * 路由:GET /health、GET /openapi.json、POST /tabs、GET /tabs、DELETE /tabs/{id}、POST /tabs/{id}/navigate、 * POST /tabs/{id}/{back,forward,refresh}、GET /tabs/{id}/snapshot、GET /tabs/{id}/screenshot(base64 PNG)、 - * POST /tabs/{id}/{click,type,select,scroll,press,extract}、GET /tabs/{id}/downloads、DELETE /sessions/{userId}。 + * POST /tabs/{id}/{click,type,select,scroll,press,extract,evaluate}、GET /tabs/{id}/downloads、DELETE /sessions/{userId}。 * 不给 site:快照为固定文本(含密码值/Cookie/passkey 供脱敏测试)。 * 给 site:模拟一个 NexusPHP 风格站点——未登录访问受保护页重定向登录页,登录页可带验证码(需人工), * 点击/输入改变页面,点"下载"把 .torrent 放进该标签页的下载列表;登录状态按 userId(profile)保存。 - * 坐标点击:页面元素按快照行排布,第 i 行(从 0 起)占 y ∈ [MOCK_ROW_Y0 + i·MOCK_ROW_H, +MOCK_ROW_H)。 + * 坐标:页面元素按快照行排布,第 i 行(从 0 起)占 y ∈ [MOCK_ROW_Y0 + i·MOCK_ROW_H, +MOCK_ROW_H);与真实上游一样 + * click 不接受 {x, y},适配器先 evaluate(elementFromPoint)打 data-umt-pt 标记再按选择器点击。 * accessKey 设置时模拟 CAMOFOX_ACCESS_KEY:除 /health 外要求 `Authorization: Bearer `,否则 401。 * 故障注入(401/超时/断连)用 MockHttp.fault。 */ @@ -58,6 +59,8 @@ export interface MockCamofoxTab { /** 元素引用 → 已输入的值 */ fields: Map; focus: string | null; + /** evaluate(elementFromPoint)打的 data-umt-pt 标记 → 元素引用 */ + marks: Map; downloads: MockCamofoxDownload[]; error: string | null; } @@ -121,6 +124,7 @@ const OPENAPI_PATHS: Record = { "/tabs/{tabId}/click": ["post"], "/tabs/{tabId}/type": ["post"], "/tabs/{tabId}/select": ["post"], + "/tabs/{tabId}/evaluate": ["post"], "/tabs/{tabId}/extract": ["post"], "/tabs/{tabId}/downloads": ["get"], "/sessions/{userId}": ["delete"], @@ -403,8 +407,11 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< const findLine = (t: MockCamofoxTab, b: Body): Line | undefined => { const lines = pageLines(t); if (typeof b.ref === "string") return lines.find((l) => l.ref === b.ref); - if (typeof b.y === "number") return lines[Math.floor((b.y - MOCK_ROW_Y0) / MOCK_ROW_H)]; - if (typeof b.selector === "string") return lines.find((l) => l.role.startsWith("button")); + if (typeof b.selector === "string") { + const mark = /^\[data-umt-pt="([^"]+)"\]$/.exec(b.selector)?.[1]; + if (mark) return lines.find((l) => l.ref === t.marks.get(mark)); + return lines.find((l) => l.role.startsWith("button")); + } return undefined; }; @@ -473,6 +480,7 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< cursor: -1, fields: new Map(), focus: null, + marks: new Map(), downloads: [], error: null, }; @@ -565,6 +573,9 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< [ /^\/tabs\/([^/]+)\/click$/, action("click", (t, b) => { + // 1.17.0 实测:只认 ref / selector,坐标点击由适配器先 evaluate 打标记 + if (typeof b.ref !== "string" && typeof b.selector !== "string") + return err(400, "ref or selector required"); const line = findLine(t, b); if (!line) return typeof b.ref === "string" ? err(422, "Unknown ref") : undefined; if (line.role.startsWith("textbox")) t.focus = line.ref; @@ -576,6 +587,8 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< /^\/tabs\/([^/]+)\/type$/, action("type", (t, b) => { if (typeof b.text !== "string") return err(400, "text required"); + if (typeof b.ref !== "string" && typeof b.selector !== "string" && b.mode !== "keyboard") + return err(400, "ref or selector required for mode=fill"); const ref = typeof b.ref === "string" ? b.ref : (findLine(t, b)?.ref ?? t.focus); if (!ref) return err(422, "No focused element"); t.fields.set(ref, b.clear === false ? (t.fields.get(ref) ?? "") + b.text : b.text); @@ -584,6 +597,21 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< return undefined; }), ], + [ + /^\/tabs\/([^/]+)\/evaluate$/, + action("evaluate", (t, b) => { + // 只模拟适配器的坐标定位表达式:elementFromPoint(x, y) + data-umt-pt 标记 + const expr = typeof b.expression === "string" ? b.expression : ""; + const pt = /elementFromPoint\((\d+), (\d+)\)/.exec(expr); + const nonce = /setAttribute\("data-umt-pt", "([^"]+)"\)/.exec(expr)?.[1]; + if (!pt || !nonce) return { json: { ok: true, result: null } }; + const line = pageLines(t)[Math.floor((Number(pt[2]) - MOCK_ROW_Y0) / MOCK_ROW_H)]; + if (!line) return { json: { ok: true, result: false } }; + t.marks.clear(); + t.marks.set(nonce, line.ref); + return { json: { ok: true, result: true } }; + }), + ], [/^\/tabs\/([^/]+)\/select$/, action("select", () => undefined)], [/^\/tabs\/([^/]+)\/scroll$/, action("scroll", () => undefined)], [ @@ -617,16 +645,20 @@ export async function createMockCamofox(opts: MockCamofoxOptions = {}): Promise< const t = tabFor(m[1]!, q.userId); if (!t) return err(404, "Tab not found"); const max = q.maxBytes ? Number(q.maxBytes) : Number.POSITIVE_INFINITY; + // 字段形状按 1.17.0 实测(lib/downloads.js getDownloadsList) const list = t.downloads.map((d) => ({ id: d.id, - filename: d.filename, url: d.url, + suggestedFilename: d.filename, mimeType: d.mimeType, - size: d.bytes.byteLength, - status: "completed", - ...(q.includeData === "true" && d.bytes.byteLength <= max - ? { data: Buffer.from(d.bytes).toString("base64") } - : { truncated: d.bytes.byteLength > max }), + bytes: d.bytes.byteLength, + createdAt: Date.now(), + failure: null, + ...(q.includeData === "true" + ? d.bytes.byteLength <= max + ? { dataBase64: Buffer.from(d.bytes).toString("base64") } + : { dataSkipped: "max_bytes_exceeded" } + : {}), })); if (q.consume === "true") t.downloads = []; return { json: { downloads: list } };