From 34e8751b45be40f7ab39b3d69d4735d7ea38c0eb Mon Sep 17 00:00:00 2001 From: root Date: Sun, 27 Sep 2026 21:27:55 +0800 Subject: [PATCH] feat(ops): W4-B single-container packaging, DATA_DIR, backup/restore - DATA_DIR (/data: agent-sessions, backups, exports, tmp); entrypoint chowns it to PUID/PGID then drops privileges via gosu; /app binary stays root read-only - compile with --allow-read --allow-write so mounted library roots are writable - preflight: DATA_DIR/session dir must be writable (fatal); enabled library roots and downloader path mappings checked read/write (warn + audit) - --role export-mappings (mapping manifest JSON, --compare row counts) and --role read-only-on (automation.readOnlyReconcile=true + audit) - scripts/backup.sh (pg_dump custom + mappings + agent sessions, SHA256SUMS, BACKUP_KEEP rotation) and scripts/restore.sh (checksum, refuses with live connections, psql single-transaction replay compatible with PG16, read-only reconcile on, row-count verification) - compose: app + postgres only, data volume, env example with key generation - docs: runbook deploy.md, backup-restore.md with verification record Co-Authored-By: Claude Opus 5.5 --- .dockerignore | 1 + .gitignore | 2 +- Dockerfile | 33 +++++-- README.md | 18 +++- apps/server/deno.json | 2 +- apps/server/src/config.ts | 23 +++++ apps/server/src/main.ts | 102 +++++++++++++++---- apps/server/src/ops.ts | 153 +++++++++++++++++++++++++++++ apps/server/src/preflight.ts | 104 ++++++++++++++++++++ apps/server/test/preflight.test.ts | 63 +++++++++++- docker-compose.yml | 24 ++++- docker/compose.env.example | 22 +++++ docker/entrypoint.sh | 37 ++++++- docs/runbook/backup-restore.md | 81 +++++++++++++++ docs/runbook/deploy.md | 148 ++++++++++++++++++++++++++++ docs/runbook/dev.md | 4 +- scripts/backup.sh | 57 +++++++++++ scripts/restore.sh | 87 ++++++++++++++++ 18 files changed, 918 insertions(+), 43 deletions(-) create mode 100644 apps/server/src/ops.ts create mode 100644 docker/compose.env.example create mode 100644 docs/runbook/backup-restore.md create mode 100644 docs/runbook/deploy.md create mode 100755 scripts/backup.sh create mode 100755 scripts/restore.sh diff --git a/.dockerignore b/.dockerignore index 37d45fd6..724195b7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,4 @@ docs !.env.example *.md !apps/**/README.md +data diff --git a/.gitignore b/.gitignore index 5192453e..d6d0bae8 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,4 @@ coverage/ .vite/ types/deno.d.ts apps/web/test/screenshots/ -data/agent-sessions/ +data/ diff --git a/Dockerfile b/Dockerfile index 366018b0..1df4ab6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,22 +10,39 @@ ARG APP_VERSION="" ENV APP_VERSION=${APP_VERSION} RUN deno task check:tokens && cd apps/web && deno task build # --node-modules-dir=none + --exclude-unused-npm:只嵌入 server 依赖图可达的 npm 包(约 110 MB,而不是整个 node_modules) -# 权限在编译时固定:--allow-read 只放 /app;不给 --allow-run / --allow-write +# 权限在编译时固定:读写不限路径(库根/下载目录由用户挂载,位置不固定;写保护靠 /app root 只读 + 非 root 运行),不给 --allow-run RUN cd apps/server && deno task compile FROM debian:trixie-slim -# 不装额外包:Deno 自带 Mozilla 根证书(外呼 HTTPS 不依赖 ca-certificates),健康探测用二进制自身的 fetch -RUN useradd --system --uid 10001 --home-dir /app --shell /usr/sbin/nologin umt +# gosu:root 修完 DATA_DIR 属主后降权;postgresql-client:备份/恢复脚本用 pg_dump/pg_restore/psql +# (trixie 自带 17 版客户端,可备份/恢复 16 版服务端)。Deno 自带根证书,不装 ca-certificates;健康探测用二进制自身的 fetch +# 国内网络可 --build-arg APT_MIRROR=mirrors.ustc.edu.cn 换源 +ARG APT_MIRROR="" +RUN if [ -n "$APT_MIRROR" ]; then sed -i "s|deb.debian.org|$APT_MIRROR|g" /etc/apt/sources.list.d/debian.sources; fi \ + && apt-get update \ + && apt-get install -y --no-install-recommends gosu postgresql-client \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system --gid 10001 umt \ + && useradd --system --uid 10001 --gid 10001 --home-dir /data --no-create-home --shell /usr/sbin/nologin umt \ + && mkdir -p /data && chown 10001:10001 /data WORKDIR /app -COPY --from=build --chown=root:root /src/dist/umt-server /app/umt-server +# /app 全部 root 属主、只读:运行用户(缺省 10001,可用 PUID/PGID 覆盖)改不了二进制与静态产物 +COPY --from=build --chown=root:root --chmod=0555 /src/dist/umt-server /app/umt-server COPY --from=build --chown=root:root /src/apps/web/build/client /app/public -COPY --chown=root:root --chmod=0755 docker/entrypoint.sh /app/entrypoint.sh +COPY --chown=root:root --chmod=0555 docker/entrypoint.sh /app/entrypoint.sh +COPY --chown=root:root --chmod=0555 scripts/backup.sh scripts/restore.sh /app/scripts/ +RUN ln -s /app/entrypoint.sh /usr/local/bin/umt ENV NODE_ENV=production \ HOST=0.0.0.0 \ PORT=4380 \ - WEB_ROOT=/app/public -USER umt + WEB_ROOT=/app/public \ + DATA_DIR=/data \ + PUID=10001 \ + PGID=10001 +# 以 root 启动只为修 /data 属主,entrypoint 随即 gosu 降权;api/worker/备份脚本都不以 root 运行 +VOLUME ["/data"] EXPOSE 4380 # 用二进制自带的 fetch 打本机回环端口,不装 curl -HEALTHCHECK --interval=15s --timeout=5s --start-period=20s --retries=3 CMD ["/app/umt-server", "--role", "probe"] +HEALTHCHECK --interval=15s --timeout=5s --start-period=30s --retries=3 CMD ["/app/umt-server", "--role", "probe"] ENTRYPOINT ["/app/entrypoint.sh"] +CMD ["serve"] diff --git a/README.md b/README.md index 564ff085..b21b92f7 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,20 @@ deno task dev # API http://localhost:4380 控制台 http://localhost:51 提交前:`deno task ci`(typecheck:tsc 7.0.2 + deno check → biome → vitest → check:tokens → web 构建 + server 编译)。 -生产/试运行:`docker compose up -d --build`(单容器 app + postgres,见 [docs/runbook/dev.md](docs/runbook/dev.md#docker-单容器部署))。 +## 部署 + +除 PostgreSQL 外全部在一个容器里(api + worker + web 静态,单端口 4380);compose 只有 `app` + `postgres`。 + +```sh +cp docker/compose.env.example .env # 填 SECRET_MASTER_KEY=$(openssl rand -hex 32)、POSTGRES_PASSWORD +docker compose up -d --build +docker compose logs app | grep 'setup token' # 首次初始化管理员 +docker compose exec app umt backup # 备份到 /data/backups/<时间戳>/ +``` + +- 应用数据卷 `/data`(Agent 会话、备份、导出);媒体库根与下载目录自行 bind mount,需对 `PUID:PGID`(缺省 10001)可读写。 +- `SECRET_MASTER_KEY` 不在备份里,和备份分开保管。 +- 详见 [docs/runbook/deploy.md](docs/runbook/deploy.md) 与 [docs/runbook/backup-restore.md](docs/runbook/backup-restore.md)。 更多见 [docs/runbook/dev.md](docs/runbook/dev.md)。 @@ -38,7 +51,8 @@ deno task dev # API http://localhost:4380 控制台 http://localhost:51 | `docs/design` | 设计系统规范 | | `docs/runbook` | 运行手册 | | `config/` | tsc 7 用的 tsconfig 基座(根目录不放 tsconfig.json,否则 `deno check` 丢 Deno 全局) | -| `docker/` | 容器入口脚本 | +| `docker/` | 容器入口脚本、compose 环境变量示例 | +| `scripts/` | 备份 / 恢复脚本(镜像内 `/app/scripts/`) | | `spikes/` | 独立兼容性验证(不属于 workspace) | 技术选型:Deno 2.9.7(锁定)+ TypeScript 7.0.2 strict,deno.json workspace(`nodeModulesDir: auto`,npm 依赖版本统一登记在根 `deno.json` 的 `imports`,`deno.lock` 锁定)。前端 Vite 7 + React Router 7 由 Deno 执行。决策见 [ADR-004](docs/adr/ADR-004-deno-runtime.md)(取代 ADR-001)。 diff --git a/apps/server/deno.json b/apps/server/deno.json index 3b8154cb..002bcc45 100644 --- a/apps/server/deno.json +++ b/apps/server/deno.json @@ -5,7 +5,7 @@ "tasks": { "dev": "deno run -A --watch --env-file=../../.env src/main.ts", "start": "deno run --allow-net --allow-env --allow-sys --allow-read --env-file=../../.env src/main.ts", - "compile": "deno compile --frozen --node-modules-dir=none --exclude-unused-npm --allow-net --allow-env --allow-sys --allow-read=/app --include ../../packages/db/migrations --output ../../dist/umt-server src/main.ts", + "compile": "deno compile --frozen --node-modules-dir=none --exclude-unused-npm --allow-net --allow-env --allow-sys --allow-read --allow-write --include ../../packages/db/migrations --output ../../dist/umt-server src/main.ts", "test": "deno run -A npm:vitest run", "typecheck": "deno run -A npm:typescript/tsc --noEmit -p tsconfig.json" } diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index 25425db1..81248435 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -1,3 +1,4 @@ +import { join, resolve } from "node:path"; import { z } from "zod"; const LogLevel = z.enum(["fatal", "error", "warn", "info", "debug", "trace", "silent"]); @@ -16,6 +17,13 @@ export const ConfigSchema = z.object({ WORKER_HEARTBEAT_MS: z.coerce.number().int().min(1000).default(30_000), /** web 静态产物目录(单容器模式):设置后同端口托管 SPA,非 /api 路径回退 index.html */ WEB_ROOT: z.string().min(1).optional(), + /** + * 应用数据目录(容器内 /data,由卷挂载):agent-sessions/、backups/、exports/、tmp/。 + * 编译后的二进制与 /app 保持只读,所有运行期写入都落在这里。相对路径按进程 cwd 解析。 + */ + DATA_DIR: z.string().min(1).default("./data"), + /** Agent JSONL 会话目录;缺省 DATA_DIR/agent-sessions(api 与 worker 必须共享同一目录) */ + AGENT_SESSION_DIR: z.string().min(1).optional(), }); export type Config = z.infer; @@ -29,4 +37,19 @@ export function loadConfig(env: Record = Deno.env.to return parsed.data; } +/** DATA_DIR 下的固定子目录 */ +export const DATA_SUBDIRS = ["agent-sessions", "backups", "exports", "tmp"] as const; + +/** 解析数据目录与会话目录为绝对路径 */ +export function dataPaths(c: Pick, cwd: string = Deno.cwd()) { + const dataDir = resolve(cwd, c.DATA_DIR); + return { + dataDir, + sessionDir: resolve(cwd, c.AGENT_SESSION_DIR ?? join(dataDir, "agent-sessions")), + backupsDir: join(dataDir, "backups"), + exportsDir: join(dataDir, "exports"), + tmpDir: join(dataDir, "tmp"), + }; +} + export const isDev = (c: Pick) => c.NODE_ENV !== "production"; diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index eb144934..f2d714bb 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -2,20 +2,36 @@ import { parseArgs } from "node:util"; import { createDb, migrateToLatest } from "@umt/db"; import { migrateEngineSchema } from "@umt/tasks"; import denoJson from "../deno.json" with { type: "json" }; -import { loadConfig } from "./config.ts"; +import { DATA_SUBDIRS, dataPaths, loadConfig } from "./config.ts"; import { bootstrap } from "./context.ts"; import { createApp } from "./http/app.ts"; import { PgEventBus } from "./http/sse-pg.ts"; import { createLogger } from "./logger.ts"; import { SERVER_MODULES } from "./modules/index.ts"; -import { runPreflight } from "./preflight.ts"; +import { diffCounts, enableReadOnlyReconcile, exportMappings, type MappingExport, writeMappingExport } from "./ops.ts"; +import { auditStorageWarnings, checkStorage, loadMountedPaths, runPreflight } from "./preflight.ts"; import { startWorker } from "./worker/index.ts"; -// api/worker/all 是常驻角色;migrate/preflight/probe 是容器入口脚本用的一次性子命令 -const Roles = ["api", "worker", "all", "migrate", "preflight", "probe"] as const; +// api/worker/all 是常驻角色;其余是容器入口 / 备份恢复脚本用的一次性子命令: +// migrate | preflight | probe +// export-mappings --out [--compare <基准清单.json>] 映射清单导出(可与基准比行数) +// read-only-on [--reason <文本>] 打开 automation.readOnlyReconcile +const Roles = ["api", "worker", "all", "migrate", "preflight", "probe", "export-mappings", "read-only-on"] as const; type Role = (typeof Roles)[number]; +const OneShot = ["migrate", "preflight", "export-mappings", "read-only-on"] as const; +type OneShotRole = (typeof OneShot)[number]; +const isOneShot = (r: Role): r is OneShotRole => (OneShot as readonly Role[]).includes(r); -const { values } = parseArgs({ args: Deno.args, options: { role: { type: "string", default: "all" } }, strict: true }); +const { values } = parseArgs({ + args: Deno.args, + options: { + role: { type: "string", default: "all" }, + out: { type: "string" }, + compare: { type: "string" }, + reason: { type: "string" }, + }, + strict: true, +}); const role = values.role as Role; if (!Roles.includes(role)) { console.error(`--role 必须是 ${Roles.join("|")}`); @@ -42,28 +58,20 @@ try { Deno.exit(2); } +// 运行期写入统一落在 DATA_DIR;Agent 会话目录经环境变量传给 agent 模块(它只读 AGENT_SESSION_DIR) +const paths = dataPaths(config); +Deno.env.set("AGENT_SESSION_DIR", paths.sessionDir); + const logger = createLogger({ level: config.LOG_LEVEL, pretty: config.NODE_ENV === "development" && Deno.stdout.isTerminal(), }); const db = createDb({ connectionString: config.DATABASE_URL }); -if (role === "migrate" || role === "preflight") { +if (isOneShot(role)) { let code = 0; try { - if (role === "migrate") { - const { results } = await migrateToLatest(db); - for (const r of results ?? []) logger.info({ migration: r.migrationName, status: r.status }, "migration"); - logger.info({ applied: results?.length ?? 0 }, "迁移完成"); - // 任务引擎自己的表(openworkflow schema);api/worker 进程启动时不再迁移 - await migrateEngineSchema(config.DATABASE_URL); - logger.info("任务引擎迁移完成"); - } else { - const problems = await runPreflight({ config, db }); - for (const p of problems) logger.error({ check: p.check }, p.message); - if (problems.length) code = 1; - else logger.info("启动自检通过"); - } + code = await runOneShot(role); } catch (err) { logger.error({ err }, `${role} 失败`); code = 1; @@ -73,6 +81,62 @@ if (role === "migrate" || role === "preflight") { Deno.exit(code); } +async function runOneShot(r: OneShotRole): Promise { + switch (r) { + case "migrate": { + const { results } = await migrateToLatest(db); + for (const m of results ?? []) logger.info({ migration: m.migrationName, status: m.status }, "migration"); + logger.info({ applied: results?.length ?? 0 }, "迁移完成"); + // 任务引擎自己的表(openworkflow schema);api/worker 进程启动时不再迁移 + await migrateEngineSchema(config.DATABASE_URL); + logger.info("任务引擎迁移完成"); + return 0; + } + case "preflight": { + const problems = await runPreflight({ config, db }); + // 库/下载目录由用户挂载:不可读写只告警(并写审计),不阻止启动;数据目录不可写才阻止 + const mounts = problems.some((p) => p.check === "db") ? [] : await loadMountedPaths(db); + const storage = await checkStorage({ ...paths, subdirs: DATA_SUBDIRS, mounts }); + problems.push(...storage.problems); + for (const w of storage.warnings) { + logger.warn({ kind: w.kind, id: w.id, path: w.path }, `挂载路径不可用: ${w.name} ${w.reason}`); + } + await auditStorageWarnings(db, storage.warnings).catch((err) => logger.warn({ err }, "存储告警写审计失败")); + for (const p of problems) logger.error({ check: p.check }, p.message); + if (problems.length) return 1; + logger.info( + { + dataDir: paths.dataDir, + sessionDir: paths.sessionDir, + mounts: mounts.length, + warnings: storage.warnings.length, + }, + "启动自检通过", + ); + return 0; + } + case "export-mappings": { + const out = values.out ?? `${paths.exportsDir}/mappings-${new Date().toISOString().replace(/[:.]/g, "-")}.json`; + const data = await exportMappings(db, denoJson.version); + await writeMappingExport(out, data); + logger.info({ out, counts: data.counts }, "映射清单已导出"); + if (!values.compare) return 0; + const base = JSON.parse(await Deno.readTextFile(values.compare)) as MappingExport; + const diff = diffCounts(base.counts, data.counts); + if (diff.length) { + logger.error({ diff }, "映射清单行数与基准不一致"); + return 3; + } + logger.info({ compare: values.compare }, "映射清单行数与基准一致"); + return 0; + } + case "read-only-on": + await enableReadOnlyReconcile(db, values.reason ?? "restore"); + logger.info("已打开只读对账模式 automation.readOnlyReconcile=true"); + return 0; + } +} + // SSE 总线:sse_event 表 + LISTEN/NOTIFY,api 与 worker 分进程时事件互相可见 let bus: PgEventBus; try { diff --git a/apps/server/src/ops.ts b/apps/server/src/ops.ts new file mode 100644 index 00000000..fd67cc0a --- /dev/null +++ b/apps/server/src/ops.ts @@ -0,0 +1,153 @@ +import { dirname } from "node:path"; +import type { DB, Kysely } from "@umt/db"; +import { sql } from "@umt/db"; + +// 运维一次性子命令(备份/恢复脚本调用):映射清单导出、恢复后打开只读对账。 +// 映射清单是"路径不携带完整身份"前提下的关键资产(需求 §8.5):数据库之外再留一份可读 JSON, +// 用于人工核对与恢复演练的行数校验;不含任何密钥(secret 表只随 pg_dump 走,主密钥永不入备份)。 + +/** 各表导出的关键列 */ +export const MAPPING_TABLES = { + library_root: ["id", "name", "media_kind", "local_path", "management_mode", "enabled", "sentinel_file"], + path_mapping: [ + "id", + "view_kind", + "downloader_instance_id", + "media_server_instance_id", + "library_root_id", + "remote_prefix", + "local_prefix", + "priority", + "enabled", + ], + media_entity: ["id", "kind", "title", "original_title", "year", "status", "merged_into_id", "updated_at"], + external_identity: [ + "id", + "media_entity_id", + "content_unit_id", + "provider", + "entity_type", + "external_id", + "status", + "evidence_level", + "locked", + "revision", + ], + path_binding: [ + "id", + "media_entity_id", + "content_unit_id", + "season_number", + "binding_kind", + "library_root_id", + "relative_path", + "template_version", + "status", + "superseded_by_id", + ], + asset_location: [ + "id", + "media_asset_id", + "location_kind", + "library_root_id", + "downloader_instance_id", + "relative_path", + "status", + "is_primary", + "size_bytes", + "inode", + "device", + "mtime", + ], + acquisition_slot: [ + "id", + "media_entity_id", + "content_unit_id", + "subscription_id", + "policy_revision_id", + "name", + "status", + "current_asset_id", + "revision", + ], + organization_record: [ + "id", + "media_entity_id", + "content_unit_id", + "download_task_id", + "source_downloader_instance_id", + "source_path", + "target_library_root_id", + "target_path", + "action", + "status", + "plan_fingerprint", + "finished_at", + ], +} as const satisfies Partial>; + +export type MappingTable = keyof typeof MAPPING_TABLES; + +export interface MappingExport { + format: "umt-mappings/1"; + exportedAt: string; + appVersion: string; + counts: Record; + tables: Record[]>; +} + +/** 按主键顺序导出映射清单 */ +export async function exportMappings(db: Kysely, appVersion: string): Promise { + const counts = {} as Record; + const tables = {} as Record[]>; + for (const [table, cols] of Object.entries(MAPPING_TABLES) as [MappingTable, readonly string[]][]) { + const { rows } = await sql< + Record + >`select ${sql.join(cols.map((c) => sql.ref(c)))} from ${sql.table(table)} order by id`.execute(db); + tables[table] = rows; + counts[table] = rows.length; + } + return { format: "umt-mappings/1", exportedAt: new Date().toISOString(), appVersion, counts, tables }; +} + +/** 写 JSON(先写临时文件再改名,避免半截文件) */ +export async function writeMappingExport(path: string, data: MappingExport): Promise { + await Deno.mkdir(dirname(path), { recursive: true }); + const tmp = `${path}.partial`; + await Deno.writeTextFile(tmp, `${JSON.stringify(data, null, 2)}\n`); + await Deno.rename(tmp, path); +} + +/** 两份清单逐表比行数,返回不一致的表 */ +export function diffCounts( + expected: Record, + actual: Record, +): Array<{ table: string; expected: number; actual: number }> { + const out: Array<{ table: string; expected: number; actual: number }> = []; + for (const table of new Set([...Object.keys(expected), ...Object.keys(actual)])) { + const e = expected[table] ?? -1; + const a = actual[table] ?? -1; + if (e !== a) out.push({ table, expected: e, actual: a }); + } + return out; +} + +/** 打开 automation.readOnlyReconcile(保留该分组其它字段)并写审计 */ +export async function enableReadOnlyReconcile(db: Kysely, reason: string): Promise { + await db.transaction().execute(async (trx) => { + await sql`insert into settings (key, value, updated_at) + values ('automation', ${JSON.stringify({ readOnlyReconcile: true })}::jsonb, now()) + on conflict (key) do update set value = settings.value || excluded.value, updated_at = now()`.execute(trx); + await trx + .insertInto("audit_event") + .values({ + actor_type: "system", + actor_id: "restore", + action: "ops.read_only_reconcile_on", + target_type: "settings", + target_id: "automation", + payload: JSON.stringify({ reason }), + }) + .execute(); + }); +} diff --git a/apps/server/src/preflight.ts b/apps/server/src/preflight.ts index 7dd3feda..b2abc473 100644 --- a/apps/server/src/preflight.ts +++ b/apps/server/src/preflight.ts @@ -43,3 +43,107 @@ export async function runPreflight({ config, db, uid = Deno.uid() }: PreflightDe } return problems; } + +// ── 存储自检:数据目录必须可写;库根 / 下载器映射路径不可写只告警 ── + +export interface MountedPath { + kind: "library_root" | "path_mapping"; + id: string; + name: string; + path: string; +} + +export interface StorageWarning extends MountedPath { + reason: string; +} + +export interface StorageCheckResult { + problems: PreflightProblem[]; + warnings: StorageWarning[]; +} + +const errText = (e: unknown) => (e as { code?: string }).code ?? (e as Error).message; + +/** 在目录里写一个探针文件再删掉;失败抛错 */ +async function probeWrite(dir: string): Promise { + const probe = join(dir, `.umt-write-probe-${Deno.pid}-${crypto.randomUUID().slice(0, 8)}`); + await Deno.writeTextFile(probe, "probe\n"); + await Deno.remove(probe); +} + +/** 读出已启用的库根与下载器路径映射在本进程视角下的路径 */ +export async function loadMountedPaths(db: Kysely): Promise { + const roots = await db + .selectFrom("library_root") + .select(["id", "name", "local_path"]) + .where("enabled", "=", true) + .execute(); + const maps = await db + .selectFrom("path_mapping") + .select(["id", "remote_prefix", "local_prefix"]) + .where("enabled", "=", true) + .where("view_kind", "=", "downloader") + .execute(); + return [ + ...roots.map((r) => ({ kind: "library_root" as const, id: r.id, name: r.name, path: r.local_path })), + ...maps.map((m) => ({ kind: "path_mapping" as const, id: m.id, name: m.remote_prefix, path: m.local_prefix })), + ]; +} + +/** + * dataDir 下建 agent-sessions/backups/exports/tmp 并逐个探测可写(失败 = problem,阻止启动); + * sessionDir 若在 dataDir 之外也单独探测;mounts 逐个检查可读可写(失败 = warning,不阻止启动)。 + */ +export async function checkStorage(opts: { + dataDir: string; + sessionDir: string; + subdirs: readonly string[]; + mounts: MountedPath[]; +}): Promise { + const problems: PreflightProblem[] = []; + const warnings: StorageWarning[] = []; + const dirs = [...opts.subdirs.map((s) => join(opts.dataDir, s))]; + if (!dirs.includes(opts.sessionDir)) dirs.push(opts.sessionDir); + for (const dir of dirs) { + try { + await Deno.mkdir(dir, { recursive: true }); + await probeWrite(dir); + } catch (e) { + problems.push({ check: "data_dir", message: `数据目录不可写: ${dir}(${errText(e)});检查卷挂载与 PUID/PGID` }); + } + } + for (const m of opts.mounts) { + try { + const st = await Deno.stat(m.path); + if (!st.isDirectory) throw Object.assign(new Error("不是目录"), { code: "ENOTDIR" }); + for await (const _ of Deno.readDir(m.path)) break; + } catch (e) { + warnings.push({ ...m, reason: `不可读: ${errText(e)}` }); + continue; + } + try { + await probeWrite(m.path); + } catch (e) { + warnings.push({ ...m, reason: `不可写: ${errText(e)}` }); + } + } + return { problems, warnings }; +} + +/** 存储告警写审计(actor=system),失败不影响启动 */ +export async function auditStorageWarnings(db: Kysely, warnings: StorageWarning[]): Promise { + if (!warnings.length) return; + await db + .insertInto("audit_event") + .values( + warnings.map((w) => ({ + actor_type: "system", + actor_id: "preflight", + action: "preflight.path_unavailable", + target_type: w.kind, + target_id: w.id, + payload: JSON.stringify({ name: w.name, path: w.path, reason: w.reason, uid: Deno.uid() }), + })), + ) + .execute(); +} diff --git a/apps/server/test/preflight.test.ts b/apps/server/test/preflight.test.ts index 86aef0c9..13e3cfd5 100644 --- a/apps/server/test/preflight.test.ts +++ b/apps/server/test/preflight.test.ts @@ -1,10 +1,11 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { DB, Kysely } from "@umt/db"; import { afterAll, describe, expect, it } from "vitest"; -import { type Config, ConfigSchema } from "../src/config.ts"; -import { runPreflight } from "../src/preflight.ts"; +import { type Config, ConfigSchema, DATA_SUBDIRS, dataPaths } from "../src/config.ts"; +import { diffCounts } from "../src/ops.ts"; +import { checkStorage, runPreflight } from "../src/preflight.ts"; const dir = mkdtempSync(join(tmpdir(), "umt-pre-")); afterAll(() => rmSync(dir, { recursive: true, force: true })); @@ -45,3 +46,59 @@ describe("启动自检", () => { expect(await runPreflight({ config, db: fakeDb(true), uid: 1000 })).toEqual([]); }); }); + +describe("存储自检", () => { + it("DATA_DIR 缺省为 ./data,会话目录缺省在其下", () => { + const p = dataPaths(cfg({}), "/srv/app"); + expect(p.dataDir).toBe("/srv/app/data"); + expect(p.sessionDir).toBe("/srv/app/data/agent-sessions"); + expect(dataPaths(cfg({ DATA_DIR: "/data", AGENT_SESSION_DIR: "/x/s" })).sessionDir).toBe("/x/s"); + }); + + it("建数据子目录并探测可写;挂载路径不存在/不可写只告警", async () => { + const dataDir = join(dir, "data"); + const ok = join(dir, "lib-ok"); + const ro = join(dir, "lib-ro"); + mkdirSync(ok); + mkdirSync(ro); + chmodSync(ro, 0o555); + const isRoot = Deno.uid() === 0; + const { problems, warnings } = await checkStorage({ + dataDir, + sessionDir: join(dataDir, "agent-sessions"), + subdirs: DATA_SUBDIRS, + mounts: [ + { kind: "library_root", id: "1", name: "ok", path: ok }, + { kind: "library_root", id: "2", name: "ro", path: ro }, + { kind: "path_mapping", id: "3", name: "/downloads", path: join(dir, "missing") }, + ], + }); + expect(problems).toEqual([]); + expect(readdirSync(dataDir).sort()).toEqual([...DATA_SUBDIRS].sort()); + // 探针文件已删除 + expect(readdirSync(ok)).toEqual([]); + expect(warnings.map((w) => w.id)).toEqual(isRoot ? ["3"] : ["2", "3"]); + chmodSync(ro, 0o755); + }); + + it("数据目录不可写 → problem", async () => { + if (Deno.uid() === 0) return; // root 无视权限位 + const locked = join(dir, "locked"); + mkdirSync(locked); + chmodSync(locked, 0o555); + const { problems } = await checkStorage({ + dataDir: join(locked, "data"), + sessionDir: join(locked, "data", "agent-sessions"), + subdirs: DATA_SUBDIRS, + mounts: [], + }); + expect(problems.every((p) => p.check === "data_dir")).toBe(true); + expect(problems.length).toBe(DATA_SUBDIRS.length); + chmodSync(locked, 0o755); + }); + + it("映射清单行数比对", () => { + expect(diffCounts({ a: 1, b: 2 }, { a: 1, b: 2 })).toEqual([]); + expect(diffCounts({ a: 1, b: 2 }, { a: 1, b: 3 })).toEqual([{ table: "b", expected: 2, actual: 3 }]); + }); +}); diff --git a/docker-compose.yml b/docker-compose.yml index 0fbb2bc9..4ca40e4c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,8 @@ -# 生产/试运行部署:app(单容器:API + worker + web 静态)+ postgres 两个服务。 -# 用法: -# export SECRET_MASTER_KEY=$(openssl rand -hex 32) POSTGRES_PASSWORD=$(openssl rand -hex 16) +# 生产/试运行部署:只有两个服务 —— app(单容器:API + worker + web 静态,单端口 4380)+ postgres。 +# 用法(详见 docs/runbook/deploy.md): +# cp docker/compose.env.example .env # 按注释生成 SECRET_MASTER_KEY / POSTGRES_PASSWORD # docker compose up -d --build +# docker compose logs app | grep -i setup # 首次启动打印一次性 setup token # 开发只起数据库用 docker-compose.dev.yml。 name: umt @@ -26,6 +27,7 @@ services: interval: 5s timeout: 3s retries: 20 + stop_grace_period: 30s logging: *logging app: @@ -33,6 +35,7 @@ services: context: . args: APP_VERSION: ${APP_VERSION:-} + APT_MIRROR: ${APT_MIRROR:-} image: uhd-media-tool:${APP_VERSION:-latest} restart: unless-stopped depends_on: @@ -41,12 +44,23 @@ services: environment: NODE_ENV: production DATABASE_URL: postgres://umt:${POSTGRES_PASSWORD}@postgres:5432/umt - SECRET_MASTER_KEY: ${SECRET_MASTER_KEY:?请设置 SECRET_MASTER_KEY(至少 32 字节随机串)} + SECRET_MASTER_KEY: ${SECRET_MASTER_KEY:?请设置 SECRET_MASTER_KEY(openssl rand -hex 32)} LOG_LEVEL: ${LOG_LEVEL:-info} + # 运行用户:DATA_DIR 由 entrypoint chown 成它;媒体库/下载目录需对它可读写 + PUID: ${PUID:-10001} + PGID: ${PGID:-10001} + BACKUP_KEEP: ${BACKUP_KEEP:-14} ports: - - "${UMT_PORT:-4380}:4380" + - "${UMT_BIND:-0.0.0.0}:${UMT_PORT:-4380}:4380" + volumes: + # 应用数据:agent-sessions/ backups/ exports/ tmp/ + - umt-data:/data + # 媒体库根与下载缓存目录按需挂载(容器内路径 = 控制台里登记的库根 / 路径映射 local 前缀): + # - /srv/media:/media + # - /srv/downloads:/downloads stop_grace_period: 15s logging: *logging volumes: umt-pg-data: + umt-data: diff --git a/docker/compose.env.example b/docker/compose.env.example new file mode 100644 index 00000000..45434508 --- /dev/null +++ b/docker/compose.env.example @@ -0,0 +1,22 @@ +# docker compose 用的环境变量(复制为部署目录下的 .env)。 +# +# SECRET_MASTER_KEY:凭证加密主密钥,生成一次后永不更换(换了之前存的站点/下载器/模型密钥全部无法解密)。 +# 生成:openssl rand -hex 32 +# 必须与数据库备份分开保管(密码管理器 / 离线介质);备份里不含它,丢了它备份里的密钥也无法恢复。 +SECRET_MASTER_KEY= +# 数据库密码:openssl rand -hex 16 +POSTGRES_PASSWORD= + +# 对外端口与绑定地址(放在反代后面时可改成 127.0.0.1) +UMT_PORT=4380 +UMT_BIND=0.0.0.0 + +# 容器运行用户:与宿主媒体目录属主一致,例如 id -u / id -g 的输出 +PUID=10001 +PGID=10001 + +# 备份保留份数(0 = 不轮转) +BACKUP_KEEP=14 +LOG_LEVEL=info +# APP_VERSION=v0.1.0 +# 国内网络构建换源(可选):APT_MIRROR=mirrors.ustc.edu.cn diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 741c8ac2..59bf275d 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,9 +1,42 @@ #!/usr/bin/env bash -# 容器入口:迁移 → 启动自检 → api 与 worker 两个进程互相看护。 -# 任一进程退出就停掉另一个并以非 0 退出,把"半死容器"变成整体退出,交给 restart 策略自愈。 +# 容器入口。 +# root 启动:建好 DATA_DIR 子目录、按 PUID/PGID 修正属主,再用 gosu 降权重跑自己(二进制与 /app 保持 root 只读)。 +# serve(缺省):迁移 → 启动自检 → api 与 worker 两个进程互相看护。 +# 任一进程退出就停掉另一个并以非 0 退出,把"半死容器"变成整体退出,交给 restart 策略自愈。 +# backup / restore <备份目录> / migrate / preflight / export-mappings ...:一次性子命令(降权后执行)。 +# 其它参数原样 exec(例如 bash)。 set -euo pipefail BIN=/app/umt-server +DATA_DIR="${DATA_DIR:-/data}" +PUID="${PUID:-10001}" +PGID="${PGID:-10001}" +export DATA_DIR TMPDIR="${TMPDIR:-$DATA_DIR/tmp}" + +if [ "$(id -u)" = "0" ]; then + if [ "$PUID" = "0" ]; then + echo "entrypoint: 不允许 PUID=0(生产环境禁止以 root 运行)" >&2 + exit 2 + fi + mkdir -p "$DATA_DIR"/{agent-sessions,backups,exports,tmp} + if [ -n "${AGENT_SESSION_DIR:-}" ]; then mkdir -p "$AGENT_SESSION_DIR"; fi + # 只在属主不对时递归 chown,避免每次启动扫全卷;媒体库/下载目录是用户挂载,不碰 + for d in "$DATA_DIR" ${AGENT_SESSION_DIR:+"$AGENT_SESSION_DIR"}; do + if [ "$(stat -c %u:%g "$d")" != "$PUID:$PGID" ] || [ -n "$(find "$d" \( ! -user "$PUID" -o ! -group "$PGID" \) -print -quit)" ]; then + chown -R "$PUID:$PGID" "$d" + fi + done + exec gosu "$PUID:$PGID" "$0" "$@" +fi + +cmd="${1:-serve}" +case "$cmd" in + serve) ;; + backup) shift; exec /app/scripts/backup.sh "$@" ;; + restore) shift; exec /app/scripts/restore.sh "$@" ;; + migrate | preflight | export-mappings | read-only-on) shift; exec "$BIN" --role "$cmd" "$@" ;; + *) exec "$@" ;; +esac "$BIN" --role migrate "$BIN" --role preflight diff --git a/docs/runbook/backup-restore.md b/docs/runbook/backup-restore.md new file mode 100644 index 00000000..a1738e35 --- /dev/null +++ b/docs/runbook/backup-restore.md @@ -0,0 +1,81 @@ +# 备份与恢复运行手册(W4-B) + +需求 §8.5:路径不携带完整身份,数据库里的映射(作品 ↔ 外部身份 ↔ 路径绑定 ↔ 资产位置 ↔ Slot ↔ 整理记录)就是关键资产。 +只靠重新扫描目录恢复不了 Slot、审批、人工纠错和历史关系,所以备份以整库为准,映射清单做人工可读的副本与恢复校验。 + +## 备份内容 + +`scripts/backup.sh`(镜像内 `/app/scripts/backup.sh`,入口命令 `umt backup`)在 `DATA_DIR/backups//` 下生成: + +| 文件 | 内容 | +|---|---| +| `db.dump` | `pg_dump --format=custom` 整库:业务表、`secret` 表(密文)、任务引擎 schema、审计 | +| `mappings.json` | `--role export-mappings` 导出的映射清单:`library_root`、`path_mapping`、`media_entity`、`external_identity`、`path_binding`、`asset_location`、`acquisition_slot`、`organization_record` 的关键列 + 每表行数(`counts`) | +| `agent-sessions.tar.gz` | Agent Pi JSONL 会话(有才打包) | +| `manifest.txt` | 时间、pg_dump 版本、服务端版本、"主密钥未包含"标记 | +| `SHA256SUMS` | 以上文件校验和,恢复前校验 | + +先写到 `.<时间戳>.partial/`,全部成功才改名,不会留下半截备份。`BACKUP_KEEP=N`(compose 缺省 14)时只保留最近 N 份。 + +**SECRET_MASTER_KEY 不在备份里。** 它和备份必须分开保管:备份泄露时密文仍然安全;主密钥丢失时备份里的站点/下载器/模型凭证无法解密(需要重新录入)。 +恢复到新机器时,`.env` 里的 `SECRET_MASTER_KEY` 必须与备份时一致。 + +## 日常备份 + +```sh +docker compose exec app umt backup # 在线备份(pg_dump 一致性快照,不停服) +docker compose exec app ls /data/backups +# 拷到机器外(示例): +docker compose cp app:/data/backups/20260927T120000Z ./offsite/ +``` + +定时:宿主 cron `0 4 * * * cd /opt/umt && docker compose exec -T app umt backup >> /var/log/umt-backup.log 2>&1`。 +单独导出映射清单:`docker compose exec app umt export-mappings`(写到 `/data/exports/`)。 + +## 恢复 + +恢复会删除并重建备份里的全部对象。**先停 app**,脚本检测到库里还有其它连接会拒绝(`--force` 可跳过,不建议)。 + +```sh +docker compose stop app +docker compose run --rm --no-deps app restore 20260927T120000Z --yes # 或绝对路径 /data/backups/... +docker compose up -d app +``` + +`restore.sh` 依次: + +1. 校验 `SHA256SUMS`; +2. `pg_restore --clean --if-exists --no-owner` 生成 SQL,经 `psql --single-transaction -v ON_ERROR_STOP=1` 回放(失败整体回滚,库保持恢复前状态)。镜像里是 17 版客户端,回放前去掉 16 版服务端不认识的 `SET transaction_timeout`; +3. 还原 `agent-sessions.tar.gz`; +4. `--role read-only-on`:把 `automation.readOnlyReconcile` 置为 true(保留 automation 其它字段),写审计 `ops.read_only_reconcile_on`; +5. 重新导出映射清单到 `DATA_DIR/tmp/restore-check-*.json`,逐表与备份里的 `counts` 比对,不一致以退出码 3 结束; +6. 打印"恢复后先对账再关闭只读"提示。 + +从别的机器恢复:把备份目录拷进目标机器的 `umt-data` 卷(`docker compose cp ./20260927T120000Z app:/data/backups/`), +`.env` 用原来的 `SECRET_MASTER_KEY`,空库上照上面的命令执行。备份早于当前镜像版本时,`up -d app` 会自动跑后续迁移。 + +## 恢复后:先对账,再关闭只读 + +只读对账模式下,自动下载提交、自动整理、订阅调度与 Agent/API Key 的自动执行一律拒绝(控制台首页有提示),查询和人工操作不受影响。 + +1. 库根:设置 → 媒体库根 → 逐个"测试"(哨兵文件在、目录可写)。 +2. 下载器:下载页核对 qB 里的任务与库里的下载任务(托管标签)一致;路径映射"验证"。 +3. 整理记录 / 资产位置:抽查最近的整理记录,目标文件确实存在。 +4. 凭证:打开一个站点/下载器"测试",能通过说明主密钥一致。 +5. 确认无误后关闭:设置 → 自动化 → 关闭"只读对账",或 + `PATCH /api/v1/settings {"automation":{"readOnlyReconcile":false}}`。 + +## 开发机上用 + +脚本不依赖容器,只要有 `pg_dump/pg_restore/psql`: + +```sh +cd apps/server && deno task compile && cd ../.. +set -a; . ./.env; set +a +DATA_DIR=./data UMT_BIN=dist/umt-server scripts/backup.sh +DATA_DIR=./data UMT_BIN=dist/umt-server scripts/restore.sh <时间戳> --yes +``` + +## 验证记录 + +见 [deploy.md#验证记录](deploy.md#验证记录)(同一次演练)。 diff --git a/docs/runbook/deploy.md b/docs/runbook/deploy.md new file mode 100644 index 00000000..84fef91a --- /dev/null +++ b/docs/runbook/deploy.md @@ -0,0 +1,148 @@ +# 部署运行手册(W4-B) + +形态:**除 PostgreSQL 外全部在一个容器里**。镜像 `uhd-media-tool` 内一个 `deno compile` 出的二进制 `umt-server`, +容器里同时跑 api 与 worker 两个进程,api 同端口托管 web 静态产物(`WEB_ROOT=/app/public`),对外只有 **4380** 一个端口。 +compose 只有 `app` + `postgres` 两个服务。 + +## 1. 构建镜像 + +```sh +docker build -t uhd-media-tool:latest . # 或 docker compose build +APP_VERSION=v0.1.0 docker compose build # 版本号写进 web 的 version.json +``` + +- 多阶段:`denoland/deno:2.9.7`(装依赖 → web 构建 → server 编译)→ `debian:trixie-slim` 运行镜像。 +- 运行镜像只额外装 `gosu`(降权)与 `postgresql-client`(备份/恢复用 pg_dump/pg_restore/psql,17 版客户端兼容 16 版服务端)。 +- `/app` 下的二进制、静态产物、脚本全部 root 属主只读;运行用户改不了它们。 +- 二进制编译时固定权限:`--allow-net --allow-env --allow-sys --allow-read --allow-write`,不给 `--allow-run` + (库根/下载目录由用户挂载,位置不固定,所以读写不按路径限制;写保护靠只读 `/app` + 非 root 运行)。 + +## 2. 启动 + +```sh +mkdir -p /opt/umt && cd /opt/umt # 部署目录(放 docker-compose.yml 与 .env) +cp <仓库>/docker/compose.env.example .env +sed -i "s/^SECRET_MASTER_KEY=.*/SECRET_MASTER_KEY=$(openssl rand -hex 32)/" .env +sed -i "s/^POSTGRES_PASSWORD=.*/POSTGRES_PASSWORD=$(openssl rand -hex 16)/" .env +docker compose up -d --build +docker compose ps # app 变成 healthy 即可访问 +curl -s localhost:4380/api/v1/health +``` + +**SECRET_MASTER_KEY** 生成一次后不再更换:站点 Cookie、下载器密码、模型 API Key 都用它加密存在 `secret` 表。 +把它存进密码管理器或离线介质,**和数据库备份分开保管**;备份里不含它,丢了它备份中的凭证无法解密。 + +容器启动顺序(`docker/entrypoint.sh`): + +1. 以 root 启动:建 `DATA_DIR`(`/data`)下的 `agent-sessions/ backups/ exports/ tmp/`,属主不是 `PUID:PGID` 时 `chown -R`, + 然后 `gosu PUID:PGID` 降权重跑自己。`PUID=0` 直接拒绝。 +2. `--role migrate`:业务表迁移 + 任务引擎 schema 迁移。 +3. `--role preflight`:密钥不是占位符、非 root、数据库可达、`WEB_ROOT/index.html` 存在、**DATA_DIR 各子目录与 Agent 会话目录可写** + (失败即退出);再逐个检查已启用的**库根**与**下载器路径映射**(`view_kind=downloader` 的 local 前缀)可读可写—— + 这些是用户挂载的目录,不可用只打 `warn` 日志并写审计 `preflight.path_unavailable`,不阻止启动。 +4. 同时起 `--role api` 与 `--role worker`;任一退出就停掉另一个并以非 0 退出,由 `restart: unless-stopped` 拉起。 + +健康检查:镜像 `HEALTHCHECK` 用二进制自带 fetch 打 `127.0.0.1:4380/api/v1/health`(`--role probe`),不装 curl。 + +## 3. 首次初始化管理员 + +api 首次启动时在日志里打印一次性 setup token(重启会换新的,旧的作废): + +```sh +docker compose logs app | grep 'setup token' +``` + +浏览器打开 `http://<主机>:4380`,登录页进入初始化模式,填 token、用户名、密码(≥10 字符)。 +命令行等价:`POST /api/v1/auth/setup {setupToken, username, password}`。之后在 设置 → API Key 创建给外部程序用的 Key。 + +## 4. 卷与路径约定 + +| 容器内路径 | 来源 | 用途 | 权限要求 | +|---|---|---|---| +| `/data` | 命名卷 `umt-data`(或宿主目录) | `agent-sessions/`(Pi JSONL 会话,崩溃恢复点)、`backups/`、`exports/`(映射清单)、`tmp/`(`TMPDIR`) | entrypoint 自动 chown 给 `PUID:PGID` | +| `/media/...` | 用户 bind mount | 媒体库根:控制台 设置 → 媒体库根 登记的 `localPath` | 需对 `PUID:PGID` 可读写;**不会被自动 chown** | +| `/downloads/...` | 用户 bind mount | 下载缓存:路径映射的 local 前缀(下载器报告的 remote 路径 → 容器内路径) | 同上 | +| `/var/lib/postgresql/data`(postgres) | 命名卷 `umt-pg-data` | 数据库 | — | + +- 媒体目录在 `docker-compose.yml` 的 `app.volumes` 里取消注释或另写 `docker-compose.override.yml`: + + ```yaml + services: + app: + volumes: + - /srv/media:/media + - /srv/downloads:/downloads + ``` + +- bind mount 的相对路径按**第一个 compose 文件所在目录**解析,不是按当前目录;不确定时写绝对路径。 + 源目录不存在时 Docker 会以 root 属主新建它,库根登记就会因哨兵写不进而失败。 +- 路径映射:下载器容器里看到的是 `/downloads/complete/...`,本容器也挂成 `/downloads` 时 remote 前缀与 local 前缀相同; + 挂到别处就登记 `remote_prefix=/downloads`、`local_prefix=<本容器内路径>`。硬链接要求下载目录与库根在同一文件系统、同一挂载点内。 +- 登记库根时服务端会在目录里写哨兵文件 `.uhd-root`,写不进去直接 400(提示检查挂载与属主)。 +- **PUID/PGID**:设成宿主媒体目录的属主(`stat -c '%u:%g' /srv/media`)。改 PUID/PGID 后重启容器,entrypoint 会把 `/data` 重新 chown。 + 也可以 `docker run --user` 直接指定(此时跳过 chown,`/data` 需事先可写)。 +- Agent 会话目录缺省 `DATA_DIR/agent-sessions`,可用 `AGENT_SESSION_DIR` 覆盖(api 与 worker 在同一容器里,天然共享)。 + +## 5. 常用环境变量 + +| 变量 | 缺省 | 说明 | +|---|---|---| +| `SECRET_MASTER_KEY` | 必填 | 凭证加密主密钥,`openssl rand -hex 32` | +| `POSTGRES_PASSWORD` | 必填 | compose 拼进 `DATABASE_URL` | +| `UMT_PORT` / `UMT_BIND` | `4380` / `0.0.0.0` | 宿主映射端口与绑定地址;放在反代后面时绑 `127.0.0.1` | +| `PUID` / `PGID` | `10001` | 运行用户 | +| `DATA_DIR` | `/data` | 应用数据目录(开发机缺省 `./data`) | +| `AGENT_SESSION_DIR` | `DATA_DIR/agent-sessions` | Agent 会话目录 | +| `BACKUP_KEEP` | `14` | 备份保留份数,0 不轮转 | +| `LOG_LEVEL` | `info` | pino 级别 | + +## 6. 升级 + +```sh +cd <仓库> && git pull +docker compose exec app umt backup # 升级前先备份 +docker compose up -d --build app # 重建镜像并替换容器;启动时自动迁移 +docker compose ps && docker compose logs --tail=50 app +``` + +迁移只前滚。升级失败要回退时:用旧镜像 + 升级前的备份恢复(见 [backup-restore.md](backup-restore.md)),不要用旧镜像直接连新库。 + +## 7. 日志与排障 + +- `docker compose logs -f app`:pino JSON,一行一条;`role` 字段区分 api / worker。json-file 驱动,单文件 10 MB × 5 份。 +- entrypoint 自身的输出(`entrypoint: ...`)说明哪个进程先退出、退出码多少。 +- 自检失败(`check=data_dir`):`/data` 卷不可写,检查是否挂了只读卷或 `--user` 与卷属主不一致。 +- 挂载路径告警(`挂载路径不可用`):看审计页 `preflight.path_unavailable` 事件,核对 bind mount 与 PUID/PGID。 +- 进容器排查:`docker compose exec app bash`(root);以运行用户执行命令用 `gosu 10001:10001 `。 +- 一次性子命令:`docker compose exec app umt migrate|preflight|export-mappings|backup`、 + `docker compose run --rm --no-deps app restore <备份>`。 + +## 验证记录 + +2026-09-27,W4-B,Docker 29 / compose v2,在干净 worktree(HEAD 8c6a922 + 本次改动)构建, +临时项目名 `umtw4b`、app 端口 127.0.0.1:4390、pg 端口 127.0.0.1:55440,`/media` 绑定一个属主 10001 的临时目录。 + +| 步骤 | 结果 | +|---|---| +| `docker build --build-arg APT_MIRROR=mirrors.ustc.edu.cn` | 成功(缓存命中 2 分 24 秒);镜像 489 MB(压缩后 107 MB):二进制 213 MB、gosu+postgresql-client 79 MB、web 2 MB | +| `compose up -d` | postgres healthy 后 app 启动,约 6 秒 healthy;21 个迁移 + 引擎迁移,`启动自检通过 dataDir=/data sessionDir=/data/agent-sessions` | +| 进程 | 两个 `umt-server`(api、worker)uid/gid 均为 10001;`/data/*` 属主 10001;`/app/umt-server` 为 `r-xr-xr-x root`,运行用户写入 → Permission denied | +| `GET /` 与 `/console/settings` | 200,SPA 由同端口托管 | +| setup token(日志)→ `POST /auth/setup` → `POST /auth/login` | 成功,拿到会话与 csrf cookie | +| 会话 + csrf `POST /api-keys {scopes:[admin]}` | 返回 `umt_…` 明文一次 | +| API Key `GET /organization-records` | 200 | +| API Key `POST /library-roots {localPath:/media}` | 成功,宿主目录出现 `.uhd-root`(属主 10001);容器内 root 属主目录登记 → 400 `库根不可写 … EACCES` | +| `PATCH /settings {pi:{provider:fake}}` → `POST /agent/runs {mode:plan_only}` | run `succeeded`,会话文件 `/data/agent-sessions/.jsonl`(18.7 KB,属主 10001) | +| 宿主 `chmod 555` 媒体目录后 `umt preflight` | 退出码 0,`warn 挂载路径不可用: verify-media 不可写: EACCES`,审计 `preflight.path_unavailable` 一条;恢复权限后 warnings=0 | +| `umt backup` | `db.dump` 245 KB + `mappings.json` + `agent-sessions.tar.gz` + `manifest.txt` + `SHA256SUMS`;pg_dump 17.11 对服务端 16.14 | +| 清库 | `drop schema public cascade` + 删 `openworkflow` schema → 0 张表 | +| app 运行中 `umt restore` | 拒绝:`数据库还有 13 个其它连接` | +| `compose stop app` → `compose run --rm --no-deps app restore --yes` | 退出码 0;63 张表恢复;抽查 admin_user 1 / api_key 1 / library_root 1 / media_entity 3 / external_identity 3 / agent_run 1 / agent_run_event 9 与备份前一致(audit_event +1 为恢复审计、settings +1 为 automation 行);映射清单逐表比对"一致";`automation={"readOnlyReconcile": true}`,审计 `ops.read_only_reconcile_on` | +| 恢复后 `up -d app` | healthy;`GET /settings` → `automation {enabled: true, readOnlyReconcile: true}` | +| `BACKUP_KEEP=3` 连续备份 3 次 | 最旧一份被轮转删除,剩 3 份 | +| `PUID=1234 PGID=1234` 空卷启动 | `/data` 及子目录 chown 为 1234,进程 uid 1234;`PUID=0` → 退出码 2 | +| `compose stop app` | 0.8 秒,api/worker 均收到 SIGTERM 正常退出 | +| 清理 | `compose down -v`、删除测试镜像;未 prune 全局 build cache | + +首次演练发现:镜像里的 pg_restore 17 直连 16 版服务端会因 `SET transaction_timeout` 失败(单事务回滚,库未被破坏), +已改为 pg_restore 输出 SQL、过滤该行后由 psql 单事务回放,复测通过。 diff --git a/docs/runbook/dev.md b/docs/runbook/dev.md index 0d8b7841..4d0091d9 100644 --- a/docs/runbook/dev.md +++ b/docs/runbook/dev.md @@ -62,9 +62,9 @@ docker compose up -d --build # 服务:app(4380)+ postgres(数据卷 umt-p curl localhost:4380/api/v1/health ``` -- 镜像:`denoland/deno:2.9.7` 构建 web 与 `deno compile` 出二进制,运行时 `debian:trixie-slim`,非 root(uid 10001),约 270 MB。 +- 镜像:`denoland/deno:2.9.7` 构建 web 与 `deno compile` 出二进制,运行时 `debian:trixie-slim` + gosu + postgresql-client,root 启动修 `/data` 属主后降权到 PUID(缺省 10001),约 490 MB。 - 入口 `docker/entrypoint.sh`:迁移 → 自检 → 同时起 api 与 worker 两个进程;任一退出则停掉另一个并以非 0 退出,由 `restart: unless-stopped` 拉起。 -- 二进制编译时固定权限:`--allow-net --allow-env --allow-sys --allow-read=/app`,不给 `--allow-run` / 写权限。 +- 二进制编译时固定权限:`--allow-net --allow-env --allow-sys --allow-read --allow-write`,不给 `--allow-run`;运行期写入落在 `DATA_DIR`(容器内 `/data`)。完整部署说明见 [deploy.md](deploy.md)。 - 构建版本号:`APP_VERSION=v0.1.0 docker compose build`(写入 web 的 `version.json`)。 ## 测试数据库 diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100755 index 00000000..ef35fdcc --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# 备份:pg_dump(自定义格式,整库,含 secret 表密文与任务引擎 schema)+ 映射清单 JSON + Agent 会话归档 +# → $DATA_DIR/backups//。SECRET_MASTER_KEY 不在备份里,必须另行保管(没有它 secret 表无法解密)。 +# +# 容器内:docker compose exec app umt backup +# 环境变量:DATABASE_URL(必需)、DATA_DIR(缺省 /data)、BACKUP_KEEP(保留最近 N 份,缺省不轮转)、 +# UMT_BIN(缺省 /app/umt-server,开发机可指向 dist/umt-server) +set -euo pipefail + +if [ "$(id -u)" = "0" ] && command -v gosu >/dev/null 2>&1; then + exec gosu "${PUID:-10001}:${PGID:-10001}" "$0" "$@" +fi + +: "${DATABASE_URL:?需要 DATABASE_URL}" +DATA_DIR="${DATA_DIR:-/data}" +BIN="${UMT_BIN:-/app/umt-server}" +KEEP="${BACKUP_KEEP:-0}" +ROOT="$DATA_DIR/backups" + +ts="$(date -u +%Y%m%dT%H%M%SZ)" +dest="$ROOT/$ts" +work="$ROOT/.$ts.partial" +mkdir -p "$work" +trap 'rm -rf "$work"' EXIT + +echo "backup: pg_dump → $dest/db.dump" +pg_dump --format=custom --compress=6 --no-owner --no-privileges --file "$work/db.dump" "$DATABASE_URL" + +echo "backup: 导出映射清单 → $dest/mappings.json" +"$BIN" --role export-mappings --out "$work/mappings.json" >&2 + +sessions="${AGENT_SESSION_DIR:-$DATA_DIR/agent-sessions}" +if [ -d "$sessions" ] && [ -n "$(ls -A "$sessions" 2>/dev/null)" ]; then + echo "backup: 归档 Agent 会话 → $dest/agent-sessions.tar.gz" + tar -C "$sessions" -czf "$work/agent-sessions.tar.gz" . +fi + +{ + echo "created_at=$ts" + echo "pg_dump=$(pg_dump --version)" + echo "server_version=$(psql "$DATABASE_URL" -Atqc 'show server_version' 2>/dev/null || echo unknown)" + echo "master_key=NOT_INCLUDED(SECRET_MASTER_KEY 需另行保管)" +} >"$work/manifest.txt" +(cd "$work" && sha256sum db.dump mappings.json $([ -f agent-sessions.tar.gz ] && echo agent-sessions.tar.gz) >SHA256SUMS) + +mv "$work" "$dest" +trap - EXIT +echo "backup: 完成 $dest ($(du -sh "$dest" | cut -f1))" + +if [ "$KEEP" -gt 0 ] 2>/dev/null; then + # 只轮转符合时间戳命名的目录,按名字(即时间)倒序保留最近 KEEP 份 + mapfile -t all < <(find "$ROOT" -mindepth 1 -maxdepth 1 -type d -name '[0-9]*T[0-9]*Z' -printf '%f\n' | sort -r) + for old in "${all[@]:$KEEP}"; do + echo "backup: 轮转删除 $ROOT/$old" + rm -rf "${ROOT:?}/$old" + done +fi diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 00000000..a9ef67d3 --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# 恢复:pg_restore 覆盖当前库 → 打开只读对账(automation.readOnlyReconcile=true)→ 映射清单行数与备份比对。 +# 会删除并重建备份里的全部对象。先停掉 app 再做: +# docker compose stop app +# docker compose run --rm --no-deps app restore /data/backups/<时间戳> --yes +# docker compose up -d app +# 选项:--yes 跳过确认;--force 即使库里还有其它连接也继续(不建议)。 +set -euo pipefail + +if [ "$(id -u)" = "0" ] && command -v gosu >/dev/null 2>&1; then + exec gosu "${PUID:-10001}:${PGID:-10001}" "$0" "$@" +fi + +: "${DATABASE_URL:?需要 DATABASE_URL}" +DATA_DIR="${DATA_DIR:-/data}" +BIN="${UMT_BIN:-/app/umt-server}" + +src="" yes=0 force=0 +for a in "$@"; do + case "$a" in + --yes) yes=1 ;; + --force) force=1 ;; + -*) echo "restore: 未知选项 $a" >&2; exit 2 ;; + *) src="$a" ;; + esac +done +if [ -z "$src" ]; then + echo "用法: restore <备份目录> [--yes] [--force] 可用备份:" >&2 + ls -1 "$DATA_DIR/backups" 2>/dev/null >&2 || true + exit 2 +fi +case "$src" in /*) ;; *) [ -d "$src" ] || src="$DATA_DIR/backups/$src" ;; esac +for f in db.dump mappings.json; do + [ -f "$src/$f" ] || { echo "restore: $src 缺少 $f" >&2; exit 2; } +done +if [ -f "$src/SHA256SUMS" ]; then + (cd "$src" && sha256sum --quiet -c SHA256SUMS) || { echo "restore: 校验和不符,备份可能损坏" >&2; exit 2; } +fi + +others="$(psql "$DATABASE_URL" -Atqc "select count(*) from pg_stat_activity where datname = current_database() and pid <> pg_backend_pid()")" +if [ "$others" != "0" ] && [ "$force" -ne 1 ]; then + echo "restore: 数据库还有 $others 个其它连接(app 是否仍在运行?)。先 docker compose stop app,或加 --force" >&2 + exit 3 +fi + +if [ "$yes" -ne 1 ]; then + read -r -p "将用 $src 覆盖当前数据库,输入 yes 继续: " ans + [ "$ans" = "yes" ] || { echo "已取消"; exit 1; } +fi + +echo "restore: pg_restore ← $src/db.dump" +# 经 psql 回放而不是 pg_restore 直连:镜像里是 17 版客户端,它会输出 16 版服务端不认识的 +# SET transaction_timeout;过滤掉这一行即可兼容 16。单事务 + ON_ERROR_STOP:失败整体回滚。 +pg_restore --clean --if-exists --no-owner --no-privileges --file - "$src/db.dump" \ + | sed '/^SET transaction_timeout = /d' \ + | psql -X -q -v ON_ERROR_STOP=1 --single-transaction -d "$DATABASE_URL" >/dev/null + +if [ -f "$src/agent-sessions.tar.gz" ]; then + sessions="${AGENT_SESSION_DIR:-$DATA_DIR/agent-sessions}" + mkdir -p "$sessions" + echo "restore: 还原 Agent 会话 → $sessions" + tar -C "$sessions" -xzf "$src/agent-sessions.tar.gz" +fi + +"$BIN" --role read-only-on --reason "restore $(basename "$src")" >&2 + +check="$DATA_DIR/tmp/restore-check-$(date -u +%Y%m%dT%H%M%SZ).json" +set +e +"$BIN" --role export-mappings --out "$check" --compare "$src/mappings.json" >&2 +cmp=$? +set -e + +cat <