feat(ops): W4-B single-container packaging, DATA_DIR, backup/restore
- DATA_DIR (/data: agent-sessions, backups, exports, tmp); entrypoint chowns it to PUID/PGID then drops privileges via gosu; /app binary stays root read-only - compile with --allow-read --allow-write so mounted library roots are writable - preflight: DATA_DIR/session dir must be writable (fatal); enabled library roots and downloader path mappings checked read/write (warn + audit) - --role export-mappings (mapping manifest JSON, --compare row counts) and --role read-only-on (automation.readOnlyReconcile=true + audit) - scripts/backup.sh (pg_dump custom + mappings + agent sessions, SHA256SUMS, BACKUP_KEEP rotation) and scripts/restore.sh (checksum, refuses with live connections, psql single-transaction replay compatible with PG16, read-only reconcile on, row-count verification) - compose: app + postgres only, data volume, env example with key generation - docs: runbook deploy.md, backup-restore.md with verification record Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
8c6a922cfb
commit
34e8751b45
18 files changed
+918
-43
No files matched your search
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
# 备份:pg_dump(自定义格式,整库,含 secret 表密文与任务引擎 schema)+ 映射清单 JSON + Agent 会话归档
|
||||
# → $DATA_DIR/backups/<UTC 时间戳>/。SECRET_MASTER_KEY 不在备份里,必须另行保管(没有它 secret 表无法解密)。
|
||||
#
|
||||
# 容器内:docker compose exec app umt backup
|
||||
# 环境变量:DATABASE_URL(必需)、DATA_DIR(缺省 /data)、BACKUP_KEEP(保留最近 N 份,缺省不轮转)、
|
||||
# UMT_BIN(缺省 /app/umt-server,开发机可指向 dist/umt-server)
|
||||
set -euo pipefail
|
||||
|
||||
if [ "$(id -u)" = "0" ] && command -v gosu >/dev/null 2>&1; then
|
||||
exec gosu "${PUID:-10001}:${PGID:-10001}" "$0" "$@"
|
||||
fi
|
||||
|
||||
: "${DATABASE_URL:?需要 DATABASE_URL}"
|
||||
DATA_DIR="${DATA_DIR:-/data}"
|
||||
BIN="${UMT_BIN:-/app/umt-server}"
|
||||
KEEP="${BACKUP_KEEP:-0}"
|
||||
ROOT="$DATA_DIR/backups"
|
||||
|
||||
ts="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
dest="$ROOT/$ts"
|
||||
work="$ROOT/.$ts.partial"
|
||||
mkdir -p "$work"
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
|
||||
echo "backup: pg_dump → $dest/db.dump"
|
||||
pg_dump --format=custom --compress=6 --no-owner --no-privileges --file "$work/db.dump" "$DATABASE_URL"
|
||||
|
||||
echo "backup: 导出映射清单 → $dest/mappings.json"
|
||||
"$BIN" --role export-mappings --out "$work/mappings.json" >&2
|
||||
|
||||
sessions="${AGENT_SESSION_DIR:-$DATA_DIR/agent-sessions}"
|
||||
if [ -d "$sessions" ] && [ -n "$(ls -A "$sessions" 2>/dev/null)" ]; then
|
||||
echo "backup: 归档 Agent 会话 → $dest/agent-sessions.tar.gz"
|
||||
tar -C "$sessions" -czf "$work/agent-sessions.tar.gz" .
|
||||
fi
|
||||
|
||||
{
|
||||
echo "created_at=$ts"
|
||||
echo "pg_dump=$(pg_dump --version)"
|
||||
echo "server_version=$(psql "$DATABASE_URL" -Atqc 'show server_version' 2>/dev/null || echo unknown)"
|
||||
echo "master_key=NOT_INCLUDED(SECRET_MASTER_KEY 需另行保管)"
|
||||
} >"$work/manifest.txt"
|
||||
(cd "$work" && sha256sum db.dump mappings.json $([ -f agent-sessions.tar.gz ] && echo agent-sessions.tar.gz) >SHA256SUMS)
|
||||
|
||||
mv "$work" "$dest"
|
||||
trap - EXIT
|
||||
echo "backup: 完成 $dest ($(du -sh "$dest" | cut -f1))"
|
||||
|
||||
if [ "$KEEP" -gt 0 ] 2>/dev/null; then
|
||||
# 只轮转符合时间戳命名的目录,按名字(即时间)倒序保留最近 KEEP 份
|
||||
mapfile -t all < <(find "$ROOT" -mindepth 1 -maxdepth 1 -type d -name '[0-9]*T[0-9]*Z' -printf '%f\n' | sort -r)
|
||||
for old in "${all[@]:$KEEP}"; do
|
||||
echo "backup: 轮转删除 $ROOT/$old"
|
||||
rm -rf "${ROOT:?}/$old"
|
||||
done
|
||||
fi
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
# 恢复:pg_restore 覆盖当前库 → 打开只读对账(automation.readOnlyReconcile=true)→ 映射清单行数与备份比对。
|
||||
# 会删除并重建备份里的全部对象。先停掉 app 再做:
|
||||
# docker compose stop app
|
||||
# docker compose run --rm --no-deps app restore /data/backups/<时间戳> --yes
|
||||
# docker compose up -d app
|
||||
# 选项:--yes 跳过确认;--force 即使库里还有其它连接也继续(不建议)。
|
||||
set -euo pipefail
|
||||
|
||||
if [ "$(id -u)" = "0" ] && command -v gosu >/dev/null 2>&1; then
|
||||
exec gosu "${PUID:-10001}:${PGID:-10001}" "$0" "$@"
|
||||
fi
|
||||
|
||||
: "${DATABASE_URL:?需要 DATABASE_URL}"
|
||||
DATA_DIR="${DATA_DIR:-/data}"
|
||||
BIN="${UMT_BIN:-/app/umt-server}"
|
||||
|
||||
src="" yes=0 force=0
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--yes) yes=1 ;;
|
||||
--force) force=1 ;;
|
||||
-*) echo "restore: 未知选项 $a" >&2; exit 2 ;;
|
||||
*) src="$a" ;;
|
||||
esac
|
||||
done
|
||||
if [ -z "$src" ]; then
|
||||
echo "用法: restore <备份目录> [--yes] [--force] 可用备份:" >&2
|
||||
ls -1 "$DATA_DIR/backups" 2>/dev/null >&2 || true
|
||||
exit 2
|
||||
fi
|
||||
case "$src" in /*) ;; *) [ -d "$src" ] || src="$DATA_DIR/backups/$src" ;; esac
|
||||
for f in db.dump mappings.json; do
|
||||
[ -f "$src/$f" ] || { echo "restore: $src 缺少 $f" >&2; exit 2; }
|
||||
done
|
||||
if [ -f "$src/SHA256SUMS" ]; then
|
||||
(cd "$src" && sha256sum --quiet -c SHA256SUMS) || { echo "restore: 校验和不符,备份可能损坏" >&2; exit 2; }
|
||||
fi
|
||||
|
||||
others="$(psql "$DATABASE_URL" -Atqc "select count(*) from pg_stat_activity where datname = current_database() and pid <> pg_backend_pid()")"
|
||||
if [ "$others" != "0" ] && [ "$force" -ne 1 ]; then
|
||||
echo "restore: 数据库还有 $others 个其它连接(app 是否仍在运行?)。先 docker compose stop app,或加 --force" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if [ "$yes" -ne 1 ]; then
|
||||
read -r -p "将用 $src 覆盖当前数据库,输入 yes 继续: " ans
|
||||
[ "$ans" = "yes" ] || { echo "已取消"; exit 1; }
|
||||
fi
|
||||
|
||||
echo "restore: pg_restore ← $src/db.dump"
|
||||
# 经 psql 回放而不是 pg_restore 直连:镜像里是 17 版客户端,它会输出 16 版服务端不认识的
|
||||
# SET transaction_timeout;过滤掉这一行即可兼容 16。单事务 + ON_ERROR_STOP:失败整体回滚。
|
||||
pg_restore --clean --if-exists --no-owner --no-privileges --file - "$src/db.dump" \
|
||||
| sed '/^SET transaction_timeout = /d' \
|
||||
| psql -X -q -v ON_ERROR_STOP=1 --single-transaction -d "$DATABASE_URL" >/dev/null
|
||||
|
||||
if [ -f "$src/agent-sessions.tar.gz" ]; then
|
||||
sessions="${AGENT_SESSION_DIR:-$DATA_DIR/agent-sessions}"
|
||||
mkdir -p "$sessions"
|
||||
echo "restore: 还原 Agent 会话 → $sessions"
|
||||
tar -C "$sessions" -xzf "$src/agent-sessions.tar.gz"
|
||||
fi
|
||||
|
||||
"$BIN" --role read-only-on --reason "restore $(basename "$src")" >&2
|
||||
|
||||
check="$DATA_DIR/tmp/restore-check-$(date -u +%Y%m%dT%H%M%SZ).json"
|
||||
set +e
|
||||
"$BIN" --role export-mappings --out "$check" --compare "$src/mappings.json" >&2
|
||||
cmp=$?
|
||||
set -e
|
||||
|
||||
cat <<MSG
|
||||
|
||||
==================================================================
|
||||
恢复完成:$(basename "$src")
|
||||
映射清单行数比对:$([ "$cmp" -eq 0 ] && echo "一致" || echo "不一致(见上方日志,退出码 $cmp)")
|
||||
只读对账模式已打开(automation.readOnlyReconcile = true):
|
||||
自动下载/整理/调度与 Agent 自动执行均被拒绝。
|
||||
恢复后先对账再关闭只读:
|
||||
1. 核对下载器任务与真实文件(控制台 → 下载 / 整理记录,库根"测试")
|
||||
2. 确认 SECRET_MASTER_KEY 与备份时一致(站点/下载器凭证能解密)
|
||||
3. 确认无误后在 设置 → 自动化 关闭"只读对账",
|
||||
或 PATCH /api/v1/settings {"automation":{"readOnlyReconcile":false}}
|
||||
==================================================================
|
||||
MSG
|
||||
exit "$cmp"
|
||||
Reference in new issue
Block a user