UHD_Media_Tool/apps/web/app/routes/console/status.tsx

import { useQueryClient } from "@tanstack/react-query";
import type { HealthLevel, SystemHealth, SystemProcess } from "@umt/contracts";
import {
  AlertTriangle,
  ChevronDown,
  CircleCheck,
  Cpu,
  Database,
  HardDrive,
  ListTodo,
  RotateCw,
  Timer,
  XCircle,
} from "lucide-react";
import { type ReactNode, useState } from "react";
import { Link } from "react-router";
import { useSystemHealth, useSystemHealthHistory } from "~/api/system.ts";
import { DataTable } from "~/components/console/data-table.tsx";
import { DetailSection } from "~/components/console/detail-sheet.tsx";
import { ErrorState, LoadingState } from "~/components/console/empty-state.tsx";
import { KvList } from "~/components/console/kv-list.tsx";
import { PageHeader } from "~/components/console/page-header.tsx";
import { StatusBadge, type StatusTone } from "~/components/console/status-badge.tsx";
import { HistoryChart } from "~/components/status/history-chart.tsx";
import { Button } from "~/components/ui/button.tsx";
import { Tabs, TabsList, TabsTrigger } from "~/components/ui/tabs.tsx";
import { formatDuration, formatRelative } from "~/lib/labels.ts";
import { qk } from "~/lib/query-keys.ts";
import { cn, formatBytes, formatDateTime } from "~/lib/utils.ts";

export const meta = () => [{ title: "运行状态 · UHD 媒体自动化控制台" }];

// 运行状态(W9-I):进程心跳、事件循环、SSE、DB、磁盘、挂载根、引擎与定时任务。阈值见 docs/runbook/ops.md。

const LEVEL: Record<HealthLevel, { label: string; tone: StatusTone }> = {
  ok: { label: "正常", tone: "success" },
  warn: { label: "需要关注", tone: "warning" },
  critical: { label: "需要处理", tone: "destructive" },
  unknown: { label: "取不到数据", tone: "neutral" },
};

const KIND_LABEL: Record<string, string> = {
  api: "API",
  worker: "Worker(io)",
  "worker:agent": "Worker(Agent)",
  all: "单进程(all)",
};

/** 横幅告警里的进程称呼 */
const KIND_NAME: Record<string, string> = {
  api: "API",
  worker: "io worker",
  "worker:agent": "Agent worker",
  all: "单进程",
};

const MOUNT_REASON: Record<string, string> = {
  root_missing: "目录不存在",
  sentinel_missing: "哨兵文件缺失",
  permission_denied: "没有权限",
  error: "检查出错",
  timeout: "检查超时",
};

const HOURS = [1, 6, 24, 168];

function Card({
  title,
  icon: Icon,
  level,
  children,
  action,
}: {
  title: string;
  icon: typeof Cpu;
  level?: HealthLevel;
  children: ReactNode;
  action?: ReactNode;
}) {
  return (
    <section className="pg-slab min-w-0 rounded-xl p-5">
      <div className="mb-3 flex items-center gap-2">
        <Icon className="h-4 w-4 text-muted-foreground" />
        <h2 className="text-body font-semibold text-foreground">{title}</h2>
        {level ? <StatusBadge tone={LEVEL[level].tone}>{LEVEL[level].label}</StatusBadge> : null}
        {action ? <span className="ml-auto">{action}</span> : null}
      </div>
      {children}
    </section>
  );
}

/** 顶部横幅:整体档位 + 需要看的几条 */
function problems(d: SystemHealth): string[] {
  const out: string[] = [];
  // 只看当前实例;已下线实例不告警
  for (const p of d.processes.filter((x) => x.stale)) {
    const line = `${KIND_NAME[p.processKind] ?? p.processKind} 45 秒没有心跳`;
    if (!out.includes(line)) out.push(line);
  }
  if (!d.processes.some((p) => !p.stale && p.processKind !== "api")) out.push("没有在线的 worker，任务不会被执行");
  const lag = d.processes.filter((p) => !p.stale && p.eventLoopLagMaxMs >= 1000);
  if (lag.length) out.push(`${lag.length} 个进程事件循环延迟超过 1 秒`);
  if (d.db.level === "warn" || d.db.level === "critical") out.push("数据库延迟或连接数偏高");
  const disks = d.disks.filter((x) => x.level === "warn" || x.level === "critical");
  if (disks.length) out.push(`${disks.length} 个磁盘用量超过阈值`);
  const mounts = d.mounts.filter((m) => m.status !== "online");
  if (mounts.length) out.push(`${mounts.length} 个挂载根不可用`);
  if (d.engine.level === "warn" || d.engine.level === "critical") out.push("任务积压或失败率偏高");
  if (d.schedules.failing.length) out.push(`${d.schedules.failing.length} 个定时任务上次运行失败`);
  return out;
}

function Banner({ d }: { d: SystemHealth }) {
  const lv = LEVEL[d.level];
  const list = problems(d);
  const ok = d.level === "ok";
  return (
    <div
      role="status"
      className={cn(
        "mb-4 flex items-start gap-3 rounded-xl border px-4 py-3",
        ok
          ? "border-success/30 bg-success/10"
          : d.level === "critical"
            ? "border-destructive/30 bg-destructive/10"
            : "border-warning/30 bg-warning/10",
      )}
    >
      {ok ? (
        <CircleCheck className="mt-0.5 h-4 w-4 shrink-0 text-success" />
      ) : d.level === "critical" ? (
        <XCircle className="mt-0.5 h-4 w-4 shrink-0 text-destructive" />
      ) : (
        <AlertTriangle className="mt-0.5 h-4 w-4 shrink-0 text-warning" />
      )}
      <div className="min-w-0 text-caption">
        <p className="font-medium text-foreground">
          {ok ? "全部正常" : lv.label}
          <span className="ml-2 font-normal text-muted-foreground">
            检查于 {formatRelative(d.checkedAt)} · API {d.api.version}，已运行 {formatDuration(d.api.uptimeSec)}
          </span>
        </p>
        {list.length ? (
          <ul className="mt-1 list-disc space-y-0.5 pl-4 text-muted-foreground">
            {list.map((x) => (
              <li key={x}>{x}</li>
            ))}
          </ul>
        ) : null}
      </div>
    </div>
  );
}

function Processes({ d }: { d: SystemHealth }) {
  return (
    <DataTable<SystemProcess>
      rows={d.processes}
      rowKey={(p) => p.instanceId}
      empty={{ title: "没有在线的进程", description: "进程启动后每 15 秒写一次心跳。" }}
      columns={[
        {
          key: "kind",
          header: "进程",
          cell: (p) => (
            <div className="min-w-0">
              <p className="flex items-center gap-2 font-medium">
                {KIND_LABEL[p.processKind] ?? p.processKind}
                {p.stale ? <StatusBadge tone="warning">无心跳</StatusBadge> : null}
              </p>
              <p className="truncate font-mono text-meta text-muted-foreground" title={p.instanceId}>
                {p.host} · pid {p.pid}
              </p>
            </div>
          ),
        },
        {
          key: "ver",
          header: "版本",
          hideBelow: "lg",
          cell: (p) => <span className="font-mono text-meta">{p.version}</span>,
        },
        {
          key: "start",
          header: "启动",
          hideBelow: "md",
          cell: (p) => (
            <span className="whitespace-nowrap text-muted-foreground" title={formatDateTime(p.startedAt)}>
              {formatRelative(p.startedAt)}
            </span>
          ),
        },
        {
          key: "beat",
          header: "最后心跳",
          cell: (p) => (
            <span className={cn("whitespace-nowrap tabular-nums", p.stale ? "text-warning" : "text-muted-foreground")}>
              {formatDuration(p.beatAgeSec)}前
            </span>
          ),
        },
        {
          key: "lag",
          header: "延迟 / 峰值",
          align: "right",
          cell: (p) => (
            <span
              className={cn(
                "whitespace-nowrap tabular-nums",
                p.eventLoopLagMaxMs >= 5000 ? "text-destructive" : p.eventLoopLagMaxMs >= 1000 ? "text-warning" : "",
              )}
            >
              {p.eventLoopLagMs} / {p.eventLoopLagMaxMs} 毫秒
            </span>
          ),
        },
        {
          key: "conc",
          header: "并发",
          hideBelow: "md",
          align: "right",
          cell: (p) =>
            p.concurrency.io === undefined && p.concurrency.agent === undefined ? (
              <span className="text-muted-foreground">—</span>
            ) : (
              <span className="whitespace-nowrap tabular-nums">
                io {p.concurrency.io ?? 0} · Agent {p.concurrency.agent ?? 0}
              </span>
            ),
        },
        {
          key: "running",
          header: "运行中",
          hideBelow: "sm",
          align: "right",
          cell: (p) => <span className="tabular-nums">{p.runningRuns ?? "—"}</span>,
        },
        {
          key: "sse",
          header: "SSE",
          hideBelow: "lg",
          align: "right",
          cell: (p) => <span className="tabular-nums">{p.sseConnections ?? "—"}</span>,
        },
        {
          key: "mem",
          header: "内存",
          hideBelow: "lg",
          align: "right",
          cell: (p) => {
            const rss = p.extra.rssBytes;
            return <span className="tabular-nums">{typeof rss === "number" ? formatBytes(rss, 0) : "—"}</span>;
          },
        },
      ]}
    />
  );
}

/** 已下线实例:已退出、容器已重建或被同容器新进程替换的旧心跳行,默认折叠,不参与告警 */
function RetiredProcesses({ rows }: { rows: SystemProcess[] }) {
  const [open, setOpen] = useState(false);
  if (!rows.length) return null;
  return (
    <div className="mt-3">
      <button
        type="button"
        aria-expanded={open}
        onClick={() => setOpen((v) => !v)}
        className="flex cursor-pointer items-center gap-1.5 text-caption text-muted-foreground hover:text-foreground"
      >
        <ChevronDown className={cn("h-3.5 w-3.5 transition-transform", !open && "-rotate-90")} />
        <span>已下线实例 {rows.length} 个</span>
        <span className="text-meta">不计入告警，24 小时后清理</span>
      </button>
      {open ? (
        <div className="mt-2">
          <DataTable<SystemProcess>
            rows={rows}
            rowKey={(p) => p.instanceId}
            empty={{ title: "没有已下线实例" }}
            columns={[
              {
                key: "id",
                header: "实例",
                cell: (p) => (
                  <div className="min-w-0">
                    <p className="flex items-center gap-2 font-medium text-muted-foreground">
                      {KIND_LABEL[p.processKind] ?? p.processKind}
                      {p.stoppedAt ? <StatusBadge tone="neutral">已退出</StatusBadge> : null}
                    </p>
                    <p className="truncate font-mono text-meta text-muted-foreground" title={p.instanceId}>
                      {p.instanceId}
                    </p>
                  </div>
                ),
              },
              {
                key: "ver",
                header: "版本",
                hideBelow: "md",
                cell: (p) => <span className="font-mono text-meta">{p.version}</span>,
              },
              {
                key: "start",
                header: "启动",
                hideBelow: "md",
                cell: (p) => (
                  <span className="whitespace-nowrap text-muted-foreground" title={formatDateTime(p.startedAt)}>
                    {formatRelative(p.startedAt)}
                  </span>
                ),
              },
              {
                key: "beat",
                header: "最后心跳",
                cell: (p) => (
                  <span className="whitespace-nowrap text-muted-foreground" title={formatDateTime(p.lastBeatAt)}>
                    {formatRelative(p.lastBeatAt)}
                  </span>
                ),
              },
            ]}
          />
        </div>
      ) : null}
    </div>
  );
}

function History() {
  const [hours, setHours] = useState(6);
  const h = useSystemHealthHistory(hours);
  return (
    <section className="pg-slab min-w-0 rounded-xl p-5">
      <div className="mb-3 flex flex-wrap items-center gap-2">
        <h2 className="text-body font-semibold text-foreground">趋势</h2>
        {h.data ? (
          <span className="text-meta text-muted-foreground">每点 {formatDuration(h.data.bucketSec)}</span>
        ) : null}
        <Tabs value={String(hours)} onValueChange={(v) => setHours(Number(v))} className="ml-auto">
          <TabsList className="pb-0">
            {HOURS.map((n) => (
              <TabsTrigger key={n} value={String(n)} className="h-7 px-2.5 text-caption">
                {n === 168 ? "7 天" : `${n} 小时`}
              </TabsTrigger>
            ))}
          </TabsList>
        </Tabs>
      </div>
      {h.isPending ? (
        <LoadingState rows={4} />
      ) : h.isError ? (
        <ErrorState plain error={h.error} onRetry={() => h.refetch()} />
      ) : (
        <div className="grid gap-6 lg:grid-cols-2">
          <DetailSection title="事件循环延迟峰值（毫秒）">
            <HistoryChart data={h.data} metric="lagMaxMs" unit="毫秒" hours={hours} />
          </DetailSection>
          <DetailSection title="SSE 连接数">
            <HistoryChart data={h.data} metric="sseConnections" unit="个" hours={hours} />
          </DetailSection>
        </div>
      )}
    </section>
  );
}

function Disks({ d }: { d: SystemHealth }) {
  const t = d.diskThresholds;
  const scope = { data_dir: "数据目录", library_root: "库根", download_dir: "下载目录" } as const;
  return (
    <Card title="磁盘" icon={HardDrive} level={worst(d.disks.map((x) => x.level))}>
      <p className="mb-3 text-meta text-muted-foreground">
        阈值：{t.warnPercent}% 需要关注，{t.criticalPercent}% 需要处理（取自缓存设置）
      </p>
      <ul className="space-y-3">
        {d.disks.map((x) => {
          const pct = x.usedRatio === null ? null : x.usedRatio * 100;
          return (
            <li key={`${x.scope}-${x.id ?? x.path}`} className="min-w-0">
              <div className="flex items-center justify-between gap-2 text-sm">
                <span className="min-w-0 truncate">
                  <span className="mr-1.5 text-meta text-muted-foreground">{scope[x.scope]}</span>
                  {x.name}
                </span>
                <StatusBadge tone={LEVEL[x.level].tone}>
                  {pct === null ? (x.error ? `不可读 ${x.error}` : "用量未知") : `${pct.toFixed(0)}%`}
                </StatusBadge>
              </div>
              <div className="relative mt-1.5 h-1.5 overflow-hidden rounded-full bg-foreground/[0.08]" aria-hidden>
                <div
                  className={cn(
                    "h-full rounded-full",
                    x.level === "critical" ? "bg-destructive" : x.level === "warn" ? "bg-warning" : "bg-foreground/60",
                  )}
                  style={{ width: `${pct ?? 0}%` }}
                />
                <span className="absolute inset-y-0 w-px bg-warning" style={{ left: `${t.warnPercent}%` }} />
                <span className="absolute inset-y-0 w-px bg-destructive" style={{ left: `${t.criticalPercent}%` }} />
              </div>
              <p className="mt-1 truncate font-mono text-meta text-muted-foreground" title={x.path}>
                {x.path}
                {x.totalBytes !== null && x.availableBytes !== null
                  ? ` · 剩余 ${formatBytes(x.availableBytes, 1)} / ${formatBytes(x.totalBytes, 1)}`
                  : ""}
              </p>
            </li>
          );
        })}
      </ul>
    </Card>
  );
}

function worst(levels: HealthLevel[]): HealthLevel {
  if (levels.includes("critical")) return "critical";
  if (levels.includes("warn")) return "warn";
  if (levels.includes("ok")) return "ok";
  return "unknown";
}

function Mounts({ d }: { d: SystemHealth }) {
  const bad = d.mounts.filter((m) => m.status !== "online").length;
  return (
    <Card title="挂载根" icon={HardDrive} level={d.mounts.length ? (bad ? "critical" : "ok") : undefined}>
      {d.mounts.length === 0 ? (
        <p className="text-caption text-muted-foreground">没有启用的库根或下载目录映射。</p>
      ) : (
        <ul className="divide-y divide-border/60">
          {d.mounts.map((m) => (
            <li key={`${m.kind}-${m.id}`} className="flex min-w-0 items-center gap-3 py-2 text-sm">
              <div className="min-w-0 flex-1">
                <p className="truncate">
                  <span className="mr-1.5 text-meta text-muted-foreground">
                    {m.kind === "library_root" ? "库根" : "下载目录"}
                  </span>
                  {m.name}
                </p>
                <p className="truncate font-mono text-meta text-muted-foreground" title={m.path}>
                  {m.path}
                  {m.sentinelChecked ? "" : " · 只查目录可读"}
                </p>
              </div>
              <StatusBadge tone={m.status === "online" ? "success" : "destructive"}>
                {m.status === "online" ? "在线" : m.status === "timeout" ? "超时" : "离线"}
                {m.reason && m.status !== "timeout" ? ` · ${MOUNT_REASON[m.reason] ?? m.reason}` : ""}
              </StatusBadge>
            </li>
          ))}
        </ul>
      )}
    </Card>
  );
}

export default function StatusPage() {
  const qc = useQueryClient();
  const h = useSystemHealth();
  const d = h.data;
  return (
    <>
      <PageHeader
        title="运行状态"
        description="进程心跳、数据库、磁盘与挂载根。15 秒刷新一次。"
        actions={
          <Button
            variant="outline"
            size="sm"
            onClick={() => qc.invalidateQueries({ queryKey: qk.all("system") })}
            disabled={h.isFetching}
          >
            <RotateCw className={h.isFetching ? "animate-spin" : undefined} />
            刷新
          </Button>
        }
      />
      {h.isPending ? (
        <LoadingState rows={8} />
      ) : h.isError ? (
        <ErrorState title="运行状态加载失败" error={h.error} onRetry={() => h.refetch()} />
      ) : d ? (
        <div className="space-y-4">
          <Banner d={d} />
          <section className="min-w-0">
            <h2 className="mb-2 flex items-center gap-2 text-body font-semibold text-foreground">
              <Cpu className="h-4 w-4 text-muted-foreground" />
              进程
              <span className="text-meta font-normal text-muted-foreground">SSE 连接合计 {d.sse.connections}</span>
            </h2>
            <Processes d={d} />
            <RetiredProcesses rows={d.retiredProcesses} />
          </section>
          <History />
          <div className="grid gap-4 md:grid-cols-2 xl:grid-cols-3">
            <Card title="数据库" icon={Database} level={d.db.level}>
              {d.db.error ? <p className="mb-2 text-caption text-destructive">{d.db.error}</p> : null}
              <KvList
                items={[
                  { label: "延迟", value: d.db.latencyMs === null ? "—" : `${d.db.latencyMs} 毫秒` },
                  { label: "库大小", value: d.db.sizeBytes === null ? "—" : formatBytes(d.db.sizeBytes, 1) },
                  {
                    label: "连接",
                    value: d.db.connections ? `${d.db.connections.total} / ${d.db.connections.max}` : "—",
                  },
                  {
                    label: "活跃 / 空闲 / 事务中空闲",
                    value: d.db.connections
                      ? `${d.db.connections.active} / ${d.db.connections.idle} / ${d.db.connections.idleInTransaction}`
                      : "—",
                  },
                ]}
              />
            </Card>
            <Card
              title="任务引擎"
              icon={ListTodo}
              level={d.engine.level}
              action={
                <Link to="/console/tasks?tab=active" className="text-meta text-muted-foreground hover:text-foreground">
                  查看任务
                </Link>
              }
            >
              <KvList
                items={[
                  { label: "待执行", value: d.engine.pending },
                  { label: "执行中", value: d.engine.running },
                  { label: "最久待执行", value: formatDuration(d.engine.oldestPendingAgeSec) },
                  {
                    label: "24 小时失败率",
                    value:
                      d.engine.failureRate24h === null
                        ? "—"
                        : `${(d.engine.failureRate24h * 100).toFixed(1)}%(${d.engine.failed24h} / ${d.engine.finished24h})`,
                  },
                ]}
              />
            </Card>
            <Card
              title="定时任务"
              icon={Timer}
              level={d.schedules.level}
              action={
                <Link to="/console/schedules" className="text-meta text-muted-foreground hover:text-foreground">
                  查看计划
                </Link>
              }
            >
              <KvList
                items={[
                  { label: "启用 / 全部", value: `${d.schedules.enabled} / ${d.schedules.total}` },
                  { label: "运行中", value: d.schedules.running },
                  { label: "24 小时失败运行", value: d.schedules.failedRuns24h },
                ]}
              />
              {d.schedules.failing.length ? (
                <ul className="mt-3 space-y-1.5">
                  {d.schedules.failing.map((f) => (
                    <li key={f.id} className="min-w-0 text-caption">
                      <Link
                        to={`/console/schedules?job=${encodeURIComponent(f.id)}`}
                        className="font-medium text-foreground hover:underline"
                      >
                        {f.name}
                      </Link>
                      <span className="ml-1.5 text-meta text-muted-foreground">{formatRelative(f.lastRunAt)}</span>
                      {f.lastError ? <p className="truncate text-meta text-destructive">{f.lastError}</p> : null}
                    </li>
                  ))}
                </ul>
              ) : null}
            </Card>
            <Disks d={d} />
            <Mounts d={d} />
          </div>
        </div>
      ) : null}
    </>
  );
}