feat(worker): IO Worker 分道(WK-1)
io worker 进程内按任务声明的 lane 分文件 / 网络 / 维护三条道,各自并发(缺省 2/4/1)、 各自排队;维护道在文件道或网络道有排队/运行时让路(连续 10 分钟放行一个),调度器内联的 清理/回填/保留期裁剪也走维护道。并发在 设置 → 自动化 → 高级 热生效,env 覆盖;旧 WORKER_CONCURRENCY 映射到网络道。运行状态页按道显示运行中/并发/排队。归属表写进 docs/runbook/dev.md,ci 与注册表逐项核对。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
rootandClaude Opus 5.5 committed
1 parent
68f53390f6
commit
da33cad81d
38 files changed
+1240
-105
No files matched your search
@@ -375,6 +375,24 @@ export type ProcessKind = z.infer<typeof ProcessKind>;
|
||||
export const HealthLevel = z.enum(["ok", "warn", "critical", "unknown"]);
|
||||
export type HealthLevel = z.infer<typeof HealthLevel>;
|
||||
|
||||
export const WorkerLane = z.enum(["file", "network", "maintenance"]);
|
||||
export type WorkerLane = z.infer<typeof WorkerLane>;
|
||||
|
||||
export const SystemWorkerLane = z.object({
|
||||
lane: WorkerLane,
|
||||
/** 当前生效的并发上限 */
|
||||
concurrency: NonNegInt,
|
||||
/** env = 环境变量覆盖(设置页改了不生效);settings = 设置页的值 */
|
||||
source: z.enum(["env", "settings"]),
|
||||
/** 本进程该道正在执行的数 */
|
||||
running: NonNegInt,
|
||||
/** 全库该道已到点等待领取的数;统计失败为 null */
|
||||
queued: NonNegInt.nullable(),
|
||||
/** 维护道:正在给文件道 / 网络道让路 */
|
||||
yielding: z.boolean().optional(),
|
||||
});
|
||||
export type SystemWorkerLane = z.infer<typeof SystemWorkerLane>;
|
||||
|
||||
export const SystemProcess = z.object({
|
||||
/** <kind>:<host>:<pid>;同一容器重启覆盖同一行 */
|
||||
instanceId: z.string(),
|
||||
@@ -394,8 +412,16 @@ export const SystemProcess = z.object({
|
||||
eventLoopLagMs: NonNegInt,
|
||||
/** 最近一个心跳周期(15s)内的最大延迟 */
|
||||
eventLoopLagMaxMs: NonNegInt,
|
||||
/** 引擎 worker 并发;api 进程为 {} */
|
||||
concurrency: z.object({ io: NonNegInt.optional(), agent: NonNegInt.optional() }),
|
||||
/** 引擎 worker 并发;api 进程为 {}。io = 三条道合计(WK-1 前是 io 组总并发) */
|
||||
concurrency: z.object({
|
||||
io: NonNegInt.optional(),
|
||||
agent: NonNegInt.optional(),
|
||||
file: NonNegInt.optional(),
|
||||
network: NonNegInt.optional(),
|
||||
maintenance: NonNegInt.optional(),
|
||||
}),
|
||||
/** io worker 各道(WK-1):并发、本进程运行中、全库排队;非 io 进程或旧心跳为 [] */
|
||||
lanes: z.array(SystemWorkerLane).default([]),
|
||||
/** 本组任务正在执行的 run 数(同组多实例时是组合计);api 进程为 null */
|
||||
runningRuns: NonNegInt.nullable(),
|
||||
/** 本进程 SSE 长连接数;不托管 HTTP 的进程为 null */
|
||||
|
||||
@@ -8,11 +8,31 @@ import { SiteRateLimit } from "./sources.ts";
|
||||
|
||||
const Percent = z.number().min(1).max(100);
|
||||
|
||||
/** 并发槽上限(与 io worker 各道开的槽位数一致;改设置只在这个范围内热生效) */
|
||||
export const WORKER_LANE_MAX = 16;
|
||||
/**
|
||||
* io worker 三条道的并发(WK-1,docs/runbook/dev.md「IO Worker 分道」):改后约 5 秒内热生效,不用重启;
|
||||
* 设置了环境变量 WORKER_FILE_CONCURRENCY / WORKER_NETWORK_CONCURRENCY(旧 WORKER_CONCURRENCY)/
|
||||
* WORKER_MAINTENANCE_CONCURRENCY 时以环境变量为准。
|
||||
*/
|
||||
export const WorkerLaneSettings = z.object({
|
||||
/** 文件道:整理、硬链接/移动、ffprobe、校验 */
|
||||
file: z.number().int().min(1).max(WORKER_LANE_MAX),
|
||||
/** 网络道:站点、RSS、TMDB、下载器、搜索(单站限流另计) */
|
||||
network: z.number().int().min(1).max(WORKER_LANE_MAX),
|
||||
/** 维护道:清理、回填、对账;文件道 / 网络道有活时让路 */
|
||||
maintenance: z.number().int().min(1).max(WORKER_LANE_MAX),
|
||||
});
|
||||
export type WorkerLaneSettings = z.infer<typeof WorkerLaneSettings>;
|
||||
export const DEFAULT_WORKER_LANES: WorkerLaneSettings = { file: 2, network: 4, maintenance: 1 };
|
||||
|
||||
export const AutomationSettings = z.object({
|
||||
/** 自动化总开关:关闭后所有自动收录/下载/整理停止,只读查询与手动操作可用 */
|
||||
enabled: z.boolean(),
|
||||
/** 恢复后的只读对账模式(REQ §8.5):开启时禁止自动写入 */
|
||||
readOnlyReconcile: z.boolean(),
|
||||
/** io worker 分道并发;旧库缺省时补 DEFAULT_WORKER_LANES */
|
||||
workerLanes: WorkerLaneSettings.default(DEFAULT_WORKER_LANES),
|
||||
});
|
||||
export type AutomationSettings = z.infer<typeof AutomationSettings>;
|
||||
|
||||
@@ -564,7 +584,7 @@ export type SettingsKey = z.infer<typeof SettingsKey>;
|
||||
|
||||
/** 缺省设置(settings 表缺 key 或缺字段时使用) */
|
||||
export const DEFAULT_SETTINGS: GlobalSettings = {
|
||||
automation: { enabled: true, readOnlyReconcile: false },
|
||||
automation: { enabled: true, readOnlyReconcile: false, workerLanes: DEFAULT_WORKER_LANES },
|
||||
cache: {
|
||||
hardThresholdPercent: 90,
|
||||
resumePercent: 85,
|
||||
@@ -646,7 +666,10 @@ export const DEFAULT_SETTINGS: GlobalSettings = {
|
||||
|
||||
/** PATCH /settings:按分组部分更新;服务端与现值合并后用 GlobalSettings 整体再校验 */
|
||||
export const GlobalSettingsUpdate = z.object({
|
||||
automation: AutomationSettings.partial().optional(),
|
||||
automation: AutomationSettings.omit({ workerLanes: true })
|
||||
.partial()
|
||||
.extend({ workerLanes: WorkerLaneSettings.partial().optional() })
|
||||
.optional(),
|
||||
cache: CacheShape.partial().optional(),
|
||||
organize: OrganizeSettings.partial().optional(),
|
||||
retention: RetentionSettings.partial().optional(),
|
||||
@@ -709,7 +732,11 @@ export function mergeSettings(current: GlobalSettings, patch: GlobalSettingsUpda
|
||||
const { webSearch: wsPatch, ...agent } = patch.agent ?? {};
|
||||
const { apiKey: _wsApiKey, ...webSearch } = wsPatch ?? {};
|
||||
return {
|
||||
automation: { ...current.automation, ...patch.automation },
|
||||
automation: {
|
||||
...current.automation,
|
||||
...patch.automation,
|
||||
workerLanes: { ...DEFAULT_WORKER_LANES, ...current.automation.workerLanes, ...patch.automation?.workerLanes },
|
||||
},
|
||||
cache: { ...current.cache, ...patch.cache },
|
||||
organize: { ...current.organize, ...patch.organize },
|
||||
retention: { ...current.retention, ...patch.retention },
|
||||
@@ -825,6 +852,7 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [
|
||||
fields: [
|
||||
"automation.enabled",
|
||||
"automation.readOnlyReconcile",
|
||||
"automation.workerLanes",
|
||||
"cache.hardThresholdPercent",
|
||||
"cache.resumePercent",
|
||||
"cache.checkIntervalSec",
|
||||
|
||||
Reference in new issue
Block a user