io worker 进程内按任务声明的 lane 分文件 / 网络 / 维护三条道,各自并发(缺省 2/4/1)、 各自排队;维护道在文件道或网络道有排队/运行时让路(连续 10 分钟放行一个),调度器内联的 清理/回填/保留期裁剪也走维护道。并发在 设置 → 自动化 → 高级 热生效,env 覆盖;旧 WORKER_CONCURRENCY 映射到网络道。运行状态页按道显示运行中/并发/排队。归属表写进 docs/runbook/dev.md,ci 与注册表逐项核对。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
115 lines
5.3 KiB
TypeScript
115 lines
5.3 KiB
TypeScript
import { registerScheduledJob } from "../schedules/registry.ts";
|
|
import type { ServerContext, ServerModule } from "../types.ts";
|
|
import { registerSourcesCapabilities } from "./capabilities.ts";
|
|
import { decisionRoutes } from "./routes/decisions.ts";
|
|
import { enrichRoutes } from "./routes/enrich.ts";
|
|
import { identityRoutes } from "./routes/identity.ts";
|
|
import { sourcesRoutes } from "./routes/sources.ts";
|
|
import { workbenchRoutes } from "./routes/workbench.ts";
|
|
import { type RuntimeOverrides, SourcesRuntime } from "./runtime.ts";
|
|
import { readOnlyReconcile, tickFeeds, tickReviews, tickSnapshots } from "./scheduler.ts";
|
|
import { tickSiteHealth, tickSiteUserData } from "./service/account.ts";
|
|
import { BACKFILL_LANGUAGES_JOB, backfillLanguages } from "./service/backfill-languages.ts";
|
|
import { registerSourcesTasks, sourcesOutboxRoutes } from "./tasks.ts";
|
|
|
|
/**
|
|
* W2-B 来源/身份/元数据:site、site_account、feed_source;ingestFeed / resolveObservation / refreshMetadata / snapshotSite;
|
|
* observation/source_release 写入与去重;身份解析、待识别池与 decision;站点工作台 API。
|
|
* 发出 feed.due、feed.gap_suspected、release.observed(自消费)与 release.resolved、decision.created/updated。
|
|
*/
|
|
export function createSourcesIdentityModule(overrides: RuntimeOverrides = {}): ServerModule & {
|
|
runtime(ctx: ServerContext): SourcesRuntime;
|
|
} {
|
|
const runtimes = new WeakMap<ServerContext, SourcesRuntime>();
|
|
const runtime = (ctx: ServerContext) => {
|
|
let rt = runtimes.get(ctx);
|
|
if (!rt) {
|
|
rt = new SourcesRuntime(ctx, overrides);
|
|
runtimes.set(ctx, rt);
|
|
}
|
|
return rt;
|
|
};
|
|
|
|
return {
|
|
name: "sources-identity",
|
|
runtime,
|
|
registerRoutes(app, ctx) {
|
|
const rt = () => runtime(ctx);
|
|
sourcesRoutes(app, rt);
|
|
workbenchRoutes(app, rt);
|
|
identityRoutes(app, rt);
|
|
enrichRoutes(app, rt);
|
|
decisionRoutes(app, rt);
|
|
},
|
|
registerTasks(registry, ctx) {
|
|
registerSourcesTasks(registry, () => runtime(ctx));
|
|
},
|
|
outboxRoutes: () => sourcesOutboxRoutes,
|
|
registerCapabilities(registry, ctx) {
|
|
registerSourcesCapabilities(registry, () => runtime(ctx));
|
|
},
|
|
/** 周期扫描登记到定时任务注册表(schedules 模块的调度器驱动);周期是总扫描频率 */
|
|
start(ctx) {
|
|
const rt = runtime(ctx);
|
|
const everyMs = overrides.schedulerIntervalMs ?? 30_000;
|
|
const job = (key: string, name: string, description: string, fn: (rt: SourcesRuntime) => Promise<number>) =>
|
|
registerScheduledJob(ctx, {
|
|
key,
|
|
defaults: { name, description, group: "collect", schedule: { type: "interval", everyMs } },
|
|
run: async () => {
|
|
if (await readOnlyReconcile(rt)) return { status: "skipped", summary: "只读对账模式" };
|
|
return { summary: `${await fn(rt)} 项` };
|
|
},
|
|
});
|
|
job("feeds.poll", "RSS 轮询", "按各 RSS 自己的间隔抓取新资源", tickFeeds);
|
|
job("sites.snapshot", "站点快照", "扫描到周期的站点并入队快照", tickSnapshots);
|
|
job("identity.review", "未识别复查", "到期的未识别条目重新识别", tickReviews);
|
|
// W9-A 站点中心:用户数据每 6 小时、健康检测每小时(可在定时任务页关闭或改周期)
|
|
registerScheduledJob(ctx, {
|
|
key: "sites.user_data",
|
|
defaults: {
|
|
name: "站点数据刷新",
|
|
description: "刷新站点账号的上传、下载和分享率;登录失效的账密账号自动重登一次",
|
|
group: "collect",
|
|
schedule: { type: "interval", everyMs: 6 * 3_600_000 },
|
|
firstRun: "after_interval",
|
|
},
|
|
run: async ({ slot }) => {
|
|
if (await readOnlyReconcile(rt)) return { status: "skipped", summary: "只读对账模式" };
|
|
return { summary: `${await tickSiteUserData(rt, slot)} 个账号` };
|
|
},
|
|
});
|
|
registerScheduledJob(ctx, {
|
|
key: "sites.health_check",
|
|
defaults: {
|
|
name: "站点健康检测",
|
|
description: "检查到期的站点能否打开、是否还在登录(每个站点按自己的检测间隔),结果在站点页",
|
|
group: "collect",
|
|
schedule: { type: "interval", everyMs: 3_600_000 },
|
|
firstRun: "after_interval",
|
|
},
|
|
run: async () => ({ summary: `${await tickSiteHealth(rt)} 个站点` }),
|
|
});
|
|
// FIX-5 存量元数据回刷:部署后首次启动即运行,跑完自动停用;可在计划页手动再执行
|
|
registerScheduledJob(ctx, {
|
|
key: BACKFILL_LANGUAGES_JOB,
|
|
lane: "maintenance",
|
|
defaults: {
|
|
name: "元数据语言回刷",
|
|
description: "按语言设置重新拉一遍作品的标题、简介和每集信息;跑完自动停用,可手动再执行",
|
|
group: "maintain",
|
|
schedule: { type: "interval", everyMs: 24 * 3_600_000 },
|
|
firstRun: "now",
|
|
},
|
|
run: async ({ runId, jobId }) => {
|
|
if (await readOnlyReconcile(rt)) return { status: "skipped", summary: "只读对账模式" };
|
|
const r = await backfillLanguages(rt, { runId, jobId, disableWhenDone: true });
|
|
return { status: r.status, summary: r.summary };
|
|
},
|
|
});
|
|
},
|
|
};
|
|
}
|
|
|
|
export const sourcesIdentityModule = createSourcesIdentityModule();
|