- Capabilities: all 14 registered (AO: inventory.query, download.plan/submit, organize.plan/execute,
policy.preview/apply; agent: task.inspect/retry/cancel); startup asserts count == 14.
Registry injects server-side authorize (automation_grant by action/scope/policy version/budget,
one-shot approval bound to plan id + payload hash + revision + expiry; model claims ignored;
automation switch), audit_event, server idempotency keys and a capability_invocation op ledger
(migration 0021) used for crash reconciliation. download.plan creates draft plans only;
download.submit promotes draft -> reserved after authorization.
- Agent session API: POST/GET /agent/runs, GET /agent/runs/{id} (events by seq, messages, report),
POST .../messages (wake), POST .../abort; automation-grants, approvals, capabilities invoke, tasks.
- runAgent task: Pi or fake provider from settings (apiKey via secrets), events persisted to
agent_run_event then published, resume + ledger reconciliation when pendingOperations > 0,
waits for goal-linked downloads/imports and reports from business tables (submitted != imported).
- Background triggers: decision.created -> bounded plan_only investigation (dedupe per decision);
decision.updated(resolved) -> resolveDecision executes chosen option.
- W2 closure: all OpenAPI routes registered in contracts api.ts (+ subset test); TMDB key from
settings secret_ref with env fallback; PUT /sites/{id}/credentials documented as alias of
/site-accounts; completion events documented as existing status events.
- e2e (real PG + OpenWorkflow + testkit mocks): RSS->library, agent delegation with grant/deny,
background continuation, worker kill -9 recovery (single qB add), auto investigation dedupe.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
169 lines
6.3 KiB
TypeScript
169 lines
6.3 KiB
TypeScript
import {
|
|
createRssFetcher,
|
|
createSiteAdapter,
|
|
createSiteRateLimiter,
|
|
createTmdbClient,
|
|
getSiteDefinition,
|
|
type RssFetcher,
|
|
type SecretResolver,
|
|
type SiteAdapter,
|
|
type SiteDefinition,
|
|
type SiteRateLimiter,
|
|
type TmdbClient,
|
|
} from "@umt/adapters";
|
|
import type { SiteRow } from "@umt/db";
|
|
import type { ServerContext } from "../types.ts";
|
|
|
|
// 模块运行时依赖:站点适配器与限流器(每站一个实例)、RSS 抓取器、TMDB 客户端、时钟。
|
|
// 测试通过 createSourcesIdentityModule(overrides) 注入 fetch / TMDB / 时钟。
|
|
|
|
export interface RuntimeOverrides {
|
|
/** 站点与 RSS 请求使用的 fetch(测试指向 mock) */
|
|
fetch?: typeof globalThis.fetch;
|
|
/** TMDB 客户端工厂;返回 null 表示未配置(只做本地召回) */
|
|
tmdb?: (() => TmdbClient | null) | undefined;
|
|
now?: () => Date;
|
|
/** RSS 抓取器选项覆盖(退避/熔断阈值) */
|
|
rss?: Parameters<typeof createRssFetcher>[0];
|
|
/** 调度 tick 间隔(毫秒),默认 30s */
|
|
schedulerIntervalMs?: number;
|
|
}
|
|
|
|
interface CachedAdapter {
|
|
sig: string;
|
|
limiter: SiteRateLimiter;
|
|
adapter: SiteAdapter;
|
|
}
|
|
|
|
export class SourcesRuntime {
|
|
readonly now: () => Date;
|
|
readonly rss: RssFetcher;
|
|
private readonly adapters = new Map<string, CachedAdapter>();
|
|
private tmdbClient: TmdbClient | null | undefined;
|
|
private tmdbSig: string | null = null;
|
|
|
|
constructor(
|
|
readonly ctx: ServerContext,
|
|
readonly overrides: RuntimeOverrides = {},
|
|
) {
|
|
this.now = overrides.now ?? (() => new Date());
|
|
this.rss = createRssFetcher({
|
|
...(overrides.fetch ? { fetch: overrides.fetch } : {}),
|
|
now: this.now,
|
|
...overrides.rss,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* TMDB 客户端:优先注入(测试);否则读设置 metadata.tmdbApiKeySecretRef(secret 现场解密)与 tmdbBaseUrl,
|
|
* 未配置时回退环境变量 TMDB_API_KEY / TMDB_ACCESS_TOKEN / TMDB_BASE_URL。设置变化(引用/地址)时重建。
|
|
*/
|
|
async tmdb(): Promise<TmdbClient | null> {
|
|
if (this.overrides.tmdb) {
|
|
if (this.tmdbClient === undefined) this.tmdbClient = this.overrides.tmdb();
|
|
return this.tmdbClient;
|
|
}
|
|
const meta = await this.ctx.settings.get("metadata").catch(() => null);
|
|
const env = (k: string) => Deno.env.get(k);
|
|
const ref = meta?.tmdbApiKeySecretRef ?? null;
|
|
const baseUrl = meta?.tmdbBaseUrl ?? env("TMDB_BASE_URL");
|
|
const sig = JSON.stringify([
|
|
ref,
|
|
baseUrl ?? null,
|
|
ref ? null : [!!env("TMDB_API_KEY"), !!env("TMDB_ACCESS_TOKEN")],
|
|
]);
|
|
if (this.tmdbClient !== undefined && this.tmdbSig === sig) return this.tmdbClient;
|
|
let apiKey: string | undefined;
|
|
let accessToken: string | undefined;
|
|
if (ref) {
|
|
try {
|
|
const secret = (await this.ctx.secrets.resolve(ref)).trim();
|
|
// v3 API Key 是 32 位十六进制;其余(v4 read access token,JWT)走 Bearer
|
|
if (/^[0-9a-f]{32}$/i.test(secret)) apiKey = secret;
|
|
else accessToken = secret;
|
|
} catch (err) {
|
|
this.ctx.logger.warn({ err: { message: (err as Error).message } }, "TMDB 密钥解密失败,按未配置处理");
|
|
}
|
|
} else {
|
|
apiKey = env("TMDB_API_KEY");
|
|
accessToken = env("TMDB_ACCESS_TOKEN");
|
|
}
|
|
this.tmdbSig = sig;
|
|
this.tmdbClient =
|
|
apiKey || accessToken
|
|
? createTmdbClient({
|
|
...(apiKey ? { apiKey } : {}),
|
|
...(accessToken ? { accessToken } : {}),
|
|
...(baseUrl ? { baseUrl } : {}),
|
|
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
|
|
})
|
|
: null;
|
|
return this.tmdbClient;
|
|
}
|
|
|
|
/**
|
|
* 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。
|
|
* 站点限流参数或基址变化时重建(签名比较)。
|
|
*/
|
|
siteAdapter(site: Pick<SiteRow, "id" | "key" | "base_url" | "rate_limit" | "config">): SiteAdapter {
|
|
const def = getSiteDefinition(site.key);
|
|
if (!def) throw new Error(`站点 ${site.key} 没有随产品发布的适配器定义`);
|
|
const sig = JSON.stringify([site.base_url, site.rate_limit, site.config]);
|
|
const hit = this.adapters.get(site.id);
|
|
if (hit && hit.sig === sig) return hit.adapter;
|
|
const limiter = createSiteRateLimiter(limiterOptions(def, site.rate_limit));
|
|
const cfg = (site.config ?? {}) as { apiBaseUrl?: string };
|
|
const baseUrl =
|
|
def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url;
|
|
const adapter = createSiteAdapter(def, {
|
|
secrets: this.siteSecrets(site.id),
|
|
limiter,
|
|
now: this.now,
|
|
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
|
|
...(baseUrl ? { baseUrl } : {}),
|
|
});
|
|
this.adapters.set(site.id, { sig, limiter, adapter });
|
|
return adapter;
|
|
}
|
|
|
|
/** 已建立的站点限流器(测试与诊断用) */
|
|
limiterOf(siteId: string): SiteRateLimiter | null {
|
|
return this.adapters.get(siteId)?.limiter ?? null;
|
|
}
|
|
|
|
/** 站点凭证:取该站第一个启用且未失效的账号,按 auth_kind 对应适配器要求的凭证类型;明文不缓存 */
|
|
private siteSecrets(siteId: string): SecretResolver {
|
|
const { db, secrets } = this.ctx;
|
|
return {
|
|
async resolve({ kind }) {
|
|
const want = kind === "cookie" ? ["cookie"] : kind === "apiKey" ? ["api_key"] : ["passkey"];
|
|
const acc = await db
|
|
.selectFrom("site_account")
|
|
.select(["secret_ref"])
|
|
.where("site_id", "=", siteId)
|
|
.where("enabled", "=", true)
|
|
.where("status", "in", ["unknown", "active"])
|
|
.where("auth_kind", "in", want as never)
|
|
.where("secret_ref", "is not", null)
|
|
.orderBy("created_at")
|
|
.executeTakeFirst();
|
|
if (!acc?.secret_ref) return null;
|
|
return secrets.resolve(acc.secret_ref);
|
|
},
|
|
};
|
|
}
|
|
}
|
|
|
|
function sameUrl(a: string, b: string): boolean {
|
|
return a.replace(/\/+$/, "") === b.replace(/\/+$/, "");
|
|
}
|
|
|
|
/** 站点行 rate_limit 优先,缺省用定义里的 umt.rateLimit */
|
|
export function limiterOptions(def: SiteDefinition, rateLimit: unknown): { minIntervalMs: number; burst: number } {
|
|
const rl = (rateLimit ?? {}) as { requestsPerMinute?: number; minIntervalMs?: number; concurrency?: number };
|
|
const minIntervalMs =
|
|
rl.minIntervalMs ??
|
|
(rl.requestsPerMinute ? Math.ceil(60_000 / rl.requestsPerMinute) : def.umt.rateLimit.minIntervalMs);
|
|
return { minIntervalMs, burst: rl.concurrency ?? def.umt.rateLimit.burst };
|
|
}
|