- Sites / accounts / feeds: registry view (/site-definitions), site CRUD with
enable/disable (history stays queryable, marked site_disabled), accounts and
feed URLs stored as secrets (only secret_ref / redacted urlDisplay exposed),
feed enable/disable/reset/fetch; one shared rate limiter + adapter per site.
- Tasks: ingestFeed (FeedFetchState/FeedCursor on feed_source, observation
feed+guid upsert with digest-change updates, source_release site+id
aggregation across feeds, auth failure -> account invalid + decision, no
retry, gap -> feed.gap_suspected -> budgeted backfill snapshot),
resolveObservation (parseRelease -> local recall via title_alias pg_trgm +
accepted identities -> TMDB -> resolveIdentity; auto binds entity /
provisional units / identities / aliases, otherwise identity_candidate pool
with evidence version and bounded review backoff; manual lock respected),
refreshMetadata (metadata_field + field_override, user locks win, paths
untouched), snapshotSite (site_snapshot coverage records).
- Events: feed.due, feed.gap_suspected, release.observed (self-consumed),
release.resolved, decision.created/updated; scheduler in worker start().
- Identity API: entity detail with redirects, manual mapping correction with
identity_revision, merge with identity_redirect, pool list/assign/dismiss/
retry, content unit numbering lock, aliases, external identities.
- Decisions: list/summary (by entity and kind)/detail/resolve; identity
options executed directly.
- Workbench: /sites/{id}/releases (snapshot first, coalesced live search with
backoff, origin + freshness) and /sites/{id}/releases/{rid} (volatile field
recheck times, refresh keeps data on failure).
- Capabilities: catalog.lookup, release.search, release.inspect,
identity.resolve.
- Migrations 0018-0020: site.adapter api, source_release.resolution_locked,
site_snapshot, identity_candidate, resolution_attempt, identity_redirect,
metadata_field, field_override.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
144 lines
5.3 KiB
TypeScript
144 lines
5.3 KiB
TypeScript
import {
|
|
createRssFetcher,
|
|
createSiteAdapter,
|
|
createSiteRateLimiter,
|
|
createTmdbClient,
|
|
getSiteDefinition,
|
|
type RssFetcher,
|
|
type SecretResolver,
|
|
type SiteAdapter,
|
|
type SiteDefinition,
|
|
type SiteRateLimiter,
|
|
type TmdbClient,
|
|
} from "@umt/adapters";
|
|
import type { SiteRow } from "@umt/db";
|
|
import type { ServerContext } from "../types.ts";
|
|
|
|
// 模块运行时依赖:站点适配器与限流器(每站一个实例)、RSS 抓取器、TMDB 客户端、时钟。
|
|
// 测试通过 createSourcesIdentityModule(overrides) 注入 fetch / TMDB / 时钟。
|
|
|
|
export interface RuntimeOverrides {
|
|
/** 站点与 RSS 请求使用的 fetch(测试指向 mock) */
|
|
fetch?: typeof globalThis.fetch;
|
|
/** TMDB 客户端工厂;返回 null 表示未配置(只做本地召回) */
|
|
tmdb?: (() => TmdbClient | null) | undefined;
|
|
now?: () => Date;
|
|
/** RSS 抓取器选项覆盖(退避/熔断阈值) */
|
|
rss?: Parameters<typeof createRssFetcher>[0];
|
|
/** 调度 tick 间隔(毫秒),默认 30s */
|
|
schedulerIntervalMs?: number;
|
|
}
|
|
|
|
interface CachedAdapter {
|
|
sig: string;
|
|
limiter: SiteRateLimiter;
|
|
adapter: SiteAdapter;
|
|
}
|
|
|
|
export class SourcesRuntime {
|
|
readonly now: () => Date;
|
|
readonly rss: RssFetcher;
|
|
private readonly adapters = new Map<string, CachedAdapter>();
|
|
private tmdbClient: TmdbClient | null | undefined;
|
|
|
|
constructor(
|
|
readonly ctx: ServerContext,
|
|
readonly overrides: RuntimeOverrides = {},
|
|
) {
|
|
this.now = overrides.now ?? (() => new Date());
|
|
this.rss = createRssFetcher({
|
|
...(overrides.fetch ? { fetch: overrides.fetch } : {}),
|
|
now: this.now,
|
|
...overrides.rss,
|
|
});
|
|
}
|
|
|
|
/** TMDB:优先注入;否则读环境变量 TMDB_API_KEY / TMDB_ACCESS_TOKEN(W2-A 设置页接管前的过渡) */
|
|
tmdb(): TmdbClient | null {
|
|
if (this.tmdbClient !== undefined) return this.tmdbClient;
|
|
if (this.overrides.tmdb) {
|
|
this.tmdbClient = this.overrides.tmdb();
|
|
return this.tmdbClient;
|
|
}
|
|
const env = (k: string) => Deno.env.get(k);
|
|
const apiKey = env("TMDB_API_KEY");
|
|
const accessToken = env("TMDB_ACCESS_TOKEN");
|
|
const baseUrl = env("TMDB_BASE_URL");
|
|
this.tmdbClient =
|
|
apiKey || accessToken
|
|
? createTmdbClient({
|
|
...(apiKey ? { apiKey } : {}),
|
|
...(accessToken ? { accessToken } : {}),
|
|
...(baseUrl ? { baseUrl } : {}),
|
|
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
|
|
})
|
|
: null;
|
|
return this.tmdbClient;
|
|
}
|
|
|
|
/**
|
|
* 站点适配器:每站一个实例,列表/搜索/详情/下载共享同一个限流器。
|
|
* 站点限流参数或基址变化时重建(签名比较)。
|
|
*/
|
|
siteAdapter(site: Pick<SiteRow, "id" | "key" | "base_url" | "rate_limit" | "config">): SiteAdapter {
|
|
const def = getSiteDefinition(site.key);
|
|
if (!def) throw new Error(`站点 ${site.key} 没有随产品发布的适配器定义`);
|
|
const sig = JSON.stringify([site.base_url, site.rate_limit, site.config]);
|
|
const hit = this.adapters.get(site.id);
|
|
if (hit && hit.sig === sig) return hit.adapter;
|
|
const limiter = createSiteRateLimiter(limiterOptions(def, site.rate_limit));
|
|
const cfg = (site.config ?? {}) as { apiBaseUrl?: string };
|
|
const baseUrl =
|
|
def.umt.kind === "mteam" ? cfg.apiBaseUrl : sameUrl(site.base_url, def.domain) ? undefined : site.base_url;
|
|
const adapter = createSiteAdapter(def, {
|
|
secrets: this.siteSecrets(site.id),
|
|
limiter,
|
|
now: this.now,
|
|
...(this.overrides.fetch ? { fetch: this.overrides.fetch } : {}),
|
|
...(baseUrl ? { baseUrl } : {}),
|
|
});
|
|
this.adapters.set(site.id, { sig, limiter, adapter });
|
|
return adapter;
|
|
}
|
|
|
|
/** 已建立的站点限流器(测试与诊断用) */
|
|
limiterOf(siteId: string): SiteRateLimiter | null {
|
|
return this.adapters.get(siteId)?.limiter ?? null;
|
|
}
|
|
|
|
/** 站点凭证:取该站第一个启用且未失效的账号,按 auth_kind 对应适配器要求的凭证类型;明文不缓存 */
|
|
private siteSecrets(siteId: string): SecretResolver {
|
|
const { db, secrets } = this.ctx;
|
|
return {
|
|
async resolve({ kind }) {
|
|
const want = kind === "cookie" ? ["cookie"] : kind === "apiKey" ? ["api_key"] : ["passkey"];
|
|
const acc = await db
|
|
.selectFrom("site_account")
|
|
.select(["secret_ref"])
|
|
.where("site_id", "=", siteId)
|
|
.where("enabled", "=", true)
|
|
.where("status", "in", ["unknown", "active"])
|
|
.where("auth_kind", "in", want as never)
|
|
.where("secret_ref", "is not", null)
|
|
.orderBy("created_at")
|
|
.executeTakeFirst();
|
|
if (!acc?.secret_ref) return null;
|
|
return secrets.resolve(acc.secret_ref);
|
|
},
|
|
};
|
|
}
|
|
}
|
|
|
|
function sameUrl(a: string, b: string): boolean {
|
|
return a.replace(/\/+$/, "") === b.replace(/\/+$/, "");
|
|
}
|
|
|
|
/** 站点行 rate_limit 优先,缺省用定义里的 umt.rateLimit */
|
|
export function limiterOptions(def: SiteDefinition, rateLimit: unknown): { minIntervalMs: number; burst: number } {
|
|
const rl = (rateLimit ?? {}) as { requestsPerMinute?: number; minIntervalMs?: number; concurrency?: number };
|
|
const minIntervalMs =
|
|
rl.minIntervalMs ??
|
|
(rl.requestsPerMinute ? Math.ceil(60_000 / rl.requestsPerMinute) : def.umt.rateLimit.minIntervalMs);
|
|
return { minIntervalMs, burst: rl.concurrency ?? def.umt.rateLimit.burst };
|
|
}
|