feat(identity): source-aware short-drama enrichment with snippet mode and hit rate (META-2)

- classify enrich sources: encyclopedia, platform pages (hongguo, kuaikaw, video
  platforms), douban (search snippet only) and other sites' search snippets
  (snippet mode, pages never opened); one extractor per source, fields carry
  source kind and confidence, merged by noisy-OR across hosts
- new cast field (applied as MediaCredit with placeholder ids)
- robots.txt check (cached per host 1 day) and login-wall/captcha give-up
- per-enrichment source attempts + GET /identity/enrich-stats, 30-day hit-rate
  table under settings -> metadata -> identity
- setting identity.enrichSnippetAllSites (default on); default domains add
  baike.com and kuaikaw.cn
- replay fixtures from the real run on the 4 user-library short dramas
  (sanitized Brave responses, hongguo page, robots.txt)

Co-Authored-By: Claude Opus 5.5 <[email protected]>

apps/server/src/modules/agent/capabilities-metadata.ts

@@ -1,4 +1,5 @@
 import { randomUUID } from "node:crypto";
+import { fetchPage, type HostResolver } from "@umt/adapters";
 import {
   CapabilityError,
   type CapabilityRegistry,
@@ -13,6 +14,7 @@ import {
   type WebFetchOutput,
   type WebSearchOutput,
 } from "@umt/contracts";
+import { pgCacheFor } from "@umt/db";
 import { resolveRunUpstream } from "../pi/index.ts";
 import {
   AUTO_ENRICH_ACTOR,
@@ -21,6 +23,7 @@ import {
   type ModelCompletion,
   suggestEnrichment,
 } from "../sources-identity/service/enrich.ts";
+import { robotsAllows } from "../sources-identity/service/enrich-sources.ts";
 import type { ServerContext } from "../types.ts";
 import { agentSettings } from "./runner.ts";
 
@@ -31,6 +34,49 @@ import { agentSettings } from "./runner.ts";
 export interface MetadataCapabilityOptions {
   /** 模型调用(测试注入 fake);返回 null = 按规则抽取 */
   complete?: EnrichDeps["complete"];
+  /** 读 robots.txt 用的 fetch / DNS 解析(测试注入 mock) */
+  fetch?: typeof fetch;
+  resolveHost?: HostResolver;
+}
+
+const ROBOTS_REGION = "enrich.robots";
+const ROBOTS_TTL_SEC = 86400;
+
+/**
+ * META-2 robots.txt:按「协议 + 主机」缓存 1 天(kv_cache region enrich.robots)。
+ * 404 / 其它 4xx = 没有限制;5xx、超时、网络错误 = 当作不允许(这次放弃该来源,不重试)。
+ */
+export async function robotsAllowedFor(
+  ctx: Pick<ServerContext, "db">,
+  url: string,
+  opts: { fetch?: typeof fetch; resolveHost?: HostResolver; signal?: AbortSignal } = {},
+): Promise<boolean> {
+  let origin: string;
+  try {
+    origin = new URL(url).origin;
+  } catch {
+    return false;
+  }
+  const cache = pgCacheFor(ctx.db);
+  let entry = await cache.get<{ txt: string | null; ok: boolean }>(ROBOTS_REGION, origin);
+  if (!entry) {
+    try {
+      const p = await fetchPage(`${origin}/robots.txt`, {
+        maxChars: 100_000,
+        timeoutMs: 8000,
+        ...(opts.fetch ? { fetch: opts.fetch } : {}),
+        ...(opts.resolveHost ? { resolve: opts.resolveHost } : {}),
+        ...(opts.signal ? { signal: opts.signal } : {}),
+      });
+      entry = { txt: p.text, ok: true };
+    } catch (e) {
+      const status = (e as { detail?: { status?: number } }).detail?.status;
+      entry = status && status >= 400 && status < 500 ? { txt: null, ok: true } : { txt: null, ok: false };
+    }
+    await cache.set(ROBOTS_REGION, origin, entry, { ttl: ROBOTS_TTL_SEC, maxsize: 2000 });
+  }
+  if (!entry.ok) return false;
+  return entry.txt ? robotsAllows(entry.txt, url) : true;
 }
 
 /** 默认 Agent 上游做一次非流式补全;fake 上游 / 未配置上游返回 null */
@@ -107,6 +153,12 @@ export function registerMetadataCapabilities(
         return { url: p.url, finalUrl: p.finalUrl, title: p.title, text: p.text, image: p.image ?? null };
       },
       complete: opts.complete ?? ((system, user, signal) => upstreamComplete(ctx, system, user, signal)),
+      robotsAllowed: (url, signal) =>
+        robotsAllowedFor(ctx, url, {
+          ...(opts.fetch ? { fetch: opts.fetch } : {}),
+          ...(opts.resolveHost ? { resolveHost: opts.resolveHost } : {}),
+          ...(signal ? { signal } : {}),
+        }),
     };
     try {
       return await suggestEnrichment(ctx, deps, input, {