From 81bf010e3e248fc41a1c69c8a966608c099e0917 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 28 Sep 2026 16:02:30 +0800 Subject: [PATCH] feat(sources,organize): site agent capabilities and organize.minSizeBytes setting (W10-pre2) - site.list / site.health_check / site.refresh_login / site.user_data capabilities over the W9-A site center services; outputs whitelisted (no secret refs, cookies, passwords, login names); execute kinds need automation grant. IDs use snake_case (CapabilityId regex); registry count 17 -> 21. - settings.organize.minSizeBytes (default 50 MiB); file-manager identify / organize and planImport take request > setting > port default. - tests and runbook notes (sites, files). Co-Authored-By: Claude Opus 5.5 --- .../acquisition-organize/__tests__/harness.ts | 7 +- .../acquisition-organize/services/organize.ts | 2 +- .../src/modules/file-manager/identify.ts | 2 +- .../src/modules/file-manager/organize.ts | 2 +- .../sources-identity/capabilities-sites.ts | 123 +++++++++++++ .../modules/sources-identity/capabilities.ts | 5 +- .../server/test/e2e/e2-agent-delegate.test.ts | 4 +- apps/server/test/e2e/harness.ts | 2 + apps/server/test/file-manager/harness.ts | 7 +- .../file-manager/identify-organize.test.ts | 10 +- .../file-manager/min-size-setting.test.ts | 73 ++++++++ .../site-capabilities.test.ts | 165 ++++++++++++++++++ docs/runbook/files.md | 2 +- docs/runbook/sites.md | 16 ++ packages/contracts/src/capabilities-io.ts | 94 +++++++++- packages/contracts/src/capability.ts | 55 +++++- packages/contracts/src/settings.ts | 5 +- packages/contracts/test/schemas.test.ts | 4 +- 18 files changed, 563 insertions(+), 15 deletions(-) create mode 100644 apps/server/src/modules/sources-identity/capabilities-sites.ts create mode 100644 apps/server/test/file-manager/min-size-setting.test.ts create mode 100644 apps/server/test/sources-identity/site-capabilities.test.ts diff --git a/apps/server/src/modules/acquisition-organize/__tests__/harness.ts b/apps/server/src/modules/acquisition-organize/__tests__/harness.ts index 6ef2e1ad..9e689fed 100644 --- a/apps/server/src/modules/acquisition-organize/__tests__/harness.ts +++ b/apps/server/src/modules/acquisition-organize/__tests__/harness.ts @@ -172,7 +172,12 @@ export async function createHarness(name: string, overrides: AoOverrides = {}): const out: Record = {}; for (const k of Object.keys(DEFAULT_SETTINGS) as SettingsKey[]) { const v = stored.get(k); - out[k] = GlobalSettings.shape[k].parse({ ...DEFAULT_SETTINGS[k], ...((v as object) ?? {}) }); + // 整理最小体积缺省跟随端口注入的 minVideoBytes(测试用小文件) + const base = + k === "organize" + ? { ...DEFAULT_SETTINGS.organize, minSizeBytes: overrides.options?.minVideoBytes ?? 100_000 } + : DEFAULT_SETTINGS[k]; + out[k] = GlobalSettings.shape[k].parse({ ...base, ...((v as object) ?? {}) }); } return out as GlobalSettings; }, diff --git a/apps/server/src/modules/acquisition-organize/services/organize.ts b/apps/server/src/modules/acquisition-organize/services/organize.ts index 41bd3a01..1123bc3b 100644 --- a/apps/server/src/modules/acquisition-organize/services/organize.ts +++ b/apps/server/src/modules/acquisition-organize/services/organize.ts @@ -524,7 +524,7 @@ export async function planImport(d: AoDeps, req: ImportRequest): Promise { const db = d.ctx.db; const { videos, truncated } = await collectVideos(d, body); - // G2:最小体积(请求 > settings.organize.minSizeBytes > 不过滤) + // G2:最小体积(请求/向导 > settings.organize.minSizeBytes,缺省 50 MiB;设置读不到时不过滤) const orgSettings = (await d.ctx.settings.get("organize").catch(() => null)) as { minSizeBytes?: unknown } | null; const minSize = body.minSizeBytes ?? (typeof orgSettings?.minSizeBytes === "number" ? orgSettings.minSizeBytes : 0); let excludedBySize = 0; diff --git a/apps/server/src/modules/file-manager/organize.ts b/apps/server/src/modules/file-manager/organize.ts index 3be54c33..0afb6724 100644 --- a/apps/server/src/modules/file-manager/organize.ts +++ b/apps/server/src/modules/file-manager/organize.ts @@ -118,7 +118,7 @@ function segmentsOf(plan: FrozenPlan | undefined, overrideDir: boolean): FilesOr ); } -/** 设置里的最小体积(settings.organize.minSizeBytes;契约未加该字段前恒为 undefined → planImport 用端口默认) */ +/** 设置里的最小体积(settings.organize.minSizeBytes,缺省 50 MiB);读不到时 undefined → planImport 再读设置 / 端口默认 */ async function settingsMinSize(d: FmDeps): Promise { const org = (await d.ctx.settings.get("organize").catch(() => null)) as { minSizeBytes?: unknown } | null; return typeof org?.minSizeBytes === "number" && org.minSizeBytes >= 0 ? org.minSizeBytes : undefined; diff --git a/apps/server/src/modules/sources-identity/capabilities-sites.ts b/apps/server/src/modules/sources-identity/capabilities-sites.ts new file mode 100644 index 00000000..03c95b25 --- /dev/null +++ b/apps/server/src/modules/sources-identity/capabilities-sites.ts @@ -0,0 +1,123 @@ +import type { CapabilityRegistry } from "@umt/agent"; +import { capabilitySpec, type SiteAccountUserData, type SiteLastHealth } from "@umt/contracts"; +import type { SiteAccountRow, SiteRow } from "@umt/db"; +import type { SourcesRuntime } from "./runtime.ts"; +import { checkSiteHealth, overallAccountStatus, refreshSiteAccount, userDataHistory } from "./service/account.ts"; + +// W10-pre2 站点能力:site.list / site.health_check / site.refresh_login / site.user_data,处理器调 W9-A 站点中心的 service。 +// 脱敏:输出按白名单逐字段拼装——不含 secret 引用、Cookie、密码、登录名、UA 与站点 config。 + +const iso = (d: Date | null) => (d ? d.toISOString() : null); + +function accountBrief(a: SiteAccountRow) { + return { + id: a.id, + label: a.label, + authKind: a.auth_kind, + enabled: a.enabled, + status: a.status, + lastLoginAt: iso(a.last_login_at), + lastVerifiedAt: iso(a.last_verified_at), + lastError: (a.last_error ?? null) as never, + userDataAt: iso(a.user_data_at), + hasSession: !!a.session_secret_ref, + hasPasskey: !!a.passkey_secret_ref || (a.auth_kind === "passkey" && !!a.secret_ref), + }; +} + +async function liveSite(r: SourcesRuntime, siteId: string): Promise { + const s = await r.ctx.db + .selectFrom("site") + .selectAll() + .where("id", "=", siteId) + .where("deleted_at", "is", null) + .executeTakeFirst(); + if (!s) throw new Error("站点不存在"); + return s; +} + +export function registerSiteCapabilities(registry: CapabilityRegistry, rt: () => SourcesRuntime): void { + registry.register(capabilitySpec("site.list"), async (input) => { + const { db } = rt().ctx; + let q = db.selectFrom("site").selectAll().where("deleted_at", "is", null); + if (!input.includeDisabled) q = q.where("enabled", "=", true); + if (input.keys?.length) q = q.where("key", "in", input.keys); + const sites = await q.orderBy("key").execute(); + const accounts = sites.length + ? await db + .selectFrom("site_account") + .selectAll() + .where( + "site_id", + "in", + sites.map((s) => s.id), + ) + .orderBy("created_at") + .execute() + : []; + return { + items: sites.map((s) => { + const mine = accounts.filter((a) => a.site_id === s.id); + return { + id: s.id, + key: s.key, + name: s.name, + adapter: s.adapter, + baseUrl: s.base_url, + enabled: s.enabled, + healthStatus: s.health_status, + lastHealth: (s.last_health ?? null) as SiteLastHealth | null, + lastHealthAt: iso(s.last_health_at), + accountStatus: overallAccountStatus(mine), + accounts: mine.map(accountBrief), + }; + }), + }; + }); + + registry.register(capabilitySpec("site.health_check"), async (input, ctx) => { + const r = rt(); + return checkSiteHealth(r, await liveSite(r, input.siteId), { type: "agent", id: ctx.actorId }); + }); + + registry.register(capabilitySpec("site.refresh_login"), async (input, ctx) => { + const r = rt(); + const s = await liveSite(r, input.siteId); + let q = r.ctx.db.selectFrom("site_account").select(["id"]).where("site_id", "=", s.id); + q = input.accountId + ? q.where("id", "=", input.accountId) + : q.where("enabled", "=", true).where("auth_kind", "!=", "passkey"); + const acc = await q.orderBy("created_at").executeTakeFirst(); + if (!acc) throw new Error(input.accountId ? "站点账号不存在" : "站点还没有凭证"); + return refreshSiteAccount(r, acc.id, { + mode: "login", + actor: { type: "agent", id: ctx.actorId }, + signal: AbortSignal.any([ctx.signal, AbortSignal.timeout(30_000)]), + }); + }); + + registry.register(capabilitySpec("site.user_data"), async (input) => { + const r = rt(); + const s = await liveSite(r, input.siteId); + let q = r.ctx.db + .selectFrom("site_account") + .select(["id", "label", "status", "user_data", "user_data_at"]) + .where("site_id", "=", s.id); + if (input.accountId) q = q.where("id", "=", input.accountId); + const accounts = await q.orderBy("created_at").execute(); + return { + siteId: s.id, + accounts: accounts.map((a) => ({ + accountId: a.id, + label: a.label, + status: a.status, + userData: (a.user_data ?? null) as SiteAccountUserData | null, + userDataAt: iso(a.user_data_at), + })), + history: await userDataHistory(r, s.id, { + ...(input.accountId ? { accountId: input.accountId } : {}), + days: input.days, + }), + }; + }); +} diff --git a/apps/server/src/modules/sources-identity/capabilities.ts b/apps/server/src/modules/sources-identity/capabilities.ts index 3d425bb0..62712cab 100644 --- a/apps/server/src/modules/sources-identity/capabilities.ts +++ b/apps/server/src/modules/sources-identity/capabilities.ts @@ -2,6 +2,7 @@ import type { CapabilityRegistry } from "@umt/agent"; import { capabilitySpec, type ExternalEntityType, type SourceRelease } from "@umt/contracts"; import { sql } from "@umt/db"; import { normalizeTitle } from "@umt/domain"; +import { registerSiteCapabilities } from "./capabilities-sites.ts"; import { toApi, toApiList } from "./rows.ts"; import type { SourcesRuntime } from "./runtime.ts"; import { assignReleaseIdentity } from "./service/identity-admin.ts"; @@ -9,10 +10,12 @@ import { resolveRelease } from "./service/resolve.ts"; import { refreshReleaseDetail } from "./service/snapshot.ts"; import { liveSearch } from "./service/workbench.ts"; -// 本模块领域的能力处理器(规格一律取 contracts):catalog.lookup / release.search / release.inspect / identity.resolve。 +// 本模块领域的能力处理器(规格一律取 contracts):catalog.lookup / release.search / release.inspect / identity.resolve; +// 站点能力 site.* 见 capabilities-sites.ts。 // identity.resolve 属 propose 类,授权回调由 W2-D 接入前会被注册中心拒绝(安全默认)。 export function registerSourcesCapabilities(registry: CapabilityRegistry, rt: () => SourcesRuntime): void { + registerSiteCapabilities(registry, rt); registry.register(capabilitySpec("catalog.lookup"), async (input) => { const { db } = rt().ctx; const local: { entity: unknown; score: number; matchedAlias: string | null }[] = []; diff --git a/apps/server/test/e2e/e2-agent-delegate.test.ts b/apps/server/test/e2e/e2-agent-delegate.test.ts index e5b5d216..cb336657 100644 --- a/apps/server/test/e2e/e2-agent-delegate.test.ts +++ b/apps/server/test/e2e/e2-agent-delegate.test.ts @@ -42,8 +42,8 @@ describe.skipIf(!hasPg)("E2 Agent 委托闭环(fake 模型 + 真实能力/授权 return r.body.data.id as string; }; - it("启动断言:17 个能力全部注册,并在能力目录里可见", async () => { - expect(e.booted.ctx.capabilities.list()).toHaveLength(17); + it("启动断言:21 个能力全部注册,并在能力目录里可见", async () => { + expect(e.booted.ctx.capabilities.list()).toHaveLength(21); const r = await e.api("GET", "/capabilities"); expect(r.body.data.items.map((c: { id: string }) => c.id).sort()).toEqual( e.booted.ctx.capabilities diff --git a/apps/server/test/e2e/harness.ts b/apps/server/test/e2e/harness.ts index 479ac2eb..014eb190 100644 --- a/apps/server/test/e2e/harness.ts +++ b/apps/server/test/e2e/harness.ts @@ -235,6 +235,8 @@ export async function createE2E(name: string, opts: E2EOptions = {}): Promise = {}; for (const k of Object.keys(DEFAULT_SETTINGS) as SettingsKey[]) { const v = stored.get(k); - out[k] = GlobalSettings.shape[k].parse({ ...DEFAULT_SETTINGS[k], ...((v as object) ?? {}) }); + // 整理最小体积缺省跟随端口注入的 minVideoBytes(测试用小文件) + const base = + k === "organize" + ? { ...DEFAULT_SETTINGS.organize, minSizeBytes: overrides.ao?.options?.minVideoBytes ?? 10 } + : DEFAULT_SETTINGS[k]; + out[k] = GlobalSettings.shape[k].parse({ ...base, ...((v as object) ?? {}) }); } return out as GlobalSettings; }, diff --git a/apps/server/test/file-manager/identify-organize.test.ts b/apps/server/test/file-manager/identify-organize.test.ts index 9426e793..59b5741b 100644 --- a/apps/server/test/file-manager/identify-organize.test.ts +++ b/apps/server/test/file-manager/identify-organize.test.ts @@ -53,7 +53,7 @@ describe.skipIf(skip)("files 识别 / 整理 / 探测 / 刮削", () => { const r = await h.api( "POST", "/files/identify", - { items: [{ root: h.dlRoot, path: "Some.Show.S01.1080p.WEB-DL" }] }, + { items: [{ root: h.dlRoot, path: "Some.Show.S01.1080p.WEB-DL" }], minSizeBytes: 0 }, READ_KEY, ); expect(r.status).toBe(200); @@ -72,6 +72,7 @@ describe.skipIf(skip)("files 识别 / 整理 / 探测 / 刮削", () => { it("identify:远端候选(catalog.lookup)", async () => { const r = await h.api("POST", "/files/identify", { items: [{ root: h.dlRoot, path: "Remote.Film.2019.1080p.mkv" }], + minSizeBytes: 0, }); expect(r.status).toBe(200); const c = r.body.data.items[0].candidates; @@ -82,7 +83,12 @@ describe.skipIf(skip)("files 识别 / 整理 / 探测 / 刮削", () => { await mkdir(join(h.dl, "many"), { recursive: true }); const names = Array.from({ length: 23 }, (_, i) => `Distinct Title ${String.fromCharCode(65 + i)}.2019.1080p.mkv`); for (const n of names) await writeFile(join(h.dl, "many", n), "x"); - const r = await h.api("POST", "/files/identify", { items: [{ root: h.dlRoot, path: "many" }] }, READ_KEY); + const r = await h.api( + "POST", + "/files/identify", + { items: [{ root: h.dlRoot, path: "many" }], minSizeBytes: 0 }, + READ_KEY, + ); expect(r.status).toBe(200); expect(r.body.data.items).toHaveLength(23); const capped = r.body.data.items.filter((i: { warnings: string[] }) => i.warnings.some((w) => w.includes("上限"))); diff --git a/apps/server/test/file-manager/min-size-setting.test.ts b/apps/server/test/file-manager/min-size-setting.test.ts new file mode 100644 index 00000000..933eafaa --- /dev/null +++ b/apps/server/test/file-manager/min-size-setting.test.ts @@ -0,0 +1,73 @@ +import { writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { DEFAULT_SETTINGS, GlobalSettings } from "@umt/contracts"; +import { sql } from "@umt/db"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createFmHarness, EXEC_KEY, type FmHarness, key, pgAvailable, READ_KEY, seedEntity } from "./harness.ts"; + +// W10-pre2:settings.organize.minSizeBytes(缺省 50 MiB)是识别与整理(planImport)的默认最小体积;请求 / 向导给的值优先。 + +const skip = !(await pgAvailable()); + +describe("settings.organize.minSizeBytes 契约", () => { + it("缺省 50 MiB;非负整数;旧的 organize 设置补缺省后仍合法", () => { + expect(DEFAULT_SETTINGS.organize.minSizeBytes).toBe(50 * 1024 * 1024); + expect(GlobalSettings.shape.organize.safeParse({ ...DEFAULT_SETTINGS.organize, minSizeBytes: -1 }).success).toBe( + false, + ); + expect( + GlobalSettings.shape.organize.parse({ ...DEFAULT_SETTINGS.organize, defaultMode: "move" }).minSizeBytes, + ).toBe(50 * 1024 * 1024); + }); +}); + +describe.skipIf(skip)("最小体积读设置(W10-pre2)", () => { + let h: FmHarness; + let film: string; + const setMin = (n: number) => + sql`insert into settings (key, value) values ('organize', ${JSON.stringify({ minSizeBytes: n })}::jsonb) + on conflict (key) do update set value = excluded.value`.execute(h.db); + + beforeAll(async () => { + h = await createFmHarness("minsize"); + film = await seedEntity(h.db, { kind: "movie", title: "Size Film", year: 2022 }); + await writeFile(join(h.dl, "Size.Film.2022.2160p.BluRay.x265-GRP.mkv"), "x".repeat(64)); + await writeFile(join(h.dl, "Size.Film.2022.1080p.small.mkv"), "0123456789ab"); + }); + afterAll(async () => { + await h?.close(); + }); + + const items = () => [ + { root: h.dlRoot, path: "Size.Film.2022.2160p.BluRay.x265-GRP.mkv", mediaEntityId: film }, + { root: h.dlRoot, path: "Size.Film.2022.1080p.small.mkv", mediaEntityId: film }, + ]; + const organize = (body: Record) => + h.api("POST", "/files/organize", { idempotencyKey: key(), action: "copy", dryRun: true, ...body }, EXEC_KEY); + + it("整理:不给 minSizeBytes 时用设置值;请求给的值优先", async () => { + await setMin(32); + const r = await organize({ items: items() }); + expect(r.status).toBe(200); + const g = r.body.data.groups[0]; + expect(g.minSizeBytes).toBe(32); + expect(g.files.map((f: { source: string }) => basename(f.source))).toEqual([ + "Size.Film.2022.2160p.BluRay.x265-GRP.mkv", + ]); + expect(g.excluded).toContainEqual(expect.objectContaining({ size: 12, reason: "below_min_size" })); + + const over = await organize({ items: items(), minSizeBytes: 0 }); + expect(over.body.data.groups[0].excluded).toEqual([]); + expect(over.body.data.groups[0].files).toHaveLength(2); + }); + + it("识别:不给 minSizeBytes 时按设置跳过小文件;请求 0 不过滤", async () => { + await setMin(32); + const body = { items: items().map(({ root, path }) => ({ root, path })) }; + const r = await h.api("POST", "/files/identify", body, READ_KEY); + expect(r.status).toBe(200); + expect(r.body.data.excludedBySize).toBe(1); + const r0 = await h.api("POST", "/files/identify", { ...body, minSizeBytes: 0 }, READ_KEY); + expect(r0.body.data.excludedBySize).toBe(0); + }); +}); diff --git a/apps/server/test/sources-identity/site-capabilities.test.ts b/apps/server/test/sources-identity/site-capabilities.test.ts new file mode 100644 index 00000000..5a9804e4 --- /dev/null +++ b/apps/server/test/sources-identity/site-capabilities.test.ts @@ -0,0 +1,165 @@ +import { randomUUID } from "node:crypto"; +import type { CapabilityRegistry, InvokeContext } from "@umt/agent"; +import { CAPABILITY_IDS, capabilitySpec } from "@umt/contracts"; +import { createMockNexusPhpSite, type MockNexusPhpSite } from "@umt/testkit"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createCapabilityRegistry } from "../../src/modules/agent/capability-registry.ts"; +import { registerSourcesCapabilities } from "../../src/modules/sources-identity/capabilities.ts"; +import { createHarness, type Harness, pgAvailable } from "./harness.ts"; + +// W10-pre2 站点能力:site.list / site.health_check / site.refresh_login / site.user_data。 +// 规格(类别 / 风险 / 重放)、execute 类要持续授权、输出不含凭证与 Cookie。 + +const up = await pgAvailable(); +const TiB = 1024 ** 4; +const SITE_IDS = ["site.list", "site.health_check", "site.refresh_login", "site.user_data"] as const; + +describe("站点能力规格", () => { + it("进入首批能力目录(17 + 4 = 21);类别 / 风险 / 重放按约定", () => { + expect(CAPABILITY_IDS).toHaveLength(21); + for (const id of SITE_IDS) expect(CAPABILITY_IDS).toContain(id); + expect(capabilitySpec("site.list")).toMatchObject({ kind: "read", riskLevel: "low", replay: "safe" }); + expect(capabilitySpec("site.user_data")).toMatchObject({ kind: "read", riskLevel: "low", replay: "safe" }); + expect(capabilitySpec("site.health_check")).toMatchObject({ + kind: "execute", + riskLevel: "low", + replay: "safe", + idempotent: true, + }); + expect(capabilitySpec("site.refresh_login")).toMatchObject({ + kind: "execute", + riskLevel: "medium", + replay: "never", + }); + }); +}); + +describe.skipIf(!up)("站点能力(mock NexusPHP)", () => { + let h: Harness; + let mock: MockNexusPhpSite; + let registry: CapabilityRegistry; + let siteId: string; + const PASSWORD = "cap-s3cret-pw"; + + const agentCtx = (): InvokeContext => ({ + actorId: "agent:site-test", + taskId: randomUUID(), + scopeIds: [], + signal: new AbortController().signal, + source: "agent", + }); + /** 输出里不许出现的东西:密码、会话 Cookie、secret 引用 */ + const assertRedacted = async (out: unknown) => { + const text = JSON.stringify(out); + expect(text).not.toContain(PASSWORD); + expect(text).not.toMatch(/secretRef|secret_ref|cookie|loginName|userAgent/i); + for (const v of h.secrets.plain.values()) expect(text).not.toContain(v); + for (const ref of h.secrets.plain.keys()) expect(text).not.toContain(ref); + }; + + beforeAll(async () => { + mock = await createMockNexusPhpSite({ username: "alice", password: PASSWORD }); + h = await createHarness("sitecaps"); + registry = createCapabilityRegistry({ logger: h.ctx.logger, db: h.db, settings: h.ctx.settings }); + registerSourcesCapabilities(registry, () => h.rt); + const r = await h.req("/sites", { + body: { + key: "audiences", + name: "观众", + adapter: "nexusphp", + baseUrl: `${mock.baseUrl}/`, + rateLimit: { minIntervalMs: 0, concurrency: 10 }, + }, + }); + siteId = ((await r.json()) as { data: { id: string } }).data.id; + const c = await h.req(`/sites/${siteId}/credentials`, { + method: "PUT", + body: { kind: "password", username: "alice", password: PASSWORD }, + }); + expect(c.status).toBe(200); + }); + afterAll(async () => { + await h?.close(); + await mock?.close(); + }); + + it("四个站点能力都挂上处理器", () => { + const ids = registry.list().map((c) => c.descriptor.id); + for (const id of SITE_IDS) expect(ids).toContain(id); + }); + + it("site.refresh_login:没有持续授权 → 拒绝,账号不动", async () => { + await expect(registry.invoke("site.refresh_login", { siteId }, agentCtx())).rejects.toThrow(); + const acc = await h.db.selectFrom("site_account").select(["status"]).where("site_id", "=", siteId).execute(); + expect(acc.map((a) => a.status)).toEqual(["unknown"]); + }); + + it("site.refresh_login:有持续授权 → 登录成功并拉用户数据;输出脱敏", async () => { + await h.db.insertInto("automation_grant").values({ action: "site.refresh_login", granted_by: "test" }).execute(); + const out = (await registry.invoke("site.refresh_login", { siteId }, agentCtx())) as { + ok: boolean; + status: string; + userData: { upload: number } | null; + }; + expect(out).toMatchObject({ ok: true, status: "active", reason: null }); + expect(out.userData?.upload).toBe(12.5 * TiB); + await assertRedacted(out); + const audits = await h.db + .selectFrom("audit_event") + .select(["action"]) + .where("target_id", "=", "site.refresh_login") + .execute(); + expect(audits.map((a) => a.action)).toEqual(expect.arrayContaining(["capability.denied", "capability.invoke"])); + }); + + it("site.list:健康、账号状态与凭证有无;不含凭证、Cookie、secret 引用", async () => { + const out = (await registry.invoke("site.list", {}, agentCtx())) as { + items: { id: string; accountStatus: string; accounts: { hasSession: boolean; status: string }[] }[]; + }; + const item = out.items.find((s) => s.id === siteId); + expect(item).toMatchObject({ key: "audiences", enabled: true, accountStatus: "active" }); + expect(item?.accounts[0]).toMatchObject({ status: "active", hasSession: true, authKind: "password" }); + await assertRedacted(out); + // 停用站点缺省不列 + await h.db.updateTable("site").set({ enabled: false }).where("id", "=", siteId).execute(); + const hidden = (await registry.invoke("site.list", {}, agentCtx())) as { items: unknown[] }; + expect(hidden.items).toHaveLength(0); + const all = (await registry.invoke("site.list", { includeDisabled: true }, agentCtx())) as { items: unknown[] }; + expect(all.items).toHaveLength(1); + await h.db.updateTable("site").set({ enabled: true }).where("id", "=", siteId).execute(); + }); + + it("site.user_data:最新用户数据 + 按天历史", async () => { + const out = (await registry.invoke("site.user_data", { siteId }, agentCtx())) as { + accounts: { userData: { seeding: number } | null }[]; + history: { upload: number }[]; + }; + expect(out.accounts).toHaveLength(1); + expect(out.accounts[0]?.userData?.seeding).toBe(31); + expect(out.history).toHaveLength(1); + expect(out.history[0]?.upload).toBe(12.5 * TiB); + await assertRedacted(out); + }); + + it("site.health_check:execute 同样要授权;授权后写回健康列", async () => { + await expect(registry.invoke("site.health_check", { siteId }, agentCtx())).rejects.toThrow(); + await h.db.insertInto("automation_grant").values({ action: "site.health_check", granted_by: "test" }).execute(); + const out = (await registry.invoke("site.health_check", { siteId }, agentCtx())) as { + ok: boolean; + healthStatus: string; + }; + expect(out).toMatchObject({ siteId, ok: true, loggedIn: true, healthStatus: "healthy" }); + await assertRedacted(out); + const s = await h.db + .selectFrom("site") + .select(["health_status", "last_health_at"]) + .where("id", "=", siteId) + .executeTakeFirstOrThrow(); + expect(s.health_status).toBe("healthy"); + expect(s.last_health_at).toBeTruthy(); + }); + + it("不存在的站点:明确报错", async () => { + await expect(registry.invoke("site.user_data", { siteId: randomUUID() }, agentCtx())).rejects.toThrow(/站点不存在/); + }); +}); diff --git a/docs/runbook/files.md b/docs/runbook/files.md index 33800660..3b9bb664 100644 --- a/docs/runbook/files.md +++ b/docs/runbook/files.md @@ -42,7 +42,7 @@ | # | 接口 | 要点 | |---|---|---| | G1 | `POST /files/entity-from-candidate {provider:"tmdb", externalId, kind:"movie"\|"tv"}`;`/files/organize` 项 `external`(与 `mediaEntityId` 二选一,都给/都不给 400) | 取 TMDB 详情建 `media_entity` + `external_identity`(accepted,`evidence_level=manual`,IMDb 同时登记)+ 别名;剧集按 TMDB 季表建季/集单元(provisional,单季 ≤200 集)。幂等:同 TMDB id 已有 accepted 映射直接复用(跟随合并),并发冲突重试一次。organize 预览阶段就地建(同一请求内同一候选只建一次),组里 `entity{provider,externalId,created,title}`。TMDB 未配置 400、无此条目 404、上游失败 502(`upstream:`)。服务在 `sources-identity/service/entity-from-candidate.ts`(`createEntityFromTmdbCandidate`),TMDB 客户端经 `SourcesRuntime.tmdb()`(设置 `metadata.tmdb*`) | -| G2 | organize `minSizeBytes`;identify `minSizeBytes` | 缺省取 `settings.organize.minSizeBytes`(设置契约加字段前恒缺省 → organize 用端口 `minVideoBytes` 50 MiB,identify 不过滤)。organize:低于阈值的视频进组 `excluded[]`,`reason=below_min_size`(sample/no_episode 等照原因名);identify:该项 `skipped="below_min_size"`、不查候选,结果 `excludedBySize` 计数;每项带 `size` | +| G2 | organize `minSizeBytes`;identify `minSizeBytes` | 缺省取 `settings.organize.minSizeBytes`(缺省 50 MiB,W10-pre2 起进契约);设置读不到时 organize 退回端口 `minVideoBytes`,identify 不过滤。请求 / 向导给的值优先,给 0 表示不过滤。自动入库的 `planImport` 同样按"请求 > 设置 > 端口"取值。设置页还没有这个字段(前端后续在「媒体库与目录 → 整理方式」加),目前只能 `PATCH /settings` `{"organize":{"minSizeBytes":<字节>}}` 修改。organize:低于阈值的视频进组 `excluded[]`,`reason=below_min_size`(sample/no_episode 等照原因名);identify:该项 `skipped="below_min_size"`、不查候选,结果 `excludedBySize` 计数;每项带 `size` | | G3 | organize `onExisting: skip\|conflict\|replace_with_verify`(缺省 conflict) | 透传 `planImport.onExisting`;选 replace 即视为人工授权(`grantReplace`),执行走现有 replace 分支(新文件核验通过后旧文件进回收站)。预览逐文件 `action`(transfer/replace/skip/hold)+ `existing`(same 同一文件已在库 / skip / replace / conflict / null)+ `skipReason`;conflict 时该组 `error.code=plan_blocked`。策略进请求指纹,换策略须重新预览 | | G4 | organize `naming: template\|keep`(缺省 template) | keep = 保留源文件名(目录仍按模板/绑定/targetDir;改名后撞名 → hold + duplicate_target)。`planImport` 新参数 `namingMode` / `namingSource`,冻结计划 `organization_record.plan.naming = {mode, templateSource: entity\|root\|global\|override, templateVersion}`(模板来源按 作品 > 库根 > 全局 回落;keep = override)。组响应同名 `naming` + `targetSegments[{text, role: root\|season\|dir, bound}]`(bound = 沿用已有路径绑定) | | G5 | identify 项 `records{count,lastRecordId,lastStatus}`;`GET /files/record-sources?recordId=`;organize `refreshMetadata`;`POST /files/scrape-batch` | 整理记录按 `organization_file_result.source_path` 回查;record-sources 把记录主视频源(多个且同在源目录下时给目录)按登记目录 hostPath 换算成 FileRef,不在登记目录/已不存在的进 `unresolved`。`refreshMetadata=true`:执行成功的组按作品去重各排一次 `refreshMetadata`(组里 `refreshTaskId`)。scrape-batch:`items`(文件/目录 → 其下已登记资产的作品)∪ `mediaEntityIds` 去重,≤200 个作品;无 active 资产 `not_registered`、不存在 `not_found`、文件不属于任何作品 `no_entity`;带幂等键 | diff --git a/docs/runbook/sites.md b/docs/runbook/sites.md index 2d26d4dc..b5c4c6b1 100644 --- a/docs/runbook/sites.md +++ b/docs/runbook/sites.md @@ -71,6 +71,22 @@ 运行时每 30 s 最多重读一次设置;设置变化会让站点适配器按新签名重建。 +## Agent 能力(W10-pre2) + +处理器在 `apps/server/src/modules/sources-identity/capabilities-sites.ts`,调用上面同一组 service。能力 ID 只允许小写 + 下划线(`CapabilityId` 正则),所以规格里的 `site.healthCheck` / `site.refreshLogin` / `site.userData` 落为下划线形式。 + +| 能力 | 类别 | 风险 | 重放 | HTTP scope | 说明 | +|---|---|---|---|---|---| +| `site.list` | read | low | safe | read | 站点健康、`lastHealth`、整体账号状态与账号摘要;缺省只列启用站点(`includeDisabled`、`keys` 可选) | +| `site.health_check` | execute | low | safe | admin | 同 `POST /sites/{id}/health-check`;幂等,只读首页并写回健康列,所以例外地声明 replay safe | +| `site.refresh_login` | execute | medium | never | admin | 同步执行 `refreshSiteAccount(mode=login)`,上限 30 s(同时跟随任务取消);`accountId` 缺省取第一个启用的非 passkey 账号 | +| `site.user_data` | read | low | safe | read | 各账号最近一次用户数据 + `site_user_data` 按天历史(`days` 缺省 30,最多 400) | + +- 授权:两个 execute 能力走通用授权,Agent / API Key / 后台调用必须有 `automation_grant`(`action` 为能力 ID 或 `*`),自动化总开关关闭或只读对账时一律拒绝;控制台管理员从 Web 直接调用视为人工授权。没有计划可绑定,所以不走单次审批。 +- 脱敏:输出按白名单逐字段拼装,没有 `secretRef` / `sessionSecretRef` / `passkeySecretRef`、Cookie、密码、登录名、UA 与站点 `config`;凭证只给 `hasSession` / `hasPasskey` 两个布尔。测试 `apps/server/test/sources-identity/site-capabilities.test.ts` 逐项断言。 +- 注册数:首批能力 17 → 21,启动断言 `assertCapabilityCount` 跟随 `CAPABILITY_IDS.length`。 +- 未接:`agent/invoke-limits.ts` 的站点并发上限还没把 `site.health_check` / `site.refresh_login` 算作外部站点调用;Agent 默认只读白名单(`agent/runner.ts`)也还没加 `site.list` / `site.user_data`。两处都在 agent 模块,留给该模块的负责人。 + ## 待核验 1. takelogin 表单字段、二次验证字段名、错误文本全部只对 `@umt/testkit` mock 站点验证过,未连接真实站点。 diff --git a/packages/contracts/src/capabilities-io.ts b/packages/contracts/src/capabilities-io.ts index 93d9d5ea..ce78dbeb 100644 --- a/packages/contracts/src/capabilities-io.ts +++ b/packages/contracts/src/capabilities-io.ts @@ -8,7 +8,20 @@ import { ExternalEntityType, ExternalIdentity } from "./identity.ts"; import { ServerMapping } from "./ops.ts"; import { OrganizationAction, OrganizationRecord } from "./organization.ts"; import { AcquisitionSlot, PolicyRevision } from "./policy.ts"; -import { SourceRelease } from "./sources.ts"; +import { + SiteAccountError, + SiteAccountOverall, + SiteAccountStatus, + SiteAccountUserData, + SiteAdapter, + SiteAuthKind, + SiteHealth, + SiteHealthResult, + SiteLastHealth, + SiteLoginResult, + SiteUserDataPoint, + SourceRelease, +} from "./sources.ts"; // 首批能力的输入/输出 schema;名字与 CAPABILITY_CATALOG 中的 inputSchema/outputSchema 一致。 // 写能力的输入只接受计划/对象 ID,不接受模型自拟的幂等键或任意路径。 @@ -236,6 +249,73 @@ export const AgentAwaitInput = z.object({ }); export const AgentAwaitOutput = z.object({ results: z.array(SubagentResult), allDone: z.boolean() }); +// ── 站点能力(W10-pre2):输出只给状态与统计,不含凭证、Cookie、secret 引用、登录名与站点 config ── + +/** 账号摘要:凭证只给"有没有" */ +export const SiteAccountBrief = z.object({ + id: Id, + label: z.string(), + authKind: SiteAuthKind, + enabled: z.boolean(), + status: SiteAccountStatus, + lastLoginAt: z.string().nullable(), + lastVerifiedAt: z.string().nullable(), + lastError: SiteAccountError.nullable(), + userDataAt: z.string().nullable(), + hasSession: z.boolean(), + hasPasskey: z.boolean(), +}); +export const SiteBrief = z.object({ + id: Id, + key: z.string(), + name: z.string(), + adapter: SiteAdapter, + baseUrl: z.string(), + enabled: z.boolean(), + healthStatus: SiteHealth, + lastHealth: SiteLastHealth.nullable(), + lastHealthAt: z.string().nullable(), + accountStatus: SiteAccountOverall, + accounts: z.array(SiteAccountBrief), +}); +export const SiteListInput = z.object({ + /** 缺省只列启用的站点 */ + includeDisabled: z.boolean().default(false), + keys: z.array(z.string().min(1)).max(100).optional(), +}); +export const SiteListOutput = z.object({ items: z.array(SiteBrief) }); + +export const SiteHealthCheckInput = z.object({ siteId: Id }); +export const SiteHealthCheckOutput = SiteHealthResult; + +export const SiteLoginRefreshInput = z.object({ + siteId: Id, + /** 缺省:该站第一个启用的非 passkey 账号 */ + accountId: Id.optional(), +}); +/** 同步执行(≤ 30 s)的登录结果 */ +export const SiteLoginRefreshOutput = SiteLoginResult; + +export const SiteUserDataInput = z.object({ + siteId: Id, + accountId: Id.optional(), + days: z.number().int().min(1).max(400).default(30), +}); +export const SiteUserDataOutput = z.object({ + siteId: Id, + /** 各账号最近一次成功的用户数据 */ + accounts: z.array( + z.object({ + accountId: Id, + label: z.string(), + status: SiteAccountStatus, + userData: SiteAccountUserData.nullable(), + userDataAt: z.string().nullable(), + }), + ), + history: z.array(SiteUserDataPoint), +}); + /** 能力 ID → 输入/输出 schema(server 注册处理器、agent 生成工具定义时共用) */ export const CAPABILITY_IO = { "catalog.lookup": { input: CatalogLookupInput, output: CatalogLookupOutput }, @@ -255,6 +335,10 @@ export const CAPABILITY_IO = { "inventory.query": { input: InventoryQueryInput, output: InventoryQueryOutput }, "agent.spawn": { input: AgentSpawnInput, output: AgentSpawnOutput }, "agent.await": { input: AgentAwaitInput, output: AgentAwaitOutput }, + "site.list": { input: SiteListInput, output: SiteListOutput }, + "site.health_check": { input: SiteHealthCheckInput, output: SiteHealthCheckOutput }, + "site.refresh_login": { input: SiteLoginRefreshInput, output: SiteLoginRefreshOutput }, + "site.user_data": { input: SiteUserDataInput, output: SiteUserDataOutput }, } as const satisfies Record; type CapabilityIo = typeof CAPABILITY_IO; @@ -309,4 +393,12 @@ export type AgentSpawnOutput = z.infer; export type AgentAwaitInput = z.infer; export type AgentAwaitOutput = z.infer; export type SubagentResult = z.infer; +export type SiteListInput = z.infer; +export type SiteListOutput = z.infer; +export type SiteHealthCheckInput = z.infer; +export type SiteHealthCheckOutput = z.infer; +export type SiteLoginRefreshInput = z.infer; +export type SiteLoginRefreshOutput = z.infer; +export type SiteUserDataInput = z.infer; +export type SiteUserDataOutput = z.infer; export type SubagentBudget = z.infer; diff --git a/packages/contracts/src/capability.ts b/packages/contracts/src/capability.ts index 5afa249b..deaeb2c1 100644 --- a/packages/contracts/src/capability.ts +++ b/packages/contracts/src/capability.ts @@ -68,6 +68,10 @@ export const CAPABILITY_IDS = [ "inventory.query", "agent.spawn", "agent.await", + "site.list", + "site.health_check", + "site.refresh_login", + "site.user_data", ] as const; export const KnownCapabilityId = z.enum(CAPABILITY_IDS); export type KnownCapabilityId = z.infer; @@ -297,6 +301,53 @@ export const CAPABILITY_CATALOG: Record true, false, ), + // W10-pre2 站点能力(W9-A 站点中心的 Agent 入口);输出一律不含凭证、Cookie 与 secret 引用 + "site.list": d( + "site.list", + "列出站点", + "列出已登记站点的健康、登录状态与账号摘要(不含凭证)", + "read", + "SiteList", + "read", + "low", + true, + false, + ), + // 只读站点首页并写回健康列,重跑无害:声明 replay safe + "site.health_check": d( + "site.health_check", + "检测站点", + "检测站点首页连通与登录态,写回站点健康状态", + "execute", + "SiteHealthCheck", + "admin", + "low", + true, + false, + "safe", + ), + "site.refresh_login": d( + "site.refresh_login", + "刷新站点登录", + "对站点账号重新登录或校验 Cookie / API Key,成功后拉取用户数据;需持续授权或审批", + "execute", + "SiteLoginRefresh", + "admin", + "medium", + true, + false, + ), + "site.user_data": d( + "site.user_data", + "查站点用户数据", + "读取站点账号的最新用户数据(上传、下载、分享率、魔力、做种)与按天历史", + "read", + "SiteUserData", + "read", + "low", + true, + false, + ), }; function d( @@ -309,6 +360,8 @@ function d( riskLevel: RiskLevel, idempotent: boolean, supportsPreview: boolean, + /** 缺省按类别推导;仅对幂等且副作用可重跑的能力显式给 safe */ + replay?: CapabilityReplay, ): CapabilityDescriptor { return { id, @@ -323,6 +376,6 @@ function d( idempotent, supportsPreview, // 有副作用的类别一律不可重放 - replay: kind === "read" || kind === "preview" ? "safe" : "never", + replay: replay ?? (kind === "read" || kind === "preview" ? "safe" : "never"), }; } diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 6c78fc8e..544c0246 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -37,6 +37,8 @@ export const OrganizeSettings = z.object({ defaultMode: OrganizeMode, /** 跨卷 Move 采用 Copy→核验→删源;EXDEV 不静默降级 */ allowCrossVolumeMove: z.boolean(), + /** 整理 / 文件管理识别时视频文件的最小体积(字节),低于它的视为样片跳过;请求(向导)里给的值优先 */ + minSizeBytes: z.number().int().nonnegative(), }); export type OrganizeSettings = z.infer; @@ -315,7 +317,7 @@ export type SettingsKey = z.infer; export const DEFAULT_SETTINGS: GlobalSettings = { automation: { enabled: true, readOnlyReconcile: false }, cache: { hardThresholdPercent: 90, resumePercent: 85, checkIntervalSec: 60, cleanupTargetPercent: 80 }, - organize: { defaultMode: "copy", allowCrossVolumeMove: true }, + organize: { defaultMode: "copy", allowCrossVolumeMove: true, minSizeBytes: 50 * 1024 * 1024 }, retention: { seedRetentionDays: 7, logRetentionDays: 180 }, pi: { defaultUpstreamId: null, @@ -548,6 +550,7 @@ export const SETTINGS_SECTIONS: readonly SettingsSectionDef[] = [ fields: [ "organize.defaultMode", "organize.allowCrossVolumeMove", + "organize.minSizeBytes", "naming.movie", "naming.tv", "naming.variety", diff --git a/packages/contracts/test/schemas.test.ts b/packages/contracts/test/schemas.test.ts index 099d9892..d21596f8 100644 --- a/packages/contracts/test/schemas.test.ts +++ b/packages/contracts/test/schemas.test.ts @@ -201,7 +201,9 @@ describe("关键合同行为", () => { expect(spec).toMatchObject(meta); expect(spec.inputSchema).toBe(C.CAPABILITY_IO[id].input); expect(spec.outputSchema).toBe(C.CAPABILITY_IO[id].output); - if (spec.kind !== "read" && spec.kind !== "preview") expect(spec.replay, id).toBe("never"); + // 例外:site.health_check 只读首页并写回健康列,幂等可重跑 + if (spec.kind !== "read" && spec.kind !== "preview" && id !== "site.health_check") + expect(spec.replay, id).toBe("never"); } });