- browser.screenshot returns the page as an image content block (≤1280px wide, JPEG quality configurable) for vision-capable upstreams; non-vision upstreams get a redacted text snapshot. Login pages / password fields are marked sensitive and withheld unless a browser.sensitive_screenshot grant exists and allowSensitive is passed. - browser.click_at / type_at / scroll / wait act in screenshot pixels; the server converts via viewport/DPR, requires automation control and an allowlisted host, and writes a thumbnail evidence file (DATA_DIR/tmp/evidence) referenced from agent_run_event. - browser.solve_challenge runs a bounded child agent_run (vision caps only, maxSteps ≤ 12, budget), judges success from page facts, stops immediately on 2FA / SMS / device confirmation, and otherwise moves the session to waiting with evidence and a takeover link. - PiAgentRuntime passes modelImage outputs as pi-ai image content when model.input has image; the ledger never stores image data; coordinate actions use per-invocation idempotency keys. - Tests: image codec, runtime image content, E14 e2e (login solved, 2FA stop, step limit, sensitive withheld). Real camofox 1.17.0 check on example.com recorded in the runbook. Co-Authored-By: Claude Opus 5.5 <[email protected]>
128 lines
4.8 KiB
TypeScript
128 lines
4.8 KiB
TypeScript
import type {
|
|
AgentBudgetKind,
|
|
AgentFailureReason,
|
|
AgentRunEvent,
|
|
AgentRunEventBody,
|
|
AgentUsageTotals,
|
|
CapabilityKind,
|
|
} from "@umt/contracts";
|
|
import type { AgentSettings } from "../settings.ts";
|
|
|
|
// 事件、用量、失败原因的类型统一取自 @umt/contracts(AgentRunEvent / AgentUsageTotals / AgentFailureReason /
|
|
// AgentBudgetKind);本文件只定义运行时端口。
|
|
|
|
export type SubagentEventBody = Extract<
|
|
AgentRunEventBody,
|
|
{ type: "subagent_spawned" | "subagent_event" | "subagent_completed" }
|
|
>;
|
|
|
|
/**
|
|
* 由服务端(而不是模型 / harness)产生、追加进会话事件流的事件:子会话镜像与会话交互
|
|
* (步骤清单、方案选择、审批卡)。与运行时自身事件共用 seq(单写者)。
|
|
*/
|
|
export type InjectedEventBody = Extract<
|
|
AgentRunEventBody,
|
|
{
|
|
type:
|
|
| "subagent_spawned"
|
|
| "subagent_event"
|
|
| "subagent_completed"
|
|
| "plan_updated"
|
|
| "options_proposed"
|
|
| "options_resolved"
|
|
| "approval_requested"
|
|
| "approval_resolved";
|
|
}
|
|
>;
|
|
|
|
/** 调用能力时使用的业务身份;授权回调据此到 automation_grant / approval 查询 */
|
|
export interface AgentActor {
|
|
actorId: string;
|
|
/** 持久任务 ID(agent_run 所属任务) */
|
|
taskId: string;
|
|
scopeIds: string[];
|
|
/** 本次运行允许触达的能力类别上限;"先给方案"模式传 ['read','preview','propose'] */
|
|
allowedKinds?: CapabilityKind[];
|
|
}
|
|
|
|
export interface CreateRunSpec {
|
|
/** = agent_run.id,同时作为 Pi 会话 ID 与 JSONL 文件名 */
|
|
runId: string;
|
|
goal: string;
|
|
systemPrompt: string;
|
|
/** 本次允许的能力 ID 白名单 */
|
|
capabilities: string[];
|
|
settings: AgentSettings;
|
|
/** 应用数据目录下的会话目录;会话文件为 sessionDir/<runId>.jsonl */
|
|
sessionDir: string;
|
|
actor: AgentActor;
|
|
/**
|
|
* 本进程第一个事件的 seq。续跑(崩溃恢复/多轮对话)时传该 run 已落库事件的 max(seq)+1
|
|
* (agent_run_event 中无记录时为 0),保证 (run_id, seq) 不冲突;默认 0。
|
|
*/
|
|
firstSeq?: number;
|
|
/**
|
|
* 子代理:本会话派生的子会话已用掉的量(工具调用 / token / 费用)。父会话总预算含子会话用量——
|
|
* 运行时在每次工具调用与每轮请求前把它加到自己的用量上再比预算。由调用方缓存,须同步返回。
|
|
*/
|
|
externalUsage?: () => ExternalUsage;
|
|
/**
|
|
* 挂起后可重放的"不可重放"能力(replay=never,如 download.submit 等审批)。给了它时,这类能力对 harness 声明为
|
|
* replay=safe,真正执行前用调用级 memo 区分首次与重放:重放时只有 replayGate 返回 true(本次恢复正是为了它,
|
|
* 如审批已决定)才重新调用注册中心(授权 / 账本照常),否则按"执行被中断、外部结果未知"返回错误(与 never 语义一致)。
|
|
*/
|
|
replayGate?: (capabilityId: string, invocationId: string) => boolean;
|
|
}
|
|
|
|
export interface ExternalUsage {
|
|
toolCalls: number;
|
|
totalTokens: number;
|
|
costUsd: number;
|
|
}
|
|
|
|
export interface RunOutcome {
|
|
status: "completed" | "failed";
|
|
reason?: AgentFailureReason;
|
|
budget?: AgentBudgetKind;
|
|
message?: string;
|
|
finalText: string;
|
|
usage: AgentUsageTotals;
|
|
}
|
|
|
|
/** 一次运行的句柄(区别于 contracts 的 AgentRun 行) */
|
|
export interface AgentRunHandle {
|
|
readonly runId: string;
|
|
readonly sessionPath: string;
|
|
/** 打开已有会话时发现的未完成操作数;>0 时应调用 resume() */
|
|
readonly pendingOperations: number;
|
|
/** 上游模型支持图片输入(视觉自动化需要);fake 模型按创建参数 */
|
|
readonly supportsImages?: boolean;
|
|
/** 从 fromSeq(含)开始回放并持续推送事件;close() 后结束 */
|
|
events(fromSeq?: number): AsyncIterable<AgentRunEvent>;
|
|
/** 本进程已产生的全部事件快照 */
|
|
history(): AgentRunEvent[];
|
|
/** 发送一条用户消息并驱动到本轮结束;text 省略时使用 goal */
|
|
prompt(text?: string): Promise<RunOutcome>;
|
|
/** 崩溃/重启后继续未完成的操作 */
|
|
resume(): Promise<RunOutcome>;
|
|
abort(reason?: string): Promise<void>;
|
|
/**
|
|
* 往本会话事件流追加一条服务端事件(子会话镜像 / 步骤清单 / 方案 / 审批),与运行时自身事件共用 seq 计数(单写者)。
|
|
* 会话已关闭时返回 false,调用方自行落库。
|
|
*/
|
|
appendEvent(body: InjectedEventBody): boolean;
|
|
/** 本会话自身已用量(不含子会话) */
|
|
stats(): { toolCalls: number; usage: AgentUsageTotals };
|
|
compact(customInstructions?: string): Promise<void>;
|
|
close(): Promise<void>;
|
|
}
|
|
|
|
/** 领域/服务层只依赖这个端口,不直接接触 Pi SDK 类型 */
|
|
export interface AgentRuntimePort {
|
|
createRun(spec: CreateRunSpec): Promise<AgentRunHandle>;
|
|
}
|
|
|
|
export function emptyUsage(): AgentUsageTotals {
|
|
return { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, totalTokens: 0, costUsd: 0 };
|
|
}
|