| 1 | package browser |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "maps" |
| 8 | "slices" |
| 9 | "strings" |
| 10 | |
| 11 | "reasonix/internal/tool" |
| 12 | ) |
| 13 | |
| 14 | func openTool(exec Executor) tool.Tool { |
| 15 | return writeTool{base: base{exec: exec, name: "browser_open", |
| 16 | description: "Open a new tab at a URL for this task and return its tabId. Tabs share the task's login partition unless temporary is set. Follow with browser_snapshot before acting on the page.", |
| 17 | schema: objectSchema([]string{"operationId", "url"}, operationIDProp(), str("url", "Absolute URL to open."), boolean("temporary", "Open in an in-memory partition that shares no cookies or logins and is discarded when the tab closes.")), |
| 18 | snip: shortSnip, |
| 19 | }, run: runOpen} |
| 20 | } |
| 21 | |
| 22 | func previewTool(exec Executor) tool.Tool { |
| 23 | return previewFileTool{writeTool{base: base{exec: exec, name: "browser_preview", |
| 24 | description: "Open or refresh a local file in this task's built-in browser and return its tabId. Use workspace for ordinary project files, presented only when a present toolCallId is available, and reference only for a path verified from the answer. Follow with browser_snapshot before acting on the page.", |
| 25 | schema: objectSchema([]string{"operationId", "path"}, operationIDProp(), |
| 26 | enum("source", "Authorization source for the file.", "workspace", "presented", "reference"), |
| 27 | str("path", "Workspace-relative, presented, or verified reference path."), |
| 28 | str("toolCallId", "Required only when source is presented.")), |
| 29 | snip: shortSnip, |
| 30 | }, run: runPreviewFile}} |
| 31 | } |
| 32 | |
| 33 | func runPreviewFile(ctx context.Context, exec Executor, args json.RawMessage) (string, error) { |
| 34 | var p struct { |
| 35 | OperationID string `json:"operationId"` |
| 36 | Source string `json:"source"` |
| 37 | Path string `json:"path"` |
| 38 | ToolCallID string `json:"toolCallId"` |
| 39 | } |
| 40 | if err := decode(args, &p); err != nil { |
| 41 | return "", err |
| 42 | } |
| 43 | if err := requireOperationID(p.OperationID); err != nil { |
| 44 | return "", err |
| 45 | } |
| 46 | if p.Source == "" { |
| 47 | p.Source = "workspace" |
| 48 | } |
| 49 | if p.Source != "workspace" && p.Source != "presented" && p.Source != "reference" { |
| 50 | return "", fmt.Errorf("source must be workspace, presented, or reference") |
| 51 | } |
| 52 | if strings.TrimSpace(p.Path) == "" { |
| 53 | return "", fmt.Errorf("path is required") |
| 54 | } |
| 55 | if p.Source == "presented" && strings.TrimSpace(p.ToolCallID) == "" { |
| 56 | return "", fmt.Errorf("toolCallId is required when source is presented") |
| 57 | } |
| 58 | previewer, ok := exec.(FilePreviewer) |
| 59 | if !ok { |
| 60 | return "", tool.Blocked("the current browser cannot preview local files") |
| 61 | } |
| 62 | tab, err := previewer.PreviewFile(ctx, FilePreviewRequest{ |
| 63 | OperationID: p.OperationID, Source: p.Source, Path: p.Path, ToolCallID: p.ToolCallID, |
| 64 | }) |
| 65 | if err != nil { |
| 66 | return "", translate(err, "browser_preview "+p.Path) |
| 67 | } |
| 68 | return "opened " + formatTab(tab) + "\nTake a browser_snapshot before acting on it.", nil |
| 69 | } |
| 70 | |
| 71 | func runOpen(ctx context.Context, exec Executor, args json.RawMessage) (string, error) { |
| 72 | var p struct { |
| 73 | OperationID string `json:"operationId"` |
| 74 | URL string `json:"url"` |
| 75 | Temporary bool `json:"temporary"` |
| 76 | } |
| 77 | if err := decode(args, &p); err != nil { |
| 78 | return "", err |
| 79 | } |
| 80 | if err := requireOperationID(p.OperationID); err != nil { |
| 81 | return "", err |
| 82 | } |
| 83 | if strings.TrimSpace(p.URL) == "" { |
| 84 | return "", fmt.Errorf("url is required") |
| 85 | } |
| 86 | tab, err := exec.Open(ctx, OpenRequest{OperationID: p.OperationID, URL: p.URL, Temporary: p.Temporary}) |
| 87 | if err != nil { |
| 88 | return "", translate(err, "browser_open "+p.URL) |
| 89 | } |
| 90 | return "opened " + formatTab(tab) + "\nTake a browser_snapshot before acting on it.", nil |
| 91 | } |
| 92 | |
| 93 | func navigateTool(exec Executor) tool.Tool { |
| 94 | return writeTool{base: base{exec: exec, name: "browser_navigate", |
| 95 | description: "Navigate a tab: go to a URL, or move back, forward, or reload. Navigation invalidates every earlier ref and documentToken for the tab, so take a new browser_snapshot afterwards.", |
| 96 | schema: objectSchema([]string{"action", "operationId", "tabId"}, operationIDProp(), tabIDProp(), |
| 97 | enum("action", "url goes to url; back, forward, and reload move through the tab's history.", NavigateURL, NavigateBack, NavigateForward, NavigateReload), |
| 98 | str("url", "Absolute URL; required when action is url and not accepted otherwise.")), |
| 99 | snip: shortSnip, |
| 100 | }, run: runNavigate} |
| 101 | } |
| 102 | |
| 103 | func runNavigate(ctx context.Context, exec Executor, args json.RawMessage) (string, error) { |
| 104 | var p struct { |
| 105 | OperationID string `json:"operationId"` |
| 106 | TabID string `json:"tabId"` |
| 107 | Action string `json:"action"` |
| 108 | URL string `json:"url"` |
| 109 | } |
| 110 | if err := decode(args, &p); err != nil { |
| 111 | return "", err |
| 112 | } |
| 113 | if err := requireOperationID(p.OperationID); err != nil { |
| 114 | return "", err |
| 115 | } |
| 116 | if err := requireTab(p.TabID); err != nil { |
| 117 | return "", err |
| 118 | } |
| 119 | switch p.Action { |
| 120 | case NavigateURL: |
| 121 | if strings.TrimSpace(p.URL) == "" { |
| 122 | return "", fmt.Errorf("url is required when action is url") |
| 123 | } |
| 124 | case NavigateBack, NavigateForward, NavigateReload: |
| 125 | if p.URL != "" { |
| 126 | return "", fmt.Errorf("url is only accepted when action is url") |
| 127 | } |
| 128 | default: |
| 129 | return "", fmt.Errorf("action must be one of url, back, forward, reload") |
| 130 | } |
| 131 | tab, err := exec.Navigate(ctx, NavigateRequest{OperationID: p.OperationID, TabID: p.TabID, URL: p.URL, Action: p.Action}) |
| 132 | if err != nil { |
| 133 | return "", translate(err, "browser_navigate "+p.Action+" on tab "+p.TabID) |
| 134 | } |
| 135 | return fmt.Sprintf("navigated (%s): %s\nEarlier refs and documentTokens for this tab are now invalid; take a new browser_snapshot.", p.Action, formatTab(tab)), nil |
| 136 | } |
| 137 | |
| 138 | func closeTool(exec Executor) tool.Tool { |
| 139 | return writeTool{base: base{exec: exec, name: "browser_close", |
| 140 | description: "Close a tab. A temporary tab's partition is discarded with its last tab.", |
| 141 | schema: objectSchema([]string{"operationId", "tabId"}, operationIDProp(), tabIDProp()), |
| 142 | snip: shortSnip, |
| 143 | }, run: runClose} |
| 144 | } |
| 145 | |
| 146 | func runClose(ctx context.Context, exec Executor, args json.RawMessage) (string, error) { |
| 147 | var p struct { |
| 148 | OperationID string `json:"operationId"` |
| 149 | TabID string `json:"tabId"` |
| 150 | } |
| 151 | if err := decode(args, &p); err != nil { |
| 152 | return "", err |
| 153 | } |
| 154 | if err := requireOperationID(p.OperationID); err != nil { |
| 155 | return "", err |
| 156 | } |
| 157 | if err := requireTab(p.TabID); err != nil { |
| 158 | return "", err |
| 159 | } |
| 160 | if err := exec.Close(ctx, CloseRequest{OperationID: p.OperationID, TabID: p.TabID}); err != nil { |
| 161 | return "", translate(err, "browser_close tab "+p.TabID) |
| 162 | } |
| 163 | return "closed tab " + p.TabID, nil |
| 164 | } |
| 165 | |
| 166 | // actArgs is the union of every reference-bound write's arguments; decodeAct |
| 167 | // still rejects fields the particular tool's schema does not declare. |
| 168 | type actArgs struct { |
| 169 | OperationID string `json:"operationId"` |
| 170 | TabID string `json:"tabId"` |
| 171 | DocumentToken string `json:"documentToken"` |
| 172 | Ref string `json:"ref"` |
| 173 | Text string `json:"text"` |
| 174 | Keys string `json:"keys"` |
| 175 | Options []string `json:"options"` |
| 176 | Files []string `json:"files"` |
| 177 | Submit bool `json:"submit"` |
| 178 | DeltaX int `json:"deltaX"` |
| 179 | DeltaY int `json:"deltaY"` |
| 180 | } |
| 181 | |
| 182 | // actSpec describes one reference-bound write beyond the operationId, tabId, |
| 183 | // and documentToken every such write carries. |
| 184 | type actSpec struct { |
| 185 | name string |
| 186 | action string |
| 187 | description string |
| 188 | required []string |
| 189 | props []property |
| 190 | validate func(actArgs) error |
| 191 | describe func(actArgs) string |
| 192 | } |
| 193 | |
| 194 | func actTool(exec Executor, spec actSpec) tool.Tool { |
| 195 | required := append([]string{"operationId", "tabId", "documentToken"}, spec.required...) |
| 196 | props := append([]property{operationIDProp(), tabIDProp(), documentTokenProp()}, spec.props...) |
| 197 | allowed := make(map[string]bool, len(props)) |
| 198 | for _, p := range props { |
| 199 | allowed[p.name] = true |
| 200 | } |
| 201 | run := func(ctx context.Context, exec Executor, args json.RawMessage) (string, error) { |
| 202 | a, err := decodeAct(args, allowed) |
| 203 | if err != nil { |
| 204 | return "", err |
| 205 | } |
| 206 | if err := spec.validate(a); err != nil { |
| 207 | return "", err |
| 208 | } |
| 209 | what := spec.describe(a) + " on tab " + a.TabID |
| 210 | res, err := exec.Act(ctx, ActRequest{ |
| 211 | OperationID: a.OperationID, TabID: a.TabID, DocumentToken: a.DocumentToken, Action: spec.action, |
| 212 | Ref: a.Ref, Text: a.Text, Keys: a.Keys, Options: a.Options, Files: a.Files, Submit: a.Submit, |
| 213 | DeltaX: a.DeltaX, DeltaY: a.DeltaY, |
| 214 | }) |
| 215 | if err != nil { |
| 216 | return "", translate(err, what) |
| 217 | } |
| 218 | return actOutcome(res, what) |
| 219 | } |
| 220 | return writeTool{base: base{exec: exec, name: spec.name, description: spec.description, schema: objectSchema(required, props...), snip: shortSnip}, run: run} |
| 221 | } |
| 222 | |
| 223 | func decodeAct(args json.RawMessage, allowed map[string]bool) (actArgs, error) { |
| 224 | var raw map[string]json.RawMessage |
| 225 | if err := decode(args, &raw); err != nil { |
| 226 | return actArgs{}, err |
| 227 | } |
| 228 | for _, name := range slices.Sorted(maps.Keys(raw)) { |
| 229 | if !allowed[name] { |
| 230 | return actArgs{}, fmt.Errorf("invalid args: unknown field %q", name) |
| 231 | } |
| 232 | } |
| 233 | var a actArgs |
| 234 | if err := json.Unmarshal(args, &a); err != nil { |
| 235 | return actArgs{}, fmt.Errorf("invalid args: %w", err) |
| 236 | } |
| 237 | if err := requireOperationID(a.OperationID); err != nil { |
| 238 | return actArgs{}, err |
| 239 | } |
| 240 | if err := requireTab(a.TabID); err != nil { |
| 241 | return actArgs{}, err |
| 242 | } |
| 243 | if err := requireDocumentToken(a.DocumentToken); err != nil { |
| 244 | return actArgs{}, err |
| 245 | } |
| 246 | return a, nil |
| 247 | } |
| 248 | |
| 249 | func actOutcome(res ActResult, what string) (string, error) { |
| 250 | switch { |
| 251 | case res.Outcome == OutcomeUnknown: |
| 252 | return "", unknownOutcome(what) |
| 253 | case res.Executed || res.Outcome == OutcomeExecuted: |
| 254 | s := "executed: " + what |
| 255 | if res.DocumentToken != "" { |
| 256 | s += "\ndocumentToken: " + res.DocumentToken |
| 257 | } |
| 258 | return s, nil |
| 259 | } |
| 260 | return "", notExecuted(what, res.Reason) |
| 261 | } |
| 262 | |
| 263 | func clickTool(exec Executor) tool.Tool { |
| 264 | return actTool(exec, actSpec{name: "browser_click", action: ActionClick, |
| 265 | description: "Click an element by its snapshot ref with trusted mouse events at its centre. Requires the documentToken of the snapshot the ref came from and a fresh operationId; if the page changed since, the call is blocked as stale and you must snapshot again.", |
| 266 | required: []string{"ref"}, |
| 267 | props: []property{refProp("Element ref from browser_snapshot, for example e12.")}, |
| 268 | validate: func(a actArgs) error { return requireRef(a.Ref) }, |
| 269 | describe: func(a actArgs) string { return "click " + a.Ref }, |
| 270 | }) |
| 271 | } |
| 272 | |
| 273 | func typeTool(exec Executor) tool.Tool { |
| 274 | return actTool(exec, actSpec{name: "browser_type", action: ActionType, |
| 275 | description: "Type text into an element by its snapshot ref using trusted key events, so controlled inputs and custom widgets behave as they would for a user. Set submit to press Enter afterwards. Never type credentials: login pages are handed over to the user.", |
| 276 | required: []string{"ref", "text"}, |
| 277 | props: []property{refProp("Element ref from browser_snapshot that accepts text."), str("text", "Text to type."), boolean("submit", "Press Enter after typing.")}, |
| 278 | validate: func(a actArgs) error { |
| 279 | if err := requireRef(a.Ref); err != nil { |
| 280 | return err |
| 281 | } |
| 282 | if a.Text == "" && !a.Submit { |
| 283 | return fmt.Errorf("text is required unless submit is set") |
| 284 | } |
| 285 | return nil |
| 286 | }, |
| 287 | describe: func(a actArgs) string { return fmt.Sprintf("type %d character(s) into %s", len(a.Text), a.Ref) }, |
| 288 | }) |
| 289 | } |
| 290 | |
| 291 | func pressTool(exec Executor) tool.Tool { |
| 292 | return actTool(exec, actSpec{name: "browser_press", action: ActionPress, |
| 293 | description: "Press a key or chord (for example Enter, Escape, Tab, Control+a) as trusted key events. Give ref to focus an element first; omit it to press on the active element.", |
| 294 | required: []string{"keys"}, |
| 295 | props: []property{str("keys", "Key name or '+'-joined chord, such as Enter or Control+a."), refProp("Optional element ref from browser_snapshot to focus first.")}, |
| 296 | validate: func(a actArgs) error { |
| 297 | if strings.TrimSpace(a.Keys) == "" { |
| 298 | return fmt.Errorf("keys is required") |
| 299 | } |
| 300 | return nil |
| 301 | }, |
| 302 | describe: func(a actArgs) string { return "press " + a.Keys }, |
| 303 | }) |
| 304 | } |
| 305 | |
| 306 | func scrollTool(exec Executor) tool.Tool { |
| 307 | return actTool(exec, actSpec{name: "browser_scroll", action: ActionScroll, |
| 308 | description: "Scroll the viewport, or the element named by ref, by deltaX and deltaY pixels; positive values scroll right and down. Take a new browser_snapshot afterwards to see newly revealed elements.", |
| 309 | props: []property{refProp("Optional scrollable element ref from browser_snapshot; omit to scroll the viewport."), integer("deltaX", "Horizontal pixels; positive scrolls right."), integer("deltaY", "Vertical pixels; positive scrolls down.")}, |
| 310 | validate: func(a actArgs) error { |
| 311 | if a.DeltaX == 0 && a.DeltaY == 0 { |
| 312 | return fmt.Errorf("deltaX or deltaY must be non-zero") |
| 313 | } |
| 314 | return nil |
| 315 | }, |
| 316 | describe: func(a actArgs) string { return fmt.Sprintf("scroll by (%d, %d)", a.DeltaX, a.DeltaY) }, |
| 317 | }) |
| 318 | } |
| 319 | |
| 320 | func selectTool(exec Executor) tool.Tool { |
| 321 | return actTool(exec, actSpec{name: "browser_select", action: ActionSelect, |
| 322 | description: "Choose options in a select element by its snapshot ref. Give option values or visible labels; more than one only for a multi-select.", |
| 323 | required: []string{"ref", "options"}, |
| 324 | props: []property{refProp("Select element ref from browser_snapshot."), strList("options", "Option values or labels to select.")}, |
| 325 | validate: func(a actArgs) error { |
| 326 | if err := requireRef(a.Ref); err != nil { |
| 327 | return err |
| 328 | } |
| 329 | return requireStrings("options", a.Options) |
| 330 | }, |
| 331 | describe: func(a actArgs) string { return fmt.Sprintf("select %s in %s", strings.Join(a.Options, ", "), a.Ref) }, |
| 332 | }) |
| 333 | } |
| 334 | |
| 335 | func uploadTool(exec Executor) tool.Tool { |
| 336 | return actTool(exec, actSpec{name: "browser_upload", action: ActionUpload, |
| 337 | description: "Attach files to a file input by its snapshot ref. Only files this task owns can be attached; a remote task stages them through the task's temporary directory first.", |
| 338 | required: []string{"ref", "files"}, |
| 339 | props: []property{refProp("File input ref from browser_snapshot."), strList("files", "Paths of task-owned files to attach.")}, |
| 340 | validate: func(a actArgs) error { |
| 341 | if err := requireRef(a.Ref); err != nil { |
| 342 | return err |
| 343 | } |
| 344 | return requireStrings("files", a.Files) |
| 345 | }, |
| 346 | describe: func(a actArgs) string { return fmt.Sprintf("upload %d file(s) to %s", len(a.Files), a.Ref) }, |
| 347 | }) |
| 348 | } |
| 349 |