| 1 | package browser |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/base64" |
| 6 | "encoding/json" |
| 7 | "fmt" |
| 8 | "io" |
| 9 | "os" |
| 10 | "strings" |
| 11 | "time" |
| 12 | |
| 13 | "reasonix/internal/tool" |
| 14 | ) |
| 15 | |
| 16 | const screenshotMaxBytes = 8 << 20 |
| 17 | |
| 18 | func tabsTool(exec Executor) tool.Tool { |
| 19 | return readTool{base: base{exec: exec, name: "browser_tabs", |
| 20 | description: "List this task's browser tabs with ID, URL, title, and loading state. Start here to find a tabId, or call browser_open when the task has no tab yet.", |
| 21 | schema: objectSchema(nil), |
| 22 | snip: listSnip, |
| 23 | }, run: runTabs} |
| 24 | } |
| 25 | |
| 26 | func runTabs(ctx context.Context, exec Executor, args json.RawMessage) (string, error) { |
| 27 | var p struct{} |
| 28 | if err := decode(args, &p); err != nil { |
| 29 | return "", err |
| 30 | } |
| 31 | tabs, err := exec.Tabs(ctx) |
| 32 | if err != nil { |
| 33 | return "", translate(err, "browser_tabs") |
| 34 | } |
| 35 | if len(tabs) == 0 { |
| 36 | return "no tabs are open for this task; call browser_open to create one", nil |
| 37 | } |
| 38 | lines := make([]string, 0, len(tabs)+1) |
| 39 | lines = append(lines, fmt.Sprintf("%d tab(s)", len(tabs))) |
| 40 | for _, t := range tabs { |
| 41 | lines = append(lines, formatTab(t)) |
| 42 | } |
| 43 | return strings.Join(lines, "\n"), nil |
| 44 | } |
| 45 | |
| 46 | func formatTab(t Tab) string { |
| 47 | s := fmt.Sprintf("tab %s: %s", t.ID, t.URL) |
| 48 | if t.Title != "" { |
| 49 | s += fmt.Sprintf(" %q", t.Title) |
| 50 | } |
| 51 | var flags []string |
| 52 | if t.Loading { |
| 53 | flags = append(flags, "loading") |
| 54 | } |
| 55 | if t.Temporary { |
| 56 | flags = append(flags, "temporary") |
| 57 | } |
| 58 | if len(flags) > 0 { |
| 59 | s += " [" + strings.Join(flags, ", ") + "]" |
| 60 | } |
| 61 | return s |
| 62 | } |
| 63 | |
| 64 | func snapshotTool(exec Executor) tool.Tool { |
| 65 | return readTool{base: base{exec: exec, name: "browser_snapshot", |
| 66 | description: "Capture the structural snapshot of a tab: an accessibility-style tree where each interactive element carries a ref such as ref=e12, plus the documentToken that binds those refs to the current document. Take a snapshot before browser_click, browser_type, browser_press, browser_scroll, browser_select, or browser_upload, and again after anything changes the page: refs and the token expire on navigation, page replacement, or user take-over. Pass selector to scope the tree to one subtree.", |
| 67 | schema: objectSchema([]string{"tabId"}, tabIDProp(), str("selector", "Optional CSS selector; only the matching subtree is captured.")), |
| 68 | snip: treeSnip, |
| 69 | }, run: runSnapshot} |
| 70 | } |
| 71 | |
| 72 | func runSnapshot(ctx context.Context, exec Executor, args json.RawMessage) (string, error) { |
| 73 | var p struct { |
| 74 | TabID string `json:"tabId"` |
| 75 | Selector string `json:"selector"` |
| 76 | } |
| 77 | if err := decode(args, &p); err != nil { |
| 78 | return "", err |
| 79 | } |
| 80 | if err := requireTab(p.TabID); err != nil { |
| 81 | return "", err |
| 82 | } |
| 83 | snap, err := exec.Snapshot(ctx, SnapshotRequest{TabID: p.TabID, Selector: p.Selector}) |
| 84 | if err != nil { |
| 85 | return "", translate(err, "browser_snapshot") |
| 86 | } |
| 87 | return fmt.Sprintf("documentToken: %s\nurl: %s\ntitle: %s\nrefs: %d%s\n\n%s", snap.DocumentToken, snap.URL, snap.Title, snap.Refs, observationText(snap.Observation), snap.Tree), nil |
| 88 | } |
| 89 | |
| 90 | func observationText(observation *Observation) string { |
| 91 | if observation == nil { |
| 92 | return "" |
| 93 | } |
| 94 | data, _ := json.Marshal(observation) |
| 95 | return "\nobservation: " + string(data) |
| 96 | } |
| 97 | |
| 98 | func downloadTool(exec Executor) tool.Tool { |
| 99 | return readTool{base: base{exec: exec, name: "browser_download", |
| 100 | description: "List the downloads of a tab, or wait up to waitSeconds for an in-progress one to finish. Each entry reports ID, URL, saved path, state, and size; the path is a task-owned file the file tools can read.", |
| 101 | schema: objectSchema([]string{"tabId"}, tabIDProp(), bounded(integer("waitSeconds", "Seconds to wait for an in-progress download to complete; 0 lists immediately."), 0, 300)), |
| 102 | snip: listSnip, |
| 103 | }, run: runDownloads} |
| 104 | } |
| 105 | |
| 106 | func runDownloads(ctx context.Context, exec Executor, args json.RawMessage) (string, error) { |
| 107 | var p struct { |
| 108 | TabID string `json:"tabId"` |
| 109 | WaitSeconds int `json:"waitSeconds"` |
| 110 | } |
| 111 | if err := decode(args, &p); err != nil { |
| 112 | return "", err |
| 113 | } |
| 114 | if err := requireTab(p.TabID); err != nil { |
| 115 | return "", err |
| 116 | } |
| 117 | if p.WaitSeconds < 0 || p.WaitSeconds > 300 { |
| 118 | return "", fmt.Errorf("waitSeconds must be between 0 and 300") |
| 119 | } |
| 120 | downloads, err := exec.Downloads(ctx, DownloadsRequest{TabID: p.TabID, WaitFor: time.Duration(p.WaitSeconds) * time.Second}) |
| 121 | if err != nil { |
| 122 | return "", translate(err, "browser_download") |
| 123 | } |
| 124 | if len(downloads) == 0 { |
| 125 | return "no downloads for tab " + p.TabID, nil |
| 126 | } |
| 127 | lines := make([]string, 0, len(downloads)+1) |
| 128 | lines = append(lines, fmt.Sprintf("%d download(s) for tab %s", len(downloads), p.TabID)) |
| 129 | for _, d := range downloads { |
| 130 | lines = append(lines, fmt.Sprintf("download %s: %s %s -> %s (%d bytes)", d.ID, d.State, d.URL, d.Path, d.Bytes)) |
| 131 | } |
| 132 | return strings.Join(lines, "\n"), nil |
| 133 | } |
| 134 | |
| 135 | type screenshot struct{ base } |
| 136 | |
| 137 | func screenshotTool(exec Executor) tool.Tool { |
| 138 | return screenshot{base{exec: exec, name: "browser_screenshot", |
| 139 | description: "Capture a PNG of a tab's viewport, of one element by ref, or of the full page, and return it as an image. Use browser_snapshot for structure and refs; use this to see layout, images, or rendering. Images over 8 MiB are not returned; capture an element or the viewport instead.", |
| 140 | schema: objectSchema([]string{"tabId"}, tabIDProp(), refProp("Optional element ref from browser_snapshot; captures only that element."), boolean("fullPage", "Capture the whole scrollable page instead of the viewport.")), |
| 141 | snip: shortSnip, |
| 142 | }} |
| 143 | } |
| 144 | |
| 145 | func (screenshot) ReadOnly() bool { return true } |
| 146 | func (screenshot) PlanModeSafe() bool { return true } |
| 147 | |
| 148 | func (t screenshot) Execute(ctx context.Context, args json.RawMessage) (string, error) { |
| 149 | text, images, err := t.ExecuteWithImages(ctx, args) |
| 150 | if err == nil && len(images) > 0 { |
| 151 | text += "\nVisual content requires a structured image channel and an image-capable model." |
| 152 | } |
| 153 | return text, err |
| 154 | } |
| 155 | |
| 156 | func (t screenshot) ExecuteWithImages(ctx context.Context, args json.RawMessage) (string, []string, error) { |
| 157 | if err := t.ready(ctx); err != nil { |
| 158 | return "", nil, err |
| 159 | } |
| 160 | var p struct { |
| 161 | TabID string `json:"tabId"` |
| 162 | Ref string `json:"ref"` |
| 163 | FullPage bool `json:"fullPage"` |
| 164 | } |
| 165 | if err := decode(args, &p); err != nil { |
| 166 | return "", nil, err |
| 167 | } |
| 168 | if err := requireTab(p.TabID); err != nil { |
| 169 | return "", nil, err |
| 170 | } |
| 171 | shot, err := t.exec.Screenshot(ctx, ScreenshotRequest{TabID: p.TabID, Ref: p.Ref, FullPage: p.FullPage}) |
| 172 | if err != nil { |
| 173 | return "", nil, translate(err, "browser_screenshot") |
| 174 | } |
| 175 | return encodeScreenshot(p.TabID, shot) |
| 176 | } |
| 177 | |
| 178 | func encodeScreenshot(tabID string, shot Screenshot) (string, []string, error) { |
| 179 | info, err := os.Stat(shot.Path) |
| 180 | if err != nil { |
| 181 | return "", nil, fmt.Errorf("read screenshot %s: %w", shot.Path, err) |
| 182 | } |
| 183 | if !info.Mode().IsRegular() { |
| 184 | return "", nil, fmt.Errorf("screenshot %s is not a regular file", shot.Path) |
| 185 | } |
| 186 | f, err := os.Open(shot.Path) |
| 187 | if err != nil { |
| 188 | return "", nil, fmt.Errorf("read screenshot %s: %w", shot.Path, err) |
| 189 | } |
| 190 | defer f.Close() |
| 191 | if err := validateScreenshot(f, shot); err != nil { |
| 192 | return "", nil, err |
| 193 | } |
| 194 | if info.Size() > screenshotMaxBytes { |
| 195 | return oversizeText(shot.Path, info.Size()), nil, nil |
| 196 | } |
| 197 | data, err := io.ReadAll(io.LimitReader(f, screenshotMaxBytes+1)) |
| 198 | if err != nil { |
| 199 | return "", nil, fmt.Errorf("read screenshot %s: %w", shot.Path, err) |
| 200 | } |
| 201 | if len(data) > screenshotMaxBytes { |
| 202 | return oversizeText(shot.Path, int64(len(data))), nil, nil |
| 203 | } |
| 204 | mime := shot.MIME |
| 205 | if mime == "" { |
| 206 | mime = "image/png" |
| 207 | } |
| 208 | text := fmt.Sprintf("[image: %s, %dx%d] screenshot of tab %s saved at %s", mime, shot.Width, shot.Height, tabID, shot.Path) |
| 209 | text += observationText(shot.Observation) |
| 210 | if shot.ObservationToken != "" { |
| 211 | text += fmt.Sprintf("\nobservationToken: %s\nCSS viewport: %dx%d; coordinate actions use CSS pixels, not image pixels.", shot.ObservationToken, shot.CSSWidth, shot.CSSHeight) |
| 212 | } |
| 213 | return text, []string{"data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data)}, nil |
| 214 | } |
| 215 | |
| 216 | func oversizeText(path string, size int64) string { |
| 217 | return fmt.Sprintf("screenshot %s is %d bytes, over the %d MiB limit for an inline image, so it was not returned. Capture one element with ref, or the viewport without fullPage, to get a smaller image.", path, size, screenshotMaxBytes>>20) |
| 218 | } |
| 219 |