| 1 | package builtin |
| 2 | |
| 3 | import ( |
| 4 | "bufio" |
| 5 | "context" |
| 6 | "encoding/base64" |
| 7 | "encoding/json" |
| 8 | "fmt" |
| 9 | stdhtml "html" |
| 10 | "io" |
| 11 | "net" |
| 12 | "net/http" |
| 13 | "net/url" |
| 14 | "regexp" |
| 15 | "strings" |
| 16 | "time" |
| 17 | "unicode" |
| 18 | |
| 19 | nethtml "golang.org/x/net/html" |
| 20 | "golang.org/x/net/proxy" |
| 21 | |
| 22 | "reasonix/internal/netclient" |
| 23 | "reasonix/internal/tool" |
| 24 | ) |
| 25 | |
| 26 | func init() { tool.RegisterBuiltin(webFetch{}) } |
| 27 | |
| 28 | type webFetch struct { |
| 29 | proxySpec netclient.ProxySpec |
| 30 | } |
| 31 | |
| 32 | const ( |
| 33 | webFetchTimeout = 15 * time.Second |
| 34 | webFetchMaxRead = 1 << 20 // 1 MiB cap before extraction |
| 35 | ) |
| 36 | |
| 37 | func (webFetch) Name() string { return "web_fetch" } |
| 38 | |
| 39 | func (webFetch) Description() string { |
| 40 | return "Fetch a URL over HTTPS/HTTP and return its text content. HTML pages are reduced to readable text (scripts, styles, tags stripped, whitespace collapsed); JSON / plain text / markdown bodies come back verbatim. Use to read documentation pages, API responses, or source files hosted somewhere the local filesystem can't reach." |
| 41 | } |
| 42 | |
| 43 | func (webFetch) Schema() json.RawMessage { |
| 44 | return json.RawMessage(`{ |
| 45 | "type":"object", |
| 46 | "properties":{ |
| 47 | "url":{"type":"string","description":"Absolute URL beginning with http:// or https://"} |
| 48 | }, |
| 49 | "required":["url"] |
| 50 | }`) |
| 51 | } |
| 52 | |
| 53 | func (webFetch) ReadOnly() bool { return true } |
| 54 | |
| 55 | // SnipHint front-loads fetched page content like a file read: keep a generous |
| 56 | // head and a short tail. |
| 57 | func (webFetch) SnipHint() tool.SnipHint { |
| 58 | return tool.SnipHint{Head: 120, Tail: 12, HeadChars: 12000, TailChars: 2000} |
| 59 | } |
| 60 | |
| 61 | var lookupFetchHost = net.DefaultResolver.LookupIPAddr |
| 62 | |
| 63 | // ssrfGuardedTransport refuses to connect to private, link-local, or unspecified |
| 64 | // addresses — the SSRF surface a prompt-injected fetch would aim at (cloud |
| 65 | // metadata at 169.254.169.254, RFC1918 internal services). Loopback is allowed: |
| 66 | // the agent can already reach localhost via bash, so a local dev server stays |
| 67 | // fetchable. The check runs at dial time on the resolved IP, so a public host |
| 68 | // that redirects or DNS-rebinds to an internal address is caught too. |
| 69 | func ssrfGuardedTransport(proxyURL string) *http.Transport { |
| 70 | dialer := &net.Dialer{Timeout: webFetchTimeout} |
| 71 | |
| 72 | tr := &http.Transport{ |
| 73 | DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { |
| 74 | return dialVettedAddress(ctx, dialer, network, addr) |
| 75 | }, |
| 76 | } |
| 77 | |
| 78 | if proxyURL != "" { |
| 79 | pu, err := url.Parse(proxyURL) |
| 80 | if err == nil && pu.Host != "" { |
| 81 | switch pu.Scheme { |
| 82 | case "http", "https": |
| 83 | // HTTP CONNECT: dial proxy → send CONNECT with the ORIGINAL |
| 84 | // hostname (not a locally-resolved IP) so the proxy handles DNS. |
| 85 | // This is essential for users whose local DNS is blocked (GFW). |
| 86 | // SSRF protection: IP literals are checked directly; domain names |
| 87 | // go through the trusted proxy which resolves them. |
| 88 | proxyDialer := dialer |
| 89 | tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { |
| 90 | host, port, err := net.SplitHostPort(addr) |
| 91 | if err != nil { |
| 92 | return nil, err |
| 93 | } |
| 94 | // SSRF check on IP literals only — domain names go through |
| 95 | // the trusted proxy which resolves them on the remote side. |
| 96 | if ip := net.ParseIP(host); ip != nil { |
| 97 | if blockedFetchIP(ip) { |
| 98 | return nil, fmt.Errorf("refusing to fetch internal address %s (resolves to %s)", host, ip) |
| 99 | } |
| 100 | } |
| 101 | // Dial the proxy (proxy address is never an SSRF target — the |
| 102 | // user configured it, and it's almost certainly an IP or a |
| 103 | // resolvable hostname reachable from the local network). |
| 104 | proxyConn, err := proxyDialer.DialContext(ctx, "tcp", pu.Host) |
| 105 | if err != nil { |
| 106 | return nil, fmt.Errorf("connect to proxy %s: %w", pu.Host, err) |
| 107 | } |
| 108 | // CONNECT the ORIGINAL hostname through the proxy, letting |
| 109 | // the proxy resolve DNS on the remote side. If this is an IP |
| 110 | // literal we already vetted it above. |
| 111 | targetAddr := net.JoinHostPort(host, port) |
| 112 | connectReq := &http.Request{ |
| 113 | Method: http.MethodConnect, |
| 114 | URL: &url.URL{Host: targetAddr}, |
| 115 | Host: targetAddr, |
| 116 | Header: make(http.Header), |
| 117 | } |
| 118 | if pu.User != nil { |
| 119 | user := pu.User.Username() |
| 120 | pass, _ := pu.User.Password() |
| 121 | auth := base64.StdEncoding.EncodeToString([]byte(user + ":" + pass)) |
| 122 | connectReq.Header.Set("Proxy-Authorization", "Basic "+auth) |
| 123 | } |
| 124 | if err := connectReq.Write(proxyConn); err != nil { |
| 125 | proxyConn.Close() |
| 126 | return nil, fmt.Errorf("write CONNECT to proxy: %w", err) |
| 127 | } |
| 128 | br := bufio.NewReader(proxyConn) |
| 129 | resp, err := http.ReadResponse(br, connectReq) |
| 130 | if err != nil { |
| 131 | proxyConn.Close() |
| 132 | return nil, fmt.Errorf("read CONNECT response: %w", err) |
| 133 | } |
| 134 | if resp.StatusCode != http.StatusOK { |
| 135 | proxyConn.Close() |
| 136 | return nil, fmt.Errorf("proxy CONNECT failed: %s", resp.Status) |
| 137 | } |
| 138 | return proxyConn, nil |
| 139 | } |
| 140 | tr.Proxy = nil |
| 141 | |
| 142 | case "socks5", "socks5h": |
| 143 | // Tunnel through SOCKS5. Dial the trusted proxy with a plain |
| 144 | // dialer (a proxy on a private/LAN address must not be rejected |
| 145 | // by the SSRF guard), then route the target through it. IP-literal |
| 146 | // targets are still SSRF-checked; hostnames are resolved by the |
| 147 | // proxy — the same boundary as the HTTP CONNECT path above. |
| 148 | var auth *proxy.Auth |
| 149 | if pu.User != nil { |
| 150 | pass, _ := pu.User.Password() |
| 151 | auth = &proxy.Auth{User: pu.User.Username(), Password: pass} |
| 152 | } |
| 153 | if sd, err := proxy.SOCKS5("tcp", pu.Host, auth, dialer); err == nil { |
| 154 | if cd, ok := sd.(proxy.ContextDialer); ok { |
| 155 | tr.Proxy = nil |
| 156 | tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { |
| 157 | host, _, err := net.SplitHostPort(addr) |
| 158 | if err != nil { |
| 159 | return nil, err |
| 160 | } |
| 161 | if ip := net.ParseIP(host); ip != nil && blockedFetchIP(ip) { |
| 162 | return nil, fmt.Errorf("refusing to fetch internal address %s (resolves to %s)", host, ip) |
| 163 | } |
| 164 | return cd.DialContext(ctx, network, addr) |
| 165 | } |
| 166 | } |
| 167 | } |
| 168 | } |
| 169 | } |
| 170 | } |
| 171 | |
| 172 | return tr |
| 173 | } |
| 174 | |
| 175 | // dialVettedAddress resolves the host locally and dials a resolved address that |
| 176 | // passes the SSRF blocklist, never the name, so DNS rebinding between the check |
| 177 | // and the connect cannot reach an internal address. |
| 178 | func dialVettedAddress(ctx context.Context, dialer *net.Dialer, network, addr string) (net.Conn, error) { |
| 179 | host, port, err := net.SplitHostPort(addr) |
| 180 | if err != nil { |
| 181 | return nil, err |
| 182 | } |
| 183 | ips, err := lookupFetchHost(ctx, host) |
| 184 | if err != nil { |
| 185 | return nil, err |
| 186 | } |
| 187 | // Each dial is pinned to one vetted address, so a blocked address in a |
| 188 | // mixed answer is skipped rather than fatal: a fake-ip resolver pairs a |
| 189 | // routable A with a unique-local AAAA for every public host. |
| 190 | var dialErr error |
| 191 | for _, ip := range ips { |
| 192 | if blockedFetchIP(ip.IP) { |
| 193 | continue |
| 194 | } |
| 195 | conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ip.IP.String(), port)) |
| 196 | if err == nil { |
| 197 | return conn, nil |
| 198 | } |
| 199 | dialErr = err |
| 200 | } |
| 201 | if dialErr != nil { |
| 202 | return nil, dialErr |
| 203 | } |
| 204 | if len(ips) == 0 { |
| 205 | return nil, fmt.Errorf("no addresses for %s", host) |
| 206 | } |
| 207 | return nil, fmt.Errorf("refusing to fetch internal address %s (resolves to %s)", host, ips[0].IP) |
| 208 | } |
| 209 | |
| 210 | type webFetchRoundTripper struct { |
| 211 | proxyURLFor func(*http.Request) (string, error) |
| 212 | } |
| 213 | |
| 214 | func (rt webFetchRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { |
| 215 | proxyURL, err := rt.proxyURLFor(req) |
| 216 | if err != nil { |
| 217 | return nil, fmt.Errorf("resolve proxy: %w", err) |
| 218 | } |
| 219 | return ssrfGuardedTransport(proxyURL).RoundTrip(req) |
| 220 | } |
| 221 | |
| 222 | func ssrfGuardedClient(proxyURLFor func(*http.Request) (string, error)) *http.Client { |
| 223 | return &http.Client{ |
| 224 | Timeout: webFetchTimeout, |
| 225 | Transport: webFetchRoundTripper{proxyURLFor: proxyURLFor}, |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | // cgnatRange is RFC 6598 shared address space (100.64.0.0/10). Go's IsPrivate |
| 230 | // doesn't cover it, yet some clouds host instance metadata there (Alibaba Cloud |
| 231 | // at 100.100.100.200), so it's an SSRF target web_fetch must refuse too. |
| 232 | var cgnatRange = mustCIDR("100.64.0.0/10") |
| 233 | |
| 234 | func mustCIDR(s string) *net.IPNet { |
| 235 | _, n, err := net.ParseCIDR(s) |
| 236 | if err != nil { |
| 237 | panic(err) |
| 238 | } |
| 239 | return n |
| 240 | } |
| 241 | |
| 242 | // blockedFetchIP reports whether ip is an address web_fetch must not reach. |
| 243 | func blockedFetchIP(ip net.IP) bool { |
| 244 | return ip.IsPrivate() || // RFC1918 + IPv6 unique-local (fc00::/7) |
| 245 | ip.IsLinkLocalUnicast() || // 169.254.0.0/16 (incl. cloud metadata) + fe80::/10 |
| 246 | ip.IsLinkLocalMulticast() || |
| 247 | ip.IsUnspecified() || // 0.0.0.0 / :: |
| 248 | cgnatRange.Contains(ip) // 100.64.0.0/10 (incl. Alibaba Cloud metadata) |
| 249 | } |
| 250 | |
| 251 | func (wf webFetch) proxyURLFor(req *http.Request) (string, error) { |
| 252 | pf, err := netclient.ProxyFunc(wf.proxySpec) |
| 253 | if err != nil { |
| 254 | return "", err |
| 255 | } |
| 256 | if pf == nil { |
| 257 | return "", nil |
| 258 | } |
| 259 | u, err := pf(req) |
| 260 | if err != nil || u == nil { |
| 261 | return "", err |
| 262 | } |
| 263 | return u.String(), nil |
| 264 | } |
| 265 | |
| 266 | func (wf webFetch) Execute(ctx context.Context, args json.RawMessage) (string, error) { |
| 267 | var p struct { |
| 268 | URL string `json:"url"` |
| 269 | } |
| 270 | if err := json.Unmarshal(args, &p); err != nil { |
| 271 | return "", fmt.Errorf("invalid args: %w", err) |
| 272 | } |
| 273 | if p.URL == "" { |
| 274 | return "", fmt.Errorf("url is required") |
| 275 | } |
| 276 | u, err := url.Parse(p.URL) |
| 277 | if err != nil || (u.Scheme != "http" && u.Scheme != "https") { |
| 278 | return "", fmt.Errorf("url must be an absolute http(s) address") |
| 279 | } |
| 280 | |
| 281 | reqCtx, cancel := context.WithTimeout(ctx, webFetchTimeout) |
| 282 | defer cancel() |
| 283 | req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, p.URL, nil) |
| 284 | if err != nil { |
| 285 | return "", fmt.Errorf("build request: %w", err) |
| 286 | } |
| 287 | // A plain UA + Accept tip the server toward returning text/HTML rather |
| 288 | // than minified asset bundles or binary content. |
| 289 | req.Header.Set("User-Agent", "reasonix-web-fetch/1.0") |
| 290 | req.Header.Set("Accept", "text/html,text/plain,text/markdown,application/json,*/*;q=0.5") |
| 291 | |
| 292 | resp, err := ssrfGuardedClient(wf.proxyURLFor).Do(req) |
| 293 | if err != nil { |
| 294 | return "", fmt.Errorf("fetch %s: %w", p.URL, err) |
| 295 | } |
| 296 | defer resp.Body.Close() |
| 297 | |
| 298 | body, err := io.ReadAll(io.LimitReader(resp.Body, webFetchMaxRead)) |
| 299 | if err != nil { |
| 300 | return "", fmt.Errorf("read body: %w", err) |
| 301 | } |
| 302 | |
| 303 | ct := strings.ToLower(resp.Header.Get("Content-Type")) |
| 304 | out := string(body) |
| 305 | if strings.Contains(ct, "text/html") || looksLikeHTML(out) { |
| 306 | out = htmlToText(out) |
| 307 | } |
| 308 | out = strings.TrimSpace(out) |
| 309 | if out == "" { |
| 310 | return fmt.Sprintf("(empty body — status %s)", resp.Status), nil |
| 311 | } |
| 312 | header := fmt.Sprintf("status %s · %s · %d bytes\n\n", resp.Status, contentTypeShort(ct), len(body)) |
| 313 | return header + out, nil |
| 314 | } |
| 315 | |
| 316 | // looksLikeHTML lets servers that misreport Content-Type still hit the HTML |
| 317 | // reducer — GitHub raw pages and many docs sites lie about content type. |
| 318 | func looksLikeHTML(s string) bool { |
| 319 | head := s |
| 320 | if len(head) > 512 { |
| 321 | head = head[:512] |
| 322 | } |
| 323 | low := strings.ToLower(head) |
| 324 | return strings.Contains(low, "<!doctype html") || strings.Contains(low, "<html") |
| 325 | } |
| 326 | |
| 327 | var ( |
| 328 | multiBlank = regexp.MustCompile(`\n[\t ]*\n([\t ]*\n)+`) |
| 329 | trailingWS = regexp.MustCompile(`[\t ]+\n`) |
| 330 | ) |
| 331 | |
| 332 | // htmlToText tokenizes HTML, drops script/style content, unescapes entities, and |
| 333 | // inserts lightweight block boundaries. It is intentionally lossy: we want to |
| 334 | // give the model readable text rather than preserve structure for re-rendering. |
| 335 | func htmlToText(s string) string { |
| 336 | w := &htmlTextWriter{} |
| 337 | tokenizer := nethtml.NewTokenizer(strings.NewReader(s)) |
| 338 | skipDepth := 0 |
| 339 | preDepth := 0 |
| 340 | for { |
| 341 | tt := tokenizer.Next() |
| 342 | switch tt { |
| 343 | case nethtml.ErrorToken: |
| 344 | return normalizeHTMLText(w.String()) |
| 345 | case nethtml.TextToken: |
| 346 | if skipDepth == 0 { |
| 347 | w.Text(string(tokenizer.Text()), preDepth > 0) |
| 348 | } |
| 349 | case nethtml.StartTagToken: |
| 350 | name, hasAttr := tokenizer.TagName() |
| 351 | tag := strings.ToLower(string(name)) |
| 352 | if tag == "script" || tag == "style" { |
| 353 | skipDepth++ |
| 354 | continue |
| 355 | } |
| 356 | if skipDepth > 0 { |
| 357 | continue |
| 358 | } |
| 359 | if tag == "a" { |
| 360 | w.StartLink(htmlAttr(tokenizer, hasAttr, "href")) |
| 361 | continue |
| 362 | } |
| 363 | w.StartTag(tag) |
| 364 | if tag == "pre" { |
| 365 | preDepth++ |
| 366 | } |
| 367 | case nethtml.SelfClosingTagToken: |
| 368 | name, _ := tokenizer.TagName() |
| 369 | tag := strings.ToLower(string(name)) |
| 370 | w.SelfClosingTag(tag) |
| 371 | case nethtml.EndTagToken: |
| 372 | name, _ := tokenizer.TagName() |
| 373 | tag := strings.ToLower(string(name)) |
| 374 | if skipDepth > 0 { |
| 375 | if tag == "script" || tag == "style" { |
| 376 | skipDepth-- |
| 377 | } |
| 378 | continue |
| 379 | } |
| 380 | if tag == "pre" && preDepth > 0 { |
| 381 | preDepth-- |
| 382 | } |
| 383 | w.EndTag(tag) |
| 384 | } |
| 385 | } |
| 386 | } |
| 387 | |
| 388 | type htmlTextWriter struct { |
| 389 | b strings.Builder |
| 390 | links []string |
| 391 | } |
| 392 | |
| 393 | func (w *htmlTextWriter) String() string { |
| 394 | return w.b.String() |
| 395 | } |
| 396 | |
| 397 | func (w *htmlTextWriter) StartTag(tag string) { |
| 398 | switch tag { |
| 399 | case "title": |
| 400 | w.ensureBlankLine() |
| 401 | w.b.WriteString("# ") |
| 402 | case "h1": |
| 403 | w.ensureBlankLine() |
| 404 | w.b.WriteString("# ") |
| 405 | case "h2": |
| 406 | w.ensureBlankLine() |
| 407 | w.b.WriteString("## ") |
| 408 | case "h3": |
| 409 | w.ensureBlankLine() |
| 410 | w.b.WriteString("### ") |
| 411 | case "h4", "h5", "h6": |
| 412 | w.ensureBlankLine() |
| 413 | w.b.WriteString("#### ") |
| 414 | case "li": |
| 415 | w.ensureNewline() |
| 416 | w.b.WriteString("- ") |
| 417 | case "pre": |
| 418 | w.ensureBlankLine() |
| 419 | w.b.WriteString("```\n") |
| 420 | case "blockquote": |
| 421 | w.ensureBlankLine() |
| 422 | w.b.WriteString("> ") |
| 423 | case "tr": |
| 424 | w.ensureNewline() |
| 425 | case "td", "th": |
| 426 | w.ensureCellBoundary() |
| 427 | default: |
| 428 | if htmlBreakTag(tag) || htmlBlockTag(tag) { |
| 429 | w.ensureNewline() |
| 430 | } |
| 431 | } |
| 432 | } |
| 433 | |
| 434 | func (w *htmlTextWriter) SelfClosingTag(tag string) { |
| 435 | if htmlBreakTag(tag) || htmlBlockTag(tag) { |
| 436 | w.ensureNewline() |
| 437 | } |
| 438 | } |
| 439 | |
| 440 | func (w *htmlTextWriter) EndTag(tag string) { |
| 441 | switch tag { |
| 442 | case "a": |
| 443 | w.EndLink() |
| 444 | case "title", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote": |
| 445 | w.ensureBlankLine() |
| 446 | case "pre": |
| 447 | w.ensureNewline() |
| 448 | w.b.WriteString("```\n") |
| 449 | w.ensureBlankLine() |
| 450 | case "li", "p", "tr": |
| 451 | w.ensureNewline() |
| 452 | case "td", "th": |
| 453 | return |
| 454 | default: |
| 455 | if htmlBlockTag(tag) { |
| 456 | w.ensureNewline() |
| 457 | } |
| 458 | } |
| 459 | } |
| 460 | |
| 461 | func (w *htmlTextWriter) StartLink(href string) { |
| 462 | w.links = append(w.links, strings.TrimSpace(href)) |
| 463 | } |
| 464 | |
| 465 | func (w *htmlTextWriter) EndLink() { |
| 466 | if len(w.links) == 0 { |
| 467 | return |
| 468 | } |
| 469 | href := w.links[len(w.links)-1] |
| 470 | w.links = w.links[:len(w.links)-1] |
| 471 | if href != "" { |
| 472 | w.b.WriteString(" (") |
| 473 | w.b.WriteString(href) |
| 474 | w.b.WriteByte(')') |
| 475 | } |
| 476 | } |
| 477 | |
| 478 | func (w *htmlTextWriter) Text(text string, pre bool) { |
| 479 | text = stdhtml.UnescapeString(text) |
| 480 | text = strings.ReplaceAll(text, "\u00a0", " ") |
| 481 | if !pre { |
| 482 | text = collapseHTMLInlineText(text) |
| 483 | } |
| 484 | if strings.TrimSpace(text) == "" { |
| 485 | if !w.lastIsSpace() { |
| 486 | w.b.WriteByte(' ') |
| 487 | } |
| 488 | return |
| 489 | } |
| 490 | if !pre && w.b.Len() > 0 && !w.lastIsSpace() && !startsWithSpaceOrPunct(text) { |
| 491 | w.b.WriteByte(' ') |
| 492 | } |
| 493 | w.b.WriteString(text) |
| 494 | } |
| 495 | |
| 496 | func (w *htmlTextWriter) ensureNewline() { |
| 497 | if w.b.Len() == 0 || w.lastByte() == '\n' { |
| 498 | return |
| 499 | } |
| 500 | w.b.WriteByte('\n') |
| 501 | } |
| 502 | |
| 503 | func (w *htmlTextWriter) ensureBlankLine() { |
| 504 | if w.b.Len() == 0 { |
| 505 | return |
| 506 | } |
| 507 | if strings.HasSuffix(w.b.String(), "\n\n") { |
| 508 | return |
| 509 | } |
| 510 | w.ensureNewline() |
| 511 | w.b.WriteByte('\n') |
| 512 | } |
| 513 | |
| 514 | func (w *htmlTextWriter) ensureCellBoundary() { |
| 515 | if w.b.Len() == 0 || w.lastByte() == '\n' { |
| 516 | return |
| 517 | } |
| 518 | if !strings.HasSuffix(w.b.String(), " | ") { |
| 519 | w.b.WriteString(" | ") |
| 520 | } |
| 521 | } |
| 522 | |
| 523 | func (w *htmlTextWriter) lastByte() byte { |
| 524 | if w.b.Len() == 0 { |
| 525 | return 0 |
| 526 | } |
| 527 | s := w.b.String() |
| 528 | return s[len(s)-1] |
| 529 | } |
| 530 | |
| 531 | func (w *htmlTextWriter) lastIsSpace() bool { |
| 532 | if w.b.Len() == 0 { |
| 533 | return false |
| 534 | } |
| 535 | return unicode.IsSpace(rune(w.lastByte())) |
| 536 | } |
| 537 | |
| 538 | func normalizeHTMLText(s string) string { |
| 539 | s = strings.ReplaceAll(s, "\r\n", "\n") |
| 540 | s = trailingWS.ReplaceAllString(s, "\n") |
| 541 | s = multiBlank.ReplaceAllString(s, "\n\n") |
| 542 | return strings.TrimSpace(s) |
| 543 | } |
| 544 | |
| 545 | func collapseHTMLInlineText(s string) string { |
| 546 | if s == "" { |
| 547 | return "" |
| 548 | } |
| 549 | leading := unicode.IsSpace([]rune(s)[0]) |
| 550 | trailing := unicode.IsSpace([]rune(s)[len([]rune(s))-1]) |
| 551 | fields := strings.Fields(s) |
| 552 | if len(fields) == 0 { |
| 553 | return " " |
| 554 | } |
| 555 | out := strings.Join(fields, " ") |
| 556 | if leading { |
| 557 | out = " " + out |
| 558 | } |
| 559 | if trailing { |
| 560 | out += " " |
| 561 | } |
| 562 | return out |
| 563 | } |
| 564 | |
| 565 | func startsWithSpaceOrPunct(s string) bool { |
| 566 | for _, r := range s { |
| 567 | return unicode.IsSpace(r) || strings.ContainsRune(".,;:!?)]}", r) |
| 568 | } |
| 569 | return false |
| 570 | } |
| 571 | |
| 572 | func htmlAttr(tokenizer *nethtml.Tokenizer, hasAttr bool, name string) string { |
| 573 | for hasAttr { |
| 574 | key, val, more := tokenizer.TagAttr() |
| 575 | if strings.EqualFold(string(key), name) { |
| 576 | return stdhtml.UnescapeString(string(val)) |
| 577 | } |
| 578 | hasAttr = more |
| 579 | } |
| 580 | return "" |
| 581 | } |
| 582 | |
| 583 | func htmlBreakTag(tag string) bool { |
| 584 | return tag == "br" || tag == "hr" |
| 585 | } |
| 586 | |
| 587 | func htmlBlockTag(tag string) bool { |
| 588 | switch tag { |
| 589 | case "address", "article", "aside", "blockquote", "body", "caption", "dd", "details", |
| 590 | "dialog", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", |
| 591 | "h1", "h2", "h3", "h4", "h5", "h6", "head", "header", "html", "li", "main", "nav", |
| 592 | "ol", "p", "pre", "section", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul": |
| 593 | return true |
| 594 | default: |
| 595 | return false |
| 596 | } |
| 597 | } |
| 598 | |
| 599 | func contentTypeShort(ct string) string { |
| 600 | if i := strings.IndexByte(ct, ';'); i >= 0 { |
| 601 | ct = ct[:i] |
| 602 | } |
| 603 | return strings.TrimSpace(ct) |
| 604 | } |
| 605 |