| 1 | package builtin |
| 2 | |
| 3 | import ( |
| 4 | "bufio" |
| 5 | "context" |
| 6 | "encoding/base64" |
| 7 | "encoding/json" |
| 8 | "fmt" |
| 9 | stdhtml "html" |
| 10 | "io" |
| 11 | "net" |
| 12 | "net/http" |
| 13 | "net/url" |
| 14 | "regexp" |
| 15 | "strings" |
| 16 | "time" |
| 17 | "unicode" |
| 18 | |
| 19 | nethtml "golang.org/x/net/html" |
| 20 | "golang.org/x/net/proxy" |
| 21 | |
| 22 | "reasonix/internal/netclient" |
| 23 | "reasonix/internal/tool" |
| 24 | ) |
| 25 | |
| 26 | func init() { tool.RegisterBuiltin(webFetch{}) } |
| 27 | |
| 28 | type webFetch struct { |
| 29 | proxySpec netclient.ProxySpec |
| 30 | } |
| 31 | |
| 32 | const ( |
| 33 | webFetchTimeout = 15 * time.Second |
| 34 | webFetchMaxRead = 1 << 20 // 1 MiB cap before extraction |
| 35 | ) |
| 36 | |
| 37 | func (webFetch) Name() string { return "web_fetch" } |
| 38 | |
| 39 | func (webFetch) Description() string { |
| 40 | return "Fetch a URL over HTTPS/HTTP and return its text content. HTML pages are reduced to readable text (scripts, styles, tags stripped, whitespace collapsed); JSON / plain text / markdown bodies come back verbatim. Use to read documentation pages, API responses, or source files hosted somewhere the local filesystem can't reach." |
| 41 | } |
| 42 | |
| 43 | func (webFetch) Schema() json.RawMessage { |
| 44 | return json.RawMessage(`{ |
| 45 | "type":"object", |
| 46 | "properties":{ |
| 47 | "url":{"type":"string","description":"Absolute URL beginning with http:// or https://"} |
| 48 | }, |
| 49 | "required":["url"] |
| 50 | }`) |
| 51 | } |
| 52 | |
| 53 | func (webFetch) ReadOnly() bool { return true } |
| 54 | |
| 55 | // SnipHint front-loads fetched page content like a file read: keep a generous |
| 56 | // head and a short tail. |
| 57 | func (webFetch) SnipHint() tool.SnipHint { |
| 58 | return tool.SnipHint{Head: 120, Tail: 12, HeadChars: 12000, TailChars: 2000} |
| 59 | } |
| 60 | |
| 61 | // ssrfGuardedTransport refuses to connect to private, link-local, or unspecified |
| 62 | // addresses — the SSRF surface a prompt-injected fetch would aim at (cloud |
| 63 | // metadata at 169.254.169.254, RFC1918 internal services). Loopback is allowed: |
| 64 | // the agent can already reach localhost via bash, so a local dev server stays |
| 65 | // fetchable. The check runs at dial time on the resolved IP, so a public host |
| 66 | // that redirects or DNS-rebinds to an internal address is caught too. |
| 67 | func ssrfGuardedTransport(proxyURL string) *http.Transport { |
| 68 | dialer := &net.Dialer{Timeout: webFetchTimeout} |
| 69 | |
| 70 | // directDialContext handles SSRF-protected direct connection (no proxy). |
| 71 | // It resolves DNS locally, checks resolved IPs against the SSRF blocklist, |
| 72 | // then dials the vetted IP directly to prevent DNS rebinding. |
| 73 | directDialContext := func(ctx context.Context, network, addr string) (net.Conn, error) { |
| 74 | host, port, err := net.SplitHostPort(addr) |
| 75 | if err != nil { |
| 76 | return nil, err |
| 77 | } |
| 78 | ips, err := net.DefaultResolver.LookupIPAddr(ctx, host) |
| 79 | if err != nil { |
| 80 | return nil, err |
| 81 | } |
| 82 | for _, ip := range ips { |
| 83 | if blockedFetchIP(ip.IP) { |
| 84 | return nil, fmt.Errorf("refusing to fetch internal address %s (resolves to %s)", host, ip.IP) |
| 85 | } |
| 86 | } |
| 87 | return dialer.DialContext(ctx, network, net.JoinHostPort(ips[0].IP.String(), port)) |
| 88 | } |
| 89 | |
| 90 | tr := &http.Transport{ |
| 91 | DialContext: directDialContext, |
| 92 | } |
| 93 | |
| 94 | if proxyURL != "" { |
| 95 | pu, err := url.Parse(proxyURL) |
| 96 | if err == nil && pu.Host != "" { |
| 97 | switch pu.Scheme { |
| 98 | case "http", "https": |
| 99 | // HTTP CONNECT: dial proxy → send CONNECT with the ORIGINAL |
| 100 | // hostname (not a locally-resolved IP) so the proxy handles DNS. |
| 101 | // This is essential for users whose local DNS is blocked (GFW). |
| 102 | // SSRF protection: IP literals are checked directly; domain names |
| 103 | // go through the trusted proxy which resolves them. |
| 104 | proxyDialer := dialer |
| 105 | tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { |
| 106 | host, port, err := net.SplitHostPort(addr) |
| 107 | if err != nil { |
| 108 | return nil, err |
| 109 | } |
| 110 | // SSRF check on IP literals only — domain names go through |
| 111 | // the trusted proxy which resolves them on the remote side. |
| 112 | if ip := net.ParseIP(host); ip != nil { |
| 113 | if blockedFetchIP(ip) { |
| 114 | return nil, fmt.Errorf("refusing to fetch internal address %s (resolves to %s)", host, ip) |
| 115 | } |
| 116 | } |
| 117 | // Dial the proxy (proxy address is never an SSRF target — the |
| 118 | // user configured it, and it's almost certainly an IP or a |
| 119 | // resolvable hostname reachable from the local network). |
| 120 | proxyConn, err := proxyDialer.DialContext(ctx, "tcp", pu.Host) |
| 121 | if err != nil { |
| 122 | return nil, fmt.Errorf("connect to proxy %s: %w", pu.Host, err) |
| 123 | } |
| 124 | // CONNECT the ORIGINAL hostname through the proxy, letting |
| 125 | // the proxy resolve DNS on the remote side. If this is an IP |
| 126 | // literal we already vetted it above. |
| 127 | targetAddr := net.JoinHostPort(host, port) |
| 128 | connectReq := &http.Request{ |
| 129 | Method: http.MethodConnect, |
| 130 | URL: &url.URL{Host: targetAddr}, |
| 131 | Host: targetAddr, |
| 132 | Header: make(http.Header), |
| 133 | } |
| 134 | if pu.User != nil { |
| 135 | user := pu.User.Username() |
| 136 | pass, _ := pu.User.Password() |
| 137 | auth := base64.StdEncoding.EncodeToString([]byte(user + ":" + pass)) |
| 138 | connectReq.Header.Set("Proxy-Authorization", "Basic "+auth) |
| 139 | } |
| 140 | if err := connectReq.Write(proxyConn); err != nil { |
| 141 | proxyConn.Close() |
| 142 | return nil, fmt.Errorf("write CONNECT to proxy: %w", err) |
| 143 | } |
| 144 | br := bufio.NewReader(proxyConn) |
| 145 | resp, err := http.ReadResponse(br, connectReq) |
| 146 | if err != nil { |
| 147 | proxyConn.Close() |
| 148 | return nil, fmt.Errorf("read CONNECT response: %w", err) |
| 149 | } |
| 150 | if resp.StatusCode != http.StatusOK { |
| 151 | proxyConn.Close() |
| 152 | return nil, fmt.Errorf("proxy CONNECT failed: %s", resp.Status) |
| 153 | } |
| 154 | return proxyConn, nil |
| 155 | } |
| 156 | tr.Proxy = nil |
| 157 | |
| 158 | case "socks5", "socks5h": |
| 159 | // Tunnel through SOCKS5. Dial the trusted proxy with a plain |
| 160 | // dialer (a proxy on a private/LAN address must not be rejected |
| 161 | // by the SSRF guard), then route the target through it. IP-literal |
| 162 | // targets are still SSRF-checked; hostnames are resolved by the |
| 163 | // proxy — the same boundary as the HTTP CONNECT path above. |
| 164 | var auth *proxy.Auth |
| 165 | if pu.User != nil { |
| 166 | pass, _ := pu.User.Password() |
| 167 | auth = &proxy.Auth{User: pu.User.Username(), Password: pass} |
| 168 | } |
| 169 | if sd, err := proxy.SOCKS5("tcp", pu.Host, auth, dialer); err == nil { |
| 170 | if cd, ok := sd.(proxy.ContextDialer); ok { |
| 171 | tr.Proxy = nil |
| 172 | tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { |
| 173 | host, _, err := net.SplitHostPort(addr) |
| 174 | if err != nil { |
| 175 | return nil, err |
| 176 | } |
| 177 | if ip := net.ParseIP(host); ip != nil && blockedFetchIP(ip) { |
| 178 | return nil, fmt.Errorf("refusing to fetch internal address %s (resolves to %s)", host, ip) |
| 179 | } |
| 180 | return cd.DialContext(ctx, network, addr) |
| 181 | } |
| 182 | } |
| 183 | } |
| 184 | } |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | return tr |
| 189 | } |
| 190 | |
| 191 | type webFetchRoundTripper struct { |
| 192 | proxyURLFor func(*http.Request) (string, error) |
| 193 | } |
| 194 | |
| 195 | func (rt webFetchRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { |
| 196 | proxyURL, err := rt.proxyURLFor(req) |
| 197 | if err != nil { |
| 198 | return nil, fmt.Errorf("resolve proxy: %w", err) |
| 199 | } |
| 200 | return ssrfGuardedTransport(proxyURL).RoundTrip(req) |
| 201 | } |
| 202 | |
| 203 | func ssrfGuardedClient(proxyURLFor func(*http.Request) (string, error)) *http.Client { |
| 204 | return &http.Client{ |
| 205 | Timeout: webFetchTimeout, |
| 206 | Transport: webFetchRoundTripper{proxyURLFor: proxyURLFor}, |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | // cgnatRange is RFC 6598 shared address space (100.64.0.0/10). Go's IsPrivate |
| 211 | // doesn't cover it, yet some clouds host instance metadata there (Alibaba Cloud |
| 212 | // at 100.100.100.200), so it's an SSRF target web_fetch must refuse too. |
| 213 | var cgnatRange = mustCIDR("100.64.0.0/10") |
| 214 | |
| 215 | func mustCIDR(s string) *net.IPNet { |
| 216 | _, n, err := net.ParseCIDR(s) |
| 217 | if err != nil { |
| 218 | panic(err) |
| 219 | } |
| 220 | return n |
| 221 | } |
| 222 | |
| 223 | // blockedFetchIP reports whether ip is an address web_fetch must not reach. |
| 224 | func blockedFetchIP(ip net.IP) bool { |
| 225 | return ip.IsPrivate() || // RFC1918 + IPv6 unique-local (fc00::/7) |
| 226 | ip.IsLinkLocalUnicast() || // 169.254.0.0/16 (incl. cloud metadata) + fe80::/10 |
| 227 | ip.IsLinkLocalMulticast() || |
| 228 | ip.IsUnspecified() || // 0.0.0.0 / :: |
| 229 | cgnatRange.Contains(ip) // 100.64.0.0/10 (incl. Alibaba Cloud metadata) |
| 230 | } |
| 231 | |
| 232 | func (wf webFetch) proxyURLFor(req *http.Request) (string, error) { |
| 233 | pf, err := netclient.ProxyFunc(wf.proxySpec) |
| 234 | if err != nil { |
| 235 | return "", err |
| 236 | } |
| 237 | if pf == nil { |
| 238 | return "", nil |
| 239 | } |
| 240 | u, err := pf(req) |
| 241 | if err != nil || u == nil { |
| 242 | return "", err |
| 243 | } |
| 244 | return u.String(), nil |
| 245 | } |
| 246 | |
| 247 | func (wf webFetch) Execute(ctx context.Context, args json.RawMessage) (string, error) { |
| 248 | var p struct { |
| 249 | URL string `json:"url"` |
| 250 | } |
| 251 | if err := json.Unmarshal(args, &p); err != nil { |
| 252 | return "", fmt.Errorf("invalid args: %w", err) |
| 253 | } |
| 254 | if p.URL == "" { |
| 255 | return "", fmt.Errorf("url is required") |
| 256 | } |
| 257 | u, err := url.Parse(p.URL) |
| 258 | if err != nil || (u.Scheme != "http" && u.Scheme != "https") { |
| 259 | return "", fmt.Errorf("url must be an absolute http(s) address") |
| 260 | } |
| 261 | |
| 262 | reqCtx, cancel := context.WithTimeout(ctx, webFetchTimeout) |
| 263 | defer cancel() |
| 264 | req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, p.URL, nil) |
| 265 | if err != nil { |
| 266 | return "", fmt.Errorf("build request: %w", err) |
| 267 | } |
| 268 | // A plain UA + Accept tip the server toward returning text/HTML rather |
| 269 | // than minified asset bundles or binary content. |
| 270 | req.Header.Set("User-Agent", "reasonix-web-fetch/1.0") |
| 271 | req.Header.Set("Accept", "text/html,text/plain,text/markdown,application/json,*/*;q=0.5") |
| 272 | |
| 273 | resp, err := ssrfGuardedClient(wf.proxyURLFor).Do(req) |
| 274 | if err != nil { |
| 275 | return "", fmt.Errorf("fetch %s: %w", p.URL, err) |
| 276 | } |
| 277 | defer resp.Body.Close() |
| 278 | |
| 279 | body, err := io.ReadAll(io.LimitReader(resp.Body, webFetchMaxRead)) |
| 280 | if err != nil { |
| 281 | return "", fmt.Errorf("read body: %w", err) |
| 282 | } |
| 283 | |
| 284 | ct := strings.ToLower(resp.Header.Get("Content-Type")) |
| 285 | out := string(body) |
| 286 | if strings.Contains(ct, "text/html") || looksLikeHTML(out) { |
| 287 | out = htmlToText(out) |
| 288 | } |
| 289 | out = strings.TrimSpace(out) |
| 290 | if out == "" { |
| 291 | return fmt.Sprintf("(empty body — status %s)", resp.Status), nil |
| 292 | } |
| 293 | header := fmt.Sprintf("status %s · %s · %d bytes\n\n", resp.Status, contentTypeShort(ct), len(body)) |
| 294 | return header + out, nil |
| 295 | } |
| 296 | |
| 297 | // looksLikeHTML lets servers that misreport Content-Type still hit the HTML |
| 298 | // reducer — GitHub raw pages and many docs sites lie about content type. |
| 299 | func looksLikeHTML(s string) bool { |
| 300 | head := s |
| 301 | if len(head) > 512 { |
| 302 | head = head[:512] |
| 303 | } |
| 304 | low := strings.ToLower(head) |
| 305 | return strings.Contains(low, "<!doctype html") || strings.Contains(low, "<html") |
| 306 | } |
| 307 | |
| 308 | var ( |
| 309 | multiBlank = regexp.MustCompile(`\n[\t ]*\n([\t ]*\n)+`) |
| 310 | trailingWS = regexp.MustCompile(`[\t ]+\n`) |
| 311 | ) |
| 312 | |
| 313 | // htmlToText tokenizes HTML, drops script/style content, unescapes entities, and |
| 314 | // inserts lightweight block boundaries. It is intentionally lossy: we want to |
| 315 | // give the model readable text rather than preserve structure for re-rendering. |
| 316 | func htmlToText(s string) string { |
| 317 | w := &htmlTextWriter{} |
| 318 | tokenizer := nethtml.NewTokenizer(strings.NewReader(s)) |
| 319 | skipDepth := 0 |
| 320 | preDepth := 0 |
| 321 | for { |
| 322 | tt := tokenizer.Next() |
| 323 | switch tt { |
| 324 | case nethtml.ErrorToken: |
| 325 | return normalizeHTMLText(w.String()) |
| 326 | case nethtml.TextToken: |
| 327 | if skipDepth == 0 { |
| 328 | w.Text(string(tokenizer.Text()), preDepth > 0) |
| 329 | } |
| 330 | case nethtml.StartTagToken: |
| 331 | name, hasAttr := tokenizer.TagName() |
| 332 | tag := strings.ToLower(string(name)) |
| 333 | if tag == "script" || tag == "style" { |
| 334 | skipDepth++ |
| 335 | continue |
| 336 | } |
| 337 | if skipDepth > 0 { |
| 338 | continue |
| 339 | } |
| 340 | if tag == "a" { |
| 341 | w.StartLink(htmlAttr(tokenizer, hasAttr, "href")) |
| 342 | continue |
| 343 | } |
| 344 | w.StartTag(tag) |
| 345 | if tag == "pre" { |
| 346 | preDepth++ |
| 347 | } |
| 348 | case nethtml.SelfClosingTagToken: |
| 349 | name, _ := tokenizer.TagName() |
| 350 | tag := strings.ToLower(string(name)) |
| 351 | w.SelfClosingTag(tag) |
| 352 | case nethtml.EndTagToken: |
| 353 | name, _ := tokenizer.TagName() |
| 354 | tag := strings.ToLower(string(name)) |
| 355 | if skipDepth > 0 { |
| 356 | if tag == "script" || tag == "style" { |
| 357 | skipDepth-- |
| 358 | } |
| 359 | continue |
| 360 | } |
| 361 | if tag == "pre" && preDepth > 0 { |
| 362 | preDepth-- |
| 363 | } |
| 364 | w.EndTag(tag) |
| 365 | } |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | type htmlTextWriter struct { |
| 370 | b strings.Builder |
| 371 | links []string |
| 372 | } |
| 373 | |
| 374 | func (w *htmlTextWriter) String() string { |
| 375 | return w.b.String() |
| 376 | } |
| 377 | |
| 378 | func (w *htmlTextWriter) StartTag(tag string) { |
| 379 | switch tag { |
| 380 | case "title": |
| 381 | w.ensureBlankLine() |
| 382 | w.b.WriteString("# ") |
| 383 | case "h1": |
| 384 | w.ensureBlankLine() |
| 385 | w.b.WriteString("# ") |
| 386 | case "h2": |
| 387 | w.ensureBlankLine() |
| 388 | w.b.WriteString("## ") |
| 389 | case "h3": |
| 390 | w.ensureBlankLine() |
| 391 | w.b.WriteString("### ") |
| 392 | case "h4", "h5", "h6": |
| 393 | w.ensureBlankLine() |
| 394 | w.b.WriteString("#### ") |
| 395 | case "li": |
| 396 | w.ensureNewline() |
| 397 | w.b.WriteString("- ") |
| 398 | case "pre": |
| 399 | w.ensureBlankLine() |
| 400 | w.b.WriteString("```\n") |
| 401 | case "blockquote": |
| 402 | w.ensureBlankLine() |
| 403 | w.b.WriteString("> ") |
| 404 | case "tr": |
| 405 | w.ensureNewline() |
| 406 | case "td", "th": |
| 407 | w.ensureCellBoundary() |
| 408 | default: |
| 409 | if htmlBreakTag(tag) || htmlBlockTag(tag) { |
| 410 | w.ensureNewline() |
| 411 | } |
| 412 | } |
| 413 | } |
| 414 | |
| 415 | func (w *htmlTextWriter) SelfClosingTag(tag string) { |
| 416 | if htmlBreakTag(tag) || htmlBlockTag(tag) { |
| 417 | w.ensureNewline() |
| 418 | } |
| 419 | } |
| 420 | |
| 421 | func (w *htmlTextWriter) EndTag(tag string) { |
| 422 | switch tag { |
| 423 | case "a": |
| 424 | w.EndLink() |
| 425 | case "title", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote": |
| 426 | w.ensureBlankLine() |
| 427 | case "pre": |
| 428 | w.ensureNewline() |
| 429 | w.b.WriteString("```\n") |
| 430 | w.ensureBlankLine() |
| 431 | case "li", "p", "tr": |
| 432 | w.ensureNewline() |
| 433 | case "td", "th": |
| 434 | return |
| 435 | default: |
| 436 | if htmlBlockTag(tag) { |
| 437 | w.ensureNewline() |
| 438 | } |
| 439 | } |
| 440 | } |
| 441 | |
| 442 | func (w *htmlTextWriter) StartLink(href string) { |
| 443 | w.links = append(w.links, strings.TrimSpace(href)) |
| 444 | } |
| 445 | |
| 446 | func (w *htmlTextWriter) EndLink() { |
| 447 | if len(w.links) == 0 { |
| 448 | return |
| 449 | } |
| 450 | href := w.links[len(w.links)-1] |
| 451 | w.links = w.links[:len(w.links)-1] |
| 452 | if href != "" { |
| 453 | w.b.WriteString(" (") |
| 454 | w.b.WriteString(href) |
| 455 | w.b.WriteByte(')') |
| 456 | } |
| 457 | } |
| 458 | |
| 459 | func (w *htmlTextWriter) Text(text string, pre bool) { |
| 460 | text = stdhtml.UnescapeString(text) |
| 461 | text = strings.ReplaceAll(text, "\u00a0", " ") |
| 462 | if !pre { |
| 463 | text = collapseHTMLInlineText(text) |
| 464 | } |
| 465 | if strings.TrimSpace(text) == "" { |
| 466 | if !w.lastIsSpace() { |
| 467 | w.b.WriteByte(' ') |
| 468 | } |
| 469 | return |
| 470 | } |
| 471 | if !pre && w.b.Len() > 0 && !w.lastIsSpace() && !startsWithSpaceOrPunct(text) { |
| 472 | w.b.WriteByte(' ') |
| 473 | } |
| 474 | w.b.WriteString(text) |
| 475 | } |
| 476 | |
| 477 | func (w *htmlTextWriter) ensureNewline() { |
| 478 | if w.b.Len() == 0 || w.lastByte() == '\n' { |
| 479 | return |
| 480 | } |
| 481 | w.b.WriteByte('\n') |
| 482 | } |
| 483 | |
| 484 | func (w *htmlTextWriter) ensureBlankLine() { |
| 485 | if w.b.Len() == 0 { |
| 486 | return |
| 487 | } |
| 488 | if strings.HasSuffix(w.b.String(), "\n\n") { |
| 489 | return |
| 490 | } |
| 491 | w.ensureNewline() |
| 492 | w.b.WriteByte('\n') |
| 493 | } |
| 494 | |
| 495 | func (w *htmlTextWriter) ensureCellBoundary() { |
| 496 | if w.b.Len() == 0 || w.lastByte() == '\n' { |
| 497 | return |
| 498 | } |
| 499 | if !strings.HasSuffix(w.b.String(), " | ") { |
| 500 | w.b.WriteString(" | ") |
| 501 | } |
| 502 | } |
| 503 | |
| 504 | func (w *htmlTextWriter) lastByte() byte { |
| 505 | if w.b.Len() == 0 { |
| 506 | return 0 |
| 507 | } |
| 508 | s := w.b.String() |
| 509 | return s[len(s)-1] |
| 510 | } |
| 511 | |
| 512 | func (w *htmlTextWriter) lastIsSpace() bool { |
| 513 | if w.b.Len() == 0 { |
| 514 | return false |
| 515 | } |
| 516 | return unicode.IsSpace(rune(w.lastByte())) |
| 517 | } |
| 518 | |
| 519 | func normalizeHTMLText(s string) string { |
| 520 | s = strings.ReplaceAll(s, "\r\n", "\n") |
| 521 | s = trailingWS.ReplaceAllString(s, "\n") |
| 522 | s = multiBlank.ReplaceAllString(s, "\n\n") |
| 523 | return strings.TrimSpace(s) |
| 524 | } |
| 525 | |
| 526 | func collapseHTMLInlineText(s string) string { |
| 527 | if s == "" { |
| 528 | return "" |
| 529 | } |
| 530 | leading := unicode.IsSpace([]rune(s)[0]) |
| 531 | trailing := unicode.IsSpace([]rune(s)[len([]rune(s))-1]) |
| 532 | fields := strings.Fields(s) |
| 533 | if len(fields) == 0 { |
| 534 | return " " |
| 535 | } |
| 536 | out := strings.Join(fields, " ") |
| 537 | if leading { |
| 538 | out = " " + out |
| 539 | } |
| 540 | if trailing { |
| 541 | out += " " |
| 542 | } |
| 543 | return out |
| 544 | } |
| 545 | |
| 546 | func startsWithSpaceOrPunct(s string) bool { |
| 547 | for _, r := range s { |
| 548 | return unicode.IsSpace(r) || strings.ContainsRune(".,;:!?)]}", r) |
| 549 | } |
| 550 | return false |
| 551 | } |
| 552 | |
| 553 | func htmlAttr(tokenizer *nethtml.Tokenizer, hasAttr bool, name string) string { |
| 554 | for hasAttr { |
| 555 | key, val, more := tokenizer.TagAttr() |
| 556 | if strings.EqualFold(string(key), name) { |
| 557 | return stdhtml.UnescapeString(string(val)) |
| 558 | } |
| 559 | hasAttr = more |
| 560 | } |
| 561 | return "" |
| 562 | } |
| 563 | |
| 564 | func htmlBreakTag(tag string) bool { |
| 565 | return tag == "br" || tag == "hr" |
| 566 | } |
| 567 | |
| 568 | func htmlBlockTag(tag string) bool { |
| 569 | switch tag { |
| 570 | case "address", "article", "aside", "blockquote", "body", "caption", "dd", "details", |
| 571 | "dialog", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", |
| 572 | "h1", "h2", "h3", "h4", "h5", "h6", "head", "header", "html", "li", "main", "nav", |
| 573 | "ol", "p", "pre", "section", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul": |
| 574 | return true |
| 575 | default: |
| 576 | return false |
| 577 | } |
| 578 | } |
| 579 | |
| 580 | func contentTypeShort(ct string) string { |
| 581 | if i := strings.IndexByte(ct, ';'); i >= 0 { |
| 582 | ct = ct[:i] |
| 583 | } |
| 584 | return strings.TrimSpace(ct) |
| 585 | } |
| 586 |