| 1 | // Package productdocs provides offline retrieval over the official Reasonix |
| 2 | // documentation embedded in the application binary. |
| 3 | package productdocs |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "crypto/sha256" |
| 9 | "encoding/binary" |
| 10 | "encoding/hex" |
| 11 | "encoding/json" |
| 12 | "fmt" |
| 13 | "hash" |
| 14 | "io/fs" |
| 15 | "path" |
| 16 | "regexp" |
| 17 | "runtime/debug" |
| 18 | "sort" |
| 19 | "strings" |
| 20 | "sync" |
| 21 | "unicode" |
| 22 | "unicode/utf8" |
| 23 | |
| 24 | "github.com/yuin/goldmark" |
| 25 | "github.com/yuin/goldmark/ast" |
| 26 | "github.com/yuin/goldmark/parser" |
| 27 | goldmarktext "github.com/yuin/goldmark/text" |
| 28 | |
| 29 | productcontent "reasonix/docs" |
| 30 | "reasonix/internal/retrieval" |
| 31 | "reasonix/internal/tool" |
| 32 | releasenotes "reasonix/release-notes" |
| 33 | ) |
| 34 | |
| 35 | const ( |
| 36 | defaultLimit = 5 |
| 37 | maxLimit = 10 |
| 38 | maxSnippet = 360 |
| 39 | maxQueryRunes = 4096 |
| 40 | scoreFloor = 0.15 |
| 41 | ) |
| 42 | |
| 43 | type document struct { |
| 44 | path string |
| 45 | source string |
| 46 | title string |
| 47 | locale string |
| 48 | audience string |
| 49 | releaseNote bool |
| 50 | releaseVersion string |
| 51 | sections []*section |
| 52 | } |
| 53 | |
| 54 | type section struct { |
| 55 | id string |
| 56 | document *document |
| 57 | heading string |
| 58 | content string |
| 59 | searchText string |
| 60 | counts map[string]int |
| 61 | headingHits map[string]int |
| 62 | length int |
| 63 | startLine int |
| 64 | endLine int |
| 65 | } |
| 66 | |
| 67 | func (d *document) displayPath() string { |
| 68 | if d.releaseNote { |
| 69 | return d.path |
| 70 | } |
| 71 | return "docs/" + d.path |
| 72 | } |
| 73 | |
| 74 | func (d *document) sourceRange(startLine, endLine int) string { |
| 75 | if d.releaseNote { |
| 76 | return fmt.Sprintf("%s rendered-lines=%d-%d", d.source, startLine, endLine) |
| 77 | } |
| 78 | return fmt.Sprintf("%s:%d-%d", d.source, startLine, endLine) |
| 79 | } |
| 80 | |
| 81 | type catalog struct { |
| 82 | digest string |
| 83 | docs []*document |
| 84 | byPath map[string]*document |
| 85 | byID map[string]*section |
| 86 | sections []*section |
| 87 | df map[string]int |
| 88 | avgLen float64 |
| 89 | releaseNotes int |
| 90 | } |
| 91 | |
| 92 | // Manifest identifies the exact documentation corpus bundled into one build. |
| 93 | // Version and Revision describe the product binary; Digest binds the sorted |
| 94 | // Markdown sources plus the structured release catalog consumed by retrieval. |
| 95 | type Manifest struct { |
| 96 | Version string `json:"version"` |
| 97 | Revision string `json:"revision"` |
| 98 | Digest string `json:"digest"` |
| 99 | Documents int `json:"documents"` |
| 100 | Sections int `json:"sections"` |
| 101 | ReleaseNotes int `json:"release_notes"` |
| 102 | } |
| 103 | |
| 104 | type searchHit struct { |
| 105 | section *section |
| 106 | score float64 |
| 107 | } |
| 108 | |
| 109 | type docsTool struct { |
| 110 | catalog *catalog |
| 111 | loadErr error |
| 112 | } |
| 113 | |
| 114 | var ( |
| 115 | queryVersionRe = regexp.MustCompile(`(?i)\bv?([0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9a-z.-]+)?)\b`) |
| 116 | |
| 117 | defaultOnce sync.Once |
| 118 | defaultCatalog *catalog |
| 119 | defaultLoadErr error |
| 120 | // linkedVersion and linkedRevision are stamped by official release builds. |
| 121 | // Keeping them here gives CLI and Desktop one shared corpus identity. |
| 122 | linkedVersion = "dev" |
| 123 | linkedRevision string |
| 124 | ) |
| 125 | |
| 126 | // NewTool returns a read-only tool backed by the documentation embedded in the |
| 127 | // current Reasonix build. Loading stays lazy so merely registering the stable |
| 128 | // schema does not add Markdown parsing work to application startup. |
| 129 | func NewTool() tool.Tool { |
| 130 | return &docsTool{} |
| 131 | } |
| 132 | |
| 133 | func loadDefaultCatalog() (*catalog, error) { |
| 134 | defaultOnce.Do(func() { |
| 135 | defaultCatalog, defaultLoadErr = loadCatalogWithReleaseNotes(productcontent.Content, releasenotes.Content) |
| 136 | }) |
| 137 | return defaultCatalog, defaultLoadErr |
| 138 | } |
| 139 | |
| 140 | // EmbeddedManifest returns the identity of the corpus compiled into this |
| 141 | // binary. Diagnostics and release verification intentionally share it. |
| 142 | func EmbeddedManifest() (Manifest, error) { |
| 143 | c, err := loadDefaultCatalog() |
| 144 | if err != nil { |
| 145 | return Manifest{}, err |
| 146 | } |
| 147 | return c.manifest(), nil |
| 148 | } |
| 149 | |
| 150 | // CommandOverview returns the local /docs help text and the identity of the |
| 151 | // exact corpus compiled into this binary. It never calls a model or the network. |
| 152 | func CommandOverview(language string) (string, error) { |
| 153 | return CommandOverviewFor(language, "/docs") |
| 154 | } |
| 155 | |
| 156 | // CommandOverviewFor is CommandOverview with the invocation name selected by |
| 157 | // the runtime resolver (for example /reasonix:docs when /docs is occupied). |
| 158 | func CommandOverviewFor(language, commandName string) (string, error) { |
| 159 | c, err := loadDefaultCatalog() |
| 160 | if err != nil { |
| 161 | return "", fmt.Errorf("load embedded documentation: %w", err) |
| 162 | } |
| 163 | commandName = "/" + strings.TrimPrefix(strings.TrimSpace(commandName), "/") |
| 164 | if commandName == "/" { |
| 165 | commandName = "/docs" |
| 166 | } |
| 167 | m := c.manifest() |
| 168 | identity := fmt.Sprintf("version=%s revision=%s digest=%s", m.Version, m.Revision, m.Digest) |
| 169 | stats := fmt.Sprintf("documents=%d sections=%d releases=%d", m.Documents, m.Sections, m.ReleaseNotes) |
| 170 | switch strings.ToLower(strings.TrimSpace(language)) { |
| 171 | case "zh", "zh-cn": |
| 172 | return fmt.Sprintf("内置 Reasonix 文档\n%s\n%s\n\n用法：%s <问题>\n示例：%s 1.19.5 更新日志\n\n搜索在本地完成，命中的版本匹配资料会交给当前配置的 AI 组织答案。", identity, stats, commandName, commandName), nil |
| 173 | case "zh-tw": |
| 174 | return fmt.Sprintf("內建 Reasonix 文件\n%s\n%s\n\n用法：%s <問題>\n範例：%s 1.19.5 更新日誌\n\n搜尋在本機完成，命中的版本匹配資料會交給目前設定的 AI 組織答案。", identity, stats, commandName, commandName), nil |
| 175 | default: |
| 176 | return fmt.Sprintf("Embedded Reasonix documentation\n%s\n%s\n\nUsage: %s <question>\nExample: %s 1.19.5 changelog\n\nSearch runs locally, then the version-matched evidence is passed to the configured AI to compose the answer.", identity, stats, commandName, commandName), nil |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | // SearchEmbedded searches the exact documentation corpus compiled into this |
| 181 | // binary. It is the host-side retrieval path used by /docs, independent of |
| 182 | // whether the configured model chooses to call the docs tool itself. |
| 183 | func SearchEmbedded(ctx context.Context, query string) (string, error) { |
| 184 | c, err := loadDefaultCatalog() |
| 185 | if err != nil { |
| 186 | return "", fmt.Errorf("load embedded documentation: %w", err) |
| 187 | } |
| 188 | return (&docsTool{catalog: c}).search(ctx, query, "auto", "all", defaultLimit) |
| 189 | } |
| 190 | |
| 191 | // SourceManifest computes the corpus identity from the source Markdown and |
| 192 | // structured release catalog used by a build. |
| 193 | func SourceManifest(docsFS, releaseNotesFS fs.FS) (Manifest, error) { |
| 194 | c, err := loadCatalogWithReleaseNotes(docsFS, releaseNotesFS) |
| 195 | if err != nil { |
| 196 | return Manifest{}, err |
| 197 | } |
| 198 | return c.manifest(), nil |
| 199 | } |
| 200 | |
| 201 | func (c *catalog) manifest() Manifest { |
| 202 | version, revision := buildIdentity() |
| 203 | return Manifest{ |
| 204 | Version: version, |
| 205 | Revision: revision, |
| 206 | Digest: "sha256:" + c.digest, |
| 207 | Documents: len(c.docs), |
| 208 | Sections: len(c.sections), |
| 209 | ReleaseNotes: c.releaseNotes, |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | func buildIdentity() (string, string) { |
| 214 | version := strings.TrimSpace(linkedVersion) |
| 215 | if version == "" { |
| 216 | version = "dev" |
| 217 | } |
| 218 | revision := strings.TrimSpace(linkedRevision) |
| 219 | if info, ok := debug.ReadBuildInfo(); ok { |
| 220 | if version == "dev" && info.Main.Version != "" && info.Main.Version != "(devel)" { |
| 221 | version = info.Main.Version |
| 222 | } |
| 223 | if revision == "" { |
| 224 | modified := false |
| 225 | for _, setting := range info.Settings { |
| 226 | switch setting.Key { |
| 227 | case "vcs.revision": |
| 228 | revision = strings.TrimSpace(setting.Value) |
| 229 | case "vcs.modified": |
| 230 | modified = setting.Value == "true" |
| 231 | } |
| 232 | } |
| 233 | if modified && revision != "" { |
| 234 | revision += "+dirty" |
| 235 | } |
| 236 | } |
| 237 | } |
| 238 | if revision == "" { |
| 239 | revision = "unknown" |
| 240 | } |
| 241 | return version, revision |
| 242 | } |
| 243 | |
| 244 | func (c *catalog) identityLine() string { |
| 245 | m := c.manifest() |
| 246 | return fmt.Sprintf("version=%s revision=%s digest=%s", m.Version, m.Revision, m.Digest) |
| 247 | } |
| 248 | |
| 249 | func (*docsTool) Name() string { return "docs" } |
| 250 | |
| 251 | func (*docsTool) Description() string { |
| 252 | return "Search and read the official documentation embedded in this exact Reasonix build. " + |
| 253 | "Use it before web search or assumptions for Reasonix setup, CLI, Desktop, configuration, permissions, MCP, memory, recovery, provider behavior, and maintainer workflows. " + |
| 254 | "Search first, then read the returned section_id when the full section is needed." |
| 255 | } |
| 256 | |
| 257 | func (*docsTool) Schema() json.RawMessage { |
| 258 | return json.RawMessage(`{ |
| 259 | "type":"object", |
| 260 | "properties":{ |
| 261 | "operation":{"type":"string","enum":["search","read","list"],"description":"search ranks relevant sections; read returns one section or lists a document's sections; list shows the embedded document catalog."}, |
| 262 | "query":{"type":"string","maxLength":4096,"description":"Question, command, configuration key, error phrase, or topic for operation=search."}, |
| 263 | "section_id":{"type":"string","description":"Exact section_id returned by search or by a document section listing. Used by operation=read."}, |
| 264 | "path":{"type":"string","description":"Exact docs/*.md path returned by search/list. With operation=read and no section_id, lists that document's sections."}, |
| 265 | "language":{"type":"string","enum":["auto","all","en","zh-CN"],"description":"Language preference. search defaults to auto from the query; list defaults to all. Explicit en or zh-CN filters results."}, |
| 266 | "audience":{"type":"string","enum":["all","user","developer","maintainer"],"description":"Optional audience filter; defaults to all."}, |
| 267 | "limit":{"type":"integer","minimum":1,"maximum":10,"description":"Maximum search results, default 5, max 10."} |
| 268 | }, |
| 269 | "required":["operation"] |
| 270 | }`) |
| 271 | } |
| 272 | |
| 273 | func (t *docsTool) Execute(ctx context.Context, args json.RawMessage) (string, error) { |
| 274 | if t.loadErr != nil { |
| 275 | return "", fmt.Errorf("load embedded documentation: %w", t.loadErr) |
| 276 | } |
| 277 | if err := ctx.Err(); err != nil { |
| 278 | return "", err |
| 279 | } |
| 280 | c := t.catalog |
| 281 | if c == nil { |
| 282 | var err error |
| 283 | c, err = loadDefaultCatalog() |
| 284 | if err != nil { |
| 285 | return "", fmt.Errorf("load embedded documentation: %w", err) |
| 286 | } |
| 287 | } |
| 288 | if c == nil { |
| 289 | return "", fmt.Errorf("embedded documentation is unavailable") |
| 290 | } |
| 291 | var in struct { |
| 292 | Operation string `json:"operation"` |
| 293 | Query string `json:"query"` |
| 294 | SectionID string `json:"section_id"` |
| 295 | Path string `json:"path"` |
| 296 | Language string `json:"language"` |
| 297 | Audience string `json:"audience"` |
| 298 | Limit int `json:"limit"` |
| 299 | } |
| 300 | if err := json.Unmarshal(args, &in); err != nil { |
| 301 | return "", fmt.Errorf("invalid arguments: %w", err) |
| 302 | } |
| 303 | language, err := normalizeLanguage(in.Language) |
| 304 | if err != nil { |
| 305 | return "", err |
| 306 | } |
| 307 | audience, err := normalizeAudience(in.Audience) |
| 308 | if err != nil { |
| 309 | return "", err |
| 310 | } |
| 311 | switch strings.ToLower(strings.TrimSpace(in.Operation)) { |
| 312 | case "search": |
| 313 | return (&docsTool{catalog: c}).search(ctx, in.Query, language, audience, in.Limit) |
| 314 | case "read": |
| 315 | return (&docsTool{catalog: c}).read(in.SectionID, in.Path) |
| 316 | case "list": |
| 317 | return (&docsTool{catalog: c}).list(language, audience), nil |
| 318 | case "": |
| 319 | return "", fmt.Errorf("operation is required") |
| 320 | default: |
| 321 | return "", fmt.Errorf("unknown operation %q", in.Operation) |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | func (*docsTool) ReadOnly() bool { return true } |
| 326 | |
| 327 | func (*docsTool) SnipHint() tool.SnipHint { |
| 328 | return tool.SnipHint{Head: 24, Tail: 6, HeadChars: 8000, TailChars: 1500} |
| 329 | } |
| 330 | |
| 331 | func (t *docsTool) search(ctx context.Context, query, language, audience string, limit int) (string, error) { |
| 332 | query = strings.TrimSpace(query) |
| 333 | if utf8.RuneCountInString(query) > maxQueryRunes { |
| 334 | return "", fmt.Errorf("query is too long: maximum %d characters", maxQueryRunes) |
| 335 | } |
| 336 | queryTerms, err := retrieval.QueryTerms(query) |
| 337 | if err != nil { |
| 338 | return "", err |
| 339 | } |
| 340 | limit = clamp(limit, defaultLimit, maxLimit) |
| 341 | preferredLanguage := language |
| 342 | if preferredLanguage == "auto" { |
| 343 | preferredLanguage = detectQueryLanguage(query) |
| 344 | } |
| 345 | queryLower := strings.ToLower(query) |
| 346 | queryVersions := queryVersionRe.FindAllStringSubmatch(queryLower, -1) |
| 347 | hits := make([]searchHit, 0, len(t.catalog.sections)) |
| 348 | for _, section := range t.catalog.sections { |
| 349 | if err := ctx.Err(); err != nil { |
| 350 | return "", err |
| 351 | } |
| 352 | if language != "auto" && language != "all" && section.document.locale != language { |
| 353 | continue |
| 354 | } |
| 355 | if audience != "all" && section.document.audience != audience { |
| 356 | continue |
| 357 | } |
| 358 | score := retrieval.BM25Score(section.counts, section.length, queryTerms, t.catalog.df, len(t.catalog.sections), t.catalog.avgLen) |
| 359 | exactReleaseVersion := false |
| 360 | for _, match := range queryVersions { |
| 361 | if len(match) > 1 && strings.EqualFold(section.document.releaseVersion, match[1]) { |
| 362 | exactReleaseVersion = true |
| 363 | break |
| 364 | } |
| 365 | } |
| 366 | if score <= 0 && !exactReleaseVersion { |
| 367 | continue |
| 368 | } |
| 369 | if exactReleaseVersion { |
| 370 | // Release-note virtual paths carry the exact requested version. Keep |
| 371 | // that stronger than generic terms such as "changelog" or "更新日志". |
| 372 | score += 100 |
| 373 | } |
| 374 | for _, term := range queryTerms { |
| 375 | if section.headingHits[term] > 0 { |
| 376 | score += 0.35 |
| 377 | } |
| 378 | } |
| 379 | if queryLower != "" && strings.Contains(strings.ToLower(section.searchText), queryLower) { |
| 380 | score += 1.5 |
| 381 | } |
| 382 | if preferredLanguage != "all" && section.document.locale == preferredLanguage { |
| 383 | score *= 1.15 |
| 384 | } |
| 385 | hits = append(hits, searchHit{section: section, score: score}) |
| 386 | } |
| 387 | sort.Slice(hits, func(i, j int) bool { |
| 388 | if hits[i].score == hits[j].score { |
| 389 | return hits[i].section.id < hits[j].section.id |
| 390 | } |
| 391 | return hits[i].score > hits[j].score |
| 392 | }) |
| 393 | hits = retrieval.KeepTopRelativeScore(hits, scoreFloor, func(hit searchHit) float64 { return hit.score }) |
| 394 | if len(hits) > limit { |
| 395 | hits = hits[:limit] |
| 396 | } |
| 397 | return formatSearchResults(query, t.catalog.identityLine(), hits), nil |
| 398 | } |
| 399 | |
| 400 | func (t *docsTool) read(sectionID, documentPath string) (string, error) { |
| 401 | sectionID = strings.TrimSpace(sectionID) |
| 402 | documentPath = strings.TrimSpace(documentPath) |
| 403 | if sectionID != "" { |
| 404 | section, ok := t.catalog.byID[sectionID] |
| 405 | if !ok { |
| 406 | return "", fmt.Errorf("unknown section_id %q; use operation=search or read with an exact path to list section ids", sectionID) |
| 407 | } |
| 408 | return fmt.Sprintf("Embedded Reasonix documentation (%s)\nsource: %s\npath: %s\nsection_id: %s\nlocale: %s\naudience: %s\nheading: %s\n\n%s", |
| 409 | t.catalog.identityLine(), section.document.sourceRange(section.startLine, section.endLine), section.document.displayPath(), section.id, |
| 410 | section.document.locale, section.document.audience, section.heading, strings.TrimSpace(section.content)), nil |
| 411 | } |
| 412 | if documentPath == "" { |
| 413 | return "", fmt.Errorf("section_id or path is required for operation=read") |
| 414 | } |
| 415 | doc, ok := t.catalog.byPath[documentPath] |
| 416 | if !ok { |
| 417 | doc, ok = t.catalog.byPath[strings.TrimPrefix(documentPath, "docs/")] |
| 418 | } |
| 419 | if !ok { |
| 420 | return "", fmt.Errorf("unknown documentation path %q; use operation=list for exact paths", documentPath) |
| 421 | } |
| 422 | var b strings.Builder |
| 423 | fmt.Fprintf(&b, "%s (%s, audience=%s)\npath: %s\nsource: %s\nbuild: %s\n", doc.title, doc.locale, doc.audience, doc.displayPath(), doc.source, t.catalog.identityLine()) |
| 424 | for _, section := range doc.sections { |
| 425 | fmt.Fprintf(&b, "\n- section_id=%s lines=%d-%d heading=%s", section.id, section.startLine, section.endLine, section.heading) |
| 426 | } |
| 427 | b.WriteString("\n\nUse operation=read with section_id to read one complete section.") |
| 428 | return b.String(), nil |
| 429 | } |
| 430 | |
| 431 | func (t *docsTool) list(language, audience string) string { |
| 432 | var b strings.Builder |
| 433 | fmt.Fprintf(&b, "Embedded Reasonix documentation catalog (%s):\n", t.catalog.identityLine()) |
| 434 | count := 0 |
| 435 | for _, doc := range t.catalog.docs { |
| 436 | if language != "auto" && language != "all" && doc.locale != language { |
| 437 | continue |
| 438 | } |
| 439 | if audience != "all" && doc.audience != audience { |
| 440 | continue |
| 441 | } |
| 442 | count++ |
| 443 | fmt.Fprintf(&b, "\n- path=%s locale=%s audience=%s sections=%d title=%s", doc.displayPath(), doc.locale, doc.audience, len(doc.sections), doc.title) |
| 444 | } |
| 445 | if count == 0 { |
| 446 | b.WriteString("\n\nNo embedded documents matched the requested filters.") |
| 447 | } else { |
| 448 | b.WriteString("\n\nUse operation=read with an exact path to list its section ids, or operation=search to rank relevant sections.") |
| 449 | } |
| 450 | return b.String() |
| 451 | } |
| 452 | |
| 453 | func loadCatalog(fsys fs.FS) (*catalog, error) { |
| 454 | return loadCatalogWithReleaseNotes(fsys, nil) |
| 455 | } |
| 456 | |
| 457 | func loadCatalogWithReleaseNotes(docsFS, releaseNotesFS fs.FS) (*catalog, error) { |
| 458 | entries, err := fs.ReadDir(docsFS, ".") |
| 459 | if err != nil { |
| 460 | return nil, err |
| 461 | } |
| 462 | sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) |
| 463 | hash := sha256.New() |
| 464 | _, _ = hash.Write([]byte("reasonix-product-docs-v1\x00")) |
| 465 | markdownParser := goldmark.DefaultParser() |
| 466 | c := &catalog{byPath: map[string]*document{}, byID: map[string]*section{}} |
| 467 | for _, entry := range entries { |
| 468 | if entry.IsDir() || !strings.HasSuffix(strings.ToLower(entry.Name()), ".md") { |
| 469 | continue |
| 470 | } |
| 471 | data, err := fs.ReadFile(docsFS, entry.Name()) |
| 472 | if err != nil { |
| 473 | return nil, fmt.Errorf("read %s: %w", entry.Name(), err) |
| 474 | } |
| 475 | if !utf8.Valid(data) { |
| 476 | return nil, fmt.Errorf("read %s: Markdown is not valid UTF-8", entry.Name()) |
| 477 | } |
| 478 | writeDigestRecord(hash, entry.Name(), data) |
| 479 | doc := parseDocumentWithParser(entry.Name(), string(data), markdownParser) |
| 480 | doc.source = "docs/" + entry.Name() |
| 481 | if len(doc.sections) == 0 { |
| 482 | continue |
| 483 | } |
| 484 | if err := c.addDocument(doc); err != nil { |
| 485 | return nil, err |
| 486 | } |
| 487 | } |
| 488 | if releaseNotesFS != nil { |
| 489 | data, err := fs.ReadFile(releaseNotesFS, "releases.json") |
| 490 | if err != nil { |
| 491 | return nil, fmt.Errorf("read release-notes/releases.json: %w", err) |
| 492 | } |
| 493 | if !utf8.Valid(data) { |
| 494 | return nil, fmt.Errorf("read release-notes/releases.json: JSON is not valid UTF-8") |
| 495 | } |
| 496 | writeDigestRecord(hash, "release-notes/releases.json", data) |
| 497 | rendered, releaseCount, err := renderReleaseDocuments(data) |
| 498 | if err != nil { |
| 499 | return nil, err |
| 500 | } |
| 501 | for _, virtual := range rendered { |
| 502 | doc := parseDocumentWithParser(virtual.path, virtual.content, markdownParser) |
| 503 | doc.source = virtual.source |
| 504 | doc.locale = virtual.locale |
| 505 | doc.audience = "user" |
| 506 | doc.releaseNote = true |
| 507 | doc.releaseVersion = virtual.version |
| 508 | if len(doc.sections) == 0 { |
| 509 | return nil, fmt.Errorf("rendered release note %s contains no sections", virtual.path) |
| 510 | } |
| 511 | if err := c.addDocument(doc); err != nil { |
| 512 | return nil, err |
| 513 | } |
| 514 | } |
| 515 | c.releaseNotes = releaseCount |
| 516 | } |
| 517 | if len(c.docs) == 0 || len(c.sections) == 0 { |
| 518 | return nil, fmt.Errorf("embedded documentation corpus is empty") |
| 519 | } |
| 520 | c.digest = hex.EncodeToString(hash.Sum(nil)) |
| 521 | counts := make([]map[string]int, 0, len(c.sections)) |
| 522 | totalLength := 0 |
| 523 | for _, section := range c.sections { |
| 524 | counts = append(counts, section.counts) |
| 525 | totalLength += section.length |
| 526 | } |
| 527 | c.df = retrieval.DocumentFrequency(counts) |
| 528 | c.avgLen = float64(totalLength) / float64(len(c.sections)) |
| 529 | if c.avgLen <= 0 { |
| 530 | c.avgLen = 1 |
| 531 | } |
| 532 | return c, nil |
| 533 | } |
| 534 | |
| 535 | func writeDigestRecord(destination hash.Hash, name string, data []byte) { |
| 536 | var size [8]byte |
| 537 | binary.BigEndian.PutUint64(size[:], uint64(len(name))) |
| 538 | _, _ = destination.Write(size[:]) |
| 539 | _, _ = destination.Write([]byte(name)) |
| 540 | binary.BigEndian.PutUint64(size[:], uint64(len(data))) |
| 541 | _, _ = destination.Write(size[:]) |
| 542 | _, _ = destination.Write(data) |
| 543 | } |
| 544 | |
| 545 | func (c *catalog) addDocument(doc *document) error { |
| 546 | if _, exists := c.byPath[doc.path]; exists { |
| 547 | return fmt.Errorf("duplicate embedded documentation path %q", doc.path) |
| 548 | } |
| 549 | c.docs = append(c.docs, doc) |
| 550 | c.byPath[doc.path] = doc |
| 551 | c.byPath[doc.displayPath()] = doc |
| 552 | for _, section := range doc.sections { |
| 553 | if _, exists := c.byID[section.id]; exists { |
| 554 | return fmt.Errorf("duplicate embedded documentation section %q", section.id) |
| 555 | } |
| 556 | c.sections = append(c.sections, section) |
| 557 | c.byID[section.id] = section |
| 558 | } |
| 559 | return nil |
| 560 | } |
| 561 | |
| 562 | func parseDocument(name, content string) *document { |
| 563 | return parseDocumentWithParser(name, content, goldmark.DefaultParser()) |
| 564 | } |
| 565 | |
| 566 | func parseDocumentWithParser(name, content string, markdownParser parser.Parser) *document { |
| 567 | source := []byte(content) |
| 568 | doc := &document{ |
| 569 | path: path.Clean(name), |
| 570 | title: strings.TrimSuffix(strings.TrimSuffix(name, ".md"), ".zh-CN"), |
| 571 | locale: detectDocumentLanguage(name, content), |
| 572 | audience: documentAudience(name), |
| 573 | } |
| 574 | type headingNode struct { |
| 575 | start int |
| 576 | level int |
| 577 | text string |
| 578 | } |
| 579 | var headingNodes []headingNode |
| 580 | root := markdownParser.Parse(goldmarktext.NewReader(source)) |
| 581 | _ = ast.Walk(root, func(node ast.Node, entering bool) (ast.WalkStatus, error) { |
| 582 | if !entering || node.Kind() != ast.KindHeading || node.Parent() == nil || node.Parent().Kind() != ast.KindDocument { |
| 583 | return ast.WalkContinue, nil |
| 584 | } |
| 585 | heading := node.(*ast.Heading) |
| 586 | if heading.Level > 4 || heading.Pos() < 0 { |
| 587 | return ast.WalkContinue, nil |
| 588 | } |
| 589 | text := strings.TrimSpace(markdownHeadingText(heading, source)) |
| 590 | if text == "" { |
| 591 | return ast.WalkContinue, nil |
| 592 | } |
| 593 | headingNodes = append(headingNodes, headingNode{start: heading.Pos(), level: heading.Level, text: text}) |
| 594 | return ast.WalkContinue, nil |
| 595 | }) |
| 596 | for _, heading := range headingNodes { |
| 597 | if heading.level == 1 { |
| 598 | doc.title = heading.text |
| 599 | break |
| 600 | } |
| 601 | } |
| 602 | lineStarts := sourceLineStarts(source) |
| 603 | var headings [4]string |
| 604 | sectionNumber := 0 |
| 605 | appendSection := func(start, end int, currentHeading string) { |
| 606 | start, end = trimSourceBounds(source, start, end) |
| 607 | if end <= start { |
| 608 | return |
| 609 | } |
| 610 | raw := string(source[start:end]) |
| 611 | sectionNumber++ |
| 612 | searchText := strings.Join([]string{doc.title, currentHeading, raw}, "\n") |
| 613 | terms := retrieval.Tokens(searchText) |
| 614 | id := fmt.Sprintf("%s::s%03d", doc.path, sectionNumber) |
| 615 | section := §ion{ |
| 616 | id: id, |
| 617 | document: doc, |
| 618 | heading: currentHeading, |
| 619 | content: raw, |
| 620 | searchText: searchText, |
| 621 | counts: retrieval.Counts(terms), |
| 622 | headingHits: retrieval.Counts(retrieval.Tokens(doc.title + " " + currentHeading)), |
| 623 | length: len(terms), |
| 624 | startLine: sourceLineNumber(lineStarts, start), |
| 625 | endLine: sourceLineNumber(lineStarts, end-1), |
| 626 | } |
| 627 | doc.sections = append(doc.sections, section) |
| 628 | } |
| 629 | if len(headingNodes) == 0 { |
| 630 | appendSection(0, len(source), doc.title) |
| 631 | return doc |
| 632 | } |
| 633 | if headingNodes[0].start > 0 { |
| 634 | appendSection(0, headingNodes[0].start, doc.title) |
| 635 | } |
| 636 | for i, heading := range headingNodes { |
| 637 | end := len(source) |
| 638 | if i+1 < len(headingNodes) { |
| 639 | end = headingNodes[i+1].start |
| 640 | } |
| 641 | headings[heading.level-1] = heading.text |
| 642 | for j := heading.level; j < len(headings); j++ { |
| 643 | headings[j] = "" |
| 644 | } |
| 645 | var trail []string |
| 646 | for _, value := range headings { |
| 647 | if value != "" { |
| 648 | trail = append(trail, value) |
| 649 | } |
| 650 | } |
| 651 | appendSection(heading.start, end, strings.Join(trail, " > ")) |
| 652 | } |
| 653 | return doc |
| 654 | } |
| 655 | |
| 656 | func markdownHeadingText(heading *ast.Heading, source []byte) string { |
| 657 | var text strings.Builder |
| 658 | _ = ast.Walk(heading, func(node ast.Node, entering bool) (ast.WalkStatus, error) { |
| 659 | if !entering { |
| 660 | return ast.WalkContinue, nil |
| 661 | } |
| 662 | switch node := node.(type) { |
| 663 | case *ast.Text: |
| 664 | text.Write(node.Value(source)) |
| 665 | if node.SoftLineBreak() { |
| 666 | text.WriteByte('\n') |
| 667 | } |
| 668 | case *ast.String: |
| 669 | text.Write(node.Value) |
| 670 | case *ast.AutoLink: |
| 671 | text.Write(node.Label(source)) |
| 672 | case *ast.RawHTML: |
| 673 | text.Write(node.Segments.Value(source)) |
| 674 | } |
| 675 | return ast.WalkContinue, nil |
| 676 | }) |
| 677 | return text.String() |
| 678 | } |
| 679 | |
| 680 | func trimSourceBounds(source []byte, start, end int) (int, int) { |
| 681 | if start < 0 { |
| 682 | start = 0 |
| 683 | } |
| 684 | if end > len(source) { |
| 685 | end = len(source) |
| 686 | } |
| 687 | if end <= start { |
| 688 | return start, start |
| 689 | } |
| 690 | trimmedLeft := bytes.TrimLeft(source[start:end], " \t\r\n") |
| 691 | start = end - len(trimmedLeft) |
| 692 | trimmed := bytes.TrimRight(trimmedLeft, " \t\r\n") |
| 693 | return start, start + len(trimmed) |
| 694 | } |
| 695 | |
| 696 | func sourceLineStarts(source []byte) []int { |
| 697 | starts := []int{0} |
| 698 | for i, value := range source { |
| 699 | if value == '\n' && i+1 < len(source) { |
| 700 | starts = append(starts, i+1) |
| 701 | } |
| 702 | } |
| 703 | return starts |
| 704 | } |
| 705 | |
| 706 | func sourceLineNumber(starts []int, offset int) int { |
| 707 | index := sort.Search(len(starts), func(i int) bool { return starts[i] > offset }) |
| 708 | if index == 0 { |
| 709 | return 1 |
| 710 | } |
| 711 | return index |
| 712 | } |
| 713 | |
| 714 | func normalizeLanguage(language string) (string, error) { |
| 715 | switch strings.ToLower(strings.TrimSpace(language)) { |
| 716 | case "", "auto": |
| 717 | return "auto", nil |
| 718 | case "all": |
| 719 | return "all", nil |
| 720 | case "en", "en-us", "english": |
| 721 | return "en", nil |
| 722 | case "zh", "zh-cn", "cn", "chinese": |
| 723 | return "zh-CN", nil |
| 724 | default: |
| 725 | return "", fmt.Errorf("unknown language %q; use auto, all, en, or zh-CN", language) |
| 726 | } |
| 727 | } |
| 728 | |
| 729 | func normalizeAudience(audience string) (string, error) { |
| 730 | switch strings.ToLower(strings.TrimSpace(audience)) { |
| 731 | case "", "all": |
| 732 | return "all", nil |
| 733 | case "user", "developer", "maintainer": |
| 734 | return strings.ToLower(strings.TrimSpace(audience)), nil |
| 735 | default: |
| 736 | return "", fmt.Errorf("unknown audience %q; use all, user, developer, or maintainer", audience) |
| 737 | } |
| 738 | } |
| 739 | |
| 740 | func detectQueryLanguage(query string) string { |
| 741 | for _, r := range query { |
| 742 | if unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul) { |
| 743 | return "zh-CN" |
| 744 | } |
| 745 | } |
| 746 | return "en" |
| 747 | } |
| 748 | |
| 749 | func detectDocumentLanguage(name, content string) string { |
| 750 | if strings.HasSuffix(strings.ToLower(name), ".zh-cn.md") { |
| 751 | return "zh-CN" |
| 752 | } |
| 753 | han, latin := 0, 0 |
| 754 | for _, r := range content { |
| 755 | switch { |
| 756 | case unicode.In(r, unicode.Han, unicode.Hiragana, unicode.Katakana, unicode.Hangul): |
| 757 | han++ |
| 758 | case unicode.Is(unicode.Latin, r): |
| 759 | latin++ |
| 760 | } |
| 761 | } |
| 762 | if han > 100 && han*4 > latin { |
| 763 | return "zh-CN" |
| 764 | } |
| 765 | return "en" |
| 766 | } |
| 767 | |
| 768 | func documentAudience(name string) string { |
| 769 | stem := strings.TrimSuffix(strings.TrimSuffix(name, ".md"), ".zh-CN") |
| 770 | switch strings.ToUpper(stem) { |
| 771 | case "RELEASING", "SIGNPATH_WINDOWS_ADMIN_SOP", "PRODUCTION_CHECKLIST", "THEME_ASSETS": |
| 772 | return "maintainer" |
| 773 | case "CHECKPOINTS", "GOAL_ENFORCEMENT", "SESSION_REFERENCE_ARCHITECTURE", "SPEC", "TASK_CONTRACT", "TOOL_CONTRACT": |
| 774 | return "developer" |
| 775 | default: |
| 776 | return "user" |
| 777 | } |
| 778 | } |
| 779 | |
| 780 | func formatSearchResults(query, identity string, hits []searchHit) string { |
| 781 | if len(hits) == 0 { |
| 782 | return fmt.Sprintf("No embedded Reasonix documentation matched %q (%s). Try fewer terms, an exact command/configuration key, language=all, or audience=all.", query, identity) |
| 783 | } |
| 784 | var b strings.Builder |
| 785 | fmt.Fprintf(&b, "Embedded Reasonix documentation results for %q (%s):\n", query, identity) |
| 786 | for i, hit := range hits { |
| 787 | section := hit.section |
| 788 | fmt.Fprintf(&b, "\n%d. score=%.3f source=%s path=%s section_id=%s locale=%s audience=%s\n heading: %s\n snippet: %s\n", |
| 789 | i+1, hit.score, section.document.sourceRange(section.startLine, section.endLine), section.document.displayPath(), section.id, |
| 790 | section.document.locale, section.document.audience, section.heading, |
| 791 | retrieval.MakeSnippet(section.searchText, query, retrieval.Unique(retrieval.Tokens(query)), maxSnippet)) |
| 792 | } |
| 793 | b.WriteString("\nUse operation=read with section_id to read the complete embedded section. Cite the source path and line range in the answer.") |
| 794 | return strings.TrimSpace(b.String()) |
| 795 | } |
| 796 | |
| 797 | func clamp(value, fallback, maximum int) int { |
| 798 | if value <= 0 { |
| 799 | return fallback |
| 800 | } |
| 801 | if value > maximum { |
| 802 | return maximum |
| 803 | } |
| 804 | return value |
| 805 | } |
| 806 |