| 1 | #!/usr/bin/env python3 |
| 2 | """check-lexicon.py — warn when user-facing English copy drifts from the lexicon. |
| 3 | |
| 4 | The product vocabulary is one word per concept across the TUI, the app, the |
| 5 | site and the docs: Agent and Fleet; Plan, Work and Operate; Permissions |
| 6 | (Ask, Auto-Review, Full Access); Tasks panel; Making room; Settings; the |
| 7 | presence words; Thinking. This script greps the English sources a person |
| 8 | actually reads for the words that decision retires, plus the engineering notes experience mark 5 keeps off product |
| 9 | surfaces (issue keys, HTTP routes, "Ctrl/Cmd"). |
| 10 | |
| 11 | Scanned: |
| 12 | - crates/localization/locales/en.json (values only, never keys) |
| 13 | - web/lib/i18n/dictionaries/en/*.ts (string literals) |
| 14 | - web/lib/content/*.ts (string literals) |
| 15 | - crates/tui/src/**/*.rs (string literals) |
| 16 | |
| 17 | The Rust scan reads string literals outside comments, test files and |
| 18 | `#[cfg(test)]` / `#[cfg(all(test, ...))]` modules and functions. It skips |
| 19 | literals passed to log, tracing, panic, assert and expect calls (developer |
| 20 | text, not copy), literals without a space (identifiers, keys, paths), |
| 21 | model-facing text (tools/, prompts/, runtime-event envelopes), and code |
| 22 | inside copy: slash commands, --flags, config keys and usage alternatives. |
| 23 | |
| 24 | Known limits: it does not see copy assembled outside crates/tui/src or split |
| 25 | across push_str calls, nor clap `--help` text written as `///` doc comments |
| 26 | (comments are masked). It cannot tell an error the user reads from one only a |
| 27 | log shows, so expect some findings to be internal text worth an allow entry. |
| 28 | The subcommand names in RUST_SUBCOMMANDS are skipped after a slash command, |
| 29 | so a retired term used as one of those subcommands is not reported. The Rust |
| 30 | scan adds about 2.5 s to a run. |
| 31 | |
| 32 | It is WARN-ONLY: it prints findings and exits 0 so it can run from |
| 33 | scripts/preflight.sh without turning a push red while the sweep finishes. |
| 34 | Pass --strict to exit 1 on any finding (for a local ratchet or a future CI |
| 35 | gate). Parser aliases, config keys and locale message identifiers are code, |
| 36 | not copy, and are out of scope. |
| 37 | |
| 38 | python3 scripts/check-lexicon.py # warn |
| 39 | python3 scripts/check-lexicon.py --strict # fail on findings |
| 40 | python3 scripts/check-lexicon.py --summary # counts only |
| 41 | python3 scripts/check-lexicon.py --rust-only # skip locale and web |
| 42 | """ |
| 43 | |
| 44 | from __future__ import annotations |
| 45 | |
| 46 | import argparse |
| 47 | import json |
| 48 | import re |
| 49 | import sys |
| 50 | from collections import Counter |
| 51 | from pathlib import Path |
| 52 | |
| 53 | ROOT = Path(__file__).resolve().parent.parent |
| 54 | EN_LOCALE = ROOT / "crates" / "localization" / "locales" / "en.json" |
| 55 | WEB_GLOBS = ( |
| 56 | "web/lib/i18n/dictionaries/en/*.ts", |
| 57 | "web/lib/content/*.ts", |
| 58 | ) |
| 59 | |
| 60 | # (label, use-instead, compiled pattern). Case matters where the retired word |
| 61 | # is also an ordinary English word ("Act" the mode vs. "act" the verb). |
| 62 | RULES: list[tuple[str, str, re.Pattern[str]]] = [ |
| 63 | # §19 modes |
| 64 | ("Act", "Work", re.compile(r"\bAct\b|\bACT\b")), |
| 65 | ("read-only lane", "Plan", re.compile(r"read-only lane", re.I)), |
| 66 | ("agent mode", "Work", re.compile(r"\bagent mode\b", re.I)), |
| 67 | ("Fleet mode", "Operate", re.compile(r"\bfleet mode\b", re.I)), |
| 68 | ("operator", "Coordinator (or Operate for the mode)", re.compile(r"\boperator\b", re.I)), |
| 69 | # §19 permissions |
| 70 | ("posture", "Permissions", re.compile(r"\bpostures?\b", re.I)), |
| 71 | ("approval policy", "Permissions", re.compile(r"\b(approval|permission) policy\b", re.I)), |
| 72 | # §16 / §19 Fleet and agents |
| 73 | ("roster", "Fleet", re.compile(r"\broster\b", re.I)), |
| 74 | ("worker", "agent", re.compile(r"(?<!Cloudflare )\bworkers?\b", re.I)), |
| 75 | ("sub-agent", "agent", re.compile(r"\bsub-?agents?\b", re.I)), |
| 76 | ("lane", "agent", re.compile(r"\blanes?\b", re.I)), |
| 77 | ("leader", "Coordinator", re.compile(r"\bleader\b", re.I)), |
| 78 | ("consultant", "Advisor", re.compile(r"\bconsultants?\b", re.I)), |
| 79 | # §19 surfaces and states |
| 80 | ("Work bar", "Tasks panel", re.compile(r"\bwork ?bar\b|\bwork dock\b|\brail panel\b", re.I)), |
| 81 | ("Bash", "Command", re.compile(r"\bBash\b")), |
| 82 | ("MCP Read/Action", "Connected app", re.compile(r"\bMCP (Read|Action)\b")), |
| 83 | ("Deny this call", "Don't allow", re.compile(r"Deny this call|\(this kind\)")), |
| 84 | ("abort", "Stop", re.compile(r"\babort(ed|s|ing)?\b", re.I)), |
| 85 | ("compaction", "Making room", re.compile(r"\bauto-?compact\w*|\bcompaction\b", re.I)), |
| 86 | ("waiting on you", "needs you", re.compile(r"waiting on you", re.I)), |
| 87 | ("unobserved", "resting", re.compile(r"\bunobserved\b", re.I)), |
| 88 | ("Reasoning", "Thinking", re.compile(r"\bReasoning\b")), |
| 89 | ("charter", "Constitution", re.compile(r"\bcharter\b", re.I)), |
| 90 | # Experience mark 5: no engineering notes on product surfaces |
| 91 | ("issue key", "(remove)", re.compile(r"\bAPPS-\d+\b|\bSHA-(?!(1|256|384|512)\b)\d+\b|\(#\d{3,}\)")), |
| 92 | ("HTTP route", "(describe what works)", re.compile(r"\b(GET|POST|PUT|DELETE|PATCH) /")), |
| 93 | ("Ctrl/Cmd", "one key notation", re.compile(r"Ctrl/Cmd")), |
| 94 | ] |
| 95 | |
| 96 | # A settings screen titled "Config" (§19: Settings). Exact values only, so |
| 97 | # "Config file:" and "config.toml" stay legal. |
| 98 | CONFIG_TITLE = re.compile(r"^\s*Config\s*$") |
| 99 | |
| 100 | # Deliberate, reviewed exceptions: {source-relative path: {key or literal: {labels}}}. |
| 101 | # Keep this short; every entry is a promise that the word is the right one. |
| 102 | ALLOW: dict[str, dict[str, set[str]]] = { |
| 103 | "crates/localization/locales/en.json": { |
| 104 | # Names the compatibility slash command the user typed. |
| 105 | "CmdSubagentsDescription": {"worker", "sub-agent"}, |
| 106 | "HomeQuickSubagents": {"worker"}, |
| 107 | }, |
| 108 | } |
| 109 | |
| 110 | # Double-quoted, single-quoted or template string literals in TS sources. |
| 111 | TS_STRING = re.compile(r'"((?:[^"\\\n]|\\.)*)"|\'((?:[^\'\\\n]|\\.)*)\'|`((?:[^`\\]|\\.)*)`') |
| 112 | |
| 113 | |
| 114 | PLACEHOLDER = re.compile(r"\{[A-Za-z_][A-Za-z0-9_]*\}") |
| 115 | |
| 116 | |
| 117 | def findings_for(text: str) -> list[tuple[str, str]]: |
| 118 | # `{posture}` is a substitution slot, not a word the reader sees. |
| 119 | text = PLACEHOLDER.sub("", text) |
| 120 | hits = [(label, instead) for label, instead, rx in RULES if rx.search(text)] |
| 121 | if CONFIG_TITLE.match(text): |
| 122 | hits.append(("Config", "Settings")) |
| 123 | return hits |
| 124 | |
| 125 | |
| 126 | def scan_locale(path: Path): |
| 127 | rel = str(path.relative_to(ROOT)) |
| 128 | allow = ALLOW.get(rel, {}) |
| 129 | data = json.loads(path.read_text(encoding="utf-8")) |
| 130 | for key, value in data.items(): |
| 131 | if not isinstance(value, str): |
| 132 | continue |
| 133 | for label, instead in findings_for(value): |
| 134 | if label in allow.get(key, set()): |
| 135 | continue |
| 136 | yield rel, key, label, instead, value |
| 137 | |
| 138 | |
| 139 | def scan_ts(path: Path): |
| 140 | rel = str(path.relative_to(ROOT)) |
| 141 | allow = ALLOW.get(rel, {}) |
| 142 | text = path.read_text(encoding="utf-8") |
| 143 | for lineno, line in enumerate(text.splitlines(), 1): |
| 144 | stripped = line.lstrip() |
| 145 | if stripped.startswith(("//", "*", "/*", "import ", "export type", "type ")): |
| 146 | continue |
| 147 | for m in TS_STRING.finditer(line): |
| 148 | value = next(g for g in m.groups() if g is not None) |
| 149 | # Skip identifiers, paths and URLs; copy has a space in it. |
| 150 | if " " not in value: |
| 151 | continue |
| 152 | for label, instead in findings_for(value): |
| 153 | if label in allow.get(value, set()): |
| 154 | continue |
| 155 | yield rel, f"L{lineno}", label, instead, value |
| 156 | |
| 157 | |
| 158 | # --- Rust string literals ------------------------------------------------- |
| 159 | |
| 160 | RUST_ROOT = ROOT / "crates" / "tui" / "src" |
| 161 | |
| 162 | # Calls whose string arguments are developer text: logs, panics, assertions. |
| 163 | RUST_INTERNAL_CALLEE = re.compile( |
| 164 | r"(\b(trace|debug|info|warn|error|panic|unreachable|todo|unimplemented" |
| 165 | r"|assert\w*|debug_assert\w*|log_\w+)!" |
| 166 | r"|\b(tracing|log|logging)::\w+(::\w+)*!?" |
| 167 | r"|\.(expect|expect_err|context|with_context)" |
| 168 | r"|#\[(?!(error|arg|command|value)\b)\w+(::\w+)*)\s*$" |
| 169 | ) |
| 170 | RUST_RAW_START = re.compile(r'b?r(#*)"') |
| 171 | # `#[cfg(test)]` or `#[cfg(all(test, ...))]`, any further attributes, then a |
| 172 | # module or function whose body is test code. |
| 173 | RUST_TEST_MOD = re.compile( |
| 174 | r"#\[cfg\((?:test|all\((?:[^\]]*,\s*)?test\b[^\]]*\))\)\]" |
| 175 | r"(?:\s*#\[[^\]]*\])*\s*(?:pub(?:\([\w:]+\))?\s+)?" |
| 176 | r"(?:mod\s+\w+|(?:const\s+|async\s+|unsafe\s+)*fn\s+\w+[^{;]*)\s*\{" |
| 177 | ) |
| 178 | |
| 179 | # Compatibility subcommand names that keep a retired word: `/config subagents`, |
| 180 | # `/constitution posture`, `/fleet workers`. They are typed, not read. |
| 181 | RUST_SUBCOMMANDS = ("subagents", "posture", "workers") |
| 182 | |
| 183 | # Code inside copy: `backticks`, /commands (with a known subcommand or an |
| 184 | # a|b alternative list), --flags, [sections], [a|b] usage alternatives, dotted |
| 185 | # or snake_case keys (including `key.{slot}`), <!-- markers --> and |
| 186 | # <placeholders> name things the user types, not words they read. A path |
| 187 | # after an HTTP verb is left in place so the HTTP-route rule still sees it. |
| 188 | RUST_CODE_TOKEN = re.compile( |
| 189 | r"`[^`]*`|<!--.*?-->|<[\w-]+>" |
| 190 | r"|(?<![\w/])(?<!GET )(?<!PUT )(?<!POST )(?<!PATCH )(?<!DELETE )/[a-z][\w-]*" |
| 191 | rf"(?: (?:{'|'.join(RUST_SUBCOMMANDS)})\b| [a-z][\w-]*(?:\|[\w-]+)+)?" |
| 192 | r"|(?<![\w-])--[a-z][\w-]*" |
| 193 | r"|\[[\w.\-]+\]|\[[^\]\n]*\|[^\]\n]*\]" |
| 194 | r"|\b\w+(?:[._](?:\w+|\{\w*\}))+" |
| 195 | ) |
| 196 | |
| 197 | # Model-facing text: tool descriptions and prompts are read by the model, not |
| 198 | # the person. Their vocabulary follows the tool contract, not §19. |
| 199 | RUST_MODEL_FACING = ( |
| 200 | "crates/tui/src/tools/", |
| 201 | "crates/tui/src/prompts/", |
| 202 | # Runtime-event envelopes and their restore projection for the model. |
| 203 | "crates/tui/src/runtime_handoff.rs", |
| 204 | ) |
| 205 | # A literal carrying a runtime-event envelope is model-facing wherever it lives. |
| 206 | RUST_MODEL_MARKER = "<codewhale:" |
| 207 | |
| 208 | # Deliberate, reviewed Rust exceptions: {path: {literal-prefix: {labels}}}. |
| 209 | # A literal matches an entry when it starts with the entry's text. Keep each |
| 210 | # entry honest: an ordinary English word, not a retired product term. |
| 211 | RUST_ALLOW: dict[str, dict[str, set[str]]] = { |
| 212 | # "roster" is a provider's model list here, not a Fleet. |
| 213 | "crates/tui/src/config.rs": {"Model '{trimmed}' is not in OpenCode Go": {"roster"}}, |
| 214 | "crates/tui/src/lib.rs": {"pinned id `{model}` is absent from": {"roster"}}, |
| 215 | "crates/tui/src/tui/model_picker.rs": {"custom · OAuth roster": {"roster"}}, |
| 216 | # A tool name in an error, not the retired Bash label. |
| 217 | "crates/tui/src/core/engine.rs": {"tool 'Bash' is not registered": {"Bash"}}, |
| 218 | # A network error, not the user stopping a turn. |
| 219 | "crates/tui/src/commands/contract.rs": {"connection aborted": {"abort"}}, |
| 220 | # Background threads, not agents. |
| 221 | "crates/tui/src/tui/window_control.rs": {"window worker panicked": {"worker"}}, |
| 222 | "crates/tui/src/tui/ui/apply.rs": {"the persistence worker is unavailable": {"worker"}}, |
| 223 | # Prompts sent to the model from UI code. |
| 224 | "crates/tui/src/tui/setup/fleet_draft.rs": {"": {"worker", "posture"}}, |
| 225 | "crates/tui/src/tui/setup/model_draft.rs": {"": {"approval policy"}}, |
| 226 | "crates/tui/src/commands/groups/core/agent.rs": {"Launch one sub-agent": {"sub-agent"}}, |
| 227 | "crates/tui/src/commands/groups/core/workflow.rs": {"{WORKFLOW_DRAFT_INSTRUCTION_PREFIX}": {"worker"}}, |
| 228 | "crates/tui/src/operate.rs": {"Keep Operate alive": {"worker"}}, |
| 229 | "crates/tui/src/fleet/worker_runtime.rs": {"- Use the policy-gated tools": {"worker"}}, |
| 230 | "crates/tui/src/fleet/roster.rs": {"Give the operator a direct second opinion": {"operator"}}, |
| 231 | "crates/tui/src/core/engine/context.rs": { |
| 232 | "[sub-agent result summarized for parent context]": {"sub-agent"}, |
| 233 | "- ... {} more sub-agent result(s)": {"sub-agent"}, |
| 234 | }, |
| 235 | "crates/tui/src/compaction.rs": { |
| 236 | "--- Additional instructions from the operator": {"operator"}, |
| 237 | }, |
| 238 | } |
| 239 | |
| 240 | |
| 241 | def is_rust_test_file(path: Path) -> bool: |
| 242 | name = path.name |
| 243 | return ( |
| 244 | name == "tests.rs" |
| 245 | or name.endswith("_tests.rs") |
| 246 | or name.startswith("test_") |
| 247 | or "tests" in path.relative_to(RUST_ROOT).parts[:-1] |
| 248 | ) |
| 249 | |
| 250 | |
| 251 | def rust_tokens(text: str): |
| 252 | """Yield (kind, start, end, value) for comments and string literals.""" |
| 253 | i, n = 0, len(text) |
| 254 | while i < n: |
| 255 | c = text[i] |
| 256 | if text.startswith("//", i): |
| 257 | j = text.find("\n", i) |
| 258 | j = n if j < 0 else j |
| 259 | yield "comment", i, j, "" |
| 260 | i = j |
| 261 | continue |
| 262 | if text.startswith("/*", i): |
| 263 | start, depth, i = i, 1, i + 2 |
| 264 | while i < n and depth: |
| 265 | if text.startswith("/*", i): |
| 266 | depth, i = depth + 1, i + 2 |
| 267 | elif text.startswith("*/", i): |
| 268 | depth, i = depth - 1, i + 2 |
| 269 | else: |
| 270 | i += 1 |
| 271 | yield "comment", start, i, "" |
| 272 | continue |
| 273 | prev = text[i - 1] if i else " " |
| 274 | m = RUST_RAW_START.match(text, i) if c in "br" else None |
| 275 | if m and not (prev.isalnum() or prev == "_"): |
| 276 | close = '"' + m.group(1) |
| 277 | j = text.find(close, m.end()) |
| 278 | j = n if j < 0 else j |
| 279 | yield "string", i, j + len(close), text[m.end():j] |
| 280 | i = j + len(close) |
| 281 | continue |
| 282 | if c == '"': |
| 283 | j = i + 1 |
| 284 | while j < n and text[j] != '"': |
| 285 | j += 2 if text[j] == "\\" else 1 |
| 286 | yield "string", i, j + 1, text[i + 1:j] |
| 287 | i = j + 1 |
| 288 | continue |
| 289 | if c == "'": |
| 290 | # Char literals ('"', '\'', '→'); anything else is a lifetime. |
| 291 | if text.startswith("\\", i + 1): |
| 292 | j = text.find("'", i + 3) |
| 293 | j = (j + 1) if j > 0 else n |
| 294 | yield "char", i, j, "" |
| 295 | i = j |
| 296 | continue |
| 297 | if i + 2 < n and text[i + 2] == "'": |
| 298 | yield "char", i, i + 3, "" |
| 299 | i += 3 |
| 300 | continue |
| 301 | i += 1 |
| 302 | |
| 303 | |
| 304 | def rust_internal(masked: str, start: int) -> bool: |
| 305 | """True when the literal at `start` is an argument of a developer-text call.""" |
| 306 | depth, j = 0, start - 1 |
| 307 | while j >= 0: |
| 308 | ch = masked[j] |
| 309 | if ch in ")]": |
| 310 | depth += 1 |
| 311 | elif ch in "([": |
| 312 | if depth: |
| 313 | depth -= 1 |
| 314 | elif RUST_INTERNAL_CALLEE.search(masked[max(0, j - 60):j]): |
| 315 | return True |
| 316 | elif ch in ";{}" and depth == 0: |
| 317 | return False |
| 318 | j -= 1 |
| 319 | return False |
| 320 | |
| 321 | |
| 322 | def scan_rust(path: Path): |
| 323 | rel = str(path.relative_to(ROOT)) |
| 324 | if rel.startswith(RUST_MODEL_FACING): |
| 325 | return |
| 326 | allow = RUST_ALLOW.get(rel, {}) |
| 327 | text = path.read_text(encoding="utf-8") |
| 328 | tokens = list(rust_tokens(text)) |
| 329 | chars = list(text) |
| 330 | for _, start, end, _ in tokens: |
| 331 | for k in range(start, min(end, len(chars))): |
| 332 | if chars[k] != "\n": |
| 333 | chars[k] = " " |
| 334 | masked = "".join(chars) |
| 335 | # Drop inline `#[cfg(test)] mod name { ... }` blocks. |
| 336 | test_spans = [] |
| 337 | for m in RUST_TEST_MOD.finditer(masked): |
| 338 | depth, j = 1, m.end() |
| 339 | while j < len(masked) and depth: |
| 340 | depth += {"{": 1, "}": -1}.get(masked[j], 0) |
| 341 | j += 1 |
| 342 | test_spans.append((m.start(), j)) |
| 343 | for kind, start, _, value in tokens: |
| 344 | if kind != "string" or " " not in value or RUST_MODEL_MARKER in value: |
| 345 | continue |
| 346 | if any(a <= start < b for a, b in test_spans): |
| 347 | continue |
| 348 | if rust_internal(masked, start): |
| 349 | continue |
| 350 | for label, instead in findings_for(RUST_CODE_TOKEN.sub(" ", value)): |
| 351 | if any(value.startswith(k) and label in v for k, v in allow.items()): |
| 352 | continue |
| 353 | lineno = text.count("\n", 0, start) + 1 |
| 354 | yield rel, f"L{lineno}", label, instead, value |
| 355 | |
| 356 | |
| 357 | def main() -> int: |
| 358 | parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) |
| 359 | parser.add_argument("--strict", action="store_true", help="exit 1 on any finding") |
| 360 | parser.add_argument("--summary", action="store_true", help="print counts only") |
| 361 | parser.add_argument("--rust-only", action="store_true", help="scan only the Rust sources") |
| 362 | args = parser.parse_args() |
| 363 | |
| 364 | found = [] |
| 365 | if not args.rust_only: |
| 366 | if EN_LOCALE.exists(): |
| 367 | found.extend(scan_locale(EN_LOCALE)) |
| 368 | for pattern in WEB_GLOBS: |
| 369 | for path in sorted(ROOT.glob(pattern)): |
| 370 | if path.name.endswith(".test.ts"): |
| 371 | continue |
| 372 | found.extend(scan_ts(path)) |
| 373 | for path in sorted(RUST_ROOT.rglob("*.rs")): |
| 374 | if is_rust_test_file(path): |
| 375 | continue |
| 376 | found.extend(scan_rust(path)) |
| 377 | |
| 378 | counts = Counter(label for _, _, label, _, _ in found) |
| 379 | if not args.summary: |
| 380 | for rel, where, label, instead, value in found: |
| 381 | excerpt = value if len(value) <= 110 else value[:107] + "..." |
| 382 | print(f"{rel}:{where}: '{label}' -> {instead}: {excerpt!r}") |
| 383 | if found: |
| 384 | tally = ", ".join(f"{label} {n}" for label, n in counts.most_common()) |
| 385 | print(f"[lexicon] {len(found)} finding(s): {tally}") |
| 386 | if args.strict: |
| 387 | return 1 |
| 388 | print("[lexicon] warn-only; pass --strict to fail") |
| 389 | else: |
| 390 | print("[lexicon] OK") |
| 391 | return 0 |
| 392 | |
| 393 | |
| 394 | if __name__ == "__main__": |
| 395 | sys.exit(main()) |
| 396 |