| 1 | #!/usr/bin/env python3 |
| 2 | """Widen exactly the runtime items the compiler asks for (runtime/TUI split). |
| 3 | |
| 4 | After `move-modules.py` moves code into `crates/runtime`, items that were |
| 5 | `pub(crate)` inside the TUI are private to the new crate. This loop runs |
| 6 | `cargo check --message-format=json` and rewrites only the definitions the |
| 7 | compiler names: |
| 8 | |
| 9 | * E0603 / E0624 / E0616 / E0451 (private item, method, field, field in a |
| 10 | struct literal) whose definition is in `crates/runtime/src`: the |
| 11 | definition's `pub(crate)` / `pub(super)` / missing visibility becomes `pub`; |
| 12 | a private `mod x;` becomes `pub mod x;`; |
| 13 | * a runtime `dead_code` warning on a `pub(crate)` item: the item is used only |
| 14 | from the TUI, so it is widened the same way; |
| 15 | * `unfulfilled_lint_expectations` in the runtime: an `#[expect(dead_code)]` |
| 16 | (or `cfg_attr(.., expect(..))`) the export made unfulfilled is deleted, but |
| 17 | only when the attribute is alone on its line. |
| 18 | |
| 19 | It never bulk-rewrites `pub(crate)`. Every widened item is printed; items in |
| 20 | security-relevant modules are flagged for human review. |
| 21 | |
| 22 | While the loop runs, the runtime manifest's `[lints] workspace = true` is |
| 23 | commented out (lints capped to warnings), because an item that is dead inside |
| 24 | the runtime would otherwise fail the runtime build under `warnings = "deny"` |
| 25 | before cargo ever reached the TUI's privacy errors. The stanza is restored |
| 26 | when the loop ends, and a final check runs with it. |
| 27 | |
| 28 | Usage: |
| 29 | python3 scripts/split/widen.py [--cargo "scripts/dev-cargo.sh"] [--max-rounds N] |
| 30 | """ |
| 31 | |
| 32 | from __future__ import annotations |
| 33 | |
| 34 | import argparse |
| 35 | import json |
| 36 | import os |
| 37 | import re |
| 38 | import shlex |
| 39 | import subprocess |
| 40 | import sys |
| 41 | from pathlib import Path |
| 42 | |
| 43 | REPO_ROOT = Path(__file__).resolve().parents[2] |
| 44 | RUNTIME_SRC = REPO_ROOT / "crates" / "runtime" / "src" |
| 45 | RUNTIME_MANIFEST = REPO_ROOT / "crates" / "runtime" / "Cargo.toml" |
| 46 | LINTS_ON = "[lints]\nworkspace = true\n" |
| 47 | LINTS_CAPPED = "# [lints] capped by scripts/split/widen.py\n# workspace = true\n" |
| 48 | PRIVACY_CODES = {"E0603", "E0624", "E0616", "E0451"} |
| 49 | SECURITY_MODULES = ( |
| 50 | "sandbox", |
| 51 | "core/authority", |
| 52 | "network_policy", |
| 53 | "repo_law", |
| 54 | "workspace_trust", |
| 55 | "oauth", |
| 56 | "credentials", |
| 57 | "mcp", |
| 58 | "runtime_api", |
| 59 | "execpolicy", |
| 60 | ) |
| 61 | ITEM_KW = r"(?:async\s+|unsafe\s+|const\s+|extern\s+\"[^\"]*\"\s+)*(?:fn|struct|enum|union|trait|type|const|static|mod|use)\b" |
| 62 | |
| 63 | |
| 64 | def run_cargo(cargo: list[str], packages: list[str], extra: list[str]) -> list[dict]: |
| 65 | cmd = [*cargo, "check", "--locked", "--message-format=json", *extra] |
| 66 | for p in packages: |
| 67 | cmd += ["-p", p] |
| 68 | proc = subprocess.run(cmd, cwd=REPO_ROOT, capture_output=True, text=True) |
| 69 | messages = [] |
| 70 | for line in proc.stdout.splitlines(): |
| 71 | try: |
| 72 | msg = json.loads(line) |
| 73 | except json.JSONDecodeError: |
| 74 | continue |
| 75 | if msg.get("reason") == "compiler-message": |
| 76 | messages.append(msg["message"]) |
| 77 | if proc.returncode != 0 and not messages: |
| 78 | sys.stderr.write(proc.stderr[-4000:]) |
| 79 | return messages |
| 80 | |
| 81 | |
| 82 | def runtime_spans(message: dict) -> list[dict]: |
| 83 | spans = list(message.get("spans", [])) |
| 84 | for child in message.get("children", []): |
| 85 | spans.extend(child.get("spans", [])) |
| 86 | out = [] |
| 87 | for span in spans: |
| 88 | path = (REPO_ROOT / span["file_name"]).resolve() |
| 89 | if str(path).startswith(str(RUNTIME_SRC)): |
| 90 | out.append(span) |
| 91 | return out |
| 92 | |
| 93 | |
| 94 | def widen_line(line: str, name: str | None) -> str | None: |
| 95 | """Rewrite one definition line to `pub`; None when it cannot be done safely.""" |
| 96 | m = re.match(r"^(\s*)pub\s*\((?:crate|super|in [^)]*)\)\s+", line) |
| 97 | if m: |
| 98 | return m.group(1) + "pub " + line[m.end() :] |
| 99 | if re.match(r"^\s*pub\s", line): |
| 100 | return None # already public: the error is about a parent module |
| 101 | m = re.match(rf"^(\s*)(?={ITEM_KW})", line) |
| 102 | if m: |
| 103 | return m.group(1) + "pub " + line[m.end() :] |
| 104 | if name: |
| 105 | m = re.match(rf"^(\s*)(?={re.escape(name)}\s*:)", line) |
| 106 | if m: |
| 107 | return m.group(1) + "pub " + line[m.end() :] |
| 108 | return None |
| 109 | |
| 110 | |
| 111 | def find_field(struct: str, field: str) -> tuple[Path, int] | None: |
| 112 | pat = re.compile(rf"\bstruct\s+{re.escape(struct)}\b") |
| 113 | for path in RUNTIME_SRC.rglob("*.rs"): |
| 114 | lines = path.read_text(encoding="utf-8").split("\n") |
| 115 | for i, line in enumerate(lines): |
| 116 | if pat.search(line): |
| 117 | for j in range(i + 1, min(i + 400, len(lines))): |
| 118 | if re.match(rf"^\s*(pub(\([^)]*\))?\s+)?{re.escape(field)}\s*:", lines[j]): |
| 119 | return path, j + 1 |
| 120 | if lines[j].startswith("}"): |
| 121 | break |
| 122 | return None |
| 123 | |
| 124 | |
| 125 | def item_line(path: Path, line_no: int) -> int: |
| 126 | """Walk down from an attribute or doc line to the item line itself.""" |
| 127 | lines = path.read_text(encoding="utf-8").split("\n") |
| 128 | i = line_no - 1 |
| 129 | while i < len(lines) and re.match(r"^\s*(#\[|///|//!)", lines[i]): |
| 130 | i += 1 |
| 131 | return i + 1 |
| 132 | |
| 133 | |
| 134 | def apply(edits: dict[tuple[Path, int], str | None]) -> list[str]: |
| 135 | """Apply per-line edits: a string replaces the line, None deletes it.""" |
| 136 | by_file: dict[Path, list[tuple[int, str | None]]] = {} |
| 137 | for (path, line_no), new in edits.items(): |
| 138 | by_file.setdefault(path, []).append((line_no, new)) |
| 139 | report = [] |
| 140 | for path, changes in by_file.items(): |
| 141 | lines = path.read_text(encoding="utf-8").split("\n") |
| 142 | for line_no, new in sorted(changes, reverse=True): |
| 143 | old = lines[line_no - 1] |
| 144 | rel = path.relative_to(REPO_ROOT) |
| 145 | flag = " [security review]" if any(f"/{m}" in f"/{rel}" for m in SECURITY_MODULES) else "" |
| 146 | if new is None: |
| 147 | del lines[line_no - 1] |
| 148 | report.append(f"removed {rel}:{line_no}: {old.strip()}") |
| 149 | else: |
| 150 | lines[line_no - 1] = new |
| 151 | report.append(f"widened {rel}:{line_no}: {new.strip()}{flag}") |
| 152 | path.write_text("\n".join(lines), encoding="utf-8") |
| 153 | return report |
| 154 | |
| 155 | |
| 156 | def plan_edits(messages: list[dict]) -> tuple[dict, list[str]]: |
| 157 | edits: dict[tuple[Path, int], str | None] = {} |
| 158 | unresolved: list[str] = [] |
| 159 | for msg in messages: |
| 160 | code = (msg.get("code") or {}).get("code") or "" |
| 161 | level = msg.get("level") |
| 162 | text = msg.get("message", "") |
| 163 | if code in PRIVACY_CODES: |
| 164 | name_m = re.search(r"`([^`]+)`", text) |
| 165 | name = name_m.group(1).split("::")[-1] if name_m else None |
| 166 | targets = [s for s in runtime_spans(msg) if not s.get("is_primary") or code == "E0603"] |
| 167 | done = False |
| 168 | for span in targets: |
| 169 | path = (REPO_ROOT / span["file_name"]).resolve() |
| 170 | line_no = item_line(path, span["line_start"]) |
| 171 | line = path.read_text(encoding="utf-8").split("\n")[line_no - 1] |
| 172 | new = widen_line(line, name) |
| 173 | if new is not None: |
| 174 | edits[(path, line_no)] = new |
| 175 | done = True |
| 176 | if not done and code == "E0616": |
| 177 | m = re.search(r"field `(\w+)` of struct `(?:[\w:]+::)?(\w+)", text) |
| 178 | hit = m and find_field(m.group(2), m.group(1)) |
| 179 | if hit: |
| 180 | path, line_no = hit |
| 181 | line = path.read_text(encoding="utf-8").split("\n")[line_no - 1] |
| 182 | new = widen_line(line, m.group(1)) |
| 183 | if new is not None: |
| 184 | edits[(path, line_no)] = new |
| 185 | done = True |
| 186 | if not done: |
| 187 | unresolved.append(msg.get("rendered") or text) |
| 188 | continue |
| 189 | if code == "dead_code" and level in ("warning", "error"): |
| 190 | for span in runtime_spans(msg): |
| 191 | path = (REPO_ROOT / span["file_name"]).resolve() |
| 192 | line_no = span["line_start"] |
| 193 | line = path.read_text(encoding="utf-8").split("\n")[line_no - 1] |
| 194 | if re.match(r"^\s*pub\s*\(", line): |
| 195 | new = widen_line(line, None) |
| 196 | if new is not None: |
| 197 | edits[(path, line_no)] = new |
| 198 | continue |
| 199 | unresolved.append(msg.get("rendered") or text) |
| 200 | continue |
| 201 | if code == "unfulfilled_lint_expectations": |
| 202 | for span in runtime_spans(msg): |
| 203 | path = (REPO_ROOT / span["file_name"]).resolve() |
| 204 | line_no = span["line_start"] |
| 205 | line = path.read_text(encoding="utf-8").split("\n")[line_no - 1].strip() |
| 206 | if line.startswith("#[") and line.endswith("]") and "expect" in line: |
| 207 | edits[(path, line_no)] = None |
| 208 | else: |
| 209 | unresolved.append(msg.get("rendered") or text) |
| 210 | continue |
| 211 | if level == "error": |
| 212 | unresolved.append(msg.get("rendered") or text) |
| 213 | return edits, unresolved |
| 214 | |
| 215 | |
| 216 | def set_lints(capped: bool) -> None: |
| 217 | text = RUNTIME_MANIFEST.read_text(encoding="utf-8") |
| 218 | old, new = (LINTS_ON, LINTS_CAPPED) if capped else (LINTS_CAPPED, LINTS_ON) |
| 219 | if old in text: |
| 220 | RUNTIME_MANIFEST.write_text(text.replace(old, new), encoding="utf-8") |
| 221 | |
| 222 | |
| 223 | def main(argv: list[str] | None = None) -> int: |
| 224 | parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) |
| 225 | parser.add_argument("--cargo", default="scripts/dev-cargo.sh") |
| 226 | parser.add_argument("--max-rounds", type=int, default=12) |
| 227 | parser.add_argument("--packages", default="codewhale-runtime,codewhale-tui") |
| 228 | parser.add_argument("--extra", default="--lib --tests") |
| 229 | args = parser.parse_args(argv) |
| 230 | cargo = shlex.split(args.cargo) |
| 231 | packages = args.packages.split(",") |
| 232 | extra = shlex.split(args.extra) |
| 233 | |
| 234 | log: list[str] = [] |
| 235 | set_lints(capped=True) |
| 236 | try: |
| 237 | for round_no in range(1, args.max_rounds + 1): |
| 238 | messages = run_cargo(cargo, packages, extra) |
| 239 | edits, unresolved = plan_edits(messages) |
| 240 | if not edits: |
| 241 | break |
| 242 | changes = apply(edits) |
| 243 | log.extend(changes) |
| 244 | print(f"round {round_no}: {len(changes)} edit(s)", flush=True) |
| 245 | finally: |
| 246 | set_lints(capped=False) |
| 247 | for line in log: |
| 248 | print(line) |
| 249 | messages = run_cargo(cargo, packages, extra) |
| 250 | edits, unresolved = plan_edits(messages) |
| 251 | remaining = [m for m in messages if m.get("level") == "error"] |
| 252 | if edits or remaining: |
| 253 | print(f"\n{len(remaining)} error(s) and {len(edits)} further edit(s) remain with lints on:") |
| 254 | for m in remaining[:40]: |
| 255 | print(m.get("rendered") or m.get("message")) |
| 256 | for (path, line_no), new in list(edits.items())[:40]: |
| 257 | print(f"pending {path.relative_to(REPO_ROOT)}:{line_no}: {new}") |
| 258 | return 1 |
| 259 | print(f"clean: {len(log)} edit(s)") |
| 260 | return 0 |
| 261 | |
| 262 | |
| 263 | if __name__ == "__main__": |
| 264 | os.environ.setdefault("CARGO_TERM_COLOR", "never") |
| 265 | sys.exit(main()) |
| 266 |