返回 CodeWhale
check-persistence-backlog-budget.py
根目录 / scripts / check-persistence-backlog-budget.py
1 #!/usr/bin/env python3
2 """Check the paused persistence backlog against one-way local ceilings.
3
4 Usage:
5 python3 scripts/check-persistence-backlog-budget.py
6 python3 scripts/check-persistence-backlog-budget.py --receipt receipt.json
7 python3 scripts/check-persistence-backlog-budget.py --update
8
9 ``--update`` is the receipt command a failing PR runs to land an intended
10 increase in the same PR: it raises only the exceeded ceilings to the measured
11 values and never lowers one, because the ceilings carry deliberate measurement
12 noise headroom. Tighten by hand, with the reason recorded in the budget.
13 """
14
15 from __future__ import annotations
16
17 import argparse
18 import json
19 import os
20 import re
21 import subprocess
22 import sys
23 from pathlib import Path
24 from typing import Any
25
26
27 ROOT = Path(__file__).resolve().parent.parent
28 MEASURE_SCRIPT = ROOT / "scripts" / "measure-persistence-backlog.py"
29 BUDGET_PATH = ROOT / "scripts" / "persistence-backlog-budget.json"
30 BASELINE_RECEIPT_PATH = ROOT / "scripts" / "persistence-backlog-baseline-receipt.json"
31 BASELINE_RECEIPT_REFERENCE = "scripts/persistence-backlog-baseline-receipt.json"
32 RECEIPT_KIND = "codewhale.persistence_backlog_receipt"
33 BUDGET_KIND = "codewhale.persistence_backlog_budget"
34 SCHEMA_VERSION = 2
35
36 FIXTURE = {
37 "fixture_id": "paused-production-channel-session-snapshot-v1",
38 "request_variant": "session_snapshot",
39 "payload_estimator": "retained-saved-session-json-bytes-v1",
40 "paused_consumer": True,
41 "requests_attempted": 128,
42 "content_bytes_per_request": 64 * 1024,
43 "single_session_id": True,
44 "expected_applied_version": 127,
45 }
46
47 REQUIRED_RECEIPT_FIELDS = (
48 "document_kind",
49 "schema_version",
50 "source_sha",
51 "source_dirty",
52 "rustc_version",
53 "cargo_version",
54 "build_profile",
55 "sample_count",
56 "fixture_id",
57 "platform",
58 "request_variant",
59 "payload_estimator",
60 "paused_consumer",
61 "requests_attempted",
62 "content_bytes_per_request",
63 "single_session_id",
64 "expected_applied_version",
65 "accepted_requests",
66 "retained_queued_requests",
67 "estimated_retained_payload_bytes",
68 "applied_version",
69 "final_version_applied",
70 "enqueue_elapsed_ns",
71 "rss_supported",
72 "rss_before_bytes",
73 "rss_during_bytes",
74 "rss_after_bytes",
75 "rss_during_delta_bytes",
76 "rss_after_delta_bytes",
77 "limitations",
78 )
79
80 CEILING_FIELDS = (
81 "retained_queued_requests",
82 "estimated_retained_payload_bytes",
83 "enqueue_elapsed_ns",
84 "rss_during_delta_bytes",
85 "rss_after_delta_bytes",
86 )
87 RSS_SAMPLE_FIELDS = ("rss_before_bytes", "rss_during_bytes", "rss_after_bytes")
88 RSS_DELTA_FIELDS = ("rss_during_delta_bytes", "rss_after_delta_bytes")
89 SUPPORTED_PLATFORMS = {"linux", "macos", "windows"}
90 SOURCE_SHA_PATTERN = re.compile(r"[0-9a-f]{40}")
91
92
93 class PersistenceBacklogError(ValueError):
94 """A receipt or budget broke the measurement contract."""
95
96
97 def load_json(path: Path, label: str) -> dict[str, Any]:
98 try:
99 value = json.loads(path.read_text(encoding="utf-8"))
100 except (OSError, json.JSONDecodeError) as error:
101 raise PersistenceBacklogError(f"invalid {label} {path}: {error}") from error
102 if not isinstance(value, dict):
103 raise PersistenceBacklogError(f"{label} must be a JSON object")
104 return value
105
106
107 def non_negative_integer(value: Any, field: str) -> int:
108 if isinstance(value, bool) or not isinstance(value, int) or value < 0:
109 raise PersistenceBacklogError(f"{field} must be a non-negative integer")
110 return value
111
112
113 def validate_frozen_field(field: str, value: Any, expected: Any) -> None:
114 if type(value) is not type(expected) or value != expected:
115 raise PersistenceBacklogError(
116 f"receipt {field} must remain {expected!r}, got {value!r}"
117 )
118
119
120 def current_source_identity() -> dict[str, Any]:
121 def run(command: list[str]) -> str:
122 result = subprocess.run(
123 command,
124 cwd=ROOT,
125 text=True,
126 capture_output=True,
127 check=False,
128 )
129 if result.returncode != 0:
130 raise PersistenceBacklogError(
131 f"source provenance command failed: {' '.join(command)}"
132 )
133 return result.stdout.strip()
134
135 return {
136 "source_sha": run(["git", "rev-parse", "HEAD"]),
137 "source_dirty": bool(
138 run(["git", "status", "--porcelain", "--untracked-files=normal"])
139 ),
140 "rustc_version": run(["rustc", "--version"]),
141 "cargo_version": run(["cargo", "--version"]),
142 "build_profile": "test",
143 "sample_count": 1,
144 }
145
146
147 def validate_receipt(
148 receipt: dict[str, Any],
149 *,
150 expected_source: dict[str, Any] | None = None,
151 require_clean_source: bool = False,
152 ) -> None:
153 missing = [field for field in REQUIRED_RECEIPT_FIELDS if field not in receipt]
154 if missing:
155 raise PersistenceBacklogError(
156 "receipt missing required field(s): " + ", ".join(missing)
157 )
158 if receipt["document_kind"] != RECEIPT_KIND:
159 raise PersistenceBacklogError(f"receipt document_kind must be {RECEIPT_KIND}")
160 if receipt["schema_version"] != SCHEMA_VERSION:
161 raise PersistenceBacklogError("receipt schema_version changed")
162 for field, expected in FIXTURE.items():
163 validate_frozen_field(field, receipt[field], expected)
164 if not isinstance(receipt["source_sha"], str) or not SOURCE_SHA_PATTERN.fullmatch(
165 receipt["source_sha"]
166 ):
167 raise PersistenceBacklogError("receipt source_sha must be an exact lowercase Git SHA")
168 if type(receipt["source_dirty"]) is not bool:
169 raise PersistenceBacklogError("receipt source_dirty must be boolean")
170 for field, prefix in (("rustc_version", "rustc "), ("cargo_version", "cargo ")):
171 if not isinstance(receipt[field], str) or not receipt[field].startswith(prefix):
172 raise PersistenceBacklogError(f"receipt {field} must be a version string")
173 validate_frozen_field("build_profile", receipt["build_profile"], "test")
174 validate_frozen_field("sample_count", receipt["sample_count"], 1)
175 if expected_source is not None:
176 for field in (
177 "source_sha",
178 "source_dirty",
179 "rustc_version",
180 "cargo_version",
181 "build_profile",
182 "sample_count",
183 ):
184 if receipt[field] != expected_source[field]:
185 raise PersistenceBacklogError(
186 f"receipt {field} does not match the checked source"
187 )
188 if require_clean_source and receipt["source_dirty"]:
189 raise PersistenceBacklogError("persistence measurement source tree is dirty")
190 platform = receipt["platform"]
191 if not isinstance(platform, str) or platform not in SUPPORTED_PLATFORMS:
192 raise PersistenceBacklogError("receipt platform is unsupported")
193
194 attempted = non_negative_integer(receipt["requests_attempted"], "requests_attempted")
195 accepted = non_negative_integer(receipt["accepted_requests"], "accepted_requests")
196 if accepted != attempted:
197 raise PersistenceBacklogError(
198 "accepted_requests must equal requests_attempted; sender rejection is not backlog improvement"
199 )
200 retained = non_negative_integer(
201 receipt["retained_queued_requests"], "retained_queued_requests"
202 )
203 if retained > accepted:
204 raise PersistenceBacklogError("retained_queued_requests exceeds accepted_requests")
205 for field in ("estimated_retained_payload_bytes", "enqueue_elapsed_ns"):
206 non_negative_integer(receipt[field], field)
207 if retained == 0 or receipt["estimated_retained_payload_bytes"] == 0:
208 raise PersistenceBacklogError(
209 "the paused channel must retain the newest request and its payload"
210 )
211 minimum_payload_bytes = retained * FIXTURE["content_bytes_per_request"]
212 if receipt["estimated_retained_payload_bytes"] < minimum_payload_bytes:
213 raise PersistenceBacklogError(
214 "estimated_retained_payload_bytes is smaller than the frozen retained content"
215 )
216 applied = non_negative_integer(
217 receipt["applied_version"], "applied_version"
218 )
219 if applied != FIXTURE["expected_applied_version"]:
220 raise PersistenceBacklogError("applied_version is not the final sent version")
221 if receipt["final_version_applied"] is not True:
222 raise PersistenceBacklogError("final_version_applied must be true")
223
224 limitations = receipt["limitations"]
225 if not isinstance(limitations, list) or not limitations or not all(
226 isinstance(item, str) and item for item in limitations
227 ):
228 raise PersistenceBacklogError("limitations must be a non-empty string array")
229
230 if not isinstance(receipt["rss_supported"], bool):
231 raise PersistenceBacklogError("rss_supported must be boolean")
232 if receipt["rss_supported"] != (platform == "macos"):
233 raise PersistenceBacklogError(
234 "rss_supported must be true exactly on the macOS measurement lane"
235 )
236 rss_fields = RSS_SAMPLE_FIELDS + RSS_DELTA_FIELDS
237 if receipt["rss_supported"]:
238 for field in rss_fields:
239 non_negative_integer(receipt[field], field)
240 before = receipt["rss_before_bytes"]
241 if receipt["rss_during_delta_bytes"] != max(
242 0, receipt["rss_during_bytes"] - before
243 ):
244 raise PersistenceBacklogError("rss_during_delta_bytes is inconsistent")
245 if receipt["rss_after_delta_bytes"] != max(
246 0, receipt["rss_after_bytes"] - before
247 ):
248 raise PersistenceBacklogError("rss_after_delta_bytes is inconsistent")
249 elif any(receipt[field] is not None for field in rss_fields):
250 raise PersistenceBacklogError("unsupported RSS fields must be null")
251
252
253 def validate_budget(budget: dict[str, Any]) -> None:
254 if budget.get("document_kind") != BUDGET_KIND:
255 raise PersistenceBacklogError(f"budget document_kind must be {BUDGET_KIND}")
256 if budget.get("schema_version") != SCHEMA_VERSION:
257 raise PersistenceBacklogError("budget schema_version changed")
258 fixture = budget.get("fixture")
259 if not isinstance(fixture, dict) or set(fixture) != set(FIXTURE):
260 raise PersistenceBacklogError("budget fixture no longer matches the frozen workload")
261 for field, expected in FIXTURE.items():
262 if type(fixture[field]) is not type(expected) or fixture[field] != expected:
263 raise PersistenceBacklogError(
264 f"budget fixture.{field} must remain {expected!r}"
265 )
266 if budget.get("baseline_receipt") != BASELINE_RECEIPT_REFERENCE:
267 raise PersistenceBacklogError("budget baseline_receipt path changed")
268 ceilings = budget.get("ceilings")
269 baseline = budget.get("baseline_observation")
270 if not isinstance(ceilings, dict) or not isinstance(baseline, dict):
271 raise PersistenceBacklogError("budget needs ceilings and baseline_observation objects")
272 for field in CEILING_FIELDS:
273 ceiling = non_negative_integer(ceilings.get(field), f"ceilings.{field}")
274 observed = non_negative_integer(
275 baseline.get(field), f"baseline_observation.{field}"
276 )
277 if observed > ceiling:
278 raise PersistenceBacklogError(
279 f"baseline_observation.{field} exceeds its ceiling"
280 )
281 baseline_accepted = non_negative_integer(
282 baseline.get("accepted_requests"), "baseline_observation.accepted_requests"
283 )
284 if baseline_accepted != FIXTURE["requests_attempted"]:
285 raise PersistenceBacklogError(
286 "baseline_observation.accepted_requests must equal requests_attempted"
287 )
288 baseline_applied = non_negative_integer(
289 baseline.get("applied_version"), "baseline_observation.applied_version"
290 )
291 if baseline_applied != FIXTURE["expected_applied_version"]:
292 raise PersistenceBacklogError(
293 "baseline_observation.applied_version must be the final sent version"
294 )
295 baseline_retained = baseline["retained_queued_requests"]
296 baseline_payload = baseline["estimated_retained_payload_bytes"]
297 if baseline_retained == 0 or baseline_payload == 0:
298 raise PersistenceBacklogError(
299 "baseline_observation must retain the final request and payload"
300 )
301 if baseline_retained > baseline_accepted:
302 raise PersistenceBacklogError(
303 "baseline_observation.retained_queued_requests exceeds accepted_requests"
304 )
305 if baseline_payload < baseline_retained * FIXTURE["content_bytes_per_request"]:
306 raise PersistenceBacklogError(
307 "baseline_observation payload is smaller than frozen retained content"
308 )
309 provenance = baseline.get("provenance")
310 if not isinstance(provenance, dict):
311 raise PersistenceBacklogError("baseline_observation needs provenance")
312 if provenance.get("platform") != "macos":
313 raise PersistenceBacklogError("baseline provenance platform must be macos")
314 if not isinstance(provenance.get("source_sha"), str) or not SOURCE_SHA_PATTERN.fullmatch(
315 provenance["source_sha"]
316 ):
317 raise PersistenceBacklogError("baseline provenance needs an exact source SHA")
318 if provenance.get("source_dirty") is not False:
319 raise PersistenceBacklogError("baseline provenance must identify a clean source tree")
320 for field, prefix in (("rustc_version", "rustc "), ("cargo_version", "cargo ")):
321 if not isinstance(provenance.get(field), str) or not provenance[field].startswith(prefix):
322 raise PersistenceBacklogError(f"baseline provenance needs {field}")
323 if provenance.get("build_profile") != "test" or not (
324 type(provenance.get("sample_count")) is int
325 and provenance["sample_count"] == 1
326 ):
327 raise PersistenceBacklogError("baseline provenance build profile/sample count changed")
328
329
330 def validate_baseline_receipt(
331 budget: dict[str, Any], baseline_receipt: dict[str, Any]
332 ) -> None:
333 validate_receipt(baseline_receipt, require_clean_source=True)
334 baseline = budget["baseline_observation"]
335 for field in ("accepted_requests", "applied_version", *CEILING_FIELDS):
336 if baseline_receipt[field] != baseline[field]:
337 raise PersistenceBacklogError(
338 f"baseline receipt {field} does not match baseline_observation"
339 )
340 provenance = baseline["provenance"]
341 for field in (
342 "platform",
343 "source_sha",
344 "source_dirty",
345 "rustc_version",
346 "cargo_version",
347 "build_profile",
348 "sample_count",
349 ):
350 if baseline_receipt[field] != provenance[field]:
351 raise PersistenceBacklogError(
352 f"baseline receipt {field} does not match baseline provenance"
353 )
354
355
356 def compare(
357 receipt: dict[str, Any],
358 budget: dict[str, Any],
359 *,
360 expected_source: dict[str, Any] | None = None,
361 require_clean_source: bool = False,
362 ) -> tuple[list[tuple[str, int, int]], list[tuple[str, int, int]]]:
363 validate_receipt(
364 receipt,
365 expected_source=expected_source,
366 require_clean_source=require_clean_source,
367 )
368 validate_budget(budget)
369 increases: list[tuple[str, int, int]] = []
370 decreases: list[tuple[str, int, int]] = []
371 for field in CEILING_FIELDS:
372 if field in RSS_DELTA_FIELDS and not receipt["rss_supported"]:
373 continue
374 current = receipt[field]
375 ceiling = budget["ceilings"][field]
376 if current > ceiling:
377 increases.append((field, current, ceiling))
378 elif current < ceiling:
379 decreases.append((field, current, ceiling))
380 return increases, decreases
381
382
383 def measure() -> dict[str, Any]:
384 env = os.environ.copy()
385 env["CARGO_NET_OFFLINE"] = "true"
386 result = subprocess.run(
387 [sys.executable, str(MEASURE_SCRIPT)],
388 cwd=ROOT,
389 env=env,
390 text=True,
391 capture_output=True,
392 check=False,
393 )
394 sys.stderr.write(result.stderr)
395 if result.returncode != 0:
396 sys.stdout.write(result.stdout)
397 raise PersistenceBacklogError("measurement command failed")
398 try:
399 receipt = json.loads(result.stdout)
400 except json.JSONDecodeError as error:
401 raise PersistenceBacklogError(f"measurement emitted invalid JSON: {error}") from error
402 if not isinstance(receipt, dict):
403 raise PersistenceBacklogError("measurement receipt must be an object")
404 return receipt
405
406
407 def update_command(receipt_path: Path | None, budget_path: Path) -> str:
408 parts = ["python3", "scripts/check-persistence-backlog-budget.py"]
409 if receipt_path is not None:
410 parts.extend(["--receipt", str(receipt_path)])
411 if budget_path != BUDGET_PATH:
412 parts.extend(["--budget", str(budget_path)])
413 parts.append("--update")
414 return " ".join(parts)
415
416
417 def raise_ceilings(
418 budget: dict[str, Any], increases: list[tuple[str, int, int]]
419 ) -> dict[str, Any]:
420 """Return a copy of ``budget`` with each exceeded ceiling set to its measurement."""
421 updated = json.loads(json.dumps(budget))
422 for field, current, _ceiling in increases:
423 updated["ceilings"][field] = current
424 validate_budget(updated)
425 return updated
426
427
428 def main() -> int:
429 parser = argparse.ArgumentParser(description=__doc__)
430 parser.add_argument("--receipt", type=Path, help="check an existing receipt")
431 parser.add_argument("--budget", type=Path, default=BUDGET_PATH)
432 parser.add_argument(
433 "--update",
434 action="store_true",
435 help="raise exceeded ceilings to the measured values (never lowers one)",
436 )
437 args = parser.parse_args()
438 try:
439 expected_source = current_source_identity()
440 receipt = load_json(args.receipt, "receipt") if args.receipt else measure()
441 budget = load_json(args.budget, "budget")
442 baseline_receipt = load_json(BASELINE_RECEIPT_PATH, "baseline receipt")
443 validate_baseline_receipt(budget, baseline_receipt)
444 increases, decreases = compare(
445 receipt,
446 budget,
447 expected_source=expected_source,
448 # An update runs while the author is mid-change; the measurement
449 # still names its exact SHA and dirty bit, so only the enforcing
450 # check insists on a clean tree.
451 require_clean_source=not args.update,
452 )
453 except PersistenceBacklogError as error:
454 print(f"[persistence-backlog-budget] ERROR: {error}", file=sys.stderr)
455 return 2
456 if args.update:
457 if not increases:
458 print(
459 "[persistence-backlog-budget] --update: no ceiling exceeded; "
460 f"{args.budget} left unchanged"
461 )
462 return 0
463 try:
464 updated = raise_ceilings(budget, increases)
465 args.budget.write_text(json.dumps(updated, indent=2) + "\n", encoding="utf-8")
466 except (OSError, PersistenceBacklogError) as error:
467 print(
468 f"[persistence-backlog-budget] ERROR: failed to update budget: {error}",
469 file=sys.stderr,
470 )
471 return 2
472 for field, current, ceiling in increases:
473 print(f"[persistence-backlog-budget] raised {field}: {ceiling} -> {current}")
474 print(
475 f"[persistence-backlog-budget] wrote {args.budget}; say why in the PR "
476 "description, or add a dated _rebaseline note to the budget."
477 )
478 return 0
479 if increases:
480 for field, current, ceiling in increases:
481 print(
482 f"[persistence-backlog-budget] FAIL: {field}={current} exceeds {ceiling}",
483 file=sys.stderr,
484 )
485 print(
486 "\nShrink the retained backlog, or if the growth is intended land the new "
487 f"ceiling in this PR:\n {update_command(args.receipt, args.budget)}",
488 file=sys.stderr,
489 )
490 return 1
491 print("[persistence-backlog-budget] PASS: one-way ceilings respected")
492 for field, current, ceiling in decreases:
493 print(f" can tighten {field}: {current} < {ceiling}")
494 return 0
495
496
497 if __name__ == "__main__":
498 raise SystemExit(main())
499
499 lines PYTHON