| 1 | #!/usr/bin/env python3 |
| 2 | """Hermetic contract tests for the persistence backlog ratchet.""" |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import copy |
| 7 | import importlib.util |
| 8 | import sys |
| 9 | import unittest |
| 10 | from pathlib import Path |
| 11 | |
| 12 | |
| 13 | ROOT = Path(__file__).resolve().parents[1] |
| 14 | SCRIPT = ROOT / "scripts" / "check-persistence-backlog-budget.py" |
| 15 | SPEC = importlib.util.spec_from_file_location("check_persistence_backlog_budget", SCRIPT) |
| 16 | assert SPEC and SPEC.loader |
| 17 | mod = importlib.util.module_from_spec(SPEC) |
| 18 | sys.modules[SPEC.name] = mod |
| 19 | SPEC.loader.exec_module(mod) |
| 20 | |
| 21 | |
| 22 | def receipt_fixture(*, rss_supported: bool = True) -> dict: |
| 23 | receipt = { |
| 24 | "document_kind": mod.RECEIPT_KIND, |
| 25 | "schema_version": mod.SCHEMA_VERSION, |
| 26 | **mod.FIXTURE, |
| 27 | "source_sha": "0123456789abcdef0123456789abcdef01234567", |
| 28 | "source_dirty": False, |
| 29 | "rustc_version": "rustc test", |
| 30 | "cargo_version": "cargo test", |
| 31 | "build_profile": "test", |
| 32 | "sample_count": 1, |
| 33 | "platform": "macos" if rss_supported else "linux", |
| 34 | "accepted_requests": 128, |
| 35 | "retained_queued_requests": 128, |
| 36 | "estimated_retained_payload_bytes": 8_500_000, |
| 37 | "applied_version": 127, |
| 38 | "final_version_applied": True, |
| 39 | "enqueue_elapsed_ns": 500_000, |
| 40 | "rss_supported": rss_supported, |
| 41 | "rss_before_bytes": 100_000_000 if rss_supported else None, |
| 42 | "rss_during_bytes": 112_000_000 if rss_supported else None, |
| 43 | "rss_after_bytes": 103_000_000 if rss_supported else None, |
| 44 | "rss_during_delta_bytes": 12_000_000 if rss_supported else None, |
| 45 | "rss_after_delta_bytes": 3_000_000 if rss_supported else None, |
| 46 | "limitations": ["macOS RSS only"], |
| 47 | } |
| 48 | return receipt |
| 49 | |
| 50 | |
| 51 | def budget_fixture(receipt: dict | None = None) -> dict: |
| 52 | receipt = receipt or receipt_fixture() |
| 53 | metrics = { |
| 54 | field: receipt[field] if receipt[field] is not None else 0 |
| 55 | for field in mod.CEILING_FIELDS |
| 56 | } |
| 57 | return { |
| 58 | "document_kind": mod.BUDGET_KIND, |
| 59 | "schema_version": mod.SCHEMA_VERSION, |
| 60 | "baseline_receipt": mod.BASELINE_RECEIPT_REFERENCE, |
| 61 | "fixture": copy.deepcopy(mod.FIXTURE), |
| 62 | "baseline_observation": { |
| 63 | "accepted_requests": receipt["accepted_requests"], |
| 64 | "applied_version": receipt["applied_version"], |
| 65 | "provenance": { |
| 66 | "platform": "macos", |
| 67 | "source_sha": receipt["source_sha"], |
| 68 | "source_dirty": False, |
| 69 | "rustc_version": receipt["rustc_version"], |
| 70 | "cargo_version": receipt["cargo_version"], |
| 71 | "build_profile": "test", |
| 72 | "sample_count": 1, |
| 73 | }, |
| 74 | **copy.deepcopy(metrics), |
| 75 | }, |
| 76 | "ceilings": copy.deepcopy(metrics), |
| 77 | } |
| 78 | |
| 79 | |
| 80 | class PersistenceBacklogBudgetTests(unittest.TestCase): |
| 81 | def test_equal_baseline_passes(self) -> None: |
| 82 | self.assertEqual( |
| 83 | mod.compare(receipt_fixture(), budget_fixture()), |
| 84 | ([], []), |
| 85 | ) |
| 86 | |
| 87 | def test_every_receipt_field_is_required(self) -> None: |
| 88 | budget = budget_fixture() |
| 89 | for field in mod.REQUIRED_RECEIPT_FIELDS: |
| 90 | with self.subTest(field=field): |
| 91 | receipt = receipt_fixture() |
| 92 | del receipt[field] |
| 93 | with self.assertRaisesRegex( |
| 94 | mod.PersistenceBacklogError, "missing required field" |
| 95 | ): |
| 96 | mod.compare(receipt, budget) |
| 97 | |
| 98 | def test_frozen_workload_cannot_be_weakened_to_fake_an_improvement(self) -> None: |
| 99 | budget = budget_fixture() |
| 100 | for field, replacement in [ |
| 101 | ("paused_consumer", False), |
| 102 | ("requests_attempted", 64), |
| 103 | ("content_bytes_per_request", 32 * 1024), |
| 104 | ("single_session_id", False), |
| 105 | ("expected_applied_version", 63), |
| 106 | ("request_variant", "clear_checkpoint"), |
| 107 | ("payload_estimator", "shallow-size"), |
| 108 | ]: |
| 109 | with self.subTest(field=field): |
| 110 | receipt = receipt_fixture() |
| 111 | receipt[field] = replacement |
| 112 | with self.assertRaisesRegex(mod.PersistenceBacklogError, field): |
| 113 | mod.compare(receipt, budget) |
| 114 | |
| 115 | def test_boolean_fixture_fields_reject_integer_aliases(self) -> None: |
| 116 | budget = budget_fixture() |
| 117 | for field in ("paused_consumer", "single_session_id"): |
| 118 | with self.subTest(field=field): |
| 119 | receipt = receipt_fixture() |
| 120 | receipt[field] = 1 |
| 121 | with self.assertRaisesRegex(mod.PersistenceBacklogError, field): |
| 122 | mod.compare(receipt, budget) |
| 123 | |
| 124 | def test_budget_boolean_and_sample_count_aliases_are_rejected(self) -> None: |
| 125 | receipt = receipt_fixture() |
| 126 | for field in ("paused_consumer", "single_session_id"): |
| 127 | with self.subTest(field=field): |
| 128 | budget = budget_fixture() |
| 129 | budget["fixture"][field] = 1 |
| 130 | with self.assertRaisesRegex(mod.PersistenceBacklogError, field): |
| 131 | mod.compare(receipt, budget) |
| 132 | |
| 133 | budget = budget_fixture() |
| 134 | budget["baseline_observation"]["provenance"]["sample_count"] = True |
| 135 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "sample count"): |
| 136 | mod.compare(receipt, budget) |
| 137 | |
| 138 | def test_every_ceiling_rejects_growth_and_accepts_tightening(self) -> None: |
| 139 | baseline = receipt_fixture() |
| 140 | baseline["retained_queued_requests"] = 64 |
| 141 | # Leave enough valid payload headroom for the retained-count subtest |
| 142 | # to change that one metric without making the receipt impossible. |
| 143 | baseline["estimated_retained_payload_bytes"] = 8_500_000 |
| 144 | budget = budget_fixture(baseline) |
| 145 | for field in mod.CEILING_FIELDS: |
| 146 | with self.subTest(field=field): |
| 147 | grown = copy.deepcopy(baseline) |
| 148 | grown[field] += 1 |
| 149 | if field == "rss_during_delta_bytes": |
| 150 | grown["rss_during_bytes"] += 1 |
| 151 | elif field == "rss_after_delta_bytes": |
| 152 | grown["rss_after_bytes"] += 1 |
| 153 | increases, _ = mod.compare(grown, budget) |
| 154 | self.assertEqual([item[0] for item in increases], [field]) |
| 155 | |
| 156 | reduced = copy.deepcopy(baseline) |
| 157 | reduced[field] -= 1 |
| 158 | if field == "rss_during_delta_bytes": |
| 159 | reduced["rss_during_bytes"] -= 1 |
| 160 | elif field == "rss_after_delta_bytes": |
| 161 | reduced["rss_after_bytes"] -= 1 |
| 162 | increases, decreases = mod.compare(reduced, budget) |
| 163 | self.assertEqual(increases, []) |
| 164 | self.assertIn(field, [item[0] for item in decreases]) |
| 165 | |
| 166 | def test_non_macos_receipt_keeps_rss_shape_but_skips_rss_ceilings(self) -> None: |
| 167 | receipt = receipt_fixture(rss_supported=False) |
| 168 | budget = budget_fixture() |
| 169 | increases, decreases = mod.compare(receipt, budget) |
| 170 | self.assertEqual(increases, []) |
| 171 | self.assertNotIn( |
| 172 | "rss_during_delta_bytes", [item[0] for item in decreases] |
| 173 | ) |
| 174 | self.assertNotIn("rss_after_delta_bytes", [item[0] for item in decreases]) |
| 175 | |
| 176 | def test_rss_delta_must_match_samples(self) -> None: |
| 177 | receipt = receipt_fixture() |
| 178 | receipt["rss_during_delta_bytes"] += 1 |
| 179 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "inconsistent"): |
| 180 | mod.compare(receipt, budget_fixture()) |
| 181 | |
| 182 | def test_sender_rejection_cannot_masquerade_as_backlog_improvement(self) -> None: |
| 183 | receipt = receipt_fixture() |
| 184 | receipt["accepted_requests"] = 1 |
| 185 | receipt["retained_queued_requests"] = 1 |
| 186 | receipt["estimated_retained_payload_bytes"] = 66_000 |
| 187 | with self.assertRaisesRegex( |
| 188 | mod.PersistenceBacklogError, "sender rejection is not backlog improvement" |
| 189 | ): |
| 190 | mod.compare(receipt, budget_fixture()) |
| 191 | |
| 192 | def test_stale_applied_version_cannot_pass_as_coalescing(self) -> None: |
| 193 | receipt = receipt_fixture() |
| 194 | receipt["applied_version"] = 126 |
| 195 | receipt["final_version_applied"] = False |
| 196 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "final sent version"): |
| 197 | mod.compare(receipt, budget_fixture()) |
| 198 | |
| 199 | def test_impossible_one_byte_retained_payload_is_rejected(self) -> None: |
| 200 | receipt = receipt_fixture() |
| 201 | receipt["retained_queued_requests"] = 1 |
| 202 | receipt["estimated_retained_payload_bytes"] = 1 |
| 203 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "frozen retained content"): |
| 204 | mod.compare(receipt, budget_fixture()) |
| 205 | |
| 206 | def test_rss_support_is_required_exactly_on_macos(self) -> None: |
| 207 | macos_without_rss = receipt_fixture(rss_supported=False) |
| 208 | macos_without_rss["platform"] = "macos" |
| 209 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "exactly on the macOS"): |
| 210 | mod.compare(macos_without_rss, budget_fixture()) |
| 211 | |
| 212 | linux_with_rss = receipt_fixture() |
| 213 | linux_with_rss["platform"] = "linux" |
| 214 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "exactly on the macOS"): |
| 215 | mod.compare(linux_with_rss, budget_fixture()) |
| 216 | |
| 217 | unknown = receipt_fixture(rss_supported=False) |
| 218 | unknown["platform"] = "unknown" |
| 219 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "unsupported"): |
| 220 | mod.compare(unknown, budget_fixture()) |
| 221 | |
| 222 | malformed = receipt_fixture(rss_supported=False) |
| 223 | malformed["platform"] = [] |
| 224 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "unsupported"): |
| 225 | mod.compare(malformed, budget_fixture()) |
| 226 | |
| 227 | def test_cli_source_identity_rejects_historical_or_dirty_receipts(self) -> None: |
| 228 | receipt = receipt_fixture() |
| 229 | expected = { |
| 230 | field: receipt[field] |
| 231 | for field in ( |
| 232 | "source_sha", |
| 233 | "source_dirty", |
| 234 | "rustc_version", |
| 235 | "cargo_version", |
| 236 | "build_profile", |
| 237 | "sample_count", |
| 238 | ) |
| 239 | } |
| 240 | historical = copy.deepcopy(receipt) |
| 241 | historical["source_sha"] = "f" * 40 |
| 242 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "checked source"): |
| 243 | mod.compare(historical, budget_fixture(), expected_source=expected) |
| 244 | |
| 245 | dirty = copy.deepcopy(receipt) |
| 246 | dirty["source_dirty"] = True |
| 247 | expected_dirty = copy.deepcopy(expected) |
| 248 | expected_dirty["source_dirty"] = True |
| 249 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "source tree is dirty"): |
| 250 | mod.compare( |
| 251 | dirty, |
| 252 | budget_fixture(), |
| 253 | expected_source=expected_dirty, |
| 254 | require_clean_source=True, |
| 255 | ) |
| 256 | |
| 257 | def test_budget_cannot_hide_a_baseline_above_its_ceiling(self) -> None: |
| 258 | budget = budget_fixture() |
| 259 | budget["baseline_observation"]["retained_queued_requests"] += 1 |
| 260 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "exceeds its ceiling"): |
| 261 | mod.compare(receipt_fixture(), budget) |
| 262 | |
| 263 | def test_budget_cannot_claim_an_empty_retained_baseline(self) -> None: |
| 264 | budget = budget_fixture() |
| 265 | budget["baseline_observation"]["retained_queued_requests"] = 0 |
| 266 | budget["baseline_observation"]["estimated_retained_payload_bytes"] = 0 |
| 267 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "retain the final"): |
| 268 | mod.compare(receipt_fixture(), budget) |
| 269 | |
| 270 | def test_raw_baseline_receipt_must_match_budget_metrics_and_provenance(self) -> None: |
| 271 | receipt = receipt_fixture() |
| 272 | budget = budget_fixture(receipt) |
| 273 | mod.validate_baseline_receipt(budget, receipt) |
| 274 | |
| 275 | stale_metric = copy.deepcopy(receipt) |
| 276 | stale_metric["enqueue_elapsed_ns"] += 1 |
| 277 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "does not match"): |
| 278 | mod.validate_baseline_receipt(budget, stale_metric) |
| 279 | |
| 280 | stale_source = copy.deepcopy(receipt) |
| 281 | stale_source["source_sha"] = "f" * 40 |
| 282 | with self.assertRaisesRegex(mod.PersistenceBacklogError, "does not match"): |
| 283 | mod.validate_baseline_receipt(budget, stale_source) |
| 284 | |
| 285 | |
| 286 | if __name__ == "__main__": |
| 287 | unittest.main() |
| 288 |