| 1 | import urllib.error |
| 2 | import unittest |
| 3 | from unittest.mock import patch, MagicMock |
| 4 | |
| 5 | from lib import http |
| 6 | |
| 7 | |
| 8 | class Test429RetryLimit(unittest.TestCase): |
| 9 | """429 retries must be capped at max_429_retries to avoid wasting latency.""" |
| 10 | |
| 11 | @patch("lib.http.urllib.request.urlopen") |
| 12 | @patch("lib.http.time.sleep") # Don't actually sleep in tests |
| 13 | def test_429_retries_limited_to_2_by_default(self, mock_sleep, mock_urlopen): |
| 14 | """With default max_429_retries=2, should attempt 2 times then raise.""" |
| 15 | error = urllib.error.HTTPError( |
| 16 | "http://example.com", 429, "Too Many Requests", {}, None |
| 17 | ) |
| 18 | mock_urlopen.side_effect = error |
| 19 | |
| 20 | with self.assertRaises(http.HTTPError) as ctx: |
| 21 | http.request("GET", "http://example.com", retries=5) |
| 22 | |
| 23 | self.assertEqual(ctx.exception.status_code, 429) |
| 24 | # Should be called exactly 2 times (initial + 1 retry), not 5 |
| 25 | self.assertEqual(mock_urlopen.call_count, 2) |
| 26 | |
| 27 | @patch("lib.http.urllib.request.urlopen") |
| 28 | @patch("lib.http.time.sleep") |
| 29 | def test_non_429_errors_still_use_full_retries(self, mock_sleep, mock_urlopen): |
| 30 | """500 errors should still retry up to the full retries count.""" |
| 31 | error = urllib.error.HTTPError( |
| 32 | "http://example.com", 500, "Internal Server Error", {}, None |
| 33 | ) |
| 34 | mock_urlopen.side_effect = error |
| 35 | |
| 36 | with self.assertRaises(http.HTTPError): |
| 37 | http.request("GET", "http://example.com", retries=3) |
| 38 | |
| 39 | self.assertEqual(mock_urlopen.call_count, 3) |
| 40 | |
| 41 | |
| 42 | def _mock_response(body: str = '{"ok": true}', status: int = 200): |
| 43 | resp = MagicMock() |
| 44 | resp.__enter__ = MagicMock(return_value=resp) |
| 45 | resp.__exit__ = MagicMock(return_value=False) |
| 46 | resp.read.return_value = body.encode("utf-8") |
| 47 | resp.status = status |
| 48 | return resp |
| 49 | |
| 50 | |
| 51 | class TestParamsEncoding(unittest.TestCase): |
| 52 | """request() should urlencode the params dict into the URL.""" |
| 53 | |
| 54 | def _sent_url(self, mock_urlopen) -> str: |
| 55 | request_arg = mock_urlopen.call_args[0][0] |
| 56 | return request_arg.full_url |
| 57 | |
| 58 | @patch("lib.http.urllib.request.urlopen") |
| 59 | def test_params_appended_to_url(self, mock_urlopen): |
| 60 | mock_urlopen.return_value = _mock_response() |
| 61 | http.get("https://api.example.com/search", params={"q": "test", "limit": 10}) |
| 62 | sent_url = self._sent_url(mock_urlopen) |
| 63 | self.assertIn("q=test", sent_url) |
| 64 | self.assertIn("limit=10", sent_url) |
| 65 | |
| 66 | @patch("lib.http.urllib.request.urlopen") |
| 67 | def test_params_appended_with_existing_query_string(self, mock_urlopen): |
| 68 | mock_urlopen.return_value = _mock_response() |
| 69 | http.get("https://api.example.com/search?api_key=secret", params={"q": "test"}) |
| 70 | sent_url = self._sent_url(mock_urlopen) |
| 71 | self.assertTrue(sent_url.startswith("https://api.example.com/search?api_key=secret&")) |
| 72 | self.assertIn("q=test", sent_url) |
| 73 | |
| 74 | @patch("lib.http.urllib.request.urlopen") |
| 75 | def test_none_values_dropped(self, mock_urlopen): |
| 76 | mock_urlopen.return_value = _mock_response() |
| 77 | http.get("https://api.example.com/search", params={"q": "test", "filter": None}) |
| 78 | sent_url = self._sent_url(mock_urlopen) |
| 79 | self.assertIn("q=test", sent_url) |
| 80 | self.assertNotIn("filter", sent_url) |
| 81 | |
| 82 | @patch("lib.http.urllib.request.urlopen") |
| 83 | def test_empty_params_leaves_url_unchanged(self, mock_urlopen): |
| 84 | mock_urlopen.return_value = _mock_response() |
| 85 | http.get("https://api.example.com/search", params={}) |
| 86 | sent_url = self._sent_url(mock_urlopen) |
| 87 | self.assertEqual(sent_url, "https://api.example.com/search") |
| 88 | |
| 89 | @patch("lib.http.urllib.request.urlopen") |
| 90 | def test_no_params_kwarg_leaves_url_unchanged(self, mock_urlopen): |
| 91 | mock_urlopen.return_value = _mock_response() |
| 92 | http.get("https://api.example.com/search") |
| 93 | sent_url = self._sent_url(mock_urlopen) |
| 94 | self.assertEqual(sent_url, "https://api.example.com/search") |
| 95 | |
| 96 | @patch("lib.http.urllib.request.urlopen") |
| 97 | def test_int_and_bool_params_stringified(self, mock_urlopen): |
| 98 | mock_urlopen.return_value = _mock_response() |
| 99 | http.get("https://api.example.com/search", params={"count": 25, "raw": True}) |
| 100 | sent_url = self._sent_url(mock_urlopen) |
| 101 | self.assertIn("count=25", sent_url) |
| 102 | self.assertIn("raw=True", sent_url) |
| 103 | |
| 104 | |
| 105 | class TestDNSResolutionRetry(unittest.TestCase): |
| 106 | """DNS resolution failures (gaierror) must retry with exponential backoff. |
| 107 | |
| 108 | Caller-passed `retries` values smaller than MIN_DNS_RETRIES are expanded |
| 109 | on the first gaierror so a transient resolution failure doesn't wipe a |
| 110 | request just because the caller passed retries=2. |
| 111 | """ |
| 112 | |
| 113 | @patch("lib.http.urllib.request.urlopen") |
| 114 | @patch("lib.http.time.sleep") |
| 115 | def test_gaierror_retries_up_to_min_dns_retries_even_when_caller_passes_fewer( |
| 116 | self, mock_sleep, mock_urlopen |
| 117 | ): |
| 118 | """Caller passed retries=2; gaierror should still get MIN_DNS_RETRIES attempts.""" |
| 119 | import socket |
| 120 | err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known")) |
| 121 | mock_urlopen.side_effect = err |
| 122 | |
| 123 | with self.assertRaises(http.HTTPError): |
| 124 | http.request("GET", "http://nonexistent.example", retries=2) |
| 125 | |
| 126 | # Caller passed retries=2, but the budget expanded to MIN_DNS_RETRIES=3. |
| 127 | self.assertEqual(mock_urlopen.call_count, http.MIN_DNS_RETRIES) |
| 128 | |
| 129 | @patch("lib.http.urllib.request.urlopen") |
| 130 | @patch("lib.http.time.sleep") |
| 131 | def test_gaierror_succeeds_after_transient_failure(self, mock_sleep, mock_urlopen): |
| 132 | """gaierror on attempt 1, then success — should NOT raise.""" |
| 133 | import socket |
| 134 | success_response = MagicMock() |
| 135 | success_response.read.return_value = b'{"ok": true}' |
| 136 | success_response.status = 200 |
| 137 | success_response.__enter__ = lambda self: self |
| 138 | success_response.__exit__ = lambda *args: None |
| 139 | |
| 140 | err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known")) |
| 141 | mock_urlopen.side_effect = [err, success_response] |
| 142 | |
| 143 | result = http.request("GET", "http://flaky.example", retries=2) |
| 144 | |
| 145 | self.assertEqual(result, {"ok": True}) |
| 146 | self.assertEqual(mock_urlopen.call_count, 2) |
| 147 | |
| 148 | @patch("lib.http.urllib.request.urlopen") |
| 149 | @patch("lib.http.time.sleep") |
| 150 | def test_gaierror_uses_exponential_backoff(self, mock_sleep, mock_urlopen): |
| 151 | """Backoff delays for gaierror should be 1s, 2s, 4s — not the linear default.""" |
| 152 | import socket |
| 153 | err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known")) |
| 154 | mock_urlopen.side_effect = err |
| 155 | |
| 156 | with self.assertRaises(http.HTTPError): |
| 157 | http.request("GET", "http://nonexistent.example", retries=3) |
| 158 | |
| 159 | # Expected sleep calls: 1s (after attempt 1), 2s (after attempt 2). |
| 160 | # No sleep after the final attempt (the loop exits to raise). |
| 161 | sleep_delays = [call.args[0] for call in mock_sleep.call_args_list] |
| 162 | self.assertEqual(sleep_delays, [1, 2]) |
| 163 | |
| 164 | @patch("lib.http.urllib.request.urlopen") |
| 165 | @patch("lib.http.time.sleep") |
| 166 | def test_non_dns_urlerror_uses_linear_backoff_not_dns_branch( |
| 167 | self, mock_sleep, mock_urlopen |
| 168 | ): |
| 169 | """A URLError that's NOT a gaierror must NOT expand the retry budget.""" |
| 170 | # ConnectionRefusedError-style URLError reason (not gaierror) |
| 171 | err = urllib.error.URLError(ConnectionRefusedError(111, "Connection refused")) |
| 172 | mock_urlopen.side_effect = err |
| 173 | |
| 174 | with self.assertRaises(http.HTTPError): |
| 175 | http.request("GET", "http://refused.example", retries=2) |
| 176 | |
| 177 | # Caller passed retries=2, and non-DNS URLError doesn't expand it. |
| 178 | self.assertEqual(mock_urlopen.call_count, 2) |
| 179 | |
| 180 | @patch("lib.http.urllib.request.urlopen") |
| 181 | @patch("lib.http.time.sleep") |
| 182 | def test_dns_widening_does_not_leak_into_subsequent_non_dns_urlerror( |
| 183 | self, mock_sleep, mock_urlopen |
| 184 | ): |
| 185 | """Mixed sequence: DNS-then-non-DNS must respect caller's original retries. |
| 186 | |
| 187 | Without the fix, the first gaierror widens effective_retries from 2 to |
| 188 | MIN_DNS_RETRIES=3, and a subsequent ConnectionRefused on attempt 1 |
| 189 | slips into a third overall attempt — exceeding what the caller asked |
| 190 | for. Each non-DNS error path must gate on the original `retries`. |
| 191 | """ |
| 192 | import socket |
| 193 | dns_err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known")) |
| 194 | conn_err = urllib.error.URLError(ConnectionRefusedError(111, "Connection refused")) |
| 195 | mock_urlopen.side_effect = [dns_err, conn_err, conn_err] # 3rd would only fire if budget leaked |
| 196 | |
| 197 | with self.assertRaises(http.HTTPError): |
| 198 | http.request("GET", "http://flaky.example", retries=2) |
| 199 | |
| 200 | # Caller asked for at most 2 attempts. DNS widening must not give us a 3rd. |
| 201 | self.assertEqual(mock_urlopen.call_count, 2) |
| 202 | |
| 203 | @patch("lib.http.urllib.request.urlopen") |
| 204 | @patch("lib.http.time.sleep") |
| 205 | def test_dns_widening_does_not_leak_into_subsequent_oserror( |
| 206 | self, mock_sleep, mock_urlopen |
| 207 | ): |
| 208 | """Mixed sequence: DNS-then-OSError must respect caller's original retries.""" |
| 209 | import socket |
| 210 | dns_err = urllib.error.URLError(socket.gaierror(-2, "Name or service not known")) |
| 211 | mock_urlopen.side_effect = [dns_err, TimeoutError("timed out"), TimeoutError("timed out")] |
| 212 | |
| 213 | with self.assertRaises(http.HTTPError): |
| 214 | http.request("GET", "http://flaky.example", retries=2) |
| 215 | |
| 216 | self.assertEqual(mock_urlopen.call_count, 2) |
| 217 |