retry with backoff
1 changed file+108−54
benchcompress/src/benchcompress/run_benchmarks/_memobin.pymodified+108−54View file
@@ -1,10 +1,44 @@
11 import json
22 import requests
3-from typing import Optional
3+import time
4+from typing import Optional, TypeVar, Callable
5+
6+T = TypeVar("T")
7+
8+
9+def _retry_with_backoff(
10+ func: Callable[..., T], num_retries: int = 4, *args, **kwargs
11+) -> T:
12+ """Execute a function with exponential backoff retry logic.
13+
14+ Args:
15+ func: Function to execute
16+ num_retries: Maximum number of retries
17+ args: Positional arguments for the function
18+ kwargs: Keyword arguments for the function
19+
20+ Returns:
21+ The function's return value
22+
23+ Raises:
24+ The last exception encountered after all retries are exhausted
25+ """
26+ last_exception = None
27+ for attempt in range(num_retries):
28+ try:
29+ return func(*args, **kwargs)
30+ except Exception as e:
31+ last_exception = e
32+ if attempt < num_retries - 1:
33+ sleep_time = 2**attempt # 1, 2, 4, 8 seconds
34+ time.sleep(sleep_time)
35+ else:
36+ raise last_exception
37+ raise RuntimeError("Unexpected: retry loop completed without return or raise")
438
539
640 def create_signed_upload_url(
7- url: str, size: int, user_id: str, memobin_api_key: str
41+ url: str, size: int, user_id: str, memobin_api_key: str, num_retries: int = 4
842 ) -> str:
943 """Create a signed upload URL for memobin.
1044
@@ -21,38 +55,42 @@ def create_signed_upload_url(
2155 ValueError: If the URL prefix is invalid
2256 requests.RequestException: If the API request fails
2357 """
24- prefix = "https://tempory.net/f/memobin/"
25- if not url.startswith(prefix):
26- raise ValueError("Invalid url. Does not have proper prefix")
2758
28- file_path = url[len(prefix) :]
29- tempory_api_url = "https://hub.tempory.net/api/uploadFile"
59+ def _create_url() -> str:
60+ prefix = "https://tempory.net/f/memobin/"
61+ if not url.startswith(prefix):
62+ raise ValueError("Invalid url. Does not have proper prefix")
63+
64+ file_path = url[len(prefix) :]
65+ tempory_api_url = "https://hub.tempory.net/api/uploadFile"
66+
67+ response = requests.post(
68+ tempory_api_url,
69+ headers={
70+ "Content-Type": "application/json",
71+ "Authorization": f"Bearer {memobin_api_key}",
72+ },
73+ json={
74+ "appName": "memobin",
75+ "filePath": file_path,
76+ "size": size,
77+ "userId": user_id,
78+ },
79+ )
3080
31- response = requests.post(
32- tempory_api_url,
33- headers={
34- "Content-Type": "application/json",
35- "Authorization": f"Bearer {memobin_api_key}",
36- },
37- json={
38- "appName": "memobin",
39- "filePath": file_path,
40- "size": size,
41- "userId": user_id,
42- },
43- )
81+ if not response.ok:
82+ raise requests.RequestException("Failed to get signed url")
4483
45- if not response.ok:
46- raise requests.RequestException("Failed to get signed url")
84+ result = response.json()
85+ upload_url = result["uploadUrl"]
86+ download_url = result["downloadUrl"]
4787
48- result = response.json()
49- upload_url = result["uploadUrl"]
50- download_url = result["downloadUrl"]
88+ if download_url != url:
89+ raise ValueError("Mismatch between download url and url")
5190
52- if download_url != url:
53- raise ValueError("Mismatch between download url and url")
91+ return upload_url
5492
55- return upload_url
93+ return _retry_with_backoff(_create_url, num_retries)
5694
5795
5896 def construct_memobin_url(
@@ -102,6 +140,7 @@ def upload_to_memobin(
102140 url: str,
103141 memobin_api_key: str,
104142 content_type: str = "application/json",
143+ num_retries: int = 4,
105144 ) -> None:
106145 """Upload data to memobin.
107146
@@ -120,17 +159,22 @@ def upload_to_memobin(
120159 data_bytes = data
121160 size = len(data_bytes)
122161
123- upload_url = create_signed_upload_url(url, size, "benchcompress", memobin_api_key)
162+ def _do_upload() -> None:
163+ upload_url = create_signed_upload_url(
164+ url, size, "benchcompress", memobin_api_key, num_retries
165+ )
124166
125- response = requests.put(
126- upload_url, data=data_bytes, headers={"Content-Type": content_type}
127- )
167+ response = requests.put(
168+ upload_url, data=data_bytes, headers={"Content-Type": content_type}
169+ )
128170
129- if not response.ok:
130- raise requests.RequestException("Failed to upload data to memobin")
171+ if not response.ok:
172+ raise requests.RequestException("Failed to upload data to memobin")
131173
174+ _retry_with_backoff(_do_upload, num_retries)
132175
133-def exists_in_memobin(url: str) -> bool:
176+
177+def exists_in_memobin(url: str, num_retries: int = 4) -> bool:
134178 """Check if a file exists in memobin using a HEAD request.
135179
136180 Args:
@@ -139,16 +183,22 @@ def exists_in_memobin(url: str) -> bool:
139183 Returns:
140184 True if the file exists, False otherwise
141185 """
142- try:
143- response = requests.head(url)
144- return (
145- 200 <= response.status_code < 300
146- ) # Any 2xx status code indicates success
147- except requests.RequestException:
148- return False
149186
187+ def _check_exists() -> bool:
188+ try:
189+ response = requests.head(url)
190+ return (
191+ 200 <= response.status_code < 300
192+ ) # Any 2xx status code indicates success
193+ except requests.RequestException:
194+ return False
195+
196+ return _retry_with_backoff(_check_exists, num_retries)
150197
151-def download_from_memobin(url: str, as_json: bool = True) -> Optional[dict | bytes]:
198+
199+def download_from_memobin(
200+ url: str, as_json: bool = True, num_retries: int = 4
201+) -> Optional[dict | bytes]:
152202 """Download data from memobin.
153203
154204 Args:
@@ -161,14 +211,18 @@ def download_from_memobin(url: str, as_json: bool = True) -> Optional[dict | byt
161211 Raises:
162212 requests.RequestException: If the download fails for a reason other than 404
163213 """
164- response = None
165- try:
166- response = requests.get(url)
167- if response.status_code == 404:
168- return None
169- response.raise_for_status()
170- return response.json() if as_json else response.content
171- except requests.RequestException as e:
172- if response and response.status_code == 404:
173- return None
174- raise e
214+
215+ def _do_download() -> Optional[dict | bytes]:
216+ response = None
217+ try:
218+ response = requests.get(url)
219+ if response.status_code == 404:
220+ return None
221+ response.raise_for_status()
222+ return response.json() if as_json else response.content
223+ except requests.RequestException as e:
224+ if response and response.status_code == 404:
225+ return None
226+ raise e
227+
228+ return _retry_with_backoff(_do_download, num_retries)