make figures
2 changed files+305−0
figures/load_results.pyadded+42−0View file
@@ -0,0 +1,42 @@
1+import requests
2+import json
3+from typing import List
4+import numpy as np
5+import matplotlib.pyplot as plt
6+from matplotlib.axes import Axes
7+from pydantic import BaseModel
8+from adjustText import adjust_text
9+
10+results_url = 'https://tempory.net/f/memobin/benchcompress/global/results.json'
11+
12+class BenchmarkResult(BaseModel):
13+ compression_ratio: float
14+ encode_time: float
15+ decode_time: float
16+ encode_mb_per_sec: float
17+ decode_mb_per_sec: float
18+ original_size: int
19+ compressed_size: int
20+ array_shape: List[int]
21+ array_dtype: str
22+ timestamp: float
23+ cache_status: str
24+ dataset: str
25+ algorithm: str
26+ algorithm_version: str
27+ dataset_version: str
28+ system_version: str
29+
30+class BenchmarkResults(BaseModel):
31+ results: List[BenchmarkResult]
32+
33+def download_results() -> dict:
34+ url = results_url
35+ print(f"Downloading results from {url}...")
36+ response = requests.get(url)
37+ response.raise_for_status()
38+ return response.json()
39+
40+def load_results_into_models(data: dict) -> BenchmarkResults:
41+ """Load the downloaded data into pydantic models."""
42+ return BenchmarkResults(**data)
figures/make_figures_iid.pyadded+263−0View file
@@ -0,0 +1,263 @@
1+import requests
2+import json
3+import os
4+import numpy as np
5+import matplotlib.pyplot as plt
6+from matplotlib.axes import Axes
7+from adjustText import adjust_text
8+from load_results import download_results, load_results_into_models, BenchmarkResults
9+
10+def ensure_image_dir():
11+ """Ensure the images directory exists relative to this script."""
12+ script_dir = os.path.dirname(os.path.abspath(__file__))
13+ image_dir = os.path.join(script_dir, 'images')
14+ os.makedirs(image_dir, exist_ok=True)
15+ return image_dir
16+
17+def get_theoretical_ratio(dataset_name: str) -> float | None:
18+ """Get theoretical compression ratio for a dataset if applicable."""
19+ if dataset_name.startswith('bernoulli-'):
20+ try:
21+ p = float(dataset_name.split('-')[1])
22+ probs = np.array([p, 1-p])
23+ entropy = -np.sum(probs * np.log2(probs)) # Shannon entropy
24+ return 8 / entropy # 8 bits per byte divided by entropy
25+ except (IndexError, ValueError):
26+ return None
27+ elif dataset_name.startswith('gaussian-q'):
28+ try:
29+ sigma = float(dataset_name.split('-q')[1])
30+ # Calculate probabilities for integers in a reasonable range around mean (0)
31+ # For 99.7% coverage, use ±4*sigma
32+ range_max = int(4 * sigma)
33+ x = np.arange(-range_max, range_max + 1)
34+ # Calculate probabilities using normal distribution PDF
35+ probs = (1/(sigma * np.sqrt(2*np.pi))) * np.exp(-(x**2)/(2*sigma**2))
36+ # Normalize probabilities to sum to 1
37+ probs = probs / np.sum(probs)
38+ # Calculate entropy
39+ nonzero_probs = probs[probs > 0] # Avoid log(0)
40+ entropy = -np.sum(nonzero_probs * np.log2(nonzero_probs))
41+ return 16 / entropy # 8 bits per byte divided by entropy
42+ except (IndexError, ValueError):
43+ return None
44+ return None
45+
46+def plot_single_compression_ratio_comparison(results: BenchmarkResults, dataset_name: str, ax: Axes):
47+ """Create a horizontal bar plot for a specific dataset."""
48+ # Filter for the specified dataset
49+ dataset_results = [r for r in results.results if r.dataset == dataset_name]
50+
51+ # Get unique algorithms and their best compression ratios
52+ algorithm_ratios = {}
53+ for r in dataset_results:
54+ current_ratio = r.compression_ratio
55+ if r.algorithm not in algorithm_ratios or current_ratio < algorithm_ratios[r.algorithm]:
56+ algorithm_ratios[r.algorithm] = current_ratio
57+
58+ # Sort by compression ratio
59+ sorted_items = sorted(algorithm_ratios.items(), key=lambda x: x[1])
60+ algorithms = [item[0] for item in sorted_items]
61+ ratios = [item[1] for item in sorted_items]
62+
63+ # Create horizontal bar plot
64+ ax.barh(algorithms, ratios)
65+ ax.set_xlabel('Compression Ratio')
66+ ax.set_ylabel('Algorithm')
67+ ax.set_title(f'Compression Ratios for {dataset_name}')
68+
69+ # Add theoretical limit line if applicable
70+ theoretical_ratio = get_theoretical_ratio(dataset_name)
71+ if theoretical_ratio is not None:
72+ ax.axvline(x=theoretical_ratio, color='red', linestyle='--')
73+ ax.legend()
74+
75+def figure_tradeoff_comparison(results: BenchmarkResults, datasets: list[str]):
76+ """Create scatter plots showing compression ratio vs speed tradeoffs."""
77+ # Create two figures - one for encode, one for decode
78+ for speed_type in ['encode', 'decode']:
79+ n_plots = len(datasets)
80+ fig, axes = plt.subplots(1, n_plots, figsize=(10*n_plots, 8))
81+ fig.suptitle(f'Compression Ratio vs {speed_type.capitalize()} Speed Tradeoff')
82+
83+ # Handle single dataset case
84+ if n_plots == 1:
85+ axes = [axes]
86+
87+ for dataset_name, ax in zip(datasets, axes):
88+ # Filter for specified dataset
89+ dataset_results = [r for r in results.results if r.dataset == dataset_name]
90+
91+ # Get best results for each algorithm
92+ algorithm_metrics = {}
93+ for r in dataset_results:
94+ speed = r.encode_mb_per_sec if speed_type == 'encode' else r.decode_mb_per_sec
95+ if r.algorithm not in algorithm_metrics:
96+ algorithm_metrics[r.algorithm] = {
97+ 'ratio': r.compression_ratio,
98+ 'speed': speed
99+ }
100+ else:
101+ # Update if this instance has better trade-off score
102+ current_score = speed / r.compression_ratio
103+ existing_score = (algorithm_metrics[r.algorithm]['speed'] /
104+ algorithm_metrics[r.algorithm]['ratio'])
105+ if current_score > existing_score:
106+ algorithm_metrics[r.algorithm] = {
107+ 'ratio': r.compression_ratio,
108+ 'speed': speed
109+ }
110+
111+ # Extract data for plotting
112+ ratios = [metrics['ratio'] for metrics in algorithm_metrics.values()]
113+ speeds = [metrics['speed'] for metrics in algorithm_metrics.values()]
114+ algorithms = list(algorithm_metrics.keys())
115+
116+ # Create scatter plot
117+ ax.scatter(ratios, speeds, alpha=0.6)
118+
119+ texts = [ax.text(ratios[i], speeds[i], alg, fontsize=9) for i, alg in enumerate(algorithms)]
120+ adjust_text(texts,
121+ arrowprops=dict(arrowstyle='->', color='gray', linewidth=0.5),
122+ ensure_inside_axes=False,
123+ ax=ax)
124+
125+ # Customize plot
126+ ax.set_xlabel('Compression Ratio (higher is better)')
127+ ax.set_ylabel(f'{speed_type.capitalize()} Speed MB/sec (higher is better)')
128+ ax.set_title(f'{dataset_name}')
129+ ax.grid(True, alpha=0.3)
130+
131+ # Add theoretical limit line if applicable
132+ theoretical_ratio = get_theoretical_ratio(dataset_name)
133+ if theoretical_ratio is not None:
134+ ax.axvline(x=theoretical_ratio, color='red', linestyle='--')
135+ ax.legend()
136+
137+ plt.tight_layout()
138+ # Save figure
139+ plt.savefig(os.path.join(ensure_image_dir(), f'iid_tradeoff_{speed_type}_{"-".join(datasets)}.png'))
140+ plt.close()
141+
142+def figure_speeds_comparison(results: BenchmarkResults, datasets: list[str]):
143+ """Create horizontal bar plots comparing encode/decode speeds for the given datasets."""
144+ # Create figure with enough subplots for all datasets
145+ n_plots = len(datasets)
146+ fig, axes = plt.subplots(1, n_plots, figsize=(10*n_plots, 6))
147+
148+ # Handle single dataset case
149+ if n_plots == 1:
150+ axes = [axes]
151+
152+ for dataset_name, ax in zip(datasets, axes):
153+ # Filter for specified dataset
154+ dataset_results = [r for r in results.results if r.dataset == dataset_name]
155+
156+ # Get unique algorithms and their speeds
157+ algorithm_speeds = {}
158+ for r in dataset_results:
159+ if r.algorithm not in algorithm_speeds:
160+ algorithm_speeds[r.algorithm] = {
161+ 'encode': r.encode_mb_per_sec,
162+ 'decode': r.decode_mb_per_sec
163+ }
164+ else:
165+ # Update if this instance has better average speed
166+ current_avg = (r.encode_mb_per_sec + r.decode_mb_per_sec) / 2
167+ existing_avg = (algorithm_speeds[r.algorithm]['encode'] +
168+ algorithm_speeds[r.algorithm]['decode']) / 2
169+ if current_avg > existing_avg:
170+ algorithm_speeds[r.algorithm] = {
171+ 'encode': r.encode_mb_per_sec,
172+ 'decode': r.decode_mb_per_sec
173+ }
174+
175+ # Sort alphabetically by algorithm name
176+ sorted_items = sorted(algorithm_speeds.items(), key=lambda x: x[0])
177+ algorithms = [item[0] for item in sorted_items]
178+ encode_speeds = [item[1]['encode'] for item in sorted_items]
179+ decode_speeds = [item[1]['decode'] for item in sorted_items]
180+
181+ # Set up positions for the bars
182+ y_pos = np.arange(len(algorithms))
183+ bar_height = 0.35
184+
185+ # Create bars
186+ ax.barh(y_pos - bar_height/2, encode_speeds, bar_height,
187+ label='Encode Speed', color='royalblue', alpha=0.8)
188+ ax.barh(y_pos + bar_height/2, decode_speeds, bar_height,
189+ label='Decode Speed', color='seagreen', alpha=0.8)
190+
191+ # Customize plot
192+ ax.set_yticks(y_pos)
193+ ax.set_yticklabels(algorithms)
194+ ax.set_xlabel('Speed (MB/sec)')
195+ ax.set_title(f'Encode/Decode Speeds for {dataset_name}')
196+ ax.legend()
197+
198+ # Add grid for readability
199+ ax.grid(True, axis='x', alpha=0.3)
200+
201+ plt.tight_layout()
202+ # Save figure
203+ plt.savefig(os.path.join(ensure_image_dir(), f'iid_speeds_comparison_{"-".join(datasets)}.png'))
204+ plt.close()
205+
206+def figure_compression_ratios_comparison(results: BenchmarkResults, datasets: list[str]):
207+ """Create horizontal bar plots comparing compression ratios for given datasets."""
208+ # Create figure with enough subplots for all datasets
209+ n_plots = len(datasets)
210+ fig, axes = plt.subplots(1, n_plots, figsize=(10*n_plots, 6))
211+
212+ # Handle single dataset case
213+ if n_plots == 1:
214+ axes = [axes]
215+
216+ # Plot comparisons for each dataset
217+ for dataset, ax in zip(datasets, axes):
218+ plot_single_compression_ratio_comparison(results, dataset, ax)
219+
220+ plt.tight_layout()
221+ # Save figure
222+ plt.savefig(os.path.join(ensure_image_dir(), f'iid_compression_ratios_{"-".join(datasets)}.png'))
223+ plt.close()
224+
225+def main():
226+ try:
227+ # Download the results
228+ raw_data = download_results()
229+
230+ # Load into pydantic models
231+ results = load_results_into_models(raw_data)
232+
233+ print(f"Successfully loaded {len(results.results)} benchmark results")
234+
235+ datasets_list = [
236+ ['bernoulli-0.1', 'bernoulli-0.5'],
237+ ['gaussian-q2', 'gaussian-q5']
238+ ]
239+
240+ for datasets in datasets_list:
241+ print("\nGenerating compression ratios plot...")
242+ figure_compression_ratios_comparison(results, datasets)
243+
244+ print("\nGenerating speed comparison plot...")
245+ figure_speeds_comparison(results, datasets)
246+
247+ print("\nGenerating tradeoff plots...")
248+ figure_tradeoff_comparison(results, datasets)
249+
250+ return results
251+
252+ except requests.RequestException as e:
253+ print(f"Error downloading results: {e}")
254+ return None
255+ except json.JSONDecodeError as e:
256+ print(f"Error parsing JSON: {e}")
257+ return None
258+ except Exception as e:
259+ print(f"Error loading data into models: {e}")
260+ return None
261+
262+if __name__ == '__main__':
263+ main()