2f82381Update benchmark results from 2025-07-24 16:44:35 [skip ci]GitHub Actions Bot 1import requests
2import json
3import os
4import numpy as np
5import matplotlib.pyplot as plt
6from matplotlib.axes import Axes
7from adjustText import adjust_text
8from load_results import download_results, load_results_into_models, BenchmarkResults
10# Font size configurations
11FONT_SIZES = {
12 'base': 14,
13 'title': 18,
14 'axis_label': 16,
15 'tick_label': 14,
16 'legend': 14,
17 'annotation': 14
18}
20figheight = 9
21figheight2 = 9
23# Set base font size for all plots
24plt.rcParams.update({'font.size': FONT_SIZES['base']})
26def ensure_image_dir():
27 """Ensure the images directory exists relative to this script."""
28 script_dir = os.path.dirname(os.path.abspath(__file__))
29 image_dir = os.path.join(script_dir, 'images')
30 os.makedirs(image_dir, exist_ok=True)
31 return image_dir
33def get_theoretical_ratio(dataset_name: str) -> float | None:
34 """Get theoretical compression ratio for a dataset if applicable."""
35 if dataset_name.startswith('bernoulli-'):
36 try:
37 p = float(dataset_name.split('-')[1])
38 probs = np.array([p, 1-p])
39 entropy = -np.sum(probs * np.log2(probs)) # Shannon entropy
40 return 8 / entropy # 8 bits per byte divided by entropy
41 except (IndexError, ValueError):
42 return None
43 elif dataset_name.startswith('gaussian-q'):
44 try:
45 sigma = float(dataset_name.split('-q')[1])
46 # Calculate probabilities for integers in a reasonable range around mean (0)
47 # For 99.7% coverage, use ±4*sigma
48 range_max = int(4 * sigma)
49 x = np.arange(-range_max, range_max + 1)
50 # Calculate probabilities using normal distribution PDF
51 probs = (1/(sigma * np.sqrt(2*np.pi))) * np.exp(-(x**2)/(2*sigma**2))
52 # Normalize probabilities to sum to 1
53 probs = probs / np.sum(probs)
54 # Calculate entropy
55 nonzero_probs = probs[probs > 0] # Avoid log(0)
56 entropy = -np.sum(nonzero_probs * np.log2(nonzero_probs))
57 return 16 / entropy # 8 bits per byte divided by entropy
58 except (IndexError, ValueError):
59 return None
60 return None
62def plot_single_compression_ratio_comparison(results: BenchmarkResults, dataset_name: str, ax: Axes):
63 """Create a horizontal bar plot for a specific dataset."""
64 # Filter for the specified dataset
65 dataset_results = [r for r in results.results if r.dataset == dataset_name]
67 # Get unique algorithms and their best compression ratios
68 algorithm_ratios = {}
69 for r in dataset_results:
70 current_ratio = r.compression_ratio
71 if r.algorithm not in algorithm_ratios or current_ratio < algorithm_ratios[r.algorithm]:
72 algorithm_ratios[r.algorithm] = current_ratio
74 # Sort by compression ratio
75 sorted_items = sorted(algorithm_ratios.items(), key=lambda x: x[1])
76 algorithms = [item[0] for item in sorted_items]
77 ratios = [item[1] for item in sorted_items]
79 # Create horizontal bar plot
80 ax.barh(algorithms, ratios)
81 ax.set_xlabel('Compression Ratio', fontsize=FONT_SIZES['axis_label'])
82 ax.set_ylabel('Algorithm', fontsize=FONT_SIZES['axis_label'])
83 ax.set_title(f'Compression Ratios for {dataset_name}', fontsize=FONT_SIZES['title'])
84 ax.tick_params(axis='both', which='major', labelsize=FONT_SIZES['tick_label'])
86 # Add theoretical limit line if applicable
87 theoretical_ratio = get_theoretical_ratio(dataset_name)
88 if theoretical_ratio is not None:
89 ax.axvline(x=theoretical_ratio, color='red', linestyle='--')
90 ax.legend()
92def figure_tradeoff_comparison(results: BenchmarkResults, datasets: list[str]):
93 """Create scatter plots showing compression ratio vs speed tradeoffs."""
94 # Create two figures - one for encode, one for decode
95 for speed_type in ['encode', 'decode']:
96 n_plots = len(datasets)
97 fig, axes = plt.subplots(1, n_plots, figsize=(10*n_plots, figheight2))
98 fig.suptitle(f'Compression Ratio vs {speed_type.capitalize()} Speed Tradeoff')
100 # Handle single dataset case
101 if n_plots == 1:
102 axes = [axes]
104 for dataset_name, ax in zip(datasets, axes):
105 # Filter for specified dataset
106 dataset_results = [r for r in results.results if r.dataset == dataset_name]
108 # Get best results for each algorithm
109 algorithm_metrics = {}
110 for r in dataset_results:
111 speed = r.encode_mb_per_sec if speed_type == 'encode' else r.decode_mb_per_sec
112 if r.algorithm not in algorithm_metrics:
113 algorithm_metrics[r.algorithm] = {
114 'ratio': r.compression_ratio,
115 'speed': speed
116 }
117 else:
118 # Update if this instance has better trade-off score
119 current_score = speed / r.compression_ratio
120 existing_score = (algorithm_metrics[r.algorithm]['speed'] /
121 algorithm_metrics[r.algorithm]['ratio'])
122 if current_score > existing_score:
123 algorithm_metrics[r.algorithm] = {
124 'ratio': r.compression_ratio,
125 'speed': speed
126 }
128 # Extract data for plotting
129 ratios = [metrics['ratio'] for metrics in algorithm_metrics.values()]
130 speeds = [metrics['speed'] for metrics in algorithm_metrics.values()]
131 algorithms = list(algorithm_metrics.keys())
133 # Create scatter plot
134 ax.scatter(ratios, speeds, alpha=0.6)
136 texts = [ax.text(ratios[i], speeds[i], alg, fontsize=FONT_SIZES['annotation']) for i, alg in enumerate(algorithms)]
137 adjust_text(texts,
138 arrowprops=dict(arrowstyle='->', color='gray', linewidth=0.5),
139 ensure_inside_axes=False,
140 ax=ax)
142 # Customize plot
143 ax.set_xlabel('Compression Ratio (higher is better)', fontsize=FONT_SIZES['axis_label'])
144 ax.set_ylabel(f'{speed_type.capitalize()} Speed MB/sec (higher is better)', fontsize=FONT_SIZES['axis_label'])
145 ax.set_title(f'{dataset_name}', fontsize=FONT_SIZES['title'])
146 ax.tick_params(axis='both', which='major', labelsize=FONT_SIZES['tick_label'])
147 ax.grid(True, alpha=0.3)
149 # Add theoretical limit line if applicable
150 theoretical_ratio = get_theoretical_ratio(dataset_name)
151 if theoretical_ratio is not None:
152 ax.axvline(x=theoretical_ratio, color='red', linestyle='--')
153 ax.legend()
155 plt.tight_layout()
156 # Save figure
157 plt.savefig(os.path.join(ensure_image_dir(), f'iid_tradeoff_{speed_type}_{"-".join(datasets)}.png'))
158 plt.close()
160def figure_speeds_comparison(results: BenchmarkResults, datasets: list[str]):
161 """Create horizontal bar plots comparing encode/decode speeds for the given datasets."""
162 # Create figure with enough subplots for all datasets
163 n_plots = len(datasets)
164 fig, axes = plt.subplots(1, n_plots, figsize=(10*n_plots, figheight))
166 # Handle single dataset case
167 if n_plots == 1:
168 axes = [axes]
170 for dataset_name, ax in zip(datasets, axes):
171 # Filter for specified dataset
172 dataset_results = [r for r in results.results if r.dataset == dataset_name]
174 # Get unique algorithms and their speeds
175 algorithm_speeds = {}
176 for r in dataset_results:
177 if r.algorithm not in algorithm_speeds:
178 algorithm_speeds[r.algorithm] = {
179 'encode': r.encode_mb_per_sec,
180 'decode': r.decode_mb_per_sec
181 }
182 else:
183 # Update if this instance has better average speed
184 current_avg = (r.encode_mb_per_sec + r.decode_mb_per_sec) / 2
185 existing_avg = (algorithm_speeds[r.algorithm]['encode'] +
186 algorithm_speeds[r.algorithm]['decode']) / 2
187 if current_avg > existing_avg:
188 algorithm_speeds[r.algorithm] = {
189 'encode': r.encode_mb_per_sec,
190 'decode': r.decode_mb_per_sec
191 }
193 # Sort alphabetically by algorithm name
194 sorted_items = sorted(algorithm_speeds.items(), key=lambda x: x[0])
195 algorithms = [item[0] for item in sorted_items]
196 encode_speeds = [item[1]['encode'] for item in sorted_items]
197 decode_speeds = [item[1]['decode'] for item in sorted_items]
199 # Set up positions for the bars
200 y_pos = np.arange(len(algorithms))
201 bar_height = 0.35
203 # Create bars
204 ax.barh(y_pos - bar_height/2, encode_speeds, bar_height,
205 label='Encode Speed', color='royalblue', alpha=0.8)
206 ax.barh(y_pos + bar_height/2, decode_speeds, bar_height,
207 label='Decode Speed', color='seagreen', alpha=0.8)
209 # Customize plot
210 ax.set_yticks(y_pos)
211 ax.set_yticklabels(algorithms)
212 ax.set_xlabel('Speed (MB/sec)', fontsize=FONT_SIZES['axis_label'])
213 ax.set_title(f'Encode/Decode Speeds for {dataset_name}', fontsize=FONT_SIZES['title'])
214 ax.tick_params(axis='both', which='major', labelsize=FONT_SIZES['tick_label'])
215 ax.legend(fontsize=FONT_SIZES['legend'])
217 # Add grid for readability
218 ax.grid(True, axis='x', alpha=0.3)
220 plt.tight_layout()
221 # Save figure
222 plt.savefig(os.path.join(ensure_image_dir(), f'iid_speeds_comparison_{"-".join(datasets)}.png'))
223 plt.close()
225def figure_compression_ratios_comparison(results: BenchmarkResults, datasets: list[str]):
226 """Create horizontal bar plots comparing compression ratios for given datasets."""
227 # Create figure with enough subplots for all datasets
228 n_plots = len(datasets)
229 fig, axes = plt.subplots(1, n_plots, figsize=(10*n_plots, figheight))
231 # Handle single dataset case
232 if n_plots == 1:
233 axes = [axes]
235 # Plot comparisons for each dataset
236 for dataset, ax in zip(datasets, axes):
237 plot_single_compression_ratio_comparison(results, dataset, ax)
239 plt.tight_layout()
240 # Save figure
241 plt.savefig(os.path.join(ensure_image_dir(), f'iid_compression_ratios_{"-".join(datasets)}.png'))
242 plt.close()
244def main():
245 try:
246 # Download the results
247 raw_data = download_results()
249 # Load into pydantic models
250 results = load_results_into_models(raw_data)
252 print(f"Successfully loaded {len(results.results)} benchmark results")
254 datasets_list = [
255 ['bernoulli-0.1', 'bernoulli-0.5'],
256 ['gaussian-q2', 'gaussian-q5']
257 ]
259 for datasets in datasets_list:
260 print("\nGenerating compression ratios plot...")
261 figure_compression_ratios_comparison(results, datasets)
263 print("\nGenerating speed comparison plot...")
264 figure_speeds_comparison(results, datasets)
266 print("\nGenerating tradeoff plots...")
267 figure_tradeoff_comparison(results, datasets)
269 return results
271 except requests.RequestException as e:
272 print(f"Error downloading results: {e}")
273 return None
274 except json.JSONDecodeError as e:
275 print(f"Error parsing JSON: {e}")
276 return None
277 except Exception as e:
278 print(f"Error loading data into models: {e}")
279 return None
281if __name__ == '__main__':
282 main()