update paper
7 changed files+391−4
.vscode/tasks.jsonmodified+1−1View file
@@ -4,7 +4,7 @@
44 {
55 "label": "Compile Paper",
66 "type": "shell",
7- "command": "./paper/compile.sh",
7+ "command": "./devel/compile_paper.sh",
88 "group": "build",
99 "presentation": {
1010 "reveal": "always",
devel/compile_paper.shadded+12−0View file
@@ -0,0 +1,12 @@
1+#!/bin/bash
2+
3+cd "$(dirname "$0")/.."
4+cd paper
5+
6+pandoc paper.md \
7+ --metadata-file=paper.yml \
8+ --bibliography=references.bib \
9+ --csl=ieee.csl \
10+ -o paper.pdf
11+
12+echo "Paper compiled successfully to paper.pdf"
paper/.gitignoreadded+1−0View file
@@ -0,0 +1 @@
1+*.pdf
paper/ieee.csladded+334−0View file
@@ -0,0 +1,334 @@
1+<?xml version="1.0" encoding="utf-8"?>
2+<style xmlns="http://purl.org/net/xbiblio/csl" class="in-text" version="1.0" demote-non-dropping-particle="sort-only">
3+ <info>
4+ <title>IEEE</title>
5+ <id>http://www.zotero.org/styles/ieee</id>
6+ <link href="http://www.zotero.org/styles/ieee" rel="self"/>
7+ <link href="https://ieeeauthorcenter.ieee.org/wp-content/uploads/IEEE-Reference-Guide.pdf" rel="documentation"/>
8+ <author>
9+ <name>Michael Berkowitz</name>
10+ <email>mberkowi@gmu.edu</email>
11+ </author>
12+ <contributor>
13+ <name>Julian Onions</name>
14+ <email>julian.onions@gmail.com</email>
15+ </contributor>
16+ <contributor>
17+ <name>Rintze Zelle</name>
18+ <uri>http://twitter.com/rintzezelle</uri>
19+ </contributor>
20+ <contributor>
21+ <name>Stephen Frank</name>
22+ <uri>http://www.zotero.org/sfrank</uri>
23+ </contributor>
24+ <contributor>
25+ <name>Sebastian Karcher</name>
26+ </contributor>
27+ <category citation-format="numeric"/>
28+ <category field="engineering"/>
29+ <category field="generic-base"/>
30+ <summary>IEEE style as per the 2021 guidelines, V 01.29.2021.</summary>
31+ <updated>2021-01-29T00:00:00+00:00</updated>
32+ <rights license="http://creativecommons.org/licenses/by-sa/3.0/">This work is licensed under a Creative Commons Attribution-ShareAlike 3.0 License</rights>
33+ </info>
34+ <locale xml:lang="en">
35+ <terms>
36+ <term name="chapter" form="short">ch.</term>
37+ <term name="presented at">presented at the</term>
38+ <term name="available at">available</term>
39+ </terms>
40+ </locale>
41+ <!-- Macros -->
42+ <macro name="edition">
43+ <choose>
44+ <if type="bill book chapter graphic legal_case legislation motion_picture paper-conference report song" match="any">
45+ <choose>
46+ <if is-numeric="edition">
47+ <group delimiter=" ">
48+ <number variable="edition" form="ordinal"/>
49+ <text term="edition" form="short"/>
50+ </group>
51+ </if>
52+ <else>
53+ <text variable="edition" text-case="capitalize-first" suffix="."/>
54+ </else>
55+ </choose>
56+ </if>
57+ </choose>
58+ </macro>
59+ <macro name="issued">
60+ <choose>
61+ <if type="article-journal report" match="any">
62+ <date variable="issued">
63+ <date-part name="month" form="short" suffix=" "/>
64+ <date-part name="year" form="long"/>
65+ </date>
66+ </if>
67+ <else-if type=" bill book chapter graphic legal_case legislation motion_picture paper-conference song thesis" match="any">
68+ <date variable="issued">
69+ <date-part name="year" form="long"/>
70+ </date>
71+ </else-if>
72+ <else>
73+ <date variable="issued">
74+ <date-part name="day" form="numeric-leading-zeros" suffix="-"/>
75+ <date-part name="month" form="short" suffix="-" strip-periods="true"/>
76+ <date-part name="year" form="long"/>
77+ </date>
78+ </else>
79+ </choose>
80+ </macro>
81+ <macro name="author">
82+ <names variable="author">
83+ <name initialize-with="." delimiter=", " and="text"/>
84+ <label form="short" prefix=", " text-case="capitalize-first"/>
85+ <substitute>
86+ <names variable="editor"/>
87+ <names variable="translator"/>
88+ </substitute>
89+ </names>
90+ </macro>
91+ <macro name="editor">
92+ <names variable="editor">
93+ <name initialize-with="." delimiter=", " and="text"/>
94+ <label form="short" prefix=", " text-case="capitalize-first"/>
95+ </names>
96+ </macro>
97+ <macro name="locators">
98+ <group delimiter=", ">
99+ <text macro="edition"/>
100+ <group delimiter=" ">
101+ <text term="volume" form="short"/>
102+ <number variable="volume" form="numeric"/>
103+ </group>
104+ <group delimiter=" ">
105+ <number variable="number-of-volumes" form="numeric"/>
106+ <text term="volume" form="short" plural="true"/>
107+ </group>
108+ <group delimiter=" ">
109+ <text term="issue" form="short"/>
110+ <number variable="issue" form="numeric"/>
111+ </group>
112+ </group>
113+ </macro>
114+ <macro name="title">
115+ <choose>
116+ <if type="bill book graphic legal_case legislation motion_picture song" match="any">
117+ <text variable="title" font-style="italic"/>
118+ </if>
119+ <else>
120+ <text variable="title" quotes="true"/>
121+ </else>
122+ </choose>
123+ </macro>
124+ <macro name="publisher">
125+ <choose>
126+ <if type=" bill book chapter graphic legal_case legislation motion_picture paper-conference song" match="any">
127+ <group delimiter=": ">
128+ <text variable="publisher-place"/>
129+ <text variable="publisher"/>
130+ </group>
131+ </if>
132+ <else>
133+ <group delimiter=", ">
134+ <text variable="publisher"/>
135+ <text variable="publisher-place"/>
136+ </group>
137+ </else>
138+ </choose>
139+ </macro>
140+ <macro name="event">
141+ <choose>
142+ <if type="paper-conference speech" match="any">
143+ <choose>
144+ <!-- Published Conference Paper -->
145+ <if variable="container-title">
146+ <group delimiter=", ">
147+ <group delimiter=" ">
148+ <text term="in"/>
149+ <text variable="container-title" font-style="italic"/>
150+ </group>
151+ <text variable="event-place"/>
152+ </group>
153+ </if>
154+ <!-- Unpublished Conference Paper -->
155+ <else>
156+ <group delimiter=", ">
157+ <group delimiter=" ">
158+ <text term="presented at"/>
159+ <text variable="event"/>
160+ </group>
161+ <text variable="event-place"/>
162+ </group>
163+ </else>
164+ </choose>
165+ </if>
166+ </choose>
167+ </macro>
168+ <macro name="access">
169+ <choose>
170+ <if variable="URL">
171+ <group delimiter=". ">
172+ <group delimiter=": ">
173+ <text term="available at" text-case="capitalize-first"/>
174+ <text variable="URL"/>
175+ </group>
176+ <group prefix="[" suffix="]" delimiter=": ">
177+ <text term="accessed" text-case="capitalize-first"/>
178+ <date variable="accessed">
179+ <date-part name="day" form="numeric-leading-zeros" suffix="-"/>
180+ <date-part name="month" form="short" suffix="-" strip-periods="true"/>
181+ <date-part name="year" form="long"/>
182+ </date>
183+ </group>
184+ </group>
185+ </if>
186+ </choose>
187+ </macro>
188+ <macro name="page">
189+ <group>
190+ <label variable="page" form="short" suffix=" "/>
191+ <text variable="page"/>
192+ </group>
193+ </macro>
194+ <macro name="citation-locator">
195+ <group delimiter=" ">
196+ <choose>
197+ <if locator="page">
198+ <label variable="locator" form="short"/>
199+ </if>
200+ <else>
201+ <label variable="locator" form="short" text-case="capitalize-first"/>
202+ </else>
203+ </choose>
204+ <text variable="locator"/>
205+ </group>
206+ </macro>
207+ <!-- Citation -->
208+ <citation collapse="citation-number">
209+ <sort>
210+ <key variable="citation-number"/>
211+ </sort>
212+ <layout delimiter=", ">
213+ <group prefix="[" suffix="]" delimiter=", ">
214+ <text variable="citation-number"/>
215+ <text macro="citation-locator"/>
216+ </group>
217+ </layout>
218+ </citation>
219+ <!-- Bibliography -->
220+ <bibliography entry-spacing="0" second-field-align="flush">
221+ <layout suffix=".">
222+ <!-- Citation Number -->
223+ <text variable="citation-number" prefix="[" suffix="]"/>
224+ <!-- Author(s) -->
225+ <text macro="author" suffix=", "/>
226+ <!-- Rest of Citation -->
227+ <group suffix=". ">
228+ <choose>
229+ <!-- Specific Formats -->
230+ <if type="article-journal">
231+ <group delimiter=", ">
232+ <text macro="title"/>
233+ <text variable="container-title" font-style="italic" form="short"/>
234+ <text macro="locators"/>
235+ <text macro="page"/>
236+ <text macro="issued"/>
237+ </group>
238+ </if>
239+ <else-if type="paper-conference">
240+ <group delimiter=", ">
241+ <text macro="title"/>
242+ <text macro="event"/>
243+ <text macro="issued"/>
244+ <text macro="locators"/>
245+ <text macro="page"/>
246+ </group>
247+ </else-if>
248+ <else-if type="report">
249+ <group delimiter=", ">
250+ <text macro="title"/>
251+ <text macro="publisher"/>
252+ <group delimiter=" ">
253+ <text variable="genre"/>
254+ <text variable="number"/>
255+ </group>
256+ <text macro="issued"/>
257+ </group>
258+ </else-if>
259+ <else-if type="thesis">
260+ <group delimiter=", ">
261+ <text macro="title"/>
262+ <text variable="genre"/>
263+ <text macro="publisher"/>
264+ <text macro="issued"/>
265+ </group>
266+ </else-if>
267+ <else-if type="webpage post-weblog post" match="any">
268+ <group delimiter=", " suffix=". ">
269+ <text macro="title"/>
270+ <text variable="container-title" font-style="italic"/>
271+ <text macro="issued"/>
272+ </group>
273+ </else-if>
274+ <else-if type="patent">
275+ <text macro="title" suffix=", "/>
276+ <text variable="number" prefix="U.S. Patent "/>
277+ <text macro="issued"/>
278+ </else-if>
279+ <!-- Generic/Fallback Formats -->
280+ <else-if type="bill book graphic legal_case legislation motion_picture report song" match="any">
281+ <group delimiter=", " suffix=". ">
282+ <text macro="title"/>
283+ <text macro="locators"/>
284+ </group>
285+ <group delimiter=", ">
286+ <text macro="publisher"/>
287+ <text macro="issued"/>
288+ <text macro="page"/>
289+ </group>
290+ </else-if>
291+ <else-if type="article-magazine article-newspaper broadcast interview manuscript map patent personal_communication song speech thesis webpage" match="any">
292+ <group delimiter=", ">
293+ <text macro="title"/>
294+ <text variable="container-title" font-style="italic"/>
295+ <text macro="locators"/>
296+ <text macro="publisher"/>
297+ <text macro="page"/>
298+ <text macro="issued"/>
299+ </group>
300+ </else-if>
301+ <else-if type="chapter paper-conference" match="any">
302+ <group delimiter=", " suffix=", ">
303+ <text macro="title"/>
304+ <group delimiter=" ">
305+ <text term="in"/>
306+ <text variable="container-title" font-style="italic"/>
307+ </group>
308+ <text macro="locators"/>
309+ </group>
310+ <text macro="editor" suffix=" "/>
311+ <group delimiter=", ">
312+ <text macro="publisher"/>
313+ <text macro="issued"/>
314+ <text macro="page"/>
315+ </group>
316+ </else-if>
317+ <else>
318+ <group delimiter=", " suffix=". ">
319+ <text macro="title"/>
320+ <text variable="container-title" font-style="italic"/>
321+ <text macro="locators"/>
322+ </group>
323+ <group delimiter=", ">
324+ <text macro="publisher"/>
325+ <text macro="page"/>
326+ <text macro="issued"/>
327+ </group>
328+ </else>
329+ </choose>
330+ </group>
331+ <text macro="access"/>
332+ </layout>
333+ </bibliography>
334+</style>
paper/paper.mdmodified+11−3View file
@@ -22,9 +22,15 @@ When assessing compression performance, it is important to consider not only fin
2222
2323 To help researchers navigate these questions, we introduce Benchcompress, a benchmarking framework designed to systematically evaluate and compare compression algorithms for numeric data arrays. By automating the testing and providing tools for analysis, Benchcompress simplifies the process of identifying which methods work best for particular datasets. In the sections that follow, we describe the system in detail, outline both standard and specialized compression strategies, and present preliminary results from our benchmarks.
2424
25-## Indepent identically distributed samples
25+## Traditional Compression Methods and Scientific Data
2626
27-For independently and identically distributed (i.i.d.) discrete data, where each sample is drawn from a discrete probability distribution (e.g., Bernoulli sampling or quantized Gaussian noise), the theoretical compressed size in bits per sample is determined by the Shannon entropy formula:
27+Traditional compression algorithms like LZW (Lempel-Ziv-Welch), LZ77, BWT (Burrows-Wheeler Transform), and Huffman coding were originally designed for text data where patterns manifest as repeated sequences of characters or words. While these methods can compress numeric data from scientific instruments, they may not be optimal for this use case. Text compression excels at identifying exact matches of recurring patterns, whereas scientific measurements often exhibit more complex relationships between values.
28+
29+For example, in text compression, finding repeated instances of common words or phrases leads to efficient encoding. In contrast, scientific data arrays often contain continuous variations where measurement noise and uncertainties mean exact repetition is rare. Even when numeric values are similar, their byte-level representations may share some common bits while differing in others. This characteristic of scientific data suggests that methods explicitly accounting for numerical relationships between values may achieve better compression ratios than general-purpose algorithms.
30+
31+## Independent identically distributed samples
32+
33+Moving from text to numeric data, we begin with the fundamental case of independently and identically distributed (i.i.d.) discrete data. For such data, where each sample is drawn from a discrete probability distribution (e.g., Bernoulli sampling or quantized Gaussian noise), the theoretical compressed size in bits per sample is determined by the Shannon entropy formula:
2834
2935 $$
3036 H(X) = -\sum_{i} p(x_i) \log_2 p(x_i).
@@ -40,7 +46,7 @@ bits per sample. This means that the optimal compression ratio for such a datase
4046
4147 ## Asymmetric numeral systems
4248
43-In practice, achieving this theoretical compression ratio requires sophisticated encoding techniques. Arithmetic encoding [ref] is one such method, but it is challenging to implement and can be computationally inefficient. A more modern and efficient alternative is asymmetric numeral systems (ANS) [ref], which closely approaches the theoretical limit and is incorporated into state-of-the-art compressors such as ZStandard [ref]. However, these large, general-use packages are primarily optimized for structured data types, such as text, rather than for numeric scientific data. In our benchmarks, we evaluate ANS using a simple, no-frills, implementation using a Python package we developed for this purpose called `simple_ans`. As anticipated, we show that ANS demonstrates superior performance when compressing i.i.d. samples from a discrete distribution.
49+In practice, achieving this theoretical compression ratio requires sophisticated encoding techniques. Arithmetic coding [@witten1987arithmetic] is one such method, but it is challenging to implement and can be computationally inefficient. A more modern and efficient alternative is asymmetric numeral systems (ANS) [@duda2013asymmetric], which closely approaches the theoretical limit and is incorporated into state-of-the-art compressors such as ZStandard [@zstd]. However, these large, general-use packages are primarily optimized for structured data types, such as text, rather than for numeric scientific data. In our benchmarks, we evaluate ANS using a simple, no-frills, implementation using a Python package we developed for this purpose called `simple_ans`. As anticipated, we show that ANS demonstrates superior performance when compressing i.i.d. samples from a discrete distribution.
4450
4551 ## Delta encoding
4652
@@ -85,3 +91,5 @@ Describe the datasets we use for benchmarking.
8591 ## Conclusion
8692
8793 [Conclusion to be added]
94+
95+## References
paper/paper.ymladded+7−0View file
@@ -0,0 +1,7 @@
1+---
2+title: "Benchmarking Compression Algorithms for Scientific Data Arrays"
3+author: "Jeremy Magland"
4+date: "January 2025"
5+bibliography: references.bib
6+csl: ieee.csl
7+---
paper/references.bibadded+25−0View file
@@ -0,0 +1,25 @@
1+@article{witten1987arithmetic,
2+ title={Arithmetic coding for data compression},
3+ author={Witten, Ian H and Neal, Radford M and Cleary, John G},
4+ journal={Communications of the ACM},
5+ volume={30},
6+ number={6},
7+ pages={520--540},
8+ year={1987},
9+ publisher={ACM}
10+}
11+
12+@article{duda2013asymmetric,
13+ title={Asymmetric numeral systems: entropy coding combining speed of Huffman coding with compression rate of arithmetic coding},
14+ author={Duda, Jarek},
15+ journal={arXiv preprint arXiv:1311.2540},
16+ year={2013}
17+}
18+
19+@misc{zstd,
20+ title={Zstandard - Fast real-time compression algorithm},
21+ author={{Meta Platforms, Inc.}},
22+ year={2023},
23+ howpublished={\url{https://github.com/facebook/zstd}},
24+ note={Accessed: 2024-01-30}
25+}