Add xerial snappy read/writer (#838)

Forked from [github.com/eapache/go-xerial-snappy](https://github.com/eapache/go-xerial-snappy). Changes: * Uses [S2](https://github.com/klauspost/compress/tree/master/s2#snappy-compatibility) for better/faster compression and decompression. * Fixes 0-length roundtrips. * Adds `DecodeCapped`, which allows decompression with capped output size. * `DecodeInto` will decode directly into destination if there is space enough. * `Encode` will now encode directly into 'dst' if it has space enough. * Fixes short snappy buffers returning `ErrMalformed`. * Renames `EncodeStream` to `Encode`. * Adds `EncodeBetter` for better than default compression at ~half the speed. Comparison (before/after): ``` BenchmarkSnappyStreamEncode-32 959010 1170 ns/op 875.15 MB/s 1280 B/op 1 allocs/op BenchmarkSnappyStreamEncode-32 1000000 1107 ns/op 925.04 MB/s 0 B/op 0 allocs/op --> Output size: 913 -> 856 bytes BenchmarkSnappyStreamEncodeBetter-32 477739 2506 ns/op 408.62 MB/s 0 B/op 0 allocs/op --> Output size: 835 bytes BenchmarkSnappyStreamEncodeMassive-32 100 10596963 ns/op 966.31 MB/s 40977 B/op 1 allocs/op BenchmarkSnappyStreamEncodeMassive-32 100 10220236 ns/op 1001.93 MB/s 0 B/op 0 allocs/op --> Output size: 2365547 -> 2256991 bytes BenchmarkSnappyStreamEncodeBetterMassive-32 69 16983314 ns/op 602.94 MB/s 0 B/op 0 allocs/op --> Output size: 2011997 bytes BenchmarkSnappyStreamDecodeInto-32 1887378 639.5 ns/op 1673.19 MB/s 1088 B/op 3 allocs/op BenchmarkSnappyStreamDecodeInto-32 2707915 436.2 ns/op 2452.99 MB/s 0 B/op 0 allocs/op BenchmarkSnappyStreamDecodeIntoMassive-32 267 4559594 ns/op 2245.81 MB/s 71120 B/op 1 allocs/op BenchmarkSnappyStreamDecodeIntoMassive-32 282 4285844 ns/op 2389.26 MB/s 0 B/op 0 allocs/op ```
klauspost · Jul 27, 2023 · 895291c · 895291c
1 parent 33c9bc8
commit 895291c
Show file tree

Hide file tree

Showing 6 changed files with 674 additions and 0 deletions.
diff --git a/snappy/xerial/LICENSE b/snappy/xerial/LICENSE
@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Evan Huus
+Copyright (c) 2023 Klaus Post
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/snappy/xerial/README.md b/snappy/xerial/README.md
@@ -0,0 +1,50 @@
+# go-xerial-snappy
+
+Xerial-compatible Snappy framing support for golang.
+
+Packages using Xerial for snappy encoding use a framing format incompatible with
+basically everything else in existence.
+
+Apps that use this format include Apache Kafka (see
+https://github.com/dpkp/kafka-python/issues/126#issuecomment-35478921 for
+details).
+
+# Fork
+
+Forked from [github.com/eapache/go-xerial-snappy](https://github.com/eapache/go-xerial-snappy).
+
+Changes:
+
+* Uses [S2](https://github.com/klauspost/compress/tree/master/s2#snappy-compatibility) for better/faster compression and decompression.
+* Fixes 0-length roundtrips.
+* Adds `DecodeCapped`, which allows decompression with capped output size.
+* `DecodeInto` will decode directly into destination if there is space enough.
+* `Encode` will now encode directly into 'dst' if it has space enough.
+* Fixes short snappy buffers returning `ErrMalformed`.
+* Renames `EncodeStream` to `Encode`.
+* Adds `EncodeBetter` for better than default compression at ~half the speed.
+
+
+Comparison (before/after):
+
+```
+BenchmarkSnappyStreamEncode-32    	  959010	      1170 ns/op	 875.15 MB/s	    1280 B/op	       1 allocs/op
+BenchmarkSnappyStreamEncode-32    	 1000000	      1107 ns/op	 925.04 MB/s	       0 B/op	       0 allocs/op
+--> Output size: 913 -> 856 bytes
+
+BenchmarkSnappyStreamEncodeBetter-32    	  477739	      2506 ns/op	 408.62 MB/s	       0 B/op	       0 allocs/op
+--> Output size: 835 bytes
+
+BenchmarkSnappyStreamEncodeMassive-32  	     100    	  10596963 ns/op	 966.31 MB/s	   40977 B/op	       1 allocs/op
+BenchmarkSnappyStreamEncodeMassive-32  	     100       	  10220236 ns/op	1001.93 MB/s	       0 B/op	       0 allocs/op
+--> Output size: 2365547 -> 2256991 bytes
+
+BenchmarkSnappyStreamEncodeBetterMassive-32    	      69	  16983314 ns/op	 602.94 MB/s	       0 B/op	       0 allocs/op
+--> Output size: 2011997 bytes
+
+BenchmarkSnappyStreamDecodeInto-32    	 1887378	       639.5 ns/op	1673.19 MB/s	    1088 B/op	       3 allocs/op
+BenchmarkSnappyStreamDecodeInto-32    	 2707915	       436.2 ns/op	2452.99 MB/s	       0 B/op	       0 allocs/op
+
+BenchmarkSnappyStreamDecodeIntoMassive-32    	     267	   4559594 ns/op	2245.81 MB/s	   71120 B/op	       1 allocs/op
+BenchmarkSnappyStreamDecodeIntoMassive-32    	     282	   4285844 ns/op	2389.26 MB/s	       0 B/op	       0 allocs/op
+```
diff --git a/snappy/xerial/fuzz_test.go b/snappy/xerial/fuzz_test.go
@@ -0,0 +1,64 @@
+package xerial
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/klauspost/compress/internal/fuzz"
+	"github.com/klauspost/compress/s2"
+)
+
+func FuzzDecode(f *testing.F) {
+	fuzz.AddFromZip(f, "testdata/FuzzDecoder.zip", fuzz.TypeGoFuzz, false)
+	const limit = 1 << 20
+	dst := make([]byte, 0, limit)
+	f.Fuzz(func(t *testing.T, data []byte) {
+		got, _ := DecodeCapped(dst[:0], data)
+		if len(got) > cap(dst) {
+			t.Fatalf("cap exceeded: %d > %d", len(got), cap(dst))
+		}
+	})
+}
+
+func FuzzEncode(f *testing.F) {
+	fuzz.AddFromZip(f, "../../s2/testdata/enc_regressions.zip", fuzz.TypeRaw, false)
+	fuzz.AddFromZip(f, "../../s2/testdata/fuzz/block-corpus-raw.zip", fuzz.TypeRaw, testing.Short())
+	fuzz.AddFromZip(f, "../../s2/testdata/fuzz/block-corpus-enc.zip", fuzz.TypeGoFuzz, testing.Short())
+
+	f.Fuzz(func(t *testing.T, data []byte) {
+		t.Run("standard", func(t *testing.T) {
+			encoded := Encode(make([]byte, 0, len(data)/2), data)
+			decoded, err := Decode(encoded)
+			if err != nil {
+				t.Errorf("input: %+v, encoded: %+v", data, encoded)
+				t.Fatal(err)
+			}
+			if !bytes.Equal(decoded, data) {
+				t.Fatal("mismatch")
+			}
+
+		})
+		t.Run("better", func(t *testing.T) {
+			encoded := EncodeBetter(make([]byte, 0, len(data)/2), data)
+			decoded, err := Decode(encoded)
+			if err != nil {
+				t.Errorf("input: %+v, encoded: %+v", data, encoded)
+				t.Fatal(err)
+			}
+			if !bytes.Equal(decoded, data) {
+				t.Fatal("mismatch")
+			}
+		})
+		t.Run("snappy", func(t *testing.T) {
+			encoded := s2.EncodeSnappy(make([]byte, 0, len(data)/2), data)
+			decoded, err := Decode(encoded)
+			if err != nil {
+				t.Errorf("input: %+v, encoded: %+v", data, encoded)
+				t.Fatal(err)
+			}
+			if !bytes.Equal(decoded, data) {
+				t.Fatal("mismatch")
+			}
+		})
+	})
+}
diff --git a/snappy/xerial/testdata/FuzzDecoder.zip b/snappy/xerial/testdata/FuzzDecoder.zip