Initial project import
This commit is contained in:
Agila Forge Importer committed
commit
23f2d29e32
11957 files changed
+1754577
No files matched your search
File diff suppressed because it is too large.
Load diff
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,457 @@
|
||||
// Implementation of Snappy compression fully in V.
|
||||
//
|
||||
// Compatible with the standard Snappy binary format. Implements level-1
|
||||
// (fast) compression using a hash-table approach over 64 KiB blocks.
|
||||
//
|
||||
// Public API:
|
||||
// compress(input []u8) []u8
|
||||
// decompress(input []u8) ![]u8
|
||||
|
||||
module snappy
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const block_size = 1 << 16 // 65 536 — max block processed at once
|
||||
const min_table_size = 1 << 8 // 256
|
||||
const max_table_size = 1 << 15 // 32 768 (fits in u16 alongside block_size-1)
|
||||
const input_margin = 15 // bytes of look-ahead the compressor needs
|
||||
const hash_magic = u32(0x1e35a7bd)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// max_compressed_length returns an upper bound on the compressed size for
|
||||
// `n` bytes of uncompressed input.
|
||||
fn max_compressed_length(n int) int {
|
||||
return 32 + n + n / 6
|
||||
}
|
||||
|
||||
// compress compresses `input` and returns the compressed bytes.
|
||||
pub fn compress(input []u8) []u8 {
|
||||
n := input.len
|
||||
mut out := []u8{cap: max_compressed_length(n)}
|
||||
|
||||
// Write the uncompressed length as a varint header.
|
||||
write_uvarint(mut out, u32(n))
|
||||
|
||||
if n == 0 {
|
||||
return out
|
||||
}
|
||||
|
||||
// Allocate a single hash table reused across every block.
|
||||
mut table := []u16{len: max_table_size}
|
||||
|
||||
mut pos := 0
|
||||
for pos < n {
|
||||
end := if pos + block_size < n { pos + block_size } else { n }
|
||||
block := input[pos..end]
|
||||
|
||||
// Zero only the slice of the table we will actually use.
|
||||
tsize := table_size_for(block.len)
|
||||
for i in 0 .. tsize {
|
||||
table[i] = 0
|
||||
}
|
||||
|
||||
compress_fragment(block, mut out, mut table, tsize)
|
||||
pos = end
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// decompress decompresses a Snappy-compressed slice.
|
||||
// It returns the original bytes, or an error if the data is malformed.
|
||||
pub fn decompress(input []u8) ![]u8 {
|
||||
// Read the varint-encoded uncompressed length from the header.
|
||||
ulen, hdr_end, ok := read_uvarint(input, 0)
|
||||
if !ok {
|
||||
return error('snappy: invalid header varint')
|
||||
}
|
||||
|
||||
// Pre-allocate the exact output size; reject obviously bogus lengths.
|
||||
ilen := int(ulen)
|
||||
if ilen < 0 {
|
||||
return error('snappy: decoded length overflow')
|
||||
}
|
||||
|
||||
// Use a pre-sized buffer with a write cursor instead of dynamic appends.
|
||||
// This eliminates per-byte reallocation checks on every literal and copy.
|
||||
mut out := []u8{len: ilen}
|
||||
mut op := 0
|
||||
mut pos := hdr_end
|
||||
|
||||
for pos < input.len {
|
||||
tag := input[pos]
|
||||
pos++
|
||||
|
||||
tag_type := tag & 0x03
|
||||
|
||||
if tag_type == 0 {
|
||||
// LITERAL
|
||||
lit_len, new_pos, lit_ok := decode_literal_len(input, tag, pos)
|
||||
if !lit_ok {
|
||||
return error('snappy: truncated literal length')
|
||||
}
|
||||
pos = new_pos
|
||||
if pos + lit_len > input.len {
|
||||
return error('snappy: literal extends past input')
|
||||
}
|
||||
if op + lit_len > ilen {
|
||||
return error('snappy: output exceeds declared length')
|
||||
}
|
||||
for i in 0 .. lit_len {
|
||||
out[op + i] = input[pos + i]
|
||||
}
|
||||
op += lit_len
|
||||
pos += lit_len
|
||||
} else if tag_type == 1 {
|
||||
// COPY_1 (2-byte offset)
|
||||
if pos >= input.len {
|
||||
return error('snappy: truncated copy-1 tag')
|
||||
}
|
||||
length := int((tag >> 2) & 0x07) + 4
|
||||
offset := int(u32(tag & 0xe0) << 3 | u32(input[pos]))
|
||||
pos++
|
||||
if offset == 0 {
|
||||
return error('snappy: zero offset in copy-1')
|
||||
}
|
||||
op = expand_copy_into(mut out, op, ilen, offset, length)!
|
||||
} else if tag_type == 2 {
|
||||
// COPY_2 (3-byte tag)
|
||||
if pos + 1 >= input.len {
|
||||
return error('snappy: truncated copy-2 tag')
|
||||
}
|
||||
length := int((tag >> 2) & 0x3f) + 1
|
||||
offset := int(u32(input[pos]) | u32(input[pos + 1]) << 8)
|
||||
pos += 2
|
||||
if offset == 0 {
|
||||
return error('snappy: zero offset in copy-2')
|
||||
}
|
||||
op = expand_copy_into(mut out, op, ilen, offset, length)!
|
||||
} else {
|
||||
// COPY_4 (5-byte tag)
|
||||
if pos + 3 >= input.len {
|
||||
return error('snappy: truncated copy-4 tag')
|
||||
}
|
||||
length := int((tag >> 2) & 0x3f) + 1
|
||||
offset := int(u32(input[pos]) | u32(input[pos + 1]) << 8 | u32(input[pos + 2]) << 16 | u32(input[
|
||||
pos + 3]) << 24)
|
||||
pos += 4
|
||||
if offset == 0 {
|
||||
return error('snappy: zero offset in copy-4')
|
||||
}
|
||||
op = expand_copy_into(mut out, op, ilen, offset, length)!
|
||||
}
|
||||
}
|
||||
|
||||
if op != ilen {
|
||||
return error('snappy: decompressed length mismatch: got ${op}, want ${ilen}')
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Compression internals
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// compress_fragment compresses a single block (≤ 64 KiB) into `out`.
|
||||
// `table` must have at least `tsize` zeroed entries (tsize is a power of 2).
|
||||
fn compress_fragment(input []u8, mut out []u8, mut table []u16, tsize int) {
|
||||
n := input.len
|
||||
if n == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// tbits is the number of bits needed to index `tsize` entries; used
|
||||
// to right-shift a 32-bit hash down to the table index.
|
||||
tbits := ilog2(tsize)
|
||||
|
||||
// ip — current input position (index into `input`)
|
||||
// lit — start of current literal run
|
||||
mut ip := 0
|
||||
mut lit := 0
|
||||
|
||||
// We need at least `input_margin` bytes of look-ahead for the main loop.
|
||||
// Very short blocks go to the slow literal-only path.
|
||||
if n >= input_margin {
|
||||
// Emit one forced literal so we have a non-empty literal buffer,
|
||||
// then begin the hash loop one byte in.
|
||||
ip = 1
|
||||
|
||||
mut next_hash := mix32(load32le(input, ip), tbits)
|
||||
|
||||
outer: for {
|
||||
// skip grows when matches are not found — we scan with
|
||||
// increasing stride to amortize the cost.
|
||||
mut skip := 32
|
||||
mut candidate := 0
|
||||
mut next_ip := ip
|
||||
|
||||
// Inner probe loop: walk forward until we find a 4-byte match.
|
||||
for {
|
||||
ip = next_ip
|
||||
hash := next_hash
|
||||
bytes_until_skip := skip >> 5
|
||||
skip++
|
||||
next_ip = ip + bytes_until_skip
|
||||
if next_ip > n - input_margin {
|
||||
break outer
|
||||
}
|
||||
next_hash = mix32(load32le(input, next_ip), tbits)
|
||||
candidate = int(table[hash])
|
||||
table[hash] = u16(ip)
|
||||
if load32le(input, ip) == load32le(input, candidate) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// We have a 4-byte match at [ip] == [candidate].
|
||||
// Emit the literal bytes accumulated since `lit`.
|
||||
write_literal(mut out, input[lit..ip])
|
||||
|
||||
// Extend the match as far as possible.
|
||||
for {
|
||||
matched := 4 + find_match_length(input, candidate + 4, ip + 4, n)
|
||||
write_copy(mut out, ip - candidate, matched)
|
||||
ip += matched
|
||||
lit = ip
|
||||
|
||||
if ip >= n - input_margin {
|
||||
break outer
|
||||
}
|
||||
|
||||
// Look for a consecutive match immediately after the copy.
|
||||
// Note: mix32 already returns a value in [0, tsize), so the
|
||||
// redundant `& tmask` from the original code is removed.
|
||||
prev_hash := mix32(load32le(input, ip - 1), tbits)
|
||||
table[prev_hash] = u16(ip - 1)
|
||||
cur_hash := mix32(load32le(input, ip), tbits)
|
||||
candidate = int(table[cur_hash])
|
||||
table[cur_hash] = u16(ip)
|
||||
if load32le(input, ip) != load32le(input, candidate) {
|
||||
// seed next_hash for the outer loop's next iteration
|
||||
next_hash = mix32(load32le(input, ip + 1), tbits)
|
||||
ip++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Emit whatever literal bytes remain.
|
||||
write_literal(mut out, input[lit..])
|
||||
}
|
||||
|
||||
// write_literal emits a LITERAL tag followed by the literal bytes.
|
||||
@[inline]
|
||||
fn write_literal(mut out []u8, lit []u8) {
|
||||
n := lit.len
|
||||
if n == 0 {
|
||||
return
|
||||
}
|
||||
n1 := n - 1
|
||||
if n1 < 60 {
|
||||
out << u8(u32(n1) << 2) // tag byte encodes length-1 directly
|
||||
} else {
|
||||
w := byte_width(n1)
|
||||
out << u8(u32(59 + w) << 2) // tag signals 1..4 extra length bytes
|
||||
mut v := n1
|
||||
for _ in 0 .. w {
|
||||
out << u8(v & 0xff)
|
||||
v >>= 8
|
||||
}
|
||||
}
|
||||
out << lit
|
||||
}
|
||||
|
||||
// write_copy chooses and emits the most compact COPY tag for the given
|
||||
// (offset, length) back-reference.
|
||||
@[inline]
|
||||
fn write_copy(mut out []u8, offset int, length int) {
|
||||
mut remaining := length
|
||||
for remaining > 0 {
|
||||
// COPY_2 handles lengths 1..64; use it as the general case.
|
||||
// COPY_1 saves a byte when offset < 2048 and length is 4..11.
|
||||
chunk := if remaining > 64 { 64 } else { remaining }
|
||||
if offset < 2048 && chunk >= 4 && chunk <= 11 {
|
||||
// COPY_1: 2 bytes total
|
||||
out << u8(u32(1) | (u32(chunk - 4) << 2) | ((u32(offset) >> 3) & u32(0xe0)))
|
||||
out << u8(offset & 0xff)
|
||||
} else if offset < 65536 {
|
||||
// COPY_2: 3 bytes total
|
||||
out << u8(2 | (u32(chunk - 1) << 2))
|
||||
out << u8(offset & 0xff)
|
||||
out << u8((offset >> 8) & 0xff)
|
||||
} else {
|
||||
// COPY_4: 5 bytes total (never reached for 64 KiB blocks, left for completeness)
|
||||
out << u8(3 | (u32(chunk - 1) << 2))
|
||||
out << u8(offset & 0xff)
|
||||
out << u8((offset >> 8) & 0xff)
|
||||
out << u8((offset >> 16) & 0xff)
|
||||
out << u8((offset >> 24) & 0xff)
|
||||
}
|
||||
remaining -= chunk
|
||||
}
|
||||
}
|
||||
|
||||
// find_match_length returns how many bytes starting at s1 and s2 are equal,
|
||||
// stopping before `limit`. Uses 4-byte word comparisons when possible.
|
||||
@[inline]
|
||||
fn find_match_length(input []u8, s1 int, s2 int, limit int) int {
|
||||
mut i := 0
|
||||
// Compare 4 bytes at a time while there is room for a full word.
|
||||
for s2 + i + 4 <= limit && s1 + i + 4 <= limit {
|
||||
if load32le(input, s1 + i) != load32le(input, s2 + i) {
|
||||
// Mismatch somewhere in this word — find the exact byte.
|
||||
for s2 + i < limit && input[s1 + i] == input[s2 + i] {
|
||||
i++
|
||||
}
|
||||
return i
|
||||
}
|
||||
i += 4
|
||||
}
|
||||
// Handle the remaining 0..3 tail bytes.
|
||||
for s2 + i < limit && input[s1 + i] == input[s2 + i] {
|
||||
i++
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Decompression internals
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// expand_copy_into copies `length` bytes from `offset` bytes before the
|
||||
// write cursor `op` in the pre-allocated `out` buffer. Returns the new
|
||||
// write cursor. Handles the overlapping (run-length) case correctly.
|
||||
@[inline]
|
||||
fn expand_copy_into(mut out []u8, op int, cap_ int, offset int, length int) !int {
|
||||
src := op - offset
|
||||
if src < 0 {
|
||||
return error('snappy: copy offset past beginning of output')
|
||||
}
|
||||
if op + length > cap_ {
|
||||
return error('snappy: output exceeds declared length')
|
||||
}
|
||||
// Both overlapping and non-overlapping copies read from indices
|
||||
// that were written in earlier iterations, so a forward byte loop
|
||||
// is correct in all cases (and required for the RLE / overlap case).
|
||||
for i in 0 .. length {
|
||||
out[op + i] = out[src + i]
|
||||
}
|
||||
return op + length
|
||||
}
|
||||
|
||||
// decode_literal_len decodes the length (in bytes) of a literal run from the
|
||||
// tag byte and returns (length, new_pos, ok). `pos` should point to the
|
||||
// byte immediately after the tag.
|
||||
@[inline]
|
||||
fn decode_literal_len(data []u8, tag u8, pos int) (int, int, bool) {
|
||||
n1 := int(tag >> 2) // length - 1, or 60..63 for extended form
|
||||
if n1 < 60 {
|
||||
return n1 + 1, pos, true
|
||||
}
|
||||
extra := n1 - 59 // number of extra bytes (1..4)
|
||||
if pos + extra > data.len {
|
||||
return 0, pos, false
|
||||
}
|
||||
mut val := 0
|
||||
for i in 0 .. extra {
|
||||
val |= int(u32(data[pos + i]) << u32(8 * i))
|
||||
}
|
||||
return val + 1, pos + extra, true
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Varint (unsigned LEB-128, protobuf-style)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// write_uvarint appends `val` as a variable-length unsigned integer.
|
||||
@[inline]
|
||||
fn write_uvarint(mut out []u8, val u32) {
|
||||
mut v := val
|
||||
for v >= 0x80 {
|
||||
out << u8(v & 0x7f | 0x80)
|
||||
v >>= 7
|
||||
}
|
||||
out << u8(v)
|
||||
}
|
||||
|
||||
// read_uvarint decodes a varint from `data` starting at `pos`.
|
||||
// Returns (value, new_pos, ok).
|
||||
@[inline]
|
||||
fn read_uvarint(data []u8, pos int) (u32, int, bool) {
|
||||
mut val := u32(0)
|
||||
mut shift := u32(0)
|
||||
mut i := pos
|
||||
for i < data.len && i - pos < 5 {
|
||||
b := data[i]
|
||||
i++
|
||||
if shift == 28 && b > 0x0f {
|
||||
return 0, pos, false
|
||||
}
|
||||
val |= u32(b & 0x7f) << shift
|
||||
if b & 0x80 == 0 {
|
||||
return val, i, true
|
||||
}
|
||||
shift += 7
|
||||
}
|
||||
return 0, pos, false
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Utilities
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// load32le reads four bytes from `data[pos..]` as a little-endian u32.
|
||||
@[inline]
|
||||
fn load32le(data []u8, pos int) u32 {
|
||||
return u32(data[pos]) | u32(data[pos + 1]) << 8 | u32(data[pos + 2]) << 16 | u32(data[pos + 3]) << 24
|
||||
}
|
||||
|
||||
// mix32 hashes a 32-bit word down to a `tbits`-bit index using a
|
||||
// multiplicative hash (same constant as the reference implementation).
|
||||
@[inline]
|
||||
fn mix32(v u32, tbits int) int {
|
||||
return int((v * hash_magic) >> u32(32 - tbits))
|
||||
}
|
||||
|
||||
// table_size_for returns the smallest power-of-two hash table size that
|
||||
// comfortably covers `n` bytes, clamped to [min_table_size, max_table_size].
|
||||
fn table_size_for(n int) int {
|
||||
mut size := min_table_size
|
||||
for size < n && size < max_table_size {
|
||||
size <<= 1
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// ilog2 returns floor(log2(n)) for n ≥ 1.
|
||||
@[inline]
|
||||
fn ilog2(n int) int {
|
||||
mut v := n
|
||||
mut r := 0
|
||||
for v > 1 {
|
||||
v >>= 1
|
||||
r++
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// byte_width returns the number of bytes needed to represent `n` (1..4).
|
||||
@[inline]
|
||||
fn byte_width(n int) int {
|
||||
if n < (1 << 8) {
|
||||
return 1
|
||||
}
|
||||
if n < (1 << 16) {
|
||||
return 2
|
||||
}
|
||||
if n < (1 << 24) {
|
||||
return 3
|
||||
}
|
||||
return 4
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
module snappy
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn round_trip(label string, input []u8) ! {
|
||||
compressed := compress(input)
|
||||
decompressed := decompress(compressed)!
|
||||
assert decompressed == input, '${label}: round-trip mismatch (input len=${input.len})'
|
||||
bound := max_compressed_length(input.len)
|
||||
assert compressed.len <= bound, '${label}: compressed size ${compressed.len} exceeds bound ${bound}'
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn test_empty() {
|
||||
round_trip('empty', []u8{}) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_single_byte() {
|
||||
round_trip('single byte', [u8(0x42)]) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_two_bytes() {
|
||||
round_trip('two bytes', [u8(0xde), 0xad]) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_all_zeros() {
|
||||
// Highly compressible — long run of the same byte.
|
||||
input := []u8{len: 4096, init: 0}
|
||||
compressed := compress(input)
|
||||
decompressed := decompress(compressed) or { panic(err) }
|
||||
assert decompressed == input, 'all-zeros round-trip failed'
|
||||
// Should compress to much less than input.
|
||||
assert compressed.len < input.len / 4, 'all-zeros should compress well'
|
||||
}
|
||||
|
||||
fn test_all_same_nonzero() {
|
||||
input := []u8{len: 1024, init: u8(0xff)}
|
||||
round_trip('all-0xff', input) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_incompressible() {
|
||||
// Pseudo-random bytes — poor compressibility.
|
||||
mut input := []u8{len: 512}
|
||||
mut seed := u32(0xdeadbeef)
|
||||
for i in 0 .. input.len {
|
||||
seed = seed * 1664525 + 1013904223 // LCG
|
||||
input[i] = u8(seed >> 24)
|
||||
}
|
||||
round_trip('pseudo-random', input) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_repeated_pattern() {
|
||||
// Short repeating pattern — exercises the copy back-reference paths.
|
||||
pattern := 'abcdefgh'.bytes()
|
||||
mut input := []u8{cap: pattern.len * 200}
|
||||
for _ in 0 .. 200 {
|
||||
input << pattern
|
||||
}
|
||||
compressed := compress(input)
|
||||
decompressed := decompress(compressed) or { panic(err) }
|
||||
assert decompressed == input, 'repeated-pattern round-trip failed'
|
||||
assert compressed.len < input.len / 2, 'repeated pattern should compress well'
|
||||
}
|
||||
|
||||
fn test_lorem_ipsum() {
|
||||
lorem := 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. ' +
|
||||
'Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ' +
|
||||
'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris ' +
|
||||
'nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in ' +
|
||||
'reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla ' +
|
||||
'pariatur. Excepteur sint occaecat cupidatat non proident, sunt in ' +
|
||||
'culpa qui officia deserunt mollit anim id est laborum.'
|
||||
mut input := []u8{cap: lorem.len * 50}
|
||||
for _ in 0 .. 50 {
|
||||
input << lorem.bytes()
|
||||
}
|
||||
round_trip('lorem-ipsum-x50', input) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_multi_block() {
|
||||
// Larger than one 64 KiB block to exercise the block-splitting path.
|
||||
mut input := []u8{len: 200_000, init: u8(index & 0xff)}
|
||||
round_trip('multi-block', input) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_max_compressed_length_bound() {
|
||||
for n in [0, 1, 100, 1000, 65536, 200_000] {
|
||||
bound := max_compressed_length(n)
|
||||
mut input := []u8{len: n, init: u8(index & 0xff)}
|
||||
compressed := compress(input)
|
||||
assert compressed.len <= bound, 'max_compressed_length(${n})=${bound} exceeded by ${compressed.len}'
|
||||
}
|
||||
}
|
||||
|
||||
fn test_decompress_invalid_varint() {
|
||||
// A stream that never terminates the varint (all bytes have MSB set).
|
||||
bad := [u8(0xff), 0xff, 0xff, 0xff, 0xff, 0xff]
|
||||
decompress(bad) or { return } // expected — an error is correct
|
||||
panic('expected decompress to fail on invalid varint')
|
||||
}
|
||||
|
||||
fn test_decompress_truncated_literal() {
|
||||
// Header says 10 bytes uncompressed; literal tag claims 10 bytes
|
||||
// but the data is truncated.
|
||||
bad := [u8(10), u8(9 << 2), u8(0x41)] // varint(10) + tag + 1 byte
|
||||
decompress(bad) or { return }
|
||||
panic('expected decompress to fail on truncated literal')
|
||||
}
|
||||
|
||||
fn test_decompress_bad_offset() {
|
||||
// A COPY_2 that references before the start of output.
|
||||
// varint(5) tag=COPY_2(len=5, _) offset=999 (past output)
|
||||
bad := [u8(5), u8(2 | ((5 - 1) << 2)), u8(0xe7), u8(0x03)]
|
||||
decompress(bad) or { return }
|
||||
panic('expected decompress to fail on out-of-range offset')
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// CRC32C — Castagnoli variant used by the Snappy framing format.
|
||||
//
|
||||
// The lookup table is pre-computed at compile time (reflected polynomial
|
||||
// 0x82F63B78). No runtime initialisation is needed.
|
||||
|
||||
module snappy
|
||||
|
||||
// crc32c_table holds all 256 pre-computed CRC32C remainders.
|
||||
// Generated with polynomial 0x82F63B78 (reflected form of 0x1EDC6F41).
|
||||
const crc32c_table = [u32(0x00000000), u32(0xf26b8303), u32(0xe13b70f7), u32(0x1350f3f4),
|
||||
u32(0xc79a971f), u32(0x35f1141c), u32(0x26a1e7e8), u32(0xd4ca64eb), u32(0x8ad958cf),
|
||||
u32(0x78b2dbcc), u32(0x6be22838), u32(0x9989ab3b), u32(0x4d43cfd0), u32(0xbf284cd3),
|
||||
u32(0xac78bf27), u32(0x5e133c24), u32(0x105ec76f), u32(0xe235446c), u32(0xf165b798),
|
||||
u32(0x030e349b), u32(0xd7c45070), u32(0x25afd373), u32(0x36ff2087), u32(0xc494a384),
|
||||
u32(0x9a879fa0), u32(0x68ec1ca3), u32(0x7bbcef57), u32(0x89d76c54), u32(0x5d1d08bf),
|
||||
u32(0xaf768bbc), u32(0xbc267848), u32(0x4e4dfb4b), u32(0x20bd8ede), u32(0xd2d60ddd),
|
||||
u32(0xc186fe29), u32(0x33ed7d2a), u32(0xe72719c1), u32(0x154c9ac2), u32(0x061c6936),
|
||||
u32(0xf477ea35), u32(0xaa64d611), u32(0x580f5512), u32(0x4b5fa6e6), u32(0xb93425e5),
|
||||
u32(0x6dfe410e), u32(0x9f95c20d), u32(0x8cc531f9), u32(0x7eaeb2fa), u32(0x30e349b1),
|
||||
u32(0xc288cab2), u32(0xd1d83946), u32(0x23b3ba45), u32(0xf779deae), u32(0x05125dad),
|
||||
u32(0x1642ae59), u32(0xe4292d5a), u32(0xba3a117e), u32(0x4851927d), u32(0x5b016189),
|
||||
u32(0xa96ae28a), u32(0x7da08661), u32(0x8fcb0562), u32(0x9c9bf696), u32(0x6ef07595),
|
||||
u32(0x417b1dbc), u32(0xb3109ebf), u32(0xa0406d4b), u32(0x522bee48), u32(0x86e18aa3),
|
||||
u32(0x748a09a0), u32(0x67dafa54), u32(0x95b17957), u32(0xcba24573), u32(0x39c9c670),
|
||||
u32(0x2a993584), u32(0xd8f2b687), u32(0x0c38d26c), u32(0xfe53516f), u32(0xed03a29b),
|
||||
u32(0x1f682198), u32(0x5125dad3), u32(0xa34e59d0), u32(0xb01eaa24), u32(0x42752927),
|
||||
u32(0x96bf4dcc), u32(0x64d4cecf), u32(0x77843d3b), u32(0x85efbe38), u32(0xdbfc821c),
|
||||
u32(0x2997011f), u32(0x3ac7f2eb), u32(0xc8ac71e8), u32(0x1c661503), u32(0xee0d9600),
|
||||
u32(0xfd5d65f4), u32(0x0f36e6f7), u32(0x61c69362), u32(0x93ad1061), u32(0x80fde395),
|
||||
u32(0x72966096), u32(0xa65c047d), u32(0x5437877e), u32(0x4767748a), u32(0xb50cf789),
|
||||
u32(0xeb1fcbad), u32(0x197448ae), u32(0x0a24bb5a), u32(0xf84f3859), u32(0x2c855cb2),
|
||||
u32(0xdeeedfb1), u32(0xcdbe2c45), u32(0x3fd5af46), u32(0x7198540d), u32(0x83f3d70e),
|
||||
u32(0x90a324fa), u32(0x62c8a7f9), u32(0xb602c312), u32(0x44694011), u32(0x5739b3e5),
|
||||
u32(0xa55230e6), u32(0xfb410cc2), u32(0x092a8fc1), u32(0x1a7a7c35), u32(0xe811ff36),
|
||||
u32(0x3cdb9bdd), u32(0xceb018de), u32(0xdde0eb2a), u32(0x2f8b6829), u32(0x82f63b78),
|
||||
u32(0x709db87b), u32(0x63cd4b8f), u32(0x91a6c88c), u32(0x456cac67), u32(0xb7072f64),
|
||||
u32(0xa457dc90), u32(0x563c5f93), u32(0x082f63b7), u32(0xfa44e0b4), u32(0xe9141340),
|
||||
u32(0x1b7f9043), u32(0xcfb5f4a8), u32(0x3dde77ab), u32(0x2e8e845f), u32(0xdce5075c),
|
||||
u32(0x92a8fc17), u32(0x60c37f14), u32(0x73938ce0), u32(0x81f80fe3), u32(0x55326b08),
|
||||
u32(0xa759e80b), u32(0xb4091bff), u32(0x466298fc), u32(0x1871a4d8), u32(0xea1a27db),
|
||||
u32(0xf94ad42f), u32(0x0b21572c), u32(0xdfeb33c7), u32(0x2d80b0c4), u32(0x3ed04330),
|
||||
u32(0xccbbc033), u32(0xa24bb5a6), u32(0x502036a5), u32(0x4370c551), u32(0xb11b4652),
|
||||
u32(0x65d122b9), u32(0x97baa1ba), u32(0x84ea524e), u32(0x7681d14d), u32(0x2892ed69),
|
||||
u32(0xdaf96e6a), u32(0xc9a99d9e), u32(0x3bc21e9d), u32(0xef087a76), u32(0x1d63f975),
|
||||
u32(0x0e330a81), u32(0xfc588982), u32(0xb21572c9), u32(0x407ef1ca), u32(0x532e023e),
|
||||
u32(0xa145813d), u32(0x758fe5d6), u32(0x87e466d5), u32(0x94b49521), u32(0x66df1622),
|
||||
u32(0x38cc2a06), u32(0xcaa7a905), u32(0xd9f75af1), u32(0x2b9cd9f2), u32(0xff56bd19),
|
||||
u32(0x0d3d3e1a), u32(0x1e6dcdee), u32(0xec064eed), u32(0xc38d26c4), u32(0x31e6a5c7),
|
||||
u32(0x22b65633), u32(0xd0ddd530), u32(0x0417b1db), u32(0xf67c32d8), u32(0xe52cc12c),
|
||||
u32(0x1747422f), u32(0x49547e0b), u32(0xbb3ffd08), u32(0xa86f0efc), u32(0x5a048dff),
|
||||
u32(0x8ecee914), u32(0x7ca56a17), u32(0x6ff599e3), u32(0x9d9e1ae0), u32(0xd3d3e1ab),
|
||||
u32(0x21b862a8), u32(0x32e8915c), u32(0xc083125f), u32(0x144976b4), u32(0xe622f5b7),
|
||||
u32(0xf5720643), u32(0x07198540), u32(0x590ab964), u32(0xab613a67), u32(0xb831c993),
|
||||
u32(0x4a5a4a90), u32(0x9e902e7b), u32(0x6cfbad78), u32(0x7fab5e8c), u32(0x8dc0dd8f),
|
||||
u32(0xe330a81a), u32(0x115b2b19), u32(0x020bd8ed), u32(0xf0605bee), u32(0x24aa3f05),
|
||||
u32(0xd6c1bc06), u32(0xc5914ff2), u32(0x37faccf1), u32(0x69e9f0d5), u32(0x9b8273d6),
|
||||
u32(0x88d28022), u32(0x7ab90321), u32(0xae7367ca), u32(0x5c18e4c9), u32(0x4f48173d),
|
||||
u32(0xbd23943e), u32(0xf36e6f75), u32(0x0105ec76), u32(0x12551f82), u32(0xe03e9c81),
|
||||
u32(0x34f4f86a), u32(0xc69f7b69), u32(0xd5cf889d), u32(0x27a40b9e), u32(0x79b737ba),
|
||||
u32(0x8bdcb4b9), u32(0x988c474d), u32(0x6ae7c44e), u32(0xbe2da0a5), u32(0x4c4623a6),
|
||||
u32(0x5f16d052), u32(0xad7d5351)]!
|
||||
|
||||
// crc32c returns the CRC-32C checksum of `data`.
|
||||
pub fn crc32c(data []u8) u32 {
|
||||
mut crc := u32(0xffffffff)
|
||||
for b in data {
|
||||
crc = crc32c_table[(crc ^ u32(b)) & 0xff] ^ (crc >> 8)
|
||||
}
|
||||
return crc ^ 0xffffffff
|
||||
}
|
||||
|
||||
// mask_crc applies the Snappy framing mask to a raw CRC32C value:
|
||||
// ((crc >> 15) | (crc << 17)) + 0xa282ead8
|
||||
// This avoids accidental matches between the checksum and compressed data.
|
||||
@[inline]
|
||||
fn mask_crc(crc u32) u32 {
|
||||
return ((crc >> 15) | (crc << 17)) + u32(0xa282ead8)
|
||||
}
|
||||
|
||||
// unmask_crc reverses the masking applied by mask_crc.
|
||||
@[inline]
|
||||
fn unmask_crc(masked u32) u32 {
|
||||
rot := masked - u32(0xa282ead8)
|
||||
return (rot >> 17) | (rot << 15)
|
||||
}
|
||||
@@ -0,0 +1,444 @@
|
||||
// Snappy framing format — streaming wrapper around raw Snappy blocks.
|
||||
//
|
||||
// Spec: https://github.com/google/snappy/blob/main/framing_format.txt
|
||||
//
|
||||
// A framed stream is a sequence of typed chunks, each with a 4-byte header:
|
||||
//
|
||||
// [type: 1 byte] [length: 3 bytes, little-endian]
|
||||
//
|
||||
// Chunk types:
|
||||
// 0xff Stream identifier — must be the first chunk; payload = "sNaPpY"
|
||||
// 0x00 Compressed data — 4-byte masked CRC32C + Snappy-compressed block
|
||||
// 0x01 Uncompressed data — 4-byte masked CRC32C + raw block
|
||||
// 0xfe Padding — payload is ignored
|
||||
// 0x80..0xcd Reserved skippable — may be silently ignored
|
||||
// 0x02..0x7f Reserved unskippable — must be treated as an error
|
||||
//
|
||||
// Public API:
|
||||
// encode_stream(input []u8) []u8
|
||||
// decode_stream(input []u8) ![]u8
|
||||
//
|
||||
// For incremental (push-based) I/O see StreamEncoder and StreamDecoder.
|
||||
|
||||
module snappy
|
||||
|
||||
import math
|
||||
import io
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const stream_identifier_magic = 'sNaPpY'.bytes()
|
||||
const chunk_type_stream_id = u8(0xff)
|
||||
const chunk_type_compressed = u8(0x00)
|
||||
const chunk_type_uncompressed = u8(0x01)
|
||||
const chunk_type_padding = u8(0xfe)
|
||||
|
||||
// Pre encoded stream header
|
||||
const stream_header = [
|
||||
chunk_type_stream_id,
|
||||
u8(stream_identifier_magic.len & 0xff),
|
||||
u8((stream_identifier_magic.len >> 8) & 0xff),
|
||||
u8((stream_identifier_magic.len >> 16) & 0xff),
|
||||
stream_identifier_magic[0],
|
||||
stream_identifier_magic[1],
|
||||
stream_identifier_magic[2],
|
||||
stream_identifier_magic[3],
|
||||
stream_identifier_magic[4],
|
||||
stream_identifier_magic[5],
|
||||
]
|
||||
|
||||
// max_chunk_data_size is the largest block of uncompressed data per chunk.
|
||||
const max_chunk_data_size = 65536
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// One-shot API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// encode_stream wraps `input` in the Snappy framing format.
|
||||
// It returns the complete byte stream. Input is split into <=64 KiB blocks automatically.
|
||||
pub fn encode_stream(input []u8) []u8 {
|
||||
// Upper bound: identifier chunk + one compressed chunk per block.
|
||||
// Each block: 4-byte header + 4-byte CRC + max_compressed_length.
|
||||
blocks := (input.len + max_chunk_data_size - 1) / max_chunk_data_size
|
||||
mut out := []u8{cap: 10 + blocks * (8 + max_compressed_length(max_chunk_data_size))}
|
||||
|
||||
out << stream_header
|
||||
|
||||
mut pos := 0
|
||||
for pos < input.len {
|
||||
end := math.min(pos + max_chunk_data_size, input.len)
|
||||
write_data_chunk(mut out, input[pos..end])
|
||||
pos = end
|
||||
}
|
||||
|
||||
// Always emit at least the identifier for empty input.
|
||||
return out
|
||||
}
|
||||
|
||||
// decode_stream decodes a Snappy framing stream.
|
||||
// It returns the concatenated uncompressed data, or an error if the stream is malformed.
|
||||
pub fn decode_stream(input []u8) ![]u8 {
|
||||
// The compressed stream is typically smaller than the uncompressed data,
|
||||
// so use a larger initial capacity estimate.
|
||||
mut out := []u8{cap: input.len * 2}
|
||||
mut pos := 0
|
||||
|
||||
// The very first chunk must be the stream identifier.
|
||||
pos = read_stream_identifier(input, pos)!
|
||||
|
||||
for pos < input.len {
|
||||
chunk_type, chunk_len, new_pos := read_chunk_header(input, pos)!
|
||||
pos = new_pos
|
||||
|
||||
if pos + chunk_len > input.len {
|
||||
return error('snappy framing: chunk body truncated at pos ${pos}')
|
||||
}
|
||||
chunk_data := input[pos..pos + chunk_len]
|
||||
pos += chunk_len
|
||||
|
||||
if chunk_type == chunk_type_compressed {
|
||||
uncompressed := decode_compressed_chunk(chunk_data)!
|
||||
out << uncompressed
|
||||
} else if chunk_type == chunk_type_uncompressed {
|
||||
raw := decode_uncompressed_chunk(chunk_data)!
|
||||
out << raw
|
||||
} else if chunk_type == chunk_type_padding || int(chunk_type) >= 0x80 {
|
||||
// 0xfe = padding, 0x80..0xfd = reserved skippable; ignore.
|
||||
continue
|
||||
} else {
|
||||
// 0x02..0x7f — reserved unskippable; must error.
|
||||
return error('snappy framing: unsupported unskippable chunk type 0x${chunk_type:02x}')
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Incremental encoder
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// StreamEncoder holds state for incremental framing-format encoding.
|
||||
// Call write() one or more times, then close() to flush the final chunk.
|
||||
// Take the encoded bytes using read().
|
||||
pub struct StreamEncoder {
|
||||
mut:
|
||||
buf []u8 // pending uncompressed bytes
|
||||
out []u8 = stream_header // accumulated encoded output, initialized with stream identifier
|
||||
closed bool
|
||||
}
|
||||
|
||||
// write appends `buf` to the encoder, flushing complete 64 KiB chunks.
|
||||
pub fn (mut enc StreamEncoder) write(buf []u8) !int {
|
||||
if enc.closed {
|
||||
return error('snappy: write on a closed stream encoder')
|
||||
}
|
||||
|
||||
enc.buf << buf
|
||||
for enc.buf.len >= max_chunk_data_size {
|
||||
block := enc.buf[..max_chunk_data_size]
|
||||
write_data_chunk(mut enc.out, block)
|
||||
enc.buf = enc.buf[max_chunk_data_size..].clone()
|
||||
}
|
||||
|
||||
return buf.len
|
||||
}
|
||||
|
||||
pub fn (mut enc StreamEncoder) read(mut buf []u8) !int {
|
||||
if enc.out.len == 0 {
|
||||
if enc.closed {
|
||||
return io.Eof{}
|
||||
}
|
||||
return 0 // not finished yet, just no data ready, the caller should feed more
|
||||
}
|
||||
|
||||
n := copy(mut buf, enc.out)
|
||||
|
||||
if n == enc.out.len {
|
||||
enc.out = []u8{}
|
||||
} else {
|
||||
enc.out = enc.out[n..].clone()
|
||||
}
|
||||
|
||||
return n
|
||||
}
|
||||
|
||||
// close flushes any remaining buffered bytes and returns the complete stream.
|
||||
// The encoder must not be used after this call.
|
||||
pub fn (mut enc StreamEncoder) close() {
|
||||
if enc.closed {
|
||||
return
|
||||
}
|
||||
|
||||
if enc.buf.len > 0 {
|
||||
write_data_chunk(mut enc.out, enc.buf)
|
||||
enc.buf = []
|
||||
}
|
||||
enc.closed = true
|
||||
}
|
||||
|
||||
// peek returns the encoded bytes produced so far without flushing.
|
||||
pub fn (enc StreamEncoder) peek() []u8 {
|
||||
return enc.out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Incremental decoder
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// StreamDecoder holds state for incremental framing-format decoding.
|
||||
// Call write() with successive chunks of compressed input, and call read() to read the decoded bytes.
|
||||
// The writer must call close() when it finishes writing the bytes.
|
||||
pub struct StreamDecoder {
|
||||
mut:
|
||||
buf []u8 // unparsed compressed bytes
|
||||
offset int // read position within buf (avoids repeated cloning)
|
||||
identified bool // have we seen the stream identifier chunk?
|
||||
err ?IError
|
||||
output []u8 // decoded bytes ready for the caller
|
||||
closed bool
|
||||
}
|
||||
|
||||
// write appends `buf` to the decoder's input buffer.
|
||||
// It processes as many complete chunks as possible and errors if the stream is malformed.
|
||||
pub fn (mut dec StreamDecoder) write(buf []u8) !int {
|
||||
if dec.closed {
|
||||
return error('snappy: write to closed stream decoder')
|
||||
}
|
||||
if err := dec.err {
|
||||
return err
|
||||
}
|
||||
dec.buf << buf
|
||||
|
||||
for {
|
||||
remaining := dec.buf.len - dec.offset
|
||||
// Need at least 4 bytes for the chunk header.
|
||||
if remaining < 4 {
|
||||
break
|
||||
}
|
||||
chunk_type, chunk_len, hdr_end := read_chunk_header(dec.buf, dec.offset) or {
|
||||
dec_err := error('snappy: failed while reading chunk header: ${err}')
|
||||
dec.err = dec_err
|
||||
return dec_err
|
||||
}
|
||||
if hdr_end + chunk_len > dec.buf.len {
|
||||
break // wait for more data
|
||||
}
|
||||
chunk_data := dec.buf[hdr_end..hdr_end + chunk_len]
|
||||
|
||||
if !dec.identified {
|
||||
if chunk_type != chunk_type_stream_id {
|
||||
dec_err := error('snappy framing: stream does not begin with identifier chunk')
|
||||
dec.err = dec_err
|
||||
return dec_err
|
||||
}
|
||||
if chunk_len != stream_identifier_magic.len {
|
||||
dec_err := error('snappy framing: bad identifier chunk length ${chunk_len}')
|
||||
dec.err = dec_err
|
||||
return dec_err
|
||||
}
|
||||
if chunk_data != stream_identifier_magic {
|
||||
dec_err := error('snappy framing: bad stream identifier payload')
|
||||
dec.err = dec_err
|
||||
return dec_err
|
||||
}
|
||||
dec.identified = true
|
||||
} else if chunk_type == chunk_type_compressed {
|
||||
uncompressed := decode_compressed_chunk(chunk_data) or {
|
||||
dec_err := error('snappy: failed while decoding compressed chunk: ${err}')
|
||||
dec.err = dec_err
|
||||
return dec_err
|
||||
}
|
||||
dec.output << uncompressed
|
||||
} else if chunk_type == chunk_type_uncompressed {
|
||||
raw := decode_uncompressed_chunk(chunk_data) or {
|
||||
dec_err := error('snappy: failed while decoding uncompressed chunk: ${err}')
|
||||
dec.err = dec_err
|
||||
return dec_err
|
||||
}
|
||||
dec.output << raw
|
||||
} else if chunk_type == chunk_type_padding || int(chunk_type) >= 0x80 {
|
||||
// skip
|
||||
} else {
|
||||
dec_err :=
|
||||
error('snappy framing: unsupported unskippable chunk type 0x${chunk_type:02x}')
|
||||
dec.err = dec_err
|
||||
return dec_err
|
||||
}
|
||||
|
||||
dec.offset = hdr_end + chunk_len
|
||||
}
|
||||
|
||||
// Compact the buffer: discard consumed bytes only when worthwhile
|
||||
// to avoid copying on every single chunk.
|
||||
if dec.offset > 0 {
|
||||
if dec.offset == dec.buf.len {
|
||||
dec.buf = []u8{}
|
||||
} else {
|
||||
dec.buf = dec.buf[dec.offset..].clone()
|
||||
}
|
||||
dec.offset = 0
|
||||
}
|
||||
|
||||
return buf.len
|
||||
}
|
||||
|
||||
// read removes and returns decoded bytes.
|
||||
// When the output buffer is empty it surfaces any error recorded during write()/close().
|
||||
// Only then does it return io.Eof{} for a cleanly terminated stream.
|
||||
pub fn (mut dec StreamDecoder) read(mut buf []u8) !int {
|
||||
// Always drain already-decoded output first so the caller never loses data.
|
||||
if dec.output.len > 0 {
|
||||
n := copy(mut buf, dec.output)
|
||||
dec.output = dec.output[n..]
|
||||
return n
|
||||
}
|
||||
|
||||
if dec.closed {
|
||||
// Surface truncation / malformed-stream errors before EOF.
|
||||
if err := dec.err {
|
||||
return err
|
||||
}
|
||||
return io.Eof{}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// close marks the stream finished and validates its terminal state.
|
||||
// Returns an error if the stream was truncated or the identifier was never received.
|
||||
// The error is also stored internally so read() will surface it after draining output.
|
||||
pub fn (mut dec StreamDecoder) close() ! {
|
||||
if dec.closed {
|
||||
return
|
||||
}
|
||||
dec.closed = true
|
||||
|
||||
remaining := dec.buf.len - dec.offset
|
||||
|
||||
if remaining > 0 {
|
||||
err :=
|
||||
error('snappy framing: stream closed with ${remaining} unprocessed bytes (truncated chunk)')
|
||||
dec.err = err
|
||||
return err
|
||||
}
|
||||
|
||||
if !dec.identified && dec.buf.len > 0 {
|
||||
err := error('snappy framing: stream closed before identifier chunk was received')
|
||||
dec.err = err
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Chunk writers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// write_data_chunk compresses `block`, chooses the compact representation,
|
||||
// and appends the chunk (type + header + masked CRC + payload) to `out`.
|
||||
fn write_data_chunk(mut out []u8, block []u8) {
|
||||
crc := mask_crc(crc32c(block))
|
||||
compressed := compress(block)
|
||||
|
||||
// Use the compressed form only when it is strictly smaller.
|
||||
if compressed.len < block.len {
|
||||
// Compressed chunk: type=0x00, length=4+compressed.len
|
||||
out << chunk_type_compressed
|
||||
write_u24le(mut out, 4 + compressed.len)
|
||||
write_u32le(mut out, crc)
|
||||
out << compressed
|
||||
} else {
|
||||
// Uncompressed chunk: type=0x01, length=4+block.len
|
||||
out << chunk_type_uncompressed
|
||||
write_u24le(mut out, 4 + block.len)
|
||||
write_u32le(mut out, crc)
|
||||
out << block
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Chunk readers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// read_stream_identifier reads and validates the identifier chunk starting at
|
||||
// `pos` and returns the position immediately after it.
|
||||
fn read_stream_identifier(data []u8, pos int) !int {
|
||||
chunk_type, chunk_len, new_pos := read_chunk_header(data, pos)!
|
||||
if chunk_type != chunk_type_stream_id {
|
||||
return error('snappy framing: stream must begin with identifier chunk, got 0x${chunk_type:02x}')
|
||||
}
|
||||
if chunk_len != stream_identifier_magic.len {
|
||||
return error('snappy framing: bad identifier chunk length ${chunk_len}')
|
||||
}
|
||||
if new_pos + chunk_len > data.len {
|
||||
return error('snappy framing: identifier chunk truncated')
|
||||
}
|
||||
magic := data[new_pos..new_pos + chunk_len]
|
||||
if magic != stream_identifier_magic {
|
||||
return error('snappy framing: bad stream identifier magic')
|
||||
}
|
||||
return new_pos + chunk_len
|
||||
}
|
||||
|
||||
// read_chunk_header reads the 4-byte chunk header at `pos` and returns
|
||||
// (chunk_type, chunk_body_length, pos_after_header).
|
||||
@[inline]
|
||||
fn read_chunk_header(data []u8, pos int) !(u8, int, int) {
|
||||
if pos + 4 > data.len {
|
||||
return error('snappy framing: truncated chunk header at pos ${pos}')
|
||||
}
|
||||
chunk_type := data[pos]
|
||||
chunk_len := int(u32(data[pos + 1]) | u32(data[pos + 2]) << 8 | u32(data[pos + 3]) << 16)
|
||||
return chunk_type, chunk_len, pos + 4
|
||||
}
|
||||
|
||||
// decode_compressed_chunk decompresses a compressed data chunk payload
|
||||
// (CRC32C[4] + snappy_data[…]) and verifies the checksum.
|
||||
fn decode_compressed_chunk(payload []u8) ![]u8 {
|
||||
if payload.len < 4 {
|
||||
return error('snappy framing: compressed chunk too short')
|
||||
}
|
||||
stored_masked_crc := u32(payload[0]) | u32(payload[1]) << 8 | u32(payload[2]) << 16 | u32(payload[3]) << 24
|
||||
uncompressed := decompress(payload[4..])!
|
||||
actual_crc := mask_crc(crc32c(uncompressed))
|
||||
if actual_crc != stored_masked_crc {
|
||||
return error('snappy framing: CRC32C mismatch in compressed chunk')
|
||||
}
|
||||
return uncompressed
|
||||
}
|
||||
|
||||
// decode_uncompressed_chunk validates and returns the payload of an
|
||||
// uncompressed data chunk (CRC32C[4] + raw_data[…]).
|
||||
fn decode_uncompressed_chunk(payload []u8) ![]u8 {
|
||||
if payload.len < 4 {
|
||||
return error('snappy framing: uncompressed chunk too short')
|
||||
}
|
||||
stored_masked_crc := u32(payload[0]) | u32(payload[1]) << 8 | u32(payload[2]) << 16 | u32(payload[3]) << 24
|
||||
raw := payload[4..].clone()
|
||||
actual_crc := mask_crc(crc32c(raw))
|
||||
if actual_crc != stored_masked_crc {
|
||||
return error('snappy framing: CRC32C mismatch in uncompressed chunk')
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Little-endian integer writers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@[inline]
|
||||
fn write_u24le(mut out []u8, v int) {
|
||||
out << u8(v & 0xff)
|
||||
out << u8((v >> 8) & 0xff)
|
||||
out << u8((v >> 16) & 0xff)
|
||||
}
|
||||
|
||||
@[inline]
|
||||
fn write_u32le(mut out []u8, v u32) {
|
||||
out << u8(v & 0xff)
|
||||
out << u8((v >> 8) & 0xff)
|
||||
out << u8((v >> 16) & 0xff)
|
||||
out << u8((v >> 24) & 0xff)
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
module snappy
|
||||
|
||||
import arrays
|
||||
import io
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn stream_round_trip(label string, input []u8) ! {
|
||||
encoded := encode_stream(input)
|
||||
decoded := decode_stream(encoded)!
|
||||
|
||||
assert decoded == input, '${label}: stream round-trip mismatch (len=${input.len})'
|
||||
}
|
||||
|
||||
// drain_decoder closes dec and returns all decoded bytes accumulated so far.
|
||||
fn drain_decoder(mut dec StreamDecoder) []u8 {
|
||||
dec.close() or { panic(err) }
|
||||
mut collected := []u8{}
|
||||
mut chunk := []u8{len: 4096}
|
||||
for {
|
||||
n := dec.read(mut chunk) or { break }
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
collected << chunk[..n]
|
||||
}
|
||||
return collected
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CRC32C tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn test_crc32c_empty() {
|
||||
// Known value: CRC32C("") = 0x00000000
|
||||
assert crc32c([]) == u32(0x00000000)
|
||||
}
|
||||
|
||||
fn test_crc32c_known_vector() {
|
||||
// CRC32C("123456789") = 0xE3069283 — standard test vector.
|
||||
assert crc32c('123456789'.bytes()) == u32(0xe3069283)
|
||||
}
|
||||
|
||||
fn test_mask_unmask_roundtrip() {
|
||||
for v in [u32(0), u32(1), u32(0xdeadbeef), u32(0xffffffff)] {
|
||||
assert unmask_crc(mask_crc(v)) == v, 'mask/unmask failed for 0x${v:08x}'
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// One-shot encode / decode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn test_stream_empty() {
|
||||
stream_round_trip('empty', []u8{}) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_stream_single_byte() {
|
||||
stream_round_trip('single byte', [u8(0x42)]) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_stream_small() {
|
||||
stream_round_trip('hello world', 'Hello, World!'.bytes()) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_stream_compressible() {
|
||||
input := []u8{len: 8192, init: u8(index % 7)}
|
||||
stream_round_trip('compressible', input) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_stream_incompressible() {
|
||||
mut input := []u8{len: 2048}
|
||||
mut seed := u32(0xc0ffee42)
|
||||
for i in 0 .. input.len {
|
||||
seed = seed * 1664525 + 1013904223
|
||||
input[i] = u8(seed >> 24)
|
||||
}
|
||||
stream_round_trip('incompressible', input) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_stream_exactly_one_block() {
|
||||
// Exactly max_chunk_data_size bytes — exercises the boundary condition.
|
||||
input := []u8{len: max_chunk_data_size, init: u8(index & 0xff)}
|
||||
stream_round_trip('one-full-block', input) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_stream_multi_block() {
|
||||
// Spans multiple 64 KiB blocks.
|
||||
input := []u8{len: max_chunk_data_size * 3 + 100, init: u8(index & 0xff)}
|
||||
stream_round_trip('multi-block', input) or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_stream_identifier_present() {
|
||||
// The encoded stream must start with the identifier chunk.
|
||||
encoded := encode_stream('test'.bytes())
|
||||
assert encoded.len >= 10, 'stream too short to contain identifier'
|
||||
assert encoded[0] == chunk_type_stream_id, 'first byte must be stream identifier type'
|
||||
// The 6-byte magic must follow the 4-byte header.
|
||||
magic := encoded[4..10]
|
||||
assert magic == stream_identifier_magic, 'stream identifier magic mismatch'
|
||||
}
|
||||
|
||||
fn test_stream_crc_mismatch_detected() {
|
||||
mut encoded := encode_stream('hello'.bytes())
|
||||
// Corrupt a byte in the CRC field of the first data chunk (after the
|
||||
// 10-byte identifier: 4-byte header + 4-byte CRC starts at byte 14).
|
||||
if encoded.len > 14 {
|
||||
encoded[14] ^= 0xff
|
||||
}
|
||||
decode_stream(encoded) or { return } // expected error
|
||||
panic('expected decode_stream to fail on CRC mismatch')
|
||||
}
|
||||
|
||||
fn test_stream_missing_identifier() {
|
||||
// A stream that starts with a compressed chunk instead of the identifier.
|
||||
bad := [u8(chunk_type_compressed), 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]
|
||||
decode_stream(bad) or { return }
|
||||
panic('expected decode_stream to fail without identifier chunk')
|
||||
}
|
||||
|
||||
fn test_stream_unskippable_reserved_chunk() {
|
||||
mut s := encode_stream('hello'.bytes())
|
||||
// Inject a reserved unskippable chunk (type=0x02) before the data.
|
||||
injected := [u8(0x02), 0x01, 0x00, 0x00, u8(0x00)] // type + 3-byte len=1 + 1 payload byte
|
||||
// Insert after the 10-byte stream identifier.
|
||||
s = arrays.reduce([s[..10], injected, s[10..]], fn (acc []u8, x []u8) []u8 {
|
||||
mut out := acc.clone()
|
||||
out << x
|
||||
return out
|
||||
})!
|
||||
decode_stream(s) or { return }
|
||||
panic('expected decode_stream to fail on unskippable reserved chunk')
|
||||
}
|
||||
|
||||
fn test_stream_padding_skipped() {
|
||||
// Build a stream manually: identifier + padding + compressed data.
|
||||
mut s := []u8{}
|
||||
// Stream identifier
|
||||
s << chunk_type_stream_id
|
||||
write_u24le(mut s, stream_identifier_magic.len)
|
||||
s << stream_identifier_magic
|
||||
// Padding chunk (type=0xfe, length=4, 4 zero bytes)
|
||||
s << chunk_type_padding
|
||||
write_u24le(mut s, 4)
|
||||
s << [u8(0), 0, 0, 0]
|
||||
// A real data chunk
|
||||
inner := encode_stream('after padding'.bytes())
|
||||
// Skip the inner identifier (first 10 bytes) and append the data chunk.
|
||||
s << inner[10..]
|
||||
|
||||
decoded := decode_stream(s) or { panic(err) }
|
||||
assert decoded == 'after padding'.bytes(), 'padding chunk should be silently skipped'
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Incremental encoder
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn test_incremental_encoder_matches_oneshot() {
|
||||
input := []u8{len: max_chunk_data_size + 500, init: u8(index & 0xff)}
|
||||
|
||||
// One-shot
|
||||
oneshot := encode_stream(input)
|
||||
|
||||
// Incremental: feed in 1000-byte pieces via write/close/peek.
|
||||
mut enc := StreamEncoder{}
|
||||
mut pos := 0
|
||||
for pos < input.len {
|
||||
end := if pos + 1000 < input.len { pos + 1000 } else { input.len }
|
||||
enc.write(input[pos..end]) or { panic(err) }
|
||||
pos = end
|
||||
}
|
||||
enc.close()
|
||||
incremental := enc.peek()
|
||||
|
||||
assert incremental == oneshot, 'incremental encoder output differs from one-shot'
|
||||
}
|
||||
|
||||
fn test_incremental_encoder_decodable() {
|
||||
input := 'Streaming is fun!'.bytes().repeat(3000)
|
||||
|
||||
mut enc := StreamEncoder{}
|
||||
enc.write(input[..input.len / 2]) or { panic(err) }
|
||||
enc.write(input[input.len / 2..]) or { panic(err) }
|
||||
enc.close()
|
||||
stream := enc.peek()
|
||||
|
||||
decoded := decode_stream(stream) or { panic(err) }
|
||||
assert decoded == input, 'incremental encoder produced undecodable stream'
|
||||
}
|
||||
|
||||
fn test_incremental_encoder_write_after_close_errors() {
|
||||
// write() on a closed encoder must return an error.
|
||||
mut enc := StreamEncoder{}
|
||||
enc.write('hello'.bytes()) or { panic(err) }
|
||||
enc.close()
|
||||
enc.write('world'.bytes()) or { return } // expected error
|
||||
panic('expected write on closed encoder to fail')
|
||||
}
|
||||
|
||||
fn test_incremental_encoder_read_drains_output() {
|
||||
// read() should drain bytes from the encoded output incrementally.
|
||||
input := []u8{len: max_chunk_data_size + 100, init: u8(index & 0xff)}
|
||||
mut enc := StreamEncoder{}
|
||||
enc.write(input) or { panic(err) }
|
||||
enc.close()
|
||||
|
||||
// Drain via read() in small pieces and reassemble.
|
||||
mut collected := []u8{}
|
||||
mut chunk := []u8{len: 256}
|
||||
for {
|
||||
n := enc.read(mut chunk) or { break }
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
collected << chunk[..n]
|
||||
}
|
||||
|
||||
expected := encode_stream(input)
|
||||
assert collected == expected, 'read() did not produce the same bytes as encode_stream'
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Incremental decoder
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn test_incremental_decoder_byte_by_byte() {
|
||||
input := 'Byte by byte delivery'.bytes().repeat(500)
|
||||
stream := encode_stream(input)
|
||||
|
||||
mut dec := StreamDecoder{}
|
||||
for b in stream {
|
||||
dec.write([b]) or { panic(err) }
|
||||
}
|
||||
assert drain_decoder(mut dec) == input, 'incremental decoder failed (byte-by-byte)'
|
||||
}
|
||||
|
||||
fn test_incremental_decoder_large_chunks() {
|
||||
input := []u8{len: max_chunk_data_size * 2 + 99, init: u8(index % 13)}
|
||||
stream := encode_stream(input)
|
||||
|
||||
mut dec := StreamDecoder{}
|
||||
mut pos := 0
|
||||
for pos < stream.len {
|
||||
end := if pos + 8192 < stream.len { pos + 8192 } else { stream.len }
|
||||
dec.write(stream[pos..end]) or { panic(err) }
|
||||
pos = end
|
||||
}
|
||||
assert drain_decoder(mut dec) == input, 'incremental decoder failed (large chunks)'
|
||||
}
|
||||
|
||||
fn test_incremental_decoder_read_drains_output() {
|
||||
// A first read() call should return all decoded bytes; a second call on a
|
||||
// closed decoder with no remaining output should return io.Eof.
|
||||
stream := encode_stream('hello world'.bytes())
|
||||
mut dec := StreamDecoder{}
|
||||
dec.write(stream) or { panic(err) }
|
||||
dec.close() or { panic(err) }
|
||||
|
||||
mut buf := []u8{len: 4096}
|
||||
|
||||
// First read — should yield the decoded payload.
|
||||
n := dec.read(mut buf) or { panic(err) }
|
||||
assert buf[..n] == 'hello world'.bytes(), 'first read should return decoded bytes'
|
||||
|
||||
// Second read — decoder is closed and output is empty; expect Eof.
|
||||
dec.read(mut buf) or {
|
||||
assert err is io.Eof, 'expected io.Eof after output is drained, got: ${err}'
|
||||
return
|
||||
}
|
||||
panic('expected io.Eof on second read after output is drained')
|
||||
}
|
||||
|
||||
fn test_incremental_decoder_write_after_close_errors() {
|
||||
// write() on a closed decoder must return an error.
|
||||
stream := encode_stream('hello'.bytes())
|
||||
mut dec := StreamDecoder{}
|
||||
dec.write(stream) or { panic(err) }
|
||||
dec.close() or { panic(err) }
|
||||
dec.write(stream) or { return } // expected error
|
||||
panic('expected write on closed decoder to fail')
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
module snappy
|
||||
|
||||
import os
|
||||
import benchmark
|
||||
|
||||
const samples_folder = os.join_path(os.dir(@FILE), 'samples')
|
||||
|
||||
fn s(fname string) string {
|
||||
return os.join_path(samples_folder, fname)
|
||||
}
|
||||
|
||||
fn test_framing_compress_decompress_alice() {
|
||||
alice := os.read_bytes(s('alice29.txt'))!
|
||||
|
||||
alice_compressed := encode_stream(alice)
|
||||
assert decode_stream(alice_compressed)! == alice, 'compressed alice should be the same as decompressed'
|
||||
}
|
||||
|
||||
fn test_precompressed_framing_alice() {
|
||||
alice_compressed := os.read_bytes(s('alice29_framing.snappy'))!
|
||||
alice := os.read_bytes(s('alice29.txt'))!
|
||||
|
||||
// Normalize line endings before comparing
|
||||
decompressed_str := decode_stream(alice_compressed)!.bytestr().replace('\r\n', '\n')
|
||||
alice_str := alice.bytestr().replace('\r\n', '\n')
|
||||
|
||||
assert decompressed_str == alice_str, 'compressed alice should be the same as decompressed'
|
||||
}
|
||||
|
||||
fn test_benchmark_framing_alice() {
|
||||
alice := os.read_bytes(s('alice29.txt'))!
|
||||
|
||||
mut b := benchmark.start()
|
||||
|
||||
encode_stream(alice)
|
||||
|
||||
b.measure('framing_alice')
|
||||
}
|
||||
|
||||
fn test_benchmark_block_alice() {
|
||||
alice := os.read_bytes(s('alice29.txt'))!
|
||||
|
||||
mut b := benchmark.start()
|
||||
|
||||
compress(alice)
|
||||
|
||||
b.measure('block_alice')
|
||||
}
|
||||
|
||||
fn test_block_compress_decompress_alice() {
|
||||
alice := os.read_bytes(s('alice29.txt'))!
|
||||
|
||||
alice_compressed := compress(alice)
|
||||
assert decompress(alice_compressed)! == alice, 'compressed alice should be the same as decompressed'
|
||||
}
|
||||
|
||||
fn test_precompressed_block_alice() {
|
||||
alice_compressed := os.read_bytes(s('alice29_block.snappy'))!
|
||||
alice := os.read_bytes(s('alice29.txt'))!
|
||||
|
||||
decompressed := decompress(alice_compressed)!
|
||||
|
||||
// Normalize line endings before comparing
|
||||
decompressed_str := decompressed.bytestr().replace('\r\n', '\n')
|
||||
alice_str := alice.bytestr().replace('\r\n', '\n')
|
||||
|
||||
assert decompressed_str == alice_str, 'compressed alice should be the same as decompressed'
|
||||
}
|
||||
Reference in new issue
Block a user