Initial project import
This commit is contained in:
commit
23f2d29e32
11957 files changed
+1754577
No files matched your search
@@ -0,0 +1,37 @@
|
||||
## Description
|
||||
|
||||
`compress.deflate` is a pure V RFC-compliant DEFLATE module.
|
||||
|
||||
Compression output format is selected by `CompressFormat` via
|
||||
`compress(data, format: ...)`:
|
||||
|
||||
- `.zlib` (RFC 1950 wrapper)
|
||||
- `.gzip` (RFC 1952 wrapper)
|
||||
- `.raw_deflate` (RFC 1951 raw stream)
|
||||
|
||||
`compress` keeps default zlib behavior, and `decompress` auto-detects all three.
|
||||
|
||||
## Interop Validation
|
||||
|
||||
Cross-validation with C/zlib is kept separate from `v test` and can be run manually:
|
||||
|
||||
```bash
|
||||
./vnew run vlib/compress/deflate/interop/deflate_interop.v
|
||||
```
|
||||
|
||||
## Example
|
||||
|
||||
```v
|
||||
import compress.deflate
|
||||
|
||||
fn main() {
|
||||
uncompressed := 'Hello world!'
|
||||
zlib_stream := deflate.compress(uncompressed.bytes())!
|
||||
gzip_stream := deflate.compress(uncompressed.bytes(), format: .gzip)!
|
||||
raw_stream := deflate.compress(uncompressed.bytes(), format: .raw_deflate)!
|
||||
assert deflate.decompress(zlib_stream)! == uncompressed.bytes()
|
||||
assert deflate.decompress(gzip_stream)! == uncompressed.bytes()
|
||||
decompressed := deflate.decompress(raw_stream)!
|
||||
assert decompressed == uncompressed.bytes()
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,291 @@
|
||||
module deflate
|
||||
|
||||
import encoding.binary
|
||||
import hash.adler32
|
||||
import hash.crc32
|
||||
|
||||
pub type ChunkCallback = fn (chunk []u8, userdata voidptr) int
|
||||
|
||||
// CompressFormat selects the output container around the RFC 1951 payload.
|
||||
pub enum CompressFormat {
|
||||
zlib
|
||||
gzip
|
||||
raw_deflate
|
||||
}
|
||||
|
||||
@[params]
|
||||
pub struct CompressParams {
|
||||
pub:
|
||||
format CompressFormat = .zlib
|
||||
}
|
||||
|
||||
pub struct RawInflateResult {
|
||||
pub:
|
||||
decoded []u8
|
||||
consumed int
|
||||
}
|
||||
|
||||
pub struct ZlibHeader {
|
||||
pub:
|
||||
payload_start int = 2
|
||||
}
|
||||
|
||||
pub struct GzipHeader {
|
||||
pub mut:
|
||||
flags u8
|
||||
payload_start int
|
||||
extra []u8
|
||||
filename []u8
|
||||
comment []u8
|
||||
modification_time u32
|
||||
operating_system u8
|
||||
}
|
||||
|
||||
// validate_zlib_header validates a RFC 1950 zlib header.
|
||||
@[direct_array_access]
|
||||
pub fn validate_zlib_header(data []u8) !ZlibHeader {
|
||||
if data.len < 6 {
|
||||
return error('invalid zlib stream: too short')
|
||||
}
|
||||
if data[0] & 0x0f != 8 {
|
||||
return error('invalid zlib stream: unsupported compression method')
|
||||
}
|
||||
if (u32(data[0]) * 256 + u32(data[1])) % 31 != 0 {
|
||||
return error('invalid zlib stream: bad header checksum')
|
||||
}
|
||||
if data[1] & 0x20 != 0 {
|
||||
return error('invalid zlib stream: preset dictionary not supported')
|
||||
}
|
||||
return ZlibHeader{}
|
||||
}
|
||||
|
||||
// validate_gzip_header validates a RFC 1952 gzip header and returns parsed fields.
|
||||
@[direct_array_access]
|
||||
pub fn validate_gzip_header(data []u8) !GzipHeader {
|
||||
if data.len < 18 {
|
||||
return error('invalid gzip stream: too short')
|
||||
}
|
||||
if data[0] != 0x1f || data[1] != 0x8b {
|
||||
return error('invalid gzip stream: bad magic')
|
||||
}
|
||||
if data[2] != 8 {
|
||||
return error('invalid gzip stream: unsupported compression method')
|
||||
}
|
||||
flg := data[3]
|
||||
if flg & 0xe0 != 0 {
|
||||
return error('invalid gzip stream: reserved flags set')
|
||||
}
|
||||
mut header := GzipHeader{
|
||||
flags: flg
|
||||
payload_start: 10
|
||||
modification_time: binary.little_endian_u32_at(data, 4)
|
||||
operating_system: data[9]
|
||||
}
|
||||
if flg & 0x04 != 0 {
|
||||
if header.payload_start + 2 > data.len {
|
||||
return error('invalid gzip stream: truncated extra')
|
||||
}
|
||||
xlen := int(u32(data[header.payload_start]) | u32(data[header.payload_start + 1]) << 8)
|
||||
header.payload_start += 2
|
||||
if header.payload_start + xlen > data.len {
|
||||
return error('invalid gzip stream: truncated extra')
|
||||
}
|
||||
header.extra = data[header.payload_start..header.payload_start + xlen]
|
||||
header.payload_start += xlen
|
||||
}
|
||||
if flg & 0x08 != 0 {
|
||||
for header.payload_start < data.len && data[header.payload_start] != 0 {
|
||||
header.filename << data[header.payload_start]
|
||||
header.payload_start++
|
||||
}
|
||||
header.payload_start++
|
||||
}
|
||||
if flg & 0x10 != 0 {
|
||||
for header.payload_start < data.len && data[header.payload_start] != 0 {
|
||||
header.comment << data[header.payload_start]
|
||||
header.payload_start++
|
||||
}
|
||||
header.payload_start++
|
||||
}
|
||||
if flg & 0x02 != 0 {
|
||||
if header.payload_start + 2 > data.len {
|
||||
return error('invalid gzip stream: truncated fhcrc')
|
||||
}
|
||||
expected_crc16 := u16(data[header.payload_start]) | (u16(data[header.payload_start + 1]) << 8)
|
||||
actual_crc16 := u16(crc32.sum(data[..header.payload_start]) & 0xffff)
|
||||
if actual_crc16 != expected_crc16 {
|
||||
return error('invalid gzip stream: header crc16 mismatch')
|
||||
}
|
||||
header.payload_start += 2
|
||||
}
|
||||
if header.payload_start + 8 > data.len {
|
||||
return error('invalid gzip stream: truncated payload')
|
||||
}
|
||||
return header
|
||||
}
|
||||
|
||||
// compress compresses data as zlib, gzip, or raw DEFLATE.
|
||||
pub fn compress(data []u8, format CompressParams) ![]u8 {
|
||||
return match format.format {
|
||||
.zlib { compress_zlib(data) }
|
||||
.gzip { compress_gzip(data) }
|
||||
.raw_deflate { deflate_compress_fixed(data)! }
|
||||
}
|
||||
}
|
||||
|
||||
pub fn compress_zlib(data []u8) ![]u8 {
|
||||
payload := deflate_compress_fixed(data)!
|
||||
cksum := adler32.sum(data)
|
||||
mut out := []u8{cap: 2 + payload.len + 4}
|
||||
out << u8(0x78) // CMF: CM=8 deflate, CINFO=7 (32K window)
|
||||
out << u8(0x9c) // FLG: default compression, FCHECK satisfies (CMF*256+FLG)%31==0
|
||||
out << payload
|
||||
out << binary.big_endian_get_u32(cksum)
|
||||
return out
|
||||
}
|
||||
|
||||
// compress_gzip compresses data into a gzip stream (RFC 1952).
|
||||
pub fn compress_gzip(data []u8) ![]u8 {
|
||||
payload := deflate_compress_fixed(data)!
|
||||
mut out := []u8{cap: 10 + payload.len + 8}
|
||||
// 10-byte gzip header: ID1 ID2 CM FLG MTIME(4) XFL OS
|
||||
out << [u8(0x1f), 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff]
|
||||
out << payload
|
||||
out << binary.little_endian_get_u32(crc32.sum(data))
|
||||
out << binary.little_endian_get_u32(u32(data.len))
|
||||
return out
|
||||
}
|
||||
|
||||
// compress_raw compresses data to a raw RFC 1951 DEFLATE stream.
|
||||
pub fn compress_raw(data []u8) ![]u8 {
|
||||
return deflate_compress_fixed(data)!
|
||||
}
|
||||
|
||||
// decompress decompresses a zlib (RFC 1950), gzip (RFC 1952), or raw DEFLATE (RFC 1951) stream.
|
||||
// The format is auto-detected.
|
||||
pub fn decompress(data []u8) ![]u8 {
|
||||
if data.len >= 2 {
|
||||
// gzip magic: 0x1f 0x8b
|
||||
if data[0] == 0x1f && data[1] == 0x8b {
|
||||
return decompress_gzip(data)
|
||||
}
|
||||
// zlib: CM=8 and header checksum passes
|
||||
if data[0] & 0x0f == 8 && (u32(data[0]) * 256 + u32(data[1])) % 31 == 0 {
|
||||
return decompress_zlib(data)
|
||||
}
|
||||
}
|
||||
// raw DEFLATE
|
||||
return inflate(data)
|
||||
}
|
||||
|
||||
// decompress_zlib decompresses a zlib stream (RFC 1950).
|
||||
// It returns the decompressed bytes in a new array.
|
||||
pub fn decompress_zlib(data []u8) ![]u8 {
|
||||
header := validate_zlib_header(data)!
|
||||
payload := data[header.payload_start..data.len - 4]
|
||||
expected := binary.big_endian_u32_at(data, data.len - 4)
|
||||
res := inflate_with_consumed(payload)!
|
||||
if res.consumed != payload.len {
|
||||
return error('invalid zlib stream: trailing data before adler32')
|
||||
}
|
||||
decoded := res.decoded
|
||||
if adler32.sum(decoded) != expected { return error('invalid zlib stream: adler32 mismatch') }
|
||||
return decoded
|
||||
}
|
||||
|
||||
// decompress_gzip decompresses a gzip stream (RFC 1952).
|
||||
// It returns the decompressed bytes in a new array.
|
||||
pub fn decompress_gzip(data []u8) ![]u8 {
|
||||
header := validate_gzip_header(data)!
|
||||
payload := data[header.payload_start..data.len - 8]
|
||||
expected_crc := binary.little_endian_u32_at(data, data.len - 8)
|
||||
expected_size := binary.little_endian_u32_at(data, data.len - 4)
|
||||
res := inflate_with_consumed(payload)!
|
||||
if res.consumed != payload.len {
|
||||
return error('invalid gzip stream: trailing data before trailer')
|
||||
}
|
||||
decoded := res.decoded
|
||||
if crc32.sum(decoded) != expected_crc {
|
||||
return error('invalid gzip stream: crc32 mismatch')
|
||||
}
|
||||
if u32(decoded.len) != expected_size {
|
||||
return error('invalid gzip stream: size mismatch')
|
||||
}
|
||||
return decoded
|
||||
}
|
||||
|
||||
// decompress_raw_with_consumed decompresses raw RFC 1951 DEFLATE data and tracks consumed bytes.
|
||||
pub fn decompress_raw_with_consumed(data []u8) !RawInflateResult {
|
||||
res := inflate_with_consumed(data)!
|
||||
return RawInflateResult{
|
||||
decoded: res.decoded
|
||||
consumed: res.consumed
|
||||
}
|
||||
}
|
||||
|
||||
// decompress_with_callback decompresses a zlib/gzip/raw stream (RFC 1950, RFC 1952) using a callback for chunked delivery.
|
||||
// The callback receives chunks of decompressed data and should return the chunk length to continue, or 0 to abort.
|
||||
// Returns the total decompressed length.
|
||||
pub fn decompress_with_callback(data []u8, cb ChunkCallback, userdata voidptr) !int {
|
||||
if data.len >= 2 {
|
||||
// gzip magic: 0x1f 0x8b
|
||||
if data[0] == 0x1f && data[1] == 0x8b {
|
||||
return decompress_gzip_with_callback(data, cb, userdata)
|
||||
}
|
||||
// zlib: CM=8 and header checksum passes
|
||||
if data[0] & 0x0f == 8 && (u32(data[0]) * 256 + u32(data[1])) % 31 == 0 {
|
||||
return decompress_zlib_with_callback(data, cb, userdata)
|
||||
}
|
||||
}
|
||||
// raw DEFLATE
|
||||
res := inflate_with_callback(data, cb, userdata)!
|
||||
return res.delivered
|
||||
}
|
||||
|
||||
fn decompress_zlib_with_callback(data []u8, cb ChunkCallback, userdata voidptr) !int {
|
||||
header := validate_zlib_header(data)!
|
||||
payload := data[header.payload_start..data.len - 4]
|
||||
expected := binary.big_endian_u32_at(data, data.len - 4)
|
||||
res := inflate_with_callback(payload, cb, userdata)!
|
||||
if res.aborted {
|
||||
return res.delivered
|
||||
}
|
||||
if res.consumed != payload.len {
|
||||
return error('invalid zlib stream: trailing data before adler32')
|
||||
}
|
||||
if adler32.sum(res.decoded) != expected {
|
||||
return error('invalid zlib stream: adler32 mismatch')
|
||||
}
|
||||
return res.delivered
|
||||
}
|
||||
|
||||
fn decompress_gzip_with_callback(data []u8, cb ChunkCallback, userdata voidptr) !int {
|
||||
header := validate_gzip_header(data)!
|
||||
payload := data[header.payload_start..data.len - 8]
|
||||
expected_crc := binary.little_endian_u32_at(data, data.len - 8)
|
||||
expected_size := binary.little_endian_u32_at(data, data.len - 4)
|
||||
res := inflate_with_callback(payload, cb, userdata)!
|
||||
if res.aborted {
|
||||
return res.delivered
|
||||
}
|
||||
if res.consumed != payload.len {
|
||||
return error('invalid gzip stream: trailing data before trailer')
|
||||
}
|
||||
if crc32.sum(res.decoded) != expected_crc {
|
||||
return error('invalid gzip stream: crc32 mismatch')
|
||||
}
|
||||
if u32(res.decoded.len) != expected_size {
|
||||
return error('invalid gzip stream: size mismatch')
|
||||
}
|
||||
return res.delivered
|
||||
}
|
||||
|
||||
fn bit_reverse(v u32, n int) u32 {
|
||||
mut r := u32(0)
|
||||
mut val := v
|
||||
for _ in 0 .. n {
|
||||
r = (r << 1) | (val & 1)
|
||||
val >>= 1
|
||||
}
|
||||
return r
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
module deflate
|
||||
|
||||
import hash.huffman
|
||||
|
||||
const deflate_hash_bits = 15
|
||||
const deflate_hash_size = 1 << deflate_hash_bits
|
||||
const deflate_max_chain = 64
|
||||
const deflate_min_match = 3
|
||||
const deflate_max_match = 258
|
||||
const deflate_window = 32768
|
||||
|
||||
// fixed_litlen_encode returns (reversed_codes, code_lengths) for fixed Huffman lit/len.
|
||||
// The LSB-first (bit-reversed) codes come straight from the shared canonical
|
||||
// builder, since the encoder writes bits LSB-first.
|
||||
fn fixed_litlen_encode() !([]u32, []int) {
|
||||
lens := fixed_litlen_lengths()
|
||||
t := huffman.build(lengths: lens, max_bits: 9, bit_order: .lsb_first)!
|
||||
return t.codes, lens
|
||||
}
|
||||
|
||||
// fixed_dist_encode returns (reversed_codes, code_lengths) for fixed Huffman distance.
|
||||
fn fixed_dist_encode() ([]u32, []int) {
|
||||
mut codes := []u32{len: 30}
|
||||
for i in 0 .. 30 {
|
||||
codes[i] = bit_reverse(u32(i), 5)
|
||||
}
|
||||
return codes, []int{len: 30, init: 5}
|
||||
}
|
||||
|
||||
fn length_code_info(length int) (int, int, int) {
|
||||
for i := length_bases.len - 1; i >= 0; i-- {
|
||||
if length >= length_bases[i] {
|
||||
return i, length - length_bases[i], length_extra_bits[i]
|
||||
}
|
||||
}
|
||||
return 0, 0, 0
|
||||
}
|
||||
|
||||
fn dist_code_info(distance int) (int, int, int) {
|
||||
for i := dist_bases.len - 1; i >= 0; i-- {
|
||||
if distance >= dist_bases[i] {
|
||||
return i, distance - dist_bases[i], dist_extra_bits[i]
|
||||
}
|
||||
}
|
||||
return 0, 0, 0
|
||||
}
|
||||
|
||||
fn hash3(data []u8, pos int) int {
|
||||
v := u32(data[pos]) | (u32(data[pos + 1]) << 8) | (u32(data[pos + 2]) << 16)
|
||||
return int((v * u32(2654435761)) >> u32(32 - deflate_hash_bits))
|
||||
}
|
||||
|
||||
@[direct_array_access]
|
||||
fn find_lz_match(data []u8, pos int, last []int, prev []int) (int, int) {
|
||||
if pos + deflate_min_match > data.len {
|
||||
return 0, 0
|
||||
}
|
||||
max_len := if pos + deflate_max_match < data.len { deflate_max_match } else { data.len - pos }
|
||||
mut best_len := 0
|
||||
mut best_off := 0
|
||||
mut i := last[hash3(data, pos)]
|
||||
mut chain := 0
|
||||
for i >= 0 && chain < deflate_max_chain {
|
||||
off := pos - i
|
||||
if off > deflate_window {
|
||||
break
|
||||
}
|
||||
mut l := 0
|
||||
for l < max_len && data[i + l] == data[pos + l] {
|
||||
l++
|
||||
}
|
||||
if l > best_len {
|
||||
best_len = l
|
||||
best_off = off
|
||||
if best_len == max_len {
|
||||
break
|
||||
}
|
||||
}
|
||||
i = prev[i]
|
||||
chain++
|
||||
}
|
||||
return best_off, best_len
|
||||
}
|
||||
|
||||
struct BitWriter {
|
||||
mut:
|
||||
buf []u8
|
||||
bits u32
|
||||
nbits int
|
||||
}
|
||||
|
||||
@[direct_array_access; inline]
|
||||
fn (mut w BitWriter) write_bits(value u32, nbits int) {
|
||||
if nbits == 0 {
|
||||
return
|
||||
}
|
||||
w.bits |= value << w.nbits
|
||||
w.nbits += nbits
|
||||
for w.nbits >= 8 {
|
||||
w.buf << u8(w.bits & 0xff)
|
||||
w.bits >>= 8
|
||||
w.nbits -= 8
|
||||
}
|
||||
}
|
||||
|
||||
fn (mut w BitWriter) flush() {
|
||||
if w.nbits > 0 {
|
||||
w.buf << u8(w.bits & 0xff)
|
||||
w.bits = 0
|
||||
w.nbits = 0
|
||||
}
|
||||
}
|
||||
|
||||
// deflate_compress_fixed compresses data to RFC 1951 DEFLATE using fixed Huffman codes.
|
||||
@[direct_array_access]
|
||||
fn deflate_compress_fixed(data []u8) ![]u8 {
|
||||
ll_codes, ll_lens := fixed_litlen_encode()!
|
||||
d_codes, d_lens := fixed_dist_encode()
|
||||
mut w := BitWriter{}
|
||||
// BFINAL=1, BTYPE=01 (fixed Huffman)
|
||||
w.write_bits(1, 1)
|
||||
w.write_bits(1, 2)
|
||||
if data.len == 0 {
|
||||
w.write_bits(ll_codes[256], ll_lens[256])
|
||||
w.flush()
|
||||
return w.buf
|
||||
}
|
||||
mut last := []int{len: deflate_hash_size, init: -1}
|
||||
mut prev := []int{len: data.len, init: -1}
|
||||
mut pos := 0
|
||||
for pos < data.len {
|
||||
off, match_len := find_lz_match(data, pos, last, prev)
|
||||
if match_len >= deflate_min_match {
|
||||
li, lext, lext_bits := length_code_info(match_len)
|
||||
sym := 257 + li
|
||||
w.write_bits(ll_codes[sym], ll_lens[sym])
|
||||
w.write_bits(u32(lext), lext_bits)
|
||||
di, dext, dext_bits := dist_code_info(off)
|
||||
w.write_bits(d_codes[di], d_lens[di])
|
||||
w.write_bits(u32(dext), dext_bits)
|
||||
for i in pos .. pos + match_len {
|
||||
if i + deflate_min_match < data.len {
|
||||
h := hash3(data, i)
|
||||
prev[i] = last[h]
|
||||
last[h] = i
|
||||
}
|
||||
}
|
||||
pos += match_len
|
||||
} else {
|
||||
w.write_bits(ll_codes[int(data[pos])], ll_lens[int(data[pos])])
|
||||
if pos + deflate_min_match < data.len {
|
||||
h := hash3(data, pos)
|
||||
prev[pos] = last[h]
|
||||
last[h] = pos
|
||||
}
|
||||
pos++
|
||||
}
|
||||
}
|
||||
w.write_bits(ll_codes[256], ll_lens[256])
|
||||
w.flush()
|
||||
return w.buf
|
||||
}
|
||||
@@ -0,0 +1,429 @@
|
||||
module deflate
|
||||
|
||||
import hash.huffman
|
||||
|
||||
// vfmt off
|
||||
// RFC 1951 length/distance decode tables
|
||||
const length_bases = [3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59,
|
||||
67, 83, 99, 115, 131, 163, 195, 227, 258]
|
||||
const length_extra_bits = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4,
|
||||
4, 5, 5, 5, 5, 0]
|
||||
const dist_bases = [1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769,
|
||||
1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577]
|
||||
const dist_extra_bits = [0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10,
|
||||
10, 11, 11, 12, 12, 13, 13]
|
||||
// code-length alphabet order (RFC 1951)
|
||||
const cl_order = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
|
||||
// vmt on
|
||||
|
||||
// fixed_litlen_lengths returns code lengths for the fixed Huffman lit/len tree (RFC 1951 §3.2.6).
|
||||
fn fixed_litlen_lengths() []int {
|
||||
mut lens := []int{len: 288}
|
||||
for i in 0 .. 144 {
|
||||
lens[i] = 8
|
||||
}
|
||||
for i in 144 .. 256 {
|
||||
lens[i] = 9
|
||||
}
|
||||
for i in 256 .. 280 {
|
||||
lens[i] = 7
|
||||
}
|
||||
for i in 280 .. 288 {
|
||||
lens[i] = 8
|
||||
}
|
||||
return lens
|
||||
}
|
||||
|
||||
// HuffTree is a MSB-first Huffman lookup table for DEFLATE decoding.
|
||||
// Indexed by the next max_bits bits read LSB-first from the stream.
|
||||
struct HuffTree {
|
||||
table []u32 // entry: (symbol << 5) | code_length; 0xFFFF_FFFF = invalid
|
||||
max_bits int
|
||||
}
|
||||
|
||||
fn build_huff_tree(lengths []int) !HuffTree {
|
||||
mut max_bits := 0
|
||||
for l in lengths {
|
||||
if l > max_bits {
|
||||
max_bits = l
|
||||
}
|
||||
}
|
||||
if max_bits == 0 {
|
||||
return HuffTree{
|
||||
table: [u32(0)]
|
||||
max_bits: 0
|
||||
}
|
||||
}
|
||||
// Build the flat LSB-first lookup table the decode loop expects directly
|
||||
// from the lengths via the shared canonical builder: entry = (symbol << 5) |
|
||||
// length, 0xFFFF_FFFF for invalid. flat_table() allocates no intermediate
|
||||
// codes array, so this matches the hand-rolled loop's cost. Over-subscribed
|
||||
// lengths now surface as an error instead of a silently corrupt table.
|
||||
table := huffman.flat_table(lengths: lengths, max_bits: max_bits, bit_order: .lsb_first)!
|
||||
return HuffTree{
|
||||
table: table
|
||||
max_bits: max_bits
|
||||
}
|
||||
}
|
||||
|
||||
struct BitReader {
|
||||
buf []u8
|
||||
mut:
|
||||
pos int
|
||||
bits u32
|
||||
nbits int
|
||||
}
|
||||
|
||||
@[direct_array_access; inline]
|
||||
fn (mut r BitReader) read_bits(n int) !u32 {
|
||||
for r.nbits < n {
|
||||
if r.pos >= r.buf.len {
|
||||
return error('inflate: unexpected end of stream')
|
||||
}
|
||||
r.bits |= u32(r.buf[r.pos]) << r.nbits
|
||||
r.pos++
|
||||
r.nbits += 8
|
||||
}
|
||||
val := r.bits & ((u32(1) << n) - 1)
|
||||
r.bits >>= u32(n)
|
||||
r.nbits -= n
|
||||
return val
|
||||
}
|
||||
|
||||
@[inline]
|
||||
fn (mut r BitReader) align_byte() {
|
||||
r.bits = 0
|
||||
r.nbits = 0
|
||||
}
|
||||
|
||||
@[direct_array_access; inline]
|
||||
fn (mut r BitReader) read_byte_raw() !u8 {
|
||||
if r.pos >= r.buf.len {
|
||||
return error('inflate: unexpected end of stream')
|
||||
}
|
||||
b := r.buf[r.pos]
|
||||
r.pos++
|
||||
return b
|
||||
}
|
||||
|
||||
@[direct_array_access; inline]
|
||||
fn (mut r BitReader) huff_decode(t HuffTree) !u32 {
|
||||
for r.nbits < t.max_bits {
|
||||
if r.pos >= r.buf.len {
|
||||
break
|
||||
}
|
||||
r.bits |= u32(r.buf[r.pos]) << r.nbits
|
||||
r.pos++
|
||||
r.nbits += 8
|
||||
}
|
||||
idx := int(r.bits & ((u32(1) << t.max_bits) - 1))
|
||||
entry := t.table[idx]
|
||||
if entry == 0xffff_ffff {
|
||||
return error('inflate: invalid Huffman code')
|
||||
}
|
||||
len_ := int(entry & 0x1f)
|
||||
if r.nbits < len_ {
|
||||
return error('inflate: unexpected end of stream')
|
||||
}
|
||||
sym := entry >> 5
|
||||
r.bits >>= u32(len_)
|
||||
r.nbits -= len_
|
||||
return sym
|
||||
}
|
||||
|
||||
struct InflateResult {
|
||||
decoded []u8
|
||||
consumed int
|
||||
}
|
||||
|
||||
struct InflateStreamResult {
|
||||
decoded []u8
|
||||
consumed int
|
||||
delivered int
|
||||
aborted bool
|
||||
}
|
||||
|
||||
struct InflateStreamState {
|
||||
mut:
|
||||
delivered int
|
||||
}
|
||||
|
||||
const inflate_callback_chunk_size = 32768
|
||||
|
||||
@[direct_array_access; inline]
|
||||
fn flush_stream_chunks(out []u8, cb ChunkCallback, userdata voidptr, mut state InflateStreamState) bool {
|
||||
for out.len - state.delivered >= inflate_callback_chunk_size {
|
||||
end := state.delivered + inflate_callback_chunk_size
|
||||
chunk := out[state.delivered..end]
|
||||
if cb(chunk, userdata) != chunk.len {
|
||||
return false
|
||||
}
|
||||
state.delivered = end
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// inflate_with_consumed decompresses raw RFC 1951 DEFLATE data and reports
|
||||
// how many input bytes were consumed by the DEFLATE bitstream.
|
||||
fn inflate_with_consumed(data []u8) !InflateResult {
|
||||
mut r := BitReader{
|
||||
buf: data
|
||||
}
|
||||
mut out := []u8{}
|
||||
fixed_ll := build_huff_tree(fixed_litlen_lengths())!
|
||||
fixed_d := build_huff_tree([]int{len: 32, init: 5})!
|
||||
for {
|
||||
bfinal := r.read_bits(1)!
|
||||
btype := r.read_bits(2)!
|
||||
match btype {
|
||||
0 {
|
||||
r.align_byte()
|
||||
len_ := int(r.read_byte_raw()!) | (int(u32(r.read_byte_raw()!) << 8))
|
||||
nlen := int(r.read_byte_raw()!) | (int(u32(r.read_byte_raw()!) << 8))
|
||||
if len_ & 0xffff != (~nlen) & 0xffff {
|
||||
return error('inflate: bad stored block length')
|
||||
}
|
||||
for _ in 0 .. len_ {
|
||||
out << r.read_byte_raw()!
|
||||
}
|
||||
}
|
||||
1 {
|
||||
inflate_block(mut r, mut out, fixed_ll, fixed_d)!
|
||||
}
|
||||
2 {
|
||||
hlit := int(r.read_bits(5)!) + 257
|
||||
hdist := int(r.read_bits(5)!) + 1
|
||||
hclen := int(r.read_bits(4)!) + 4
|
||||
mut cl_lens := []int{len: 19}
|
||||
for i in 0 .. hclen {
|
||||
cl_lens[cl_order[i]] = int(r.read_bits(3)!)
|
||||
}
|
||||
cl_tree := build_huff_tree(cl_lens)!
|
||||
mut all_lens := []int{}
|
||||
for all_lens.len < hlit + hdist {
|
||||
sym := r.huff_decode(cl_tree)!
|
||||
if sym <= 15 {
|
||||
all_lens << int(sym)
|
||||
} else if sym == 16 {
|
||||
if all_lens.len == 0 {
|
||||
return error('inflate: repeat with empty history')
|
||||
}
|
||||
rep := int(r.read_bits(2)!) + 3
|
||||
last := all_lens[all_lens.len - 1]
|
||||
for _ in 0 .. rep {
|
||||
all_lens << last
|
||||
}
|
||||
} else if sym == 17 {
|
||||
rep := int(r.read_bits(3)!) + 3
|
||||
for _ in 0 .. rep {
|
||||
all_lens << 0
|
||||
}
|
||||
} else if sym == 18 {
|
||||
rep := int(r.read_bits(7)!) + 11
|
||||
for _ in 0 .. rep {
|
||||
all_lens << 0
|
||||
}
|
||||
} else {
|
||||
return error('inflate: bad code length symbol')
|
||||
}
|
||||
}
|
||||
ll_tree := build_huff_tree(all_lens[..hlit])!
|
||||
d_tree := build_huff_tree(all_lens[hlit..])!
|
||||
inflate_block(mut r, mut out, ll_tree, d_tree)!
|
||||
}
|
||||
else {
|
||||
return error('inflate: reserved block type')
|
||||
}
|
||||
}
|
||||
|
||||
if bfinal == 1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
consumed := r.pos - (r.nbits >> 3)
|
||||
return InflateResult{
|
||||
decoded: out
|
||||
consumed: consumed
|
||||
}
|
||||
}
|
||||
|
||||
// inflate decompresses raw RFC 1951 DEFLATE data.
|
||||
fn inflate(data []u8) ![]u8 {
|
||||
res := inflate_with_consumed(data)!
|
||||
return res.decoded
|
||||
}
|
||||
|
||||
// inflate_with_callback decompresses raw RFC 1951 DEFLATE data and streams output through `cb`.
|
||||
// It still tracks consumed input bytes for container-level validation.
|
||||
fn inflate_with_callback(data []u8, cb ChunkCallback, userdata voidptr) !InflateStreamResult {
|
||||
mut r := BitReader{
|
||||
buf: data
|
||||
}
|
||||
mut out := []u8{}
|
||||
mut state := InflateStreamState{}
|
||||
mut aborted := false
|
||||
fixed_ll := build_huff_tree(fixed_litlen_lengths())!
|
||||
fixed_d := build_huff_tree([]int{len: 32, init: 5})!
|
||||
for {
|
||||
bfinal := r.read_bits(1)!
|
||||
btype := r.read_bits(2)!
|
||||
match btype {
|
||||
0 {
|
||||
r.align_byte()
|
||||
len_ := int(r.read_byte_raw()!) | (int(u32(r.read_byte_raw()!) << 8))
|
||||
nlen := int(r.read_byte_raw()!) | (int(u32(r.read_byte_raw()!) << 8))
|
||||
if len_ & 0xffff != (~nlen) & 0xffff {
|
||||
return error('inflate: bad stored block length')
|
||||
}
|
||||
for _ in 0 .. len_ {
|
||||
out << r.read_byte_raw()!
|
||||
if !flush_stream_chunks(out, cb, userdata, mut state) {
|
||||
aborted = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
1 {
|
||||
if !inflate_block_stream(mut r, mut out, fixed_ll, fixed_d, cb, userdata, mut state)! {
|
||||
aborted = true
|
||||
}
|
||||
}
|
||||
2 {
|
||||
hlit := int(r.read_bits(5)!) + 257
|
||||
hdist := int(r.read_bits(5)!) + 1
|
||||
hclen := int(r.read_bits(4)!) + 4
|
||||
mut cl_lens := []int{len: 19}
|
||||
for i in 0 .. hclen {
|
||||
cl_lens[cl_order[i]] = int(r.read_bits(3)!)
|
||||
}
|
||||
cl_tree := build_huff_tree(cl_lens)!
|
||||
mut all_lens := []int{}
|
||||
for all_lens.len < hlit + hdist {
|
||||
sym := r.huff_decode(cl_tree)!
|
||||
if sym <= 15 {
|
||||
all_lens << int(sym)
|
||||
} else if sym == 16 {
|
||||
if all_lens.len == 0 {
|
||||
return error('inflate: repeat with empty history')
|
||||
}
|
||||
rep := int(r.read_bits(2)!) + 3
|
||||
last := all_lens[all_lens.len - 1]
|
||||
for _ in 0 .. rep {
|
||||
all_lens << last
|
||||
}
|
||||
} else if sym == 17 {
|
||||
rep := int(r.read_bits(3)!) + 3
|
||||
for _ in 0 .. rep {
|
||||
all_lens << 0
|
||||
}
|
||||
} else if sym == 18 {
|
||||
rep := int(r.read_bits(7)!) + 11
|
||||
for _ in 0 .. rep {
|
||||
all_lens << 0
|
||||
}
|
||||
} else {
|
||||
return error('inflate: bad code length symbol')
|
||||
}
|
||||
}
|
||||
ll_tree := build_huff_tree(all_lens[..hlit])!
|
||||
d_tree := build_huff_tree(all_lens[hlit..])!
|
||||
if !inflate_block_stream(mut r, mut out, ll_tree, d_tree, cb, userdata, mut state)! {
|
||||
aborted = true
|
||||
}
|
||||
}
|
||||
else {
|
||||
return error('inflate: reserved block type')
|
||||
}
|
||||
}
|
||||
|
||||
if aborted || bfinal == 1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !aborted && out.len > state.delivered {
|
||||
chunk := unsafe { out[state.delivered..] }
|
||||
if cb(chunk, userdata) != chunk.len {
|
||||
aborted = true
|
||||
} else {
|
||||
state.delivered = out.len
|
||||
}
|
||||
}
|
||||
consumed := r.pos - (r.nbits >> 3)
|
||||
return InflateStreamResult{
|
||||
decoded: out
|
||||
consumed: consumed
|
||||
delivered: state.delivered
|
||||
aborted: aborted
|
||||
}
|
||||
}
|
||||
|
||||
@[direct_array_access]
|
||||
fn inflate_block(mut r BitReader, mut out []u8, ll HuffTree, dist HuffTree) ! {
|
||||
for {
|
||||
sym := r.huff_decode(ll)!
|
||||
if sym == 256 {
|
||||
break
|
||||
}
|
||||
if sym < 256 {
|
||||
out << u8(sym)
|
||||
} else {
|
||||
li := int(sym) - 257
|
||||
if li < 0 || li >= length_bases.len {
|
||||
return error('inflate: invalid length symbol ${sym}')
|
||||
}
|
||||
length := length_bases[li] + int(r.read_bits(length_extra_bits[li])!)
|
||||
dsym := r.huff_decode(dist)!
|
||||
di := int(dsym)
|
||||
if di >= dist_bases.len {
|
||||
return error('inflate: invalid distance symbol ${dsym}')
|
||||
}
|
||||
distance := dist_bases[di] + int(r.read_bits(dist_extra_bits[di])!)
|
||||
if distance > out.len {
|
||||
return error('inflate: distance past output start')
|
||||
}
|
||||
base := out.len - distance
|
||||
for i in 0 .. length {
|
||||
out << out[base + i]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@[direct_array_access]
|
||||
fn inflate_block_stream(mut r BitReader, mut out []u8, ll HuffTree, dist HuffTree, cb ChunkCallback, userdata voidptr, mut state InflateStreamState) !bool {
|
||||
for {
|
||||
sym := r.huff_decode(ll)!
|
||||
if sym == 256 {
|
||||
break
|
||||
}
|
||||
if sym < 256 {
|
||||
out << u8(sym)
|
||||
if !flush_stream_chunks(out, cb, userdata, mut state) {
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
li := int(sym) - 257
|
||||
if li < 0 || li >= length_bases.len {
|
||||
return error('inflate: invalid length symbol ${sym}')
|
||||
}
|
||||
length := length_bases[li] + int(r.read_bits(length_extra_bits[li])!)
|
||||
dsym := r.huff_decode(dist)!
|
||||
di := int(dsym)
|
||||
if di >= dist_bases.len {
|
||||
return error('inflate: invalid distance symbol ${dsym}')
|
||||
}
|
||||
distance := dist_bases[di] + int(r.read_bits(dist_extra_bits[di])!)
|
||||
if distance > out.len {
|
||||
return error('inflate: distance past output start')
|
||||
}
|
||||
base := out.len - distance
|
||||
for i in 0 .. length {
|
||||
out << out[base + i]
|
||||
if !flush_stream_chunks(out, cb, userdata, mut state) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
module deflate
|
||||
|
||||
fn test_zlib_roundtrip() {
|
||||
data := 'Hello world!'.bytes()
|
||||
compressed := compress(data)!
|
||||
assert compressed[0] == 0x78 && compressed[1] == 0x9c // zlib header
|
||||
assert decompress(compressed)! == data
|
||||
}
|
||||
|
||||
fn test_gzip_roundtrip() {
|
||||
data := 'Hello gzip!'.repeat(10).bytes()
|
||||
compressed := compress(data, format: .gzip)!
|
||||
assert compressed[0] == 0x1f && compressed[1] == 0x8b // gzip magic
|
||||
assert decompress(compressed)! == data
|
||||
}
|
||||
|
||||
fn test_raw_deflate_roundtrip() {
|
||||
data := 'raw deflate'.repeat(20).bytes()
|
||||
raw := compress(data, format: .raw_deflate)!
|
||||
decoded := decompress(raw)! // auto-detected as raw
|
||||
assert decoded == data
|
||||
}
|
||||
|
||||
fn test_decompress_auto_detects_all_formats() {
|
||||
data := 'multi-format detection test'.repeat(5).bytes()
|
||||
assert decompress(compress(data)!)! == data
|
||||
assert decompress(compress(data, format: .gzip)!)! == data
|
||||
assert decompress(compress(data, format: .raw_deflate)!)! == data
|
||||
}
|
||||
|
||||
fn test_wrapper_helpers_match_unified_api() {
|
||||
data := 'wrapper compatibility'.repeat(8).bytes()
|
||||
assert compress(data)! == compress(data, format: .zlib)!
|
||||
assert compress_gzip(data)! == compress(data, format: .gzip)!
|
||||
assert compress_raw(data)! == compress(data, format: .raw_deflate)!
|
||||
}
|
||||
|
||||
fn test_roundtrip_repeated() {
|
||||
data := 'abcabc'.repeat(100).bytes()
|
||||
compressed := compress(data)!
|
||||
assert compressed.len < data.len
|
||||
assert decompress(compressed)! == data
|
||||
}
|
||||
|
||||
fn test_bad_compression_method_fails() {
|
||||
bad := [u8(0x79), 0x18, 0x00, 0x00, 0x00, 0x00]
|
||||
decompress(bad) or {
|
||||
assert err.msg().len > 0
|
||||
return
|
||||
}
|
||||
assert false
|
||||
}
|
||||
|
||||
fn test_corrupt_checksum_fails() {
|
||||
mut enc := compress(('hello world').repeat(10).bytes())!
|
||||
// flip a byte in the adler32 footer
|
||||
enc[enc.len - 1] ^= 0xff
|
||||
decompress(enc) or {
|
||||
assert err.msg().contains('adler32')
|
||||
return
|
||||
}
|
||||
assert false
|
||||
}
|
||||
|
||||
fn test_truncated_zlib_payload_fails() {
|
||||
decompress([u8(0x78), 0x9c, 0x03, 0x00, 0x00, 0x00, 0x01]) or {
|
||||
assert err.msg().contains('unexpected end of stream')
|
||||
return
|
||||
}
|
||||
assert false
|
||||
}
|
||||
|
||||
fn test_zlib_inserted_bytes_before_adler_fails() {
|
||||
enc := compress('zlib injected trailer bytes'.repeat(4).bytes())!
|
||||
mut bad := []u8{cap: enc.len + 2}
|
||||
bad << enc[..enc.len - 4]
|
||||
bad << [u8(0xaa), 0x55]
|
||||
bad << enc[enc.len - 4..]
|
||||
decompress(bad) or {
|
||||
assert err.msg() == 'invalid zlib stream: trailing data before adler32'
|
||||
return
|
||||
}
|
||||
assert false
|
||||
}
|
||||
|
||||
fn test_gzip_inserted_bytes_before_trailer_fails() {
|
||||
enc := compress('gzip injected trailer bytes'.repeat(4).bytes(), format: .gzip)!
|
||||
mut bad := []u8{cap: enc.len + 1}
|
||||
bad << enc[..enc.len - 8]
|
||||
bad << u8(0x42)
|
||||
bad << enc[enc.len - 8..]
|
||||
decompress(bad) or {
|
||||
assert err.msg() == 'invalid gzip stream: trailing data before trailer'
|
||||
return
|
||||
}
|
||||
assert false
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
module main
|
||||
|
||||
import compress.deflate
|
||||
import os
|
||||
|
||||
fn choose_cc() string {
|
||||
for cc in ['cc', 'gcc', 'clang'] {
|
||||
if os.execute('${cc} --version').exit_code == 0 {
|
||||
return cc
|
||||
}
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
fn compile_c_ref(workdir string) !string {
|
||||
cc := choose_cc()
|
||||
if cc == '' {
|
||||
return error('no C compiler found')
|
||||
}
|
||||
src := os.join_path(@DIR, 'deflate_ref.c')
|
||||
bin := os.join_path(workdir, 'deflate_cross_validate')
|
||||
res := os.execute('${cc} -O2 ${os.quoted_path(src)} -lz -o ${os.quoted_path(bin)}')
|
||||
if res.exit_code != 0 {
|
||||
return error('C compile failed: ${res.output}')
|
||||
}
|
||||
return bin
|
||||
}
|
||||
|
||||
fn main() {
|
||||
base_tmp := os.join_path(os.temp_dir(), 'v_deflate_interop')
|
||||
os.mkdir_all(base_tmp) or {
|
||||
eprintln('could not create base temp dir: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
workdir := os.join_path(base_tmp, 'run_${os.getpid()}')
|
||||
os.mkdir_all(workdir) or {
|
||||
eprintln('could not create work dir: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
defer {
|
||||
os.rmdir_all(workdir) or {}
|
||||
}
|
||||
|
||||
bin := compile_c_ref(workdir) or {
|
||||
eprintln('Skipping C cross-validation: ${err.msg()}')
|
||||
return
|
||||
}
|
||||
|
||||
input := 'deflate C cross-validation payload'.repeat(50).bytes()
|
||||
ip := os.join_path(workdir, 'xval_in.bin')
|
||||
cp := os.join_path(workdir, 'xval_c_zlib.bin')
|
||||
gp := os.join_path(workdir, 'xval_c_gzip.bin')
|
||||
vp := os.join_path(workdir, 'xval_v_zlib.bin')
|
||||
vgp := os.join_path(workdir, 'xval_v_gzip.bin')
|
||||
dp := os.join_path(workdir, 'xval_dec.bin')
|
||||
|
||||
os.write_file_array(ip, input) or {
|
||||
eprintln('write input failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
|
||||
res1 :=
|
||||
os.execute('${os.quoted_path(bin)} compress ${os.quoted_path(ip)} ${os.quoted_path(cp)}')
|
||||
if res1.exit_code != 0 {
|
||||
eprintln('C zlib compress failed: ${res1.output}')
|
||||
exit(1)
|
||||
}
|
||||
c_zlib := os.read_bytes(cp) or {
|
||||
eprintln('read C zlib stream failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
v_decoded_zlib := deflate.decompress(c_zlib) or {
|
||||
eprintln('V decompress of C zlib failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
if v_decoded_zlib != input {
|
||||
eprintln('C zlib -> V mismatch')
|
||||
exit(1)
|
||||
}
|
||||
println('OK: C zlib -> V decompress')
|
||||
|
||||
res2 := os.execute('${os.quoted_path(bin)} gzip ${os.quoted_path(ip)} ${os.quoted_path(gp)}')
|
||||
if res2.exit_code != 0 {
|
||||
eprintln('C gzip failed: ${res2.output}')
|
||||
exit(1)
|
||||
}
|
||||
c_gzip := os.read_bytes(gp) or {
|
||||
eprintln('read C gzip stream failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
v_decoded_gzip := deflate.decompress(c_gzip) or {
|
||||
eprintln('V decompress of C gzip failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
if v_decoded_gzip != input {
|
||||
eprintln('C gzip -> V mismatch')
|
||||
exit(1)
|
||||
}
|
||||
println('OK: C gzip -> V decompress')
|
||||
|
||||
v_zlib := deflate.compress(input) or {
|
||||
eprintln('V zlib compress failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
os.write_file_array(vp, v_zlib) or {
|
||||
eprintln('write V zlib failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
res3 :=
|
||||
os.execute('${os.quoted_path(bin)} decompress ${os.quoted_path(vp)} ${os.quoted_path(dp)}')
|
||||
if res3.exit_code != 0 {
|
||||
eprintln('C decompress of V zlib failed: ${res3.output}')
|
||||
exit(1)
|
||||
}
|
||||
c_unzlib := os.read_bytes(dp) or {
|
||||
eprintln('read C zlib decompressed output failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
if c_unzlib != input {
|
||||
eprintln('V zlib -> C mismatch')
|
||||
exit(1)
|
||||
}
|
||||
println('OK: V zlib -> C decompress')
|
||||
|
||||
v_gzip := deflate.compress(input, format: .gzip) or {
|
||||
eprintln('V gzip compress failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
os.write_file_array(vgp, v_gzip) or {
|
||||
eprintln('write V gzip failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
res4 := os.execute('${os.quoted_path(bin)} gunzip ${os.quoted_path(vgp)} ${os.quoted_path(dp)}')
|
||||
if res4.exit_code != 0 {
|
||||
eprintln('C gunzip of V gzip failed: ${res4.output}')
|
||||
exit(1)
|
||||
}
|
||||
c_ungzip := os.read_bytes(dp) or {
|
||||
eprintln('read C gzip decompressed output failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
if c_ungzip != input {
|
||||
eprintln('V gzip -> C mismatch')
|
||||
exit(1)
|
||||
}
|
||||
println('OK: V gzip -> C decompress')
|
||||
|
||||
if os.execute('python3 --version').exit_code == 0 {
|
||||
py_src := os.join_path(@DIR, 'deflate_ref.py')
|
||||
py_driver := os.join_path(workdir, 'deflate_ref.py')
|
||||
os.cp(py_src, py_driver) or {
|
||||
eprintln('copy Python reference script failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
|
||||
py_zp := os.join_path(workdir, 'xval_py_zlib.bin')
|
||||
py_gp := os.join_path(workdir, 'xval_py_gzip.bin')
|
||||
|
||||
res5 :=
|
||||
os.execute('python3 ${os.quoted_path(py_driver)} compress ${os.quoted_path(ip)} ${os.quoted_path(py_zp)}')
|
||||
if res5.exit_code != 0 {
|
||||
eprintln('Python zlib compress failed: ${res5.output}')
|
||||
exit(1)
|
||||
}
|
||||
py_zlib := os.read_bytes(py_zp) or {
|
||||
eprintln('read Python zlib stream failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
if deflate.decompress(py_zlib) or { []u8{} } != input {
|
||||
eprintln('Python zlib -> V mismatch')
|
||||
exit(1)
|
||||
}
|
||||
println('OK: Python zlib -> V decompress')
|
||||
|
||||
res6 :=
|
||||
os.execute('python3 ${os.quoted_path(py_driver)} gzip ${os.quoted_path(ip)} ${os.quoted_path(py_gp)}')
|
||||
if res6.exit_code != 0 {
|
||||
eprintln('Python gzip failed: ${res6.output}')
|
||||
exit(1)
|
||||
}
|
||||
py_gzip := os.read_bytes(py_gp) or {
|
||||
eprintln('read Python gzip stream failed: ${err.msg()}')
|
||||
exit(1)
|
||||
}
|
||||
if deflate.decompress(py_gzip) or { []u8{} } != input {
|
||||
eprintln('Python gzip -> V mismatch')
|
||||
exit(1)
|
||||
}
|
||||
println('OK: Python gzip -> V decompress')
|
||||
|
||||
py_unz := os.join_path(workdir, 'xval_py_unz.bin')
|
||||
res7 :=
|
||||
os.execute('python3 ${os.quoted_path(py_driver)} decompress ${os.quoted_path(vp)} ${os.quoted_path(py_unz)}')
|
||||
if res7.exit_code != 0 {
|
||||
eprintln('Python decompress of V zlib failed: ${res7.output}')
|
||||
exit(1)
|
||||
}
|
||||
if (os.read_bytes(py_unz) or { []u8{} }) != input {
|
||||
eprintln('V zlib -> Python mismatch')
|
||||
exit(1)
|
||||
}
|
||||
println('OK: V zlib -> Python decompress')
|
||||
|
||||
py_ungz := os.join_path(workdir, 'xval_py_ungz.bin')
|
||||
res8 :=
|
||||
os.execute('python3 ${os.quoted_path(py_driver)} gunzip ${os.quoted_path(vgp)} ${os.quoted_path(py_ungz)}')
|
||||
if res8.exit_code != 0 {
|
||||
eprintln('Python gunzip of V gzip failed: ${res8.output}')
|
||||
exit(1)
|
||||
}
|
||||
if (os.read_bytes(py_ungz) or { []u8{} }) != input {
|
||||
eprintln('V gzip -> Python mismatch')
|
||||
exit(1)
|
||||
}
|
||||
println('OK: V gzip -> Python decompress')
|
||||
} else {
|
||||
eprintln('Skipping Python cross-validation: python3 not found')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <zlib.h>
|
||||
|
||||
static int rf(const char *p, unsigned char **o, size_t *n) {
|
||||
FILE *f = fopen(p, "rb");
|
||||
if (!f) return 1;
|
||||
fseek(f, 0, SEEK_END);
|
||||
long s = ftell(f);
|
||||
fseek(f, 0, SEEK_SET);
|
||||
*o = malloc(*n = (size_t)s);
|
||||
if (!*o) {
|
||||
fclose(f);
|
||||
return 1;
|
||||
}
|
||||
if (fread(*o, 1, *n, f) != *n) {
|
||||
free(*o);
|
||||
fclose(f);
|
||||
return 1;
|
||||
}
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int wf(const char *p, const unsigned char *b, size_t n) {
|
||||
FILE *f = fopen(p, "wb");
|
||||
if (!f) return 1;
|
||||
if (fwrite(b, 1, n, f) != n) {
|
||||
fclose(f);
|
||||
return 1;
|
||||
}
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc != 4) {
|
||||
fputs("usage: xval compress|decompress|gzip|gunzip in out\n", stderr);
|
||||
return 2;
|
||||
}
|
||||
unsigned char *in;
|
||||
size_t in_n;
|
||||
if (rf(argv[2], &in, &in_n)) {
|
||||
fputs("read error\n", stderr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (strcmp(argv[1], "compress") == 0) {
|
||||
uLongf cap = compressBound((uLong)in_n);
|
||||
unsigned char *out = malloc(cap);
|
||||
if (!out) return 1;
|
||||
if (compress2(out, &cap, in, (uLong)in_n, Z_DEFAULT_COMPRESSION) != Z_OK) {
|
||||
fputs("compress2 failed\n", stderr);
|
||||
free(in);
|
||||
free(out);
|
||||
return 1;
|
||||
}
|
||||
wf(argv[3], out, (size_t)cap);
|
||||
free(out);
|
||||
} else if (strcmp(argv[1], "decompress") == 0) {
|
||||
uLongf cap = in_n * 8 + 65536;
|
||||
unsigned char *out = malloc(cap);
|
||||
if (!out) return 1;
|
||||
if (uncompress(out, &cap, in, (uLong)in_n) != Z_OK) {
|
||||
fputs("uncompress failed\n", stderr);
|
||||
free(in);
|
||||
free(out);
|
||||
return 1;
|
||||
}
|
||||
wf(argv[3], out, (size_t)cap);
|
||||
free(out);
|
||||
} else if (strcmp(argv[1], "gzip") == 0) {
|
||||
z_stream s;
|
||||
memset(&s, 0, sizeof(s));
|
||||
if (deflateInit2(&s, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 | 16, 8, Z_DEFAULT_STRATEGY) != Z_OK) {
|
||||
free(in);
|
||||
return 1;
|
||||
}
|
||||
uLongf cap = deflateBound(&s, (uLong)in_n) + 32;
|
||||
unsigned char *out = malloc(cap);
|
||||
if (!out) {
|
||||
deflateEnd(&s);
|
||||
free(in);
|
||||
return 1;
|
||||
}
|
||||
s.next_in = in;
|
||||
s.avail_in = (uInt)in_n;
|
||||
s.next_out = out;
|
||||
s.avail_out = (uInt)cap;
|
||||
if (deflate(&s, Z_FINISH) != Z_STREAM_END) {
|
||||
fputs("gzip deflate failed\n", stderr);
|
||||
deflateEnd(&s);
|
||||
free(in);
|
||||
free(out);
|
||||
return 1;
|
||||
}
|
||||
wf(argv[3], out, (size_t)s.total_out);
|
||||
deflateEnd(&s);
|
||||
free(out);
|
||||
} else if (strcmp(argv[1], "gunzip") == 0) {
|
||||
z_stream s;
|
||||
memset(&s, 0, sizeof(s));
|
||||
if (inflateInit2(&s, 15 | 16) != Z_OK) {
|
||||
free(in);
|
||||
return 1;
|
||||
}
|
||||
uLongf cap = in_n * 8 + 65536;
|
||||
unsigned char *out = malloc(cap);
|
||||
if (!out) {
|
||||
inflateEnd(&s);
|
||||
free(in);
|
||||
return 1;
|
||||
}
|
||||
s.next_in = in;
|
||||
s.avail_in = (uInt)in_n;
|
||||
s.next_out = out;
|
||||
s.avail_out = (uInt)cap;
|
||||
if (inflate(&s, Z_FINISH) != Z_STREAM_END) {
|
||||
fputs("gunzip inflate failed\n", stderr);
|
||||
inflateEnd(&s);
|
||||
free(in);
|
||||
free(out);
|
||||
return 1;
|
||||
}
|
||||
wf(argv[3], out, (size_t)s.total_out);
|
||||
inflateEnd(&s);
|
||||
free(out);
|
||||
}
|
||||
|
||||
free(in);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import sys, zlib
|
||||
|
||||
def rf(p):
|
||||
with open(p, "rb") as f: return f.read()
|
||||
|
||||
def wf(p, b):
|
||||
with open(p, "wb") as f: f.write(b)
|
||||
|
||||
if len(sys.argv) != 4:
|
||||
print("usage: py_ref compress|decompress|gzip|gunzip in out", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
mode, inp, outp = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
data = rf(inp)
|
||||
if mode == "compress":
|
||||
wf(outp, zlib.compress(data))
|
||||
elif mode == "decompress":
|
||||
wf(outp, zlib.decompress(data))
|
||||
elif mode == "gzip":
|
||||
co = zlib.compressobj(level=6, wbits=16 + zlib.MAX_WBITS)
|
||||
wf(outp, co.compress(data) + co.flush())
|
||||
elif mode == "gunzip":
|
||||
wf(outp, zlib.decompress(data, 16 + zlib.MAX_WBITS))
|
||||
else:
|
||||
print("unknown mode", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
Reference in new issue
Block a user