Initial project import

tools/v/vlib/compress/bzip2/README.md

@@ -0,0 +1,51 @@
+# compress.bzip2
+
+Pure V bzip2 encoder/decoder.
+
+## Features
+
+- Pure V implementation (no C wrappers)
+- bzip2 stream compression (`BZh1`..`BZh9`)
+- bzip2 stream decompression
+- Block and stream CRC validation
+- Deterministic output for identical input/params
+
+## API
+
+```v
+import compress.bzip2
+
+compressed := bzip2.compress('hello'.bytes())!
+plain := bzip2.decompress(compressed)!
+assert plain.bytestr() == 'hello'
+```
+
+With params:
+
+```v
+import compress.bzip2
+
+data := 'hello'.bytes()
+compressed := bzip2.compress(data, block_size: 1)!
+plain := bzip2.decompress(compressed, verify_crc: true)!
+assert plain.bytestr() == 'hello'
+```
+
+## Notes
+
+- Randomized legacy bzip2 blocks are intentionally rejected.
+- API currently works on full byte slices (no streaming interface yet).
+
+## Test
+
+```bash
+v test vlib/compress/bzip2/
+```
+
+## Proof
+
+- tested against the official bzip2 C library using a variety of inputs and parameters.
+- deterministic and produces identical output for identical input and parameters.
+- includes CRC validation to ensure data integrity during compression and decompression.
+- rejects randomized legacy bzip2 blocks, ensuring only valid bzip2 streams are processed.
+- tested with a variety of inputs, including edge cases, to ensure robustness and reliability.

tools/v/vlib/compress/bzip2/bzip2_test.v

@@ -0,0 +1,136 @@
+module bzip2
+
+import encoding.hex
+
+fn must_decode_hex(s string) []u8 {
+	return hex.decode(s) or { panic(err) }
+}
+
+fn test_roundtrip_empty_input() {
+	src := []u8{}
+	compressed := compress(src) or { panic(err) }
+	decompressed := decompress(compressed) or { panic(err) }
+	assert decompressed == src
+}
+
+fn test_roundtrip_small_text() {
+	src := 'hello world\n'.bytes()
+	compressed := compress(src) or { panic(err) }
+	decompressed := decompress(compressed) or { panic(err) }
+	assert decompressed == src
+}
+
+fn test_roundtrip_binary_data() {
+	mut src := []u8{len: 4096}
+	for i in 0 .. src.len {
+		src[i] = u8((i * 17 + 13) & 0xff)
+	}
+	compressed := compress(src, block_size: 1) or { panic(err) }
+	decompressed := decompress(compressed) or { panic(err) }
+	assert decompressed == src
+}
+
+fn test_roundtrip_long_runs() {
+	src := 'a'.repeat(2000).bytes()
+	compressed := compress(src) or { panic(err) }
+	decompressed := decompress(compressed) or { panic(err) }
+	assert decompressed == src
+}
+
+fn test_decompress_known_python_vector_empty() {
+	bz2 := must_decode_hex('425a683917724538509000000000')
+	plain := decompress(bz2) or { panic(err) }
+	assert plain == []u8{}
+}
+
+fn test_decompress_known_python_vector_hello() {
+	bz2 :=
+		must_decode_hex('425a68393141592653594eece83600000251800010400006449080200031064c4101a7a9a580bb9431f8bb9229c28482776741b0')
+	plain := decompress(bz2) or { panic(err) }
+	assert plain.bytestr() == 'hello world\n'
+}
+
+fn test_decompress_known_python_vector_text() {
+	bz2 :=
+		must_decode_hex('425a6839314159265359dc01b0d8000002d9800010410120080a00cc20200021a4d3688cd0806800e28a3de49f0b16b10d177245385090dc01b0d8')
+	plain := decompress(bz2) or { panic(err) }
+	assert plain.bytestr() == '1.test\ncopy ©\n'
+}
+
+fn test_decompress_known_python_vector_repeated_a() {
+	bz2 :=
+		must_decode_hex('425a6839314159265359ca3d8dfb000000010420000400200021008283177245385090ca3d8dfb')
+	plain := decompress(bz2) or { panic(err) }
+	assert plain == 'a'.repeat(200).bytes()
+}
+
+fn test_decompress_rejects_invalid_header() {
+	_ := decompress('not-bzip2'.bytes()) or {
+		assert err.msg().contains('invalid header')
+		return
+	}
+	assert false
+}
+
+fn test_decompress_rejects_crc_mismatch() {
+	mut bz2 :=
+		must_decode_hex('425a68393141592653594eece83600000251800010400006449080200031064c4101a7a9a580bb9431f8bb9229c28482776741b0')
+	// Corrupt the stored block CRC field (bytes 10..13 in a single-block stream).
+	bz2[10] ^= 0x01
+	_ := decompress(bz2) or {
+		assert err.msg().contains('crc mismatch')
+		return
+	}
+	assert false
+}
+
+fn test_selector_count_limit_boundaries() {
+	below_limit := selector_count_from_symbol_count(900050) or { panic(err) }
+	at_limit := selector_count_from_symbol_count(900100) or { panic(err) }
+	assert below_limit == 18001
+	assert at_limit == 18002
+
+	_ := selector_count_from_symbol_count(900101) or {
+		assert err.msg().contains('invalid selector count')
+		return
+	}
+	assert false
+}
+
+fn test_block_output_limit_guard() {
+	ensure_block_output_limit(0, 100000, 100000) or { panic(err) }
+	ensure_block_output_limit(99999, 1, 100000) or { panic(err) }
+
+	ensure_block_output_limit(100000, 1, 100000) or {
+		assert err.msg().contains('block output exceeds declared block size')
+		return
+	}
+	assert false
+}
+
+fn test_find_rle1_block_end_for_four_byte_runs() {
+	mut src := []u8{cap: 100000}
+	for i in 0 .. 25000 {
+		b := u8(i & 0xff)
+		for _ in 0 .. 4 {
+			src << b
+		}
+	}
+	end := find_rle1_block_end(src, 0, 100000)
+	assert end == 80000
+	assert rle1_encode(src[0..end]).len == 100000
+	assert rle1_encode(src[0..end + 4]).len > 100000
+}
+
+fn test_roundtrip_block_size_1_four_byte_runs() {
+	mut src := []u8{cap: 100000}
+	for i in 0 .. 25000 {
+		b := u8(i & 0xff)
+		for _ in 0 .. 4 {
+			src << b
+		}
+	}
+	compressed := compress(src, block_size: 1) or { panic(err) }
+	decompressed := decompress(compressed) or { panic(err) }
+	assert decompressed == src
+}

tools/v/vlib/compress/bzip2/interop/README.md

@@ -0,0 +1,26 @@
+# bzip2 interop checker
+
+This helper verifies interoperability between:
+
+- V module: `compress.bzip2`
+- System CLI: `bzip2`
+- Python stdlib: `bz2`
+
+It runs deterministic test vectors and compression levels, compresses each vector with all
+three producers, then cross-decompresses every produced stream with all three decoders.
+
+A case passes only if every decompressed output is byte-identical to the original input.
+
+Current levels are `1`, `6`, and `9` (see `compression_levels` in
+`vlib/compress/bzip2/interop/bzip2_interop_check.vsh`).
+
+## Run
+
+```bash
+v run vlib/compress/bzip2/interop/bzip2_interop_check.vsh
+```
+
+## Requirements
+
+- `bzip2` available in `PATH`
+- `python3` with `bz2`

tools/v/vlib/compress/bzip2/interop/bzip2_interop_check.vsh

@@ -0,0 +1,166 @@
+#!/usr/bin/env -S v
+
+import compress.bzip2
+
+const compression_levels = [1, 6, 9]!
+
+struct TestVector {
+	name string
+	data []u8
+}
+
+fn main() {
+	ensure_tools() or {
+		eprintln('SKIP: ${err.msg()}')
+		exit(2)
+	}
+	vectors := make_test_vectors()
+	mut total_checks := 0
+	mut total_runs := 0
+	tmp_root := join_path(temp_dir(), 'v_bzip2_interop_${getpid()}')
+	mkdir_all(tmp_root) or { panic(err) }
+	defer {
+		rmdir_all(tmp_root) or {}
+	}
+	for level in compression_levels {
+		for i, vec in vectors {
+			total_checks += run_case(tmp_root, level, i, vec) or {
+				eprintln('FAIL: ${vec.name} (level=${level}): ${err}')
+				exit(1)
+				0
+			}
+			total_runs++
+			println('ok ${total_runs}/${vectors.len * compression_levels.len}: ${vec.name} (level=${level}, ${vec.data.len} bytes)')
+		}
+	}
+	println('PASS: ${vectors.len} vectors x ${compression_levels.len} levels, ${total_checks} cross-checks')
+}
+
+fn ensure_tools() ! {
+	must_succeed('bzip2 --help >/dev/null 2>&1', 'system bzip2 command is not available')!
+	must_succeed("python3 -c 'import bz2' >/dev/null 2>&1",
+		'python3 with bz2 module is not available')!
+}
+
+fn make_test_vectors() []TestVector {
+	mut vectors := []TestVector{}
+	vectors << TestVector{'empty', []u8{}}
+	vectors << TestVector{'ascii_text', 'The quick brown fox jumps over the lazy dog.\n'.repeat(64).bytes()}
+	vectors << TestVector{'repeated_byte', []u8{len: 10000, init: `A`}}
+	vectors << TestVector{'all_bytes_x4', all_bytes_repeated(4)}
+	vectors << TestVector{'lcg_64k', lcg_bytes(65536)}
+	return vectors
+}
+
+fn run_case(tmp_root string, level int, case_idx int, vec TestVector) !int {
+	case_dir := join_path(tmp_root, 'case_l${level}_${case_idx:02}_${vec.name}')
+	mkdir_all(case_dir)!
+
+	v_bz2 := bzip2.compress(vec.data, block_size: level)!
+	cli_bz2 := cli_compress(case_dir, vec.data, level)!
+	py_bz2 := py_compress(case_dir, vec.data, level)!
+
+	mut checks := 0
+
+	producers := {
+		'v':   v_bz2
+		'cli': cli_bz2
+		'py':  py_bz2
+	}
+	for producer, compressed in producers {
+		v_plain := bzip2.decompress(compressed)!
+		assert_equal_bytes('v.decompress(${producer}.compress, level=${level})', vec.data, v_plain)!
+		checks++
+
+		cli_plain := cli_decompress(case_dir, producer, compressed)!
+		assert_equal_bytes('cli.decompress(${producer}.compress, level=${level})', vec.data,
+			cli_plain)!
+		checks++
+
+		py_plain := py_decompress(case_dir, producer, compressed)!
+		assert_equal_bytes('py.decompress(${producer}.compress, level=${level})', vec.data,
+			py_plain)!
+		checks++
+	}
+	return checks
+}
+
+fn cli_compress(case_dir string, plain []u8, level int) ![]u8 {
+	in_path := join_path(case_dir, 'plain.in')
+	out_path := join_path(case_dir, 'cli_l${level}.bz2')
+	write_file_array(in_path, plain)!
+	must_succeed('bzip2 -${level} -c -- ${shell_quote(in_path)} > ${shell_quote(out_path)}',
+		'bzip2 compression failed')!
+	return read_bytes(out_path)!
+}
+
+fn cli_decompress(case_dir string, producer string, compressed []u8) ![]u8 {
+	in_path := join_path(case_dir, '${producer}.for_cli.bz2')
+	out_path := join_path(case_dir, '${producer}.from_cli.out')
+	write_file_array(in_path, compressed)!
+	must_succeed('bzip2 -d -c -- ${shell_quote(in_path)} > ${shell_quote(out_path)}',
+		'bzip2 decompression failed')!
+	return read_bytes(out_path)!
+}
+
+fn py_compress(case_dir string, plain []u8, level int) ![]u8 {
+	in_path := join_path(case_dir, 'plain_py.in')
+	out_path := join_path(case_dir, 'py_l${level}.bz2')
+	write_file_array(in_path, plain)!
+	py_code := 'import bz2, pathlib, sys; p=pathlib.Path(sys.argv[1]); o=pathlib.Path(sys.argv[2]); l=int(sys.argv[3]); o.write_bytes(bz2.compress(p.read_bytes(), compresslevel=l))'
+	must_succeed('python3 -c ${shell_quote(py_code)} ${shell_quote(in_path)} ${shell_quote(out_path)} ${level}',
+		'python bz2 compression failed')!
+	return read_bytes(out_path)!
+}
+
+fn py_decompress(case_dir string, producer string, compressed []u8) ![]u8 {
+	in_path := join_path(case_dir, '${producer}.for_py.bz2')
+	out_path := join_path(case_dir, '${producer}.from_py.out')
+	write_file_array(in_path, compressed)!
+	py_code := 'import bz2, pathlib, sys; p=pathlib.Path(sys.argv[1]); o=pathlib.Path(sys.argv[2]); o.write_bytes(bz2.decompress(p.read_bytes()))'
+	must_succeed('python3 -c ${shell_quote(py_code)} ${shell_quote(in_path)} ${shell_quote(out_path)}',
+		'python bz2 decompression failed')!
+	return read_bytes(out_path)!
+}
+
+fn all_bytes_repeated(times int) []u8 {
+	mut out := []u8{cap: 256 * times}
+	for _ in 0 .. times {
+		for i in 0 .. 256 {
+			out << u8(i)
+		}
+	}
+	return out
+}
+
+fn lcg_bytes(n int) []u8 {
+	mut out := []u8{len: n}
+	mut x := u32(0x12345678)
+	for i in 0 .. n {
+		x = x * u32(1664525) + u32(1013904223)
+		out[i] = u8((x >> 16) & u32(0xff))
+	}
+	return out
+}
+
+fn assert_equal_bytes(label string, expected []u8, got []u8) ! {
+	if expected.len != got.len {
+		return error('${label}: length mismatch expected=${expected.len} got=${got.len}')
+	}
+	for i in 0 .. expected.len {
+		if expected[i] != got[i] {
+			return error('${label}: byte mismatch at offset ${i}')
+		}
+	}
+}
+
+fn must_succeed(command string, context string) ! {
+	res := execute(command)
+	if res.exit_code != 0 {
+		return error('${context}\ncommand: ${command}\nexit_code: ${res.exit_code}\n${res.output}')
+	}
+}
+
+fn shell_quote(s string) string {
+	return "'${s.replace("'", "'\\''")}'"
+}