Initial project import

tools/v/vlib/compress/lz/interop/README.md

@@ -0,0 +1,44 @@
+## LZ Interop Validation (V, C, Python)
+
+This tool validates:
+
+- V compress/decompress roundtrips for all formats: `lz77`, `lz78`, `lzw`,
+  `lz4`, `lzss`, `lzma`, `lzjb`
+- a C `lz77`-like reference implementation
+- a Python `lz77`-like reference implementation
+
+The C/Python references are intentionally simple so the benchmark is easy to run
+without external dependencies.
+
+### Run
+
+```bash
+cd /home/jalon/git/v
+./vnew run vlib/compress/lz/interop/lz_interop.v 2>&1
+```
+
+Optional args:
+
+1. validation rounds (default: `40`)
+2. input size in bytes (default: `524288`)
+
+Example:
+
+```bash
+cd /home/jalon/git/v
+./vnew run vlib/compress/lz/interop/lz_interop.v 25 262144 2>&1
+```
+
+### Output
+
+The tool prints validation status lines only (no timing output).
+
+When helpers are available, it also cross-validates compression/expansion
+interoperability with V (`lz77`) in both directions:
+
+- V compress -> C decompress
+- C compress -> V decompress
+- V compress -> Python decompress
+- Python compress -> V decompress
+
+If `cc`/`gcc` or `python3` are missing, that row is skipped with a message.

tools/v/vlib/compress/lz/interop/lz77_ref.c

@@ -0,0 +1,256 @@
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#define MIN_MATCH 3
+#define MAX_LITERAL 128
+
+static const uint8_t STREAM_MAGIC[4] = {0x56, 0x4c, 0x5a, 0x31};
+static const uint8_t FORMAT_LZ77 = 0;
+
+typedef struct {
+	uint8_t *data;
+	size_t len;
+	size_t cap;
+} Buffer;
+
+static void die(const char *msg) {
+	fprintf(stderr, "%s\n", msg);
+	exit(1);
+}
+
+static void buf_init(Buffer *b, size_t cap) {
+	b->data = (uint8_t *)malloc(cap > 0 ? cap : 1);
+	if (!b->data) {
+		die("allocation failed");
+	}
+	b->len = 0;
+	b->cap = cap > 0 ? cap : 1;
+}
+
+static void buf_push(Buffer *b, uint8_t v) {
+	if (b->len >= b->cap) {
+		size_t new_cap = b->cap * 2;
+		uint8_t *n = (uint8_t *)realloc(b->data, new_cap);
+		if (!n) {
+			die("reallocation failed");
+		}
+		b->data = n;
+		b->cap = new_cap;
+	}
+	b->data[b->len++] = v;
+}
+
+static void buf_append(Buffer *b, const uint8_t *src, size_t len) {
+	for (size_t i = 0; i < len; i++) {
+		buf_push(b, src[i]);
+	}
+}
+
+static Buffer read_all(const char *path) {
+	FILE *f = fopen(path, "rb");
+	if (!f) {
+		die("could not open input file");
+	}
+	if (fseek(f, 0, SEEK_END) != 0) {
+		fclose(f);
+		die("could not seek input file");
+	}
+	long sz = ftell(f);
+	if (sz < 0) {
+		fclose(f);
+		die("could not read input file size");
+	}
+	if (fseek(f, 0, SEEK_SET) != 0) {
+		fclose(f);
+		die("could not rewind input file");
+	}
+	Buffer in;
+	buf_init(&in, (size_t)sz + 1);
+	in.len = (size_t)sz;
+	if (in.len > 0 && fread(in.data, 1, in.len, f) != in.len) {
+		fclose(f);
+		free(in.data);
+		die("could not read input file");
+	}
+	fclose(f);
+	return in;
+}
+
+static void write_all(const char *path, const uint8_t *data, size_t len) {
+	FILE *f = fopen(path, "wb");
+	if (!f) {
+		die("could not open output file");
+	}
+	if (len > 0 && fwrite(data, 1, len, f) != len) {
+		fclose(f);
+		die("could not write output file");
+	}
+	fclose(f);
+}
+
+static void write_uvarint(Buffer *out, uint64_t value) {
+	uint64_t v = value;
+	while (v >= 0x80) {
+		buf_push(out, (uint8_t)(v & 0x7f) | 0x80);
+		v >>= 7;
+	}
+	buf_push(out, (uint8_t)v);
+}
+
+static int read_uvarint(const uint8_t *data, size_t len, size_t *pos, uint64_t *value) {
+	uint64_t out = 0;
+	uint32_t shift = 0;
+	while (*pos < len && shift <= 63) {
+		uint8_t b = data[*pos];
+		(*pos)++;
+		out |= ((uint64_t)(b & 0x7f)) << shift;
+		if ((b & 0x80) == 0) {
+			*value = out;
+			return 1;
+		}
+		shift += 7;
+	}
+	return 0;
+}
+
+static Buffer compress_lz77(const uint8_t *in, size_t in_len) {
+	Buffer out;
+	buf_init(&out, in_len + 32);
+	buf_append(&out, STREAM_MAGIC, 4);
+	buf_push(&out, FORMAT_LZ77);
+	write_uvarint(&out, (uint64_t)in_len);
+
+	for (size_t i = 0; i < in_len;) {
+		size_t lit_len = in_len - i;
+		if (lit_len > MAX_LITERAL) {
+			lit_len = MAX_LITERAL;
+		}
+		buf_push(&out, (uint8_t)(lit_len - 1));
+		buf_append(&out, in + i, lit_len);
+		i += lit_len;
+	}
+	return out;
+}
+
+static Buffer decompress_lz77(const uint8_t *in, size_t in_len) {
+	if (in_len < 6 || memcmp(in, STREAM_MAGIC, 4) != 0) {
+		die("bad magic");
+	}
+	if (in[4] != FORMAT_LZ77) {
+		die("format mismatch");
+	}
+	size_t pos = 5;
+	uint64_t expected_len_u64 = 0;
+	if (!read_uvarint(in, in_len, &pos, &expected_len_u64)) {
+		die("bad length varint");
+	}
+	size_t expected_len = (size_t)expected_len_u64;
+
+	Buffer out;
+	buf_init(&out, expected_len + 16);
+	while (pos < in_len) {
+		uint8_t control = in[pos++];
+		if ((control & 0x80) == 0) {
+			size_t lit_len = (size_t)(control & 0x7f) + 1;
+			if (pos + lit_len > in_len) {
+				die("truncated literal");
+			}
+			buf_append(&out, in + pos, lit_len);
+			pos += lit_len;
+		} else {
+			size_t match_len = (size_t)(control & 0x7f) + MIN_MATCH;
+			uint64_t offset_u64 = 0;
+			if (!read_uvarint(in, in_len, &pos, &offset_u64)) {
+				die("bad match offset");
+			}
+			size_t offset = (size_t)offset_u64;
+			if (offset == 0 || offset > out.len) {
+				die("bad offset");
+			}
+			size_t base = out.len - offset;
+			for (size_t k = 0; k < match_len; k++) {
+				buf_push(&out, out.data[base + k]);
+			}
+		}
+	}
+	if (out.len != expected_len) {
+		die("length mismatch");
+	}
+	return out;
+}
+
+static int64_t now_ms(void) {
+	return (int64_t)((double)clock() * 1000.0 / (double)CLOCKS_PER_SEC);
+}
+
+int main(int argc, char **argv) {
+	if (argc < 2) {
+		fprintf(stderr,
+			"usage:\n"
+			"  %s bench <input.bin> <iterations>\n"
+			"  %s compress <input.bin> <output.bin>\n"
+			"  %s decompress <input.bin> <output.bin>\n",
+			argv[0], argv[0], argv[0]);
+		return 1;
+	}
+	if (strcmp(argv[1], "bench") == 0) {
+		if (argc < 4) {
+			fprintf(stderr, "usage: %s bench <input.bin> <iterations>\n", argv[0]);
+			return 1;
+		}
+		int iterations = atoi(argv[3]);
+		if (iterations <= 0) {
+			fprintf(stderr, "iterations must be > 0\n");
+			return 1;
+		}
+		Buffer input = read_all(argv[2]);
+		int64_t start = now_ms();
+		for (int i = 0; i < iterations; i++) {
+			Buffer comp = compress_lz77(input.data, input.len);
+			Buffer decomp = decompress_lz77(comp.data, comp.len);
+			if (decomp.len != input.len || memcmp(decomp.data, input.data, input.len) != 0) {
+				fprintf(stderr, "roundtrip mismatch\n");
+				return 1;
+			}
+			free(comp.data);
+			free(decomp.data);
+		}
+		int64_t elapsed = now_ms() - start;
+		printf("ms=%lld\n", (long long)elapsed);
+		free(input.data);
+		return 0;
+	}
+
+	if (strcmp(argv[1], "compress") == 0) {
+		if (argc < 4) {
+			fprintf(stderr, "usage: %s compress <input.bin> <output.bin>\n", argv[0]);
+			return 1;
+		}
+		Buffer input = read_all(argv[2]);
+		Buffer comp = compress_lz77(input.data, input.len);
+		write_all(argv[3], comp.data, comp.len);
+		free(input.data);
+		free(comp.data);
+		return 0;
+	}
+
+	if (strcmp(argv[1], "decompress") == 0) {
+		if (argc < 4) {
+			fprintf(stderr, "usage: %s decompress <input.bin> <output.bin>\n", argv[0]);
+			return 1;
+		}
+		Buffer input = read_all(argv[2]);
+		Buffer dec = decompress_lz77(input.data, input.len);
+		write_all(argv[3], dec.data, dec.len);
+		free(input.data);
+		free(dec.data);
+		return 0;
+	}
+
+	fprintf(stderr, "unknown mode: %s\n", argv[1]);
+	return 1;
+}
+

tools/v/vlib/compress/lz/interop/lz77_ref.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+import sys
+import time
+
+MIN_MATCH = 3
+MAX_LITERAL = 128
+STREAM_MAGIC = b'VLZ1'
+FORMAT_LZ77 = 0
+
+
+def encode_uvarint(value: int) -> bytes:
+    out = bytearray()
+    v = value
+    while v >= 0x80:
+        out.append((v & 0x7F) | 0x80)
+        v >>= 7
+    out.append(v)
+    return bytes(out)
+
+
+def decode_uvarint(data: bytes, pos: int):
+    value = 0
+    shift = 0
+    i = pos
+    while i < len(data) and shift <= 63:
+        b = data[i]
+        i += 1
+        value |= (b & 0x7F) << shift
+        if (b & 0x80) == 0:
+            return value, i
+        shift += 7
+    raise ValueError('bad length varint')
+
+
+def compress_lz77(data: bytes) -> bytes:
+    out = bytearray()
+    out.extend(STREAM_MAGIC)
+    out.append(FORMAT_LZ77)
+    out.extend(encode_uvarint(len(data)))
+    i = 0
+    while i < len(data):
+        lit_len = min(MAX_LITERAL, len(data) - i)
+        out.append(lit_len - 1)
+        out.extend(data[i : i + lit_len])
+        i += lit_len
+    return bytes(out)
+
+
+def decompress_lz77(data: bytes) -> bytes:
+    if len(data) < 6 or data[:4] != STREAM_MAGIC:
+        raise ValueError('bad magic')
+    if data[4] != FORMAT_LZ77:
+        raise ValueError('format mismatch')
+
+    expected_len, pos = decode_uvarint(data, 5)
+    out = bytearray()
+    while pos < len(data):
+        control = data[pos]
+        pos += 1
+        if (control & 0x80) == 0:
+            literal_len = (control & 0x7F) + 1
+            if pos + literal_len > len(data):
+                raise ValueError('truncated literal')
+            out.extend(data[pos : pos + literal_len])
+            pos += literal_len
+        else:
+            length = (control & 0x7F) + MIN_MATCH
+            off, pos = decode_uvarint(data, pos)
+            if off == 0 or off > len(out):
+                raise ValueError('bad offset')
+            base = len(out) - off
+            for k in range(length):
+                out.append(out[base + k])
+    if len(out) != expected_len:
+        raise ValueError('length mismatch')
+    return bytes(out)
+
+
+def main() -> int:
+    if len(sys.argv) < 2:
+        print(
+            f'usage:\n'
+            f'  {sys.argv[0]} bench <input.bin> <iterations>\n'
+            f'  {sys.argv[0]} compress <input.bin> <output.bin>\n'
+            f'  {sys.argv[0]} decompress <input.bin> <output.bin>',
+            file=sys.stderr,
+        )
+        return 1
+
+    mode = sys.argv[1]
+    if mode == 'bench':
+        if len(sys.argv) < 4:
+            print(f'usage: {sys.argv[0]} bench <input.bin> <iterations>', file=sys.stderr)
+            return 1
+        input_path = sys.argv[2]
+        iterations = int(sys.argv[3])
+        if iterations <= 0:
+            print('iterations must be > 0', file=sys.stderr)
+            return 1
+        with open(input_path, 'rb') as f:
+            data = f.read()
+        start = time.perf_counter()
+        for _ in range(iterations):
+            enc = compress_lz77(data)
+            dec = decompress_lz77(enc)
+            if dec != data:
+                print('roundtrip mismatch', file=sys.stderr)
+                return 1
+        elapsed_ms = int((time.perf_counter() - start) * 1000)
+        print(f'ms={elapsed_ms}')
+        return 0
+
+    if mode == 'compress':
+        if len(sys.argv) < 4:
+            print(f'usage: {sys.argv[0]} compress <input.bin> <output.bin>', file=sys.stderr)
+            return 1
+        with open(sys.argv[2], 'rb') as f:
+            data = f.read()
+        out = compress_lz77(data)
+        with open(sys.argv[3], 'wb') as f:
+            f.write(out)
+        return 0
+
+    if mode == 'decompress':
+        if len(sys.argv) < 4:
+            print(f'usage: {sys.argv[0]} decompress <input.bin> <output.bin>', file=sys.stderr)
+            return 1
+        with open(sys.argv[2], 'rb') as f:
+            data = f.read()
+        out = decompress_lz77(data)
+        with open(sys.argv[3], 'wb') as f:
+            f.write(out)
+        return 0
+
+    print(f'unknown mode: {mode}', file=sys.stderr)
+    return 1
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())
+

tools/v/vlib/compress/lz/interop/lz_interop.v

@@ -0,0 +1,172 @@
+module main
+
+import compress.lz
+import os
+
+const default_data_size = 512 * 1024
+
+fn main() {
+	mut data_size := default_data_size
+	if os.args.len > 1 {
+		parsed := int(os.args[1].i32())
+		if parsed > 0 {
+			data_size = parsed
+		}
+	}
+	data := `z`.repeat(data_size).bytes()
+
+	println('LZ interop input: ${data.len} bytes')
+
+	for format in [lz.Format.lz77, .lz78, .lzw, .lz4, .lzss, .lzma, .lzma2, .lzjb] {
+		validate_v_roundtrip(data, format) or {
+			eprintln('V validation failed for ${format}: ${err.msg()}')
+			exit(1)
+		}
+		println('V roundtrip (${format}): OK')
+	}
+
+	tmp_dir := os.join_path(os.temp_dir(), 'v_lz_interop')
+	os.mkdir_all(tmp_dir) or {
+		eprintln('Could not create temp directory ${tmp_dir}: ${err.msg()}')
+		exit(1)
+	}
+	defer {
+		os.rmdir_all(tmp_dir) or {
+			eprintln('Could not remove temp directory ${tmp_dir}: ${err.msg()}')
+		}
+	}
+	input_path := os.join_path(tmp_dir, 'input.bin')
+	os.write_file_array(input_path, data) or {
+		eprintln('Could not write input file ${input_path}: ${err.msg()}')
+		exit(1)
+	}
+	mut c_bin := ''
+	if bin := compile_c_runner() {
+		c_bin = bin
+	} else {
+		eprintln('Skipping C benchmark: ${err.msg()}')
+	}
+	python_ok := has_python3()
+	if !python_ok {
+		eprintln('Skipping Python benchmark: python3 is not available')
+	}
+
+	if c_bin.len > 0 {
+		cross_validate_v_c(c_bin, data, input_path, tmp_dir) or {
+			eprintln('Cross-validation V<->C failed: ${err.msg()}')
+			exit(1)
+		}
+		println('Cross-validation: V<->C compress/decompress OK')
+	} else {
+		println('Cross-validation: skipped V<->C (requires C compiler)')
+	}
+
+	if python_ok {
+		cross_validate_v_python(data, input_path, tmp_dir) or {
+			eprintln('Cross-validation V<->Python failed: ${err.msg()}')
+			exit(1)
+		}
+		println('Cross-validation: V<->Python compress/decompress OK')
+	} else {
+		println('Cross-validation: skipped V<->Python (requires python3)')
+	}
+}
+
+fn validate_v_roundtrip(data []u8, format lz.Format) ! {
+	encoded := lz.compress(data, format)!
+	decoded := lz.decompress(encoded, format)!
+	if decoded != data {
+		return error('roundtrip mismatch for ${format}')
+	}
+}
+
+fn compile_c_runner() !string {
+	cc := choose_cc()
+	if cc.len == 0 {
+		return error('no C compiler found (tried cc, gcc, and clang)')
+	}
+	bin_path := os.join_path(os.temp_dir(), 'lz77_ref_bench')
+	c_src := os.join_path(@DIR, 'lz77_ref.c')
+	compile_cmd := '${cc} -O3 -std=c99 ${os.quoted_path(c_src)} -o ${os.quoted_path(bin_path)}'
+	compile_res := os.execute(compile_cmd)
+	if compile_res.exit_code != 0 {
+		return error('C compile failed: ${compile_res.output.trim_space()}')
+	}
+	return bin_path
+}
+
+fn choose_cc() string {
+	for cc in ['cc', 'gcc', 'clang'] {
+		if os.execute('${cc} --version').exit_code == 0 {
+			return cc
+		}
+	}
+	return ''
+}
+
+fn has_python3() bool {
+	return os.execute('python3 --version').exit_code == 0
+}
+
+fn cross_validate_v_c(c_bin string, original []u8, input_path string, tmp_dir string) ! {
+	v_encoded := os.join_path(tmp_dir, 'v_encoded.bin')
+	c_decoded := os.join_path(tmp_dir, 'c_decoded.bin')
+	c_encoded := os.join_path(tmp_dir, 'c_encoded.bin')
+
+	v_stream := lz.compress_lz77(original)!
+	os.write_file_array(v_encoded, v_stream)!
+
+	mut res :=
+		os.execute('${os.quoted_path(c_bin)} decompress ${os.quoted_path(v_encoded)} ${os.quoted_path(c_decoded)}')
+	if res.exit_code != 0 {
+		return error('C decompress(V output) failed: ${res.output.trim_space()}')
+	}
+	validate_equal_files(input_path, c_decoded, 'V->C')!
+
+	res =
+		os.execute('${os.quoted_path(c_bin)} compress ${os.quoted_path(input_path)} ${os.quoted_path(c_encoded)}')
+	if res.exit_code != 0 {
+		return error('C compress failed: ${res.output.trim_space()}')
+	}
+	c_encoded_data := os.read_bytes(c_encoded)!
+	v_decoded := lz.decompress_lz77(c_encoded_data)!
+	if v_decoded != original {
+		return error('C->V output mismatch')
+	}
+}
+
+fn cross_validate_v_python(original []u8, input_path string, tmp_dir string) ! {
+	v_encoded := os.join_path(tmp_dir, 'v_encoded_for_py.bin')
+	py_decoded := os.join_path(tmp_dir, 'py_decoded.bin')
+	py_encoded := os.join_path(tmp_dir, 'py_encoded.bin')
+	py_script := os.join_path(@DIR, 'lz77_ref.py')
+
+	v_stream := lz.compress_lz77(original)!
+	os.write_file_array(v_encoded, v_stream)!
+
+	mut res :=
+		os.execute('python3 ${os.quoted_path(py_script)} decompress ${os.quoted_path(v_encoded)} ${os.quoted_path(py_decoded)}')
+	if res.exit_code != 0 {
+		return error('Python decompress(V output) failed: ${res.output.trim_space()}')
+	}
+	validate_equal_files(input_path, py_decoded, 'V->Python')!
+
+	res =
+		os.execute('python3 ${os.quoted_path(py_script)} compress ${os.quoted_path(input_path)} ${os.quoted_path(py_encoded)}')
+	if res.exit_code != 0 {
+		return error('Python compress failed: ${res.output.trim_space()}')
+	}
+	py_encoded_data := os.read_bytes(py_encoded)!
+	v_decoded := lz.decompress_lz77(py_encoded_data)!
+	if v_decoded != original {
+		return error('Python->V output mismatch')
+	}
+}
+
+fn validate_equal_files(expected_path string, actual_path string, tag string) ! {
+	expected := os.read_bytes(expected_path)!
+	actual := os.read_bytes(actual_path)!
+	if expected != actual {
+		return error('${tag} output mismatch')
+	}
+}