Initial project import

tools/v/vlib/compress/zstd/README.md

@@ -0,0 +1,25 @@
+## Description
+
+`compress.zstd` is a module that assists in the compression and
+decompression of binary data using `zstd` compression.
+
+## Examples
+
+Decode a `.zst` file from disk:
+
+```v
+import os
+import compress.zstd
+
+fn main() {
+	compressed := os.read_bytes('document.zst') or { panic(err) }
+	decompressed := zstd.decompress(compressed) or { panic(err) }
+	println(decompressed.bytestr())
+}
+```
+
+For a runnable example in this repository, see
+[examples/read_zstd_file.v](examples/read_zstd_file.v).
+
+`read_zstd_files_test.v` is a test file. Run it with
+`v test read_zstd_files_test.v`, not with `v read_zstd_files_test.v`.

tools/v/vlib/compress/zstd/examples/read_zstd_file.v

@@ -0,0 +1,13 @@
+module main
+
+import os
+import compress.zstd
+
+const sample_file = os.join_path(os.dir(@FILE), '..', 'samples', 'known.zst')
+
+fn main() {
+	compressed := os.read_bytes(sample_file) or { panic(err) }
+	decompressed := zstd.decompress(compressed) or { panic(err) }
+	println('decoded `${sample_file}`:')
+	println(decompressed.bytestr())
+}

tools/v/vlib/compress/zstd/read_zstd_files_test.v

@@ -0,0 +1,41 @@
+import os
+import compress.zstd
+
+const samples_folder = os.join_path(os.dir(@FILE), 'samples')
+
+fn s(fname string) string {
+	return os.join_path(samples_folder, fname)
+}
+
+fn read_and_decode_file(fpath string) !([]u8, string) {
+	compressed := os.read_bytes(fpath)!
+	decoded := zstd.decompress(compressed)!
+	content := decoded.bytestr()
+	return compressed, content
+}
+
+fn test_reading_and_decoding_a_known_zstded_file() {
+	compressed, content := read_and_decode_file(s('known.zst'))!
+	assert compressed#[0..3] == [u8(40), 181, 47]
+	assert compressed#[-5..] == [u8(10), 78, 32, 170, 44]
+	assert content.contains('## Description')
+	assert content.contains('## Examples:')
+	assert content.ends_with('```')
+}
+
+fn test_decoding_all_samples_files() {
+	for zstd_file in os.walk_ext(samples_folder, '.zst') {
+		_, content := read_and_decode_file(zstd_file)!
+		assert content.len > 0, 'decoded content should not be empty: `${content}`'
+	}
+}
+
+fn test_reading_zstd_files_compressed_with_different_compress_level() {
+	_, content1 := read_and_decode_file(s('readme_level_1.zst'))!
+	_, content5 := read_and_decode_file(s('readme_level_5.zst'))!
+	_, content9 := read_and_decode_file(s('readme_level_9.zst'))!
+	_, content19 := read_and_decode_file(s('readme_level_19.zst'))!
+	assert content19 == content9
+	assert content9 == content5
+	assert content5 == content1
+}

tools/v/vlib/compress/zstd/zstd.c.v

@@ -0,0 +1,649 @@
+// zstd(https://github.com/facebook/zstd) is a fast real-time compression algorithm developed by Facebook.
+// zstd compression/decompression
+module zstd
+
+import os
+import encoding.binary
+
+#flag -I @VMODROOT/thirdparty/zstd
+#include "zstd.c"	// msvc can't compile multiple source files, so included
+
+const frame_header_size_max = 18
+const content_size_unknown = u64(-1)
+const content_size_error = u64(-2)
+
+const buf_in_size = 1024 * 1024
+const buf_out_size = 1024 * 1024
+
+fn C.ZSTD_versionNumber() u32
+fn C.ZSTD_versionString() charptr
+
+fn C.ZSTD_compress(voidptr, usize, voidptr, usize, i32) usize
+fn C.ZSTD_decompress(voidptr, usize, voidptr, usize) usize
+fn C.ZSTD_getFrameContentSize(voidptr, usize) u64
+fn C.ZSTD_findFrameCompressedSize(voidptr, usize) usize
+fn C.ZSTD_compressBound(usize) usize
+fn C.ZSTD_isError(usize) u32
+fn C.ZSTD_getErrorName(usize) charptr
+fn C.ZSTD_minCLevel() i32
+fn C.ZSTD_maxCLevel() i32
+fn C.ZSTD_defaultCLevel() i32
+fn C.ZSTD_createCCtx() &C.ZSTD_CCtx_s
+fn C.ZSTD_freeCCtx(voidptr) usize
+fn C.ZSTD_compressCCtx(voidptr, voidptr, usize, voidptr, usize, i32) usize
+fn C.ZSTD_createDCtx() &C.ZSTD_DCtx_s
+fn C.ZSTD_freeDCtx(voidptr) usize
+fn C.ZSTD_decompressDCtx(voidptr, voidptr, usize, voidptr, usize) usize
+
+// note : new strategies _might_ be added in the future. Only the order (from fast to strong) is guaranteed
+pub enum Strategy {
+	default  = 0
+	fast     = 1
+	dfast    = 2
+	greedy   = 3
+	lazy     = 4
+	lazy2    = 5
+	btlazy2  = 6
+	btopt    = 7
+	btultra  = 8
+	btultra2 = 9
+}
+
+pub enum CParameter {
+	// compression parameters
+	// Note: When compressing with a ZSTD_CDict these parameters are superseded
+	// by the parameters used to construct the ZSTD_CDict.
+	// See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict).
+	//
+	// Set compression parameters according to pre-defined cLevel table.
+	// Note that exact compression parameters are dynamically determined,
+	// depending on both compression level and srcSize (when known).
+	// Default level is ZSTD_CLEVEL_DEFAULT==3.
+	// Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
+	// Note 1 : it's possible to pass a negative compression level.
+	// Note 2 : setting a level does not automatically set all other compression parameters
+	// to default. Setting this will however eventually dynamically impact the compression
+	// parameters which have not been manually set. The manually set
+	// ones will 'stick'.
+	compression_level = 100
+	// Advanced compression parameters :
+	// It's possible to pin down compression parameters to some specific values.
+	// In which case, these values are no longer dynamically selected by the compressor
+	//
+	// Maximum allowed back-reference distance, expressed as power of 2.
+	// This will set a memory budget for streaming decompression,
+	// with larger values requiring more memory
+	// and typically compressing more.
+	// Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
+	// Special: value 0 means "use default windowLog".
+	// Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
+	// requires explicitly allowing such size at streaming decompression stage.
+	window_log = 101
+	// Size of the initial probe table, as a power of 2.
+	// Resulting memory usage is (1 << (hashLog+2)).
+	// Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
+	// Larger tables improve compression ratio of strategies <= dFast,
+	// and improve speed of strategies > dFast.
+	// Special: value 0 means "use default hashLog".
+	hash_log = 102
+	// Size of the multi-probe search table, as a power of 2.
+	// Resulting memory usage is (1 << (chainLog+2)).
+	// Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
+	// Larger tables result in better and slower compression.
+	// This parameter is useless for "fast" strategy.
+	// It's still useful when using "dfast" strategy,
+	// in which case it defines a secondary probe table.
+	// Special: value 0 means "use default chainLog".
+	chain_log = 103
+	// Number of search attempts, as a power of 2.
+	// More attempts result in better and slower compression.
+	// This parameter is useless for "fast" and "dFast" strategies.
+	// Special: value 0 means "use default searchLog".
+	search_log = 104
+	// Minimum size of searched matches.
+	// Note that Zstandard can still find matches of smaller size,
+	// it just tweaks its search algorithm to look for this size and larger.
+	// Larger values increase compression and decompression speed, but decrease ratio.
+	// Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
+	// Note that currently, for all strategies < btopt, effective minimum is 4.
+	// , for all strategies > fast, effective maximum is 6.
+	// Special: value 0 means "use default minMatchLength".
+	min_match = 105
+	// Impact of this field depends on strategy.
+	// For strategies btopt, btultra & btultra2:
+	// Length of Match considered "good enough" to stop search.
+	// Larger values make compression stronger, and slower.
+	// For strategy fast:
+	// Distance between match sampling.
+	// Larger values make compression faster, and weaker.
+	// Special: value 0 means "use default targetLength".
+	target_length = 106
+	// See ZSTD_strategy enum definition.
+	// The higher the value of selected strategy, the more complex it is,
+	// resulting in stronger and slower compression.
+	// Special: value 0 means "use default strategy".
+	strategy = 107
+	// v1.5.6+
+	// Attempts to fit compressed block size into approximately targetCBlockSize.
+	// Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX.
+	// Note that it's not a guarantee, just a convergence target (default:0).
+	// No target when targetCBlockSize == 0.
+	// This is helpful in low bandwidth streaming environments to improve end-to-end latency,
+	// when a client can make use of partial documents (a prominent example being Chrome).
+	// Note: this parameter is stable since v1.5.6.
+	// It was present as an experimental parameter in earlier versions,
+	// but it's not recommended using it with earlier library versions
+	// due to massive performance regressions.
+	target_c_block_size = 130
+	// LDM mode parameters
+	// Enable long distance matching.
+	// This parameter is designed to improve compression ratio
+	// for large inputs, by finding large matches at long distance.
+	// It increases memory usage and window size.
+	// Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
+	// except when expressly set to a different value.
+	// Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
+	// compression strategy >= ZSTD_btopt (== compression level 16+)
+	enable_long_distance_matching = 160
+	// Size of the table for long distance matching, as a power of 2.
+	// Larger values increase memory usage and compression ratio,
+	// but decrease compression speed.
+	// Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
+	// default: windowlog - 7.
+	// Special: value 0 means "automatically determine hashlog".
+	ldm_hash_log = 161
+	// Minimum match size for long distance matcher.
+	// Larger/too small values usually decrease compression ratio.
+	// Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
+	// Special: value 0 means "use default value" (default: 64).
+	ldm_min_match = 162
+	// log size of each bucket in the ldm hash table for collision resolution.
+	// Larger values improve collision resolution but decrease compression speed.
+	// The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
+	// Special: value 0 means "use default value" (default: 3).
+	ldm_bucket_size_log = 163
+	// Frequency of inserting/looking up entries into the LDM hash table.
+	// Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
+	// Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
+	// Larger values improve compression speed.
+	// Deviating far from default value will likely result in a compression ratio decrease.
+	// Special: value 0 means "automatically determine hashRateLog".
+	ldm_hash_rate_log = 164
+	// frame parameters
+	// Content size will be written into frame header _whenever known_ (default:1)
+	// Content size must be known at the beginning of compression.
+	// This is automatically the case when using ZSTD_compress2(),
+	// For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize()
+	content_size_flag = 200
+	// A 32-bits checksum of content is written at end of frame (default:0)
+	checksum_flag = 201
+	// When applicable, dictionary's ID is written into frame header (default:1)
+	dict_id_flag = 202
+	// multi-threading parameters
+	// These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
+	// Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
+	// In a situation where it's unknown if the linked library supports multi-threading or not,
+	// setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
+	//
+	// Select how many threads will be spawned to compress in parallel.
+	// When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
+	// ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
+	// while compression is performed in parallel, within worker thread(s).
+	// (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
+	// in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
+	// More workers improve speed, but also increase memory usage.
+	// Default value is `0`, aka "single-threaded mode" : no worker is spawned,
+	// compression is performed inside Caller's thread, and all invocations are blocking
+	nb_workers = 400
+	// Size of a compression job. This value is enforced only when nbWorkers >= 1.
+	// Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
+	// 0 means default, which is dynamically determined based on compression parameters.
+	// Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
+	// The minimum size is automatically and transparently enforced.
+	job_size = 401
+	// Control the overlap size, as a fraction of window size.
+	// The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
+	// It helps preserve compression ratio, while each job is compressed in parallel.
+	// This value is enforced only when nbWorkers >= 1.
+	// Larger values increase compression ratio, but decrease speed.
+	// Possible values range from 0 to 9 :
+	// - 0 means "default" : value will be determined by the library, depending on strategy
+	// - 1 means "no overlap"
+	// - 9 means "full overlap", using a full window size.
+	// Each intermediate rank increases/decreases load size by a factor 2 :
+	// 9: full window;  8: w/2;  7: w/4;  6: w/8;  5:w/16;  4: w/32;  3:w/64;  2:w/128;  1:no overlap;  0:default
+	// default value varies between 6 and 9, depending on strategy
+	overlap_log = 402
+	// note : additional experimental parameters are also available
+	// within the experimental section of the API.
+	// At the time of this writing, they include :
+	// zstd_c_rsyncable
+	// zstd_c_format
+	// zstd_c_force_max_window
+	// zstd_c_force_attach_dict
+	// zstd_c_literal_compression_mode
+	// zstd_c_target_c_block_size
+	// zstd_c_src_size_hint
+	// zstd_c_enable_dedicated_dict_search
+	// zstd_c_stable_in_buffer
+	// zstd_c_stable_out_buffer
+	// zstd_c_block_delimiters
+	// zstd_c_validate_sequences
+	// zstd_c_use_block_splitter
+	// zstd_c_use_row_match_finder
+	// zstd_c_prefetch_c_dict_tables
+	// zstd_c_enable_seq_producer_fallback
+	// zstd_c_max_block_size
+	// Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+	// note : never ever use experimentalParam? names directly;
+	//        also, the enums values themselves are unstable and can still change.
+	//
+	experimental_param1 = 500
+	experimental_param2 = 10
+	experimental_param3 = 1000
+	experimental_param4 = 1001
+	experimental_param5 = 1002
+	// experimental_param6  = 1003 is now ZSTD_c_targetCBlockSize
+	experimental_param7  = 1004
+	experimental_param8  = 1005
+	experimental_param9  = 1006
+	experimental_param10 = 1007
+	experimental_param11 = 1008
+	experimental_param12 = 1009
+	experimental_param13 = 1010
+	experimental_param14 = 1011
+	experimental_param15 = 1012
+	experimental_param16 = 1013
+	experimental_param17 = 1014
+	experimental_param18 = 1015
+	experimental_param19 = 1016
+	experimental_param20 = 1017
+}
+
+pub struct Bounds {
+pub:
+	error       usize
+	lower_bound int
+	upper_bound int
+}
+
+fn C.ZSTD_cParam_getBounds(CParameter) Bounds
+fn C.ZSTD_CCtx_setParameter(voidptr, CParameter, i32) usize
+fn C.ZSTD_CCtx_setPledgedSrcSize(voidptr, u64) usize
+
+pub enum ResetDirective {
+	session_only           = 1
+	parameters             = 2
+	session_and_parameters = 3
+}
+
+fn C.ZSTD_CCtx_reset(voidptr, ResetDirective) usize
+fn C.ZSTD_compress2(voidptr, voidptr, usize, voidptr, usize) usize
+
+pub enum DParameter {
+	// Select a size limit (in power of 2) beyond which
+	// the streaming API will refuse to allocate memory buffer
+	// in order to protect the host from unreasonable memory requirements.
+	// This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
+	// By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
+	// Special: value 0 means "use default maximum windowLog".
+	window_log_max = 100
+	// note : additional experimental parameters are also available
+	// within the experimental section of the API.
+	// At the time of this writing, they include :
+	// ZSTD_d_format
+	// zstd_d_stable_out_buffer
+	// zstd_d_force_ignore_checksum
+	// zstd_d_ref_multipled_dicts
+	// zstd_d_disable_huffman_assembly
+	// Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
+	// note : never ever use experimentalParam? names directly
+	experimental_param1 = 1000
+	experimental_param2 = 1001
+	experimental_param3 = 1002
+	experimental_param4 = 1003
+	experimental_param5 = 1004
+	experimental_param6 = 1005
+}
+
+fn C.ZSTD_dParam_getBounds(DParameter) Bounds
+fn C.ZSTD_DCtx_setParameter(voidptr, DParameter, i32) usize
+fn C.ZSTD_DCtx_reset(voidptr, ResetDirective) usize
+
+// streaming compression
+pub struct InBuffer {
+pub mut:
+	src  voidptr
+	size usize
+	pos  usize
+}
+
+pub struct OutBuffer {
+pub mut:
+	dst  voidptr
+	size usize
+	pos  usize
+}
+
+fn C.ZSTD_createCStream() voidptr
+fn C.ZSTD_freeCStream(voidptr) usize
+
+pub enum EndDirective {
+	// collect more data, encoder decides when to output compressed result, for optimal compression ratio
+	continue = 0
+	// flush any data provided so far,
+	// it creates (at least) one new block, that can be decoded immediately on reception;
+	// frame will continue: any future data can still reference previously compressed data, improving compression.
+	// note : multithreaded compression will block to flush as much output as possible.
+	flush = 1
+	// flush any remaining data _and_ close current frame.
+	// note that frame is only closed after compressed data is fully flushed (return value == 0).
+	// After that point, any additional data starts a new frame.
+	// note : each frame is independent (does not reference any content from previous frame).
+	// note : multithreaded compression will block to flush as much output as possible.
+	end = 2
+}
+
+fn C.ZSTD_compressStream2(voidptr, &OutBuffer, &InBuffer, EndDirective) usize
+fn C.ZSTD_CStreamInSize() usize
+fn C.ZSTD_CStreamOutSize() usize
+fn C.ZSTD_initCStream(voidptr, i32) usize
+fn C.ZSTD_compressStream(voidptr, &OutBuffer, &InBuffer) usize
+fn C.ZSTD_flushStream(voidptr, &OutBuffer) usize
+fn C.ZSTD_endStream(voidptr, &OutBuffer) usize
+
+// streaming decompression
+fn C.ZSTD_createDStream() voidptr
+fn C.ZSTD_freeDStream(voidptr) usize
+fn C.ZSTD_initDStream(voidptr) usize
+fn C.ZSTD_decompressStream(voidptr, &OutBuffer, &InBuffer) usize
+fn C.ZSTD_DStreamInSize() usize
+fn C.ZSTD_DStreamOutSize() usize
+
+// version_number returns runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE).
+pub fn version_number() u32 {
+	return C.ZSTD_versionNumber()
+}
+
+// version_string returns runtime library version, like "1.5.7".
+pub fn version_string() string {
+	return unsafe { tos_clone(C.ZSTD_versionString()) }
+}
+
+// is_error tells if a `usize` function result is an error code.
+pub fn is_error(code usize) bool {
+	return C.ZSTD_isError(code) == 1
+}
+
+// get_error_name provides readable string from an error code.
+pub fn get_error_name(code usize) string {
+	return unsafe { tos_clone(&u8(C.ZSTD_getErrorName(code))) }
+}
+
+// check_zstd checks the zstd error code, and return a error string.
+pub fn check_error(code usize) ! {
+	if is_error(code) {
+		return error(get_error_name(code))
+	}
+}
+
+// min_c_level returns minimum negative compression level allowed.
+pub fn min_c_level() int {
+	return C.ZSTD_minCLevel()
+}
+
+// max_c_level returns maximum compression level available.
+pub fn max_c_level() int {
+	return C.ZSTD_maxCLevel()
+}
+
+// default_c_level returns default compression level.
+pub fn default_c_level() int {
+	return C.ZSTD_defaultCLevel()
+}
+
+@[params]
+pub struct CompressParams {
+pub:
+	// 1~22
+	compression_level int = default_c_level()
+	// how many threads will be spawned to compress in parallel
+	nb_threads    int      = 1
+	checksum_flag bool     = true
+	strategy      Strategy = .default
+}
+
+// compresses an array of bytes using zstd and returns the compressed bytes in a new array
+// extra compression parameters can be set by `params`
+// Example: b := 'abcdef'.repeat(1000).bytes(); cmpr := zstd.compress(b, compression_level: 10)!; assert cmpr.len < b.len; dc := zstd.decompress(cmpr)!; assert b == dc
+pub fn compress(data []u8, params CompressParams) ![]u8 {
+	dst_capacity := C.ZSTD_compressBound(data.len)
+	check_error(dst_capacity)!
+	mut dst := []u8{len: int(dst_capacity)}
+	mut cctx := new_cctx(params)!
+	defer {
+		cctx.free_cctx()
+	}
+	size := C.ZSTD_compress2(cctx.ctx, dst.data, dst.len, data.data, data.len)
+	check_error(size)!
+	return dst[..int(size)]
+}
+
+@[params]
+pub struct DecompressParams {
+pub:
+	window_log_max int
+}
+
+// decompresses an array of bytes using zstd and returns the decompressed bytes in a new array
+// extra decompression parameters can be set by `params`
+// Example: b := 'abcdef'.repeat(1000).bytes(); cmpr := zstd.compress(b, compression_level: 10)!; assert cmpr.len < b.len; dc := zstd.decompress(cmpr)!; assert b == dc
+pub fn decompress(data []u8, params DecompressParams) ![]u8 {
+	dst_capacity := C.ZSTD_getFrameContentSize(data.data, frame_header_size_max)
+	if dst_capacity == content_size_unknown {
+		return error('The size cannot be determined, try use streaming mode to decompress data?')
+	} else if dst_capacity == content_size_error {
+		return error('An error occurred (e.g. invalid magic number, srcSize too small)')
+	} else if dst_capacity == 0 {
+		return error('The frame is valid but empty')
+	}
+	mut dst := []u8{len: int(dst_capacity)}
+	decompressed_size := C.ZSTD_decompress(dst.data, dst.len, data.data, data.len)
+	check_error(decompressed_size)!
+	return dst[..int(decompressed_size)]
+}
+
+pub struct CCtx {
+mut:
+	ctx &C.ZSTD_CCtx_s
+}
+
+// new_cctx create a compression context.
+// extra compression parameters can be set by `params`
+pub fn new_cctx(params CompressParams) !&CCtx {
+	mut ctx := C.ZSTD_createCCtx()
+	if isnil(ctx) {
+		return error('new_cctx() failed!')
+	}
+	mut cctx := &CCtx{ctx}
+	cctx.set(.compression_level, params.compression_level)!
+	$if !(tinyc && windows) {
+		// TODO: tinyc on windows doesn't support multiple thread
+		cctx.set(.nb_workers, params.nb_threads)!
+	}
+	cctx.set(.checksum_flag, if params.checksum_flag { 1 } else { 0 })!
+	cctx.set(.strategy, int(params.strategy))!
+	return cctx
+}
+
+// set_parameter set compression parameter `c_param` to value `val`
+pub fn (mut c CCtx) set(c_param CParameter, val int) ! {
+	check_error(C.ZSTD_CCtx_setParameter(c.ctx, c_param, val))!
+}
+
+// compress_stream2 do stream compress on `input`, and store compressed data in `output`.
+// `mode`:
+// 	.zstd_e_continue => continue stream compression.
+// 	.zstd_e_flush => flush data
+// 	.zstd_e_end => it is the last frame
+pub fn (mut c CCtx) compress_stream2(output &OutBuffer, input &InBuffer, mode EndDirective) !usize {
+	res := C.ZSTD_compressStream2(c.ctx, output, input, mode)
+	check_error(res)!
+	return res
+}
+
+// free_cctx free a compression context.
+pub fn (mut c CCtx) free_cctx() usize {
+	return C.ZSTD_freeCCtx(c.ctx)
+}
+
+struct C.ZSTD_CCtx_s {}
+
+struct C.ZSTD_DCtx_s {}
+
+pub struct DCtx {
+mut:
+	ctx &C.ZSTD_DCtx_s
+}
+
+// new_dctx creates a decompression context.
+// extra decompression parameters can be set by `params`
+pub fn new_dctx(params DecompressParams) !&DCtx {
+	mut ctx := C.ZSTD_createDCtx()
+	if isnil(ctx) {
+		return error('new_dctx() failed!')
+	}
+	mut dctx := &DCtx{ctx}
+	dctx.set(.window_log_max, params.window_log_max)!
+	return dctx
+}
+
+// set_parameter sets decompression parameter `d_param` to value `val`
+pub fn (mut d DCtx) set(d_param DParameter, val int) ! {
+	check_error(C.ZSTD_DCtx_setParameter(d.ctx, d_param, val))!
+}
+
+// decompress_stream do stream decompress on `input`, and store decompressed data in `output`.
+// return remaining bytes in `input` stream
+pub fn (mut d DCtx) decompress_stream(output &OutBuffer, input &InBuffer) !usize {
+	res := C.ZSTD_decompressStream(d.ctx, output, input)
+	check_error(res)!
+	return res
+}
+
+// free_cctx free a compression context
+pub fn (mut d DCtx) free_dctx() usize {
+	return C.ZSTD_freeDCtx(d.ctx)
+}
+
+// store_array compress an `array`'s data, and store it to file `fname`.
+// extra compression parameters can be set by `params`
+// WARNING: Because struct padding, some data in struct may be marked unused.
+// So, when `store_array`, it will cause memory fsanitize fail with 'use-of-uninitialized-value'.
+// It can be safely ignored.
+// For example, following struct may cause memory fsanitize fail:
+// struct MemoryTrace {
+// 	operation u8
+// 	address   u64
+// 	size      u8
+// }
+// By changing it into following , you can pass the memory fsanitize check :
+// struct MemoryTrace {
+// 	operation u64
+// 	address   u64
+// 	size      u64
+// }
+pub fn store_array[T](fname string, array []T, params CompressParams) ! {
+	mut fout := os.open_file(fname, 'wb')!
+	mut cctx := new_cctx(params)!
+	defer {
+		cctx.free_cctx()
+		fout.close()
+	}
+
+	mut buf_out := []u8{len: buf_out_size}
+	mut input := &InBuffer{}
+	mut output := &OutBuffer{}
+	// first, write the array.len to file
+	mut len_buf := []u8{len: 8}
+	binary.little_endian_put_u64(mut len_buf, u64(array.len))
+	input.src = len_buf.data
+	input.size = 8
+	input.pos = 0
+	output.dst = buf_out.data
+	output.size = buf_out_size
+	output.pos = 0
+	mut remaining := cctx.compress_stream2(output, input, .flush)!
+	fout.write(buf_out[..int(output.pos)])!
+	// then, write the array.data to file
+	input.src = array.data
+	input.size = usize(array.len * int(sizeof(T)))
+	input.pos = 0
+	output.dst = buf_out.data
+	output.size = buf_out_size
+	output.pos = 0
+	for {
+		output.dst = buf_out.data
+		output.size = buf_out_size
+		output.pos = 0
+		remaining = cctx.compress_stream2(output, input, .end)!
+		fout.write(buf_out[..int(output.pos)])!
+		if remaining == 0 {
+			break
+		}
+	}
+}
+
+// load_array return an array which data is decompressed from a file `fname`.
+// extra decompression parameters can be set by `params`
+pub fn load_array[T](fname string, params DecompressParams) ![]T {
+	mut fin := os.open_file(fname, 'rb')!
+	mut dctx := new_dctx(params)!
+	defer {
+		dctx.free_dctx()
+		fin.close()
+	}
+
+	mut buf_in := []u8{len: buf_in_size}
+	mut len_buf := []u8{len: 8}
+	mut input := &InBuffer{}
+	mut output := &OutBuffer{}
+	mut last_ret := usize(0)
+	mut last_chunk := false
+	// first, read the array.len from file
+	mut read_len := fin.read(mut buf_in)!
+	last_chunk = read_len < buf_in.len
+	input.src = buf_in.data
+	input.size = usize(read_len)
+	input.pos = 0
+	output.dst = len_buf.data
+	output.size = usize(len_buf.len)
+	output.pos = 0
+	last_ret = dctx.decompress_stream(output, input)!
+	len := binary.little_endian_u64(len_buf)
+	// then, read the array.data from file
+	mut result := []T{len: int(len)}
+	output.dst = result.data
+	output.size = usize(result.len) * sizeof(T)
+	output.pos = 0
+	last_ret = dctx.decompress_stream(output, input)!
+	for !last_chunk {
+		read_len = fin.read(mut buf_in)!
+		last_chunk = read_len < buf_in.len
+		input.size = usize(read_len)
+		input.pos = 0
+		for input.pos < input.size {
+			last_ret = dctx.decompress_stream(output, input)!
+		}
+		if read_len < buf_in.len {
+			break
+		}
+	}
+	if last_ret != 0 {
+		// The last return value from ZSTD_decompressStream did not end on a
+		// frame, but we reached the end of the file! We assume this is an
+		// error, and the input was truncated.
+		return error('EOF before end of stream: ${last_ret}')
+	}
+	return result
+}

tools/v/vlib/compress/zstd/zstd_d_musl.c.v

@@ -0,0 +1,4 @@
+module zstd
+
+// As MUSL has no `qsort_r()` as GLIBC, so fallback to `qsort()`
+#flag -DZSTD_USE_C90_QSORT

tools/v/vlib/compress/zstd/zstd_test.v

@@ -0,0 +1,364 @@
+module zstd
+
+import os
+
+const samples_folder = os.join_path(os.dir(@FILE), 'samples')
+
+fn s(fname string) string {
+	return os.join_path(samples_folder, fname)
+}
+
+fn test_zstd() {
+	assert version_number() >= 10505
+
+	uncompressed := 'Hello world!'.repeat(10000)
+	compressed := compress(uncompressed.bytes())!
+	decompressed := decompress(compressed)!
+	assert decompressed == uncompressed.bytes()
+}
+
+fn test_zstd_deferent_compression_level() {
+	uncompressed := 'Hello world!'.repeat(10000)
+
+	compressed_1000 := compress(uncompressed.bytes(), compression_level: 1000)!
+	decompressed_1000 := decompress(compressed_1000)!
+	assert decompressed_1000 == uncompressed.bytes()
+
+	compressed_0 := compress(uncompressed.bytes(), compression_level: 0)!
+	decompressed_0 := decompress(compressed_0)!
+	assert decompressed_0 == uncompressed.bytes()
+
+	compressed_1 := compress(uncompressed.bytes(), compression_level: 1)!
+	decompressed_1 := decompress(compressed_1)!
+	assert decompressed_1 == uncompressed.bytes()
+
+	compressed_15 := compress(uncompressed.bytes(), compression_level: 15)!
+	decompressed_15 := decompress(compressed_15)!
+	assert decompressed_15 == uncompressed.bytes()
+}
+
+fn test_zstd_nb_threads() {
+	uncompressed := 'Hello world!'.repeat(10000)
+
+	compressed_0 := compress(uncompressed.bytes(), nb_threads: 0)!
+	decompressed_0 := decompress(compressed_0)!
+	assert decompressed_0 == uncompressed.bytes()
+
+	compressed_1 := compress(uncompressed.bytes(), nb_threads: 1)!
+	decompressed_1 := decompress(compressed_1)!
+	assert decompressed_1 == uncompressed.bytes()
+
+	compressed_15 := compress(uncompressed.bytes(), nb_threads: 15)!
+	decompressed_15 := decompress(compressed_15)!
+	assert decompressed_15 == uncompressed.bytes()
+}
+
+fn test_zstd_checksum_flag() {
+	uncompressed := 'Hello world!'.repeat(10000)
+
+	compressed_true := compress(uncompressed.bytes(), checksum_flag: true)!
+	decompressed_true := decompress(compressed_true)!
+	assert decompressed_true == uncompressed.bytes()
+
+	compressed_false := compress(uncompressed.bytes(), checksum_flag: false)!
+	decompressed_false := decompress(compressed_false)!
+	assert decompressed_false == uncompressed.bytes()
+}
+
+fn test_zstd_deferent_strategy() {
+	uncompressed := 'Hello world!'.repeat(10000)
+
+	compressed_default := compress(uncompressed.bytes(), strategy: .default)!
+	decompressed_default := decompress(compressed_default)!
+	assert decompressed_default == uncompressed.bytes()
+
+	compressed_fast := compress(uncompressed.bytes(), strategy: .fast)!
+	decompressed_fast := decompress(compressed_fast)!
+	assert decompressed_fast == uncompressed.bytes()
+
+	compressed_dfast := compress(uncompressed.bytes(), strategy: .dfast)!
+	decompressed_dfast := decompress(compressed_dfast)!
+	assert decompressed_dfast == uncompressed.bytes()
+
+	compressed_greedy := compress(uncompressed.bytes(), strategy: .greedy)!
+	decompressed_greedy := decompress(compressed_greedy)!
+	assert decompressed_greedy == uncompressed.bytes()
+
+	compressed_lazy := compress(uncompressed.bytes(), strategy: .lazy)!
+	decompressed_lazy := decompress(compressed_lazy)!
+	assert decompressed_lazy == uncompressed.bytes()
+
+	compressed_lazy2 := compress(uncompressed.bytes(), strategy: .lazy2)!
+	decompressed_lazy2 := decompress(compressed_lazy2)!
+	assert decompressed_lazy2 == uncompressed.bytes()
+
+	compressed_btlazy2 := compress(uncompressed.bytes(), strategy: .btlazy2)!
+	decompressed_btlazy2 := decompress(compressed_btlazy2)!
+	assert decompressed_btlazy2 == uncompressed.bytes()
+
+	compressed_btopt := compress(uncompressed.bytes(), strategy: .btopt)!
+	decompressed_btopt := decompress(compressed_btopt)!
+	assert decompressed_btopt == uncompressed.bytes()
+
+	compressed_btultra := compress(uncompressed.bytes(), strategy: .btultra)!
+	decompressed_btultra := decompress(compressed_btultra)!
+	assert decompressed_btultra == uncompressed.bytes()
+
+	compressed_btultra2 := compress(uncompressed.bytes(), strategy: .btultra2)!
+	decompressed_btultra2 := decompress(compressed_btultra2)!
+	assert decompressed_btultra2 == uncompressed.bytes()
+}
+
+fn compress_file(fname string, oname string, params CompressParams) ! {
+	mut fin := os.open_file(fname, 'rb')!
+	mut fout := os.open_file(oname, 'wb')!
+	defer {
+		fin.close()
+		fout.close()
+	}
+
+	mut buf_in := []u8{len: buf_in_size}
+	mut buf_out := []u8{len: buf_out_size}
+
+	mut cctx := new_cctx(params)!
+	defer {
+		cctx.free_cctx()
+	}
+
+	mut last_chunk := false
+	mut input := &InBuffer{
+		src:  buf_in.data
+		size: 0
+		pos:  0
+	}
+	mut output := &OutBuffer{
+		dst:  buf_out.data
+		size: 0
+		pos:  0
+	}
+	for !last_chunk {
+		read_len := fin.read(mut buf_in)!
+		last_chunk = read_len < buf_in_size
+		mode := if last_chunk {
+			EndDirective.end
+		} else {
+			EndDirective.continue
+		}
+
+		mut finished := false
+		input.src = buf_in.data
+		input.size = usize(read_len)
+		input.pos = 0
+		for !finished {
+			output.dst = buf_out.data
+			output.size = buf_out_size
+			output.pos = 0
+			remaining := cctx.compress_stream2(output, input, mode)!
+			fout.write(buf_out[..int(output.pos)])!
+			finished = if last_chunk { remaining == 0 } else { input.pos == input.size }
+		}
+
+		if input.pos != input.size {
+			return error('Impossible: zstd only returns 0 when the input is completely consumed!')
+		}
+	}
+}
+
+fn decompress_file(fname string, oname string, params DecompressParams) ! {
+	mut fin := os.open_file(fname, 'rb')!
+	mut fout := os.open_file(oname, 'wb')!
+	defer {
+		fin.close()
+		fout.close()
+	}
+
+	mut buf_in := []u8{len: buf_in_size}
+	mut buf_out := []u8{len: buf_out_size}
+
+	mut dctx := new_dctx(params)!
+	defer {
+		dctx.free_dctx()
+	}
+
+	mut input := &InBuffer{
+		src:  buf_in.data
+		size: 0
+		pos:  0
+	}
+	mut output := &OutBuffer{
+		dst:  buf_out.data
+		size: 0
+		pos:  0
+	}
+
+	mut last_ret := usize(0)
+	for {
+		read_len := fin.read(mut buf_in)!
+		input.src = buf_in.data
+		input.size = usize(read_len)
+		input.pos = 0
+		for input.pos < input.size {
+			output.dst = buf_out.data
+			output.size = buf_out_size
+			output.pos = 0
+			ret := dctx.decompress_stream(output, input)!
+			fout.write(buf_out[..int(output.pos)])!
+			last_ret = ret
+		}
+		if read_len < buf_in.len {
+			break
+		}
+	}
+	if last_ret != 0 {
+		/* The last return value from DecompressStream did not end on a
+         * frame, but we reached the end of the file! We assume this is an
+         * error, and the input was truncated.
+         */
+		return error('EOF before end of stream: ${last_ret}')
+	}
+}
+
+// zstd stream mode test
+fn test_zstd_stream() {
+	decompress_file(s('readme_level_19.zst'), s('tmp_file1'))!
+	compress_file(s('tmp_file1'), s('tmp_file.zstd'),
+		compression_level: 6
+		nb_threads:        1
+		checksum_flag:     true
+	)!
+	decompress_file(s('tmp_file.zstd'), s('tmp_file2'))!
+	file1 := os.read_file(s('tmp_file1'))!
+	assert file1.contains('## Acknowledgement')
+	assert file1.contains('## Troubleshooting')
+	file2 := os.read_file(s('tmp_file2'))!
+	assert file1 == file2
+	os.rm(s('tmp_file1'))!
+	os.rm(s('tmp_file2'))!
+	os.rm(s('tmp_file.zstd'))!
+}
+
+// store_array compress an `array`'s data, and store it to file `fname`.
+// extra compression parameters can be set by `params`
+// WARNING: Because struct padding, some data in struct may be marked unused.
+// So, when `store_array`, it will cause memory fsanitize fail with 'use-of-uninitialized-value'.
+// It can be safely ignored.
+// For example, following struct may cause memory fsanitize fail:
+// struct MemoryTrace {
+// 	operation u8
+// 	address   u64
+// 	size      u8
+// }
+// By changing it into following , you can pass the memory fsanitize check :
+// struct MemoryTrace {
+// 	operation u64
+// 	address   u64
+// 	size      u64
+// }
+struct MemoryTrace {
+	operation u64
+	address   u64
+	size      u64
+}
+
+fn store_array_test(fname string) ! {
+	// Create a big array
+	mut store_memory_trace := []MemoryTrace{cap: 1000}
+	for i in 0 .. 1000 {
+		store_memory_trace << MemoryTrace{
+			operation: u64(`L`)
+			address:   u64(i)
+			size:      u8(i % 8)
+		}
+	}
+	store_array[MemoryTrace](fname, store_memory_trace, compression_level: 8)!
+}
+
+fn load_array_test(fname string) ! {
+	load_memory_trace := load_array[MemoryTrace](fname)!
+	for i in 0 .. 1000 {
+		assert load_memory_trace[i].operation == `L`
+		assert load_memory_trace[i].address == i
+		assert load_memory_trace[i].size == u8(i % 8)
+	}
+}
+
+fn test_zstd_store_load_array() {
+	store_array_test(s('mem_trace.zstd'))!
+	load_array_test(s('mem_trace.zstd'))!
+	os.rm(s('mem_trace.zstd'))!
+}
+
+fn assert_decompress_error(data []u8, reason string) ! {
+	decompress(data) or {
+		assert err.msg() == reason
+		return
+	}
+	return error('did not error')
+}
+
+fn test_zstd_invalid_too_small() {
+	assert_decompress_error([]u8{},
+		'An error occurred (e.g. invalid magic number, srcSize too small)')!
+}
+
+fn test_zstd_invalid_magic_numbers() {
+	assert_decompress_error([]u8{len: 100},
+		'An error occurred (e.g. invalid magic number, srcSize too small)')!
+}
+
+fn test_zstd_invalid_compression() {
+	mut data := []u8{len: 100}
+	data[0] = 0x1f
+	data[1] = 0x8b
+	assert_decompress_error(data,
+		'An error occurred (e.g. invalid magic number, srcSize too small)')!
+}
+
+fn test_zstd_with_corruption1() {
+	uncompressed := 'Hello world!'
+	mut compressed := compress(uncompressed.bytes())!
+	compressed[5] = u8(0x7A)
+	assert_decompress_error(compressed, 'Data corruption detected')!
+}
+
+fn test_zstd_with_corruption2() {
+	uncompressed := 'Hello world!'
+	mut compressed := compress(uncompressed.bytes())!
+	compressed[6] = u8(0x7A)
+	assert_decompress_error(compressed, 'Destination buffer is too small')!
+}
+
+fn test_zstd_with_corruption3() {
+	uncompressed := 'Hello world!'
+	mut compressed := compress(uncompressed.bytes())!
+	compressed[7] = u8(0x7A)
+	assert_decompress_error(compressed, 'Src size is incorrect')!
+}
+
+fn test_zstd_with_corruption4() {
+	uncompressed := 'Hello world!'
+	mut compressed := compress(uncompressed.bytes())!
+	compressed[8] = u8(0x7A)
+	assert_decompress_error(compressed, 'Src size is incorrect')!
+}
+
+fn test_zstd_with_corruption5() {
+	uncompressed := 'Hello world!'
+	mut compressed := compress(uncompressed.bytes())!
+	compressed[9] = u8(0x7A)
+	assert_decompress_error(compressed, "Restored data doesn't match checksum")!
+}
+
+fn test_zstd_with_corruption6() {
+	uncompressed := 'Hello world!'
+	mut compressed := compress(uncompressed.bytes())!
+	compressed[10] = u8(0x7A)
+	assert_decompress_error(compressed, "Restored data doesn't match checksum")!
+}
+
+fn test_zstd_with_corruption7() {
+	uncompressed := 'Hello world!'
+	mut compressed := compress(uncompressed.bytes())!
+	compressed[compressed.len - 1] += 1
+	assert_decompress_error(compressed, "Restored data doesn't match checksum")!
+}