Initial project import

tools/v/vlib/io/string_reader/string_reader.v

@@ -0,0 +1,329 @@
+@[deprecated: 'If you use it, create a local module named `string_reader`, and copy https://github.com/vlang/v/blob/f24d49259db5d2d7211e79bcf6b6f507d22847f3/vlib/io/string_reader/string_reader.v there.']
+@[deprecated_after: '2026-01-06']
+module string_reader
+
+import io
+import strings
+
+@[params]
+pub struct StringReaderParams {
+pub:
+	// the reader interface
+	reader ?io.Reader
+	// initialize the builder with this source string
+	source ?string
+	// if no source is given the string builder is initialized with this size
+	initial_size int
+}
+
+// StringReader is able to read data from a Reader interface and/or source string to a dynamically
+// growing buffer using a string builder. Unlike the BufferedReader, StringReader will
+// keep the entire contents of the buffer in memory, allowing the incoming data to be reused
+// and read in an efficient matter. The StringReader will not set a maximum capacity to the string
+// builders buffer and could grow very large.
+pub struct StringReader {
+mut:
+	reader ?io.Reader
+	offset int // current offset in the buffer
+pub mut:
+	end_of_stream bool // whether we reached the end of the upstream reader
+	builder       strings.Builder
+}
+
+// new creates a new StringReader and sets the string builder size to `initial_size`.
+// If a source
+pub fn StringReader.new(params StringReaderParams) StringReader {
+	mut r := StringReader{
+		reader: params.reader
+	}
+
+	if source := params.source {
+		r.builder = strings.new_builder(source.len)
+		r.builder.write_string(source)
+		if params.reader == none {
+			// There is no upstream reader; only the preloaded buffer can be consumed.
+			r.end_of_stream = true
+		}
+	} else {
+		r.builder = strings.new_builder(params.initial_size)
+	}
+
+	return r
+}
+
+// needs_fill returns whether the buffer needs refilling
+pub fn (r StringReader) needs_fill() bool {
+	return r.offset >= r.builder.len
+}
+
+// needs_fill_until returns whether the buffer needs refilling in order to read
+// `n` bytes
+pub fn (r StringReader) needs_fill_until(n int) bool {
+	return r.offset + n > r.builder.len
+}
+
+// fill_bufer tries to read data into the buffer until either a 0 length read or if read_to_end_of_stream
+// is true then the end of the stream. It returns the number of bytes read
+pub fn (mut r StringReader) fill_buffer(read_till_end_of_stream bool) !int {
+	if r.end_of_stream {
+		return io.Eof{}
+	}
+
+	mut reader := r.reader or { return error('reader is not set') }
+
+	start := r.builder.len
+	mut end := start
+
+	// make sure there is enough room in the string builder
+	unsafe { r.builder.grow_len(io.read_all_len) }
+	defer {
+		// shrink the length of the buffer to the total of bytes read
+		r.builder.go_back(r.builder.len - end)
+	}
+
+	for {
+		read := reader.read(mut r.builder[end..]) or {
+			r.end_of_stream = true
+			break
+		}
+		end += read
+
+		if !read_till_end_of_stream || read == 0 {
+			break
+		} else if r.builder.len == end {
+			unsafe { r.builder.grow_len(io.read_all_grow_len) }
+		}
+	}
+
+	if end == start {
+		return io.Eof{}
+	}
+
+	return end - start
+}
+
+// fill_buffer_until tries read `n` amount of bytes from the reader into the buffer
+// and returns the actual number of bytes read
+pub fn (mut r StringReader) fill_buffer_until(n int) !int {
+	if r.end_of_stream {
+		return io.Eof{}
+	}
+
+	mut reader := r.reader or { return error('reader is not set') }
+
+	start := r.builder.len
+	// make sure there is enough room in the string builder
+	if n > io.read_all_len {
+		unsafe { r.builder.grow_len(io.read_all_len) }
+	} else {
+		unsafe { r.builder.grow_len(n) }
+	}
+
+	mut end := start
+	defer {
+		// shrink the length of the buffer to the total of bytes read
+		r.builder.go_back(r.builder.len - end)
+	}
+	for {
+		read := reader.read(mut r.builder[end..]) or {
+			r.end_of_stream = true
+			break
+		}
+		end += read
+
+		if read == 0 || end - start == n {
+			break
+		} else if r.builder.len == end {
+			remaining := n - (end - start)
+			if remaining > io.read_all_grow_len {
+				unsafe { r.builder.grow_len(io.read_all_grow_len) }
+			} else {
+				unsafe { r.builder.grow_len(remaining) }
+			}
+		}
+	}
+
+	if end == start {
+		return io.Eof{}
+	}
+	return end - start
+}
+
+// read_all_bytes reads all bytes from a reader until either a 0 length read or if read_to_end_of_stream
+// is true then the end of the stream. It returns a copy of the read data
+pub fn (mut r StringReader) read_all_bytes(read_till_end_of_stream bool) ![]u8 {
+	start := r.offset
+	// ignore Eof error from fill buffer
+	r.fill_buffer(read_till_end_of_stream) or {}
+	r.offset = r.builder.len
+	// check if there was still data in the buffer, but the reader has reached its end of stream
+	if start == r.offset {
+		return io.Eof{}
+	}
+
+	return r.get_part(start, r.offset - start)!
+}
+
+// read_all reads all bytes from a reader until either a 0 length read or if read_to_end_of_stream
+// is true then the end of the stream. It produces a string from the read data
+pub fn (mut r StringReader) read_all(read_till_end_of_stream bool) !string {
+	buf := r.read_all_bytes(read_till_end_of_stream)!
+	return unsafe { tos(buf.data, buf.len) }
+}
+
+// read_bytes tries to read n amount of bytes from the reader
+pub fn (mut r StringReader) read_bytes(n int) ![]u8 {
+	start := r.offset
+
+	if r.needs_fill_until(n) {
+		actual_read := r.fill_buffer_until(n - (r.builder.len - r.offset))!
+		r.offset += actual_read
+	} else {
+		r.offset += n
+	}
+
+	return r.get_part(start, r.offset - start)!
+}
+
+// read_bytes tries to read `n` amount of bytes from the reader and produces a string
+// from the read data
+pub fn (mut r StringReader) read_string(n int) !string {
+	buf := r.read_bytes(n)!
+	return unsafe { tos(buf.data, buf.len) }
+}
+
+// read implements the Reader interface
+pub fn (mut r StringReader) read(mut buf []u8) !int {
+	if buf.len == 0 {
+		return 0
+	}
+
+	available := r.builder.len - r.offset
+	if available < buf.len && !r.end_of_stream {
+		if r.reader != none {
+			missing := buf.len - available
+			r.fill_buffer_until(missing) or {
+				if err !is io.Eof {
+					return err
+				}
+			}
+		}
+	}
+
+	available_after_fill := r.builder.len - r.offset
+	if available_after_fill == 0 {
+		if r.reader == none && !r.end_of_stream {
+			return error('reader is not set')
+		}
+		return io.Eof{}
+	}
+
+	read := if available_after_fill < buf.len { available_after_fill } else { buf.len }
+	start := r.offset
+	r.offset += read
+
+	copy(mut buf, r.builder[start..start + read])
+	return read
+}
+
+// read_line attempts to read a line from the reader.
+// It reads until it finds the specified delimiter or the end of stream.
+// The returned string does not include the delimiter.
+// When the delimiter is `\n`, a preceding `\r` is treated as part of CRLF
+// and is also omitted from the returned string.
+@[direct_array_access]
+pub fn (mut r StringReader) read_line(config io.BufferedReadLineConfig) !string {
+	if r.end_of_stream && r.needs_fill() {
+		return io.Eof{}
+	}
+
+	start := r.offset
+	for {
+		if r.needs_fill() {
+			r.fill_buffer(false) or {
+				// we are at the end of the stream
+				if r.offset == start {
+					return io.Eof{}
+				}
+				return r.get_string_part(start, r.offset - start)!
+			}
+		}
+		// try to find a newline character
+		mut i := r.offset
+		for ; i < r.builder.len; i++ {
+			c := r.builder[i]
+			if c == config.delim {
+				// great, we hit something
+				// do some checking for whether we hit \r\n or just \n
+				mut x := i
+				if i > start && config.delim == `\n` && r.builder[i - 1] == `\r` {
+					x--
+				}
+				r.offset = i + 1
+				return r.get_string_part(start, x - start)!
+			}
+		}
+		r.offset = i
+	}
+
+	return io.Eof{}
+}
+
+// write implements the Writer interface
+pub fn (mut r StringReader) write(buf []u8) !int {
+	return r.builder.write(buf)!
+}
+
+// get_data returns a copy of the buffer
+@[inline]
+pub fn (r StringReader) get_data() []u8 {
+	unsafe {
+		mut x := malloc_noscan(r.builder.len)
+		vmemcpy(x, &u8(r.builder.data), r.builder.len)
+		return x.vbytes(r.builder.len)
+	}
+}
+
+// get get_part returns a copy of a part of the buffer from `start` till `start` + `n`
+pub fn (r StringReader) get_part(start int, n int) ![]u8 {
+	if start + n > r.builder.len {
+		return io.Eof{}
+	}
+
+	unsafe {
+		mut x := malloc_noscan(n)
+		vmemcpy(x, &u8(r.builder.data) + start, n)
+		return x.vbytes(n)
+	}
+}
+
+// get_string produces a string from all the bytes in the buffer
+@[inline]
+pub fn (r StringReader) get_string() string {
+	return r.builder.spart(0, r.builder.len)
+}
+
+// get_string_part produces a string from `start` till `start` + `n` of the buffer
+pub fn (r StringReader) get_string_part(start int, n int) !string {
+	if start + n > r.builder.len {
+		return io.Eof{}
+	}
+
+	return r.builder.spart(start, n)
+}
+
+// flush clears the stringbuilder and returns the resulting string and the stringreaders
+// offset is reset to 0
+pub fn (mut r StringReader) flush() string {
+	r.offset = 0
+	return r.builder.str()
+}
+
+// free frees the memory block used for the string builders buffer,
+// a new string builder with size 0 is initialized and the stringreaders offset is reset to 0
+@[unsafe]
+pub fn (mut r StringReader) free() {
+	unsafe { r.builder.free() }
+	r.builder = strings.new_builder(0)
+	r.offset = 0
+}

tools/v/vlib/io/string_reader/string_reader_test.v

@@ -0,0 +1,409 @@
+module string_reader
+
+import io
+
+struct Buf {
+pub:
+	bytes []u8
+mut:
+	i int
+}
+
+struct TwoByteReader {
+mut:
+	data string
+	pos  int
+}
+
+fn (mut b Buf) read(mut buf []u8) !int {
+	if !(b.i < b.bytes.len) {
+		return io.Eof{}
+	}
+	n := copy(mut buf, b.bytes[b.i..])
+	b.i += n
+	return n
+}
+
+fn (mut r TwoByteReader) read(mut buf []u8) !int {
+	if r.pos >= r.data.len {
+		return io.Eof{}
+	}
+	min := int_min(int_min(r.data.len - r.pos, 2), buf.len)
+	for i in 0 .. min {
+		buf[i] = r.data[r.pos]
+		r.pos++
+	}
+	return min
+}
+
+fn read_from_reader_interface(mut r io.Reader, mut buf []u8) !int {
+	return r.read(mut buf)
+}
+
+fn test_read_all() {
+	mut reader := StringReader.new()
+
+	if _ := reader.read_all(false) {
+		assert false, 'should return io.Eof'
+	} else {
+		assert err is io.Eof
+	}
+
+	contents := 'test string'
+	buf := Buf{
+		bytes: contents.bytes()
+	}
+
+	reader = StringReader.new(reader: buf)
+	data := reader.read_all(false)!
+
+	assert data == contents
+}
+
+fn test_read_bytes() {
+	mut reader := StringReader.new()
+
+	if _ := reader.read_bytes(4) {
+		assert false, 'should return io.Eof'
+	} else {
+		assert err.msg() == 'reader is not set'
+	}
+
+	buf := Buf{
+		bytes: '12345678'.bytes()
+	}
+
+	reader = StringReader.new(reader: buf)
+	mut data := reader.read_bytes(4)!
+	assert data == buf.bytes[..4]
+
+	data = reader.read_bytes(4)!
+	assert data == buf.bytes[4..8]
+}
+
+fn test_read_line() {
+	mut reader := StringReader.new()
+
+	if _ := reader.read_line() {
+		assert false, 'should return io.Eof'
+	} else {
+		assert err is io.Eof
+	}
+
+	buf := Buf{
+		bytes: 'first line\r\nsecond line\n'.bytes()
+	}
+
+	reader = StringReader.new(reader: buf)
+
+	assert reader.read_line()! == 'first line'
+	assert reader.read_line()! == 'second line'
+}
+
+fn test_from_string() {
+	mut reader := StringReader.new(source: 'test')
+	assert reader.read_all(false)! == 'test'
+
+	if _ := reader.read_all(false) {
+		assert false, 'should return Eof'
+	} else {
+		assert err is io.Eof
+	}
+}
+
+fn test_read_from_source_read() {
+	source := 'line 1\nline 2\n'
+	mut reader := StringReader.new(source: source)
+	mut buf := []u8{len: 100}
+	read := reader.read(mut buf)!
+	assert read == source.len
+	assert buf[..read].bytestr() == source
+	if _ := reader.read(mut buf) {
+		assert false, 'should return io.Eof'
+	} else {
+		assert err is io.Eof
+	}
+}
+
+fn test_read_from_source_read_via_reader_interface() {
+	source := 'line 1\nline 2\n'
+	mut reader := StringReader.new(source: source)
+	mut buf := []u8{len: 100}
+	read := read_from_reader_interface(mut reader, mut buf)!
+	assert read == source.len
+	assert buf[..read].bytestr() == source
+	if _ := read_from_reader_interface(mut reader, mut buf) {
+		assert false, 'should return io.Eof'
+	} else {
+		assert err is io.Eof
+	}
+}
+
+fn test_from_string_read_byte_one_by_one() {
+	mut reader := StringReader.new(source: 'test')
+	assert reader.read_bytes(1)![0].ascii_str() == 't'
+	assert reader.read_bytes(1)![0].ascii_str() == 'e'
+	assert reader.read_bytes(1)![0].ascii_str() == 's'
+	assert reader.read_bytes(1)![0].ascii_str() == 't'
+
+	if _ := reader.read_all(false) {
+		assert false, 'should return Eof'
+	} else {
+		assert err is io.Eof
+	}
+}
+
+fn test_from_string_and_reader() {
+	buf := Buf{
+		bytes: 'buffer'.bytes()
+	}
+
+	mut reader := StringReader.new(reader: buf, source: 'string')
+
+	assert reader.read_all(false)! == 'stringbuffer'
+}
+
+fn test_flush() {
+	mut reader := StringReader.new(source: 'flushed data')
+
+	str := reader.flush()
+	assert str == 'flushed data'
+
+	assert reader.offset == 0
+	assert reader.builder.len == 0
+}
+
+fn test_fill_buffer_true() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.fill_buffer(true)! == 5
+	assert reader.get_string() == '12345'
+}
+
+fn test_fill_buffer_false() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.fill_buffer(false)! == 2
+	assert reader.get_string() == '12'
+}
+
+fn test_fill_buffer_until() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.fill_buffer_until(1)! == 1
+	assert reader.builder.len == 1
+	assert reader.fill_buffer_until(2)! == 2
+	assert reader.builder.len == 3
+	assert reader.fill_buffer_until(2)! == 2
+	assert reader.builder.len == 5
+	assert reader.get_string() == '12345'
+}
+
+fn test_fill_buffer_until_one() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.fill_buffer_until(1)! == 1
+	assert reader.builder.len == 1
+	assert reader.fill_buffer_until(1)! == 1
+	assert reader.builder.len == 2
+	assert reader.fill_buffer_until(1)! == 1
+	assert reader.builder.len == 3
+	assert reader.fill_buffer_until(1)! == 1
+	assert reader.builder.len == 4
+	assert reader.fill_buffer_until(1)! == 1
+	assert reader.builder.len == 5
+	assert reader.get_string() == '12345'
+}
+
+fn test_fill_buffer_until_many() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.fill_buffer_until(1)! == 1
+	assert reader.builder.len == 1
+	assert reader.fill_buffer_until(12)! == 4
+	assert reader.builder.len == 5
+	assert reader.fill_buffer_until(123) or { -1 } == -1
+	assert reader.builder.len == 5
+	assert reader.get_string() == '12345'
+}
+
+fn test_read_all_bytes_false() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_all_bytes(false)! == [u8(49), 50]
+	assert reader.read_all_bytes(false)! == [u8(51), 52]
+	assert reader.read_all_bytes(false)! == [u8(53)]
+}
+
+fn test_read_all_bytes_true() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_all_bytes(false)! == [u8(49), 50]
+	assert reader.read_all_bytes(true)! == [u8(51), 52, 53]
+}
+
+fn test_read_all_false() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_all(false)! == '12'
+	assert reader.read_all(false)! == '34'
+	assert reader.read_all(false)! == '5'
+}
+
+fn test_read_all_true() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_all(false)! == '12'
+	assert reader.read_all(true)! == '345'
+}
+
+fn test_read_bytes_1() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_bytes(1)! == [u8(49)]
+	assert reader.read_bytes(2)! == [u8(50), 51]
+	assert reader.read_bytes(2)! == [u8(52), 53]
+	assert reader.read_bytes(2) or { [u8(255)] } == [u8(255)]
+}
+
+fn test_read_bytes_many() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_bytes(1)! == [u8(49)]
+	assert reader.builder.len == 1
+	assert reader.offset == 1
+	assert reader.read_bytes(123)! == [u8(50), 51, 52, 53]
+	assert reader.builder.len == 5
+	assert reader.offset == 5
+}
+
+fn test_read_string() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_string(1)! == '1'
+	assert reader.read_string(2)! == '23'
+	assert reader.read_string(3)! == '45'
+}
+
+fn test_read() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	mut o1 := []u8{len: 1}
+	mut o5 := []u8{len: 5}
+	mut res := reader.read(mut o1)!
+	assert res == 1
+	assert o1[..res] == [u8(49)]
+	res = reader.read(mut o5)!
+	assert res == 4
+	assert o5[..res] == [u8(50), 51, 52, 53]
+}
+
+fn test_read_many() {
+	mut two := TwoByteReader{
+		data: '12345'
+	}
+	mut reader := StringReader.new(reader: two)
+	mut o4 := []u8{len: 4}
+	mut res := reader.read(mut o4)!
+	assert res == 4
+	assert o4[..res] == [u8(49), 50, 51, 52]
+	res = reader.read(mut o4)!
+	assert res == 1
+	assert o4[..res] == [u8(53)]
+	res = reader.read(mut o4) or { -1 }
+	assert res == -1
+	assert o4 == [u8(53), 50, 51, 52]
+}
+
+fn test_read_line_2() {
+	mut two := TwoByteReader{
+		data: '12345\n67890'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_line()! == '12345'
+	assert reader.read_line()! == '67890'
+}
+
+fn test_read_line_3() {
+	mut two := TwoByteReader{
+		data: '12345\t67890'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_line(delim: `\t`)! == '12345'
+	assert reader.read_line()! == '67890'
+}
+
+fn test_read_line_4() {
+	mut two := TwoByteReader{
+		data: '12345\t67890'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_line(delim: `\t`)! == '12345'
+	assert reader.read_line(delim: `\t`)! == '67890'
+}
+
+fn test_read_line_5() {
+	mut two := TwoByteReader{
+		data: '\n12345\n67890'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_line()! == ''
+	assert reader.read_line()! == '12345'
+	assert reader.read_line()! == '67890'
+}
+
+fn test_read_line_6() {
+	mut two := TwoByteReader{
+		data: '\r\n12345\r\n67890'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_line()! == ''
+	assert reader.read_line()! == '12345'
+	assert reader.read_line()! == '67890'
+}
+
+fn test_read_line_7() {
+	mut two := TwoByteReader{
+		data: '1234567890'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_line(delim: `6`)! == '12345'
+	assert reader.read_line(delim: `6`)! == '7890'
+	assert reader.read_line() or { '777' } == '777'
+}
+
+fn test_read_line_8() {
+	mut two := TwoByteReader{
+		data: '12345\r\n67890'
+	}
+	mut reader := StringReader.new(reader: two)
+	assert reader.read_bytes(6)! == [u8(49), 50, 51, 52, 53, 13]
+	reader.read_line()! // \n
+	assert reader.read_line()! == '67890'
+}