Initial project import
This commit is contained in:
Agila Forge Importer committed
commit
23f2d29e32
11957 files changed
+1754577
No files matched your search
@@ -0,0 +1,6 @@
|
||||
## Description
|
||||
|
||||
`strings` provides utilities for efficiently processing large strings.
|
||||
|
||||
If you got here looking for methods available on the `string` struct, those
|
||||
methods are found in the `builtin` module.
|
||||
@@ -0,0 +1,470 @@
|
||||
// Copyright (c) 2019-2024 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module strings
|
||||
|
||||
// strings.Builder is used to efficiently append many strings to a large
|
||||
// dynamically growing buffer, then use the resulting large string. Using
|
||||
// a string builder is much better for performance/memory usage than doing
|
||||
// constantly string concatenation.
|
||||
pub type Builder = []u8
|
||||
|
||||
// new_builder returns a new string builder, with an initial capacity of `initial_size`.
|
||||
pub fn new_builder(initial_size int) Builder {
|
||||
mut res := Builder([]u8{cap: initial_size})
|
||||
unsafe { res.flags.set(.noslices) }
|
||||
return res
|
||||
}
|
||||
|
||||
// reuse_as_plain_u8_array allows using the Builder instance as a plain []u8 return value.
|
||||
// It is useful, when you have accumulated data in the builder, that you want to
|
||||
// pass/access as []u8 later, without copying or freeing the buffer.
|
||||
// NB: you *should NOT use* the string builder instance after calling this method.
|
||||
// Use only the return value after calling this method.
|
||||
@[unsafe]
|
||||
pub fn (mut b Builder) reuse_as_plain_u8_array() []u8 {
|
||||
unsafe { b.flags.clear(.noslices) }
|
||||
return *b
|
||||
}
|
||||
|
||||
// write_ptr writes `len` bytes provided byteptr to the accumulated buffer
|
||||
@[unsafe]
|
||||
pub fn (mut b Builder) write_ptr(ptr &u8, len int) {
|
||||
if len == 0 {
|
||||
return
|
||||
}
|
||||
unsafe { b.push_many(ptr, len) }
|
||||
}
|
||||
|
||||
// write_rune appends a single rune to the accumulated buffer
|
||||
@[manualfree]
|
||||
pub fn (mut b Builder) write_rune(r rune) {
|
||||
mut buffer := [5]u8{}
|
||||
res := unsafe { utf32_to_str_no_malloc(u32(r), mut &buffer[0]) }
|
||||
if res.len == 0 {
|
||||
return
|
||||
}
|
||||
unsafe { b.push_many(res.str, res.len) }
|
||||
}
|
||||
|
||||
// write_runes appends all the given runes to the accumulated buffer.
|
||||
pub fn (mut b Builder) write_runes(runes []rune) {
|
||||
mut buffer := [5]u8{}
|
||||
for r in runes {
|
||||
res := unsafe { utf32_to_str_no_malloc(u32(r), mut &buffer[0]) }
|
||||
if res.len == 0 {
|
||||
continue
|
||||
}
|
||||
unsafe { b.push_many(res.str, res.len) }
|
||||
}
|
||||
}
|
||||
|
||||
// write_u8 appends a single `data` byte to the accumulated buffer
|
||||
@[inline]
|
||||
pub fn (mut b Builder) write_u8(data u8) {
|
||||
b << data
|
||||
}
|
||||
|
||||
// write_byte appends a single `data` byte to the accumulated buffer
|
||||
@[inline]
|
||||
pub fn (mut b Builder) write_byte(data u8) {
|
||||
b << data
|
||||
}
|
||||
|
||||
// write_decimal appends a decimal representation of the number `n` into the builder `b`,
|
||||
// without dynamic allocation. The higher order digits come first, i.e. 6123 will be written
|
||||
// with the digit `6` first, then `1`, then `2` and `3` last.
|
||||
@[direct_array_access]
|
||||
pub fn (mut b Builder) write_decimal(n i64) {
|
||||
if n == 0 {
|
||||
b.write_u8(0x30)
|
||||
return
|
||||
}
|
||||
if n == min_i64 {
|
||||
b.write_string(n.str())
|
||||
return
|
||||
}
|
||||
|
||||
mut buf := [25]u8{}
|
||||
mut x := if n < 0 { -n } else { n }
|
||||
mut i := 24
|
||||
for x != 0 {
|
||||
nextx := x / 10
|
||||
r := x % 10
|
||||
buf[i] = u8(r) + 0x30
|
||||
x = nextx
|
||||
i--
|
||||
}
|
||||
if n < 0 {
|
||||
buf[i] = `-`
|
||||
i--
|
||||
}
|
||||
unsafe { b.write_ptr(&buf[i + 1], 24 - i) }
|
||||
}
|
||||
|
||||
// write implements the io.Writer interface, that is why it returns how many bytes were written to the string builder.
|
||||
pub fn (mut b Builder) write(data []u8) !int {
|
||||
if data.len == 0 {
|
||||
return 0
|
||||
}
|
||||
unsafe { b.push_many(data.data, data.len) }
|
||||
return data.len
|
||||
}
|
||||
|
||||
// drain_builder writes all of the `other` builder content, then re-initialises
|
||||
// `other`, so that the `other` strings builder is ready to receive new content.
|
||||
@[manualfree]
|
||||
pub fn (mut b Builder) drain_builder(mut other Builder, other_new_cap int) {
|
||||
if other.len > 0 {
|
||||
b << *other
|
||||
}
|
||||
unsafe { other.free() }
|
||||
other = new_builder(other_new_cap)
|
||||
}
|
||||
|
||||
// byte_at returns a byte, located at a given index `i`.
|
||||
// Note: it can panic, if there are not enough bytes in the strings builder yet.
|
||||
@[inline]
|
||||
pub fn (b &Builder) byte_at(n int) u8 {
|
||||
return unsafe { (&[]u8(b))[n] }
|
||||
}
|
||||
|
||||
// write appends the string `s` to the buffer
|
||||
@[expand_simple_interpolation; inline]
|
||||
pub fn (mut b Builder) write_string(s string) {
|
||||
if s.len == 0 {
|
||||
return
|
||||
}
|
||||
unsafe { b.push_many(s.str, s.len) }
|
||||
// for c in s {
|
||||
// b.buf << c
|
||||
// }
|
||||
// b.buf << []u8(s) // TODO
|
||||
}
|
||||
|
||||
// write_string2 appends the strings `s1` and `s2` to the buffer.
|
||||
@[inline]
|
||||
pub fn (mut b Builder) write_string2(s1 string, s2 string) {
|
||||
if s1.len != 0 {
|
||||
unsafe { b.push_many(s1.str, s1.len) }
|
||||
}
|
||||
if s2.len != 0 {
|
||||
unsafe { b.push_many(s2.str, s2.len) }
|
||||
}
|
||||
}
|
||||
|
||||
// go_back discards the last `n` bytes from the buffer.
|
||||
pub fn (mut b Builder) go_back(n int) {
|
||||
b.trim(b.len - n)
|
||||
}
|
||||
|
||||
// spart returns a part of the buffer as a string
|
||||
@[inline]
|
||||
pub fn (b &Builder) spart(start_pos int, n int) string {
|
||||
unsafe {
|
||||
mut x := malloc_noscan(n + 1)
|
||||
vmemcpy(x, &u8(b.data) + start_pos, n)
|
||||
x[n] = 0
|
||||
return tos(x, n)
|
||||
}
|
||||
}
|
||||
|
||||
// cut_last cuts the last `n` bytes from the buffer and returns them.
|
||||
pub fn (mut b Builder) cut_last(n int) string {
|
||||
cut_pos := b.len - n
|
||||
res := b.spart(cut_pos, n)
|
||||
b.trim(cut_pos)
|
||||
return res
|
||||
}
|
||||
|
||||
// cut_to cuts the string after `pos` and returns it.
|
||||
// if `pos` is superior to builder length, returns an empty string
|
||||
// and cancel further operations
|
||||
pub fn (mut b Builder) cut_to(pos int) string {
|
||||
if pos > b.len {
|
||||
return ''
|
||||
}
|
||||
return b.cut_last(b.len - pos)
|
||||
}
|
||||
|
||||
// go_back_to resets the buffer to the given position `pos`.
|
||||
// Note: pos should be < than the existing buffer length.
|
||||
pub fn (mut b Builder) go_back_to(pos int) {
|
||||
b.trim(pos)
|
||||
}
|
||||
|
||||
// writeln appends the string `s`, and then a newline character.
|
||||
@[inline]
|
||||
pub fn (mut b Builder) writeln(s string) {
|
||||
// for c in s {
|
||||
// b.buf << c
|
||||
// }
|
||||
if s != '' {
|
||||
unsafe { b.push_many(s.str, s.len) }
|
||||
}
|
||||
// b.buf << []u8(s) // TODO
|
||||
b << u8(`\n`)
|
||||
}
|
||||
|
||||
// writeln2 appends two strings: `s1` + `\n`, and `s2` + `\n`, to the buffer.
|
||||
@[inline]
|
||||
pub fn (mut b Builder) writeln2(s1 string, s2 string) {
|
||||
if s1 != '' {
|
||||
unsafe { b.push_many(s1.str, s1.len) }
|
||||
}
|
||||
b << u8(`\n`)
|
||||
if s2 != '' {
|
||||
unsafe { b.push_many(s2.str, s2.len) }
|
||||
}
|
||||
b << u8(`\n`)
|
||||
}
|
||||
|
||||
// last_n(5) returns 'world'
|
||||
// buf == 'hello world'
|
||||
pub fn (b &Builder) last_n(n int) string {
|
||||
if n > b.len {
|
||||
return ''
|
||||
}
|
||||
return b.spart(b.len - n, n)
|
||||
}
|
||||
|
||||
// after(6) returns 'world'
|
||||
// buf == 'hello world'
|
||||
pub fn (b &Builder) after(n int) string {
|
||||
if n >= b.len {
|
||||
return ''
|
||||
}
|
||||
return b.spart(n, b.len - n)
|
||||
}
|
||||
|
||||
// str returns a copy of all of the accumulated buffer content.
|
||||
// Note: after a call to b.str(), the builder b will be empty, and could be used again.
|
||||
// The returned string *owns* its own separate copy of the accumulated data that was in
|
||||
// the string builder, before the .str() call.
|
||||
pub fn (mut b Builder) str() string {
|
||||
b << u8(0)
|
||||
bcopy := unsafe { &u8(memdup_noscan(b.data, b.len)) }
|
||||
s := unsafe { bcopy.vstring_with_len(b.len - 1) }
|
||||
b.clear()
|
||||
return s
|
||||
}
|
||||
|
||||
// ensure_cap ensures that the buffer has enough space for at least `n` bytes by growing the buffer if necessary.
|
||||
pub fn (mut b Builder) ensure_cap(n int) {
|
||||
// Work through the underlying array pointer, instead of taking a pointer
|
||||
// cast to the alias receiver. This keeps self-hosted builds from generating
|
||||
// an invalid `&b` cast in C.
|
||||
mut arr := unsafe { &[]u8(b) }
|
||||
arr.ensure_cap(n)
|
||||
}
|
||||
|
||||
// grow_len grows the length of the buffer by `n` bytes if necessary
|
||||
@[unsafe]
|
||||
pub fn (mut b Builder) grow_len(n int) {
|
||||
if n <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
new_len := b.len + n
|
||||
b.ensure_cap(new_len)
|
||||
unsafe {
|
||||
b.len = new_len
|
||||
}
|
||||
}
|
||||
|
||||
// free frees the memory block, used for the buffer.
|
||||
// Note: do not use the builder, after a call to free().
|
||||
@[unsafe]
|
||||
pub fn (mut b Builder) free() {
|
||||
if b.data != 0 {
|
||||
mut arr := unsafe { &[]u8(b) }
|
||||
unsafe { arr.free() }
|
||||
}
|
||||
}
|
||||
|
||||
// write_repeated_rune appends multiple copies of the same rune to the accumulated buffer
|
||||
@[direct_array_access]
|
||||
pub fn (mut b Builder) write_repeated_rune(r rune, count int) {
|
||||
if count <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Convert rune to UTF-8 bytes once
|
||||
mut buffer := [5]u8{}
|
||||
res := unsafe { utf32_to_str_no_malloc(u32(r), mut &buffer[0]) }
|
||||
if res.len == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if res.len == 1 {
|
||||
b.ensure_cap(b.len + count)
|
||||
unsafe {
|
||||
vmemset(&u8(b.data) + b.len, buffer[0], count)
|
||||
b.len += count
|
||||
}
|
||||
return
|
||||
} else {
|
||||
total_needed := count * res.len
|
||||
b.ensure_cap(b.len + total_needed)
|
||||
|
||||
mut dest := unsafe { &u8(b.data) + b.len }
|
||||
for _ in 0 .. count {
|
||||
unsafe {
|
||||
vmemcpy(dest, res.str, res.len)
|
||||
dest += res.len
|
||||
}
|
||||
}
|
||||
unsafe {
|
||||
b.len += total_needed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// IndentParam holds configuration parameters for the indent() function
|
||||
@[params]
|
||||
pub struct IndentParam {
|
||||
pub mut:
|
||||
block_start rune = `{` // Character that starts a new block (+ indent)
|
||||
block_end rune = `}` // Character that ends a new block (- indent)
|
||||
indent_char rune = ` ` // Character used for indentation (space or tab)
|
||||
indent_count int = 4 // Number of indent_char per indentation level
|
||||
starting_level int // Initial indentation level (0 = no initial indent)
|
||||
}
|
||||
|
||||
// IndentState represents the current parsing state of the indent() function
|
||||
enum IndentState {
|
||||
normal // Normal state, processing regular characters
|
||||
in_string // Inside a string literal, ignoring formatting characters
|
||||
}
|
||||
|
||||
// indent formats a string by applying structured indentation based on block delimiters.
|
||||
// It processes the input string `s` and writes the formatted output to the `Builder` `b`.
|
||||
// The function preserves content inside string literals (both single and double quotes) and
|
||||
// configures indentation behavior through the `param` structure.
|
||||
//
|
||||
// Key behaviors:
|
||||
// 1. Removes existing indentation at the beginning of lines.
|
||||
// 2. Applies new indentation based on block nesting levels.
|
||||
// 3. Ignores block delimiters and formatting characters inside string literals.
|
||||
// 4. Keeps empty blocks (e.g., {}) on the same line.
|
||||
// 5. Inserts newlines after `block_start` and before `block_end` (except for empty blocks).
|
||||
// 6. Maintains existing line breaks from the input.
|
||||
//
|
||||
// Example:
|
||||
// ```v
|
||||
// import strings
|
||||
// input := 'User{name:"John" settings:{theme:"dark"}}'
|
||||
// mut b := strings.new_builder(64)
|
||||
// b.indent(input, indent_count: 2)
|
||||
// println(b.str()) // Formatted output: 'User{\n name:"John" settings:{\n theme:"dark"\n }\n}'
|
||||
// ```
|
||||
@[direct_array_access]
|
||||
pub fn (mut b Builder) indent(s string, param IndentParam) {
|
||||
if s.len == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
mut state := IndentState.normal
|
||||
mut indent_level := param.starting_level
|
||||
mut string_char := `\0`
|
||||
mut at_line_start := true
|
||||
for i := 0; i < s.len; i++ {
|
||||
c := rune(s[i])
|
||||
match state {
|
||||
// Normal state: process characters outside of string literals
|
||||
.normal {
|
||||
match c {
|
||||
`"`, `'` { // Note: quote characters for editor display "
|
||||
state = .in_string
|
||||
string_char = c
|
||||
// Add indentation if at the start of a line
|
||||
if at_line_start {
|
||||
b.write_repeated_rune(param.indent_char,
|
||||
indent_level * param.indent_count)
|
||||
at_line_start = false
|
||||
}
|
||||
// Write the opening quote
|
||||
b.write_rune(c)
|
||||
}
|
||||
param.block_start {
|
||||
// Start of a new block
|
||||
// Add indentation if at the start of a line
|
||||
if at_line_start {
|
||||
b.write_repeated_rune(param.indent_char,
|
||||
indent_level * param.indent_count)
|
||||
at_line_start = false
|
||||
}
|
||||
|
||||
// Write the block start character
|
||||
b.write_rune(c)
|
||||
|
||||
// Check for empty block (e.g., {})
|
||||
// Empty blocks stay on the same line
|
||||
if i + 1 < s.len && s[i + 1] == param.block_end {
|
||||
b.write_rune(param.block_end)
|
||||
i++
|
||||
} else {
|
||||
// Non-empty block: increase indentation and add newline
|
||||
indent_level++
|
||||
b.write_rune(`\n`)
|
||||
at_line_start = true
|
||||
}
|
||||
}
|
||||
param.block_end {
|
||||
// End of a block
|
||||
// Decrease indentation level (but not below 0)
|
||||
if indent_level > 0 {
|
||||
indent_level--
|
||||
}
|
||||
|
||||
// If not at the start of a line, add a newline
|
||||
if !at_line_start {
|
||||
b.write_rune(`\n`)
|
||||
}
|
||||
|
||||
// Add indentation for the block end
|
||||
b.write_repeated_rune(param.indent_char, indent_level * param.indent_count)
|
||||
at_line_start = false
|
||||
|
||||
b.write_rune(c)
|
||||
}
|
||||
` `, `\t`, `\r`, `\n` {
|
||||
// Whitespace characters
|
||||
// Only write whitespace if not at the start of a line
|
||||
if !at_line_start {
|
||||
b.write_rune(c)
|
||||
}
|
||||
|
||||
// Newline resets the line start flag
|
||||
if c == `\n` {
|
||||
at_line_start = true
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Any other character
|
||||
// Add indentation if at the start of a line
|
||||
if at_line_start {
|
||||
b.write_repeated_rune(param.indent_char,
|
||||
indent_level * param.indent_count)
|
||||
at_line_start = false
|
||||
}
|
||||
b.write_rune(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
.in_string {
|
||||
// Inside a string literal: preserve all characters as-is
|
||||
b.write_rune(c)
|
||||
|
||||
// Check for string termination
|
||||
// The character must match the opening quote and not be escaped
|
||||
if c == string_char {
|
||||
if s[i - 1] != `\\` {
|
||||
state = .normal
|
||||
string_char = `\0`
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
// Copyright (c) 2019-2024 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module strings
|
||||
|
||||
/*
|
||||
pub struct Builder {
|
||||
mut:
|
||||
buf []u8
|
||||
pub mut:
|
||||
len int
|
||||
initial_size int = 1
|
||||
}*/
|
||||
|
||||
pub type Builder = []u8
|
||||
|
||||
pub fn new_builder(initial_size int) Builder {
|
||||
return []u8{cap: initial_size}
|
||||
}
|
||||
|
||||
pub fn (mut b Builder) write_byte(data u8) {
|
||||
b << data
|
||||
}
|
||||
|
||||
pub fn (mut b Builder) write_u8(data u8) {
|
||||
b << data
|
||||
}
|
||||
|
||||
pub fn (mut b Builder) write(data []u8) ?int {
|
||||
if data.len == 0 {
|
||||
return 0
|
||||
}
|
||||
b << data
|
||||
return data.len
|
||||
}
|
||||
|
||||
pub fn (b &Builder) byte_at(n int) u8 {
|
||||
unsafe {
|
||||
return b[n]
|
||||
}
|
||||
}
|
||||
|
||||
pub fn (mut b Builder) write_string(s string) {
|
||||
if s == '' {
|
||||
return
|
||||
}
|
||||
|
||||
for c in s {
|
||||
b << c
|
||||
}
|
||||
}
|
||||
|
||||
pub fn (mut b Builder) writeln(s string) {
|
||||
if s != '' {
|
||||
b.write_string(s)
|
||||
}
|
||||
|
||||
b << 10
|
||||
}
|
||||
|
||||
pub fn (mut b Builder) str() string {
|
||||
s := ''
|
||||
|
||||
#for (const c of b.val.arr.arr)
|
||||
#s.str += String.fromCharCode(+c)
|
||||
b.clear()
|
||||
return s
|
||||
}
|
||||
|
||||
pub fn (mut b Builder) cut_last(n int) string {
|
||||
cut_pos := b.len - n
|
||||
x := unsafe { b[cut_pos..] }
|
||||
res := x.bytestr()
|
||||
b.trim(cut_pos)
|
||||
return res
|
||||
}
|
||||
|
||||
pub fn (mut b Builder) go_back_to(pos int) {
|
||||
b.trim(pos)
|
||||
}
|
||||
|
||||
// go_back discards the last `n` bytes from the buffer.
|
||||
pub fn (mut b Builder) go_back(n int) {
|
||||
b.trim(b.len - n)
|
||||
}
|
||||
|
||||
// cut_to cuts the string after `pos` and returns it.
|
||||
// if `pos` is superior to builder length, returns an empty string
|
||||
// and cancel further operations
|
||||
pub fn (mut b Builder) cut_to(pos int) string {
|
||||
if pos > b.len {
|
||||
return ''
|
||||
}
|
||||
return b.cut_last(b.len - pos)
|
||||
}
|
||||
|
||||
pub fn (mut b Builder) write_runes(runes []rune) {
|
||||
for r in runes {
|
||||
res := r.str()
|
||||
#res.str = String.fromCharCode(r.val)
|
||||
b << res.bytes()
|
||||
}
|
||||
}
|
||||
|
||||
// after(6) returns 'world'
|
||||
// buf == 'hello world'
|
||||
pub fn (mut b Builder) after(n int) string {
|
||||
if n >= b.len {
|
||||
return ''
|
||||
}
|
||||
|
||||
x := unsafe { b[n..b.len] }
|
||||
return x.bytestr()
|
||||
}
|
||||
|
||||
// last_n(5) returns 'world'
|
||||
// buf == 'hello world'
|
||||
pub fn (mut b Builder) last_n(n int) string {
|
||||
if n >= b.len {
|
||||
return ''
|
||||
}
|
||||
|
||||
x := unsafe { b[b.len - n..b.len] }
|
||||
return x.bytestr()
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// vtest build: present_node?
|
||||
import strings
|
||||
|
||||
type MyInt = int
|
||||
|
||||
const maxn = 100000
|
||||
|
||||
fn test_sb() {
|
||||
mut sb := strings.new_builder(100)
|
||||
sb.write_string('hi')
|
||||
sb.write_string('!')
|
||||
sb.write_string('hello')
|
||||
assert sb.len == 8
|
||||
sb_end := sb.str()
|
||||
assert sb_end == 'hi!hello'
|
||||
assert sb.len == 0
|
||||
///
|
||||
sb = strings.new_builder(10)
|
||||
sb.write_string('a')
|
||||
sb.write_string('b')
|
||||
assert sb.len == 2
|
||||
assert sb.str() == 'ab'
|
||||
// Test interpolation optimization
|
||||
sb = strings.new_builder(10)
|
||||
x := 10
|
||||
y := MyInt(20)
|
||||
sb.writeln('x = ${x} y = ${y}')
|
||||
res := sb.str()
|
||||
assert res[res.len - 1] == `\n`
|
||||
println('"${res}"')
|
||||
assert res.trim_space() == 'x = 10 y = 20'
|
||||
|
||||
sb = strings.new_builder(10)
|
||||
sb.write_string('x = ${x} y = ${y}')
|
||||
assert sb.str() == 'x = 10 y = 20'
|
||||
//$if !windows {
|
||||
sb = strings.new_builder(10)
|
||||
sb.write_string('123456')
|
||||
last_2 := sb.cut_last(2)
|
||||
assert last_2 == '56'
|
||||
final_sb := sb.str()
|
||||
assert final_sb == '1234'
|
||||
|
||||
sb.clear()
|
||||
assert sb.str() == ''
|
||||
//}
|
||||
}
|
||||
|
||||
fn test_big_sb() {
|
||||
mut sb := strings.new_builder(100)
|
||||
mut sb2 := strings.new_builder(10000)
|
||||
for i in 0 .. maxn {
|
||||
sb.writeln(i.str())
|
||||
sb2.write_string('+')
|
||||
}
|
||||
s := sb.str()
|
||||
lines := s.split_into_lines()
|
||||
assert lines.len == maxn
|
||||
assert lines[0] == '0'
|
||||
assert lines[1] == '1'
|
||||
assert lines[777] == '777'
|
||||
assert lines[98765] == '98765'
|
||||
println(sb2.len)
|
||||
assert sb2.len == maxn
|
||||
}
|
||||
|
||||
fn test_byte_write() {
|
||||
mut sb := strings.new_builder(100)
|
||||
temp_str := 'byte testing'
|
||||
mut count := 0
|
||||
for word in temp_str {
|
||||
sb.write_u8(word)
|
||||
count++
|
||||
assert count == sb.len
|
||||
}
|
||||
sb_final := sb.str()
|
||||
assert sb_final == temp_str
|
||||
}
|
||||
|
||||
fn test_strings_builder_reuse() {
|
||||
mut sb := strings.new_builder(256)
|
||||
sb.write_string('world')
|
||||
assert sb.str() == 'world'
|
||||
sb.write_string('hello')
|
||||
assert sb.str() == 'hello'
|
||||
}
|
||||
|
||||
fn test_cut_to() {
|
||||
mut sb := strings.new_builder(16)
|
||||
sb.write_string('hello')
|
||||
assert sb.cut_to(3) == 'lo'
|
||||
assert sb.len == 3
|
||||
assert sb.cut_to(3) == ''
|
||||
assert sb.len == 3
|
||||
assert sb.cut_to(0) == 'hel'
|
||||
assert sb.cut_to(32) == ''
|
||||
assert sb.len == 0
|
||||
}
|
||||
@@ -0,0 +1,417 @@
|
||||
import strings
|
||||
|
||||
type MyInt = int
|
||||
|
||||
const maxn = 100000
|
||||
|
||||
fn test_sb() {
|
||||
mut sb := strings.new_builder(100)
|
||||
sb.write_string('hi')
|
||||
sb.write_string('!')
|
||||
sb.write_string('hello')
|
||||
assert sb.len == 8
|
||||
sb_end := sb.str()
|
||||
assert sb_end == 'hi!hello'
|
||||
assert sb.len == 0
|
||||
///
|
||||
sb = strings.new_builder(10)
|
||||
sb.write_string('a')
|
||||
sb.write_string('b')
|
||||
assert sb.len == 2
|
||||
assert sb.str() == 'ab'
|
||||
// Test interpolation optimization
|
||||
sb = strings.new_builder(10)
|
||||
x := 10
|
||||
y := MyInt(20)
|
||||
sb.writeln('x = ${x} y = ${y}')
|
||||
res := sb.str()
|
||||
assert res[res.len - 1] == `\n`
|
||||
println('"${res}"')
|
||||
assert res.trim_space() == 'x = 10 y = 20'
|
||||
|
||||
sb = strings.new_builder(10)
|
||||
sb.write_string('x = ${x} y = ${y}')
|
||||
assert sb.str() == 'x = 10 y = 20'
|
||||
//$if !windows {
|
||||
sb = strings.new_builder(10)
|
||||
sb.write_string('123456')
|
||||
last_2 := sb.cut_last(2)
|
||||
assert last_2 == '56'
|
||||
final_sb := sb.str()
|
||||
assert final_sb == '1234'
|
||||
//}
|
||||
sb.clear()
|
||||
assert sb.str() == ''
|
||||
}
|
||||
|
||||
fn test_big_sb() {
|
||||
mut sb := strings.new_builder(100)
|
||||
mut sb2 := strings.new_builder(10000)
|
||||
for i in 0 .. maxn {
|
||||
sb.writeln(i.str())
|
||||
sb2.write_string('+')
|
||||
}
|
||||
s := sb.str()
|
||||
lines := s.split_into_lines()
|
||||
assert lines.len == maxn
|
||||
assert lines[0] == '0'
|
||||
assert lines[1] == '1'
|
||||
assert lines[777] == '777'
|
||||
assert lines[98765] == '98765'
|
||||
println(sb2.len)
|
||||
assert sb2.len == maxn
|
||||
}
|
||||
|
||||
fn test_byte_write() {
|
||||
mut sb := strings.new_builder(100)
|
||||
temp_str := 'byte testing'
|
||||
mut count := 0
|
||||
for word in temp_str {
|
||||
sb.write_u8(word)
|
||||
count++
|
||||
assert count == sb.len
|
||||
}
|
||||
sb_final := sb.str()
|
||||
assert sb_final == temp_str
|
||||
}
|
||||
|
||||
fn test_strings_builder_reuse() {
|
||||
mut sb := strings.new_builder(256)
|
||||
sb.write_string('world')
|
||||
assert sb.str() == 'world'
|
||||
sb.write_string('hello')
|
||||
assert sb.str() == 'hello'
|
||||
}
|
||||
|
||||
fn test_cut_to() {
|
||||
mut sb := strings.new_builder(16)
|
||||
sb.write_string('hello')
|
||||
assert sb.cut_to(3) == 'lo'
|
||||
assert sb.len == 3
|
||||
assert sb.cut_to(3) == ''
|
||||
assert sb.len == 3
|
||||
assert sb.cut_to(0) == 'hel'
|
||||
assert sb.cut_to(32) == ''
|
||||
assert sb.len == 0
|
||||
}
|
||||
|
||||
fn test_write_rune() {
|
||||
mut sb := strings.new_builder(10)
|
||||
sb.write_rune(`h`)
|
||||
sb.write_rune(`e`)
|
||||
sb.write_rune(`l`)
|
||||
sb.write_rune(`l`)
|
||||
sb.write_rune(`o`)
|
||||
x := sb.str()
|
||||
assert x == 'hello'
|
||||
}
|
||||
|
||||
fn test_write_runes() {
|
||||
mut sb := strings.new_builder(20)
|
||||
sb.write_runes([`h`, `e`, `l`, `l`, `o`])
|
||||
sb.write_rune(` `)
|
||||
sb.write_runes([`w`, `o`, `r`, `l`, `d`])
|
||||
x := sb.str()
|
||||
assert x == 'hello world'
|
||||
}
|
||||
|
||||
fn test_ensure_cap() {
|
||||
mut sb := strings.new_builder(0)
|
||||
assert sb.cap == 0
|
||||
sb.ensure_cap(10)
|
||||
assert sb.cap >= 10
|
||||
old_cap := sb.cap
|
||||
sb.ensure_cap(10)
|
||||
assert sb.cap == old_cap
|
||||
sb.ensure_cap(15)
|
||||
assert sb.cap >= 15
|
||||
old_cap2 := sb.cap
|
||||
sb.ensure_cap(10)
|
||||
assert sb.cap == old_cap2
|
||||
sb.ensure_cap(-1)
|
||||
assert sb.cap == old_cap2
|
||||
}
|
||||
|
||||
fn test_drain_builder() {
|
||||
mut sb := strings.new_builder(0)
|
||||
mut target_sb := strings.new_builder(0)
|
||||
assert sb.cap == 0
|
||||
assert target_sb.cap == 0
|
||||
|
||||
sb.write_string('abc')
|
||||
assert sb.len == 3
|
||||
|
||||
target_sb.drain_builder(mut sb, 0)
|
||||
assert sb.len == 0
|
||||
assert target_sb.len == 3
|
||||
assert target_sb.str() == 'abc'
|
||||
}
|
||||
|
||||
@[manualfree]
|
||||
fn sb_i64_str(n i64) string {
|
||||
mut sb := strings.new_builder(24)
|
||||
defer {
|
||||
unsafe { sb.free() }
|
||||
}
|
||||
sb.write_decimal(n)
|
||||
return sb.str()
|
||||
}
|
||||
|
||||
fn test_write_decimal() {
|
||||
assert sb_i64_str(0) == '0'
|
||||
assert sb_i64_str(1) == '1'
|
||||
assert sb_i64_str(-1) == '-1'
|
||||
assert sb_i64_str(1001) == '1001'
|
||||
assert sb_i64_str(-1001) == '-1001'
|
||||
assert sb_i64_str(1234567890) == '1234567890'
|
||||
assert sb_i64_str(-1234567890) == '-1234567890'
|
||||
assert sb_i64_str(9223372036854775807) == '9223372036854775807'
|
||||
assert sb_i64_str(-9223372036854775807) == '-9223372036854775807'
|
||||
assert sb_i64_str(min_i64) == '-9223372036854775808'
|
||||
}
|
||||
|
||||
fn test_grow_len() {
|
||||
mut sb := strings.new_builder(10)
|
||||
assert sb.len == 0
|
||||
assert sb.cap == 10
|
||||
|
||||
sb.write_string('0123456789')
|
||||
assert sb.len == 10
|
||||
|
||||
unsafe { sb.grow_len(-5) }
|
||||
assert sb.len == 10
|
||||
assert sb.cap == 10
|
||||
|
||||
unsafe { sb.grow_len(10) }
|
||||
assert sb.len == 20
|
||||
assert sb.cap == 20
|
||||
|
||||
sb.ensure_cap(35)
|
||||
assert sb.len == 20
|
||||
assert sb.cap >= 35
|
||||
cap_after_ensure := sb.cap
|
||||
|
||||
unsafe { sb.grow_len(5) }
|
||||
assert sb.len == 25
|
||||
assert sb.cap == cap_after_ensure
|
||||
}
|
||||
|
||||
fn test_write_repeated_rune() {
|
||||
mut sb := strings.new_builder(20)
|
||||
sb.write_repeated_rune(`h`, 5)
|
||||
sb.write_repeated_rune(`w`, 5)
|
||||
sb.write_repeated_rune(`√`, 5)
|
||||
sb.write_rune(` `)
|
||||
x := sb.str()
|
||||
assert x == 'hhhhhwwwww√√√√√ '
|
||||
}
|
||||
|
||||
struct IndentTest {
|
||||
param strings.IndentParam
|
||||
input string
|
||||
output string
|
||||
}
|
||||
|
||||
// vfmt off
|
||||
const indent_test_data = [
|
||||
IndentTest{
|
||||
input: 'User1 {
|
||||
name: "John"
|
||||
settings: {
|
||||
theme: "dark"
|
||||
language: "en"
|
||||
}
|
||||
}'
|
||||
output: 'User1 {
|
||||
name: "John"
|
||||
settings: {
|
||||
theme: "dark"
|
||||
language: "en"
|
||||
}
|
||||
}'
|
||||
},
|
||||
IndentTest{
|
||||
input: 'User2{name:"John" settings:{theme:"dark" language:"en" hobbies:["reading","sports"]}}'
|
||||
output: 'User2{
|
||||
name:"John" settings:{
|
||||
theme:"dark" language:"en" hobbies:["reading","sports"]
|
||||
}
|
||||
}'
|
||||
},
|
||||
IndentTest{
|
||||
input: 'message {text: "Hello {world}!" count: 5 nested: {data: "Test {inner}"}}'
|
||||
output: 'message {
|
||||
text: "Hello {world}!" count: 5 nested: {
|
||||
data: "Test {inner}"
|
||||
}
|
||||
}'
|
||||
},
|
||||
IndentTest{
|
||||
input: 'if x > 0 {println("Positive") for i in 0..x {println(i) if i % 2 == 0 {println("even")}}} else {println("Not positive")}'
|
||||
output: 'if x > 0 {
|
||||
println("Positive") for i in 0..x {
|
||||
println(i) if i % 2 == 0 {
|
||||
println("even")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
println("Not positive")
|
||||
}'
|
||||
},
|
||||
IndentTest{
|
||||
input: 'config{database:{host:"localhost" port:5432} server:{port:8080 routes:{api:"/api" static:"/static"}} log:{level:"info"}}'
|
||||
output: 'config{
|
||||
database:{
|
||||
host:"localhost" port:5432
|
||||
} server:{
|
||||
port:8080 routes:{
|
||||
api:"/api" static:"/static"
|
||||
}
|
||||
} log:{
|
||||
level:"info"
|
||||
}
|
||||
}'
|
||||
},
|
||||
IndentTest{
|
||||
input: "MyS{
|
||||
a: 0
|
||||
b: 0
|
||||
son: Son{
|
||||
k: ''
|
||||
m: ''
|
||||
}
|
||||
}
|
||||
"
|
||||
output: "MyS{
|
||||
a: 0
|
||||
b: 0
|
||||
son: Son{
|
||||
k: ''
|
||||
m: ''
|
||||
}
|
||||
}
|
||||
"
|
||||
},
|
||||
IndentTest{
|
||||
input: 'config {message: "He said: \\"Hello World!\\"" code: "if (x) { return \\"value\\"; }"}'
|
||||
output: 'config {
|
||||
message: "He said: \\"Hello World!\\"" code: "if (x) { return \\"value\\"; }"
|
||||
}'
|
||||
},
|
||||
IndentTest {
|
||||
input : 'data {text: "String with {curly braces} and \\"quotes\\"" nested: {value: "Inner \\"quoted\\" string"}}'
|
||||
output : 'data {
|
||||
text: "String with {curly braces} and \\"quotes\\"" nested: {
|
||||
value: "Inner \\"quoted\\" string"
|
||||
}
|
||||
}'
|
||||
},
|
||||
IndentTest {
|
||||
input : 'escape {backslash: "Path: C:\\\\Users\\\\Test" newline: "Line1\\nLine2" tab: "Column1\\tColumn2"}'
|
||||
output : 'escape {
|
||||
backslash: "Path: C:\\\\Users\\\\Test" newline: "Line1\\nLine2" tab: "Column1\\tColumn2"
|
||||
}'
|
||||
},
|
||||
IndentTest {
|
||||
input : 'nested {outer: "Outer \\"quote\\" with {braces}" inner: {value: "Inner string with \\\\backslash and \\"quotes\\""}}'
|
||||
output : 'nested {
|
||||
outer: "Outer \\"quote\\" with {braces}" inner: {
|
||||
value: "Inner string with \\\\backslash and \\"quotes\\""
|
||||
}
|
||||
}'
|
||||
},
|
||||
IndentTest {
|
||||
input : 'complex {str1: "\\"Escaped quotes\\"" str2: "Backslash: \\\\" str3: "Mixed: \\"quoted\\" and \\\\backslash" regex: "Pattern: \\\\d+\\\\.\\\\d+"}'
|
||||
output : 'complex {
|
||||
str1: "\\"Escaped quotes\\"" str2: "Backslash: \\\\" str3: "Mixed: \\"quoted\\" and \\\\backslash" regex: "Pattern: \\\\d+\\\\.\\\\d+"
|
||||
}'
|
||||
},
|
||||
IndentTest {
|
||||
input : 'json {data: "{\\"name\\": \\"John\\", \\"age\\": 30, \\"city\\": \\"New York\\"}"}'
|
||||
output : 'json {
|
||||
data: "{\\"name\\": \\"John\\", \\"age\\": 30, \\"city\\": \\"New York\\"}"
|
||||
}'
|
||||
},
|
||||
IndentTest {
|
||||
input : 'code {function: "fn test() { if (x) { return \\\"value\\\"; } }" error: "Error: \\\"File not found\\\" at line 10"}'
|
||||
output : 'code {
|
||||
function: "fn test() { if (x) { return \\\"value\\\"; } }" error: "Error: \\\"File not found\\\" at line 10"
|
||||
}'
|
||||
},
|
||||
IndentTest {
|
||||
input : 'multiline {message: "Line 1\\nLine 2\\nLine 3 with {braces}\\nLine 4 with \\\"quotes\\\""}'
|
||||
output : 'multiline {
|
||||
message: "Line 1\\nLine 2\\nLine 3 with {braces}\\nLine 4 with \\\"quotes\\\""
|
||||
}'
|
||||
},
|
||||
IndentTest {
|
||||
input : 'extreme {str: "\\\\\\\\\\\\\\"Quadruple escaped\\\\\\\\\\\\\\""}'
|
||||
output : 'extreme {
|
||||
str: "\\\\\\\\\\\\\\"Quadruple escaped\\\\\\\\\\\\\\""
|
||||
}'
|
||||
},
|
||||
IndentTest{
|
||||
param: strings.IndentParam {
|
||||
block_start: `[`
|
||||
block_end: `]`
|
||||
}
|
||||
input: 'message [text: "Hello {world}!" count: 5 nested: [data: "Test {inner}"]]'
|
||||
output: 'message [
|
||||
text: "Hello {world}!" count: 5 nested: [
|
||||
data: "Test {inner}"
|
||||
]
|
||||
]'
|
||||
},
|
||||
IndentTest{
|
||||
param: strings.IndentParam {
|
||||
block_start: `[`
|
||||
block_end: `]`
|
||||
indent_char: `#`
|
||||
}
|
||||
input: 'message [text: "Hello {world}!" count: 5 nested: [data: "Test {inner}"]]'
|
||||
output: 'message [
|
||||
####text: "Hello {world}!" count: 5 nested: [
|
||||
########data: "Test {inner}"
|
||||
####]
|
||||
]'
|
||||
},
|
||||
IndentTest{
|
||||
param: strings.IndentParam {
|
||||
block_start: `[`
|
||||
block_end: `]`
|
||||
indent_char: `\t`
|
||||
indent_count: 2
|
||||
}
|
||||
input: 'message [text: "Hello {world}!" count: 5 nested: [data: "Test {inner}"]]'
|
||||
output: 'message [
|
||||
\t\ttext: "Hello {world}!" count: 5 nested: [
|
||||
\t\t\t\tdata: "Test {inner}"
|
||||
\t\t]
|
||||
]'
|
||||
},
|
||||
IndentTest{
|
||||
param: strings.IndentParam {
|
||||
block_start: `[`
|
||||
block_end: `]`
|
||||
indent_char: `#`
|
||||
indent_count: 1
|
||||
starting_level: 2
|
||||
}
|
||||
input: 'message [text: "Hello {world}!" count: 5 nested: [data: "Test {inner}"]]'
|
||||
output: '##message [
|
||||
###text: "Hello {world}!" count: 5 nested: [
|
||||
####data: "Test {inner}"
|
||||
###]
|
||||
##]'
|
||||
},
|
||||
]
|
||||
|
||||
// vfmt on
|
||||
|
||||
fn test_indent() {
|
||||
for t in indent_test_data {
|
||||
mut sb := strings.new_builder(256)
|
||||
sb.indent(t.input, t.param)
|
||||
assert sb.str() == t.output
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
module lorem
|
||||
|
||||
const lorem_corpora = {
|
||||
'lorem': lorem_corpus
|
||||
'poe': poe_corpus
|
||||
'darwin': darwin_corpus
|
||||
'bard': shakespeare_corpus
|
||||
}
|
||||
|
||||
const lorem_corpus = '
|
||||
lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor
|
||||
incididunt ut labore et dolore magna aliqua Ut enim ad minim veniam quis
|
||||
nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat
|
||||
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore
|
||||
eu fugiat nulla pariatur Excepteur sint occaecat cupidatat non proident sunt
|
||||
in culpa qui officia deserunt mollit anim id est laborum
|
||||
'
|
||||
|
||||
const poe_corpus = '
|
||||
once upon a midnight dreary while I pondered weak and weary
|
||||
over many a quaint and curious volume of forgotten lore
|
||||
while I nodded nearly napping suddenly there came a tapping
|
||||
as of someone gently rapping at my chamber door
|
||||
'
|
||||
|
||||
const darwin_corpus = '
|
||||
when we look to the individuals of the same variety or sub variety of our
|
||||
older cultivated plants and animals one of the first points which strikes
|
||||
us is that they generally differ much more from each other than do the
|
||||
individuals of any one species or variety in a state of nature
|
||||
'
|
||||
|
||||
const shakespeare_corpus = '
|
||||
to be or not to be that is the question
|
||||
all the worlds a stage and all the men and women merely players
|
||||
the lady doth protest too much methinks
|
||||
a rose by any other name would smell as sweet
|
||||
et tu brute
|
||||
if music be the food of love play on
|
||||
now is the winter of our discontent
|
||||
we are such stuff as dreams are made on
|
||||
brevity is the soul of wit
|
||||
some are born great some achieve greatness and some have greatness thrust upon them
|
||||
cry havoc and let slip the dogs of war
|
||||
all that glisters is not gold
|
||||
the fault dear brutus is not in our stars but in ourselves
|
||||
to thine own self be true
|
||||
lord what fools these mortals be
|
||||
shall i compare thee to a summers day
|
||||
'
|
||||
|
||||
// ---------------- Seed Phrases ----------------
|
||||
|
||||
const lorem_seed_phrases = [
|
||||
'in the beginning',
|
||||
'once upon a time',
|
||||
'it was the first',
|
||||
'when we consider',
|
||||
'there was a moment',
|
||||
]
|
||||
|
||||
// ---------------- Text Generation ----------------
|
||||
|
||||
// LoremCfg
|
||||
// - `corpus_name` random if not specified
|
||||
// - `seed_text` random if not specified
|
||||
// - `rng_seed` random if not specified
|
||||
@[params]
|
||||
pub struct LoremCfg {
|
||||
pub:
|
||||
words_per_sentence int = 10
|
||||
sentences_per_paragraph int = 5
|
||||
paragraphs int = 3
|
||||
markov_order int = 2
|
||||
corpus_name string
|
||||
seed_text string
|
||||
rng_seed int
|
||||
}
|
||||
|
||||
// lorum - pseudo-random text using a Markov chain built from
|
||||
// one of several embedded corpora. It produces structured output in the
|
||||
// form of paragraphs and sentences, with configurable parameters for:
|
||||
//
|
||||
// - Markov order (n-gram size)
|
||||
// - Words per sentence
|
||||
// - Sentences per paragraph
|
||||
// - Paragraph count
|
||||
// - Optional seed phrases and RNG seed
|
||||
// - Optional corpus selection
|
||||
//
|
||||
pub fn generate(cfg LoremCfg) string {
|
||||
// Initialize LCG with provided seed or default
|
||||
// If rng_seed is 0, we use a default seed (making it deterministic by default)
|
||||
initial_seed := if cfg.rng_seed != 0 { u32(cfg.rng_seed) } else { u32(123456789) }
|
||||
mut rng := LorumRNG{
|
||||
seed: initial_seed
|
||||
}
|
||||
|
||||
seed := match cfg.seed_text != '' {
|
||||
true { cfg.seed_text }
|
||||
else { lorem_random_seed_phrase(mut rng) }
|
||||
}
|
||||
|
||||
corpus := lorem_select_corpus(mut rng, cfg.corpus_name)
|
||||
tokens := lorem_tokenize(corpus)
|
||||
|
||||
if tokens.len <= cfg.markov_order {
|
||||
eprintln('corpus too small for selected order')
|
||||
return ''
|
||||
}
|
||||
|
||||
model := lorem_build_markov(tokens, cfg.markov_order)
|
||||
|
||||
mut state := lorem_tokenize(seed)
|
||||
if state.len < cfg.markov_order {
|
||||
start := rng.intn(tokens.len - cfg.markov_order)
|
||||
state = tokens[start..start + cfg.markov_order].clone()
|
||||
}
|
||||
|
||||
mut out := []string{}
|
||||
|
||||
for pi in 0 .. cfg.paragraphs {
|
||||
if pi != 0 {
|
||||
out << '\n\n'
|
||||
}
|
||||
sentences := lorem_vary(mut rng, cfg.sentences_per_paragraph, 1)
|
||||
|
||||
for si in 0 .. sentences {
|
||||
if si != 0 {
|
||||
out << ' '
|
||||
}
|
||||
words := lorem_vary(mut rng, cfg.words_per_sentence, 3)
|
||||
mut sentence := []string{}
|
||||
|
||||
for _ in 0 .. words {
|
||||
key := state.join('\u0001')
|
||||
nexts := model[key] or {
|
||||
start := rng.intn(tokens.len - cfg.markov_order)
|
||||
state = tokens[start..start + cfg.markov_order].clone()
|
||||
continue
|
||||
}
|
||||
|
||||
next := nexts[rng.intn(nexts.len)]
|
||||
sentence << next
|
||||
|
||||
state = state[1..].clone()
|
||||
state << next
|
||||
}
|
||||
|
||||
if sentence.len > 0 {
|
||||
out << sentence.join(' ').capitalize()
|
||||
out << '.'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return out.join('')
|
||||
}
|
||||
|
||||
// ---------------- Utilities ----------------
|
||||
|
||||
// LorumRNG is a simple internal PRNG used to avoid importing the `rand` module,
|
||||
// which would cause an import cycle (strings -> rand -> time -> ... -> strings).
|
||||
struct LorumRNG {
|
||||
mut:
|
||||
seed u32
|
||||
}
|
||||
|
||||
// intn returns a pseudo-random number in [0, max).
|
||||
// It is a minimal implementation sufficient for text generation.
|
||||
fn (mut r LorumRNG) intn(max int) int {
|
||||
if max <= 0 {
|
||||
return 0
|
||||
}
|
||||
r.seed = r.seed * 1664525 + 1013904223
|
||||
return int(r.seed % u32(max))
|
||||
}
|
||||
|
||||
fn lorem_vary(mut rng LorumRNG, base int, min int) int {
|
||||
delta := int(f32(base) * 0.2)
|
||||
if delta == 0 {
|
||||
return base
|
||||
}
|
||||
offset := rng.intn(delta * 2 + 1) - delta
|
||||
val := base + offset
|
||||
return if val < min { min } else { val }
|
||||
}
|
||||
|
||||
fn lorem_select_corpus(mut rng LorumRNG, name string) string {
|
||||
if name != '' {
|
||||
if corpus := lorem_corpora[name] {
|
||||
return corpus
|
||||
}
|
||||
eprintln('unknown corpus: ${name}')
|
||||
exit(1)
|
||||
}
|
||||
|
||||
keys := lorem_corpora.keys()
|
||||
key := keys[rng.intn(keys.len)]
|
||||
return lorem_corpora[key]
|
||||
}
|
||||
|
||||
fn lorem_random_seed_phrase(mut rng LorumRNG) string {
|
||||
return lorem_seed_phrases[rng.intn(lorem_seed_phrases.len)]
|
||||
}
|
||||
|
||||
fn lorem_tokenize(text string) []string {
|
||||
return text
|
||||
.replace_each(['\n', ' ', '\t', ' '])
|
||||
.split(' ')
|
||||
.filter(it.len > 0)
|
||||
}
|
||||
|
||||
fn lorem_build_markov(tokens []string, order int) map[string][]string {
|
||||
mut model := map[string][]string{}
|
||||
for i in 0 .. tokens.len - order {
|
||||
key := tokens[i..i + order].join('\u0001')
|
||||
if key !in model {
|
||||
model[key] = []string{}
|
||||
}
|
||||
model[key] << tokens[i + order]
|
||||
}
|
||||
return model
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
module lorem
|
||||
|
||||
fn test_lorem_generate_basic() {
|
||||
output := generate(LoremCfg{
|
||||
paragraphs: 2
|
||||
sentences_per_paragraph: 3
|
||||
words_per_sentence: 5
|
||||
})
|
||||
assert output.len > 0
|
||||
// 2 paragraphs should be separated by "\n\n"
|
||||
assert output.count('\n\n') == 1
|
||||
}
|
||||
|
||||
fn test_lorem_generate_deterministic() {
|
||||
cfg := LoremCfg{
|
||||
rng_seed: 12345
|
||||
paragraphs: 1
|
||||
}
|
||||
out1 := generate(cfg)
|
||||
out2 := generate(cfg)
|
||||
assert out1 == out2
|
||||
}
|
||||
|
||||
fn test_lorem_generate_counts() {
|
||||
cfg := LoremCfg{
|
||||
paragraphs: 3
|
||||
}
|
||||
output := generate(cfg)
|
||||
// There should be 2 separators for 3 paragraphs
|
||||
assert output.count('\n\n') == 2
|
||||
}
|
||||
|
||||
fn test_lorem_custom_corpus() {
|
||||
// 'bard' is shakespeare
|
||||
cfg := LoremCfg{
|
||||
corpus_name: 'bard'
|
||||
rng_seed: 999
|
||||
paragraphs: 1
|
||||
}
|
||||
output := generate(cfg)
|
||||
assert output.len > 0
|
||||
// Hard to check exact content due to randomness, but it should not crash
|
||||
}
|
||||
|
||||
fn test_lorem_vary() {
|
||||
// lorem_vary is private, effectively testing internal logic
|
||||
// base 10, min 5
|
||||
// delta = 10 * 0.2 = 2
|
||||
// range = [-2, 2]
|
||||
// result = 10 + [-2, 2] -> [8, 12]
|
||||
// We can't guarantee a specific value, but we can check bounds
|
||||
mut rng := LorumRNG{
|
||||
seed: 12345
|
||||
}
|
||||
for _ in 0 .. 100 {
|
||||
val := lorem_vary(mut rng, 10, 5)
|
||||
assert val >= 8
|
||||
assert val <= 12
|
||||
}
|
||||
}
|
||||
|
||||
fn test_lorem_tokenize() {
|
||||
text := 'Hello\nWorld Test'
|
||||
tokens := lorem_tokenize(text)
|
||||
assert tokens == ['Hello', 'World', 'Test']
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
module strings
|
||||
|
||||
@[inline]
|
||||
fn min(a int, b int, c int) int {
|
||||
mut m := a
|
||||
if b < m {
|
||||
m = b
|
||||
}
|
||||
if c < m {
|
||||
m = c
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
@[inline]
|
||||
fn max2(a int, b int) int {
|
||||
if a < b {
|
||||
return b
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
@[inline]
|
||||
fn min2(a int, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
@[inline]
|
||||
fn abs2(a int, b int) int {
|
||||
if a < b {
|
||||
return b - a
|
||||
}
|
||||
return a - b
|
||||
}
|
||||
|
||||
// levenshtein_distance uses the Levenshtein Distance algorithm to calculate
|
||||
// the distance between between two strings `a` and `b` (lower is closer).
|
||||
@[direct_array_access]
|
||||
pub fn levenshtein_distance(a string, b string) int {
|
||||
if a.len == 0 {
|
||||
return b.len
|
||||
}
|
||||
if b.len == 0 {
|
||||
return a.len
|
||||
}
|
||||
if a == b {
|
||||
return 0
|
||||
}
|
||||
mut row := []int{len: a.len + 1, init: index}
|
||||
for i := 1; i < b.len + 1; i++ {
|
||||
mut prev := i
|
||||
for j := 1; j < a.len + 1; j++ {
|
||||
mut current := row[j - 1] // match
|
||||
if b[i - 1] != a[j - 1] {
|
||||
// insertion, substitution, deletion
|
||||
current = min(row[j - 1] + 1, prev + 1, row[j] + 1)
|
||||
}
|
||||
row[j - 1] = prev
|
||||
prev = current
|
||||
}
|
||||
row[a.len] = prev
|
||||
}
|
||||
return row[a.len]
|
||||
}
|
||||
|
||||
// levenshtein_distance_percentage uses the Levenshtein Distance algorithm to calculate how similar two strings are as a percentage (higher is closer).
|
||||
pub fn levenshtein_distance_percentage(a string, b string) f32 {
|
||||
d := levenshtein_distance(a, b)
|
||||
l := if a.len >= b.len { a.len } else { b.len }
|
||||
return (1.00 - f32(d) / f32(l)) * 100.00
|
||||
}
|
||||
|
||||
// dice_coefficient implements the Sørensen–Dice coefficient.
|
||||
// It finds the similarity between two strings, and returns a coefficient
|
||||
// between 0.0 (not similar) and 1.0 (exact match).
|
||||
pub fn dice_coefficient(s1 string, s2 string) f32 {
|
||||
if s1.len == 0 || s2.len == 0 {
|
||||
return 0.0
|
||||
}
|
||||
if s1 == s2 {
|
||||
return 1.0
|
||||
}
|
||||
if s1.len < 2 || s2.len < 2 {
|
||||
return 0.0
|
||||
}
|
||||
a := if s1.len > s2.len { s1 } else { s2 }
|
||||
b := if a == s1 { s2 } else { s1 }
|
||||
mut first_bigrams := map[string]int{}
|
||||
for i in 0 .. a.len - 1 {
|
||||
bigram := a[i..i + 2]
|
||||
q := if bigram in first_bigrams { first_bigrams[bigram] + 1 } else { 1 }
|
||||
first_bigrams[bigram] = q
|
||||
}
|
||||
mut intersection_size := 0
|
||||
for i in 0 .. b.len - 1 {
|
||||
bigram := b[i..i + 2]
|
||||
count := if bigram in first_bigrams { first_bigrams[bigram] } else { 0 }
|
||||
if count > 0 {
|
||||
first_bigrams[bigram] = count - 1
|
||||
intersection_size++
|
||||
}
|
||||
}
|
||||
return (2.0 * f32(intersection_size)) / (f32(a.len) + f32(b.len) - 2)
|
||||
}
|
||||
|
||||
// hamming_distance uses the Hamming Distance algorithm to calculate
|
||||
// the distance between two strings `a` and `b` (lower is closer).
|
||||
@[direct_array_access]
|
||||
pub fn hamming_distance(a string, b string) int {
|
||||
if a.len == 0 && b.len == 0 {
|
||||
return 0
|
||||
}
|
||||
mut match_len := min2(a.len, b.len)
|
||||
mut diff_count := abs2(a.len, b.len)
|
||||
for i in 0 .. match_len {
|
||||
if a[i] != b[i] {
|
||||
diff_count++
|
||||
}
|
||||
}
|
||||
return diff_count
|
||||
}
|
||||
|
||||
// hamming_similarity uses the Hamming Distance algorithm to calculate the distance between two strings `a` and `b`.
|
||||
// It returns a coefficient between 0.0 (not similar) and 1.0 (exact match).
|
||||
pub fn hamming_similarity(a string, b string) f32 {
|
||||
l := max2(a.len, b.len)
|
||||
if l == 0 {
|
||||
// Both are empty strings, should return 1.0
|
||||
return 1.0
|
||||
}
|
||||
d := hamming_distance(a, b)
|
||||
return 1.00 - f32(d) / f32(l)
|
||||
}
|
||||
|
||||
// jaro_similarity uses the Jaro Distance algorithm to calculate
|
||||
// the distance between two strings `a` and `b`.
|
||||
// It returns a coefficient between 0.0 (not similar) and 1.0 (exact match).
|
||||
@[direct_array_access]
|
||||
pub fn jaro_similarity(a string, b string) f64 {
|
||||
a_len := a.len
|
||||
b_len := b.len
|
||||
if a_len == 0 && b_len == 0 {
|
||||
// Both are empty strings, should return 1.0
|
||||
return 1.0
|
||||
}
|
||||
if a_len == 0 || b_len == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Maximum distance upto which matching is allowed
|
||||
match_distance := max2(max2(a_len, b_len) / 2 - 1, 0)
|
||||
|
||||
mut a_matches := []bool{len: a_len}
|
||||
mut b_matches := []bool{len: b_len}
|
||||
mut matches := 0
|
||||
mut transpositions := 0.0
|
||||
|
||||
// Traverse through the first string
|
||||
for i in 0 .. a_len {
|
||||
start := max2(0, i - match_distance)
|
||||
end := min2(b_len, i + match_distance + 1)
|
||||
for k in start .. end {
|
||||
// If there is a match
|
||||
if b_matches[k] {
|
||||
continue
|
||||
}
|
||||
if a[i] != b[k] {
|
||||
continue
|
||||
}
|
||||
a_matches[i] = true
|
||||
b_matches[k] = true
|
||||
matches++
|
||||
break
|
||||
}
|
||||
}
|
||||
// If there is no match
|
||||
if matches == 0 {
|
||||
return 0
|
||||
}
|
||||
mut k := 0
|
||||
// Count number of occurrences where two characters match but
|
||||
// there is a third matched character in between the indices
|
||||
for i in 0 .. a_len {
|
||||
if !a_matches[i] {
|
||||
continue
|
||||
}
|
||||
// Find the next matched character in second string
|
||||
for !b_matches[k] {
|
||||
k++
|
||||
}
|
||||
if a[i] != b[k] {
|
||||
transpositions++
|
||||
}
|
||||
k++
|
||||
}
|
||||
transpositions /= 2
|
||||
return (matches / f64(a_len) + matches / f64(b_len) + (matches - transpositions) / matches) / 3
|
||||
}
|
||||
|
||||
// jaro_winkler_similarity uses the Jaro Winkler Distance algorithm to calculate
|
||||
// the distance between two strings `a` and `b`.
|
||||
// It returns a coefficient between 0.0 (not similar) and 1.0 (exact match).
|
||||
// The scaling factor(`p=0.1`) in Jaro-Winkler gives higher weight to prefix
|
||||
// similarities, making it especially effective for cases where slight misspellings
|
||||
// or prefixes are common.
|
||||
@[direct_array_access]
|
||||
pub fn jaro_winkler_similarity(a string, b string) f64 {
|
||||
// Maximum of 4 characters are allowed in prefix
|
||||
mut lmax := min2(4, min2(a.len, b.len))
|
||||
mut l := 0
|
||||
for i in 0 .. lmax {
|
||||
if a[i] == b[i] {
|
||||
l++
|
||||
}
|
||||
}
|
||||
js := jaro_similarity(a, b)
|
||||
// select a multiplier (Winkler suggested p=0.1) for the relative importance of the prefix for the word similarity
|
||||
p := 0.1
|
||||
ws := js + f64(l) * p * (1 - js)
|
||||
return ws
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// vtest build: present_node?
|
||||
import strings
|
||||
|
||||
fn test_levenshtein_distance() {
|
||||
assert strings.levenshtein_distance('', '') == 0
|
||||
assert strings.levenshtein_distance('one', 'one') == 0
|
||||
assert strings.levenshtein_distance('', 'two') == 3
|
||||
assert strings.levenshtein_distance('three', '') == 5
|
||||
assert strings.levenshtein_distance('bananna', '') == 7
|
||||
assert strings.levenshtein_distance('cats', 'hats') == 1
|
||||
assert strings.levenshtein_distance('hugs', 'shrugs') == 2
|
||||
assert strings.levenshtein_distance('broom', 'shroom') == 2
|
||||
assert strings.levenshtein_distance('flomax', 'volmax') == 3
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import strings
|
||||
|
||||
fn test_levenshtein_distance() {
|
||||
assert strings.levenshtein_distance('', '') == 0
|
||||
assert strings.levenshtein_distance('one', 'one') == 0
|
||||
assert strings.levenshtein_distance('', 'two') == 3
|
||||
assert strings.levenshtein_distance('three', '') == 5
|
||||
assert strings.levenshtein_distance('bananna', '') == 7
|
||||
assert strings.levenshtein_distance('cats', 'hats') == 1
|
||||
assert strings.levenshtein_distance('hugs', 'shrugs') == 2
|
||||
assert strings.levenshtein_distance('broom', 'shroom') == 2
|
||||
assert strings.levenshtein_distance('flomax', 'volmax') == 3
|
||||
assert strings.levenshtein_distance('ab', 'cd') == 2
|
||||
}
|
||||
|
||||
fn test_hamming_distance() {
|
||||
assert strings.hamming_distance('', '') == 0
|
||||
assert strings.hamming_distance('one', 'one') == 0
|
||||
assert strings.hamming_distance('', 'two') == 3
|
||||
assert strings.hamming_distance('three', '') == 5
|
||||
assert strings.hamming_distance('bananna', '') == 7
|
||||
assert strings.hamming_distance('cats', 'hats') == 1
|
||||
assert strings.hamming_distance('hugs', 'shrugs') == 6
|
||||
assert strings.hamming_distance('broom', 'shroom') == 5
|
||||
assert strings.hamming_distance('flomax', 'volmax') == 3
|
||||
assert strings.hamming_distance('ab', 'cd') == 2
|
||||
}
|
||||
|
||||
fn test_hamming_similarity() {
|
||||
assert strings.hamming_similarity('', '') == 1.0
|
||||
assert strings.hamming_similarity('one', 'one') == 1.0
|
||||
assert strings.hamming_similarity('', 'two') == 0
|
||||
assert strings.hamming_similarity('three', '') == 0
|
||||
assert strings.hamming_similarity('bananna', '') == 0
|
||||
assert strings.hamming_similarity('cats', 'hats') == 0.75
|
||||
assert strings.hamming_similarity('hugs', 'shrugs') == 0
|
||||
assert strings.hamming_similarity('broom', 'shroom') == 0.1666666865348816
|
||||
assert strings.hamming_similarity('flomax', 'volmax') == 0.5
|
||||
assert strings.hamming_similarity('ab', 'cd') == 0
|
||||
}
|
||||
|
||||
fn test_jaro_similarity() {
|
||||
assert strings.jaro_similarity('', '') == 1
|
||||
assert strings.jaro_similarity('one', 'one') == 1
|
||||
assert strings.jaro_similarity('', 'two') == 0
|
||||
assert strings.jaro_similarity('three', '') == 0
|
||||
assert strings.jaro_similarity('bananna', '') == 0
|
||||
assert strings.jaro_similarity('MARTHA', 'MARHTA') == 0.9444444444444445
|
||||
assert strings.jaro_similarity('DIXON', 'DICKSONX') == 0.7666666666666666
|
||||
assert strings.jaro_similarity('JELLYFISH', 'SMELLYFISH') == 0.8962962962962964
|
||||
assert strings.jaro_similarity('a', 'a') == 1
|
||||
assert strings.jaro_similarity('a', 'b') == 0
|
||||
}
|
||||
|
||||
fn test_jaro_winkler_similarity() {
|
||||
assert strings.jaro_winkler_similarity('', '') == 1
|
||||
assert strings.jaro_winkler_similarity('one', 'one') == 1
|
||||
assert strings.jaro_winkler_similarity('', 'two') == 0
|
||||
assert strings.jaro_winkler_similarity('three', '') == 0
|
||||
assert strings.jaro_winkler_similarity('bananna', '') == 0
|
||||
assert strings.jaro_winkler_similarity('accomodate', 'accommodate') == 0.9818181818181818
|
||||
assert strings.jaro_winkler_similarity('accomodate', 'accompanist') == 0.8672727272727273
|
||||
assert strings.jaro_winkler_similarity('untill', 'huntsville') == 0.8666666666666667
|
||||
assert strings.jaro_winkler_similarity('wich', 'wichita') == 0.9142857142857143
|
||||
assert strings.jaro_winkler_similarity('a', 'a') == 1
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
module strings
|
||||
|
||||
// strings.repeat - fill a string with `n` repetitions of the character `c`
|
||||
@[direct_array_access]
|
||||
pub fn repeat(c u8, n int) string {
|
||||
if n <= 0 {
|
||||
return ''
|
||||
}
|
||||
mut bytes := unsafe { malloc_noscan(n + 1) }
|
||||
unsafe {
|
||||
C.memset(bytes, c, n)
|
||||
bytes[n] = 0
|
||||
}
|
||||
return unsafe { bytes.vstring_with_len(n) }
|
||||
}
|
||||
|
||||
// strings.repeat_string - gives you `n` repetitions of the substring `s`
|
||||
// Note: strings.repeat, that repeats a single byte, is between 2x
|
||||
// and 24x faster than strings.repeat_string called for a 1 char string.
|
||||
@[direct_array_access]
|
||||
pub fn repeat_string(s string, n int) string {
|
||||
if n <= 0 || s.len == 0 {
|
||||
return ''
|
||||
}
|
||||
slen := s.len
|
||||
blen := slen * n
|
||||
mut bytes := unsafe { malloc_noscan(blen + 1) }
|
||||
for bi in 0 .. n {
|
||||
bislen := bi * slen
|
||||
for si in 0 .. slen {
|
||||
unsafe {
|
||||
bytes[bislen + si] = s[si]
|
||||
}
|
||||
}
|
||||
}
|
||||
unsafe {
|
||||
bytes[blen] = 0
|
||||
}
|
||||
return unsafe { bytes.vstring_with_len(blen) }
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
module strings
|
||||
|
||||
pub fn repeat(c u8, n int) string {
|
||||
if n <= 0 {
|
||||
return ''
|
||||
}
|
||||
arr := [c].repeat(n)
|
||||
return arr.bytestr()
|
||||
}
|
||||
|
||||
pub fn repeat_string(s string, n int) string {
|
||||
res := ''
|
||||
#res.str = s.str.repeat(n.valueOf())
|
||||
|
||||
return res
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
module strings
|
||||
|
||||
// import rand
|
||||
// random returns a random string with `n` characters
|
||||
/*
|
||||
pub fn random(n int) string {
|
||||
buf := vmalloc(n)
|
||||
for i in 0..n {
|
||||
buf[i] = rand.next()
|
||||
}
|
||||
return tos(buf)
|
||||
}
|
||||
*/
|
||||
|
||||
// find_between_pair_byte returns the string found between the pair of marks defined
|
||||
// by `start` and `end`.
|
||||
// As opposed to the `find_between`, `all_after*`, `all_before*` methods defined on the
|
||||
// `string` type, this function can extract content between *nested* marks in `input`.
|
||||
// If `start` and `end` marks are nested in `input`, the characters
|
||||
// between the *outermost* mark pair is returned. It is expected that `start` and `end`
|
||||
// marks are *balanced*, meaning that the amount of `start` marks equal the
|
||||
// amount of `end` marks in the `input`. An empty string is returned otherwise.
|
||||
// Using two identical marks as `start` and `end` results in undefined output behavior.
|
||||
// find_between_pair_byte is the fastest in the find_between_pair_* family of functions.
|
||||
// Example: assert strings.find_between_pair_u8('(V) (NOT V)',`(`,`)`) == 'V'
|
||||
// Example: assert strings.find_between_pair_u8('s {X{Y}} s',`{`,`}`) == 'X{Y}'
|
||||
pub fn find_between_pair_u8(input string, start u8, end u8) string {
|
||||
mut marks := 0
|
||||
mut start_index := -1
|
||||
for i, b in input {
|
||||
if b == start {
|
||||
if start_index == -1 {
|
||||
start_index = i + 1
|
||||
}
|
||||
marks++
|
||||
continue
|
||||
}
|
||||
if start_index > 0 {
|
||||
if b == end {
|
||||
marks--
|
||||
if marks == 0 {
|
||||
return input[start_index..i]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
// find_between_pair_rune returns the string found between the pair of marks defined by `start` and `end`.
|
||||
// As opposed to the `find_between`, `all_after*`, `all_before*` methods defined on the
|
||||
// `string` type, this function can extract content between *nested* marks in `input`.
|
||||
// If `start` and `end` marks are nested in `input`, the characters
|
||||
// between the *outermost* mark pair is returned. It is expected that `start` and `end`
|
||||
// marks are *balanced*, meaning that the amount of `start` marks equal the
|
||||
// amount of `end` marks in the `input`. An empty string is returned otherwise.
|
||||
// Using two identical marks as `start` and `end` results in undefined output behavior.
|
||||
// find_between_pair_rune is inbetween the fastest and slowest in the find_between_pair_* family of functions.
|
||||
// Example: assert strings.find_between_pair_rune('(V) (NOT V)',`(`,`)`) == 'V'
|
||||
// Example: assert strings.find_between_pair_rune('s {X{Y}} s',`{`,`}`) == 'X{Y}'
|
||||
pub fn find_between_pair_rune(input string, start rune, end rune) string {
|
||||
mut marks := 0
|
||||
mut start_index := -1
|
||||
runes := input.runes()
|
||||
for i, r in runes {
|
||||
if r == start {
|
||||
if start_index == -1 {
|
||||
start_index = i + 1
|
||||
}
|
||||
marks++
|
||||
continue
|
||||
}
|
||||
if start_index > 0 {
|
||||
if r == end {
|
||||
marks--
|
||||
if marks == 0 {
|
||||
return runes[start_index..i].string()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
// find_between_pair_string returns the string found between the pair of marks defined by `start` and `end`.
|
||||
// As opposed to the `find_between`, `all_after*`, `all_before*` methods defined on the
|
||||
// `string` type, this function can extract content between *nested* marks in `input`.
|
||||
// If `start` and `end` marks are nested in `input`, the characters
|
||||
// between the *outermost* mark pair is returned. It is expected that `start` and `end`
|
||||
// marks are *balanced*, meaning that the amount of `start` marks equal the
|
||||
// amount of `end` marks in the `input`. An empty string is returned otherwise.
|
||||
// Using two identical marks as `start` and `end` results in undefined output behavior.
|
||||
// find_between_pair_string is the slowest in the find_between_pair_* function family.
|
||||
// Example: assert strings.find_between_pair_string('/*V*/ /*NOT V*/','/*','*/') == 'V'
|
||||
// Example: assert strings.find_between_pair_string('s {{X{{Y}}}} s','{{','}}') == 'X{{Y}}'
|
||||
pub fn find_between_pair_string(input string, start string, end string) string {
|
||||
mut start_index := -1
|
||||
mut marks := 0
|
||||
start_runes := start.runes()
|
||||
end_runes := end.runes()
|
||||
runes := input.runes()
|
||||
mut i := 0
|
||||
for ; i < runes.len; i++ {
|
||||
start_slice := runes#[i..i + start_runes.len]
|
||||
if start_slice == start_runes {
|
||||
i = i + start_runes.len - 1
|
||||
if start_index < 0 {
|
||||
start_index = i + 1
|
||||
}
|
||||
marks++
|
||||
continue
|
||||
}
|
||||
if start_index > 0 {
|
||||
end_slice := runes#[i..i + end_runes.len]
|
||||
if end_slice == end_runes {
|
||||
marks--
|
||||
if marks == 0 {
|
||||
return runes[start_index..i].string()
|
||||
}
|
||||
i = i + end_runes.len - 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
// split_capital returns an array containing the contents of `s` split by capital letters.
|
||||
// Example: assert strings.split_capital('XYZ') == ['X', 'Y', 'Z']
|
||||
// Example: assert strings.split_capital('XYStar') == ['X', 'Y', 'Star']
|
||||
pub fn split_capital(s string) []string {
|
||||
mut res := []string{}
|
||||
mut word_start := 0
|
||||
for idx, c in s {
|
||||
if c.is_capital() {
|
||||
if word_start != idx {
|
||||
res << s#[word_start..idx]
|
||||
}
|
||||
word_start = idx
|
||||
continue
|
||||
}
|
||||
}
|
||||
if word_start != s.len {
|
||||
res << s#[word_start..]
|
||||
}
|
||||
return res
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// vtest build: present_node?
|
||||
import strings
|
||||
|
||||
fn test_repeat() {
|
||||
assert strings.repeat(`x`, 10) == 'xxxxxxxxxx'
|
||||
assert strings.repeat(`a`, 1) == 'a'
|
||||
assert strings.repeat(`a`, 0) == ''
|
||||
}
|
||||
|
||||
fn test_repeat_string() {
|
||||
assert strings.repeat_string('abc', 3) == 'abcabcabc'
|
||||
assert strings.repeat_string('abc', 1) == 'abc'
|
||||
assert strings.repeat_string('abc', 0) == ''
|
||||
assert strings.repeat_string('', 200) == ''
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import strings
|
||||
|
||||
fn test_repeat() {
|
||||
assert strings.repeat(`x`, 10) == 'xxxxxxxxxx'
|
||||
assert strings.repeat(`a`, 1) == 'a'
|
||||
assert strings.repeat(`a`, 0) == ''
|
||||
}
|
||||
|
||||
fn test_repeat_string() {
|
||||
assert strings.repeat_string('abc', 3) == 'abcabcabc'
|
||||
assert strings.repeat_string('abc', 1) == 'abc'
|
||||
assert strings.repeat_string('abc', 0) == ''
|
||||
assert strings.repeat_string('', 200) == ''
|
||||
}
|
||||
|
||||
const test_rune_and_byte = [
|
||||
'xxx[ok1]xxx',
|
||||
'xxx[[ok2]okok]',
|
||||
'xxx[ok3[[[ok]okok]]]',
|
||||
'yyy[ok4]',
|
||||
'[]',
|
||||
']',
|
||||
'[',
|
||||
'yyy[ok5][]zzz',
|
||||
'yyy[xxx',
|
||||
'xxx[xxx
|
||||
xxx]',
|
||||
]
|
||||
|
||||
const test_strings = [
|
||||
'xxx/*ok1*/xxx',
|
||||
'xxx/*/*ok2*/okok*/',
|
||||
'xxx/*ok3/*/*/*ok*/okok*/*/*/',
|
||||
'yyy/*ok4*/',
|
||||
'/**/',
|
||||
'*/',
|
||||
'/*',
|
||||
'yyy/*ok5*//**/zzz',
|
||||
'yyy/*xxx',
|
||||
'xxx/*xxx
|
||||
xxx*/xxx',
|
||||
]
|
||||
|
||||
const expected_rune_and_byte_outputs = [
|
||||
'ok1',
|
||||
'[ok2]okok',
|
||||
'ok3[[[ok]okok]]',
|
||||
'ok4',
|
||||
'',
|
||||
'',
|
||||
'',
|
||||
'ok5',
|
||||
'',
|
||||
'xxx
|
||||
xxx',
|
||||
]
|
||||
|
||||
const expected_string_outputs = [
|
||||
'ok1',
|
||||
'/*ok2*/okok',
|
||||
'ok3/*/*/*ok*/okok*/*/',
|
||||
'ok4',
|
||||
'',
|
||||
'',
|
||||
'',
|
||||
'ok5',
|
||||
'',
|
||||
'xxx
|
||||
xxx',
|
||||
]
|
||||
|
||||
fn test_find_between_pair_family() {
|
||||
assert strings.find_between_pair_rune('xx♡ok❦yy', `♡`, `❦`) == 'ok'
|
||||
assert strings.find_between_pair_u8('xx{ok}yy', `{`, `}`) == 'ok'
|
||||
assert strings.find_between_pair_string('xx/*ok*/yy', '/*', '*/') == 'ok'
|
||||
assert strings.find_between_pair_u8('xx{ok}yy', `{`, `}`) == 'ok'
|
||||
assert strings.find_between_pair_string('xxxxokyyyy', 'xxx', 'yyy') == 'xok'
|
||||
|
||||
for i, tstr in test_rune_and_byte {
|
||||
e1 := strings.find_between_pair_rune(tstr, `[`, `]`)
|
||||
e2 := expected_rune_and_byte_outputs[i]
|
||||
assert '${e1}' == '${e2}'
|
||||
}
|
||||
|
||||
for i, tstr in test_rune_and_byte {
|
||||
e1 := strings.find_between_pair_u8(tstr, `[`, `]`)
|
||||
e2 := expected_rune_and_byte_outputs[i]
|
||||
assert '${e1}' == '${e2}'
|
||||
}
|
||||
|
||||
for i, tstr in test_strings {
|
||||
e1 := strings.find_between_pair_string(tstr, '/*', '*/')
|
||||
e2 := expected_string_outputs[i]
|
||||
assert '${e1}' == '${e2}'
|
||||
}
|
||||
}
|
||||
|
||||
fn test_split_capital() {
|
||||
assert strings.split_capital('') == []
|
||||
assert strings.split_capital('abc') == ['abc']
|
||||
assert strings.split_capital('X') == ['X']
|
||||
assert strings.split_capital('XX') == ['X', 'X']
|
||||
assert strings.split_capital('XYZ') == ['X', 'Y', 'Z']
|
||||
assert strings.split_capital('JohnWilliams') == ['John', 'Williams']
|
||||
assert strings.split_capital('JDStar') == ['J', 'D', 'Star']
|
||||
assert strings.split_capital('cpDumpRotarySpring') == ['cp', 'Dump', 'Rotary', 'Spring']
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
module textscanner
|
||||
|
||||
// TextScanner simplifies writing small scanners/parsers.
|
||||
// It helps by providing safe methods to scan texts character by character, peek for the next characters, go back, etc.
|
||||
pub struct TextScanner {
|
||||
pub:
|
||||
input string
|
||||
ilen int
|
||||
pub mut:
|
||||
pos int // current position; pos is *always* kept in [0,ilen]
|
||||
}
|
||||
|
||||
// new returns a stack allocated instance of TextScanner.
|
||||
pub fn new(input string) TextScanner {
|
||||
return TextScanner{
|
||||
input: input
|
||||
ilen: input.len
|
||||
}
|
||||
}
|
||||
|
||||
// free frees all allocated resources.
|
||||
@[unsafe]
|
||||
pub fn (mut ss TextScanner) free() {
|
||||
unsafe {
|
||||
ss.input.free()
|
||||
}
|
||||
}
|
||||
|
||||
// remaining returns how many characters remain from current position.
|
||||
@[inline]
|
||||
pub fn (ss &TextScanner) remaining() int {
|
||||
return ss.ilen - ss.pos
|
||||
}
|
||||
|
||||
// next returns the next character code from the input text.
|
||||
// next returns `-1` if it can't reach the next character.
|
||||
// next advances the scanner position.
|
||||
@[direct_array_access; inline]
|
||||
pub fn (mut ss TextScanner) next() int {
|
||||
if ss.pos < ss.ilen {
|
||||
opos := ss.pos
|
||||
ss.pos++
|
||||
return ss.input[opos]
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// skip skips one character ahead; `skip()` is slightly faster than `.next()`.
|
||||
// `skip()` does not return a result.
|
||||
@[inline]
|
||||
pub fn (mut ss TextScanner) skip() {
|
||||
if ss.pos < ss.ilen {
|
||||
ss.pos++
|
||||
}
|
||||
}
|
||||
|
||||
// skip_n skips ahead `n` characters, stopping at the end of the input.
|
||||
@[inline]
|
||||
pub fn (mut ss TextScanner) skip_n(n int) {
|
||||
ss.pos += n
|
||||
if ss.pos > ss.ilen {
|
||||
ss.pos = ss.ilen
|
||||
}
|
||||
}
|
||||
|
||||
// peek returns the *next* character code from the input text.
|
||||
// peek returns `-1` if it can't peek the next character.
|
||||
// unlike `next()`, `peek()` does not change the state of the scanner.
|
||||
@[direct_array_access; inline]
|
||||
pub fn (ss &TextScanner) peek() int {
|
||||
if ss.pos < ss.ilen {
|
||||
return ss.input[ss.pos]
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// peek_u8 returns the *next* character code from the input text, as a byte/u8.
|
||||
// unlike `next()`, `peek_u8()` does not change the state of the scanner.
|
||||
// Note: peek_u8 returns `0`, if it can't peek the next character.
|
||||
// Note: use `peek()`, instead of `peek_u8()`, if your input itself can
|
||||
// legitimately contain bytes with value `0`.
|
||||
@[direct_array_access; inline]
|
||||
pub fn (ss &TextScanner) peek_u8() u8 {
|
||||
if ss.pos < ss.ilen {
|
||||
return ss.input[ss.pos]
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// peek_n returns the character code from the input text at position + `n`.
|
||||
// peek_n returns `-1` if it can't peek `n` characters ahead.
|
||||
// ts.peek_n(0) == ts.current() .
|
||||
// ts.peek_n(1) == ts.peek() .
|
||||
@[direct_array_access; inline]
|
||||
pub fn (ss &TextScanner) peek_n(n int) int {
|
||||
if ss.pos + n < ss.ilen {
|
||||
return ss.input[ss.pos + n]
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// peek_n_u8 returns the character code from the input text, at position + `n`,
|
||||
// as a byte/u8.
|
||||
// Note: peek_n_u8 returns `0`, if it can't peek the next character.
|
||||
// Note: use `peek_n()`, instead of `peek_n_u8()`, if your input itself can
|
||||
// legitimately contain bytes with value `0`.
|
||||
@[direct_array_access; inline]
|
||||
pub fn (ss &TextScanner) peek_n_u8(n int) u8 {
|
||||
if ss.pos + n < ss.ilen {
|
||||
return ss.input[ss.pos + n]
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// back goes back one character from the current scanner position.
|
||||
@[inline]
|
||||
pub fn (mut ss TextScanner) back() {
|
||||
if ss.pos > 0 {
|
||||
ss.pos--
|
||||
}
|
||||
}
|
||||
|
||||
// back_n goes back `n` characters from the current scanner position.
|
||||
pub fn (mut ss TextScanner) back_n(n int) {
|
||||
ss.pos -= n
|
||||
if ss.pos < 0 {
|
||||
ss.pos = 0
|
||||
}
|
||||
if ss.pos > ss.ilen {
|
||||
ss.pos = ss.ilen
|
||||
}
|
||||
}
|
||||
|
||||
// peek_back returns the *previous* character code from the input text.
|
||||
// peek_back returns `-1` if it can't peek the previous character.
|
||||
// unlike `back()`, `peek_back()` does not change the state of the scanner.
|
||||
@[direct_array_access; inline]
|
||||
pub fn (ss &TextScanner) peek_back() int {
|
||||
return ss.peek_back_n(1)
|
||||
}
|
||||
|
||||
// peek_back_n returns the character code from the input text at position - `n`.
|
||||
// peek_back_n returns `-1` if it can't peek `n` characters back.
|
||||
// ts.peek_back_n(0) == ts.current()
|
||||
// ts.peek_back_n(1) == ts.peek_back()
|
||||
@[direct_array_access; inline]
|
||||
pub fn (ss &TextScanner) peek_back_n(n int) int {
|
||||
offset := n + 1
|
||||
if ss.pos >= offset {
|
||||
return ss.input[ss.pos - offset]
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// current returns the current character code from the input text.
|
||||
// current returns `-1` at the start of the input text.
|
||||
// Note: after `c := ts.next()`, `ts.current()` will also return `c`.
|
||||
@[direct_array_access; inline]
|
||||
pub fn (mut ss TextScanner) current() int {
|
||||
if ss.pos > 0 {
|
||||
return ss.input[ss.pos - 1]
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// reset resets the internal state of the scanner.
|
||||
// After calling .reset(), .next() will start reading
|
||||
// again from the start of the input text.
|
||||
pub fn (mut ss TextScanner) reset() {
|
||||
ss.pos = 0
|
||||
}
|
||||
|
||||
// goto_end has the same effect as `for ts.next() != -1 {}`.
|
||||
// i.e. after calling .goto_end(), the scanner will be at
|
||||
// the end of the input text. Further .next() calls will
|
||||
// return -1, unless you go back.
|
||||
pub fn (mut ss TextScanner) goto_end() {
|
||||
ss.pos = ss.ilen
|
||||
}
|
||||
|
||||
// skip_whitespace advances the scanner pass any space characters in the input.
|
||||
pub fn (mut ss TextScanner) skip_whitespace() {
|
||||
for ss.ilen - ss.pos > 0 && ss.peek_u8().is_space() {
|
||||
ss.next()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
// vtest build: present_node?
|
||||
import strings.textscanner
|
||||
|
||||
fn test_remaining() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.remaining() == 3
|
||||
s.next()
|
||||
s.next()
|
||||
assert s.remaining() == 1
|
||||
s.next()
|
||||
assert s.remaining() == 0
|
||||
s.next()
|
||||
s.next()
|
||||
assert s.remaining() == 0
|
||||
s.reset()
|
||||
assert s.remaining() == 3
|
||||
}
|
||||
|
||||
fn test_next() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
assert s.next() == -1
|
||||
assert s.next() == -1
|
||||
assert s.next() == -1
|
||||
}
|
||||
|
||||
fn test_skip() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.next() == `a`
|
||||
s.skip()
|
||||
assert s.next() == `c`
|
||||
assert s.next() == -1
|
||||
|
||||
s.reset()
|
||||
assert s.peek() == `a`
|
||||
s.skip()
|
||||
assert s.peek() == `b`
|
||||
s.skip()
|
||||
assert s.peek() == `c`
|
||||
s.skip()
|
||||
assert s.peek() == -1
|
||||
}
|
||||
|
||||
fn test_skip_n() {
|
||||
mut s := textscanner.new('abc')
|
||||
s.skip_n(2)
|
||||
assert s.next() == `c`
|
||||
assert s.next() == -1
|
||||
|
||||
s.reset()
|
||||
assert s.peek() == `a`
|
||||
s.skip_n(2)
|
||||
assert s.peek() == `c`
|
||||
s.skip_n(2)
|
||||
assert s.peek() == -1
|
||||
|
||||
s.reset()
|
||||
assert s.peek() == `a`
|
||||
s.skip_n(3)
|
||||
assert s.peek() == -1
|
||||
|
||||
s.reset()
|
||||
assert s.peek() == `a`
|
||||
s.skip_n(4)
|
||||
assert s.peek() == -1
|
||||
}
|
||||
|
||||
fn test_peek() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.peek() == `a`
|
||||
assert s.peek() == `a`
|
||||
assert s.peek() == `a`
|
||||
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
assert s.next() == -1
|
||||
}
|
||||
|
||||
fn test_peek_n() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.peek_n(0) == `a`
|
||||
assert s.peek_n(1) == `b`
|
||||
assert s.peek_n(2) == `c`
|
||||
assert s.peek_n(3) == -1
|
||||
assert s.peek_n(4) == -1
|
||||
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
assert s.next() == -1
|
||||
}
|
||||
|
||||
fn test_back() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.next() == `a`
|
||||
s.back()
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
s.back()
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
assert s.next() == -1
|
||||
}
|
||||
|
||||
fn test_back_n() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.next() == `a`
|
||||
s.back_n(10)
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
s.back_n(2)
|
||||
assert s.next() == `b`
|
||||
}
|
||||
|
||||
fn test_peek_back() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
// check that calling .peek_back() multiple times
|
||||
// does not change the state:
|
||||
assert s.peek_back() == `a`
|
||||
assert s.peek_back() == `a`
|
||||
assert s.peek_back() == `a`
|
||||
// advance, then peek_back again:
|
||||
assert s.next() == `c`
|
||||
assert s.peek_back() == `b`
|
||||
// peeking before the start:
|
||||
s.reset()
|
||||
assert s.peek_back() == -1
|
||||
// peeking right at the end:
|
||||
s.goto_end()
|
||||
assert s.peek_back() == `b`
|
||||
}
|
||||
|
||||
fn test_peek_back_n() {
|
||||
mut s := textscanner.new('abc')
|
||||
s.goto_end()
|
||||
assert s.peek_back_n(0) == `c`
|
||||
assert s.peek_back_n(1) == `b`
|
||||
assert s.peek_back_n(2) == `a`
|
||||
assert s.peek_back_n(3) == -1
|
||||
assert s.peek_back_n(4) == -1
|
||||
}
|
||||
|
||||
fn test_reset() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.next() == `a`
|
||||
s.next()
|
||||
s.next()
|
||||
assert s.next() == -1
|
||||
s.reset()
|
||||
assert s.next() == `a`
|
||||
}
|
||||
|
||||
fn test_current() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.current() == -1
|
||||
assert s.next() == `a`
|
||||
assert s.current() == `a`
|
||||
assert s.current() == `a`
|
||||
assert s.peek_back() == -1
|
||||
assert s.next() == `b`
|
||||
assert s.current() == `b`
|
||||
assert s.current() == `b`
|
||||
assert s.peek_back() == `a`
|
||||
assert s.next() == `c`
|
||||
assert s.current() == `c`
|
||||
assert s.next() == -1
|
||||
assert s.current() == `c`
|
||||
assert s.next() == -1
|
||||
assert s.current() == `c`
|
||||
s.reset()
|
||||
assert s.current() == -1
|
||||
assert s.next() == `a`
|
||||
assert s.current() == `a`
|
||||
}
|
||||
|
||||
fn test_goto_end() {
|
||||
mut s := textscanner.new('abc')
|
||||
s.goto_end()
|
||||
assert s.current() == `c`
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
import strings.textscanner
|
||||
|
||||
fn test_remaining() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.remaining() == 3
|
||||
s.next()
|
||||
s.next()
|
||||
assert s.remaining() == 1
|
||||
s.next()
|
||||
assert s.remaining() == 0
|
||||
s.next()
|
||||
s.next()
|
||||
assert s.remaining() == 0
|
||||
s.reset()
|
||||
assert s.remaining() == 3
|
||||
}
|
||||
|
||||
fn test_next() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
assert s.next() == -1
|
||||
assert s.next() == -1
|
||||
assert s.next() == -1
|
||||
}
|
||||
|
||||
fn test_skip() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.next() == `a`
|
||||
s.skip()
|
||||
assert s.next() == `c`
|
||||
assert s.next() == -1
|
||||
|
||||
s.reset()
|
||||
assert s.peek() == `a`
|
||||
s.skip()
|
||||
assert s.peek() == `b`
|
||||
s.skip()
|
||||
assert s.peek() == `c`
|
||||
s.skip()
|
||||
assert s.peek() == -1
|
||||
}
|
||||
|
||||
fn test_skip_n() {
|
||||
mut s := textscanner.new('abc')
|
||||
s.skip_n(2)
|
||||
assert s.next() == `c`
|
||||
assert s.next() == -1
|
||||
|
||||
s.reset()
|
||||
assert s.peek() == `a`
|
||||
s.skip_n(2)
|
||||
assert s.peek() == `c`
|
||||
s.skip_n(2)
|
||||
assert s.peek() == -1
|
||||
|
||||
s.reset()
|
||||
assert s.peek() == `a`
|
||||
s.skip_n(3)
|
||||
assert s.peek() == -1
|
||||
|
||||
s.reset()
|
||||
assert s.peek() == `a`
|
||||
s.skip_n(4)
|
||||
assert s.peek() == -1
|
||||
}
|
||||
|
||||
fn test_peek() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.peek() == `a`
|
||||
assert s.peek() == `a`
|
||||
assert s.peek() == `a`
|
||||
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
assert s.next() == -1
|
||||
}
|
||||
|
||||
fn test_peek_n() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.peek_n(0) == `a`
|
||||
assert s.peek_n(1) == `b`
|
||||
assert s.peek_n(2) == `c`
|
||||
assert s.peek_n(3) == -1
|
||||
assert s.peek_n(4) == -1
|
||||
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
assert s.next() == -1
|
||||
}
|
||||
|
||||
fn test_back() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.next() == `a`
|
||||
s.back()
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
s.back()
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
assert s.next() == -1
|
||||
}
|
||||
|
||||
fn test_back_n() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.next() == `a`
|
||||
s.back_n(10)
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
s.back_n(2)
|
||||
assert s.next() == `b`
|
||||
}
|
||||
|
||||
fn test_peek_back() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
// check that calling .peek_back() multiple times
|
||||
// does not change the state:
|
||||
assert s.peek_back() == `a`
|
||||
assert s.peek_back() == `a`
|
||||
assert s.peek_back() == `a`
|
||||
// advance, then peek_back again:
|
||||
assert s.next() == `c`
|
||||
assert s.peek_back() == `b`
|
||||
// peeking before the start:
|
||||
s.reset()
|
||||
assert s.peek_back() == -1
|
||||
// peeking right at the end:
|
||||
s.goto_end()
|
||||
assert s.peek_back() == `b`
|
||||
}
|
||||
|
||||
fn test_peek_back_n() {
|
||||
mut s := textscanner.new('abc')
|
||||
s.goto_end()
|
||||
assert s.peek_back_n(0) == `c`
|
||||
assert s.peek_back_n(1) == `b`
|
||||
assert s.peek_back_n(2) == `a`
|
||||
assert s.peek_back_n(3) == -1
|
||||
assert s.peek_back_n(4) == -1
|
||||
}
|
||||
|
||||
fn test_reset() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.next() == `a`
|
||||
s.next()
|
||||
s.next()
|
||||
assert s.next() == -1
|
||||
s.reset()
|
||||
assert s.next() == `a`
|
||||
}
|
||||
|
||||
fn test_current() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.current() == -1
|
||||
assert s.next() == `a`
|
||||
assert s.current() == `a`
|
||||
assert s.current() == `a`
|
||||
assert s.peek_back() == -1
|
||||
assert s.next() == `b`
|
||||
assert s.current() == `b`
|
||||
assert s.current() == `b`
|
||||
assert s.peek_back() == `a`
|
||||
assert s.next() == `c`
|
||||
assert s.current() == `c`
|
||||
assert s.next() == -1
|
||||
assert s.current() == `c`
|
||||
assert s.next() == -1
|
||||
assert s.current() == `c`
|
||||
s.reset()
|
||||
assert s.current() == -1
|
||||
assert s.next() == `a`
|
||||
assert s.current() == `a`
|
||||
}
|
||||
|
||||
fn test_goto_end() {
|
||||
mut s := textscanner.new('abc')
|
||||
s.goto_end()
|
||||
assert s.current() == `c`
|
||||
}
|
||||
|
||||
fn test_skip_whitespace() {
|
||||
mut s := textscanner.new('abc d \n xyz')
|
||||
assert s.current() == -1
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
s.skip_whitespace()
|
||||
assert s.next() == `d`
|
||||
s.skip_whitespace()
|
||||
assert s.next() == `x`
|
||||
assert s.next() == `y`
|
||||
assert s.next() == `z`
|
||||
}
|
||||
|
||||
fn test_peek_u8() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.peek_u8() == `a`
|
||||
assert !s.peek_u8().is_digit()
|
||||
assert s.next() == `a`
|
||||
assert s.peek_u8() == `b`
|
||||
}
|
||||
|
||||
fn test_peek_n_u8() {
|
||||
mut s := textscanner.new('abc')
|
||||
assert s.peek_n_u8(0) == `a`
|
||||
assert s.peek_n_u8(1) == `b`
|
||||
assert s.peek_n_u8(2) == `c`
|
||||
assert s.peek_n_u8(3) == 0
|
||||
assert s.peek_n_u8(4) == 0
|
||||
}
|
||||
Reference in new issue
Block a user