Initial project import

tools/v/vlib/encoding/utf8/east_asian/east_asian_width_test.v

@@ -0,0 +1,23 @@
+module east_asian
+
+fn test_width() {
+	assert east_asian_width_property_at('A', 0) == .narrow
+	assert east_asian_width_property_at('Ａ', 0) == .full
+	assert east_asian_width_property_at('ｱ', 0) == .half
+	assert east_asian_width_property_at('ア', 0) == .wide
+	assert east_asian_width_property_at('☆', 0) == .ambiguous
+	assert east_asian_width_property_at('ج', 0) == .neutral
+	assert display_width('abc', 1) == 3
+	assert display_width('ひらがな', 1) == 8
+	assert display_width('カタカナ', 1) == 8
+	assert display_width('ｶﾀｶﾅ', 1) == 4
+	assert display_width('한글', 1) == 4
+	assert display_width('한자', 1) == 4
+	assert display_width('漢字', 1) == 4
+	assert display_width('简体字', 1) == 6
+	assert display_width('繁體字', 1) == 6
+	assert display_width('अरबी लिपि', 1) == 9
+	assert display_width('☆', 1) == 1
+	assert display_width('☆', 2) == 2
+	assert display_width('🐈👽📛', 1) == 6
+}

tools/v/vlib/encoding/utf8/utf8.v

@@ -0,0 +1,13 @@
+module utf8
+
+import encoding.utf8.validate as impl
+
+// validate_str reports if str consists of valid UTF-8 runes
+pub fn validate_str(str string) bool {
+	return impl.utf8_string(str)
+}
+
+// validate reports if data consists of valid UTF-8 runes
+pub fn validate(data &u8, len int) bool {
+	return impl.utf8_data(data, len)
+}

tools/v/vlib/encoding/utf8/utf8_is_punct_test.v

@@ -0,0 +1,33 @@
+module main
+
+import encoding.utf8
+
+fn test_true() {
+	assert utf8.is_punct('!', 0)
+	assert utf8.is_rune_punct(`!`)
+	assert utf8.is_punct('.', 0)
+	assert utf8.is_rune_punct(`.`)
+	assert utf8.is_punct(',', 0)
+	assert utf8.is_rune_punct(`,`)
+	assert utf8.is_punct(';', 0)
+	assert utf8.is_rune_punct(`;`)
+	assert utf8.is_punct("'", 0)
+	assert utf8.is_rune_punct(`'`)
+	assert utf8.is_punct(',', 0)
+	assert utf8.is_rune_punct(`,`)
+	assert utf8.is_punct('/', 0)
+	assert utf8.is_rune_punct(`/`)
+	assert utf8.is_punct('*', 0)
+	assert utf8.is_rune_punct(`*`)
+}
+
+fn test_false() {
+	assert !utf8.is_punct('a', 0)
+	assert !utf8.is_rune_punct(`a`)
+	assert !utf8.is_punct('ç', 0)
+	assert !utf8.is_rune_punct(`ç`)
+	assert !utf8.is_punct('á', 0)
+	assert !utf8.is_rune_punct(`á`)
+	assert !utf8.is_punct('-', 0)
+	assert !utf8.is_rune_punct(`-`)
+}

tools/v/vlib/encoding/utf8/utf8_util_test.v

@@ -0,0 +1,137 @@
+import encoding.utf8
+
+fn test_utf8_util() {
+	// string test
+	src := 'ăĂ ôÔ testo 怔' //_\u1E5A\u1E5B<=>\u1F49\u1F41<=>\u0128\u012a\u012c" // len 29 runes, raw 49 bytes
+	src_upper := 'ĂĂ ÔÔ TESTO Æ€”' //_\u1E5A\u1E5A<=>\u1F49\u1F49<=>\u0128\u012A\u012C"
+	src_lower := 'ăă ôô testo 怔' //_\u1E5B\u1E5B<=>\u1F41\u1F41<=>\u0129\u012B\u012D"
+	upper := utf8.to_upper(src)
+	lower := utf8.to_lower(src)
+	assert upper == src_upper
+	assert lower == src_lower
+
+	assert utf8.to_upper('абвёabc12｛') == 'АБВЁABC12｛'
+	assert utf8.to_lower('АБВЁABC12｛') == 'абвёabc12｛'
+
+	// test len function
+	assert utf8.len('') == 0
+	assert utf8.len('pippo') == 5
+	assert utf8.len(src) == 15 // 29
+	assert src.len == 24 // 49
+
+	// western punctuation
+	a := '.abc?abcòàè.'
+	assert utf8.is_punct(a, 0) == true
+	assert utf8.is_punct('b', 0) == false
+	assert utf8.is_rune_punct(0x002E) == true
+	assert utf8.is_punct(a, 4) == true // ?
+	assert utf8.is_punct(a, 14) == true // last .
+	assert utf8.is_punct(a, 12) == false // è
+	println('OK western')
+
+	// global punctuation
+	b := '.ĂĂa. ÔÔ TESTO Æ€'
+	assert utf8.is_global_punct(b, 0) == true
+	assert utf8.is_global_punct('.', 0) == true
+	assert utf8.is_rune_punct(0x002E) == true
+	assert utf8.is_global_punct(b, 6) == true // .
+	assert utf8.is_global_punct(b, 1) == false // a
+
+	// test utility functions
+	assert utf8.get_rune(b, 0) == 0x002E
+	c := 'a©★🚀'
+	assert utf8.get_rune(c, 0) == `a` // 1 byte
+	assert utf8.get_rune(c, 1) == `©` // 2 bytes
+	assert utf8.get_rune(c, 3) == `★` // 3 bytes
+	assert utf8.get_rune(c, 6) == `🚀` // 4 bytes
+}
+
+fn test_raw_indexing() {
+	a := '我是V Lang!'
+
+	// test non ascii characters
+	assert utf8.raw_index(a, 0) == '我'
+	assert utf8.raw_index(a, 1) == '是'
+
+	// test ascii characters
+	assert utf8.raw_index(a, 2) == 'V'
+	assert utf8.raw_index(a, 3) == ' '
+	assert utf8.raw_index(a, 4) == 'L'
+	assert utf8.raw_index(a, 5) == 'a'
+	assert utf8.raw_index(a, 6) == 'n'
+	assert utf8.raw_index(a, 7) == 'g'
+	assert utf8.raw_index(a, 8) == '!'
+
+	// test differnt utf8 byte lengths
+	c := 'a©★🚀'
+	assert utf8.raw_index(c, 0) == 'a' // 1 byte
+	assert utf8.raw_index(c, 1) == '©' // 2 bytes
+	assert utf8.raw_index(c, 2) == '★' // 3 bytes
+	assert utf8.raw_index(c, 3) == '🚀' // 4 bytes
+}
+
+fn test_reversed() {
+	a := '我是V Lang!'
+	b := '你好世界hello world'
+	assert utf8.reverse(a) == '!gnaL V是我'
+	assert utf8.reverse(b) == 'dlrow olleh界世好你'
+}
+
+fn test_is_control() {
+	for ra in `a` .. `z` {
+		assert utf8.is_control(ra) == false
+	}
+
+	for ra in `A` .. `Z` {
+		assert utf8.is_control(ra) == false
+	}
+
+	assert utf8.is_control('\x01'.runes()[0]) == true
+	assert utf8.is_control('\u0100'.runes()[0]) == false
+}
+
+fn test_is_letter() {
+	for ra in `a` .. `z` {
+		assert utf8.is_letter(ra) == true
+	}
+
+	for ra in `A` .. `Z` {
+		assert utf8.is_letter(ra) == true
+	}
+
+	assert utf8.is_letter(`ɀ`) == true
+	assert utf8.is_letter(`ȶ`) == true
+	assert utf8.is_letter(`ȹ`) == true
+}
+
+fn test_is_space() {
+	for ra in `a` .. `z` {
+		assert utf8.is_space(ra) == false
+	}
+
+	for ra in `A` .. `Z` {
+		assert utf8.is_space(ra) == false
+	}
+
+	assert utf8.is_space(`\u202f`) == true
+	assert utf8.is_space(`\u2009`) == true
+	assert utf8.is_space(`\u00A0`) == true
+}
+
+fn test_is_number() {
+	for ra in `a` .. `z` {
+		assert utf8.is_number(ra) == false
+	}
+
+	for ra in `A` .. `Z` {
+		assert utf8.is_number(ra) == false
+	}
+
+	for ra in `0` .. `1` {
+		assert utf8.is_number(ra) == true
+	}
+
+	assert utf8.is_number(`\u2164`) == true
+	assert utf8.is_number(`\u2188`) == true
+	assert utf8.is_number(`\u3029`) == true
+}

tools/v/vlib/encoding/utf8/validate/encoding_utf8_test.v

@@ -0,0 +1,88 @@
+import encoding.utf8.validate
+
+fn test_validate_str() {
+	assert validate.utf8_string('añçá') == true
+	assert validate.utf8_string('\x61\xC3\xB1\xC3\xA7\xC3\xA1') == true
+
+	assert validate.utf8_string('\x01') == true
+	assert validate.utf8_string('\x7e') == true
+	assert validate.utf8_string('\x7f') == true
+	assert validate.utf8_string('\xc2\x80') == true
+	assert validate.utf8_string('\xc2\x81') == true
+	assert validate.utf8_string('\xc2\xbf') == true
+	assert validate.utf8_string('\xc3\x80') == true
+	assert validate.utf8_string('\xc3\x81') == true
+	assert validate.utf8_string('\xc3\x88') == true
+	assert validate.utf8_string('\xc3\x90') == true
+	assert validate.utf8_string('\xc3\xa0') == true
+	assert validate.utf8_string('\xc3\xb0') == true
+	assert validate.utf8_string('\xc3\xb8') == true
+	assert validate.utf8_string('\xc3\xbf') == true
+	assert validate.utf8_string('\xc4\x80') == true
+	assert validate.utf8_string('\xdf\xbf') == true
+	assert validate.utf8_string('\xd0\x80') == true
+	assert validate.utf8_string('\xe0\xa0\x80') == true
+	assert validate.utf8_string('\xe0\xa0\x81') == true
+	assert validate.utf8_string('\xe1\x80\x80') == true
+	assert validate.utf8_string('\xed\x80\x80') == true
+	assert validate.utf8_string('\xed\x9f\xbf') == true
+	assert validate.utf8_string('\xee\x80\x80') == true
+	assert validate.utf8_string('\xef\xbf\xbe') == true
+	assert validate.utf8_string('\xef\xbf\xbf') == true
+	assert validate.utf8_string('\xf0\x90\x80\x80') == true
+	assert validate.utf8_string('\xf0\x90\x80\x81') == true
+	assert validate.utf8_string('\xf1\x80\x80\x80') == true
+	assert validate.utf8_string('\xf4\x8f\xbf\xbe') == true
+	assert validate.utf8_string('\xf4\x8f\xbf\xbf') == true
+	assert validate.utf8_string('\xef\xbf\xbd') == true
+}
+
+fn test_validate_invalid_str() {
+	assert validate.utf8_string('\xC0\xC1') == false
+	assert validate.utf8_string('\xF5\xFF') == false
+	assert validate.utf8_string('\xE0\xEF') == false
+
+	// xx
+	assert validate.utf8_string('\x91\x80\x80\x80') == false
+
+	// s1
+	assert validate.utf8_string('\xC2\x7F\x80\x80') == false
+	assert validate.utf8_string('\xC2\xC0\x80\x80') == false
+	assert validate.utf8_string('\xDF\x7F\x80\x80') == false
+	assert validate.utf8_string('\xDF\xC0\x80\x80') == false
+
+	// s2
+	assert validate.utf8_string('\xE0\x9F\xBF\x80') == false
+	assert validate.utf8_string('\xE0\xA0\x7F\x80') == false
+	assert validate.utf8_string('\xE0\xBF\xC0\x80') == false
+	assert validate.utf8_string('\xE0\xC0\x80\x80') == false
+
+	// s3
+	assert validate.utf8_string('\xE1\x7F\xBF\x80') == false
+	assert validate.utf8_string('\xE1\x80\x7F\x80') == false
+	assert validate.utf8_string('\xE1\xBF\xC0\x80') == false
+	assert validate.utf8_string('\xE1\xC0\x80\x80') == false
+
+	// s4
+	assert validate.utf8_string('\xED\x7F\xBF\x80') == false
+	assert validate.utf8_string('\xED\x80\x7F\x80') == false
+	assert validate.utf8_string('\xED\x9F\xC0\x80') == false
+	assert validate.utf8_string('\xED\xA0\x80\x80') == false
+
+	// s5
+	assert validate.utf8_string('\xF0\x8F\xBF\xBF') == false
+	assert validate.utf8_string('\xF0\x90\x7F\xBF') == false
+	assert validate.utf8_string('\xF0\x90\x80\x7F') == false
+	assert validate.utf8_string('\xF0\xBF\xBF\xC0') == false
+	assert validate.utf8_string('\xF0\xBF\xC0\x80') == false
+	assert validate.utf8_string('\xF0\xC0\x80\x80') == false
+
+	// s6
+	assert validate.utf8_string('\xF1\x7F\xBF\xBF') == false
+	assert validate.utf8_string('\xF1\x80\x7F\xBF') == false
+	assert validate.utf8_string('\xF1\x80\x80\x7F') == false
+	assert validate.utf8_string('\xF1\xBF\xBF\xC0') == false
+	assert validate.utf8_string('\xF1\xBF\xC0\x80') == false
+	assert validate.utf8_string('\xF1\xC0\x80\x80') == false
+	assert validate.utf8_string('\xED\xEF\xBF\x89') == false
+}

tools/v/vlib/encoding/utf8/validate/validate_utf8.v

@@ -0,0 +1,45 @@
+module validate
+
+// http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
+
+// vfmt off
+const utf8d = [
+	u8(0), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+	7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
+	7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 10, 3, 3, 3, 3,
+	3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, 11, 6, 6, 6, 5, 8, 8, 8, 8, 8, 8, 8,
+	8, 8, 8, 8, 0, 12, 24, 36, 60, 96, 84, 12, 12, 12, 48, 72, 12, 12, 12,
+	12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 0, 12, 12, 12, 12, 12, 0, 12,
+	0, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 24, 12, 12, 12, 12, 12,
+	12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 24, 12, 12, 12, 12, 12, 12, 12,
+	24, 12, 12, 12, 12, 12, 12, 12, 12, 12, 36, 12, 36, 12, 12, 12, 36, 12,
+	12, 12, 12, 12, 36, 12, 36, 12, 12, 12, 36, 12, 12, 12, 12, 12, 12, 12,
+	12, 12, 12 ]!
+// vfmt on
+
+// utf8_string returns true, if the given string `s` consists only of valid UTF-8 runes
+pub fn utf8_string(s string) bool {
+	return utf8_data(s.str, s.len)
+}
+
+// utf8_data returns true, if the given `data` block, with length `len` bytes, consists only of valid UTF-8 runes
+@[direct_array_access]
+pub fn utf8_data(data &u8, len int) bool {
+	mut state := 0
+
+	for i := 0; i < len; i++ {
+		b := unsafe { data[i] }
+		state = utf8d[256 + state + utf8d[b]]
+		if state == 12 {
+			return false
+		}
+	}
+	return state == 0
+}