Initial project import

tools/v/vlib/v3/token/position.v

@@ -0,0 +1,166 @@
+module token
+
+pub struct Pos {
+pub:
+	offset int
+	id     int
+}
+
+pub fn (p Pos) str() string {
+	return '{ offset: ${p.offset}, id: ${p.id} }'
+}
+
+pub fn (p Pos) is_valid() bool {
+	return p.id > 0
+}
+
+pub struct Position {
+pub:
+	filename string
+	offset   int
+	line     int
+	column   int
+}
+
+pub fn (p Position) str() string {
+	return '${p.filename}:${p.line}:${p.column}'
+}
+
+pub struct File {
+pub:
+	name string
+	base int
+	size int
+mut:
+	line_offsets []int = [0]
+	id_counter   &int  = unsafe { nil }
+}
+
+pub struct FileSet {
+mut:
+	base       int = 1
+	id_counter int
+	files      []&File
+}
+
+pub fn FileSet.new() &FileSet {
+	return &FileSet{}
+}
+
+pub fn (mut fs FileSet) add_file(filename string, base_ int, size int) &File {
+	mut base := if base_ < 0 { fs.base } else { base_ }
+	if base < fs.base {
+		panic('invalid base ${base} (should be >= ${fs.base}')
+	}
+	file := &File{
+		name:       filename
+		base:       base
+		size:       size
+		id_counter: &fs.id_counter
+	}
+	if size < 0 {
+		panic('invalid size ${size} (should be >= 0)')
+	}
+	base += size + 1
+	if base < 0 {
+		panic('token.Pos offset overflow (> 2G of source code in file set)')
+	}
+	fs.base = base
+	fs.files << file
+	return file
+}
+
+fn search_files(files []&File, x int) int {
+	mut min, mut max := 0, files.len
+	for min < max {
+		mid := (min + max) / 2
+		if files[mid].base <= x {
+			min = mid + 1
+		} else {
+			max = mid
+		}
+	}
+	return min - 1
+}
+
+pub fn (mut fs FileSet) file(pos Pos) &File {
+	i := search_files(fs.files, pos.offset)
+	if i >= 0 {
+		file := fs.files[i]
+		if pos.offset <= file.base + file.size {
+			return file
+		}
+	}
+	dump(fs)
+	panic('cannot find file for pos: ${pos}')
+}
+
+@[inline]
+pub fn (mut f File) add_line(offset int) {
+	f.line_offsets << offset
+}
+
+@[inline]
+pub fn (f &File) line_count() int {
+	return f.line_offsets.len
+}
+
+pub fn (f &File) line_start(line int) int {
+	idx := line - 1
+	if idx < 0 || idx >= f.line_offsets.len {
+		panic('invalid line `${line}` (must be > 0 & < ${f.line_count()})')
+	}
+	return f.line_offsets[idx]
+}
+
+pub fn (f &File) line(pos Pos) int {
+	return f.find_line(pos.offset - f.base)
+}
+
+pub fn (mut f File) pos(offset int) Pos {
+	if offset > f.size {
+		panic('invalid offset')
+	}
+	mut current_id := 0
+	mut next_id := 0
+	unsafe {
+		current_id = *f.id_counter
+	}
+	next_id = current_id + 1
+	unsafe {
+		*f.id_counter = next_id
+	}
+	return Pos{
+		offset: f.base + offset
+		id:     next_id
+	}
+}
+
+pub fn (f &File) position(pos Pos) Position {
+	offset := pos.offset - f.base
+	line, column := f.find_line_and_column(offset)
+	return Position{
+		filename: f.name
+		offset:   offset
+		line:     line
+		column:   column
+	}
+}
+
+pub fn (f &File) find_line_and_column(pos int) (int, int) {
+	line := f.find_line(pos)
+	return line, pos - f.line_offsets[line - 1] + 1
+}
+
+pub fn (f &File) find_line(pos int) int {
+	mut min, mut max := 0, f.line_offsets.len
+	for min < max {
+		mid := (min + max) / 2
+		if f.line_offsets[mid] <= pos {
+			min = mid + 1
+		} else {
+			max = mid
+		}
+	}
+	return min
+}

tools/v/vlib/v3/token/token.v

@@ -0,0 +1,380 @@
+module token
+
+pub enum Token {
+	amp        // &
+	and        // &&
+	and_assign // &=
+	arrow      // <-
+	assign     // =
+	attribute
+	bit_not // ~
+	char    // `A` - rune
+	colon   // :
+	comma   // ,
+	comment
+	dec         // --
+	decl_assign // :=
+	div         // /
+	div_assign  // /=
+	dollar      // $
+	dot         // .
+	dotdot      // ..
+	ellipsis    // ...
+	eof
+	eq   // ==
+	ge   // >=
+	gt   // >
+	hash // #
+	inc  // ++
+	key_as
+	key_asm
+	key_assert
+	key_atomic
+	key_break
+	key_const
+	key_continue
+	key_defer
+	key_dump
+	key_else
+	key_enum
+	key_false
+	key_fn
+	key_for
+	key_global
+	key_go
+	key_goto
+	key_if
+	key_import
+	key_in
+	key_interface
+	key_is
+	key_isreftype
+	key_likely
+	key_lock
+	key_match
+	key_module
+	key_mut
+	key_nil
+	key_none
+	key_offsetof
+	key_or
+	key_pub
+	key_return
+	key_rlock
+	key_select
+	key_shared
+	key_sizeof
+	key_spawn
+	key_static
+	key_struct
+	key_true
+	key_type
+	key_typeof
+	key_union
+	key_unlikely
+	key_unsafe
+	key_volatile
+	lcbr                        // {
+	le                          // <=
+	left_shift                  // <<
+	left_shift_assign           // >>=
+	logical_or                  // ||
+	lpar                        // (
+	lsbr                        // [
+	lt                          // <
+	minus                       // -
+	minus_assign                // -=
+	mod                         // %
+	mod_assign                  // %=
+	mul                         // *
+	mul_assign                  // *=
+	name                        // user
+	ne                          // !=
+	not                         // !
+	not_in                      // !in
+	not_is                      // !is
+	number                      // 123
+	or_assign                   // |=
+	pipe                        // |
+	plus                        // +
+	plus_assign                 // +=
+	question                    // ?
+	rcbr                        // }
+	right_shift                 // >>
+	right_shift_assign          // <<=
+	right_shift_unsigned        // >>>
+	right_shift_unsigned_assign // >>>=
+	rpar                        // )
+	rsbr                        // ]
+	semicolon                   // ;
+	str_dollar
+	string // 'foo'
+	unknown
+	xor        // ^
+	xor_assign // ^=
+}
+
+pub enum BindingPower {
+	lowest
+	logical_or  // ||
+	logical_and // &&
+	compare     // ==, !=, <, <=, >, >=, in, !in, is, !is
+	bit_or      // |
+	bit_xor     // ^
+	shift       // <<, >>, >>>
+	add         // +, -
+	product     // *, /, %, &
+	highest
+}
+
+const token_and_id = 1
+const token_arrow_id = 3
+const token_assign_id = 4
+const token_bit_not_id = 6
+const token_dec_id = 11
+const token_decl_assign_id = 12
+const token_div_id = 13
+const token_div_assign_id = 14
+const token_eq_id = 20
+const token_ge_id = 21
+const token_gt_id = 22
+const token_inc_id = 24
+const token_key_in_id = 44
+const token_key_is_id = 46
+const token_le_id = 74
+const token_left_shift_id = 75
+const token_left_shift_assign_id = 76
+const token_logical_or_id = 77
+const token_lt_id = 80
+const token_minus_id = 81
+const token_minus_assign_id = 82
+const token_mod_id = 83
+const token_mod_assign_id = 84
+const token_mul_id = 85
+const token_mul_assign_id = 86
+const token_ne_id = 88
+const token_not_id = 89
+const token_not_in_id = 90
+const token_not_is_id = 91
+const token_or_assign_id = 93
+const token_pipe_id = 94
+const token_plus_id = 95
+const token_plus_assign_id = 96
+const token_right_shift_id = 99
+const token_right_shift_assign_id = 100
+const token_right_shift_unsigned_id = 101
+const token_right_shift_unsigned_assign_id = 102
+const token_xor_id = 109
+const token_xor_assign_id = 110
+
+@[inline]
+pub fn (t Token) left_binding_power() BindingPower {
+	tv := int(t)
+	if tv == token_logical_or_id {
+		return BindingPower.logical_or
+	}
+	if tv == token_and_id {
+		return BindingPower.logical_and
+	}
+	if tv == token_eq_id || tv == token_ne_id || tv == token_lt_id || tv == token_le_id
+		|| tv == token_gt_id || tv == token_ge_id || tv == token_key_in_id || tv == token_not_in_id
+		|| tv == token_key_is_id || tv == token_not_is_id {
+		return BindingPower.compare
+	}
+	if tv == token_pipe_id {
+		return BindingPower.bit_or
+	}
+	if tv == token_xor_id {
+		return BindingPower.bit_xor
+	}
+	if tv == token_left_shift_id || tv == token_right_shift_id
+		|| tv == token_right_shift_unsigned_id {
+		return BindingPower.shift
+	}
+	if tv == token_plus_id || tv == token_minus_id {
+		return BindingPower.add
+	}
+	if tv == token_mul_id || tv == token_div_id || tv == token_mod_id || tv == 0 {
+		return BindingPower.product
+	}
+	return BindingPower.lowest
+}
+
+@[inline]
+pub fn (t Token) right_binding_power() BindingPower {
+	return unsafe { BindingPower((int(t.left_binding_power()) + 1)) }
+}
+
+@[inline]
+pub fn (t Token) is_keyword() bool {
+	return int(t) >= int(Token.key_as) && int(t) <= int(Token.key_volatile)
+}
+
+@[inline]
+pub fn (t Token) is_prefix() bool {
+	tv := int(t)
+	return tv == token_minus_id || tv == 0 || tv == token_and_id || tv == token_mul_id
+		|| tv == token_not_id || tv == token_bit_not_id || tv == token_arrow_id
+}
+
+@[inline]
+pub fn (t Token) is_infix() bool {
+	tv := int(t)
+	return tv == 0 || tv == token_and_id || tv == token_arrow_id || tv == token_div_id
+		|| tv == token_eq_id || tv == token_ge_id || tv == token_gt_id || tv == token_key_in_id
+		|| tv == token_key_is_id || tv == token_le_id || tv == token_left_shift_id
+		|| tv == token_logical_or_id || tv == token_lt_id || tv == token_minus_id
+		|| tv == token_mod_id || tv == token_mul_id || tv == token_ne_id || tv == token_not_in_id
+		|| tv == token_not_is_id || tv == token_pipe_id || tv == token_plus_id
+		|| tv == token_right_shift_id || tv == token_right_shift_unsigned_id || tv == token_xor_id
+}
+
+@[inline]
+pub fn (t Token) is_postfix() bool {
+	tv := int(t)
+	return tv == token_dec_id || tv == token_inc_id
+}
+
+@[inline]
+pub fn (t Token) is_assignment() bool {
+	tv := int(t)
+	return tv == 2 || tv == token_assign_id || tv == token_decl_assign_id
+		|| tv == token_div_assign_id || tv == token_left_shift_assign_id
+		|| tv == token_minus_assign_id || tv == token_mod_assign_id || tv == token_mul_assign_id
+		|| tv == token_or_assign_id || tv == token_plus_assign_id
+		|| tv == token_right_shift_assign_id || tv == token_right_shift_unsigned_assign_id
+		|| tv == token_xor_assign_id
+}
+
+@[inline]
+pub fn (t Token) is_overloadable() bool {
+	tv := int(t)
+	return tv == token_div_id || tv == token_eq_id || tv == token_ge_id || tv == token_gt_id
+		|| tv == token_le_id || tv == token_lt_id || tv == token_minus_id || tv == token_mod_id
+		|| tv == token_mul_id || tv == token_ne_id || tv == token_pipe_id || tv == token_plus_id
+		|| tv == token_xor_id
+}
+
+@[inline]
+pub fn (t Token) is_comparison() bool {
+	tv := int(t)
+	return tv == token_eq_id || tv == token_ge_id || tv == token_gt_id || tv == token_key_in_id
+		|| tv == token_key_is_id || tv == token_le_id || tv == token_lt_id || tv == token_ne_id
+		|| tv == token_not_in_id || tv == token_not_is_id
+}
+
+pub fn (t Token) str() string {
+	return match t {
+		.amp { '&' }
+		.and { '&&' }
+		.and_assign { '&=' }
+		.arrow { '<-' }
+		.assign { '=' }
+		.attribute { '@[' }
+		.bit_not { '~' }
+		.char { 'char' }
+		.colon { ':' }
+		.comma { ',' }
+		.comment { '// comment' }
+		.dec { '--' }
+		.decl_assign { ':=' }
+		.div { '/' }
+		.div_assign { '/=' }
+		.dollar { '$' }
+		.dot { '.' }
+		.dotdot { '..' }
+		.ellipsis { '...' }
+		.eof { 'eof' }
+		.eq { '==' }
+		.ge { '>=' }
+		.gt { '>' }
+		.hash { '#' }
+		.inc { '++' }
+		.key_as { 'as' }
+		.key_asm { 'asm' }
+		.key_assert { 'assert' }
+		.key_atomic { 'atomic' }
+		.key_break { 'break' }
+		.key_const { 'const' }
+		.key_continue { 'continue' }
+		.key_defer { 'defer' }
+		.key_dump { 'dump' }
+		.key_else { 'else' }
+		.key_enum { 'enum' }
+		.key_false { 'false' }
+		.key_fn { 'fn' }
+		.key_for { 'for' }
+		.key_global { '__global' }
+		.key_go { 'go' }
+		.key_goto { 'goto' }
+		.key_if { 'if' }
+		.key_import { 'import' }
+		.key_in { 'in' }
+		.key_interface { 'interface' }
+		.key_is { 'is' }
+		.key_isreftype { 'isreftype' }
+		.key_likely { '_likely_' }
+		.key_lock { 'lock' }
+		.key_match { 'match' }
+		.key_module { 'module' }
+		.key_mut { 'mut' }
+		.key_nil { 'nil' }
+		.key_none { 'none' }
+		.key_offsetof { '__offsetof' }
+		.key_or { 'or' }
+		.key_pub { 'pub' }
+		.key_return { 'return' }
+		.key_rlock { 'rlock' }
+		.key_select { 'select' }
+		.key_shared { 'shared' }
+		.key_sizeof { 'sizeof' }
+		.key_spawn { 'spawn' }
+		.key_static { 'static' }
+		.key_struct { 'struct' }
+		.key_true { 'true' }
+		.key_type { 'type' }
+		.key_typeof { 'typeof' }
+		.key_union { 'union' }
+		.key_unlikely { '_unlikely_' }
+		.key_unsafe { 'unsafe' }
+		.key_volatile { 'volatile' }
+		.lcbr { '{' }
+		.le { '<=' }
+		.left_shift { '<<' }
+		.left_shift_assign { '<<=' }
+		.logical_or { '||' }
+		.lpar { '(' }
+		.lsbr { '[' }
+		.lt { '<' }
+		.minus { '-' }
+		.minus_assign { '-=' }
+		.mod { '%' }
+		.mod_assign { '%=' }
+		.mul { '*' }
+		.mul_assign { '*=' }
+		.name { 'name' }
+		.ne { '!=' }
+		.not { '!' }
+		.not_in { '!in' }
+		.not_is { '!is' }
+		.number { 'number' }
+		.or_assign { '|=' }
+		.pipe { '|' }
+		.plus { '+' }
+		.plus_assign { '+=' }
+		.question { '?' }
+		.rcbr { '}' }
+		.right_shift { '>>' }
+		.right_shift_assign { '>>=' }
+		.right_shift_unsigned { '>>>' }
+		.right_shift_unsigned_assign { '>>>=' }
+		.rpar { ')' }
+		.rsbr { ']' }
+		.semicolon { ';' }
+		.str_dollar { '\${' }
+		.string { 'string' }
+		.unknown { 'unknown' }
+		.xor { '^' }
+		.xor_assign { '^=' }
+	}
+}

tools/v/vlib/v3/token/util.v

@@ -0,0 +1,183 @@
+module token
+
+// vfmt off
+@[direct_array_access]
+pub fn Token.from_string_tinyv(name string) Token {
+	match name.len {
+		2 {
+			match name[0] {
+				`a` {
+					match name[1] {
+						`s` { return .key_as }
+						else { return .name }
+					}
+				}
+				`f` {
+					match name[1] {
+						`n` { return .key_fn }
+						else { return .name }
+					}
+				}
+				`g` {
+					match name[1] {
+						`o` { return .key_go }
+						else { return .name }
+					}
+				}
+				`i` {
+					match name[1] {
+						`f` { return .key_if }
+						`n` { return .key_in }
+						`s` { return .key_is }
+						else { return .name }
+					}
+				}
+				`o` {
+					match name[1] {
+						`r` { return .key_or }
+						else { return .name }
+					}
+				}
+				else { return .name }
+			}
+			return .name
+		}
+		3 {
+			match name[0] {
+				`a` {
+					if name[1] == `s` && name[2] == `m` { return .key_asm } else { return .name }
+				}
+				`f` {
+					if name[1] == `o` && name[2] == `r` { return .key_for } else { return .name }
+				}
+				`m` {
+					if name[1] == `u` && name[2] == `t` { return .key_mut } else { return .name }
+				}
+				`n` {
+					if name[1] == `i` && name[2] == `l` { return .key_nil } else { return .name }
+				}
+				`p` {
+					if name[1] == `u` && name[2] == `b` { return .key_pub } else { return .name }
+				}
+				else { return .name }
+			}
+			return .name
+		}
+		4 {
+			match name[0] {
+				`d` {
+					if name[1] == `u` && name[2] == `m` && name[3] == `p` { return .key_dump } else { return .name }
+				}
+				`e` {
+					match name[1] {
+						`l` { if name[2] == `s` && name[3] == `e` { return .key_else } else { return .name } }
+						`n` { if name[2] == `u` && name[3] == `m` { return .key_enum } else { return .name } }
+						else { return .name }
+					}
+				}
+				`g` {
+					if name[1] == `o` && name[2] == `t` && name[3] == `o` { return .key_goto } else { return .name }
+				}
+				`l` {
+					if name[1] == `o` && name[2] == `c` && name[3] == `k` { return .key_lock } else { return .name }
+				}
+				`n` {
+					if name[1] == `o` && name[2] == `n` && name[3] == `e` { return .key_none } else { return .name }
+				}
+				`t` {
+					match name[1] {
+						`r` { if name[2] == `u` && name[3] == `e` { return .key_true } else { return .name } }
+						`y` { if name[2] == `p` && name[3] == `e` { return .key_type } else { return .name } }
+						else { return .name }
+					}
+				}
+				else { return .name }
+			}
+		}
+		5 {
+			match name[0] {
+				`b` { if name[1] == `r` && name[2] == `e` && name[3] == `a` && name[4] == `k` { return .key_break } else { return .name } }
+				`c` { if name[1] == `o` && name[2] == `n` && name[3] == `s` && name[4] == `t` { return .key_const } else { return .name } }
+				`d` { if name[1] == `e` && name[2] == `f` && name[3] == `e` && name[4] == `r` { return .key_defer } else { return .name } }
+				`f` { if name[1] == `a` && name[2] == `l` && name[3] == `s` && name[4] == `e` { return .key_false } else { return .name } }
+				`m` { if name[1] == `a` && name[2] == `t` && name[3] == `c` && name[4] == `h` { return .key_match } else { return .name } }
+				`r` { if name[1] == `l` && name[2] == `o` && name[3] == `c` && name[4] == `k` { return .key_rlock } else { return .name } }
+				`s` { if name[1] == `p` && name[2] == `a` && name[3] == `w` && name[4] == `n` { return .key_spawn } else { return .name } }
+				`u` { if name[1] == `n` && name[2] == `i` && name[3] == `o` && name[4] == `n` { return .key_union } else { return .name } }
+				else { return .name }
+			}
+		}
+		6 {
+			match name[0] {
+				`a` {
+					match name[1] {
+						`s` { if name[2] == `s` && name[3] == `e` && name[4] == `r` && name[5] == `t` { return .key_assert } else { return .name } }
+						`t` { if name[2] == `o` && name[3] == `m` && name[4] == `i` && name[5] == `c` { return .key_atomic } else { return .name } }
+						else { return .name }
+					}
+				}
+				`i` { if name[1] == `m` && name[2] == `p` && name[3] == `o` && name[4] == `r` && name[5] == `t` { return .key_import } else { return .name } }
+				`m` { if name[1] == `o` && name[2] == `d` && name[3] == `u` && name[4] == `l` && name[5] == `e` { return .key_module } else { return .name } }
+				`r` { if name[1] == `e` && name[2] == `t` && name[3] == `u` && name[4] == `r` && name[5] == `n` { return .key_return } else { return .name } }
+				`s` {
+					match name[1] {
+						`e` { if name[2] == `l` && name[3] == `e` && name[4] == `c` && name[5] == `t` { return .key_select } else { return .name } }
+						`h` { if name[2] == `a` && name[3] == `r` && name[4] == `e` && name[5] == `d` { return .key_shared } else { return .name } }
+						`i` { if name[2] == `z` && name[3] == `e` && name[4] == `o` && name[5] == `f` { return .key_sizeof } else { return .name } }
+						`t` {
+							match name[2] {
+								`a` { if name[3] == `t` && name[4] == `i` && name[5] == `c` { return .key_static } else { return .name } }
+								`r` { if name[3] == `u` && name[4] == `c` && name[5] == `t` { return .key_struct } else { return .name } }
+								else { return .name }
+							}
+						}
+						else { return .name }
+					}
+				}
+				`t` { if name[1] == `y` && name[2] == `p` && name[3] == `e` && name[4] == `o` && name[5] == `f` { return .key_typeof } else { return .name } }
+				`u` { if name[1] == `n` && name[2] == `s` && name[3] == `a` && name[4] == `f` && name[5] == `e` { return .key_unsafe } else { return .name } }
+				else { return .name }
+			}
+		}
+		8 {
+			match name[0] {
+				`_` {
+					match name[1] {
+						`_` { if name[2] == `g` && name[3] == `l` && name[4] == `o` && name[5] == `b` && name[6] == `a` && name[7] == `l` { return .key_global } else { return .name } }
+						`l` { if name[2] == `i` && name[3] == `k` && name[4] == `e` && name[5] == `l` && name[6] == `y` && name[7] == `_` { return .key_likely } else { return .name } }
+						else { return .name }
+					}
+				}
+				`c` { if name[1] == `o` && name[2] == `n` && name[3] == `t` && name[4] == `i` && name[5] == `n` && name[6] == `u` && name[7] == `e` { return .key_continue } else { return .name } }
+				`v` { if name[1] == `o` && name[2] == `l` && name[3] == `a` && name[4] == `t` && name[5] == `i` && name[6] == `l` && name[7] == `e` { return .key_volatile } else { return .name } }
+				else { return .name }
+			}
+		}
+		9 {
+			match name[0] {
+				`i` {
+					match name[1] {
+						`n` { if name[2] == `t` && name[3] == `e` && name[4] == `r` && name[5] == `f` && name[6] == `a` && name[7] == `c` && name[8] == `e` { return .key_interface } else { return .name } }
+						`s` { if name[2] == `r` && name[3] == `e` && name[4] == `f` && name[5] == `t` && name[6] == `y` && name[7] == `p` && name[8] == `e` { return .key_isreftype } else { return .name } }
+						else { return .name }
+					}
+				}
+				else { return .name }
+			}
+		}
+		10 {
+			match name[0] {
+				`_` {
+					match name[1] {
+						`_` { if name[2] == `o` && name[3] == `f` && name[4] == `f` && name[5] == `s` && name[6] == `e` && name[7] == `t` && name[8] == `o` && name[9] == `f` { return .key_offsetof } else { return .name } }
+						`u` { if name[2] == `n` && name[3] == `l` && name[4] == `i` && name[5] == `k` && name[6] == `e` && name[7] == `l` && name[8] == `y` && name[9] == `_` { return .key_unlikely } else { return .name } }
+						else { return .name }
+					}
+				}
+				else { return .name }
+			}
+		}
+		else { return .name }
+	}
+}
+// vfmt on