Initial project import

This commit is contained in:
Agila Forge Importer committed 2026-07-13 15:20:32 +00:00
commit 23f2d29e32
11957 files changed
+1754577

No files matched your search

@@ -0,0 +1,225 @@
// Copyright (c) 2020-2024 Joe Conigliaro. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module token
// TODO: finish fileset / file / base pos etc
// Pos encodes a source position with a unique ID for use as a map key.
// offset: file-set global byte offset (for error messages, file lookup)
// id: unique sequential ID (for expr_types map key — avoids collisions)
pub struct Pos {
pub:
offset int
id int
}
// str returns a readable summary of the position key fields.
pub fn (p Pos) str() string {
return '{ offset: ${p.offset}, id: ${p.id} }'
}
pub fn (p Pos) is_valid() bool {
return p.id > 0
}
pub struct Position {
pub:
filename string
offset int
line int
column int
}
pub fn (p Position) str() string {
return '${p.filename}:${p.line}:${p.column}'
}
pub struct File {
pub:
name string
base int
size int
mut:
line_offsets []int = [0] // start of each line
id_counter &int = unsafe { nil } // shared global counter for unique expression IDs
}
pub struct FileSet {
mut:
base int = 1 // reserve 0 for no position
id_counter int // global counter for unique expression IDs across all files
// files shared []&File
files []&File
}
pub fn FileSet.new() &FileSet {
return &FileSet{}
}
// TODO:
pub fn (mut fs FileSet) add_file(filename string, base_ int, size int) &File {
// eprintln('>>> add_file fs: ${voidptr(fs)} | filename: ${filename} | base_: ${base_} | size: ${size}')
mut base := if base_ < 0 { fs.base } else { base_ }
// eprintln('>>> add_file fs: ${voidptr(fs)} | base: ${base:10} | fs.base: ${fs.base} | base_: ${base_:10} | size: ${size:10} | filename: ${filename}')
if base < fs.base {
panic('invalid base ${base} (should be >= ${fs.base}')
}
file := &File{
name: filename
base: base
size: size
id_counter: &fs.id_counter
}
if size < 0 {
panic('invalid size ${size} (should be >= 0)')
}
base += size + 1 // +1 because EOF also has a position
if base < 0 {
panic('token.Pos offset overflow (> 2G of source code in file set)')
}
// add the file to the file set
// fs.base = base
// lock fs.files {
// // TODO: add shared to fs.base (fix compiler errors first)
// fs.files << file
// }
// fs.last = file
fs.base = base
fs.files << file
return file
}
fn search_files(files []&File, x int) int {
// binary search
mut min, mut max := 0, files.len
for min < max {
mid := (min + max) / 2
// println('# min: ${min}, mid: ${mid}, max: ${max}')
if files[mid].base <= x {
min = mid + 1
} else {
max = mid
}
}
return min - 1
// linear search
// for i := files.len-1; i>=0; i-- {
// file := files[i]
// if file.base < x && x <= file.base + file.size {
// // println('found file for pos `${x}` i = ${i}:')
// // dump(file)
// return i
// }
// }
// return -1
}
pub fn (mut fs FileSet) file(pos Pos) &File {
// eprintln('>>>>>>>>> file fs: ${voidptr(fs)} | pos: ${pos}')
// lock fs.files
// last file
// if last_file := fs.files.last() {
// // p_int
// if last_file.base <= int(pos) && int(p) <- f.base+f.size {
// return last_file
// }
// }
// i := search_files(lock fs.files { fs.files }, pos)
i := search_files(fs.files, pos.offset)
if i >= 0 {
file := fs.files[i]
if pos.offset <= file.base + file.size {
// we could store last and retrieve and try above
return file
}
}
dump(fs)
panic('cannot find file for pos: ${pos}')
}
// pub fn new_file(filename string) File {
// return File{
// name: filename
// }
// }
@[inline]
pub fn (mut f File) add_line(offset int) {
f.line_offsets << offset
}
@[inline]
pub fn (f &File) line_count() int {
return f.line_offsets.len
}
pub fn (f &File) line_start(line int) int {
idx := line - 1
if idx < 0 || idx >= f.line_offsets.len {
panic('invalid line `${line}` (must be > 0 & < ${f.line_count()})')
}
return f.line_offsets[idx]
}
pub fn (f &File) line(pos Pos) int {
return f.find_line(pos.offset - f.base)
}
pub fn (mut f File) pos(offset int) Pos {
if offset > f.size {
panic('invalid offset')
}
mut current_id := 0
mut next_id := 0
unsafe {
current_id = *f.id_counter
}
next_id = current_id + 1
unsafe {
*f.id_counter = next_id
}
return Pos{
offset: f.base + offset
id: next_id
}
}
pub fn (f &File) position(pos Pos) Position {
offset := pos.offset - f.base
line, column := f.find_line_and_column(offset)
return Position{
filename: f.name
offset: offset
line: line
column: column
}
}
// return (line, column) when passed pos
pub fn (f &File) find_line_and_column(pos int) (int, int) {
line := f.find_line(pos)
return line, pos - f.line_offsets[line - 1] + 1
}
// return line when passed pos (binary search)
// NOTE: only used for error conditions
// therefore search speed is not an issue
pub fn (f &File) find_line(pos int) int {
mut min, mut max := 0, f.line_offsets.len
for min < max {
mid := (min + max) / 2
// println('# min: ${min}, mid: ${mid}, max: ${max}')
if f.line_offsets[mid] <= pos {
min = mid + 1
} else {
max = mid
}
}
return min
}
+377
View File
@@ -0,0 +1,377 @@
// Copyright (c) 2020-2024 Joe Conigliaro. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module token
pub enum Token {
amp // &
and // &&
and_assign // &=
arrow // <-
assign // =
// at // @
attribute
bit_not // ~
char // `A` - rune
colon // :
comma // ,
comment
dec // --
decl_assign // :=
div // /
div_assign // /=
dollar // $
dot // .
dotdot // ..
ellipsis // ...
eof
eq // ==
ge // >=
gt // >
hash // #
inc // ++
key_as
key_asm
key_assert
key_atomic
key_break
key_const
key_continue
key_defer
key_dump
key_else
key_enum
key_false
key_fn
key_for
key_global
key_go
key_goto
key_if
key_import
key_in
key_interface
key_is
key_isreftype
key_likely
key_lock
key_match
key_module
key_mut
key_nil
key_none
key_offsetof
key_or
key_pub
key_return
key_rlock
key_select
key_shared
key_sizeof
key_spawn
key_static
key_struct
key_true
key_type
key_typeof
key_union
key_unlikely
key_unsafe
key_volatile
lcbr // {
le // <=
left_shift // <<
left_shift_assign // >>=
logical_or // ||
lpar // (
lsbr // [
lt // <
minus // -
minus_assign // -=
mod // %
mod_assign // %=
mul // *
mul_assign // *=
name // user
ne // !=
not // !
not_in // !in
not_is // !is
number // 123
or_assign // |=
pipe // |
plus // +
plus_assign // +=
question // ?
rcbr // }
right_shift // >>
right_shift_assign // <<=
right_shift_unsigned // >>>
right_shift_unsigned_assign // >>>=
rpar // )
rsbr // ]
semicolon // ;
str_dollar
string // 'foo'
unknown
xor // ^
xor_assign // ^=
}
pub enum BindingPower {
lowest
logical_or // ||
logical_and // &&
compare // ==, !=, <, <=, >, >=, in, !in, is, !is
bit_or // |
bit_xor // ^
shift // <<, >>, >>>
add // +, -
product // *, /, %, &
highest
}
@[inline]
pub fn (t Token) left_binding_power() BindingPower {
return match t {
// `||`
.logical_or {
BindingPower.logical_or
}
// `&&`
.and {
BindingPower.logical_and
}
// `==` | `!=` | `<` | `<=` | `>` | `>=` | `in` | `!in` | `is` | `!is`
.eq, .ne, .lt, .le, .gt, .ge, .key_in, .not_in, .key_is, .not_is {
BindingPower.compare
}
// `|`
.pipe {
BindingPower.bit_or
}
// `^`
.xor {
BindingPower.bit_xor
}
// `<<` | `>>` | `>>>`
.left_shift, .right_shift, .right_shift_unsigned {
BindingPower.shift
}
// `+` | `-`
.plus, .minus {
BindingPower.add
}
// `*` | `/` | `%` | `&`
.mul, .div, .mod, .amp {
BindingPower.product
}
else {
BindingPower.lowest
}
}
}
// TODO: double check / fix this. just use what is needed instead of this
@[inline]
pub fn (t Token) right_binding_power() BindingPower {
return unsafe { BindingPower((int(t.left_binding_power()) + 1)) }
}
@[inline]
pub fn (t Token) is_keyword() bool {
return int(t) >= int(Token.key_as) && int(t) <= int(Token.key_volatile)
}
@[inline]
pub fn (t Token) is_prefix() bool {
return t in [.minus, .amp, .mul, .not, .bit_not, .arrow]
}
@[inline]
pub fn (t Token) is_infix() bool {
return t in [
.amp,
.and,
.arrow,
.div,
// .dot,
.eq,
.ge,
.gt,
// .key_as,
.key_in,
.key_is,
.le,
.left_shift,
.logical_or,
.lt,
.minus,
.mod,
.mul,
.ne,
.not_in,
.not_is,
.pipe,
.plus,
.right_shift,
.right_shift_unsigned,
.xor,
]
}
@[inline]
pub fn (t Token) is_postfix() bool {
return t in [.dec, .inc]
// return t in [.dec, .inc, .not, .question]
}
@[inline]
pub fn (t Token) is_assignment() bool {
return t in [
.and_assign,
.assign,
.decl_assign,
.div_assign,
.left_shift_assign,
.minus_assign,
.mod_assign,
.mul_assign,
.or_assign,
.plus_assign,
.right_shift_assign,
.right_shift_unsigned_assign,
.xor_assign,
]
}
@[inline]
pub fn (t Token) is_overloadable() bool {
// `/` | `==` | `>=` | `>` | `<=` | `<` | `-` | `%` | `*` | `!=` | `|` | `+` | `^`
return t in [.div, .eq, .ge, .gt, .le, .lt, .minus, .mod, .mul, .ne, .pipe, .plus, .xor]
}
@[inline]
pub fn (t Token) is_comparison() bool {
// `==` | `>=` | `>` | `in` | `is` | `<=` | `<` | `!=` | `!in` | `!is`
return t in [.eq, .ge, .gt, .key_in, .key_is, .le, .lt, .ne, .not_in, .not_is]
}
// NOTE: probably switch back to map again later.
// for dev this is easier to see if any tokens are missing.
pub fn (t Token) str() string {
return match t {
.amp { '&' }
.and { '&&' }
.and_assign { '&=' }
.arrow { '<-' }
.assign { '=' }
// .at { '@' }
.attribute { '@[' }
.bit_not { '~' }
.char { 'char' }
.colon { ':' }
.comma { ',' }
.comment { '// comment' }
.dec { '--' }
.decl_assign { ':=' }
.div { '/' }
.div_assign { '/=' }
.dollar { '$' }
.dot { '.' }
.dotdot { '..' }
.ellipsis { '...' }
.eof { 'eof' }
.eq { '==' }
.ge { '>=' }
.gt { '>' }
.hash { '#' }
.inc { '++' }
.key_as { 'as' }
.key_asm { 'asm' }
.key_assert { 'assert' }
.key_atomic { 'atomic' }
.key_break { 'break' }
.key_const { 'const' }
.key_continue { 'continue' }
.key_defer { 'defer' }
.key_dump { 'dump' }
.key_else { 'else' }
.key_enum { 'enum' }
.key_false { 'false' }
.key_fn { 'fn' }
.key_for { 'for' }
.key_global { '__global' }
.key_go { 'go' }
.key_goto { 'goto' }
.key_if { 'if' }
.key_import { 'import' }
.key_in { 'in' }
.key_interface { 'interface' }
.key_is { 'is' }
.key_isreftype { 'isreftype' }
.key_likely { '_likely_' }
.key_lock { 'lock' }
.key_match { 'match' }
.key_module { 'module' }
.key_mut { 'mut' }
.key_nil { 'nil' }
.key_none { 'none' }
.key_offsetof { '__offsetof' }
.key_or { 'or' }
.key_pub { 'pub' }
.key_return { 'return' }
.key_rlock { 'rlock' }
.key_select { 'select' }
.key_shared { 'shared' }
.key_sizeof { 'sizeof' }
.key_spawn { 'spawn' }
.key_static { 'static' }
.key_struct { 'struct' }
.key_true { 'true' }
.key_type { 'type' }
.key_typeof { 'typeof' }
.key_union { 'union' }
.key_unlikely { '_unlikely_' }
.key_unsafe { 'unsafe' }
.key_volatile { 'volatile' }
.lcbr { '{' }
.le { '<=' }
.left_shift { '<<' }
.left_shift_assign { '<<=' }
.logical_or { '||' }
.lpar { '(' }
.lsbr { '[' }
.lt { '<' }
.minus { '-' }
.minus_assign { '-=' }
.mod { '%' }
.mod_assign { '%=' }
.mul { '*' }
.mul_assign { '*=' }
.name { 'name' }
.ne { '!=' }
.not { '!' }
.not_in { '!in' }
.not_is { '!is' }
.number { 'number' }
.or_assign { '|=' }
.pipe { '|' }
.plus { '+' }
.plus_assign { '+=' }
.question { '?' }
.rcbr { '}' }
.right_shift { '>>' }
.right_shift_assign { '>>=' }
.right_shift_unsigned { '>>>' }
.right_shift_unsigned_assign { '>>>=' }
.rpar { ')' }
.rsbr { ']' }
.semicolon { ';' }
.str_dollar { '\${' }
.string { 'string' }
.unknown { 'unknown' }
.xor { '^' }
.xor_assign { '^=' }
}
}
+463
View File
@@ -0,0 +1,463 @@
module token
// NOTE: add keyword tokens here
// TODO: allow overriding this method in main v compiler
// that is why this method was renamed from `from_string`
// vfmt off
@[direct_array_access]
pub fn Token.from_string_tinyv(name string) Token {
match name.len {
2 {
match name[0] {
`a` {
match name[1] {
`s` { return .key_as }
else { return .name }
}
}
`f` {
match name[1] {
`n` { return .key_fn }
else { return .name }
}
}
`g` {
match name[1] {
`o` { return .key_go }
else { return .name }
}
}
`i` {
match name[1] {
`f` { return .key_if }
`n` { return .key_in }
`s` { return .key_is }
else { return .name }
}
}
`o` {
match name[1] {
`r` { return .key_or }
else { return .name }
}
}
else {
return .name
}
}
return .name
}
3 {
match name[0] {
`a` {
if name[1] == `s` && name[2] == `m` {
return .key_asm
} else {
return .name
}
}
`f` {
if name[1] == `o` && name[2] == `r` {
return .key_for
} else {
return .name
}
}
`m` {
if name[1] == `u` && name[2] == `t` {
return .key_mut
} else {
return .name
}
}
`n` {
if name[1] == `i` && name[2] == `l` {
return .key_nil
} else {
return .name
}
}
`p` {
if name[1] == `u` && name[2] == `b` {
return .key_pub
} else {
return .name
}
}
else {
return .name
}
}
return .name
}
4 {
match name[0] {
`d` {
if name[1] == `u` && name[2] == `m` && name[3] == `p` {
return .key_dump
} else {
return .name
}
}
`e` {
match name[1] {
`l` {
if name[2] == `s` && name[3] == `e` {
return .key_else
} else {
return .name
}
}
`n` {
if name[2] == `u` && name[3] == `m` {
return .key_enum
} else {
return .name
}
}
else {
return .name
}
}
}
`g` {
if name[1] == `o` && name[2] == `t` && name[3] == `o` {
return .key_goto
} else {
return .name
}
}
`l` {
if name[1] == `o` && name[2] == `c` && name[3] == `k` {
return .key_lock
} else {
return .name
}
}
`n` {
if name[1] == `o` && name[2] == `n` && name[3] == `e` {
return .key_none
} else {
return .name
}
}
`t` {
match name[1] {
`r` {
if name[2] == `u` && name[3] == `e` {
return .key_true
} else {
return .name
}
}
`y` {
if name[2] == `p` && name[3] == `e` {
return .key_type
} else {
return .name
}
}
else {
return .name
}
}
}
else {
return .name
}
}
}
5 {
match name[0] {
`b` {
if name[1] == `r` && name[2] == `e` && name[3] == `a` && name[4] == `k` {
return .key_break
} else {
return .name
}
}
`c` {
if name[1] == `o` && name[2] == `n` && name[3] == `s` && name[4] == `t` {
return .key_const
} else {
return .name
}
}
`d` {
if name[1] == `e` && name[2] == `f` && name[3] == `e` && name[4] == `r` {
return .key_defer
} else {
return .name
}
}
`f` {
if name[1] == `a` && name[2] == `l` && name[3] == `s` && name[4] == `e` {
return .key_false
} else {
return .name
}
}
`m` {
if name[1] == `a` && name[2] == `t` && name[3] == `c` && name[4] == `h` {
return .key_match
} else {
return .name
}
}
`r` {
if name[1] == `l` && name[2] == `o` && name[3] == `c` && name[4] == `k` {
return .key_rlock
} else {
return .name
}
}
`s` {
if name[1] == `p` && name[2] == `a` && name[3] == `w` && name[4] == `n` {
return .key_spawn
} else {
return .name
}
}
`u` {
if name[1] == `n` && name[2] == `i` && name[3] == `o` && name[4] == `n` {
return .key_union
} else {
return .name
}
}
else {
return .name
}
}
}
6 {
match name[0] {
`a` {
match name[1] {
`s` {
if name[2] == `s` && name[3] == `e` && name[4] == `r` && name[5] == `t` {
return .key_assert
} else {
return .name
}
}
`t` {
if name[2] == `o` && name[3] == `m` && name[4] == `i` && name[5] == `c` {
return .key_atomic
} else {
return .name
}
}
else {
return .name
}
}
}
`i` {
if name[1] == `m` && name[2] == `p` && name[3] == `o` && name[4] == `r`
&& name[5] == `t` {
return .key_import
} else {
return .name
}
}
`m` {
if name[1] == `o` && name[2] == `d` && name[3] == `u` && name[4] == `l`
&& name[5] == `e` {
return .key_module
} else {
return .name
}
}
`r` {
if name[1] == `e` && name[2] == `t` && name[3] == `u` && name[4] == `r`
&& name[5] == `n` {
return .key_return
} else {
return .name
}
}
`s` {
match name[1] {
`e` {
if name[2] == `l` && name[3] == `e` && name[4] == `c` && name[5] == `t` {
return .key_select
} else {
return .name
}
}
`h` {
if name[2] == `a` && name[3] == `r` && name[4] == `e` && name[5] == `d` {
return .key_shared
} else {
return .name
}
}
`i` {
if name[2] == `z` && name[3] == `e` && name[4] == `o` && name[5] == `f` {
return .key_sizeof
} else {
return .name
}
}
`t` {
match name[2] {
`a` {
if name[3] == `t` && name[4] == `i` && name[5] == `c` {
return .key_static
} else {
return .name
}
}
`r` {
if name[3] == `u` && name[4] == `c` && name[5] == `t` {
return .key_struct
} else {
return .name
}
}
else {
return .name
}
}
}
else {
return .name
}
}
}
`t` {
if name[1] == `y` && name[2] == `p` && name[3] == `e` && name[4] == `o`
&& name[5] == `f` {
return .key_typeof
} else {
return .name
}
}
`u` {
if name[1] == `n` && name[2] == `s` && name[3] == `a` && name[4] == `f`
&& name[5] == `e` {
return .key_unsafe
} else {
return .name
}
}
else {
return .name
}
}
}
8 {
match name[0] {
`_` {
match name[1] {
`_` {
if name[2] == `g` && name[3] == `l` && name[4] == `o` && name[5] == `b`
&& name[6] == `a` && name[7] == `l` {
return .key_global
} else {
return .name
}
}
`l` {
if name[2] == `i` && name[3] == `k` && name[4] == `e` && name[5] == `l`
&& name[6] == `y` && name[7] == `_` {
return .key_likely
} else {
return .name
}
}
else {
return .name
}
}
}
`c` {
if name[1] == `o` && name[2] == `n` && name[3] == `t` && name[4] == `i`
&& name[5] == `n` && name[6] == `u` && name[7] == `e` {
return .key_continue
} else {
return .name
}
}
`v` {
if name[1] == `o` && name[2] == `l` && name[3] == `a` && name[4] == `t`
&& name[5] == `i` && name[6] == `l` && name[7] == `e` {
return .key_volatile
} else {
return .name
}
}
else {
return .name
}
}
}
9 {
match name[0] {
`i` {
match name[1] {
`n` {
if name[2] == `t` && name[3] == `e` && name[4] == `r` && name[5] == `f`
&& name[6] == `a` && name[7] == `c` && name[8] == `e` {
return .key_interface
} else {
return .name
}
}
`s` {
if name[2] == `r` && name[3] == `e` && name[4] == `f` && name[5] == `t`
&& name[6] == `y` && name[7] == `p` && name[8] == `e` {
return .key_isreftype
} else {
return .name
}
}
else {
return .name
}
}
}
else {
return .name
}
}
}
10 {
match name[0] {
`_` {
match name[1] {
`_` {
if name[2] == `o` && name[3] == `f` && name[4] == `f` && name[5] == `s`
&& name[6] == `e` && name[7] == `t` && name[8] == `o`
&& name[9] == `f` {
return .key_offsetof
} else {
return .name
}
}
`u` {
if name[2] == `n` && name[3] == `l` && name[4] == `i` && name[5] == `k`
&& name[6] == `e` && name[7] == `l` && name[8] == `y`
&& name[9] == `_` {
return .key_unlikely
} else {
return .name
}
}
else {
return .name
}
}
}
else {
return .name
}
}
}
else {
return .name
}
}
}
// vfmt on