agila_open_tool/tools/v/vlib/v3/scanner/scanner.v

module scanner

import v3.token
import v3.pref

@[flag]
pub enum Mode {
	normal
	scan_comments
	skip_interpolation
}

pub struct Scanner {
	pref               &pref.Preferences
	mode               Mode
	skip_interpolation bool
mut:
	file        &token.File = unsafe { nil }
	insert_semi bool
pub mut:
	src                 string
	offset              int
	pos                 int
	lit                 string
	in_str_incomplete   bool
	in_str_inter        bool
	str_inter_cbr_depth int
	str_quote           u8
}

@[direct_array_access; inline]
fn (s &Scanner) peek_byte(n int) u8 {
	idx := s.offset + n
	if idx < 0 || idx >= s.src.len {
		return 0
	}
	return s.src[idx]
}

pub fn new_scanner(prefs &pref.Preferences, mode Mode) Scanner {
	unsafe {
		return Scanner{
			pref:               prefs
			mode:               mode
			skip_interpolation: mode.has(.skip_interpolation)
		}
	}
}

pub fn (mut s Scanner) init(file &token.File, src string) {
	s.offset = 0
	s.pos = 0
	s.lit = ''
	s.insert_semi = false
	s.in_str_incomplete = false
	s.in_str_inter = false
	s.str_inter_cbr_depth = 0
	s.str_quote = 0
	s.file = unsafe { file }
	s.src = src
}

fn (mut s Scanner) scan_char_literal(quote u8) token.Token {
	s.offset++
	for s.offset < s.src.len {
		c2 := s.src[s.offset]
		if c2 == quote {
			break
		}
		if c2 == `\\` && s.offset + 1 < s.src.len {
			s.offset += 2
			continue
		}
		s.offset++
	}
	mut end := s.offset
	if s.offset < s.src.len && s.src[s.offset] == quote {
		end = s.offset
		s.offset++
	}
	s.lit = s.src[s.pos + 1..end]
	s.insert_semi = true
	return .char
}

pub fn (s &Scanner) current_file() &token.File {
	return unsafe { s.file }
}

@[direct_array_access]
pub fn (mut s Scanner) scan() token.Token {
	if s.in_str_incomplete {
		s.in_str_incomplete = false
		s.pos = s.offset
		s.string_literal(false, s.str_quote)
		s.lit = s.src[s.pos..s.offset]
		return .string
	}
	start:
	s.whitespace()
	if s.offset == s.src.len {
		s.lit = ''
		if s.insert_semi {
			s.insert_semi = false
			return .semicolon
		}
		s.file.add_line(s.offset)
		return .eof
	}
	c := s.src[s.offset]
	s.pos = s.offset
	s.lit = ''
	preserve_insert_semi := s.insert_semi
	s.insert_semi = false
	if c == `\n` {
		s.lit = ''
		return .semicolon
	} else if c == `/` {
		c2 := s.peek_byte(1)
		if c2 == `/` || c2 == `*` {
			if preserve_insert_semi {
				s.insert_semi = true
			}
			s.comment()
			if !s.mode.has(.scan_comments) {
				unsafe {
					goto start
				}
			}
			s.lit = s.src[s.pos..s.offset]
			return .comment
		} else if c2 == `=` {
			s.offset += 2
			return .div_assign
		}
		s.offset++
		return .div
	} else if c >= `0` && c <= `9` {
		s.number()
		s.lit = s.src[s.pos..s.offset]
		s.insert_semi = true
		return .number
	} else if (c >= `a` && c <= `z`) || (c >= `A` && c <= `Z`) || c == `_` || c == `@` {
		s.offset++
		if c == `@` && s.peek_byte(0) == `[` {
			s.offset++
			return .attribute
		}
		for s.offset < s.src.len {
			c3 := s.src[s.offset]
			if c3.is_alnum() || c3 == `_` {
				s.offset++
				continue
			}
			break
		}
		s.lit = s.src[s.pos..s.offset]
		if s.lit == 'c' && s.offset < s.src.len && s.src[s.offset] == `'` {
			s.pos = s.offset
			tok := s.scan_char_literal(`'`)
			s.lit = 'c:${s.lit}'
			return tok
		}
		if s.lit == 'r' && s.offset < s.src.len
			&& (s.src[s.offset] == `'` || s.src[s.offset] == `"`) {
			quote := s.src[s.offset]
			s.offset++
			if !s.in_str_inter {
				s.str_quote = quote
			}
			s.string_literal(true, quote)
			s.lit = s.src[s.pos..s.offset]
			s.insert_semi = true
			return .string
		}
		tok := token.Token.from_string_tinyv(s.lit)
		if tok in [.key_break, .key_continue, .key_nil, .key_none, .key_return, .key_false, .key_true,
			.name] {
			s.insert_semi = true
		}
		return tok
	} else if c == `'` || c == `"` {
		s.offset++
		if !s.in_str_inter {
			s.str_quote = c
		}
		s.string_literal(s.in_str_inter || (s.offset >= 2 && s.src[s.offset - 2] == `r`), c)
		s.lit = s.src[s.pos..s.offset]
		s.insert_semi = true
		return .string
	} else if c == `\`` {
		return s.scan_char_literal(c)
	}
	s.lit = ''
	s.offset++
	match c {
		`.` {
			c2 := s.peek_byte(0)
			if c2 >= `0` && c2 <= `9` {
				s.number()
				s.lit = s.src[s.pos..s.offset]
				return .number
			} else if c2 == `.` {
				s.offset++
				if s.peek_byte(0) == `.` {
					s.offset++
					return .ellipsis
				}
				return .dotdot
			}
			return .dot
		}
		`:` {
			if s.peek_byte(0) == `=` {
				s.offset++
				return .decl_assign
			}
			return .colon
		}
		`!` {
			c2 := s.peek_byte(0)
			if c2 == `=` {
				s.offset++
				return .ne
			} else if c2 == `i` {
				c3 := s.peek_byte(1)
				c4 := s.peek_byte(2)
				c4_is_space := c4 == ` ` || c4 == `\t`
				if c3 == `n` && c4_is_space {
					s.offset += 2
					return .not_in
				} else if c3 == `s` && c4_is_space {
					s.offset += 2
					return .not_is
				}
			}
			s.insert_semi = true
			return .not
		}
		`=` {
			if s.peek_byte(0) == `=` {
				s.offset++
				return .eq
			}
			return .assign
		}
		`+` {
			c2 := s.peek_byte(0)
			if c2 == `+` {
				s.offset++
				return .inc
			} else if c2 == `=` {
				s.offset++
				return .plus_assign
			}
			return .plus
		}
		`-` {
			c2 := s.peek_byte(0)
			if c2 == `-` {
				s.offset++
				return .dec
			} else if c2 == `=` {
				s.offset++
				return .minus_assign
			}
			return .minus
		}
		`%` {
			if s.peek_byte(0) == `=` {
				s.offset++
				return .mod_assign
			}
			return .mod
		}
		`*` {
			if s.peek_byte(0) == `=` {
				s.offset++
				return .mul_assign
			}
			return .mul
		}
		`^` {
			if s.peek_byte(0) == `=` {
				s.offset++
				return .xor_assign
			}
			return .xor
		}
		`&` {
			c2 := s.peek_byte(0)
			if c2 == `&` {
				if s.peek_byte(1) == `=` {
					s.offset += 2
					return .and_assign
				}
				s.offset++
				return .and
			} else if c2 == `=` {
				s.offset++
				return .and_assign
			}
			return .amp
		}
		`|` {
			c2 := s.peek_byte(0)
			if c2 == `|` {
				if s.peek_byte(1) == `=` {
					s.offset += 2
					return .or_assign
				}
				s.offset++
				return .logical_or
			} else if c2 == `=` {
				s.offset++
				return .or_assign
			}
			return .pipe
		}
		`<` {
			c2 := s.peek_byte(0)
			if c2 == `<` {
				s.offset++
				if s.peek_byte(0) == `=` {
					s.offset++
					return .left_shift_assign
				}
				return .left_shift
			} else if c2 == `=` {
				s.offset++
				return .le
			} else if c2 == `-` {
				s.offset++
				return .arrow
			}
			return .lt
		}
		`>` {
			c2 := s.peek_byte(0)
			if c2 == `>` {
				s.offset++
				c3 := s.peek_byte(0)
				if c3 == `>` {
					s.offset++
					if s.peek_byte(0) == `=` {
						s.offset++
						return .right_shift_unsigned_assign
					}
					return .right_shift_unsigned
				} else if c3 == `=` {
					s.offset++
					return .right_shift_assign
				}
				return .right_shift
			} else if c2 == `=` {
				s.offset++
				return .ge
			}
			return .gt
		}
		`#` {
			if s.peek_byte(0) == `[` {
				s.offset++
				return .lsbr
			}
			s.offset++
			start := s.offset
			for s.offset < s.src.len && s.src[s.offset] != `\n` {
				s.offset++
			}
			s.lit = s.src[start..s.offset].trim_space()
			s.insert_semi = true
			return .hash
		}
		`~` {
			return .bit_not
		}
		`,` {
			return .comma
		}
		`$` {
			if s.in_str_inter {
				return .str_dollar
			}
			return .dollar
		}
		`{` {
			if s.in_str_inter {
				s.str_inter_cbr_depth++
			}
			return .lcbr
		}
		`}` {
			if s.in_str_inter {
				s.str_inter_cbr_depth--
				if s.str_inter_cbr_depth == 0 {
					s.in_str_incomplete = true
					s.in_str_inter = false
				}
			}
			s.insert_semi = true
			return .rcbr
		}
		`(` {
			return .lpar
		}
		`)` {
			s.insert_semi = true
			return .rpar
		}
		`[` {
			return .lsbr
		}
		`]` {
			s.insert_semi = true
			return .rsbr
		}
		`;` {
			return .semicolon
		}
		`?` {
			s.insert_semi = true
			return .question
		}
		else {
			return .unknown
		}
	}
}

@[direct_array_access]
fn (mut s Scanner) whitespace() {
	for s.offset < s.src.len {
		c := s.src[s.offset]
		if c == ` ` || c == `\t` || c == `\r` {
			s.offset++
			continue
		} else if c == `\n` {
			if s.insert_semi {
				return
			}
			s.offset++
			s.file.add_line(s.offset)
			continue
		}
		break
	}
}

@[direct_array_access]
fn (mut s Scanner) line() {
	for s.offset < s.src.len {
		if s.src[s.offset] == `\n` {
			break
		}
		s.offset++
	}
}

@[direct_array_access]
fn (mut s Scanner) comment() {
	s.offset++
	if s.offset >= s.src.len {
		return
	}
	c := s.src[s.offset]
	if c == `/` {
		s.line()
	} else if c == `*` {
		s.offset++
		mut ml_comment_depth := 1
		for s.offset < s.src.len {
			c2 := s.src[s.offset]
			c3 := s.peek_byte(1)
			if c2 == `\n` {
				s.offset++
				s.file.add_line(s.offset)
			} else if c2 == `/` && c3 == `*` {
				s.offset += 2
				ml_comment_depth++
			} else if c2 == `*` && c3 == `/` {
				s.offset += 2
				ml_comment_depth--
				if ml_comment_depth == 0 {
					break
				}
			} else {
				s.offset++
			}
		}
	}
}

@[direct_array_access]
fn (mut s Scanner) string_literal(scan_as_raw bool, c_quote u8) {
	if scan_as_raw {
		for s.offset < s.src.len {
			c := s.src[s.offset]
			if c == c_quote {
				break
			}
			if c == `\n` {
				s.offset++
				s.file.add_line(s.offset)
				continue
			}
			s.offset++
		}
		if s.offset < s.src.len {
			s.offset++
		}
		return
	}
	for s.offset < s.src.len {
		c := s.src[s.offset]
		if c == `\\` {
			s.offset += 2
			continue
		} else if c == `\n` {
			s.offset++
			s.file.add_line(s.offset)
			continue
		} else if c == `$` && s.peek_byte(1) == `{` {
			s.in_str_inter = true
			if s.skip_interpolation {
				s.str_inter_cbr_depth++
				s.offset += 2
				continue
			} else {
				return
			}
		} else if s.skip_interpolation && s.in_str_inter {
			if c == `{` {
				s.str_inter_cbr_depth++
			} else if c == `}` {
				s.str_inter_cbr_depth--
				if s.str_inter_cbr_depth == 0 {
					s.in_str_inter = false
				}
			}
		} else if c == c_quote && !s.in_str_inter {
			s.offset++
			break
		}
		s.offset++
	}
}

@[direct_array_access]
fn (mut s Scanner) number() {
	if s.offset < s.src.len && s.src[s.offset] == `0` {
		s.offset++
		c := s.peek_byte(0)
		if c == `b` || c == `B` {
			s.offset++
			for s.offset < s.src.len {
				c2 := s.src[s.offset]
				if c2 == `0` || c2 == `1` || c2 == `_` {
					s.offset++
					continue
				}
				return
			}
			return
		} else if c == `x` || c == `X` {
			s.offset++
			for s.offset < s.src.len {
				c2 := s.src[s.offset]
				if (c2 >= `0` && c2 <= `9`) || (c2 >= `a` && c2 <= `f`)
					|| (c2 >= `A` && c2 <= `F`) || c2 == `_` {
					s.offset++
					continue
				}
				return
			}
			return
		} else if c == `o` || c == `O` {
			s.offset++
			for s.offset < s.src.len {
				c2 := s.src[s.offset]
				if (c2 >= `0` && c2 <= `7`) || c2 == `_` {
					s.offset++
					continue
				}
				return
			}
			return
		}
	}
	mut has_decimal := false
	mut has_exponent := false
	for s.offset < s.src.len {
		c := s.src[s.offset]
		if (c >= `0` && c <= `9`) || c == `_` {
			s.offset++
			continue
		} else if !has_decimal && c == `.` {
			next := s.peek_byte(1)
			if next != `.` && next >= `0` && next <= `9` {
				has_decimal = true
				s.offset++
				continue
			}
		} else if !has_exponent && (c == `e` || c == `E`) {
			has_exponent = true
			s.offset++
			if s.offset < s.src.len && (s.src[s.offset] == `+` || s.src[s.offset] == `-`) {
				s.offset++
			}
			continue
		}
		break
	}
}