Initial project import
This commit is contained in:
Agila Forge Importer committed
commit
23f2d29e32
11957 files changed
+1754577
No files matched your search
@@ -0,0 +1,160 @@
|
||||
## Description
|
||||
|
||||
`toml` is a fully fledged [TOML v1.0.0](https://toml.io/en/v1.0.0) compatible parser written in pure V.
|
||||
The module is tested against the [official compliance tests](https://github.com/toml-lang/compliance).
|
||||
|
||||
## Usage
|
||||
|
||||
Parsing files or `string`s containing TOML is easy.
|
||||
|
||||
Simply import the `toml` module and do:
|
||||
```v ignore
|
||||
doc1 := toml.parse_text(<string content>) or { panic(err) }
|
||||
doc2 := toml.parse_file(<file path>) or { panic(err) }
|
||||
```
|
||||
|
||||
## Example
|
||||
|
||||
Here we parse the official [TOML example](https://github.com/toml-lang/toml/blob/3b11f6921da7b6f5db37af039aa021fee450c091/README.md#Example)
|
||||
and print out some values.
|
||||
|
||||
```v
|
||||
import toml
|
||||
|
||||
const toml_text = '# This is a TOML document.
|
||||
|
||||
title = "TOML Example"
|
||||
|
||||
[owner]
|
||||
name = "Tom Preston-Werner"
|
||||
dob = 1979-05-27T07:32:00-08:00 # First class dates
|
||||
|
||||
[database]
|
||||
server = "192.168.1.1"
|
||||
ports = [ 8000, 8001, 8002 ]
|
||||
connection_max = 5000
|
||||
enabled = true
|
||||
|
||||
[servers]
|
||||
|
||||
# Indentation (tabs and/or spaces) is allowed but not required
|
||||
[servers.alpha]
|
||||
ip = "10.0.0.1"
|
||||
dc = "eqdc10"
|
||||
|
||||
[servers.beta]
|
||||
ip = "10.0.0.2"
|
||||
dc = "eqdc10"
|
||||
|
||||
[clients]
|
||||
data = [ ["gamma", "delta"], [1, 2] ]
|
||||
|
||||
# Line breaks are OK when inside arrays
|
||||
hosts = [
|
||||
"alpha",
|
||||
"omega"
|
||||
]'
|
||||
|
||||
fn main() {
|
||||
doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
title := doc.value('title').string()
|
||||
println('title: "${title}"')
|
||||
ip := doc.value('servers.alpha.ip').string()
|
||||
println('Server IP: "${ip}"')
|
||||
}
|
||||
```
|
||||
|
||||
## Value retrieval
|
||||
|
||||
The `toml` module supports easy retrieval of values from
|
||||
TOML documents by using a small and simple query syntax
|
||||
as argument to the `value()` function.
|
||||
|
||||
Keys in map entries are denoted by `.` and array entries
|
||||
uses `[<int>]`. Quoted keys are also supported by using
|
||||
the delimiters `"` or `'`.
|
||||
|
||||
`doc.value('table.array[0].a."b.c"')`
|
||||
|
||||
To query for a value that might not be in the document you
|
||||
can use the `.default_to(...)` function to provide a
|
||||
default value.
|
||||
|
||||
For cases where a default value might not be appropriate or
|
||||
to check if a value exists you can use `doc.value_opt('query')!`
|
||||
instead.
|
||||
|
||||
```v
|
||||
import toml
|
||||
|
||||
const toml_text = '
|
||||
val = true
|
||||
|
||||
[table]
|
||||
array = [
|
||||
{ a = "A" },
|
||||
{ b = "B" }
|
||||
]
|
||||
'
|
||||
|
||||
doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
|
||||
assert doc.value('val').bool() == true
|
||||
assert doc.value('table.array[0].a').string() == 'A'
|
||||
|
||||
// Provides a default value
|
||||
assert doc.value('non.existing').default_to(false).bool() == false
|
||||
|
||||
// Check if value exist
|
||||
// doc.value_opt('should.exist') or { ... }
|
||||
// or
|
||||
if value := doc.value_opt('table.array[1].b') {
|
||||
assert value.string() == 'B'
|
||||
}
|
||||
|
||||
// You can pass parts of the TOML document around
|
||||
// and still use .value()/.value_opt() to get the values
|
||||
arr := doc.value('table.array')
|
||||
assert arr.value('[1].b').string() == 'B'
|
||||
```
|
||||
|
||||
### Conversion
|
||||
|
||||
Any TOML value can be converted to a V equivalent type.
|
||||
|
||||
TOML values are represented as the `toml.Any` sum-type that
|
||||
can be converted to a V type.
|
||||
|
||||
| TOML value | V conversion (via `toml.Any`) |
|
||||
| -------------------------- | ----------------------------- |
|
||||
| val = "Basic string" | .string() |
|
||||
| val = 'Literal string' | .string() |
|
||||
| val = true | .bool() |
|
||||
| val = 1979-05-27T07:32:00Z | .datetime() (toml.DateTime) |
|
||||
| val = 1979-05-27 | .date() (toml.Date) |
|
||||
| val = 07:32:59 | .time() (toml.Time) |
|
||||
| val = 3.14 | .f32() / .f64() |
|
||||
| val = 100 | .int() / .i64() / .u64() |
|
||||
|
||||
Read more about values in the [TOML specification](https://toml.io/en/v1.0.0#spec).
|
||||
|
||||
## TOML to JSON
|
||||
|
||||
The `toml.to` module supports easy serialization of any TOML to JSON.
|
||||
|
||||
```v
|
||||
import toml
|
||||
import toml.to
|
||||
|
||||
const toml_text = '
|
||||
val = true
|
||||
[table]
|
||||
array = [
|
||||
{ a = "A" },
|
||||
{ b = "B" }
|
||||
]
|
||||
'
|
||||
|
||||
doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
assert to.json(doc) == '{ "val": true, "table": { "array": [ { "a": "A" }, { "b": "B" } ] } }'
|
||||
```
|
||||
@@ -0,0 +1,516 @@
|
||||
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module toml
|
||||
|
||||
// Pretty much all the same builtin types as the `json2.Any` type plus `DateTime`,`Date`,`Time`
|
||||
pub type Any = Date
|
||||
| DateTime
|
||||
| Null
|
||||
| Time
|
||||
| []Any
|
||||
| bool
|
||||
| f32
|
||||
| f64
|
||||
| i64
|
||||
| int
|
||||
| map[string]Any
|
||||
| string
|
||||
| u64
|
||||
|
||||
// string returns `Any` as a string.
|
||||
pub fn (a Any) string() string {
|
||||
match a {
|
||||
// NOTE if `.clone()` is not used here:
|
||||
// string { return a as string }
|
||||
// ... certain call-patterns to this function will cause a memory corruption.
|
||||
// See `tests/toml_memory_corruption_test.v` for a matching regression test.
|
||||
string { return (a as string).clone() }
|
||||
DateTime { return a.str().clone() }
|
||||
Date { return a.str().clone() }
|
||||
Time { return a.str().clone() }
|
||||
else { return a.str().clone() }
|
||||
}
|
||||
}
|
||||
|
||||
// to_toml returns `Any` as a TOML encoded value.
|
||||
pub fn (a Any) to_toml() string {
|
||||
match a {
|
||||
map[string]Any {
|
||||
// TODO: more format control?
|
||||
return a.to_inline_toml()
|
||||
}
|
||||
[]Any {
|
||||
return a.to_toml()
|
||||
}
|
||||
bool, f32, f64, i64, int, u64 {
|
||||
return a.str().clone()
|
||||
}
|
||||
// NOTE if `.clone()` is not used here:
|
||||
// string { return a as string }
|
||||
// ... certain call-patterns to this function will cause a memory corruption.
|
||||
// See `tests/toml_memory_corruption_test.v` for a matching regression test.
|
||||
string {
|
||||
return '"' + (a as string).clone() + '"'
|
||||
}
|
||||
DateTime {
|
||||
return a.str().clone()
|
||||
}
|
||||
Date {
|
||||
return a.str().clone()
|
||||
}
|
||||
Time {
|
||||
return a.str().clone()
|
||||
}
|
||||
else {
|
||||
return a.str().clone()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// int returns `Any` as an 32-bit integer.
|
||||
pub fn (a Any) int() int {
|
||||
match a {
|
||||
int { return a }
|
||||
i64, f32, f64, bool { return int(a) }
|
||||
// time.Time { return int(0) } // TODO
|
||||
else { return 0 }
|
||||
}
|
||||
}
|
||||
|
||||
// i64 returns `Any` as a 64-bit integer.
|
||||
pub fn (a Any) i64() i64 {
|
||||
match a {
|
||||
i64 { return a }
|
||||
int, f32, f64, bool { return i64(a) }
|
||||
// time.Time { return i64(0) } // TODO
|
||||
else { return 0 }
|
||||
}
|
||||
}
|
||||
|
||||
// u64 returns `Any` as a 64-bit unsigned integer.
|
||||
pub fn (a Any) u64() u64 {
|
||||
match a {
|
||||
u64 { return a }
|
||||
int, i64, f32, f64, bool { return u64(a) }
|
||||
// time.Time { return u64(0) } // TODO
|
||||
else { return 0 }
|
||||
}
|
||||
}
|
||||
|
||||
// f32 returns `Any` as a 32-bit float.
|
||||
pub fn (a Any) f32() f32 {
|
||||
match a {
|
||||
f32 { return a }
|
||||
int, i64, f64 { return f32(a) }
|
||||
// time.Time { return f32(0) } // TODO
|
||||
else { return 0.0 }
|
||||
}
|
||||
}
|
||||
|
||||
// f64 returns `Any` as a 64-bit float.
|
||||
pub fn (a Any) f64() f64 {
|
||||
match a {
|
||||
f64 { return a }
|
||||
int, i64, f32 { return f64(a) }
|
||||
// time.Time { return f64(0) } // TODO
|
||||
else { return 0.0 }
|
||||
}
|
||||
}
|
||||
|
||||
// array returns `Any` as an array.
|
||||
pub fn (a Any) array() []Any {
|
||||
if a is []Any {
|
||||
return a
|
||||
} else if a is map[string]Any {
|
||||
mut arr := []Any{}
|
||||
for _, v in a {
|
||||
arr << v
|
||||
}
|
||||
return arr
|
||||
}
|
||||
return [a]
|
||||
}
|
||||
|
||||
// as_map returns `Any` as a map (TOML table).
|
||||
pub fn (a Any) as_map() map[string]Any {
|
||||
if a is map[string]Any {
|
||||
return a
|
||||
} else if a is []Any {
|
||||
mut mp := map[string]Any{}
|
||||
for i, fi in a {
|
||||
mp['${i}'] = fi
|
||||
}
|
||||
return mp
|
||||
}
|
||||
return {
|
||||
'0': a
|
||||
}
|
||||
}
|
||||
|
||||
// bool returns `Any` as a boolean.
|
||||
pub fn (a Any) bool() bool {
|
||||
match a {
|
||||
bool { return a }
|
||||
string { return a.bool() }
|
||||
else { return false }
|
||||
}
|
||||
}
|
||||
|
||||
// date returns `Any` as a `toml.Date` struct.
|
||||
pub fn (a Any) date() Date {
|
||||
match a {
|
||||
// string { } // TODO
|
||||
// NOTE `.clone()` is to avoid memory corruption see `pub fn (a Any) string() string`
|
||||
Date { return Date{a.str().clone()} }
|
||||
else { return Date{''} }
|
||||
}
|
||||
}
|
||||
|
||||
// time returns `Any` as a `toml.Time` struct.
|
||||
pub fn (a Any) time() Time {
|
||||
match a {
|
||||
// string { } // TODO
|
||||
// NOTE `.clone()` is to avoid memory corruption see `pub fn (a Any) string() string`
|
||||
Time { return Time{a.str().clone()} }
|
||||
else { return Time{''} }
|
||||
}
|
||||
}
|
||||
|
||||
// datetime returns `Any` as a `toml.DateTime` struct.
|
||||
pub fn (a Any) datetime() DateTime {
|
||||
match a {
|
||||
// string { } // TODO
|
||||
// NOTE `.clone()` is to avoid memory corruption see `pub fn (a Any) string() string`
|
||||
DateTime { return DateTime{a.str().clone()} }
|
||||
else { return DateTime{''} }
|
||||
}
|
||||
}
|
||||
|
||||
// default_to returns `value` if `a Any` is `Null`.
|
||||
// This can be used to set default values when retrieving
|
||||
// values. E.g.: `toml_doc.value('wrong.key').default_to(123).int()`
|
||||
pub fn (a Any) default_to(value Any) Any {
|
||||
match a {
|
||||
Null { return value }
|
||||
else { return a }
|
||||
}
|
||||
}
|
||||
|
||||
// value queries a value from the map.
|
||||
// `key` supports a small query syntax scheme:
|
||||
// Maps can be queried in "dotted" form e.g. `a.b.c`.
|
||||
// quoted keys are supported as `a."b.c"` or `a.'b.c'`.
|
||||
// Arrays can be queried with `a[0].b[1].[2]`.
|
||||
pub fn (m map[string]Any) value(key string) Any {
|
||||
return Any(m).value(key)
|
||||
}
|
||||
|
||||
// as_strings returns the contents of the map
|
||||
// as `map[string]string`
|
||||
pub fn (m map[string]Any) as_strings() map[string]string {
|
||||
mut result := map[string]string{}
|
||||
for k, v in m {
|
||||
result[k] = v.string()
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// to_toml returns the contents of the map
|
||||
// as a TOML encoded `string`.
|
||||
pub fn (m map[string]Any) to_toml() string {
|
||||
mut toml_text := ''
|
||||
for k, v in m {
|
||||
key := if k.contains(' ') { '"${k}"' } else { k }
|
||||
toml_text += '${key} = ${v.to_toml()}\n'
|
||||
}
|
||||
toml_text = toml_text.trim_right('\n')
|
||||
return toml_text
|
||||
}
|
||||
|
||||
// to_inline_toml returns the contents of the map
|
||||
// as an inline table encoded TOML `string`.
|
||||
pub fn (m map[string]Any) to_inline_toml() string {
|
||||
mut toml_text := '{'
|
||||
mut i := 1
|
||||
for k, v in m {
|
||||
key := if k.contains(' ') { '"${k}"' } else { k }
|
||||
delimiter := if i < m.len { ',' } else { '' }
|
||||
toml_text += ' ${key} = ${v.to_toml()}${delimiter}'
|
||||
i++
|
||||
}
|
||||
return toml_text + ' }'
|
||||
}
|
||||
|
||||
// value queries a value from the array.
|
||||
// `key` supports a small query syntax scheme:
|
||||
// The array can be queried with `[0].b[1].[2]`.
|
||||
// Maps can be queried in "dotted" form e.g. `a.b.c`.
|
||||
// quoted keys are supported as `a."b.c"` or `a.'b.c'`.
|
||||
pub fn (a []Any) value(key string) Any {
|
||||
return Any(a).value(key)
|
||||
}
|
||||
|
||||
// as_strings returns the contents of the array
|
||||
// as `[]string`
|
||||
pub fn (a []Any) as_strings() []string {
|
||||
mut sa := []string{}
|
||||
for any in a {
|
||||
sa << any.string()
|
||||
}
|
||||
return sa
|
||||
}
|
||||
|
||||
// to_toml returns the contents of the array
|
||||
// as a TOML encoded `string`.
|
||||
pub fn (a []Any) to_toml() string {
|
||||
mut toml_text := '[\n'
|
||||
for any in a {
|
||||
toml_text += ' ' + any.to_toml() + ',\n'
|
||||
}
|
||||
toml_text = toml_text.trim_right(',\n')
|
||||
return toml_text + '\n]'
|
||||
}
|
||||
|
||||
// value queries a value from the `Any` type.
|
||||
// `key` supports a small query syntax scheme:
|
||||
// Maps can be queried in "dotted" form e.g. `a.b.c`.
|
||||
// quoted keys are supported as `a."b.c"` or `a.'b.c'`.
|
||||
// Arrays can be queried with `a[0].b[1].[2]`.
|
||||
pub fn (a Any) value(key string) Any {
|
||||
key_split := parse_dotted_key(key) or { return null }
|
||||
return a.value_(a, key_split)
|
||||
}
|
||||
|
||||
// value_opt queries a value from the current element's tree. Returns an error
|
||||
// if the key is not valid or there is no value for the key.
|
||||
pub fn (a Any) value_opt(key string) !Any {
|
||||
key_split := parse_dotted_key(key) or { return error('invalid dotted key') }
|
||||
x := a.value_(a, key_split)
|
||||
if x is Null {
|
||||
return error('no value for key')
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
// value_ returns the `Any` value found at `key`.
|
||||
fn (a Any) value_(value Any, key []string) Any {
|
||||
if key.len == 0 {
|
||||
return null
|
||||
}
|
||||
mut any_value := null
|
||||
k, index := parse_array_key(key[0])
|
||||
if k == '' {
|
||||
arr := value as []Any
|
||||
any_value = arr[index] or { return null }
|
||||
}
|
||||
if value is map[string]Any {
|
||||
any_value = value[k] or { return null }
|
||||
if index > -1 {
|
||||
arr := any_value as []Any
|
||||
any_value = arr[index] or { return null }
|
||||
}
|
||||
}
|
||||
if key.len <= 1 {
|
||||
return any_value
|
||||
}
|
||||
match any_value {
|
||||
map[string]Any, []Any {
|
||||
return a.value_(any_value, key[1..])
|
||||
}
|
||||
else {
|
||||
return value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reflect returns `T` with `T.<field>`'s value set to the
|
||||
// value of any 1st level TOML key by the same name.
|
||||
pub fn (a Any) reflect[T]() T {
|
||||
mut reflected := T{}
|
||||
$for field in T.fields {
|
||||
mut toml_field_name := field.name
|
||||
mut skip := false
|
||||
// Remapping of field names, for example:
|
||||
// TOML: 'assert = "ok"'
|
||||
// V: User { asrt string @[toml: 'assert'] }
|
||||
// User.asrt == 'ok'
|
||||
for attr in field.attrs {
|
||||
if attr == 'skip' {
|
||||
skip = true
|
||||
break
|
||||
}
|
||||
if attr.starts_with('toml:') {
|
||||
toml_field_name = attr.all_after(':').trim_space()
|
||||
}
|
||||
}
|
||||
value := a.value(toml_field_name)
|
||||
// only set the field's value when value != null and !skip, else field got it's default value
|
||||
if !skip && value != null {
|
||||
$if field.typ is string {
|
||||
reflected.$(field.name) = value.string()
|
||||
} $else $if field.typ is bool {
|
||||
reflected.$(field.name) = value.bool()
|
||||
} $else $if field.typ is int {
|
||||
reflected.$(field.name) = value.int()
|
||||
} $else $if field.typ is f32 {
|
||||
reflected.$(field.name) = value.f32()
|
||||
} $else $if field.typ is f64 {
|
||||
reflected.$(field.name) = value.f64()
|
||||
} $else $if field.typ is i64 {
|
||||
reflected.$(field.name) = value.i64()
|
||||
} $else $if field.typ is u64 {
|
||||
reflected.$(field.name) = value.u64()
|
||||
} $else $if field.typ is Any {
|
||||
reflected.$(field.name) = value
|
||||
} $else $if field.typ is DateTime {
|
||||
reflected.$(field.name) = value.datetime()
|
||||
} $else $if field.typ is Date {
|
||||
reflected.$(field.name) = value.date()
|
||||
} $else $if field.typ is Time {
|
||||
reflected.$(field.name) = value.time()
|
||||
}
|
||||
// Arrays of primitive types
|
||||
$else $if field.typ is []string {
|
||||
any_array := value.array()
|
||||
reflected.$(field.name) = any_array.as_strings()
|
||||
} $else $if field.typ is []bool {
|
||||
any_array := value.array()
|
||||
mut arr := []bool{cap: any_array.len}
|
||||
for any_value in any_array {
|
||||
arr << any_value.bool()
|
||||
}
|
||||
reflected.$(field.name) = arr
|
||||
} $else $if field.typ is []int {
|
||||
any_array := value.array()
|
||||
mut arr := []int{cap: any_array.len}
|
||||
for any_value in any_array {
|
||||
arr << any_value.int()
|
||||
}
|
||||
reflected.$(field.name) = arr
|
||||
} $else $if field.typ is []f32 {
|
||||
any_array := value.array()
|
||||
mut arr := []f32{cap: any_array.len}
|
||||
for any_value in any_array {
|
||||
arr << any_value.f32()
|
||||
}
|
||||
reflected.$(field.name) = arr
|
||||
} $else $if field.typ is []f64 {
|
||||
any_array := value.array()
|
||||
mut arr := []f64{cap: any_array.len}
|
||||
for any_value in any_array {
|
||||
arr << any_value.f64()
|
||||
}
|
||||
reflected.$(field.name) = arr
|
||||
} $else $if field.typ is []i64 {
|
||||
any_array := value.array()
|
||||
mut arr := []i64{cap: any_array.len}
|
||||
for any_value in any_array {
|
||||
arr << any_value.i64()
|
||||
}
|
||||
reflected.$(field.name) = arr
|
||||
} $else $if field.typ is []u64 {
|
||||
any_array := value.array()
|
||||
mut arr := []u64{cap: any_array.len}
|
||||
for any_value in any_array {
|
||||
arr << any_value.u64()
|
||||
}
|
||||
reflected.$(field.name) = arr
|
||||
} $else $if field.typ is []Any {
|
||||
reflected.$(field.name) = value.array()
|
||||
} $else $if field.typ is []DateTime {
|
||||
any_array := value.array()
|
||||
mut arr := []DateTime{cap: any_array.len}
|
||||
for any_value in any_array {
|
||||
arr << any_value.datetime()
|
||||
}
|
||||
reflected.$(field.name) = arr
|
||||
} $else $if field.typ is []Date {
|
||||
any_array := value.array()
|
||||
mut arr := []Date{cap: any_array.len}
|
||||
for any_value in any_array {
|
||||
arr << any_value.date()
|
||||
}
|
||||
reflected.$(field.name) = arr
|
||||
} $else $if field.typ is []Time {
|
||||
any_array := value.array()
|
||||
mut arr := []Time{cap: any_array.len}
|
||||
for any_value in any_array {
|
||||
arr << any_value.time()
|
||||
}
|
||||
reflected.$(field.name) = arr
|
||||
}
|
||||
// String key maps of primitive types
|
||||
$else $if field.typ is map[string]string {
|
||||
any_map := value.as_map()
|
||||
reflected.$(field.name) = any_map.as_strings()
|
||||
} $else $if field.typ is map[string]bool {
|
||||
any_map := value.as_map()
|
||||
mut type_map := map[string]bool{}
|
||||
for k, any_value in any_map {
|
||||
type_map[k] = any_value.bool()
|
||||
}
|
||||
reflected.$(field.name) = type_map.clone()
|
||||
} $else $if field.typ is map[string]int {
|
||||
any_map := value.as_map()
|
||||
mut type_map := map[string]int{}
|
||||
for k, any_value in any_map {
|
||||
type_map[k] = any_value.int()
|
||||
}
|
||||
reflected.$(field.name) = type_map.clone()
|
||||
} $else $if field.typ is map[string]f32 {
|
||||
any_map := value.as_map()
|
||||
mut type_map := map[string]f32{}
|
||||
for k, any_value in any_map {
|
||||
type_map[k] = any_value.f32()
|
||||
}
|
||||
reflected.$(field.name) = type_map.clone()
|
||||
} $else $if field.typ is map[string]f64 {
|
||||
any_map := value.as_map()
|
||||
mut type_map := map[string]f64{}
|
||||
for k, any_value in any_map {
|
||||
type_map[k] = any_value.f64()
|
||||
}
|
||||
reflected.$(field.name) = type_map.clone()
|
||||
} $else $if field.typ is map[string]i64 {
|
||||
any_map := value.as_map()
|
||||
mut type_map := map[string]i64{}
|
||||
for k, any_value in any_map {
|
||||
type_map[k] = any_value.i64()
|
||||
}
|
||||
reflected.$(field.name) = type_map.clone()
|
||||
} $else $if field.typ is map[string]u64 {
|
||||
any_map := value.as_map()
|
||||
mut type_map := map[string]u64{}
|
||||
for k, any_value in any_map {
|
||||
type_map[k] = any_value.u64()
|
||||
}
|
||||
reflected.$(field.name) = type_map.clone()
|
||||
} $else $if field.typ is map[string]Any {
|
||||
reflected.$(field.name) = value.as_map()
|
||||
} $else $if field.typ is map[string]DateTime {
|
||||
any_map := value.as_map()
|
||||
mut type_map := map[string]DateTime{}
|
||||
for k, any_value in any_map {
|
||||
type_map[k] = any_value.datetime()
|
||||
}
|
||||
reflected.$(field.name) = type_map.clone()
|
||||
} $else $if field.typ is map[string]Date {
|
||||
any_map := value.as_map()
|
||||
mut type_map := map[string]Date{}
|
||||
for k, any_value in any_map {
|
||||
type_map[k] = any_value.date()
|
||||
}
|
||||
reflected.$(field.name) = type_map.clone()
|
||||
} $else $if field.typ is map[string]Time {
|
||||
any_map := value.as_map()
|
||||
mut type_map := map[string]Time{}
|
||||
for k, any_value in any_map {
|
||||
type_map[k] = any_value.time()
|
||||
}
|
||||
reflected.$(field.name) = type_map.clone()
|
||||
}
|
||||
}
|
||||
}
|
||||
return reflected
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module ast
|
||||
|
||||
import toml.input
|
||||
|
||||
// Root represents the root structure of any parsed TOML text snippet or file.
|
||||
@[heap]
|
||||
pub struct Root {
|
||||
pub:
|
||||
input input.Config // User input configuration
|
||||
pub mut:
|
||||
comments []Comment
|
||||
table Value
|
||||
// errors []errors.Error // all the checker errors in the file
|
||||
}
|
||||
|
||||
// str returns the string representation of the root node.
|
||||
pub fn (r Root) str() string {
|
||||
mut s := typeof(r).name + '{\n'
|
||||
s += ' input: ${r.input}\n'
|
||||
s += ' table: ${r.table}\n'
|
||||
s += '}'
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module ast
|
||||
|
||||
import toml.token
|
||||
import strconv
|
||||
|
||||
// Key is a sumtype representing all types of keys that
|
||||
// can be found in a TOML document.
|
||||
pub type Key = Bare | Bool | Null | Number | Quoted
|
||||
|
||||
// str returns the string representation of the key. This is implemented
|
||||
// by all the variants of Key.
|
||||
pub fn (k Key) str() string {
|
||||
return k.text.clone()
|
||||
}
|
||||
|
||||
// Value is a sumtype representing all possible value types
|
||||
// found in a TOML document.
|
||||
pub type Value = Bool
|
||||
| Date
|
||||
| DateTime
|
||||
| Null
|
||||
| Number
|
||||
| Quoted
|
||||
| Time
|
||||
| []Value
|
||||
| map[string]Value
|
||||
|
||||
// str outputs the value in JSON-like format for eased
|
||||
// debugging
|
||||
pub fn (v Value) str() string {
|
||||
match v {
|
||||
Quoted, Date, DateTime, Time {
|
||||
return '"${v.text}"'
|
||||
}
|
||||
Bool, Null, Number {
|
||||
return v.text
|
||||
}
|
||||
map[string]Value {
|
||||
mut str := '{'
|
||||
for key, val in v {
|
||||
str += ' "${key}": ${val},'
|
||||
}
|
||||
str = str.trim_right(',')
|
||||
str += ' }'
|
||||
return str
|
||||
}
|
||||
[]Value {
|
||||
mut str := '['
|
||||
for val in v {
|
||||
str += ' ${val},'
|
||||
}
|
||||
str = str.trim_right(',')
|
||||
str += ' ]'
|
||||
return str
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DateTimeType is a sumtype representing all possible date types
|
||||
// found in a TOML document.
|
||||
pub type DateTimeType = Date | DateTime | Time
|
||||
|
||||
// str returns the `string` representation of the `DateTimeType` type.
|
||||
pub fn (dtt DateTimeType) str() string {
|
||||
return dtt.text
|
||||
}
|
||||
|
||||
// Comment is the data representation of a TOML comment (`# This is a comment`).
|
||||
pub struct Comment {
|
||||
pub:
|
||||
text string
|
||||
pos token.Pos
|
||||
}
|
||||
|
||||
// str returns the `string` representation of the `Comment` type.
|
||||
pub fn (c Comment) str() string {
|
||||
mut s := typeof(c).name + '{\n'
|
||||
s += ' text: \'${c.text}\'\n'
|
||||
s += ' pos: ${c.pos}\n'
|
||||
s += '}'
|
||||
return s
|
||||
}
|
||||
|
||||
// Null is used in sumtype checks as a "default" value when nothing else is possible.
|
||||
pub struct Null {
|
||||
pub:
|
||||
text string
|
||||
pos token.Pos
|
||||
}
|
||||
|
||||
// str returns the `string` representation of the `Null` type
|
||||
pub fn (n Null) str() string {
|
||||
return n.text
|
||||
}
|
||||
|
||||
// Quoted is the data representation of a TOML quoted type (`"quoted-key" = "I'm a quoted value"`).
|
||||
// Quoted types can appear both as keys and values in TOML documents.
|
||||
pub struct Quoted {
|
||||
pub mut:
|
||||
text string
|
||||
pub:
|
||||
pos token.Pos
|
||||
is_multiline bool
|
||||
quote u8
|
||||
}
|
||||
|
||||
// str returns the `string` representation of the `Quoted` type.
|
||||
pub fn (q Quoted) str() string {
|
||||
mut str := typeof(q).name + '{\n'
|
||||
str += ' text: \'${q.text}\'\n'
|
||||
str += ' pos: ${q.pos}\n'
|
||||
str += ' is_multiline: ${q.is_multiline}\n'
|
||||
str += ' quote: \'${q.quote}\'\n'
|
||||
str += '}'
|
||||
return str
|
||||
}
|
||||
|
||||
// Bare is the data representation of a TOML bare type (`bare_key = ...`).
|
||||
// Bare types can appear only as keys in TOML documents. Otherwise they take the
|
||||
// form of Bool or Numbers.
|
||||
pub struct Bare {
|
||||
pub:
|
||||
text string
|
||||
pos token.Pos
|
||||
}
|
||||
|
||||
// str returns the `string` representation of the `Bare` type.
|
||||
pub fn (b Bare) str() string {
|
||||
mut str := typeof(b).name + '{\n'
|
||||
str += ' text: \'${b.text}\'\n'
|
||||
str += ' pos: ${b.pos}\n'
|
||||
str += '}'
|
||||
return str
|
||||
}
|
||||
|
||||
// Bool is the data representation of a TOML boolean type (`... = true`).
|
||||
// Bool types can appear only as values in TOML documents. Keys named `true` or `false`
|
||||
// are considered as Bare types.
|
||||
pub struct Bool {
|
||||
pub:
|
||||
text string
|
||||
pos token.Pos
|
||||
}
|
||||
|
||||
// str returns the `string` representation of the `Bool` type.
|
||||
pub fn (b Bool) str() string {
|
||||
mut str := typeof(b).name + '{\n'
|
||||
str += ' text: \'${b.text}\'\n'
|
||||
str += ' pos: ${b.pos}\n'
|
||||
str += '}'
|
||||
return str
|
||||
}
|
||||
|
||||
// Number is the data representation of a TOML number type (`25 = 5e2`).
|
||||
// Number types can appear both as keys and values in TOML documents.
|
||||
// Number can be integers, floats, infinite, NaN - they can have exponents (`5e2`) and be sign prefixed (`+2`).
|
||||
pub struct Number {
|
||||
pub:
|
||||
pos token.Pos
|
||||
pub mut:
|
||||
text string
|
||||
}
|
||||
|
||||
// str returns the `string` representation of the `Number` type.
|
||||
pub fn (n Number) str() string {
|
||||
mut str := typeof(n).name + '{\n'
|
||||
str += ' text: \'${n.text}\'\n'
|
||||
str += ' pos: ${n.pos}\n'
|
||||
str += '}'
|
||||
return str
|
||||
}
|
||||
|
||||
// i64 returns the `n Number` as an `i64` value.
|
||||
pub fn (n Number) i64() i64 {
|
||||
if n.text.starts_with('0x') {
|
||||
hex := n.text.all_after('0x').to_upper().replace('_', '')
|
||||
return strconv.parse_int(hex, 16, 64) or { i64(0) }
|
||||
} else if n.text.starts_with('0o') {
|
||||
oct := n.text.all_after('0o').replace('_', '')
|
||||
return strconv.parse_int(oct, 8, 64) or { i64(0) }
|
||||
} else if n.text.starts_with('0b') {
|
||||
bin := n.text.all_after('0b').replace('_', '')
|
||||
return strconv.parse_int(bin, 2, 64) or { i64(0) }
|
||||
}
|
||||
return strconv.parse_int(n.text, 0, 64) or { i64(0) }
|
||||
}
|
||||
|
||||
// f64 returns the `n Number` as an `f64` value.
|
||||
pub fn (n Number) f64() f64 {
|
||||
return n.text.replace('_', '').f64()
|
||||
}
|
||||
|
||||
// Date is the data representation of a TOML date type (`YYYY-MM-DD`).
|
||||
// Date types can appear both as keys and values in TOML documents.
|
||||
// Keys named like dates e.g. `1980-12-29` are considered Bare key types.
|
||||
pub struct Date {
|
||||
pub:
|
||||
text string
|
||||
pos token.Pos
|
||||
}
|
||||
|
||||
// str returns the `string` representation of the `Date` type.
|
||||
pub fn (d Date) str() string {
|
||||
mut str := typeof(d).name + '{\n'
|
||||
str += ' text: \'${d.text}\'\n'
|
||||
str += ' pos: ${d.pos}\n'
|
||||
str += '}'
|
||||
return str
|
||||
}
|
||||
|
||||
// Time is the data representation of a TOML time type (`HH:MM:SS.milli`).
|
||||
// Time types can appear only as values in TOML documents.
|
||||
pub struct Time {
|
||||
pub:
|
||||
text string
|
||||
offset int
|
||||
pos token.Pos
|
||||
}
|
||||
|
||||
// str returns the `string` representation of the `Time` type.
|
||||
pub fn (t Time) str() string {
|
||||
mut str := typeof(t).name + '{\n'
|
||||
str += ' text: \'${t.text}\'\n'
|
||||
str += ' offset: \'${t.offset}\'\n'
|
||||
str += ' pos: ${t.pos}\n'
|
||||
str += '}'
|
||||
return str
|
||||
}
|
||||
|
||||
// DateTime is the data representation of a TOML date-time type (`YYYY-MM-DDTHH:MM:SS.milli`).
|
||||
// DateTime types can appear only as values in TOML documents.
|
||||
pub struct DateTime {
|
||||
pub mut:
|
||||
text string
|
||||
pub:
|
||||
pos token.Pos
|
||||
date Date
|
||||
time Time
|
||||
}
|
||||
|
||||
// str returns the `string` representation of the `DateTime` type.
|
||||
pub fn (dt DateTime) str() string {
|
||||
mut str := typeof(dt).name + '{\n'
|
||||
str += ' text: \'${dt.text}\'\n'
|
||||
str += ' date: \'${dt.date}\'\n'
|
||||
str += ' time: \'${dt.time}\'\n'
|
||||
str += ' pos: ${dt.pos}\n'
|
||||
str += '}'
|
||||
return str
|
||||
}
|
||||
|
||||
// EOF is the data representation of the end of the TOML document.
|
||||
pub struct EOF {
|
||||
pub:
|
||||
pos token.Pos
|
||||
}
|
||||
|
||||
// str returns the `string` representation of the `EOF` type.
|
||||
pub fn (e EOF) str() string {
|
||||
mut str := typeof(e).name + '{\n'
|
||||
str += ' pos: ${e.pos}\n'
|
||||
str += '}'
|
||||
return str
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
module walker
|
||||
|
||||
import toml.ast
|
||||
|
||||
// Visitor defines a visit method which is invoked by the walker on each Value node it encounters.
|
||||
pub interface Visitor {
|
||||
visit(value &ast.Value) !
|
||||
}
|
||||
|
||||
// Modifier defines a modify method which is invoked by the walker on each Value node it encounters.
|
||||
pub interface Modifier {
|
||||
modify(mut value ast.Value) !
|
||||
}
|
||||
|
||||
pub type InspectorFn = fn (value &ast.Value, data voidptr) !
|
||||
|
||||
struct Inspector {
|
||||
inspector_callback InspectorFn = unsafe { nil }
|
||||
mut:
|
||||
data voidptr
|
||||
}
|
||||
|
||||
// visit calls the inspector callback on the specified Value node.
|
||||
pub fn (i &Inspector) visit(value &ast.Value) ! {
|
||||
i.inspector_callback(value, i.data) or { return err }
|
||||
}
|
||||
|
||||
// inspect traverses and checks the AST Value node on a depth-first order and based on the data given
|
||||
pub fn inspect(value &ast.Value, data voidptr, inspector_callback InspectorFn) ! {
|
||||
walk(Inspector{inspector_callback, data}, value)!
|
||||
}
|
||||
|
||||
// walk traverses the AST using the given visitor
|
||||
@[autofree_bug; manualfree]
|
||||
pub fn walk(visitor Visitor, value &ast.Value) ! {
|
||||
if value is map[string]ast.Value {
|
||||
value_map := value as map[string]ast.Value
|
||||
for _, val in value_map {
|
||||
walk(visitor, &val)!
|
||||
}
|
||||
}
|
||||
if value is []ast.Value {
|
||||
value_array := value as []ast.Value
|
||||
for val in value_array {
|
||||
walk(visitor, &val)!
|
||||
}
|
||||
} else {
|
||||
visitor.visit(value)!
|
||||
}
|
||||
}
|
||||
|
||||
// walk_and_modify traverses the AST using the given modifier and lets the visitor
|
||||
// modify the contents.
|
||||
@[autofree_bug; manualfree]
|
||||
pub fn walk_and_modify(modifier Modifier, mut value ast.Value) ! {
|
||||
if value is map[string]ast.Value {
|
||||
mut value_map := value as map[string]ast.Value
|
||||
for _, mut val in value_map {
|
||||
walk_and_modify(modifier, mut &val)!
|
||||
}
|
||||
}
|
||||
if value is []ast.Value {
|
||||
mut value_array := value as []ast.Value
|
||||
for mut val in value_array {
|
||||
walk_and_modify(modifier, mut &val)!
|
||||
}
|
||||
} else {
|
||||
modifier.modify(mut value)!
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,665 @@
|
||||
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module checker
|
||||
|
||||
import toml.ast
|
||||
import toml.ast.walker
|
||||
import toml.util
|
||||
import toml.token
|
||||
import toml.scanner
|
||||
import encoding.utf8
|
||||
import time
|
||||
import strconv
|
||||
|
||||
pub const allowed_basic_escape_chars = [`u`, `U`, `b`, `t`, `n`, `f`, `r`, `"`, `\\`]
|
||||
|
||||
// utf8_max is the largest inclusive value of the Unicodes scalar value ranges.
|
||||
const utf8_max = 0x10FFFF
|
||||
|
||||
fn allowed_basic_escape_char_list() []u8 {
|
||||
return [u8(`u`), `U`, `b`, `t`, `n`, `f`, `r`, `"`, `\\`]
|
||||
}
|
||||
|
||||
fn toml_parse_time(s string) !time.Time {
|
||||
if s.len > 3 && s[2] == `:` {
|
||||
// complete the partial time, with an arbitrary date:
|
||||
return time.parse_rfc3339('0001-01-01T' + s)
|
||||
}
|
||||
if s.len == 10 {
|
||||
// complete the partial date, with zero time and zero timezone
|
||||
return time.parse_rfc3339(s + 'T00:00:00Z')
|
||||
}
|
||||
return time.parse_rfc3339(s)!
|
||||
}
|
||||
|
||||
// Checker checks a tree of TOML `ast.Value`'s for common errors.
|
||||
pub struct Checker {
|
||||
pub:
|
||||
scanner &scanner.Scanner = unsafe { nil }
|
||||
}
|
||||
|
||||
// check checks the `ast.Value` and all it's children
|
||||
// for common errors.
|
||||
pub fn (c &Checker) check(n &ast.Value) ! {
|
||||
walker.walk(c, n)!
|
||||
}
|
||||
|
||||
fn (c &Checker) visit(value &ast.Value) ! {
|
||||
match value {
|
||||
ast.Bool {
|
||||
c.check_boolean(value)!
|
||||
}
|
||||
ast.Number {
|
||||
c.check_number(value)!
|
||||
}
|
||||
ast.Quoted {
|
||||
c.check_quoted(value)!
|
||||
}
|
||||
ast.DateTime {
|
||||
c.check_date_time(value)!
|
||||
}
|
||||
ast.Date {
|
||||
c.check_date(value)!
|
||||
}
|
||||
ast.Time {
|
||||
c.check_time(value)!
|
||||
}
|
||||
else {}
|
||||
}
|
||||
}
|
||||
|
||||
// excerpt returns a string of the token's surroundings
|
||||
fn (c &Checker) excerpt(tp token.Pos) string {
|
||||
return c.scanner.excerpt(tp.pos, 10)
|
||||
}
|
||||
|
||||
// is_hex_bin_oct_prefixed returns true if `hbo` has either
|
||||
// of: `0x`, `0o` or `0b` - as a prefix.
|
||||
// Example: assert is_hex_bin_oct_prefixed('0xAF') == true
|
||||
// Example: assert is_hex_bin_oct_prefixed('xAF') == false
|
||||
fn is_hex_bin_oct_prefixed(hbo string) bool {
|
||||
return hbo.len > 2 && (hbo.starts_with('0x') || hbo.starts_with('0o') || hbo.starts_with('0b'))
|
||||
}
|
||||
|
||||
// has_repeating returns true if `str` has one or more repeating
|
||||
// `rune` characters provided in `repeats`.
|
||||
// Example: assert has_repeating('hello__v.', [`.`,`_`]) == true
|
||||
// Example: assert has_repeating('hello_v.', [`.`,`_`]) == false
|
||||
fn has_repeating(str string, repeats []rune) bool {
|
||||
for i, r in str {
|
||||
if r in repeats && i + 1 < str.len {
|
||||
if r == str[i + 1] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// check_number returns an error if `num` is not a valid TOML number.
|
||||
fn (c &Checker) check_number(num ast.Number) ! {
|
||||
lit := num.text
|
||||
lit_lower_case := lit.to_lower()
|
||||
if lit in ['0', '0.0', '+0', '-0', '+0.0', '-0.0', '0e0', '+0e0', '-0e0', '0e00'] {
|
||||
return
|
||||
}
|
||||
|
||||
if lit.contains('_') {
|
||||
if lit.starts_with('_') || lit.ends_with('_') {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" can not start or end with `_` in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
if lit.contains('__') {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" can not have more than one underscore (`_`) in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
}
|
||||
|
||||
mut hex_bin_oct := is_hex_bin_oct_prefixed(lit)
|
||||
mut is_bin, mut is_oct, mut is_hex := false, false, false
|
||||
is_float := lit_lower_case.all_before('e').contains('.')
|
||||
has_exponent_notation := lit_lower_case.contains('e')
|
||||
float_decimal_index := lit.index_('.')
|
||||
// mut is_first_digit := u8(lit[0]).is_digit()
|
||||
mut ascii := u8(lit[0]).ascii_str()
|
||||
is_sign_prefixed := lit[0] in [`+`, `-`]
|
||||
mut lit_sans_sign := lit
|
||||
if is_sign_prefixed { // +/- ...
|
||||
lit_sans_sign = lit[1..]
|
||||
hex_bin_oct = is_hex_bin_oct_prefixed(lit_sans_sign)
|
||||
if hex_bin_oct {
|
||||
ascii = u8(lit[0]).ascii_str()
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" (hex, octal and binary) can not start with `${ascii}` in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
if lit.len > 1 && lit_sans_sign.starts_with('0') && !lit_sans_sign.starts_with('0.') {
|
||||
ascii = u8(lit_sans_sign[0]).ascii_str()
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" can not start with `${ascii}` in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
} else {
|
||||
if !hex_bin_oct {
|
||||
if !is_float && lit[0] == `0` {
|
||||
if lit[1] in [`B`, `O`, `X`] {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" only lowercase notation in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" can not start with a zero in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
|
||||
if is_float && lit[0] == `0` && float_decimal_index > 1 {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" can not start with a zero in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if has_repeating(lit, [`_`, `.`, `b`, `o`, `x`]) {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" can not have ${scanner.digit_extras} as repeating characters in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
|
||||
if hex_bin_oct {
|
||||
is_bin = lit_sans_sign.starts_with('0b')
|
||||
is_oct = lit_sans_sign.starts_with('0o')
|
||||
is_hex = lit_sans_sign.starts_with('0x')
|
||||
|
||||
lit_sans_sign_and_type_prefix := lit_sans_sign[2..]
|
||||
|
||||
if lit_sans_sign_and_type_prefix.starts_with('_')
|
||||
|| lit_sans_sign_and_type_prefix.ends_with('_') {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" can not start or end with `_` in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
|
||||
if is_bin {
|
||||
if !c.is_valid_binary_literal(lit_sans_sign_and_type_prefix) {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" is not a valid binary number in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
} else if is_oct {
|
||||
if !c.is_valid_octal_literal(lit_sans_sign_and_type_prefix) {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" is not a valid octal number in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
} else {
|
||||
if !c.is_valid_hex_literal(lit_sans_sign_and_type_prefix) {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" is not a valid hexadecimal number in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if has_exponent_notation {
|
||||
if lit_lower_case.all_after('e').starts_with('_')
|
||||
|| lit_lower_case.all_before('e').ends_with('_') {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' the exponent in "${lit}" can not start nor end with an underscore in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
if lit_lower_case.all_after('e').contains('.') {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" (with exponent) can not have a decimal point in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
if !is_hex && lit_lower_case.count('e') > 1 {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" (with exponent) can only have one exponent in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
}
|
||||
|
||||
if is_float {
|
||||
if lit.count('.') > 1 {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" (float) can only have one decimal point in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
last := lit[lit.len - 1]
|
||||
if last in scanner.digit_extras {
|
||||
ascii = u8(last).ascii_str()
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" (float) can not end with `${ascii}` in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
if lit.contains('_.') || lit.contains('._') {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" (float) can not have underscores before or after the decimal point in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
if lit_lower_case.contains('e.') || lit.contains('.e') {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" (float) can not have decimal points on either side of the exponent notation in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
// Check if it contains other chars than the allowed
|
||||
for r in lit {
|
||||
if r !in [`0`, `1`, `2`, `3`, `4`, `5`, `6`, `7`, `8`, `9`, `.`, `e`, `E`, `-`, `+`,
|
||||
`_`] {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" (float) can not contain `${u8(r).ascii_str()}` in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if lit.len > 1 && lit.starts_with('0') && lit[1] !in [`b`, `o`, `x`] {
|
||||
ascii = u8(lit[0]).ascii_str()
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' numbers like "${lit}" can not start with `${ascii}` in ...${c.excerpt(num.pos)}...')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// is_valid_binary_literal returns true if `num` is valid TOML binary literal.
|
||||
fn (c &Checker) is_valid_binary_literal(num string) bool {
|
||||
for ch in num {
|
||||
if ch == `_` {
|
||||
continue
|
||||
}
|
||||
if !(ch >= `0` && ch <= `1`) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// is_valid_octal_literal returns true if `num` is valid TOML octal literal.
|
||||
fn (c &Checker) is_valid_octal_literal(num string) bool {
|
||||
for ch in num {
|
||||
if ch == `_` {
|
||||
continue
|
||||
}
|
||||
if !(ch >= `0` && ch <= `7`) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// is_valid_hex_literal returns true if `num` is valid TOML hexadecimal literal.
|
||||
fn (c &Checker) is_valid_hex_literal(num string) bool {
|
||||
for ch in num {
|
||||
if ch == `_` {
|
||||
continue
|
||||
}
|
||||
if !ch.is_hex_digit() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// check_boolean returns an error if `b` is not a valid TOML boolean.
|
||||
fn (c &Checker) check_boolean(b ast.Bool) ! {
|
||||
lit := b.text
|
||||
if lit in ['true', 'false'] {
|
||||
return
|
||||
}
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' boolean values like "${lit}" can only be `true` or `false` literals, not `${lit}` in ...${c.excerpt(b.pos)}...')
|
||||
}
|
||||
|
||||
// check_date_time returns an error if `dt` is not a valid TOML date-time string (RFC 3339).
|
||||
// See also https://ijmacd.github.io/rfc3339-iso8601 for a more
|
||||
// visual representation of the RFC 3339 format.
|
||||
fn (c &Checker) check_date_time(dt ast.DateTime) ! {
|
||||
lit := dt.text
|
||||
mut split := []string{}
|
||||
// RFC 3339 Date-Times can be split via 4 separators (` `, `_`, `T` and `t`).
|
||||
if lit.to_lower().contains_any(' _t') {
|
||||
if lit.contains(' ') {
|
||||
split = lit.split(' ')
|
||||
} else if lit.contains('_') {
|
||||
split = lit.split('_')
|
||||
} else if lit.contains('T') {
|
||||
split = lit.split('T')
|
||||
} else if lit.contains('t') {
|
||||
split = lit.split('t')
|
||||
}
|
||||
// Validate the split into date and time parts.
|
||||
if split.len != 2 {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" contains too many date/time separators in ...${c.excerpt(dt.pos)}...')
|
||||
}
|
||||
// Re-use date and time validation code for detailed testing of each part
|
||||
c.check_date(ast.Date{
|
||||
text: split[0]
|
||||
pos: token.Pos{
|
||||
len: split[0].len
|
||||
line_nr: dt.pos.line_nr
|
||||
pos: dt.pos.pos
|
||||
col: dt.pos.col
|
||||
}
|
||||
})!
|
||||
c.check_time(ast.Time{
|
||||
text: split[1]
|
||||
pos: token.Pos{
|
||||
len: split[1].len
|
||||
line_nr: dt.pos.line_nr
|
||||
pos: dt.pos.pos + split[0].len
|
||||
col: dt.pos.col + split[0].len
|
||||
}
|
||||
})!
|
||||
// Simulate a time offset if it's missing then it can be checked. Already toml supports local time and rfc3339 don't.
|
||||
mut has_time_offset := false
|
||||
for ch in lit#[19..] {
|
||||
if ch in [u8(`-`), `+`, `Z`] {
|
||||
has_time_offset = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
mut lit_with_offset := lit
|
||||
if !has_time_offset {
|
||||
lit_with_offset += 'Z'
|
||||
}
|
||||
|
||||
toml_parse_time(lit_with_offset) or {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" is not a valid RFC 3339 Date-Time format string "${err}". In ...${c.excerpt(dt.pos)}...')
|
||||
}
|
||||
} else {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" is not a valid RFC 3339 Date-Time format string in ...${c.excerpt(dt.pos)}...')
|
||||
}
|
||||
}
|
||||
|
||||
// check_time returns an error if `date` is not a valid TOML date string (RFC 3339).
|
||||
fn (c &Checker) check_date(date ast.Date) ! {
|
||||
lit := date.text
|
||||
parts := lit.split('-')
|
||||
if parts.len != 3 {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" is not a valid RFC 3339 Date format string in ...${c.excerpt(date.pos)}...')
|
||||
}
|
||||
yyyy := parts[0]
|
||||
if yyyy.len != 4 {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" does not have a valid RFC 3339 year indication in ...${c.excerpt(date.pos)}...')
|
||||
}
|
||||
mm := parts[1]
|
||||
if mm.len != 2 {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" does not have a valid RFC 3339 month indication in ...${c.excerpt(date.pos)}...')
|
||||
}
|
||||
dd := parts[2]
|
||||
if dd.len != 2 {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" does not have a valid RFC 3339 day indication in ...${c.excerpt(date.pos)}...')
|
||||
}
|
||||
if mm.int() == 2 {
|
||||
ddi := dd.int()
|
||||
if ddi > 28 {
|
||||
if ddi == 29 {
|
||||
yyyyi := yyyy.int()
|
||||
if !(yyyyi % 4 == 0 && (yyyyi % 100 != 0 || yyyyi % 400 == 0)) {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" is not a valid RFC 3339 date: ${yyyy} is not a leap year so February can not have 29 days in it ...${c.excerpt(date.pos)}...')
|
||||
}
|
||||
} else {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" is not a valid RFC 3339 date: February can not have more that 28 or 29 days in it ...${c.excerpt(date.pos)}...')
|
||||
}
|
||||
}
|
||||
}
|
||||
toml_parse_time(lit) or {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" is not a valid RFC 3339 Date format string "${err}". In ...${c.excerpt(date.pos)}...')
|
||||
}
|
||||
}
|
||||
|
||||
// check_time returns an error if `t` is not a valid TOML time string (RFC 3339).
|
||||
fn (c &Checker) check_time(t ast.Time) ! {
|
||||
lit := t.text
|
||||
// Split any offsets from the time
|
||||
mut offset_splitter := if lit.contains('+') { '+' } else { '-' }
|
||||
parts := lit.split(offset_splitter)
|
||||
mut hhmmss := parts[0].all_before('.')
|
||||
// Check for 2 digits in all fields
|
||||
mut check_length := 8
|
||||
if hhmmss.to_upper().ends_with('Z') {
|
||||
check_length++
|
||||
}
|
||||
if hhmmss.len != check_length {
|
||||
starts_with_zero := hhmmss.starts_with('0')
|
||||
if !starts_with_zero {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" must be zero prefixed in ...${c.excerpt(t.pos)}...')
|
||||
}
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" is not a valid RFC 3339 Time format string in ...${c.excerpt(t.pos)}...')
|
||||
}
|
||||
|
||||
if parts.len > 1 {
|
||||
// Offset
|
||||
offset_parts := parts[1].split(':')
|
||||
if offset_parts.len != 2 {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${parts[1]}" is not a valid RFC 3339 time offset specifier in ...${c.excerpt(t.pos)}...')
|
||||
}
|
||||
hh := offset_parts[0].int()
|
||||
if hh < 0 || hh > 24 {
|
||||
pos := token.Pos{
|
||||
...t.pos
|
||||
pos: t.pos.pos + check_length
|
||||
}
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${hh}" hour specifier in "${parts[1]}" should be between 00 and 24 in ...${c.excerpt(pos)}...')
|
||||
}
|
||||
mm := offset_parts[1].int()
|
||||
if mm < 0 || mm > 59 {
|
||||
pos := token.Pos{
|
||||
...t.pos
|
||||
pos: t.pos.pos + check_length
|
||||
}
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${mm}" second specifier in "${parts[1]}" should be between 00 and 59 in ...${c.excerpt(pos)}...')
|
||||
}
|
||||
}
|
||||
// Simulate a time offset if it's missing then it can be checked. Already toml supports local time and rfc3339 don't.
|
||||
mut has_time_offset := false
|
||||
for ch in parts[0]#[8..] {
|
||||
if ch in [u8(`-`), `+`, `Z`] {
|
||||
has_time_offset = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
mut part_with_offset := parts[0]
|
||||
if !has_time_offset {
|
||||
part_with_offset += 'Z'
|
||||
}
|
||||
|
||||
toml_parse_time(part_with_offset) or {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' "${lit}" is not a valid RFC 3339 Time format string "${err}". In ...${c.excerpt(t.pos)}...')
|
||||
}
|
||||
}
|
||||
|
||||
// check_quoted returns an error if `q` is not a valid quoted TOML string.
|
||||
pub fn (c &Checker) check_quoted(q ast.Quoted) ! {
|
||||
lit := q.text
|
||||
quote := q.quote.ascii_str()
|
||||
triple_quote := quote + quote + quote
|
||||
if q.is_multiline && lit.ends_with(triple_quote) && !lit.ends_with('\\' + triple_quote) {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' string values like "${lit}" has unbalanced quote literals `${quote}` in ...${c.excerpt(q.pos)}...')
|
||||
}
|
||||
c.check_quoted_escapes(q)!
|
||||
c.check_utf8_validity(q)!
|
||||
}
|
||||
|
||||
// check_quoted_escapes returns an error for any disallowed escape sequences.
|
||||
// Delimiters in TOML has significant meaning:
|
||||
// '/''' delimits *literal* strings (WYSIWYG / What-you-see-is-what-you-get)
|
||||
// "/""" delimits *basic* strings
|
||||
// Allowed escapes in *basic* strings are:
|
||||
// \b - backspace (U+0008)
|
||||
// \t - tab (U+0009)
|
||||
// \n - linefeed (U+000A)
|
||||
// \f - form feed (U+000C)
|
||||
// \r - carriage return (U+000D)
|
||||
// \" - quote (U+0022)
|
||||
// \\ - backslash (U+005C)
|
||||
// \uXXXX - Unicode (U+XXXX)
|
||||
// \UXXXXXXXX - Unicode (U+XXXXXXXX)
|
||||
fn (c &Checker) check_quoted_escapes(q ast.Quoted) ! {
|
||||
// Setup a scanner in stack memory for easier navigation.
|
||||
mut s := scanner.new_simple_text(q.text)!
|
||||
|
||||
// See https://toml.io/en/v1.0.0#string for more info on string types.
|
||||
is_basic := q.quote == `\"`
|
||||
contains_newlines := q.text.contains('\n')
|
||||
for {
|
||||
ch := s.next()
|
||||
if ch == scanner.end_of_text {
|
||||
break
|
||||
}
|
||||
ch_byte := u8(ch)
|
||||
if ch == `\\` {
|
||||
next_ch := u8(s.at())
|
||||
|
||||
if next_ch == `\\` {
|
||||
s.next()
|
||||
continue
|
||||
}
|
||||
|
||||
escape := ch_byte.ascii_str() + next_ch.ascii_str()
|
||||
if is_basic {
|
||||
if q.is_multiline {
|
||||
if next_ch == ` ` {
|
||||
if !contains_newlines {
|
||||
st := s.state()
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' can not escape whitespaces in multi-line strings (`\\ `) at `${escape}` (${st.line_nr},${st.col}) in ...${c.excerpt(q.pos)}...')
|
||||
}
|
||||
// Rest of line must only be space chars from this point on
|
||||
for {
|
||||
ch_ := s.next()
|
||||
if ch_ == `\n` {
|
||||
break
|
||||
}
|
||||
if !(ch_ == ` ` || ch_ == `\t`) {
|
||||
st := s.state()
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' invalid character `${u8(ch_).ascii_str()}` after `${escape}` at (${st.line_nr},${st.col}) in ...${c.excerpt(q.pos)}...')
|
||||
}
|
||||
}
|
||||
}
|
||||
if next_ch in [`\t`, `\r`, `\n`, ` `] {
|
||||
s.next()
|
||||
continue
|
||||
}
|
||||
}
|
||||
if next_ch !in allowed_basic_escape_char_list() {
|
||||
st := s.state()
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' unknown basic string escape character `${next_ch.ascii_str()}` in `${escape}` (${st.line_nr},${st.col}) in ...${c.excerpt(q.pos)}...')
|
||||
}
|
||||
}
|
||||
// Check Unicode escapes
|
||||
if is_basic && escape.to_lower() == '\\u' {
|
||||
// Long type Unicode (\UXXXXXXXX) is a maximum of 10 chars: '\' + 'U' + 8 hex characters
|
||||
// we pass in 10 characters from the `u`/`U` which is the longest possible sequence
|
||||
// of 9 chars plus one extra.
|
||||
if s.remaining() >= 10 {
|
||||
pos := s.state().pos
|
||||
c.check_unicode_escape(s.text[pos..pos + 11]) or {
|
||||
st := s.state()
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' escaped Unicode is invalid. ${err.msg().capitalize()} (${st.line_nr},${st.col}) in ...${c.excerpt(q.pos)}...')
|
||||
}
|
||||
} else {
|
||||
pos := s.state().pos
|
||||
c.check_unicode_escape(s.text[pos..]) or {
|
||||
st := s.state()
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' escaped Unicode is invalid. ${err.msg().capitalize()} (${st.line_nr},${st.col}) in ...${c.excerpt(q.pos)}...')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check_utf8_string returns an error if `str` is not valid UTF-8.
|
||||
fn (c &Checker) check_utf8_validity(q ast.Quoted) ! {
|
||||
lit := q.text
|
||||
if !utf8.validate_str(lit) {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' the string value "${lit}" is not valid UTF-8 in ...${c.excerpt(q.pos)}...')
|
||||
}
|
||||
}
|
||||
|
||||
// validate_utf8_codepoint_string returns an error if `str` is not a valid Unicode code point.
|
||||
// `str` is expected to be a `string` containing *only* hex values.
|
||||
// Any preludes or prefixes like `0x` could pontentially yield wrong results.
|
||||
fn validate_utf8_codepoint_string(str string) ! {
|
||||
int_val := strconv.parse_int(str, 16, 64) or { i64(-1) }
|
||||
if int_val > utf8_max || int_val < 0 {
|
||||
return error('Unicode code point `${str}` is outside the valid Unicode scalar value ranges.')
|
||||
}
|
||||
// Check if the Unicode value is actually in the valid Unicode scalar value ranges.
|
||||
// TODO: should probably be transferred / implemented in `utf8.validate(...)` also?
|
||||
if !((int_val >= 0x0000 && int_val <= 0xD7FF) || (int_val >= 0xE000 && int_val <= 0x10FFFF)) {
|
||||
return error('Unicode code point `${str}` is not a valid Unicode scalar value.')
|
||||
}
|
||||
bytes := str.bytes()
|
||||
if !utf8.validate(bytes.data, bytes.len) {
|
||||
return error('Unicode code point `${str}` is not a valid UTF-8 code point.')
|
||||
}
|
||||
}
|
||||
|
||||
// check_unicode_escape returns an error if `esc_unicode` is not
|
||||
// a valid Unicode escape sequence. `esc_unicode` is expected to be
|
||||
// prefixed with either `u` or `U`.
|
||||
fn (c &Checker) check_unicode_escape(esc_unicode string) ! {
|
||||
if esc_unicode.len < 5 || !esc_unicode.to_lower().starts_with('u') {
|
||||
// Makes sure the input to this function is actually valid.
|
||||
return error('`${esc_unicode}` is not a valid escaped Unicode sequence.')
|
||||
}
|
||||
is_long_esc_type := esc_unicode.starts_with('U')
|
||||
mut sequence := esc_unicode[1..]
|
||||
hex_digits_len := if is_long_esc_type { 8 } else { 4 }
|
||||
if sequence.len < hex_digits_len {
|
||||
return error('Unicode escape sequence `${esc_unicode}` should be at least ${hex_digits_len} in length.')
|
||||
}
|
||||
sequence = sequence[..hex_digits_len]
|
||||
// TODO: not enforced in BurnSushi testsuite??
|
||||
// if !sequence.is_upper() {
|
||||
// return error('Unicode escape sequence `${esc_unicode}` is not in all uppercase.')
|
||||
//}
|
||||
validate_utf8_codepoint_string(sequence.to_upper())!
|
||||
if is_long_esc_type {
|
||||
// Long escape type checks
|
||||
} else {
|
||||
// Short escape type checks
|
||||
}
|
||||
}
|
||||
|
||||
// check_comment returns an error if the contents of `comment` isn't
|
||||
// a valid TOML comment.
|
||||
pub fn (c &Checker) check_comment(comment ast.Comment) ! {
|
||||
lit := comment.text
|
||||
// Setup a scanner in stack memory for easier navigation.
|
||||
mut s := scanner.new_simple_text(lit)!
|
||||
for {
|
||||
ch := s.next()
|
||||
if ch == scanner.end_of_text {
|
||||
break
|
||||
}
|
||||
ch_byte := u8(ch)
|
||||
// Check for carriage return
|
||||
if ch_byte == 0x0D {
|
||||
st := s.state()
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' carriage return character `${ch_byte.hex()}` is not allowed in comments (${st.line_nr},${st.col}).')
|
||||
}
|
||||
// Check for control characters (allow TAB)
|
||||
if util.is_illegal_ascii_control_character(ch_byte) {
|
||||
st := s.state()
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' control character `${ch_byte.hex()}` is not allowed (${st.line_nr},${st.col}) "${u8(s.at()).ascii_str()}" near ...${s.excerpt(st.pos, 10)}...')
|
||||
}
|
||||
}
|
||||
|
||||
// Check for bad UTF-8 encoding
|
||||
if !utf8.validate_str(lit) {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' comment "${lit}" is not valid UTF-8 in ...${c.excerpt(comment.pos)}...')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module decoder
|
||||
|
||||
import toml.ast
|
||||
import toml.ast.walker
|
||||
import toml.token
|
||||
import toml.scanner
|
||||
import strconv
|
||||
|
||||
// utf8_max is the largest inclusive value of the Unicodes scalar value ranges.
|
||||
const utf8_max = 0x10FFFF
|
||||
|
||||
// Decoder decode special sequences in a tree of TOML `ast.Value`'s.
|
||||
pub struct Decoder {
|
||||
pub:
|
||||
scanner &scanner.Scanner = unsafe { nil }
|
||||
}
|
||||
|
||||
// decode decodes certain `ast.Value`'s and all it's children.
|
||||
pub fn (d Decoder) decode(mut n ast.Value) ! {
|
||||
walker.walk_and_modify(d, mut n)!
|
||||
}
|
||||
|
||||
fn (d Decoder) modify(mut value ast.Value) ! {
|
||||
match value {
|
||||
ast.Quoted {
|
||||
mut v := &(value as ast.Quoted)
|
||||
d.decode_quoted(mut v)!
|
||||
}
|
||||
ast.Number {
|
||||
mut v := &(value as ast.Number)
|
||||
d.decode_number(mut v)!
|
||||
}
|
||||
ast.DateTime {
|
||||
mut v := &(value as ast.DateTime)
|
||||
d.decode_date_time(mut v)!
|
||||
}
|
||||
else {}
|
||||
}
|
||||
}
|
||||
|
||||
// excerpt returns a string of the token's surroundings
|
||||
fn (d Decoder) excerpt(tp token.Pos) string {
|
||||
return d.scanner.excerpt(tp.pos, 10)
|
||||
}
|
||||
|
||||
// decode_quoted returns an error if `q` is not a valid quoted TOML string.
|
||||
fn (d Decoder) decode_quoted(mut q ast.Quoted) ! {
|
||||
decode_quoted_escapes(mut q)!
|
||||
}
|
||||
|
||||
// decode_number decodes the `n ast.Number` into valid TOML.
|
||||
fn (d Decoder) decode_number(mut n ast.Number) ! {
|
||||
if n.text == '-nan' || n.text == '+nan' {
|
||||
n.text = 'nan'
|
||||
}
|
||||
}
|
||||
|
||||
// decode_quoted_escapes returns an error for any disallowed escape sequences.
|
||||
// Delimiters in TOML has significant meaning:
|
||||
// '/''' delimits *literal* strings (WYSIWYG / What-you-see-is-what-you-get)
|
||||
// "/""" delimits *basic* strings
|
||||
// Allowed escapes in *basic* strings are:
|
||||
// \b - backspace (U+0008)
|
||||
// \t - tab (U+0009)
|
||||
// \n - linefeed (U+000A)
|
||||
// \f - form feed (U+000C)
|
||||
// \r - carriage return (U+000D)
|
||||
// \" - quote (U+0022)
|
||||
// \\ - backslash (U+005C)
|
||||
// \uXXXX - Unicode (U+XXXX)
|
||||
// \UXXXXXXXX - Unicode (U+XXXXXXXX)
|
||||
pub fn decode_quoted_escapes(mut q ast.Quoted) ! {
|
||||
// Setup a scanner in stack memory for easier navigation.
|
||||
mut eat_whitespace := false
|
||||
// TODO: use string builder
|
||||
mut decoded_s := ''
|
||||
// See https://toml.io/en/v1.0.0#string for more info on string types.
|
||||
is_basic := q.quote == `\"`
|
||||
if !is_basic {
|
||||
return
|
||||
}
|
||||
|
||||
mut s := scanner.new_simple_text(q.text)!
|
||||
q.text = q.text.replace('\\"', '"')
|
||||
|
||||
for ch := s.next(); ch != scanner.end_of_text; ch = s.next() {
|
||||
ch_byte := u8(ch)
|
||||
if eat_whitespace && ch_byte.is_space() {
|
||||
continue
|
||||
}
|
||||
eat_whitespace = false
|
||||
|
||||
if ch == `\\` {
|
||||
ch_next := s.at()
|
||||
ch_next_byte := u8(ch_next)
|
||||
|
||||
if q.is_multiline {
|
||||
if ch_next_byte.is_space() {
|
||||
eat_whitespace = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
match rune(ch_next) {
|
||||
`\\`, `"` {
|
||||
decoded_s += ch_next_byte.ascii_str()
|
||||
s.next()
|
||||
continue
|
||||
}
|
||||
`n` {
|
||||
decoded_s += '\n'
|
||||
s.next()
|
||||
continue
|
||||
}
|
||||
`t` {
|
||||
decoded_s += '\t'
|
||||
s.next()
|
||||
continue
|
||||
}
|
||||
`b` {
|
||||
decoded_s += '\b'
|
||||
s.next()
|
||||
continue
|
||||
}
|
||||
`r` {
|
||||
decoded_s += '\r'
|
||||
s.next()
|
||||
continue
|
||||
}
|
||||
`f` {
|
||||
decoded_s += '\f'
|
||||
s.next()
|
||||
continue
|
||||
}
|
||||
else {}
|
||||
}
|
||||
|
||||
escape := ch_byte.ascii_str() + ch_next_byte.ascii_str()
|
||||
// Decode unicode escapes
|
||||
if escape.to_lower() == '\\u' {
|
||||
is_valid_short := u8(s.peek(1)).is_hex_digit() && u8(s.peek(2)).is_hex_digit()
|
||||
&& u8(s.peek(3)).is_hex_digit() && u8(s.peek(4)).is_hex_digit()
|
||||
|
||||
if is_valid_short {
|
||||
is_valid_long := u8(s.peek(5)).is_hex_digit() && u8(s.peek(6)).is_hex_digit()
|
||||
&& u8(s.peek(7)).is_hex_digit() && u8(s.peek(8)).is_hex_digit()
|
||||
// If it's a long type Unicode (\UXXXXXXXX) with a maximum of 10 chars: '\' + 'U' + 8 hex characters
|
||||
// we pass in 10 characters from the `u`/`U` which is the longest possible sequence
|
||||
// of 9 chars plus one extra.
|
||||
// Else it's a short sequence (\uXXXX) with a maximum of 6 chars: '\' + 'U' + 4 hex characters.
|
||||
mut decoded := ''
|
||||
mut sequence_length := 0
|
||||
mut unicode_val := 0
|
||||
mut slen := if is_valid_long { 10 } else { 6 }
|
||||
if slen <= s.remaining() {
|
||||
pos := s.state().pos
|
||||
sequence := s.text#[pos..pos + slen + 1]
|
||||
decoded, unicode_val, sequence_length = decode_unicode_escape(sequence) or {
|
||||
decoded_s += escape
|
||||
continue
|
||||
}
|
||||
if unicode_val > utf8_max || unicode_val < 0 {
|
||||
decoded_s += escape
|
||||
continue
|
||||
}
|
||||
// Check if the Unicode value is actually in the valid Unicode scalar value ranges.
|
||||
if !((unicode_val >= 0x0000 && unicode_val <= 0xD7FF)
|
||||
|| (unicode_val >= 0xE000 && unicode_val <= utf8_max)) {
|
||||
decoded_s += escape
|
||||
continue
|
||||
}
|
||||
decoded_s += decoded
|
||||
replacement := s.text[pos..pos + sequence_length + 1]
|
||||
s.skip_n(replacement.len)
|
||||
continue
|
||||
} else {
|
||||
pos := s.state().pos
|
||||
sequence := s.text[pos..]
|
||||
decoded, _, _ = decode_unicode_escape(sequence) or {
|
||||
decoded_s += escape
|
||||
continue
|
||||
}
|
||||
decoded_s += decoded
|
||||
s.skip_n(s.text[pos..].len)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
decoded_s += ch_byte.ascii_str()
|
||||
}
|
||||
q.text = decoded_s
|
||||
}
|
||||
|
||||
// decode_unicode_escape decodes the Unicode escape sequence `esc_unicode`.
|
||||
// The sequence is expected to be prefixed with either `u` or `U`.
|
||||
// decode_unicode_escape returns the decoded rune as
|
||||
// a string, it's integer value and it's length.
|
||||
fn decode_unicode_escape(esc_unicode string) !(string, int, int) {
|
||||
is_long_esc_type := esc_unicode.starts_with('U')
|
||||
mut sequence := esc_unicode[1..]
|
||||
hex_digits_len := if is_long_esc_type { 8 } else { 4 }
|
||||
mut sequence_len := hex_digits_len
|
||||
|
||||
sequence = sequence[..hex_digits_len]
|
||||
|
||||
mut unicode_point := sequence
|
||||
if unicode_point.len < 8 {
|
||||
unicode_point = '0'.repeat(8 - unicode_point.len) + unicode_point
|
||||
}
|
||||
i64_val := strconv.parse_int(unicode_point, 16, 0)!
|
||||
rn := rune(i64_val)
|
||||
return '${rn}', int(i64_val), sequence_len
|
||||
}
|
||||
|
||||
// decode_date_time decodes the `dt ast.DateTime`.
|
||||
fn (d Decoder) decode_date_time(mut dt ast.DateTime) ! {
|
||||
// Expand milliseconds that are only 1 char
|
||||
if dt.text.contains('.') {
|
||||
yymmddhhmmss := dt.text.all_before('.')
|
||||
rest := dt.text.all_after('.')
|
||||
z := if rest.contains('Z') { 'Z' } else { '' }
|
||||
mut ms := rest
|
||||
mut offset := ''
|
||||
if rest.contains('+') {
|
||||
offset = '+' + rest.all_after('+')
|
||||
ms = rest.all_before('+')
|
||||
} else if rest.contains('-') {
|
||||
offset = '-' + rest.all_after('-')
|
||||
ms = rest.all_before('-')
|
||||
}
|
||||
if z != '' {
|
||||
ms = ms.replace('Z', '')
|
||||
}
|
||||
if ms.len > 1 {
|
||||
return
|
||||
}
|
||||
ms = ms + '0'.repeat(4 - ms.len) + z
|
||||
dt.text = yymmddhhmmss + '.' + ms + offset
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module input
|
||||
|
||||
import os
|
||||
|
||||
// Config is used to configure input to the toml module.
|
||||
// Only one of the fields `text` and `file_path` is allowed to be set at time of configuration.
|
||||
pub struct Config {
|
||||
pub:
|
||||
text string // TOML text
|
||||
file_path string // '/path/to/file.toml'
|
||||
}
|
||||
|
||||
// read_input returns either Config.text or the read file contents of Config.file_path
|
||||
// depending on which one is not empty.
|
||||
pub fn (c Config) read_input() !string {
|
||||
if c.file_path != '' && c.text != '' {
|
||||
return error(@MOD + '.' + @FN +
|
||||
' ${typeof(c).name} should contain only one of the fields `file_path` OR `text` filled out')
|
||||
}
|
||||
if c.file_path == '' && c.text == '' {
|
||||
// TODO: passing both empty is used *a lot* by `./v vlib/toml/tests/toml_lang_test.v`; investigate why.
|
||||
return ''
|
||||
}
|
||||
if c.text != '' {
|
||||
return c.text
|
||||
}
|
||||
text := os.read_file(c.file_path) or {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' Could not read "${c.file_path}": "${err.msg()}"')
|
||||
}
|
||||
return text
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,741 @@
|
||||
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module scanner
|
||||
|
||||
import toml.input
|
||||
import toml.token
|
||||
import toml.util
|
||||
|
||||
pub const digit_extras = [`_`, `.`, `x`, `o`, `b`, `e`, `E`]
|
||||
pub const end_of_text = u32(~0)
|
||||
|
||||
// Scanner contains the necessary fields for the state of the scan process.
|
||||
// the task the scanner does is also referred to as "lexing" or "tokenizing".
|
||||
// The Scanner methods are based on much of the work in `vlib/strings/textscanner`.
|
||||
pub struct Scanner {
|
||||
pub:
|
||||
config Config
|
||||
text string // the input TOML text
|
||||
mut:
|
||||
col int // current column number (x coordinate)
|
||||
line_nr int = 1 // current line number (y coordinate)
|
||||
pos int // current flat/index position in the `text` field
|
||||
header_len int // Length, how many bytes of header was found
|
||||
// Quirks
|
||||
is_left_of_assign bool = true // indicates if the scanner is on the *left* side of an assignment
|
||||
}
|
||||
|
||||
// State is a read-only copy of the scanner's internal state.
|
||||
// See also `Scanner.state()`.
|
||||
pub struct State {
|
||||
pub:
|
||||
col int // current column number (x coordinate)
|
||||
line_nr int = 1 // current line number (y coordinate)
|
||||
pos int // current flat/index position in the `text` field
|
||||
}
|
||||
|
||||
// Config is used to configure a Scanner instance.
|
||||
// Only one of the fields `text` and `file_path` is allowed to be set at time of configuration.
|
||||
pub struct Config {
|
||||
pub:
|
||||
input input.Config
|
||||
tokenize_formatting bool = true // if true, generate tokens for `\n`, ` `, `\t`, `\r` etc.
|
||||
}
|
||||
|
||||
// new_scanner returns a new *heap* allocated `Scanner` instance, based on the file in config.input.file_path,
|
||||
// or based on the text in config.input.text .
|
||||
pub fn new_scanner(config Config) !&Scanner {
|
||||
mut s := &Scanner{
|
||||
config: config
|
||||
text: config.input.read_input()!
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// new_simple returns a new *stack* allocated `Scanner` instance.
|
||||
pub fn new_simple(config Config) !Scanner {
|
||||
return Scanner{
|
||||
config: config
|
||||
text: config.input.read_input()!
|
||||
}
|
||||
}
|
||||
|
||||
// new_simple_text returns a new *stack* allocated `Scanner` instance
|
||||
// ready for parsing TOML in `text`.
|
||||
pub fn new_simple_text(text string) !Scanner {
|
||||
in_config := input.Config{
|
||||
text: text
|
||||
}
|
||||
config := Config{
|
||||
input: in_config
|
||||
}
|
||||
return Scanner{
|
||||
config: config
|
||||
text: config.input.read_input()!
|
||||
}
|
||||
}
|
||||
|
||||
// new_simple_file returns a new *stack* allocated `Scanner` instance
|
||||
// ready for parsing TOML in file read from `path`.
|
||||
pub fn new_simple_file(path string) !Scanner {
|
||||
in_config := input.Config{
|
||||
file_path: path
|
||||
}
|
||||
config := Config{
|
||||
input: in_config
|
||||
}
|
||||
return Scanner{
|
||||
config: config
|
||||
text: config.input.read_input()!
|
||||
}
|
||||
}
|
||||
|
||||
// scan returns the next token from the input.
|
||||
@[direct_array_access]
|
||||
pub fn (mut s Scanner) scan() !token.Token {
|
||||
s.validate_and_skip_headers()!
|
||||
|
||||
for {
|
||||
c := s.next()
|
||||
byte_c := u8(c)
|
||||
if c == end_of_text {
|
||||
s.inc_line_number()
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'reached EOF')
|
||||
return s.new_token(.eof, '', 1)
|
||||
}
|
||||
|
||||
ascii := byte_c.ascii_str()
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'current char "${ascii}"')
|
||||
|
||||
if byte_c == u8(0x0) {
|
||||
s.reset()
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' NULL control character `${c.hex()}` is not allowed at (${s.line_nr},${s.col}) "${ascii}" near ...${s.excerpt(s.pos, 5)}...')
|
||||
}
|
||||
|
||||
is_sign := c == `+` || c == `-`
|
||||
|
||||
// (+/-)nan & (+/-)inf
|
||||
peek_1 := s.peek(1)
|
||||
peek_2 := s.peek(2)
|
||||
is_nan := c == `n` && s.at() == `a` && peek_1 == `n`
|
||||
is_inf := !is_nan && c == `i` && s.at() == `n` && peek_1 == `f`
|
||||
is_signed_nan := is_sign && s.at() == `n` && peek_1 == `a` && peek_2 == `n`
|
||||
is_signed_inf := !is_signed_nan && is_sign && s.at() == `i` && peek_1 == `n`
|
||||
&& peek_2 == `f`
|
||||
if !s.is_left_of_assign && (is_nan || is_inf || is_signed_nan || is_signed_inf) {
|
||||
num := s.extract_nan_or_inf_number()!
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified a special number "${num}" (${num.len})')
|
||||
return s.new_token(.number, num, num.len)
|
||||
}
|
||||
|
||||
is_signed_number := is_sign && u8(s.at()).is_digit() && !u8(s.peek(-1)).is_digit()
|
||||
is_digit := byte_c.is_digit()
|
||||
if is_digit || is_signed_number {
|
||||
num := s.extract_number()!
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified a number "${num}" (${num.len})')
|
||||
return s.new_token(.number, num, num.len)
|
||||
}
|
||||
|
||||
if util.is_key_char(byte_c) {
|
||||
key := s.extract_key()
|
||||
if u8(s.peek(1)) != `=` && (key == 'true' || key == 'false') {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified a boolean "${key}" (${key.len})')
|
||||
return s.new_token(.boolean, key, key.len)
|
||||
}
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified a bare key "${key}" (${key.len})')
|
||||
return s.new_token(.bare, key, key.len)
|
||||
}
|
||||
|
||||
match rune(c) {
|
||||
` `, `\t`, `\n`, `\r` {
|
||||
if c == `\n` {
|
||||
s.inc_line_number()
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'incremented line nr to ${s.line_nr}')
|
||||
} else if c == `\r` {
|
||||
// CR should always be followed by a `\n`
|
||||
if s.at() != `\n` {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' missing newline/linefeed character after "\\c" carriage return at (${s.line_nr},${s.col}) "${ascii}" near ...${s.excerpt(s.pos, 5)}...')
|
||||
}
|
||||
}
|
||||
// Date-Time in RFC 3339 is allowed to have a space between the date and time in supplement to the 'T'
|
||||
// so we allow space characters to slip through to the parser if the space is between two digits...
|
||||
// util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, '"'+u8(s.peek(-1)).ascii_str()+'" < "${ascii}" > "'+u8(s.at()).ascii_str()+'"')
|
||||
if c == ` ` && u8(s.peek(-1)).is_digit() && u8(s.at()).is_digit() {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified, what could be, a space between a RFC 3339 date and time ("${ascii}") (${ascii.len})')
|
||||
return s.new_token(token.Kind.whitespace, ascii, ascii.len)
|
||||
}
|
||||
if s.config.tokenize_formatting {
|
||||
mut kind := token.Kind.whitespace
|
||||
if c == `\t` {
|
||||
kind = token.Kind.tab
|
||||
} else if c == `\r` {
|
||||
kind = token.Kind.cr
|
||||
} else if c == `\n` {
|
||||
kind = token.Kind.nl
|
||||
}
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified formatting character ("${ascii}") (${ascii.len})')
|
||||
return s.new_token(kind, ascii, ascii.len)
|
||||
} else {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'skipping " ", "\\t" or "\\n" ("${ascii}") (${ascii.len})')
|
||||
}
|
||||
continue
|
||||
}
|
||||
`-` {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified minus "${ascii}" (${ascii.len})')
|
||||
return s.new_token(.minus, ascii, ascii.len)
|
||||
}
|
||||
`_` {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified underscore "${ascii}" (${ascii.len})')
|
||||
return s.new_token(.underscore, ascii, ascii.len)
|
||||
}
|
||||
`+` {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified plus "${ascii}" (${ascii.len})')
|
||||
return s.new_token(.plus, ascii, ascii.len)
|
||||
}
|
||||
`=` {
|
||||
s.is_left_of_assign = false
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified assignment "${ascii}" (${ascii.len})')
|
||||
return s.new_token(.assign, ascii, ascii.len)
|
||||
}
|
||||
`"`, `'` { // ... some string "/'
|
||||
ident_string := s.extract_string()!
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified quoted string `${ident_string}`')
|
||||
return s.new_token(.quoted, ident_string, ident_string.len)
|
||||
}
|
||||
`#` {
|
||||
hash := s.ignore_line()!
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified comment hash "${hash}" (${hash.len})')
|
||||
return s.new_token(.hash, hash, hash.len + 1)
|
||||
}
|
||||
`{` {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified left curly bracket "${ascii}" (${ascii.len})')
|
||||
return s.new_token(.lcbr, ascii, ascii.len)
|
||||
}
|
||||
`}` {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified right curly bracket "${ascii}" (${ascii.len})')
|
||||
return s.new_token(.rcbr, ascii, ascii.len)
|
||||
}
|
||||
`[` {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified left square bracket "${ascii}" (${ascii.len})')
|
||||
return s.new_token(.lsbr, ascii, ascii.len)
|
||||
}
|
||||
`]` {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified right square bracket "${ascii}" (${ascii.len})')
|
||||
return s.new_token(.rsbr, ascii, ascii.len)
|
||||
}
|
||||
`:` {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified colon "${ascii}" (${ascii.len})')
|
||||
return s.new_token(.colon, ascii, ascii.len)
|
||||
}
|
||||
`,` {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified comma "${ascii}" (${ascii.len})')
|
||||
return s.new_token(.comma, ascii, ascii.len)
|
||||
}
|
||||
`.` {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified period "${ascii}" (${ascii.len})')
|
||||
return s.new_token(.period, ascii, ascii.len)
|
||||
}
|
||||
else {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' could not scan character `${ascii}` / ${c} at ${s.pos} (${s.line_nr},${s.col}) near ...${s.excerpt(s.pos, 5)}...')
|
||||
}
|
||||
}
|
||||
}
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'unknown character code at ${s.pos} (${s.line_nr},${s.col}) near ...${s.excerpt(s.pos,
|
||||
5)}...')
|
||||
return s.new_token(.unknown, '', 0)
|
||||
}
|
||||
|
||||
// free frees all allocated resources.
|
||||
@[unsafe]
|
||||
pub fn (mut s Scanner) free() {
|
||||
unsafe {
|
||||
s.text.free()
|
||||
}
|
||||
}
|
||||
|
||||
// remaining returns how many characters remain in the text input.
|
||||
@[inline]
|
||||
pub fn (s &Scanner) remaining() int {
|
||||
return s.text.len - s.pos
|
||||
}
|
||||
|
||||
// next returns the next character code from the input text.
|
||||
// next returns `end_of_text` if it can't reach the next character.
|
||||
@[direct_array_access; inline]
|
||||
pub fn (mut s Scanner) next() u32 {
|
||||
if s.pos < s.text.len {
|
||||
opos := s.pos
|
||||
s.pos++
|
||||
s.col++
|
||||
c := s.text[opos]
|
||||
return c
|
||||
}
|
||||
return end_of_text
|
||||
}
|
||||
|
||||
// skip skips one character ahead.
|
||||
@[inline]
|
||||
pub fn (mut s Scanner) skip() {
|
||||
if s.pos + 1 < s.text.len {
|
||||
s.pos++
|
||||
s.col++
|
||||
}
|
||||
}
|
||||
|
||||
// skip_n skips ahead `n` characters.
|
||||
// If the skip goes out of bounds from the length of `Scanner.text`,
|
||||
// the scanner position will be sat to the last character possible.
|
||||
@[inline]
|
||||
pub fn (mut s Scanner) skip_n(n int) {
|
||||
s.pos += n
|
||||
if s.pos > s.text.len {
|
||||
s.pos = s.text.len
|
||||
}
|
||||
s.col = s.pos
|
||||
}
|
||||
|
||||
// at returns the *current* character code from the input text.
|
||||
// at returns `end_of_text` if it can't get the current character.
|
||||
// unlike `next()`, `at()` does not change the state of the scanner.
|
||||
@[direct_array_access; inline]
|
||||
pub fn (s &Scanner) at() u32 {
|
||||
if s.pos < s.text.len {
|
||||
return s.text[s.pos]
|
||||
}
|
||||
return end_of_text
|
||||
}
|
||||
|
||||
// at_crlf returns `true` if the scanner is at a `\r` character
|
||||
// and the next character is a `\n`.
|
||||
fn (s &Scanner) at_crlf() bool {
|
||||
return s.at() == `\r` && s.peek(1) == `\n`
|
||||
}
|
||||
|
||||
// peek returns the character code from the input text at position + `n`.
|
||||
// peek returns `end_of_text` if it can't peek `n` characters ahead.
|
||||
@[direct_array_access; inline]
|
||||
pub fn (s &Scanner) peek(n int) u32 {
|
||||
if s.pos + n < s.text.len {
|
||||
// Allow peeking back - needed for spaces between date and time in RFC 3339 format :/
|
||||
if n - 1 < 0 && s.pos + n - 1 >= 0 {
|
||||
// util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'LOOKING BAAAA-AACK - OOOVER MY SHOOOOULDEEEER "${s.text[s.pos + n-1]}"')
|
||||
return s.text[s.pos + n - 1]
|
||||
}
|
||||
return s.text[s.pos + n]
|
||||
}
|
||||
return end_of_text
|
||||
}
|
||||
|
||||
// reset resets the internal state of the scanner.
|
||||
pub fn (mut s Scanner) reset() {
|
||||
s.pos = 0
|
||||
s.col = 0
|
||||
s.line_nr = 1
|
||||
s.header_len = 0
|
||||
}
|
||||
|
||||
// new_token returns a new `token.Token`.
|
||||
@[inline]
|
||||
fn (mut s Scanner) new_token(kind token.Kind, lit string, len int) token.Token {
|
||||
// println('new_token(${lit})')
|
||||
mut col := s.col - len + 1
|
||||
if s.line_nr == 1 {
|
||||
col -= s.header_len
|
||||
}
|
||||
return token.Token{
|
||||
kind: kind
|
||||
lit: lit
|
||||
col: if col < 1 { 1 } else { col }
|
||||
line_nr: s.line_nr + 1
|
||||
pos: s.pos - s.header_len - len + 1
|
||||
len: len
|
||||
}
|
||||
}
|
||||
|
||||
// ignore_line forwards the scanner to the end of the current line.
|
||||
@[direct_array_access; inline]
|
||||
fn (mut s Scanner) ignore_line() !string {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, ' ignoring until EOL...')
|
||||
start := s.pos
|
||||
for c := s.at(); c != end_of_text && c != `\n`; c = s.at() {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'skipping "${u8(c).ascii_str()} / ${c}"')
|
||||
if s.at_crlf() {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'letting `\\r\\n` slip through')
|
||||
break
|
||||
}
|
||||
s.next()
|
||||
}
|
||||
return s.text[start..s.pos]
|
||||
}
|
||||
|
||||
// inc_line_number increases the internal line number.
|
||||
@[inline]
|
||||
fn (mut s Scanner) inc_line_number() {
|
||||
s.col = 0
|
||||
s.line_nr++
|
||||
s.is_left_of_assign = true
|
||||
}
|
||||
|
||||
// extract_key parses and returns a TOML key as a string.
|
||||
@[direct_array_access; inline]
|
||||
fn (mut s Scanner) extract_key() string {
|
||||
s.pos--
|
||||
s.col--
|
||||
start := s.pos
|
||||
for s.pos < s.text.len {
|
||||
c := u8(s.at())
|
||||
if !(util.is_key_char(c) || c.is_digit() || c in [`_`, `-`]) {
|
||||
break
|
||||
}
|
||||
s.pos++
|
||||
s.col++
|
||||
}
|
||||
key := s.text[start..s.pos]
|
||||
return key
|
||||
}
|
||||
|
||||
// extract_string collects and returns a string containing
|
||||
// any bytes recognized as a TOML string.
|
||||
// TOML strings are everything found between two double or single quotation marks (`"`/`'`).
|
||||
@[direct_array_access; inline]
|
||||
fn (mut s Scanner) extract_string() !string {
|
||||
// extract_string is called when the scanner has already reached
|
||||
// a byte that is the start of a string so we rewind it to start at the correct
|
||||
s.pos--
|
||||
s.col--
|
||||
quote := u8(s.at())
|
||||
start := s.pos
|
||||
mut lit := quote.ascii_str()
|
||||
|
||||
is_multiline := s.text[s.pos + 1] == quote && s.text[s.pos + 2] == quote
|
||||
// Check for escaped multiline quote
|
||||
if is_multiline {
|
||||
mls := s.extract_multiline_string()!
|
||||
return mls
|
||||
}
|
||||
|
||||
for {
|
||||
s.pos++
|
||||
s.col++
|
||||
|
||||
if s.pos >= s.text.len {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' unfinished single-line string literal `${quote.ascii_str()}` started at ${start} (${s.line_nr},${s.col}) "${u8(s.at()).ascii_str()}" near ...${s.excerpt(s.pos, 5)}...')
|
||||
}
|
||||
|
||||
c := u8(s.at())
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'c: `${c.ascii_str()}` / ${c} (quote type: ${quote}/${quote.ascii_str()})')
|
||||
|
||||
// Check for escaped chars
|
||||
if c == u8(92) {
|
||||
esc, skip := s.handle_escapes(quote, is_multiline)
|
||||
lit += esc
|
||||
if skip > 0 {
|
||||
s.pos += skip
|
||||
s.col += skip
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Check for control characters (allow TAB)
|
||||
if util.is_illegal_ascii_control_character(c) {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' control character `${c.hex()}` is not allowed at ${start} (${s.line_nr},${s.col}) "${u8(s.at()).ascii_str()}" near ...${s.excerpt(s.pos, 5)}...')
|
||||
}
|
||||
|
||||
if c == quote {
|
||||
s.pos++
|
||||
s.col++
|
||||
return lit + quote.ascii_str()
|
||||
}
|
||||
|
||||
lit += c.ascii_str()
|
||||
|
||||
// Don't eat multiple lines in single-line mode
|
||||
if lit.contains('\n') {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' unfinished single-line string literal `${quote.ascii_str()}` started at ${start} (${s.line_nr},${s.col}) "${u8(s.at()).ascii_str()}" near ...${s.excerpt(s.pos, 5)}...')
|
||||
}
|
||||
}
|
||||
return lit
|
||||
}
|
||||
|
||||
// extract_multiline_string collects and returns a string containing
|
||||
// any bytes recognized as a TOML string.
|
||||
// TOML strings are everything found between two double or single quotation marks (`"`/`'`).
|
||||
@[direct_array_access; inline]
|
||||
fn (mut s Scanner) extract_multiline_string() !string {
|
||||
// extract_multiline_string is called from extract_string so we know the 3 first
|
||||
// characters is the quotes
|
||||
quote := u8(s.at())
|
||||
start := s.pos
|
||||
mut lit := quote.ascii_str() + quote.ascii_str() + quote.ascii_str()
|
||||
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'multi-line `${quote.ascii_str()}${s.text[
|
||||
s.pos + 1].ascii_str()}${s.text[s.pos + 2].ascii_str()}` string started at pos ${start} (${s.line_nr},${s.col}) (quote type: ${quote.ascii_str()} / ${quote})')
|
||||
|
||||
s.pos += 2
|
||||
s.col += 2
|
||||
|
||||
for {
|
||||
s.pos++
|
||||
s.col++
|
||||
|
||||
if s.pos >= s.text.len {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' unfinished multi-line string literal (${quote.ascii_str()}${quote.ascii_str()}${quote.ascii_str()}) started at ${start} (${s.line_nr},${s.col}) "${u8(s.at()).ascii_str()}" near ...${s.excerpt(s.pos, 5)}...')
|
||||
}
|
||||
|
||||
c := u8(s.at())
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'c: `${c.ascii_str()}` / ${c} (quote type: ${quote}/${quote.ascii_str()})')
|
||||
|
||||
if c == `\r` && s.peek(1) == `\n` {
|
||||
continue
|
||||
}
|
||||
if c == `\n` {
|
||||
s.inc_line_number()
|
||||
lit += c.ascii_str()
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'c: `\\n` / ${c}')
|
||||
continue
|
||||
}
|
||||
// Check for escaped chars
|
||||
if c == u8(92) {
|
||||
esc, skip := s.handle_escapes(quote, true)
|
||||
lit += esc
|
||||
if skip > 0 {
|
||||
s.pos += skip
|
||||
s.col += skip
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Check for control characters (allow TAB)
|
||||
if util.is_illegal_ascii_control_character(c) {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' control character `${c.hex()}` is not allowed at ${start} (${s.line_nr},${s.col}) "${u8(s.at()).ascii_str()}" near ...${s.excerpt(s.pos, 5)}...')
|
||||
}
|
||||
|
||||
if c == quote {
|
||||
if s.peek(1) == quote && s.peek(2) == quote {
|
||||
if s.peek(3) == end_of_text {
|
||||
s.pos += 3
|
||||
s.col += 3
|
||||
lit += quote.ascii_str() + quote.ascii_str() + quote.ascii_str()
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'returning at ${c.ascii_str()} `${lit}`')
|
||||
return lit
|
||||
} else if s.peek(3) != quote {
|
||||
// lit += c.ascii_str()
|
||||
// lit += quote.ascii_str()
|
||||
s.pos += 3
|
||||
s.col += 3
|
||||
lit += quote.ascii_str() + quote.ascii_str() + quote.ascii_str()
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'returning at ${c.ascii_str()} `${lit}`')
|
||||
return lit
|
||||
}
|
||||
}
|
||||
}
|
||||
lit += c.ascii_str()
|
||||
}
|
||||
return lit
|
||||
}
|
||||
|
||||
// handle_escapes returns any escape character sequence.
|
||||
// For escape sequence validation see `Checker.check_quoted_escapes`.
|
||||
fn (mut s Scanner) handle_escapes(quote u8, is_multiline bool) (string, int) {
|
||||
c := u8(s.at())
|
||||
mut lit := c.ascii_str()
|
||||
is_literal_string := quote == `'`
|
||||
if !is_literal_string {
|
||||
if s.peek(1) == `u` && u8(s.peek(2)).is_hex_digit() && u8(s.peek(3)).is_hex_digit()
|
||||
&& u8(s.peek(4)).is_hex_digit() && u8(s.peek(5)).is_hex_digit() {
|
||||
lit += s.text[s.pos + 1..s.pos + 6] //.ascii_str()
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'gulp escaped unicode `${lit}`')
|
||||
return lit, 5
|
||||
} else if s.peek(1) == quote {
|
||||
if (!is_multiline && s.peek(2) == `\n`)
|
||||
|| (is_multiline && s.peek(2) == quote && s.peek(3) == quote && s.peek(4) == `\n`) {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'ignore special case escaped `${lit}` at end of string')
|
||||
return '', 0
|
||||
}
|
||||
lit += quote.ascii_str()
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'gulp escaped `${lit}`')
|
||||
return lit, 1
|
||||
}
|
||||
}
|
||||
if is_literal_string {
|
||||
if s.peek(1) == quote {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'ignore escape `${lit}${u8(s.peek(1)).ascii_str()}` in literal string')
|
||||
return '', 0
|
||||
}
|
||||
}
|
||||
|
||||
lit += u8(s.peek(1)).ascii_str()
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'gulp escaped `${lit}`')
|
||||
return lit, 1
|
||||
}
|
||||
|
||||
// extract_number collects and returns a string containing
|
||||
// any bytes recognized as a TOML number except for "(+/-)nan" and "(+/-)inf".
|
||||
// TOML numbers can include digits 0-9 and `_`.
|
||||
@[direct_array_access; inline]
|
||||
fn (mut s Scanner) extract_number() !string {
|
||||
// extract_number is called when the scanner has already reached
|
||||
// a byte that is a number or +/- - so we rewind it to start at the correct
|
||||
// position to get the complete number. Even if it's only one digit
|
||||
s.pos--
|
||||
s.col--
|
||||
start := s.pos
|
||||
|
||||
mut c := s.at()
|
||||
is_digit := u8(c).is_digit()
|
||||
if !(is_digit || c in [`+`, `-`]) {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' ${u8(c).ascii_str()} is not a number at ${s.excerpt(s.pos, 10)}')
|
||||
}
|
||||
s.pos++
|
||||
s.col++
|
||||
for s.pos < s.text.len {
|
||||
c = s.at()
|
||||
// Adjust scanner position to floating point numbers
|
||||
mut float_precision := 0
|
||||
if c == `.` {
|
||||
mut i := 1
|
||||
for c_ := u8(s.peek(i)); c_ != end_of_text && c_ != `\n`; c_ = u8(s.peek(i)) {
|
||||
if !c_.is_digit() && c_ != `,` {
|
||||
float_precision = 0
|
||||
break
|
||||
}
|
||||
float_precision++
|
||||
i++
|
||||
}
|
||||
}
|
||||
s.pos += float_precision
|
||||
s.col += float_precision
|
||||
// Handle signed exponent notation. I.e.: 3e2, 3E2, 3e-2, 3E+2, 3e0, 3.1e2, 3.1E2, -1E-1
|
||||
if c in [`e`, `E`] && s.peek(1) in [`+`, `-`] && u8(s.peek(2)).is_digit() {
|
||||
s.pos += 2
|
||||
s.col += 2
|
||||
}
|
||||
c = s.at()
|
||||
if !(u8(c).is_hex_digit() || c in digit_extras) || (c == `.` && s.is_left_of_assign) {
|
||||
break
|
||||
}
|
||||
s.pos++
|
||||
s.col++
|
||||
}
|
||||
key := s.text[start..s.pos]
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified number "${key}" in range [${start} .. ${s.pos}]')
|
||||
return key
|
||||
}
|
||||
|
||||
// extract_nan_or_inf_number collects and returns a string containing
|
||||
// any bytes recognized as infinity or not-a-number TOML numbers.
|
||||
@[direct_array_access; inline]
|
||||
fn (mut s Scanner) extract_nan_or_inf_number() !string {
|
||||
// extract_nan_or_inf_number is called when the scanner has already identified that
|
||||
// +/- or 'nan'/'inf' bytes is up but we rewind it to start at the correct position
|
||||
s.pos--
|
||||
s.col--
|
||||
start := s.pos
|
||||
|
||||
mut c := s.at()
|
||||
if c !in [`+`, `-`, `n`, `i`] {
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' ${u8(c).ascii_str()} is not a number at ${s.excerpt(s.pos, 10)}')
|
||||
}
|
||||
s.pos++
|
||||
s.col++
|
||||
for s.pos < s.text.len {
|
||||
c = s.at()
|
||||
if c !in [`n`, `a`, `i`, `f`] {
|
||||
break
|
||||
}
|
||||
s.pos++
|
||||
s.col++
|
||||
}
|
||||
key := s.text[start..s.pos]
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN,
|
||||
'identified special number "${key}" in range [${start} .. ${s.pos}]')
|
||||
return key
|
||||
}
|
||||
|
||||
// excerpt returns a string excerpt of the input text centered
|
||||
// at `pos`. The `margin` argument defines how many chacters
|
||||
// on each side of `pos` is returned
|
||||
pub fn (s &Scanner) excerpt(pos int, margin int) string {
|
||||
start := if pos > 0 && pos >= margin { pos - margin } else { 0 }
|
||||
end := if pos + margin < s.text.len { pos + margin } else { s.text.len }
|
||||
return s.text[start..end].replace('\n', r'\n')
|
||||
}
|
||||
|
||||
// state returns a read-only view of the scanner's internal state.
|
||||
pub fn (s &Scanner) state() State {
|
||||
return State{
|
||||
col: s.col
|
||||
line_nr: s.line_nr
|
||||
pos: s.pos
|
||||
}
|
||||
}
|
||||
|
||||
fn (mut s Scanner) validate_and_skip_headers() ! {
|
||||
// UTF-16 / UTF-32 headers (BE/LE)
|
||||
s.check_utf16_or_32_bom()!
|
||||
|
||||
// NICE-TO-HAVE-TODO Check other types of (UTF-?) headers and yield an error. TOML is UTF-8 only.
|
||||
|
||||
// Skip optional UTF-8 header, if any.
|
||||
if s.at() == 0xEF && s.peek(1) == 0xBB && s.peek(2) == 0xBF {
|
||||
util.printdbg(@MOD + '.' + @STRUCT + '.' + @FN, 'skipping UTF-8 byte order mark (BOM)')
|
||||
s.header_len = 3
|
||||
s.skip_n(s.header_len)
|
||||
}
|
||||
|
||||
// Check after we've skipped UTF-8 BOM
|
||||
s.check_utf16_or_32_bom()!
|
||||
}
|
||||
|
||||
fn (mut s Scanner) check_utf16_or_32_bom() ! {
|
||||
if (s.at() == 0xFF && s.peek(1) == 0xFE && s.peek(2) == 0x00 && s.peek(3) == 0x00)
|
||||
|| (s.at() == 0x00 && s.peek(1) == 0x00 && s.peek(2) == 0xFE && s.peek(3) == 0xFF) {
|
||||
s.header_len = 4
|
||||
s.skip_n(s.header_len)
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' UTF-32 is not a valid TOML encoding at ${s.pos} (${s.line_nr},${s.col}) near ...${s.excerpt(s.pos, 5)}...')
|
||||
}
|
||||
if (s.at() == 0xFE && s.peek(1) == 0xFF) || (s.at() == 0xFF && s.peek(1) == 0xFE) {
|
||||
s.header_len = 2
|
||||
s.skip_n(s.header_len)
|
||||
return error(@MOD + '.' + @STRUCT + '.' + @FN +
|
||||
' UTF-16 is not a valid TOML encoding at ${s.pos} (${s.line_nr},${s.col}) near ...${s.excerpt(s.pos, 5)}...')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import toml.input
|
||||
import toml.scanner
|
||||
|
||||
const scan_input = input.Config{
|
||||
text: 'abc'
|
||||
}
|
||||
|
||||
fn test_remaining() {
|
||||
mut s := scanner.new_scanner(input: scan_input) or { panic(err) }
|
||||
assert s.remaining() == 3
|
||||
s.next()
|
||||
s.next()
|
||||
assert s.remaining() == 1
|
||||
s.next()
|
||||
assert s.remaining() == 0
|
||||
s.next()
|
||||
s.next()
|
||||
assert s.remaining() == 0
|
||||
s.reset()
|
||||
assert s.remaining() == 3
|
||||
}
|
||||
|
||||
fn test_next() {
|
||||
mut s := scanner.new_scanner(input: scan_input) or { panic(err) }
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
assert s.next() == scanner.end_of_text
|
||||
assert s.next() == scanner.end_of_text
|
||||
assert s.next() == scanner.end_of_text
|
||||
}
|
||||
|
||||
fn test_skip() {
|
||||
mut s := scanner.new_scanner(input: scan_input) or { panic(err) }
|
||||
assert s.next() == `a`
|
||||
s.skip()
|
||||
assert s.next() == `c`
|
||||
assert s.next() == scanner.end_of_text
|
||||
}
|
||||
|
||||
fn test_skip_n() {
|
||||
mut s := scanner.new_scanner(input: scan_input) or { panic(err) }
|
||||
s.skip_n(2)
|
||||
assert s.next() == `c`
|
||||
assert s.next() == scanner.end_of_text
|
||||
}
|
||||
|
||||
fn test_at() {
|
||||
mut s := scanner.new_scanner(input: scan_input) or { panic(err) }
|
||||
assert s.at() == `a`
|
||||
assert s.at() == `a`
|
||||
assert s.at() == `a`
|
||||
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
assert s.next() == scanner.end_of_text
|
||||
}
|
||||
|
||||
fn test_peek() {
|
||||
mut s := scanner.new_scanner(input: scan_input) or { panic(err) }
|
||||
assert s.peek(0) == `a`
|
||||
assert s.peek(1) == `b`
|
||||
assert s.peek(2) == `c`
|
||||
assert s.peek(3) == scanner.end_of_text
|
||||
assert s.peek(4) == scanner.end_of_text
|
||||
|
||||
assert s.next() == `a`
|
||||
assert s.next() == `b`
|
||||
assert s.next() == `c`
|
||||
assert s.next() == scanner.end_of_text
|
||||
}
|
||||
|
||||
fn test_reset() {
|
||||
mut s := scanner.new_scanner(input: scan_input) or { panic(err) }
|
||||
assert s.next() == `a`
|
||||
s.next()
|
||||
s.next()
|
||||
assert s.next() == scanner.end_of_text
|
||||
s.reset()
|
||||
assert s.next() == `a`
|
||||
}
|
||||
|
||||
const multiline_string_input = input.Config{
|
||||
text: '"""abc\r\ndef\n123"""'
|
||||
}
|
||||
|
||||
fn test_multiline_string() {
|
||||
mut s := scanner.new_scanner(input: multiline_string_input) or { panic(err) }
|
||||
tok := s.scan()!
|
||||
assert tok.kind == .quoted
|
||||
assert tok.lit.contains('abc')
|
||||
assert tok.lit.contains('def')
|
||||
assert tok.lit.contains('123')
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import os
|
||||
import toml
|
||||
import toml.to
|
||||
|
||||
const toml_table_text = '
|
||||
[[products]]
|
||||
name = "Hammer"
|
||||
sku = 738594937
|
||||
|
||||
[[products]] # empty table within the array
|
||||
|
||||
[[products]]
|
||||
name = "Nail"
|
||||
sku = 284758393
|
||||
|
||||
color = "gray"'
|
||||
|
||||
fn test_tables() {
|
||||
mut toml_doc := toml.parse_text(toml_table_text) or { panic(err) }
|
||||
|
||||
toml_json := to.json(toml_doc)
|
||||
|
||||
assert toml_json == os.read_file(
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.out') or { panic(err) }
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import os
|
||||
import toml
|
||||
import toml.to
|
||||
|
||||
const toml_text = '[[albums]]
|
||||
name = "Born to Run"
|
||||
|
||||
[[albums.songs]]
|
||||
name = "Jungleland"
|
||||
|
||||
[[albums.songs]]
|
||||
name = "Meeting Across the River"
|
||||
|
||||
[[albums]]
|
||||
name = "Born in the USA"
|
||||
|
||||
[[albums.songs]]
|
||||
name = "Glory Days"
|
||||
|
||||
[[albums.songs]]
|
||||
name = "Dancing in the Dark"'
|
||||
|
||||
const fprefix = os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))
|
||||
|
||||
fn test_nested_array_of_tables() {
|
||||
mut toml_doc := toml.parse_text(toml_text)!
|
||||
|
||||
toml_json := to.json(toml_doc)
|
||||
eprintln(toml_json)
|
||||
|
||||
assert toml_json == os.read_file(fprefix + '.out')!
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import os
|
||||
import toml
|
||||
import toml.to
|
||||
|
||||
const toml_text = '[[a]]
|
||||
[[a.b]]
|
||||
[a.b.c]
|
||||
d = "val0"
|
||||
[[a.b]]
|
||||
[a.b.c]
|
||||
d = "val1"
|
||||
'
|
||||
|
||||
fn test_nested_array_of_tables() {
|
||||
mut toml_doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
|
||||
toml_json := to.json(toml_doc)
|
||||
|
||||
eprintln(toml_json)
|
||||
assert toml_json == os.read_file(
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.out') or { panic(err) }
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import os
|
||||
import toml
|
||||
import toml.to
|
||||
|
||||
fn test_array_of_tables_edge_case_file() {
|
||||
toml_file :=
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.toml'
|
||||
toml_doc := toml.parse_file(toml_file) or { panic(err) }
|
||||
|
||||
toml_json := to.json(toml_doc)
|
||||
out_file :=
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.out'
|
||||
out_file_json := os.read_file(out_file) or { panic(err) }
|
||||
println(toml_json)
|
||||
assert toml_json == out_file_json
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import os
|
||||
import toml
|
||||
import toml.to
|
||||
|
||||
const fprefix = os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))
|
||||
|
||||
fn test_array_of_tables_edge_case_file() {
|
||||
toml_doc := toml.parse_file(os.real_path(fprefix + '.toml'))!
|
||||
|
||||
toml_json := to.json(toml_doc)
|
||||
|
||||
out_file_json := os.read_file(os.real_path(fprefix + '.out'))!
|
||||
println(toml_json)
|
||||
assert toml_json == out_file_json
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import toml
|
||||
|
||||
// Complete text from the example in the README.md:
|
||||
// https://github.com/toml-lang/toml/blob/3b11f6921da7b6f5db37af039aa021fee450c091/README.md#Example
|
||||
const toml_text = '# This is a TOML document.
|
||||
title = "TOML Example"
|
||||
[owner]
|
||||
name = "Tom Preston-Werner"
|
||||
dob = 1979-05-27T07:32:00-08:00 # First class dates
|
||||
[database]
|
||||
server = "192.168.1.1"
|
||||
ports = [ 8000, 8001, 8002 ]
|
||||
connection_max = 5000
|
||||
enabled = true
|
||||
[servers]
|
||||
# Indentation (tabs and/or spaces) is allowed but not required
|
||||
[servers.alpha]
|
||||
ip = "10.0.0.1"
|
||||
dc = "eqdc10"
|
||||
[servers.beta]
|
||||
ip = "10.0.0.2"
|
||||
dc = "eqdc10"
|
||||
[clients]
|
||||
data=[["gamma","delta"],[1,2]]
|
||||
# Line breaks are OK when inside arrays
|
||||
hosts = [
|
||||
"alpha",
|
||||
"omega"
|
||||
]'
|
||||
|
||||
fn test_parse_compact_text() {
|
||||
toml_doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
|
||||
title := toml_doc.value('title')
|
||||
assert title == toml.Any('TOML Example')
|
||||
assert title as string == 'TOML Example'
|
||||
|
||||
database := toml_doc.value('database') as map[string]toml.Any
|
||||
db_serv := database['server'] or {
|
||||
panic('could not access "server" index in "database" variable')
|
||||
}
|
||||
assert db_serv as string == '192.168.1.1'
|
||||
|
||||
assert toml_doc.value('owner.name') as string == 'Tom Preston-Werner'
|
||||
|
||||
assert toml_doc.value('database.server') as string == '192.168.1.1'
|
||||
|
||||
database_ports := toml_doc.value('database.ports') as []toml.Any
|
||||
assert database_ports[0] as i64 == 8000
|
||||
assert database_ports[1] as i64 == 8001
|
||||
assert database_ports[2] as i64 == 8002
|
||||
assert database_ports[0].int() == 8000
|
||||
assert database_ports[1].int() == 8001
|
||||
assert database_ports[2].int() == 8002
|
||||
|
||||
assert toml_doc.value('database.connection_max') as i64 == 5000
|
||||
assert toml_doc.value('database.enabled') as bool == true
|
||||
|
||||
assert toml_doc.value('servers.alpha.ip').string() == '10.0.0.1'
|
||||
assert toml_doc.value('servers.alpha.dc').string() == 'eqdc10'
|
||||
|
||||
assert toml_doc.value('servers.beta.ip').string() == '10.0.0.2'
|
||||
assert toml_doc.value('servers.beta.dc').string() == 'eqdc10'
|
||||
|
||||
clients_data := (toml_doc.value('clients.data') as []toml.Any)
|
||||
// dump(clients_data)
|
||||
// assert false
|
||||
gamma_delta_array := clients_data[0] as []toml.Any
|
||||
digits_array := clients_data[1] as []toml.Any
|
||||
assert gamma_delta_array[0].string() == 'gamma'
|
||||
assert gamma_delta_array[1].string() == 'delta'
|
||||
assert digits_array[0].int() == 1
|
||||
assert digits_array[1].int() == 2
|
||||
|
||||
clients_hosts := (toml_doc.value('clients.hosts') as []toml.Any).as_strings()
|
||||
assert clients_hosts[0] == 'alpha'
|
||||
assert clients_hosts[1] == 'omega'
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import toml
|
||||
|
||||
fn test_crlf() {
|
||||
str_value := 'test string'
|
||||
mut toml_txt := 'crlf_string = "test string"\r\n
|
||||
# Comment with CRLF is not allowed'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
|
||||
value := toml_doc.value('crlf_string')
|
||||
assert value == toml.Any(str_value)
|
||||
assert value as string == str_value
|
||||
assert value.string() == str_value
|
||||
}
|
||||
|
||||
fn test_crlf_is_parsable_just_like_lf() {
|
||||
crlf_content := '# a comment\r\ntitle = "TOML Example"\r\n[database]\r\nserver = "192.168.1.1"\r\nports = [ 8000, 8001, 8002 ]\r\n'
|
||||
all := [crlf_content, crlf_content.replace('\r\n', '\n')]
|
||||
for content in all {
|
||||
res := toml.parse_text(content)!
|
||||
assert res.value('title') == toml.Any('TOML Example')
|
||||
assert (res.value('database') as map[string]toml.Any)['server']! == toml.Any('192.168.1.1')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import toml
|
||||
|
||||
fn test_dates() {
|
||||
toml_txt := '
|
||||
# Offset Date-Time
|
||||
odt1 = 1979-05-27T07:32:00Z
|
||||
odt2 = 1979-05-27T00:32:00-07:00
|
||||
odt3 = 1979-05-27T00:32:00.999999-07:00
|
||||
odt4 = 1979-05-27 07:32:00Z
|
||||
# Local Date-Time
|
||||
ldt1 = 1979-05-27T07:32:00
|
||||
ldt2 = 1979-05-27T00:32:00.999999
|
||||
# Local Date
|
||||
ld1 = 1979-05-27
|
||||
# Local Time
|
||||
lt1 = 07:32:00
|
||||
lt2 = 00:32:00.999999
|
||||
'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
|
||||
// Re-use vars
|
||||
mut odt_time := toml.DateTime{'1979-05-27T07:32:00Z'}
|
||||
mut odt_str := toml_doc.value('odt1').string()
|
||||
|
||||
// odt1 test section
|
||||
assert odt_str == '1979-05-27T07:32:00Z'
|
||||
odt1 := toml_doc.value('odt1')
|
||||
assert odt1.datetime() == odt_time
|
||||
|
||||
// odt2 test section
|
||||
odt_time = toml.DateTime{'1979-05-27T00:32:00-07:00'}
|
||||
odt2 := toml_doc.value('odt2')
|
||||
assert odt2.datetime() == odt_time
|
||||
|
||||
// odt3 test section
|
||||
odt_time = toml.DateTime{'1979-05-27T00:32:00.999999-07:00'}
|
||||
odt3 := toml_doc.value('odt3')
|
||||
assert odt3.datetime() == odt_time
|
||||
|
||||
// odt4 test section
|
||||
odt_time = toml.DateTime{'1979-05-27 07:32:00Z'}
|
||||
odt4 := toml_doc.value('odt4')
|
||||
assert odt4.datetime() == odt_time
|
||||
|
||||
// ldt1 test section
|
||||
odt_time = toml.DateTime{'1979-05-27T07:32:00'}
|
||||
ldt1 := toml_doc.value('ldt1')
|
||||
assert ldt1.datetime() == odt_time
|
||||
|
||||
// ldt2 test section
|
||||
odt_time = toml.DateTime{'1979-05-27T00:32:00.999999'}
|
||||
ldt2 := toml_doc.value('ldt2')
|
||||
assert ldt2.datetime() == odt_time
|
||||
|
||||
// ld1 test section
|
||||
od_time := toml.Date{'1979-05-27'}
|
||||
ld1 := toml_doc.value('ld1')
|
||||
assert ld1.date() == od_time
|
||||
assert ld1.string() == '1979-05-27'
|
||||
|
||||
// lt1 test section
|
||||
mut ot_time := toml.Time{'07:32:00'}
|
||||
lt1 := toml_doc.value('lt1')
|
||||
assert lt1.time() == ot_time
|
||||
assert lt1.string() == '07:32:00'
|
||||
|
||||
// lt2 test section
|
||||
ot_time = toml.Time{'00:32:00.999999'}
|
||||
lt2 := toml_doc.value('lt2')
|
||||
assert lt2.time() == ot_time
|
||||
assert lt2.string() == '00:32:00.999999'
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import toml
|
||||
|
||||
fn test_default_to() {
|
||||
default_value := 4321
|
||||
mut toml_txt := 'var = 1234'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
value := toml_doc.value('tar').default_to(default_value).int()
|
||||
assert value == default_value
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import toml
|
||||
|
||||
const toml_text = '# This TOML can reflect/decode to a struct
|
||||
val_string = "test"
|
||||
val_bool = false
|
||||
val_int = 456
|
||||
val_i64 = 4567
|
||||
val_u64 = 45678
|
||||
val_f32 = 200.2
|
||||
val_f64 = 2000.2
|
||||
val_datetime = 2024-10-05 09:10:20.000
|
||||
val_date = 2099-09-09
|
||||
val_time = 22:22:22.222
|
||||
'
|
||||
|
||||
const toml_all_default_text = '# This TOML can reflect/decode to a struct all with default values'
|
||||
|
||||
struct Test {
|
||||
val_string string = 'abcd'
|
||||
val_bool bool = true
|
||||
val_int int = 123
|
||||
val_i64 i64 = 1234
|
||||
val_u64 u64 = 12345
|
||||
val_f32 f32 = 100.1
|
||||
val_f64 f64 = 1000.1
|
||||
val_datetime toml.DateTime = toml.DateTime{'1980-07-11 21:23:42.123'}
|
||||
val_date toml.Date = toml.Date{'1977-07-07'}
|
||||
val_time toml.Time = toml.Time{'11:11:11.111'}
|
||||
}
|
||||
|
||||
fn test_reflect_default_values() {
|
||||
toml_has_values := toml.parse_text(toml_text) or { panic(err) }
|
||||
test_has_values := toml_has_values.reflect[Test]()
|
||||
|
||||
assert test_has_values.val_string == 'test'
|
||||
assert test_has_values.val_bool == false
|
||||
assert test_has_values.val_int == 456
|
||||
assert test_has_values.val_i64 == 4567
|
||||
assert test_has_values.val_u64 == 45678
|
||||
assert test_has_values.val_f32 == 200.2
|
||||
assert test_has_values.val_f64 == 2000.2
|
||||
assert test_has_values.val_datetime == toml.DateTime{'2024-10-05 09:10:20.000'}
|
||||
assert test_has_values.val_date == toml.Date{'2099-09-09'}
|
||||
assert test_has_values.val_time == toml.Time{'22:22:22.222'}
|
||||
|
||||
toml_all_default_values := toml.parse_text(toml_all_default_text) or { panic(err) }
|
||||
test_all_default_values := toml_all_default_values.reflect[Test]()
|
||||
|
||||
assert test_all_default_values.val_string == 'abcd'
|
||||
assert test_all_default_values.val_bool == true
|
||||
assert test_all_default_values.val_int == 123
|
||||
assert test_all_default_values.val_i64 == 1234
|
||||
assert test_all_default_values.val_u64 == 12345
|
||||
assert test_all_default_values.val_f32 == 100.1
|
||||
assert test_all_default_values.val_f64 == 1000.1
|
||||
assert test_all_default_values.val_datetime == toml.DateTime{'1980-07-11 21:23:42.123'}
|
||||
assert test_all_default_values.val_date == toml.Date{'1977-07-07'}
|
||||
assert test_all_default_values.val_time == toml.Time{'11:11:11.111'}
|
||||
}
|
||||
|
||||
fn test_decode_struct_default_values() {
|
||||
test_has_values := toml.decode[Test](toml_text) or { panic(err) }
|
||||
assert test_has_values.val_string == 'test'
|
||||
assert test_has_values.val_bool == false
|
||||
assert test_has_values.val_int == 456
|
||||
assert test_has_values.val_i64 == 4567
|
||||
assert test_has_values.val_u64 == 45678
|
||||
assert test_has_values.val_f32 == 200.2
|
||||
assert test_has_values.val_f64 == 2000.2
|
||||
assert test_has_values.val_datetime == toml.DateTime{'2024-10-05 09:10:20.000'}
|
||||
assert test_has_values.val_date == toml.Date{'2099-09-09'}
|
||||
assert test_has_values.val_time == toml.Time{'22:22:22.222'}
|
||||
|
||||
test_all_default_values := toml.decode[Test](toml_all_default_text) or { panic(err) }
|
||||
assert test_all_default_values.val_string == 'abcd'
|
||||
assert test_all_default_values.val_bool == true
|
||||
assert test_all_default_values.val_int == 123
|
||||
assert test_all_default_values.val_i64 == 1234
|
||||
assert test_all_default_values.val_u64 == 12345
|
||||
assert test_all_default_values.val_f32 == 100.1
|
||||
assert test_all_default_values.val_f64 == 1000.1
|
||||
assert test_all_default_values.val_datetime == toml.DateTime{'1980-07-11 21:23:42.123'}
|
||||
assert test_all_default_values.val_date == toml.Date{'1977-07-07'}
|
||||
assert test_all_default_values.val_time == toml.Time{'11:11:11.111'}
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
import toml
|
||||
|
||||
enum JobTitle {
|
||||
worker
|
||||
executive
|
||||
manager
|
||||
}
|
||||
|
||||
struct Pet {
|
||||
name string
|
||||
nicknames []string
|
||||
age u64
|
||||
income int
|
||||
height f32
|
||||
has_furr bool
|
||||
title JobTitle
|
||||
address Address
|
||||
meal_frequency map[string]int
|
||||
// *¹ Currently it is only possible to decode a single nested struct generically.
|
||||
// As soon as we decode another nested struct (e.g. within this struct, like `contact` below)
|
||||
// or only one nested struct within another struct, it results in wrong values or errors.
|
||||
// Related issue: https://github.com/vlang/v/issues/18110
|
||||
// contact Contact
|
||||
}
|
||||
|
||||
struct Address {
|
||||
street string
|
||||
city string
|
||||
}
|
||||
|
||||
// *¹
|
||||
/*
|
||||
struct Contact {
|
||||
phone string
|
||||
}*/
|
||||
|
||||
struct AnyStruct {
|
||||
val toml.Any
|
||||
}
|
||||
|
||||
struct Employee {
|
||||
mut:
|
||||
name string
|
||||
age int
|
||||
salary f32
|
||||
is_human bool
|
||||
title JobTitle
|
||||
}
|
||||
|
||||
struct Arrs {
|
||||
strs []string
|
||||
bools []bool
|
||||
ints []int
|
||||
i64s []i64
|
||||
u64s []u64
|
||||
f32s []f32
|
||||
f64s []f64
|
||||
dts []toml.DateTime
|
||||
dates []toml.Date
|
||||
times []toml.Time
|
||||
}
|
||||
|
||||
// individual because toml.decode[Foo](str)! == foo is false
|
||||
struct AnyArr {
|
||||
arr []toml.Any
|
||||
}
|
||||
|
||||
struct Item {
|
||||
name string
|
||||
}
|
||||
|
||||
struct Doc {
|
||||
items []Item
|
||||
}
|
||||
|
||||
struct MapDoc {
|
||||
items map[string]Item
|
||||
}
|
||||
|
||||
fn test_encode_and_decode() {
|
||||
// *¹
|
||||
// p := Pet{'Mr. Scratchy McEvilPaws', ['Freddy', 'Fred', 'Charles'], 8, -1, 0.8, true, .manager, Address{'1428 Elm Street', 'Springwood'}, Contact{'123-456-7890'}}
|
||||
p := Pet{'Mr. Scratchy McEvilPaws', ['Freddy', 'Fred', 'Charles'], 8, -1, 0.8, true, .manager, Address{'1428 Elm Street', 'Springwood'}, {
|
||||
'bones': 2
|
||||
'kibble': 5
|
||||
}}
|
||||
s := 'name = "Mr. Scratchy McEvilPaws"
|
||||
nicknames = [
|
||||
"Freddy",
|
||||
"Fred",
|
||||
"Charles"
|
||||
]
|
||||
age = 8
|
||||
income = -1
|
||||
height = 0.8
|
||||
has_furr = true
|
||||
title = 2
|
||||
address = { street = "1428 Elm Street", city = "Springwood" }
|
||||
meal_frequency = { bones = 2, kibble = 5 }'
|
||||
// contact = { phone = "123-456-7890" }' // *¹
|
||||
|
||||
assert toml.encode[Pet](p) == s
|
||||
assert toml.decode[Pet](s)! == p
|
||||
}
|
||||
|
||||
fn test_encode_and_decode_any() {
|
||||
a := AnyStruct{toml.Any(10)}
|
||||
s := 'val = 10'
|
||||
|
||||
assert toml.encode[AnyStruct](a) == s
|
||||
assert toml.decode[AnyStruct](s)!.val.int() == 10
|
||||
}
|
||||
|
||||
pub fn (e Employee) to_toml() string {
|
||||
mut mp := map[string]toml.Any{}
|
||||
mp['name'] = toml.Any(e.name)
|
||||
mp['age'] = toml.Any(e.age)
|
||||
mp['is_human'] = toml.Any(e.is_human)
|
||||
// Change some values to assert that the custom method is used instead of generic encoding.
|
||||
mp['salary'] = toml.Any(f32(e.salary) + 5000.0)
|
||||
mp['title'] = toml.Any(int(e.title) + 1)
|
||||
return mp.to_toml()
|
||||
}
|
||||
|
||||
pub fn (mut e Employee) from_toml(any toml.Any) {
|
||||
mp := any.as_map()
|
||||
e.name = mp['name'] or { toml.Any('') }.string()
|
||||
e.age = mp['age'] or { toml.Any(0) }.int()
|
||||
e.is_human = mp['is_human'] or { toml.Any(false) }.bool()
|
||||
// Change some values to assert that the custom method is used instead of generic decoding.
|
||||
e.salary = mp['salary'] or { toml.Any(0) }.f32() - 15000.0
|
||||
e.title = unsafe { JobTitle(mp['title'] or { toml.Any(0) }.int() - 2) }
|
||||
}
|
||||
|
||||
fn test_custom_encode_and_decode() {
|
||||
x := Employee{'Peter', 28, 95000.5, true, .executive}
|
||||
s := toml.encode[Employee](x)
|
||||
eprintln('Employee x: ${s}')
|
||||
assert s == r'name = "Peter"
|
||||
age = 28
|
||||
is_human = true
|
||||
salary = 100000.5
|
||||
title = 2'
|
||||
|
||||
y := toml.decode[Employee](s) or {
|
||||
println(err)
|
||||
assert false
|
||||
return
|
||||
}
|
||||
eprintln('Employee y: ${y}')
|
||||
assert y.name == 'Peter'
|
||||
assert y.age == 28
|
||||
assert y.salary == 85000.5
|
||||
assert y.is_human == true
|
||||
assert y.title == .worker
|
||||
}
|
||||
|
||||
struct Example1 {
|
||||
arr []Problem
|
||||
}
|
||||
|
||||
struct Example2 {
|
||||
arr []Problem
|
||||
}
|
||||
|
||||
struct Problem {
|
||||
x int
|
||||
}
|
||||
|
||||
pub fn (example Example1) to_toml() string {
|
||||
return '[This is Valid]'
|
||||
}
|
||||
|
||||
pub fn (problem Problem) to_toml() string {
|
||||
return 'a problem'
|
||||
}
|
||||
|
||||
fn test_custom_encode_of_complex_struct() {
|
||||
assert toml.encode(Example1{}) == '[This is Valid]'
|
||||
assert toml.encode(Example2{[Problem{}, Problem{}]}) == 'arr = [
|
||||
"a problem",
|
||||
"a problem"
|
||||
]'
|
||||
}
|
||||
|
||||
struct Example3 {
|
||||
arr_arr [][]Problem
|
||||
}
|
||||
|
||||
struct Example4 {
|
||||
mp map[string]Problem
|
||||
}
|
||||
|
||||
pub fn (example Example3) to_toml() string {
|
||||
return '[This is Valid]'
|
||||
}
|
||||
|
||||
pub fn (example Example4) to_toml() string {
|
||||
return '[This is Valid]'
|
||||
}
|
||||
|
||||
fn test_custom_encode_of_nested_complex_struct() {
|
||||
assert toml.encode(Example3{}) == '[This is Valid]'
|
||||
assert toml.encode(Example4{}) == '[This is Valid]'
|
||||
}
|
||||
|
||||
struct Example5 {
|
||||
mp map[string]Problem
|
||||
}
|
||||
|
||||
fn test_map_encode_of_complex_struct() {
|
||||
mut mp := map[string]Problem{}
|
||||
mp['key_one'] = Problem{}
|
||||
mp['key_two'] = Problem{}
|
||||
assert toml.encode(Example5{ mp: mp }) == 'mp = { key_one = "a problem", key_two = "a problem" }'
|
||||
}
|
||||
|
||||
struct Example6 {
|
||||
ptr voidptr
|
||||
r rune
|
||||
}
|
||||
|
||||
fn test_encode_for_exotic_types() {
|
||||
assert toml.encode(Example6{ ptr: unsafe { &voidptr(nil) }, r: `🚀` }) == 'ptr = "0x0"\nr = "🚀"'
|
||||
}
|
||||
|
||||
fn test_array_encode_decode() {
|
||||
a := Arrs{
|
||||
strs: ['foo', 'bar']
|
||||
bools: [true, false]
|
||||
ints: [-1, 2]
|
||||
i64s: [i64(-2)]
|
||||
u64s: [u64(123)]
|
||||
f32s: [f32(1.0), f32(2.5)]
|
||||
f64s: [100000.5, -123.0]
|
||||
dts: [toml.DateTime{'1979-05-27T07:32:00Z'}, toml.DateTime{'1979-05-27T07:32:00Z'}]
|
||||
dates: [toml.Date{'1979-05-27'}, toml.Date{'2022-12-31'}]
|
||||
times: [toml.Time{'07:32:59'}, toml.Time{'17:32:04'}]
|
||||
}
|
||||
|
||||
s := 'strs = [
|
||||
"foo",
|
||||
"bar"
|
||||
]
|
||||
bools = [
|
||||
true,
|
||||
false
|
||||
]
|
||||
ints = [
|
||||
-1,
|
||||
2
|
||||
]
|
||||
i64s = [
|
||||
-2
|
||||
]
|
||||
u64s = [
|
||||
123
|
||||
]
|
||||
f32s = [
|
||||
1.0,
|
||||
2.5
|
||||
]
|
||||
f64s = [
|
||||
100000.5,
|
||||
-123.0
|
||||
]
|
||||
dts = [
|
||||
1979-05-27T07:32:00Z,
|
||||
1979-05-27T07:32:00Z
|
||||
]
|
||||
dates = [
|
||||
1979-05-27,
|
||||
2022-12-31
|
||||
]
|
||||
times = [
|
||||
07:32:59,
|
||||
17:32:04
|
||||
]'
|
||||
|
||||
assert toml.encode[Arrs](a) == s
|
||||
assert toml.decode[Arrs](s)! == a
|
||||
|
||||
any_a := AnyArr{[toml.Any(10), 20, 30]}
|
||||
any_s := 'arr = [
|
||||
10,
|
||||
20,
|
||||
30
|
||||
]'
|
||||
|
||||
assert toml.encode[AnyArr](any_a) == any_s
|
||||
assert toml.decode[AnyArr](any_s)!.arr.map(it.int()) == [10, 20, 30]
|
||||
}
|
||||
|
||||
fn test_array_of_struct_decode() {
|
||||
doc := Doc{
|
||||
items: [
|
||||
Item{
|
||||
name: 'item #1'
|
||||
},
|
||||
Item{
|
||||
name: 'item #2'
|
||||
},
|
||||
Item{
|
||||
name: 'item #3'
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
encoded := toml.encode[Doc](doc)
|
||||
assert toml.decode[Doc](encoded)! == doc
|
||||
|
||||
array_of_tables := '[[items]]
|
||||
name = "item #1"
|
||||
|
||||
[[items]]
|
||||
name = "item #2"
|
||||
|
||||
[[items]]
|
||||
name = "item #3"'
|
||||
assert toml.decode[Doc](array_of_tables)! == doc
|
||||
}
|
||||
|
||||
fn test_map_of_struct_decode() {
|
||||
doc := MapDoc{
|
||||
items: {
|
||||
'first': Item{
|
||||
name: 'first one'
|
||||
}
|
||||
'second': Item{
|
||||
name: 'second one'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
encoded := toml.encode[MapDoc](doc)
|
||||
assert toml.decode[MapDoc](encoded)! == doc
|
||||
|
||||
table_doc := '[items.first]
|
||||
name = "first one"
|
||||
|
||||
[items.second]
|
||||
name = "second one"'
|
||||
assert toml.decode[MapDoc](table_doc)! == doc
|
||||
}
|
||||
|
||||
fn test_decode_doc() {
|
||||
doc := toml.parse_text('name = "Peter"
|
||||
age = 28
|
||||
is_human = true
|
||||
salary = 100000.5
|
||||
title = 2')!
|
||||
e := doc.decode[Employee]()!
|
||||
assert e.name == 'Peter'
|
||||
assert e.age == 28
|
||||
assert e.salary == 100000.5
|
||||
assert e.is_human == true
|
||||
assert e.title == .manager
|
||||
}
|
||||
|
||||
fn test_unsupported_type() {
|
||||
s := 'name = "Peter"'
|
||||
err_msg := 'toml.decode: expected struct, found '
|
||||
if _ := toml.decode[string](s) {
|
||||
assert false
|
||||
} else {
|
||||
assert err.msg() == err_msg + 'string'
|
||||
}
|
||||
if _ := toml.decode[[]string](s) {
|
||||
assert false
|
||||
} else {
|
||||
assert err.msg() == err_msg + '[]string'
|
||||
}
|
||||
if _ := toml.decode[int](s) {
|
||||
assert false
|
||||
} else {
|
||||
assert err.msg() == err_msg + 'int'
|
||||
}
|
||||
if _ := toml.decode[[]f32](s) {
|
||||
assert false
|
||||
} else {
|
||||
assert err.msg() == err_msg + '[]f32'
|
||||
}
|
||||
// ...
|
||||
|
||||
doc := toml.parse_text('name = "Peter"')!
|
||||
assert doc.value('name').string() == 'Peter'
|
||||
if _ := doc.decode[string]() {
|
||||
assert false
|
||||
} else {
|
||||
assert err.msg() == 'Doc.decode: expected struct, found string'
|
||||
}
|
||||
}
|
||||
|
||||
fn test_date_array_decode_with_spaces() {
|
||||
// test space after yyyy-mm-dd is not date-time separator but white to consume. issue # 25279
|
||||
s := '
|
||||
dates = [ 1979-05-27 , 2022-12-31 ]
|
||||
'
|
||||
a := Arrs{
|
||||
dates: [toml.Date{'1979-05-27'}, toml.Date{'2022-12-31'}]
|
||||
}
|
||||
assert toml.decode[Arrs](s)! == a
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
import os
|
||||
import toml
|
||||
import toml.ast
|
||||
import x.json2
|
||||
|
||||
const hide_oks = os.getenv('VTEST_HIDE_OK') == '1'
|
||||
|
||||
// Can be set to `true` to process tests that stress test the parser
|
||||
// by having large data amounts - these pass - but slow down the test run
|
||||
const do_large_files = os.getenv('VTEST_TOML_DO_LARGE_FILES') == '1'
|
||||
|
||||
// Can be set to `true` to process tests that triggers a slow conversion
|
||||
// process that uses `python` to convert from YAML to JSON.
|
||||
const do_yaml_conversion = os.getenv('VTEST_TOML_DO_YAML_CONVERSION') == '1'
|
||||
|
||||
// Instructions for developers:
|
||||
// The actual tests and data can be obtained by doing:
|
||||
// `git clone --depth 1 https://github.com/iarna/toml-spec-tests.git vlib/toml/tests/testdata/iarna/toml-test`
|
||||
// See also the CI toml tests
|
||||
// Kept for easier handling of future updates to the tests
|
||||
const valid_exceptions = []string{}
|
||||
const invalid_exceptions = []string{}
|
||||
|
||||
const valid_value_exceptions = [
|
||||
'values/spec-date-time-3.toml',
|
||||
'values/spec-date-time-4.toml',
|
||||
'values/spec-readme-example.toml',
|
||||
'values/spec-date-time-6.toml',
|
||||
'values/spec-date-time-5.toml',
|
||||
'values/spec-date-time-1.toml',
|
||||
'values/spec-date-time-2.toml',
|
||||
'values/qa-table-inline-nested-1000.toml',
|
||||
'values/qa-array-inline-nested-1000.toml',
|
||||
]
|
||||
|
||||
const yaml_value_exceptions = [
|
||||
'values/spec-float-5.toml', // YAML: "1e6", V: 1000000
|
||||
'values/spec-float-9.toml', // YAML: "-0e0", V: 0
|
||||
'values/spec-float-6.toml', // YAML: "-2E-2", V: -0.02
|
||||
'values/spec-float-4.toml', // YAML: "5e+22", V: 50000000000000004000000
|
||||
]
|
||||
|
||||
const jq = os.find_abs_path_of_executable('jq') or { '' }
|
||||
const python = os.find_abs_path_of_executable('python') or { '' }
|
||||
const compare_work_dir_root = os.join_path(os.vtmp_dir(), 'toml_iarna')
|
||||
// From: https://stackoverflow.com/a/38266731/1904615
|
||||
const jq_normalize = r'# Apply f to composite entities recursively using keys[], and to atoms
|
||||
def sorted_walk(f):
|
||||
. as $in
|
||||
| if type == "object" then
|
||||
reduce keys[] as $key
|
||||
( {}; . + { ($key): ($in[$key] | sorted_walk(f)) } ) | f
|
||||
elif type == "array" then map( sorted_walk(f) ) | f
|
||||
else f
|
||||
end;
|
||||
|
||||
def normalize: sorted_walk(if type == "array" then sort else . end);
|
||||
|
||||
normalize'
|
||||
|
||||
fn run(args []string) !string {
|
||||
res := os.execute(args.join(' '))
|
||||
if res.exit_code != 0 {
|
||||
return error('${args[0]} failed with return code ${res.exit_code}.\n${res.output}')
|
||||
}
|
||||
return res.output
|
||||
}
|
||||
|
||||
// test_iarna_toml_spec_tests run though 'testdata/iarna/toml-test/*' if found.
|
||||
fn test_iarna_toml_spec_tests() {
|
||||
this_file := @FILE
|
||||
test_root := os.join_path(os.dir(this_file), 'testdata', 'iarna')
|
||||
if os.is_dir(test_root) {
|
||||
valid_test_files := os.walk_ext(os.join_path(test_root, 'values'), '.toml')
|
||||
println('Testing ${valid_test_files.len} valid TOML files...')
|
||||
mut valid := 0
|
||||
mut e := 0
|
||||
for i, valid_test_file in valid_test_files {
|
||||
mut relative := valid_test_file.all_after('iarna').trim_left(os.path_separator)
|
||||
$if windows {
|
||||
relative = relative.replace('/', '\\')
|
||||
}
|
||||
|
||||
if !do_large_files && valid_test_file.contains('qa-') {
|
||||
e++
|
||||
println('SKIP [${i + 1}/${valid_test_files.len}] "${valid_test_file}" LARGE FILE...')
|
||||
continue
|
||||
}
|
||||
|
||||
if relative in valid_exceptions {
|
||||
e++
|
||||
idx := valid_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${valid_test_files.len}] "${valid_test_file}" VALID EXCEPTION [${idx}/${valid_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
|
||||
if !hide_oks {
|
||||
println('OK [${i + 1}/${valid_test_files.len}] "${valid_test_file}"...')
|
||||
}
|
||||
toml_doc := toml.parse_file(valid_test_file)!
|
||||
valid++
|
||||
}
|
||||
println('${valid}/${valid_test_files.len} TOML files were parsed correctly')
|
||||
if valid_exceptions.len > 0 {
|
||||
println('TODO Skipped parsing of ${e} valid TOML files...')
|
||||
}
|
||||
|
||||
// If the command-line tool `jq` is installed, value tests can be run as well.
|
||||
if jq != '' {
|
||||
println('Testing value output of ${valid_test_files.len} valid TOML files using "${jq}"...')
|
||||
|
||||
if os.exists(compare_work_dir_root) {
|
||||
os.rmdir_all(compare_work_dir_root)!
|
||||
}
|
||||
os.mkdir_all(compare_work_dir_root)!
|
||||
|
||||
jq_normalize_path := os.join_path(compare_work_dir_root, 'normalize.jq')
|
||||
os.write_file(jq_normalize_path, jq_normalize)!
|
||||
|
||||
valid = 0
|
||||
e = 0
|
||||
for i, valid_test_file in valid_test_files {
|
||||
mut relative := valid_test_file.all_after('iarna').trim_left(os.path_separator)
|
||||
$if windows {
|
||||
relative = relative.replace('/', '\\')
|
||||
}
|
||||
|
||||
// Skip the file if we know it can't be parsed or we know that the value retrieval needs work.
|
||||
if relative in valid_exceptions {
|
||||
e++
|
||||
idx := valid_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${valid_test_files.len}] "${valid_test_file}" VALID EXCEPTION [${idx}/${valid_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
|
||||
if relative in valid_value_exceptions {
|
||||
e++
|
||||
idx := valid_value_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${valid_test_files.len}] "${valid_test_file}" VALID VALUE EXCEPTION [${idx}/${valid_value_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
|
||||
valid_test_file_name := os.file_name(valid_test_file).all_before_last('.')
|
||||
uses_json_format := os.exists(valid_test_file.all_before_last('.') + '.json')
|
||||
|
||||
// Use python to convert the YAML files to json - it yields some inconsistencies
|
||||
// so we skip some of them
|
||||
mut converted_from_yaml := false
|
||||
mut converted_json_path := ''
|
||||
if !uses_json_format {
|
||||
$if windows {
|
||||
println('N/A [${i + 1}/${valid_test_files.len}] "${valid_test_file}"...')
|
||||
continue
|
||||
}
|
||||
if python == '' {
|
||||
println('N/A [${i + 1}/${valid_test_files.len}] "${valid_test_file}"...')
|
||||
continue
|
||||
}
|
||||
if !do_yaml_conversion || relative in yaml_value_exceptions {
|
||||
e++
|
||||
idx := yaml_value_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${valid_test_files.len}] "${valid_test_file}" YAML VALUE EXCEPTION [${idx}/${valid_value_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
|
||||
if !do_large_files && valid_test_file.contains('qa-') {
|
||||
e++
|
||||
println('SKIP [${i + 1}/${valid_test_files.len}] "${valid_test_file}" LARGE FILE...')
|
||||
continue
|
||||
}
|
||||
|
||||
iarna_yaml_path := valid_test_file.all_before_last('.') + '.yaml'
|
||||
if os.exists(iarna_yaml_path) {
|
||||
converted_json_path = os.join_path(compare_work_dir_root,
|
||||
'${valid_test_file_name}.yaml.json')
|
||||
run([python, '-c',
|
||||
"'import sys, yaml, json; json.dump(yaml.load(sys.stdin, Loader=yaml.FullLoader), sys.stdout, indent=4)'",
|
||||
'<', iarna_yaml_path, '>', converted_json_path]) or {
|
||||
contents := os.read_file(iarna_yaml_path)!
|
||||
// NOTE there's known errors with the python convention method.
|
||||
// For now we just ignore them as it's a broken tool - not a wrong test-case.
|
||||
// Uncomment this print to see/check them.
|
||||
// eprintln(err.msg() + '\n${contents}')
|
||||
e++
|
||||
println('ERR [${i + 1}/${valid_test_files.len}] "${valid_test_file}" EXCEPTION [${e}/${valid_value_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
converted_from_yaml = true
|
||||
}
|
||||
}
|
||||
|
||||
if !hide_oks {
|
||||
println('OK [${i + 1}/${valid_test_files.len}] "${valid_test_file}"...')
|
||||
}
|
||||
toml_doc := toml.parse_file(valid_test_file)!
|
||||
|
||||
v_toml_json_path := os.join_path(compare_work_dir_root,
|
||||
'${valid_test_file_name}.v.json')
|
||||
iarna_toml_json_path := os.join_path(compare_work_dir_root,
|
||||
'${valid_test_file_name}.json')
|
||||
|
||||
os.write_file(v_toml_json_path, to_iarna(toml_doc.ast.table, converted_from_yaml))!
|
||||
|
||||
if converted_json_path == '' {
|
||||
converted_json_path = valid_test_file.all_before_last('.') + '.json'
|
||||
}
|
||||
iarna_json := os.read_file(converted_json_path)!
|
||||
os.write_file(iarna_toml_json_path, iarna_json)!
|
||||
|
||||
v_normalized_json := run([jq, '-S', '-f "${jq_normalize_path}"', v_toml_json_path]) or {
|
||||
contents := os.read_file(v_toml_json_path)!
|
||||
panic(err.msg() + '\n${contents}')
|
||||
}
|
||||
cmd := [jq, '-S', '-f "${jq_normalize_path}"', iarna_toml_json_path]
|
||||
iarna_normalized_json := run(cmd) or {
|
||||
contents := os.read_file(v_toml_json_path)!
|
||||
panic(err.msg() + '\n${contents}\n\ncmd: ${cmd.join(' ')}')
|
||||
}
|
||||
|
||||
assert iarna_normalized_json == v_normalized_json
|
||||
|
||||
valid++
|
||||
}
|
||||
println('${valid}/${valid_test_files.len} TOML files were parsed correctly and value checked')
|
||||
if valid_value_exceptions.len > 0 {
|
||||
println('TODO Skipped value checks of ${e} valid TOML files...')
|
||||
}
|
||||
}
|
||||
|
||||
invalid_test_files := os.walk_ext(os.join_path(test_root, 'errors'), '.toml')
|
||||
println('Testing ${invalid_test_files.len} invalid TOML files...')
|
||||
mut invalid := 0
|
||||
e = 0
|
||||
for i, invalid_test_file in invalid_test_files {
|
||||
mut relative := invalid_test_file.all_after('iarna').trim_left(os.path_separator)
|
||||
$if windows {
|
||||
relative = relative.replace('/', '\\')
|
||||
}
|
||||
if relative in invalid_exceptions {
|
||||
e++
|
||||
idx := invalid_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${invalid_test_files.len}] "${invalid_test_file}" INVALID EXCEPTION [${idx}/${invalid_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
if !hide_oks {
|
||||
println('OK [${i + 1}/${invalid_test_files.len}] "${invalid_test_file}"...')
|
||||
}
|
||||
if toml_doc := toml.parse_file(invalid_test_file) {
|
||||
content_that_should_have_failed := os.read_file(invalid_test_file)!
|
||||
println(' This TOML should have failed:\n${'-'.repeat(40)}\n${content_that_should_have_failed}\n${'-'.repeat(40)}')
|
||||
assert false
|
||||
} else {
|
||||
if !hide_oks {
|
||||
println(' ${err.msg()}')
|
||||
}
|
||||
assert true
|
||||
}
|
||||
invalid++
|
||||
}
|
||||
println('${invalid}/${invalid_test_files.len} TOML files were parsed correctly')
|
||||
if invalid_exceptions.len > 0 {
|
||||
println('TODO Skipped parsing of ${invalid_exceptions.len} invalid TOML files...')
|
||||
}
|
||||
} else {
|
||||
println('No test data directory found in "${test_root}"')
|
||||
assert true
|
||||
}
|
||||
}
|
||||
|
||||
// to_iarna_time
|
||||
fn to_iarna_time(time_str string) string {
|
||||
if time_str.contains('.') {
|
||||
date_and_time := time_str.all_before('.')
|
||||
mut ms := time_str.all_after('.')
|
||||
z := if ms.contains('Z') { 'Z' } else { '' }
|
||||
ms = ms.replace('Z', '')
|
||||
if ms.len > 3 {
|
||||
ms = ms[..3]
|
||||
}
|
||||
return date_and_time + '.' + ms + z
|
||||
} else {
|
||||
return time_str + '.000'
|
||||
}
|
||||
}
|
||||
|
||||
// to_iarna returns a iarna compatible json string converted from the `value` ast.Value.
|
||||
fn to_iarna(value ast.Value, skip_value_map bool) string {
|
||||
match value {
|
||||
ast.Quoted {
|
||||
json_text := json2.Any(value.text).json_str()
|
||||
if skip_value_map {
|
||||
return json_text
|
||||
}
|
||||
return '{ "type": "string", "value": ${json_text} }'
|
||||
}
|
||||
ast.DateTime {
|
||||
// Normalization for json
|
||||
mut json_text := json2.Any(value.text).json_str().to_upper().replace(' ', 'T')
|
||||
typ := if json_text.ends_with('Z"') || json_text.all_after('T').contains('-')
|
||||
|| json_text.all_after('T').contains('+') {
|
||||
'datetime'
|
||||
} else {
|
||||
'datetime-local'
|
||||
}
|
||||
// NOTE test suite inconsistency.
|
||||
// It seems it's implementation specific how time and
|
||||
// date-time values are represented in detail. For now we follow the toml-lang format
|
||||
// that expands to 6 digits which is also a valid RFC 3339 representation.
|
||||
json_text = to_iarna_time(json_text[1..json_text.len - 1])
|
||||
if skip_value_map {
|
||||
return json_text
|
||||
}
|
||||
return '{ "type": "${typ}", "value": "${json_text}" }'
|
||||
}
|
||||
ast.Date {
|
||||
json_text := json2.Any(value.text).json_str()
|
||||
if skip_value_map {
|
||||
return json_text
|
||||
}
|
||||
return '{ "type": "date", "value": ${json_text} }'
|
||||
}
|
||||
ast.Time {
|
||||
mut json_text := json2.Any(value.text).json_str()
|
||||
// Note: Removes the quotes of the encoded JSON string - Ned
|
||||
json_text = to_iarna_time(json_text[1..json_text.len - 1])
|
||||
if skip_value_map {
|
||||
return json_text
|
||||
}
|
||||
return '{ "type": "time", "value": "${json_text}" }'
|
||||
}
|
||||
ast.Bool {
|
||||
json_text := json2.Any(value.text.bool()).json_str()
|
||||
if skip_value_map {
|
||||
return json_text
|
||||
}
|
||||
return '{ "type": "bool", "value": "${json_text}" }'
|
||||
}
|
||||
ast.Null {
|
||||
json_text := json2.Any(value.text).json_str()
|
||||
if skip_value_map {
|
||||
return json_text
|
||||
}
|
||||
return '{ "type": "null", "value": ${json_text} }'
|
||||
}
|
||||
ast.Number {
|
||||
if value.text.contains('inf') {
|
||||
mut json_text :=
|
||||
value.text.replace('inf', '1.7976931348623157e+308') // Inconsistency ???
|
||||
if skip_value_map {
|
||||
return '${json_text}'
|
||||
}
|
||||
return '{ "type": "float", "value": "${json_text}" }'
|
||||
}
|
||||
if value.text.contains('nan') {
|
||||
mut json_text := 'null'
|
||||
if skip_value_map {
|
||||
return '${json_text}'
|
||||
}
|
||||
return '{ "type": "float", "value": "${json_text}" }'
|
||||
}
|
||||
if !value.text.starts_with('0x')
|
||||
&& (value.text.contains('.') || value.text.to_lower().contains('e')) {
|
||||
mut val := '${value.f64()}'.replace('.e+', '.0e') // json notation
|
||||
if !val.contains('.') && val != '0' { // json notation
|
||||
val += '.0'
|
||||
}
|
||||
if skip_value_map {
|
||||
return '${val}'
|
||||
}
|
||||
return '{ "type": "float", "value": "${val}" }'
|
||||
}
|
||||
v := value.i64()
|
||||
if skip_value_map {
|
||||
return '${v}'
|
||||
}
|
||||
return '{ "type": "integer", "value": "${v}" }'
|
||||
}
|
||||
map[string]ast.Value {
|
||||
mut str := '{ '
|
||||
for key, val in value {
|
||||
json_key := json2.Any(key).json_str()
|
||||
str += ' ${json_key}: ${to_iarna(val, skip_value_map)},'
|
||||
}
|
||||
str = str.trim_right(',')
|
||||
str += ' }'
|
||||
return str
|
||||
}
|
||||
[]ast.Value {
|
||||
mut str := '[ '
|
||||
for val in value {
|
||||
str += ' ${to_iarna(val, skip_value_map)},'
|
||||
}
|
||||
str = str.trim_right(',')
|
||||
str += ' ]\n'
|
||||
return str
|
||||
}
|
||||
}
|
||||
|
||||
return '<error>'
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import toml
|
||||
|
||||
struct Address {
|
||||
street string
|
||||
city string
|
||||
}
|
||||
|
||||
fn test_inline() {
|
||||
a := Address{'Elm Street', 'Springwood'}
|
||||
|
||||
mut mp := map[string]toml.Any{}
|
||||
mp['street'] = toml.Any(a.street)
|
||||
mp['city'] = toml.Any(a.city)
|
||||
|
||||
assert mp.to_inline_toml() == '{ street = "Elm Street", city = "Springwood" }'
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import os
|
||||
import toml
|
||||
import toml.to
|
||||
|
||||
const fprefix = os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))
|
||||
|
||||
fn test_parse() {
|
||||
toml_doc := toml.parse_file(os.real_path(fprefix + '.toml'))!
|
||||
|
||||
toml_json := to.json(toml_doc)
|
||||
println(toml_json)
|
||||
|
||||
out_file_json := os.read_file(os.real_path(fprefix + '.out'))!
|
||||
assert toml_json == out_file_json
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import os
|
||||
import toml
|
||||
import toml.to
|
||||
|
||||
fn test_parse() {
|
||||
toml_file :=
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.toml'
|
||||
toml_doc := toml.parse_file(toml_file) or { panic(err) }
|
||||
|
||||
toml_json := to.json(toml_doc)
|
||||
out_file :=
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.out'
|
||||
out_file_json := os.read_file(out_file) or { panic(err) }
|
||||
println(toml_json)
|
||||
assert toml_json == out_file_json
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import os
|
||||
import toml
|
||||
import toml.to
|
||||
|
||||
fn path_by_extension(ext string) string {
|
||||
return os.join_path(os.dir(@VEXE), 'vlib/toml/tests/testdata/key_test.${ext}')
|
||||
}
|
||||
|
||||
fn test_keys() {
|
||||
toml_doc := toml.parse_file(path_by_extension('toml'))!
|
||||
|
||||
mut value := toml_doc.value('34-11')
|
||||
assert value.int() == 23
|
||||
|
||||
value = toml_doc.value('1.2')
|
||||
assert value.int() == 3
|
||||
|
||||
value = toml_doc.value('34-12.2')
|
||||
assert value.int() == 42
|
||||
|
||||
toml_json := to.json(toml_doc)
|
||||
out_file_json := os.read_file(path_by_extension('out'))!
|
||||
println(toml_json)
|
||||
assert toml_json == out_file_json
|
||||
|
||||
if x := toml_doc.value_opt('unknown key') {
|
||||
assert false
|
||||
} else {
|
||||
assert err.msg() == 'no value for key'
|
||||
}
|
||||
if x := toml_doc.value_opt("'a") {
|
||||
assert false
|
||||
} else {
|
||||
assert err.msg() == 'invalid dotted key'
|
||||
}
|
||||
}
|
||||
|
||||
fn test_parse_dotted_key() {
|
||||
assert toml.parse_dotted_key('')! == []
|
||||
assert toml.parse_dotted_key('abc')! == ['abc']
|
||||
assert toml.parse_dotted_key('tube.test."test.test".h."i.j."."k"')! == ['tube', 'test',
|
||||
'test.test', 'h', 'i.j.', 'k']
|
||||
if x := toml.parse_dotted_key("'some unclosed string") {
|
||||
assert false
|
||||
} else {
|
||||
assert err.msg().starts_with('parse_dotted_key: could not parse key, missing closing string delimiter')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import os
|
||||
import toml
|
||||
|
||||
// Instructions for developers:
|
||||
// The large (1MB) TOML file can be obtained by doing:
|
||||
// `wget https://gist.githubusercontent.com/Larpon/89b0e3d94c6903851ff15559e5df7a05/raw/62a1f87a4e37bf157f2e0bfb32d85d840c98e422/large_toml_file_test.toml`
|
||||
// `mv large_toml_file_test.toml vlib/toml/tests/testdata/`
|
||||
|
||||
const toml_file = os.join_path_single(@VEXEROOT,
|
||||
'vlib/toml/tests/testdata/large_toml_file_test.toml')
|
||||
|
||||
fn test_large_file() {
|
||||
if !os.exists(toml_file) {
|
||||
eprintln('skipping ${@FILE} since ${toml_file} is missing.')
|
||||
return
|
||||
}
|
||||
println('Testing parsing of large (${os.file_size(toml_file)} bytes) "${toml_file}"...')
|
||||
doc := toml.parse_file(toml_file) or { panic(err) }
|
||||
assert true
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import toml
|
||||
|
||||
const toml_text = '
|
||||
[db]
|
||||
enabled = true
|
||||
|
||||
[servers]
|
||||
# Indentation (tabs and/or spaces) is allowed but not required
|
||||
[servers.alpha]
|
||||
ip = "10.0.0.1"
|
||||
dc = "eqdc10"
|
||||
|
||||
[servers.beta]
|
||||
ip = "10.0.0.2"
|
||||
dc = "eqdc10"
|
||||
|
||||
[servers.alpha.tricky]
|
||||
ip = "10.0.0.100"
|
||||
|
||||
[firewall.rules]
|
||||
block = true
|
||||
|
||||
[firewall.rules.limit]
|
||||
ip = "10.0.0.101"
|
||||
'
|
||||
|
||||
fn test_parse() {
|
||||
toml_doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
// dump(toml_doc.ast)
|
||||
// assert false
|
||||
|
||||
assert toml_doc.value('db.enabled').bool()
|
||||
// TODO: make this work
|
||||
assert toml_doc.value('servers.alpha.ip').string() == '10.0.0.1'
|
||||
assert toml_doc.value('servers.alpha.dc').string() == 'eqdc10'
|
||||
|
||||
assert toml_doc.value('servers.beta.ip').string() == '10.0.0.2'
|
||||
assert toml_doc.value('servers.beta.dc').string() == 'eqdc10'
|
||||
|
||||
assert toml_doc.value('servers.alpha.tricky.ip').string() == '10.0.0.100'
|
||||
assert toml_doc.value('firewall.rules.limit.ip').string() == '10.0.0.101'
|
||||
assert toml_doc.value('firewall.rules.block').bool() == true
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// vtest retry: 3
|
||||
import toml
|
||||
|
||||
fn test_quoted_keys() {
|
||||
str_value := 'V rocks!'
|
||||
toml_txt := 'a."b.c" = "V rocks!"'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
|
||||
value := toml_doc.value('a."b.c"')
|
||||
assert value == toml.Any(str_value)
|
||||
assert value as string == str_value
|
||||
assert value.string() == str_value
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import toml
|
||||
|
||||
fn test_quoted_string_crlf() {
|
||||
toml_txt := 'str1 = """tcc \\\r\nabc \\\r\n123"""'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
|
||||
value := toml_doc.value('str1').string()
|
||||
assert value == 'tcc abc 123'
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import toml
|
||||
|
||||
const toml_text = '# This TOML can reflect to a struct
|
||||
name = "Tom"
|
||||
age = 45
|
||||
height = 1.97
|
||||
|
||||
birthday = 1980-04-23
|
||||
|
||||
strings = [
|
||||
"v matures",
|
||||
"like rings",
|
||||
"spread in the",
|
||||
"water"
|
||||
]
|
||||
|
||||
bools = [true, false, true, true]
|
||||
|
||||
bools_br = [
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
true
|
||||
]
|
||||
|
||||
floats = [0.0, 1.0, 2.0, 3.0]
|
||||
|
||||
floats_br = [
|
||||
0.0,
|
||||
1.05,
|
||||
2.025,
|
||||
3.5001
|
||||
]
|
||||
|
||||
int_map = {"a" = 0, "b" = 1, "c" = 2, "d" = 3}
|
||||
|
||||
[bio]
|
||||
text = "Tom has done many great things"
|
||||
years_of_service = 5
|
||||
|
||||
[field_remap]
|
||||
txt = "I am remapped"
|
||||
uint64 = 100
|
||||
|
||||
[config]
|
||||
data = [ 1, 2, 3 ]
|
||||
levels = { "info" = 1, "warn" = 2, "critical" = 3 }
|
||||
'
|
||||
|
||||
struct FieldRemap {
|
||||
text string @[toml: 'txt']
|
||||
num u64 @[toml: 'uint64']
|
||||
}
|
||||
|
||||
struct Bio {
|
||||
text string
|
||||
years_of_service int
|
||||
}
|
||||
|
||||
struct User {
|
||||
name string
|
||||
age int
|
||||
height f64
|
||||
birthday toml.Date
|
||||
strings []string
|
||||
bools []bool
|
||||
bools_br []bool
|
||||
floats []f32
|
||||
floats_br []f32
|
||||
int_map map[string]int
|
||||
|
||||
config toml.Any
|
||||
mut:
|
||||
bio Bio
|
||||
remap FieldRemap
|
||||
}
|
||||
|
||||
fn test_reflect() {
|
||||
toml_doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
|
||||
mut user := toml_doc.reflect[User]()
|
||||
user.bio = toml_doc.value('bio').reflect[Bio]()
|
||||
user.remap = toml_doc.value('field_remap').reflect[FieldRemap]()
|
||||
|
||||
assert user.name == 'Tom'
|
||||
assert user.age == 45
|
||||
assert user.height == 1.97
|
||||
assert user.birthday.str() == '1980-04-23'
|
||||
assert user.strings == ['v matures', 'like rings', 'spread in the', 'water']
|
||||
assert user.bools == [true, false, true, true]
|
||||
assert user.bools_br == [true, false, true, true]
|
||||
assert user.floats == [f32(0.0), 1.0, 2.0, 3.0]
|
||||
assert user.floats_br == [f32(0.0), 1.05, 2.025, 3.5001]
|
||||
assert user.int_map == {
|
||||
'a': 0
|
||||
'b': 1
|
||||
'c': 2
|
||||
'd': 3
|
||||
}
|
||||
assert user.bio.text == 'Tom has done many great things'
|
||||
assert user.bio.years_of_service == 5
|
||||
|
||||
assert user.remap.text == 'I am remapped'
|
||||
assert user.remap.num == 100
|
||||
|
||||
assert user.config.value('data[0]').int() == 1
|
||||
assert user.config.value('levels.warn').int() == 2
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import toml
|
||||
|
||||
fn test_spaced_keys() {
|
||||
str_value := 'V rocks!'
|
||||
|
||||
toml_txt := '
|
||||
"o" . pq . r = "Yuk"
|
||||
|
||||
[[ a . "b.c" ]]
|
||||
d . e = "V rocks!"
|
||||
|
||||
[ tube . test . "test.test" ]
|
||||
h . "i.j." . "k" = "Cryptic"
|
||||
'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
mut value := toml_doc.value('a."b.c"[0].d.e')
|
||||
assert value == toml.Any(str_value)
|
||||
assert value as string == str_value
|
||||
assert value.string() == str_value
|
||||
|
||||
value = toml_doc.value('"o".pq.r')
|
||||
assert value.string() == 'Yuk'
|
||||
|
||||
value = toml_doc.value('tube.test."test.test".h."i.j."."k"')
|
||||
assert value.string() == 'Cryptic'
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import os
|
||||
import toml
|
||||
|
||||
const toml_multiline_text_1 = 'multi1 = """one"""
|
||||
multi2 = """one
|
||||
two"""
|
||||
multi3 = """
|
||||
one
|
||||
two
|
||||
three"""
|
||||
multi4 = """
|
||||
one
|
||||
two
|
||||
three
|
||||
four
|
||||
"""'
|
||||
|
||||
const toml_multiline_text_2 = "multi1 = '''one'''
|
||||
multi2 = '''one
|
||||
two'''
|
||||
multi3 = '''
|
||||
one
|
||||
two
|
||||
three'''
|
||||
multi4 = '''
|
||||
one
|
||||
two
|
||||
three
|
||||
four
|
||||
'''"
|
||||
|
||||
const toml_unicode_escapes = r'short = "\u03B4"
|
||||
long = "\U000003B4"'
|
||||
|
||||
fn test_multiline_strings() {
|
||||
mut toml_doc := toml.parse_text(toml_multiline_text_1) or { panic(err) }
|
||||
|
||||
mut value := toml_doc.value('multi1')
|
||||
assert value.string() == 'one'
|
||||
value = toml_doc.value('multi2')
|
||||
assert value.string() == 'one\ntwo'
|
||||
value = toml_doc.value('multi3')
|
||||
assert value.string() == 'one\ntwo\nthree'
|
||||
value = toml_doc.value('multi4')
|
||||
assert value.string() == 'one\ntwo\nthree\nfour\n'
|
||||
|
||||
toml_doc = toml.parse_text(toml_multiline_text_2) or { panic(err) }
|
||||
value = toml_doc.value('multi1')
|
||||
assert value.string() == 'one'
|
||||
value = toml_doc.value('multi2')
|
||||
assert value.string() == 'one\ntwo'
|
||||
value = toml_doc.value('multi3')
|
||||
assert value.string() == 'one\ntwo\nthree'
|
||||
value = toml_doc.value('multi4')
|
||||
assert value.string() == 'one\ntwo\nthree\nfour\n'
|
||||
|
||||
toml_file :=
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.toml'
|
||||
toml_doc = toml.parse_file(toml_file) or { panic(err) }
|
||||
value = toml_doc.value('lit_one')
|
||||
assert value.string() == "'one quote'"
|
||||
value = toml_doc.value('lit_two')
|
||||
assert value.string() == "''two quotes''"
|
||||
value = toml_doc.value('mismatch1')
|
||||
assert value.string() == 'aaa' + "'''" + 'bbb'
|
||||
value = toml_doc.value('mismatch2')
|
||||
assert value.string() == 'aaa' + '"""' + 'bbb'
|
||||
}
|
||||
|
||||
fn test_unicode_escapes() {
|
||||
mut toml_doc := toml.parse_text(toml_unicode_escapes) or { panic(err) }
|
||||
|
||||
mut value := toml_doc.value('short')
|
||||
assert value.string() == '\u03B4' // <- This escape is handled by V
|
||||
value = toml_doc.value('long')
|
||||
assert value.string() == 'δ' // <- for the long escape we compare with the unicode point
|
||||
}
|
||||
|
||||
fn test_literal_strings() {
|
||||
toml_file :=
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.toml'
|
||||
toml_doc := toml.parse_file(toml_file) or { panic(err) }
|
||||
|
||||
assert toml_doc.value('lit1').string() == r'\' // '\'
|
||||
assert toml_doc.value('lit2').string() == r'\\' // '\\'
|
||||
assert toml_doc.value('lit3').string() == r'\tricky\' // '\tricky\'
|
||||
|
||||
// NOTE to Windows users: git is set to use Unix EOLs for all TOML files (*.toml) in the repo.
|
||||
// See `.gitattributes` in the project root for the rule in action.
|
||||
// These lines would look like this on Windows:
|
||||
// assert toml_doc.value('ml_lit1').string() == '\r\n\\'
|
||||
assert toml_doc.value('ml_lit1').string() == '\\'
|
||||
assert toml_doc.value('ml_lit2').string() == '\\\n\\'
|
||||
assert toml_doc.value('ml_lit3').string() == '\\\ntricky\\\n'
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import toml
|
||||
|
||||
const toml_table_text = 'inline = {a.b = 42}
|
||||
|
||||
many.dots.here.dot.dot.dot = {a.b.c = 1, a.b.d = 2}
|
||||
|
||||
a = { a.b = 1 }
|
||||
b = { "a"."b" = 1 }
|
||||
c = { a . b = 1 }
|
||||
d = { \'a\' . "b" = 1 }
|
||||
e = {a.b=1}
|
||||
|
||||
[tbl]
|
||||
a.b.c = {d.e=1}
|
||||
|
||||
[tbl.x]
|
||||
a.b.c = {d.e=1}
|
||||
|
||||
[[arr]]
|
||||
t = {a.b=1}
|
||||
T = {a.b=1}
|
||||
|
||||
[[arr]]
|
||||
t = {a.b=2}
|
||||
T = {a.b=2}'
|
||||
|
||||
fn test_tables() {
|
||||
mut toml_doc := toml.parse_text(toml_table_text) or { panic(err) }
|
||||
|
||||
mut value := toml_doc.value('inline.a.b')
|
||||
assert value.int() == 42
|
||||
|
||||
value = toml_doc.value('many.dots.here.dot.dot.dot.a.b.c')
|
||||
assert value.int() == 1
|
||||
|
||||
value = toml_doc.value('many.dots.here.dot.dot.dot.a.b.d')
|
||||
assert value.int() == 2
|
||||
|
||||
value = toml_doc.value('a.a.b')
|
||||
assert value.int() == 1
|
||||
|
||||
value = toml_doc.value('b.a.b')
|
||||
assert value.int() == 1
|
||||
|
||||
value = toml_doc.value('c.a.b')
|
||||
assert value.int() == 1
|
||||
|
||||
value = toml_doc.value('d.a.b')
|
||||
assert value.int() == 1
|
||||
|
||||
value = toml_doc.value('e.a.b')
|
||||
assert value.int() == 1
|
||||
|
||||
value = toml_doc.value('tbl.a.b.c.d.e')
|
||||
assert value.int() == 1
|
||||
|
||||
value = toml_doc.value('tbl.x.a.b.c.d.e')
|
||||
assert value.int() == 1
|
||||
|
||||
mut m := toml_doc.value('tbl') as map[string]toml.Any
|
||||
|
||||
value = m.value('a.b.c.d.e')
|
||||
assert value.int() == 1
|
||||
|
||||
value = m.value('x.a.b.c.d.e')
|
||||
assert value.int() == 1
|
||||
|
||||
arr := toml_doc.value('arr') as []toml.Any
|
||||
|
||||
for i := 0; i < arr.len; i++ {
|
||||
entry := (arr[i] as map[string]toml.Any)
|
||||
value = entry.value('t.a.b')
|
||||
assert value.int() == i + 1
|
||||
value = entry.value('T.a.b')
|
||||
assert value.int() == i + 1
|
||||
}
|
||||
|
||||
arr0 := arr[0] as map[string]toml.Any
|
||||
value = arr0.value('t.a.b')
|
||||
assert value.int() == 1
|
||||
value = arr0.value('T.a.b')
|
||||
assert value.int() == 1
|
||||
|
||||
arr1 := arr[1] as map[string]toml.Any
|
||||
value = arr1.value('t.a.b')
|
||||
assert value.int() == 2
|
||||
value = arr1.value('T.a.b')
|
||||
assert value.int() == 2
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
iarna/
|
||||
toml_lang/
|
||||
toml_rs/
|
||||
large_toml_file_test.toml
|
||||
@@ -0,0 +1 @@
|
||||
{ "products": [ { "name": "Hammer", "sku": 738594937 }, { }, { "name": "Nail", "sku": 284758393, "color": "gray" } ] }
|
||||
@@ -0,0 +1 @@
|
||||
{ "albums": [ { "name": "Born to Run", "songs": [ { "name": "Jungleland" }, { "name": "Meeting Across the River" } ] }, { "name": "Born in the USA", "songs": [ { "name": "Glory Days" }, { "name": "Dancing in the Dark" } ] } ] }
|
||||
@@ -0,0 +1 @@
|
||||
{ "a": [ { "b": [ { "c": { "d": "val0" } }, { "c": { "d": "val1" } } ] } ] }
|
||||
@@ -0,0 +1 @@
|
||||
{ "Toml": [ { "ID": "1", "Index": 0, "Range": [ 0, 1, 2, 3 ], "Greeting": "Hello!", "Name": { "First": "Dolores", "Last": "Alvarado" }, "Friends": [ { "ID": 0, "Name": "Tom" }, { "ID": 1, "Name": "Dollie" } ] }, { "ID": "2", "Index": 1, "Range": [ 0, 1, 2, 3 ], "Greeting": "Hep!", "Name": { "First": "Rush", "Last": "Washington" }, "Friends": [ { "ID": 0, "Name": "Staci" }, { "ID": 1, "Name": "Dollie" } ] } ] }
|
||||
@@ -0,0 +1,33 @@
|
||||
[[Toml]]
|
||||
ID = "1"
|
||||
Index = 0
|
||||
Range = [0, 1, 2, 3]
|
||||
Greeting = "Hello!"
|
||||
[Toml.Name]
|
||||
First = "Dolores"
|
||||
Last = "Alvarado"
|
||||
|
||||
[[Toml.Friends]]
|
||||
ID = 0
|
||||
Name = "Tom"
|
||||
|
||||
[[Toml.Friends]]
|
||||
ID = 1
|
||||
Name = "Dollie"
|
||||
|
||||
[[Toml]]
|
||||
ID = "2"
|
||||
Index = 1
|
||||
Range = [0, 1, 2, 3]
|
||||
Greeting = "Hep!"
|
||||
[Toml.Name]
|
||||
First = "Rush"
|
||||
Last = "Washington"
|
||||
|
||||
[[Toml.Friends]]
|
||||
ID = 0
|
||||
Name = "Staci"
|
||||
|
||||
[[Toml.Friends]]
|
||||
ID = 1
|
||||
Name = "Dollie"
|
||||
@@ -0,0 +1 @@
|
||||
{ "albums": [ { "songs": [ { }, { } ] } ], "artists": [ { "home": { "address": { } } } ] }
|
||||
@@ -0,0 +1,6 @@
|
||||
[[ albums ]]
|
||||
[[ albums.songs ]]
|
||||
[[ albums.songs ]]
|
||||
|
||||
[[ artists ]]
|
||||
[ artists.home.address ]
|
||||
@@ -0,0 +1 @@
|
||||
{ "ひらがな": "ひら" }
|
||||
@@ -0,0 +1 @@
|
||||
"ひらがな" = 'ひら'
|
||||
@@ -0,0 +1 @@
|
||||
{ "v": true, "animal": { "type": { "name": "pug" } }, "inline": { "a": 4, "b.c": 6, "b": { "c": 7 } }, "db": { "t": true }, "ij": { "a": { "i": 1, "j": 2 }, "b": { "i": "3", "j": "4" } }, "fruit": { "apple": { "color": "red", "taste": { "sweet": true }, "texture": { "smooth": true } } } }
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
|
||||
v = true
|
||||
|
||||
animal = { type.name = "pug" }
|
||||
|
||||
inline = { "a" = 4, "b.c" = 6, b.c = 7 }
|
||||
|
||||
[db]
|
||||
t = true
|
||||
|
||||
[ij]
|
||||
[ij.a]
|
||||
i = 1
|
||||
j = 2
|
||||
|
||||
[ij.b]
|
||||
i = "3"
|
||||
j = "4"
|
||||
|
||||
[fruit]
|
||||
apple.color = "red"
|
||||
apple.taste.sweet = true
|
||||
|
||||
[fruit.apple.texture]
|
||||
smooth = true
|
||||
@@ -0,0 +1 @@
|
||||
{ "34-11": 23, "1": { "2": 3 }, "34-12": { "2": 42 }, "34-20": { }, "5": { "6": { } }, "34-13": { "2": { } } }
|
||||
@@ -0,0 +1,9 @@
|
||||
34-11 = 23 # came out as "34-11 " = 23
|
||||
1.2 = 3 # came out as "1.2" = 3
|
||||
34-12.2 = 42 # came out as "34-12.2" = 42
|
||||
|
||||
[34-20]
|
||||
|
||||
[5.6]
|
||||
|
||||
[34-13.2]
|
||||
@@ -0,0 +1,27 @@
|
||||
# Make sure that quotes inside multiline strings are allowed, including right
|
||||
# after the opening '''/""" and before the closing '''/"""
|
||||
|
||||
lit_one = ''''one quote''''
|
||||
lit_two = '''''two quotes'''''
|
||||
lit_one_space = ''' 'one quote' '''
|
||||
lit_two_space = ''' ''two quotes'' '''
|
||||
|
||||
one = """"one quote""""
|
||||
two = """""two quotes"""""
|
||||
one_space = """ "one quote" """
|
||||
two_space = """ ""two quotes"" """
|
||||
|
||||
mismatch1 = """aaa'''bbb"""
|
||||
mismatch2 = '''aaa"""bbb'''
|
||||
|
||||
lit1 = '\'
|
||||
lit2 = '\\'
|
||||
lit3 = '\tricky\'
|
||||
|
||||
ml_lit1 = '''
|
||||
\'''
|
||||
ml_lit2 = '''\
|
||||
\'''
|
||||
ml_lit3 = '''\
|
||||
tricky\
|
||||
'''
|
||||
@@ -0,0 +1 @@
|
||||
{ "title": "TOML Example", "owner": { "name": "Tom Preston-Werner", "dob": "1979-05-27T07:32:00-08:00" }, "database": { "server": "192.168.1.1", "ports": [ 8000, 8001, 8002 ], "connection_max": 5000, "enabled": true }, "servers": { "alpha": { "ip": "10.0.0.1", "dc": "eqdc10" }, "beta": { "ip": "10.0.0.2", "dc": "eqdc10" } }, "clients": { "data": [ [ "gamma", "delta" ], [ 1, 2 ] ], "hosts": [ "alpha", "omega" ] } }
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
# This is a TOML document.
|
||||
|
||||
title = "TOML Example"
|
||||
|
||||
[owner]
|
||||
name = "Tom Preston-Werner"
|
||||
dob = 1979-05-27T07:32:00-08:00 # First class dates
|
||||
|
||||
[database]
|
||||
server = "192.168.1.1"
|
||||
ports = [ 8000, 8001, 8002 ]
|
||||
connection_max = 5000
|
||||
enabled = true
|
||||
|
||||
[servers]
|
||||
|
||||
# Indentation (tabs and/or spaces) is allowed but not required
|
||||
[servers.alpha]
|
||||
ip = "10.0.0.1"
|
||||
dc = "eqdc10"
|
||||
|
||||
[servers.beta]
|
||||
ip = "10.0.0.2"
|
||||
dc = "eqdc10"
|
||||
|
||||
[clients]
|
||||
data = [ ["gamma", "delta"], [1, 2] ]
|
||||
|
||||
# Line breaks are OK when inside arrays
|
||||
hosts = [
|
||||
"alpha",
|
||||
"omega"
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,33 @@
|
||||
# This is a TOML document with an UTF-8 BOM header.
|
||||
|
||||
title = "TOML Example"
|
||||
|
||||
[owner]
|
||||
name = "Tom Preston-Werner"
|
||||
dob = 1979-05-27T07:32:00-08:00 # First class dates
|
||||
|
||||
[database]
|
||||
server = "192.168.1.1"
|
||||
ports = [ 8000, 8001, 8002 ]
|
||||
connection_max = 5000
|
||||
enabled = true
|
||||
|
||||
[servers]
|
||||
|
||||
# Indentation (tabs and/or spaces) is allowed but not required
|
||||
[servers.alpha]
|
||||
ip = "10.0.0.1"
|
||||
dc = "eqdc10"
|
||||
|
||||
[servers.beta]
|
||||
ip = "10.0.0.2"
|
||||
dc = "eqdc10"
|
||||
|
||||
[clients]
|
||||
data = [ ["gamma", "delta"], [1, 2] ]
|
||||
|
||||
# Line breaks are OK when inside arrays
|
||||
hosts = [
|
||||
"alpha",
|
||||
"omega"
|
||||
]
|
||||
@@ -0,0 +1,15 @@
|
||||
import toml
|
||||
|
||||
struct TestStruct {
|
||||
foo int
|
||||
bar bool @[skip]
|
||||
baz string = 'def' @[toml: barbaz]
|
||||
}
|
||||
|
||||
fn test_toml_attr_encode() {
|
||||
assert toml.encode(TestStruct{}) == 'foo = 0\nbarbaz = "def"'
|
||||
}
|
||||
|
||||
fn test_toml_attr_decode() {
|
||||
assert toml.decode[TestStruct]('foo = 0\nbarbaz = "def"')! == TestStruct{}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import os
|
||||
import toml
|
||||
import toml.to
|
||||
import toml.ast
|
||||
|
||||
const empty_toml_document = toml.Doc{
|
||||
ast: &ast.Root(unsafe { nil })
|
||||
}
|
||||
|
||||
const toml_text_with_utf8_bom = os.read_file(os.real_path(os.join_path(os.dir(@FILE), 'testdata',
|
||||
|
||||
'toml_with_utf8_bom' + '.toml'))) or { panic(err) }
|
||||
const toml_text_with_utf16_bom = os.read_file(os.real_path(os.join_path(os.dir(@FILE), 'testdata',
|
||||
|
||||
'toml_with_utf16_bom' + '.toml'))) or { panic(err) }
|
||||
const toml_text_with_utf32_bom = os.read_file(os.real_path(os.join_path(os.dir(@FILE), 'testdata',
|
||||
|
||||
'toml_with_utf32_bom' + '.toml'))) or { panic(err) }
|
||||
|
||||
fn test_toml_with_bom() {
|
||||
toml_doc := toml.parse_text(toml_text_with_utf8_bom) or { panic(err) }
|
||||
toml_json := to.json(toml_doc)
|
||||
|
||||
title := toml_doc.value('title')
|
||||
assert title == toml.Any('TOML Example')
|
||||
assert title as string == 'TOML Example'
|
||||
|
||||
owner := toml_doc.value('owner') as map[string]toml.Any
|
||||
any_name := owner.value('name')
|
||||
assert any_name.string() == 'Tom Preston-Werner'
|
||||
|
||||
database := toml_doc.value('database') as map[string]toml.Any
|
||||
db_serv := database['server'] or {
|
||||
panic('could not access "server" index in "database" variable')
|
||||
}
|
||||
assert db_serv as string == '192.168.1.1'
|
||||
|
||||
// Re-cycle bad_toml_doc
|
||||
mut bad_toml_doc := empty_toml_document
|
||||
bad_toml_doc = toml.parse_text(toml_text_with_utf16_bom) or {
|
||||
println(' ${err.msg()}')
|
||||
assert true
|
||||
empty_toml_document
|
||||
}
|
||||
|
||||
bad_toml_doc = toml.parse_text(toml_text_with_utf32_bom) or {
|
||||
println(' ${err.msg()}')
|
||||
assert true
|
||||
empty_toml_document
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
// Instructions for developers:
|
||||
// The actual tests and data can be obtained by doing:
|
||||
// `git clone -n https://github.com/toml-lang/toml-test.git vlib/toml/tests/testdata/toml_lang`
|
||||
// `git -C vlib/toml/tests/testdata/toml_lang reset --hard 229ce2e
|
||||
// See also the CI toml tests
|
||||
import os
|
||||
import toml
|
||||
import toml.ast
|
||||
import x.json2
|
||||
|
||||
const test_root = os.join_path(@VEXEROOT, 'vlib/toml/tests/testdata/toml_lang/tests')
|
||||
const test_files_file = os.join_path(test_root, 'files-toml-1.0.0')
|
||||
|
||||
const hide_oks = os.getenv('VTEST_HIDE_OK') == '1'
|
||||
const no_jq = os.getenv('VNO_JQ') == '1'
|
||||
|
||||
// Kept for easier lookup and handling of future updates to the tests.
|
||||
// NOTE: entries in this list, except `do_not_remove`, are valid TOML that the parser should work with, but currently does not.
|
||||
const valid_exceptions = [
|
||||
'do_not_remove',
|
||||
]
|
||||
// NOTE: entries in this list, except `do_not_remove`, are tests of invalid TOML that should have the parser fail, but currently does not.
|
||||
const invalid_exceptions = [
|
||||
'do_not_remove',
|
||||
]
|
||||
const valid_value_exceptions = [
|
||||
'do_not_remove',
|
||||
]
|
||||
const jq_not_equal = [
|
||||
'do_not_remove',
|
||||
]
|
||||
|
||||
const jq = os.find_abs_path_of_executable('jq') or { '' }
|
||||
const compare_work_dir_root = os.join_path(os.vtmp_dir(), 'toml_toml_lang')
|
||||
// From: https://stackoverflow.com/a/38266731/1904615
|
||||
const jq_normalize = r'# Apply f to composite entities recursively using keys[], and to atoms
|
||||
def sorted_walk(f):
|
||||
. as $in
|
||||
| if type == "object" then
|
||||
reduce keys[] as $key
|
||||
( {}; . + { ($key): ($in[$key] | sorted_walk(f)) } ) | f
|
||||
elif type == "array" then map( sorted_walk(f) ) | f
|
||||
else f
|
||||
end;
|
||||
|
||||
def normalize: sorted_walk(if type == "array" then sort else . end);
|
||||
|
||||
normalize'
|
||||
|
||||
fn run(args []string) !string {
|
||||
res := os.execute(args.join(' '))
|
||||
if res.exit_code != 0 {
|
||||
return error('${args[0]} failed with return code ${res.exit_code}.\n${res.output}')
|
||||
}
|
||||
return res.output
|
||||
}
|
||||
|
||||
// test_toml_lang_tomltest run though 'testdata/toml_lang/toml-test/*' if found.
|
||||
fn test_toml_lang_tomltest() {
|
||||
eprintln('> running ${@LOCATION}')
|
||||
if !os.is_dir(test_root) {
|
||||
println('No test data directory found in "${test_root}"')
|
||||
assert true
|
||||
return
|
||||
}
|
||||
test_files_list := os.read_lines(test_files_file) or {
|
||||
panic('Could not read "${test_files_file}" with test files list. It should reside in "${test_root}": ${err.msg()}')
|
||||
}
|
||||
|
||||
valid_folder := 'valid'
|
||||
valid_test_files :=
|
||||
test_files_list.filter(it.starts_with('valid/') && it.ends_with('.toml')).map(it.replace('\\', '/'))
|
||||
invalid_folder := 'invalid'
|
||||
invalid_test_files :=
|
||||
test_files_list.filter(it.starts_with('invalid/') && it.ends_with('.toml')).map(it.replace('\\', '/'))
|
||||
|
||||
assert valid_test_files.len > 0, 'Expected a list of *valid* test files'
|
||||
assert invalid_test_files.len > 0, 'Expected a list of *invalid* test files'
|
||||
|
||||
assert os.is_file(os.join_path(test_root, valid_test_files[0])), 'Expected at least one file from the valid list to be present'
|
||||
assert os.is_file(os.join_path(test_root, invalid_test_files[0])), 'Expected at least one file from the invalid list to present'
|
||||
|
||||
println('\nTesting ${valid_test_files.len} valid TOML files...')
|
||||
mut valid := 0
|
||||
mut e := 0
|
||||
for i, relative_valid_test_file in valid_test_files {
|
||||
relative := relative_valid_test_file.all_after(valid_folder).trim_left('/')
|
||||
valid_test_file := os.join_path(test_root, relative_valid_test_file)
|
||||
if relative in valid_exceptions {
|
||||
e++
|
||||
idx := valid_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${valid_test_files.len}] "${valid_test_file}" VALID EXCEPTION [${idx}/${valid_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
// eprintln('>>> trying to parse: ${valid_test_file} | relative: ${relative}')
|
||||
toml_doc := toml.parse_file(valid_test_file) or {
|
||||
eprintln('>>> error while parsing: ${valid_test_file}')
|
||||
panic(err)
|
||||
}
|
||||
valid++
|
||||
if !hide_oks {
|
||||
println('OK [${i + 1}/${valid_test_files.len}] "${valid_test_file}"...')
|
||||
}
|
||||
}
|
||||
println('${valid}/${valid_test_files.len} TOML files were parsed correctly')
|
||||
if valid_exceptions.len > 0 {
|
||||
println('TODO Skipped parsing of ${valid_exceptions.len} valid TOML files...')
|
||||
}
|
||||
|
||||
// If the command-line tool `jq` is installed, value tests can be run as well.
|
||||
if jq != '' && !no_jq {
|
||||
println('\nTesting value output of ${valid_test_files.len} valid TOML files using "${jq}"...')
|
||||
|
||||
if os.exists(compare_work_dir_root) {
|
||||
os.rmdir_all(compare_work_dir_root)!
|
||||
}
|
||||
os.mkdir_all(compare_work_dir_root)!
|
||||
|
||||
jq_normalize_path := os.join_path(compare_work_dir_root, 'normalize.jq')
|
||||
os.write_file(jq_normalize_path, jq_normalize)!
|
||||
|
||||
valid = 0
|
||||
e = 0
|
||||
for i, relative_valid_test_file in valid_test_files {
|
||||
relative := relative_valid_test_file.all_after(valid_folder).trim_left('/')
|
||||
valid_test_file := os.join_path(test_root, relative_valid_test_file)
|
||||
// Skip the file if we know it can't be parsed or we know that the value retrieval needs work.
|
||||
if relative in valid_exceptions {
|
||||
e++
|
||||
idx := valid_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${valid_test_files.len}] "${valid_test_file}" VALID EXCEPTION [${idx}/${valid_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
if relative in valid_value_exceptions {
|
||||
e++
|
||||
idx := valid_value_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${valid_test_files.len}] "${valid_test_file}" VALID VALUE EXCEPTION [${idx}/${valid_value_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
|
||||
if !hide_oks {
|
||||
println('OK [${i + 1}/${valid_test_files.len}] "${valid_test_file}"...')
|
||||
}
|
||||
toml_doc := toml.parse_file(valid_test_file)!
|
||||
// eprintln(' relative: ${relative} parsed')
|
||||
|
||||
v_toml_json_path := os.join_path(compare_work_dir_root,
|
||||
|
||||
os.file_name(valid_test_file).all_before_last('.') + '.v.json')
|
||||
bs_toml_json_path := os.join_path(compare_work_dir_root,
|
||||
|
||||
os.file_name(valid_test_file).all_before_last('.') + '.json')
|
||||
|
||||
os.write_file(v_toml_json_path, to_toml_lang(toml_doc.ast.table))!
|
||||
|
||||
bs_json := os.read_file(valid_test_file.all_before_last('.') + '.json')!
|
||||
|
||||
os.write_file(bs_toml_json_path, bs_json)!
|
||||
|
||||
v_normalized_json := run([jq, '-S', '-f "${jq_normalize_path}"', v_toml_json_path]) or {
|
||||
contents := os.read_file(v_toml_json_path)!
|
||||
panic(err.msg() + '\n${contents}')
|
||||
}
|
||||
bs_normalized_json := run([jq, '-S', '-f "${jq_normalize_path}"', bs_toml_json_path]) or {
|
||||
contents := os.read_file(v_toml_json_path)!
|
||||
panic(err.msg() + '\n${contents}')
|
||||
}
|
||||
|
||||
if relative in jq_not_equal {
|
||||
e++
|
||||
eprintln('>>> skipped: relative: ${relative} in jq_not_equal, bs_normalized_json != bs_normalized_json')
|
||||
continue
|
||||
}
|
||||
|
||||
if bs_normalized_json != v_normalized_json {
|
||||
e++
|
||||
eprintln('>>> error: relative: ${relative}, bs_normalized_json != bs_normalized_json')
|
||||
continue
|
||||
}
|
||||
|
||||
assert bs_normalized_json == v_normalized_json
|
||||
valid++
|
||||
}
|
||||
} else {
|
||||
println('> Skipping json conversion tests, since jq: ${jq} | no_jq: ${no_jq}')
|
||||
}
|
||||
println('${valid}/${valid_test_files.len} TOML files were parsed correctly and value checked')
|
||||
if valid_value_exceptions.len > 0 {
|
||||
println('TODO Skipped value checks of ${valid_value_exceptions.len} valid TOML files...')
|
||||
}
|
||||
|
||||
println('\nTesting ${invalid_test_files.len} invalid TOML files...')
|
||||
mut invalid := 0
|
||||
e = 0
|
||||
for i, relative_invalid_test_file in invalid_test_files {
|
||||
relative := relative_invalid_test_file.all_after(invalid_folder).trim_left('/')
|
||||
invalid_test_file := os.join_path(test_root, relative_invalid_test_file)
|
||||
if relative in invalid_exceptions {
|
||||
e++
|
||||
idx := invalid_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${invalid_test_files.len}] "${invalid_test_file}" INVALID EXCEPTION [${idx}/${invalid_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
if !hide_oks {
|
||||
println('OK [${i + 1}/${invalid_test_files.len}] "${invalid_test_file}"...')
|
||||
}
|
||||
if toml_doc := toml.parse_file(invalid_test_file) {
|
||||
content_that_should_have_failed := os.read_file(invalid_test_file)!
|
||||
println(' This TOML from relative: ${relative}, invalid_test_file: ${invalid_test_file}, should have failed:\n${'-'.repeat(100)}\n${content_that_should_have_failed}\n${'-'.repeat(100)}')
|
||||
assert false
|
||||
} else {
|
||||
if !hide_oks {
|
||||
println(' ${err.msg()}')
|
||||
}
|
||||
assert true
|
||||
}
|
||||
invalid++
|
||||
}
|
||||
println('${invalid}/${invalid_test_files.len} invalid TOML files correctly had parsing errors')
|
||||
if invalid_exceptions.len > 0 {
|
||||
println('TODO Skipped parsing of ${invalid_exceptions.len} invalid TOML files...')
|
||||
}
|
||||
}
|
||||
|
||||
// to_toml_lang returns a toml-lang compatible json string converted from the `value` ast.Value.
|
||||
fn to_toml_lang(value ast.Value) string {
|
||||
match value {
|
||||
ast.Quoted {
|
||||
json_text := json2.Any(value.text).json_str()
|
||||
return '{ "type": "string", "value": ${json_text} }'
|
||||
}
|
||||
ast.DateTime {
|
||||
// Normalization for json
|
||||
json_text := json2.Any(value.text).json_str().to_upper().replace(' ', 'T')
|
||||
|
||||
// Note: Since encoding strings in JSON now automatically includes quotes,
|
||||
// I added a somewhat a workaround by adding an ending quote in order to
|
||||
// recognize properly the date time type. - Ned
|
||||
typ := if json_text.ends_with('Z"') || json_text.all_after('T').contains('-')
|
||||
|| json_text.all_after('T').contains('+') {
|
||||
'datetime'
|
||||
} else {
|
||||
'datetime-local'
|
||||
}
|
||||
return '{ "type": "${typ}", "value": ${json_text} }'
|
||||
}
|
||||
ast.Date {
|
||||
json_text := json2.Any(value.text).json_str()
|
||||
return '{ "type": "date-local", "value": ${json_text} }'
|
||||
}
|
||||
ast.Time {
|
||||
json_text := json2.Any(value.text).json_str()
|
||||
return '{ "type": "time-local", "value": ${json_text} }'
|
||||
}
|
||||
ast.Bool {
|
||||
json_text := json2.Any(value.text.bool()).json_str()
|
||||
return '{ "type": "bool", "value": "${json_text}" }'
|
||||
}
|
||||
ast.Null {
|
||||
json_text := json2.Any(value.text).json_str()
|
||||
return '{ "type": "null", "value": ${json_text} }'
|
||||
}
|
||||
ast.Number {
|
||||
if value.text.contains('inf') || value.text.contains('nan') {
|
||||
return '{ "type": "float", "value": "${value.text}" }'
|
||||
}
|
||||
if !value.text.starts_with('0x')
|
||||
&& (value.text.contains('.') || value.text.to_lower().contains('e')) {
|
||||
mut val := '${value.f64()}'.replace('.e+', '.0e') // JSON notation
|
||||
if !val.contains('.') && val != '0' { // JSON notation
|
||||
val += '.0'
|
||||
}
|
||||
// Since https://github.com/vlang/v/pull/16079 V's string conversion of a zero (0) will
|
||||
// output "0.0" for float types - the JSON test suite data, however, expects "0" for floats
|
||||
// The following is a correction for that inconsistency
|
||||
if val == '0.0' {
|
||||
val = '0'
|
||||
}
|
||||
return '{ "type": "float", "value": "${val}" }'
|
||||
}
|
||||
return '{ "type": "integer", "value": "${value.i64()}" }'
|
||||
}
|
||||
map[string]ast.Value {
|
||||
mut str := '{ '
|
||||
for key, val in value {
|
||||
json_key := json2.Any(key).json_str()
|
||||
str += ' ${json_key}: ${to_toml_lang(val)},'
|
||||
}
|
||||
str = str.trim_right(',')
|
||||
str += ' }'
|
||||
return str
|
||||
}
|
||||
[]ast.Value {
|
||||
mut str := '[ '
|
||||
for val in value {
|
||||
str += ' ${to_toml_lang(val)},'
|
||||
}
|
||||
str = str.trim_right(',')
|
||||
str += ' ]\n'
|
||||
return str
|
||||
}
|
||||
}
|
||||
|
||||
return '<error>'
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// This tests the `toml` module for a known memory corruption.
|
||||
// The BUG shows below if no string `.clone()` nor any garbage-collection is done...
|
||||
import os
|
||||
import toml
|
||||
|
||||
const toml_text = os.read_file(os.real_path(os.join_path(os.dir(@FILE), 'testdata', 'toml_test')) +
|
||||
'.toml') or { panic(err) }
|
||||
|
||||
fn test_toml_known_memory_corruption() {
|
||||
toml_doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
|
||||
owner := toml_doc.value('owner') as map[string]toml.Any
|
||||
any_name := owner.value('name')
|
||||
// This assert code path will cause the corruption.
|
||||
assert any_name.string() == 'Tom Preston-Werner'
|
||||
|
||||
// This code then triggered the bug before the fix.
|
||||
// Also see note in toml/any.v in function `pub fn (a Any) string() string`
|
||||
assert toml_doc.value('owner.name') as string == 'Tom Preston-Werner'
|
||||
|
||||
// Repeat the pattern
|
||||
assert any_name.string() == 'Tom Preston-Werner'
|
||||
assert toml_doc.value('owner.name') as string == 'Tom Preston-Werner'
|
||||
}
|
||||
|
||||
fn test_toml_known_memory_corruption_2() {
|
||||
toml_txt := '
|
||||
# Local Date-Time
|
||||
ldt1 = 1979-05-27T07:32:00
|
||||
ldt2 = 1979-05-27T00:32:00.999999
|
||||
# Local Date
|
||||
ld1 = 1979-05-27
|
||||
# Local Time
|
||||
lt1 = 07:32:00
|
||||
lt2 = 00:32:00.999999
|
||||
'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
|
||||
// ldt1 test section
|
||||
odt_time := toml.DateTime{'1979-05-27T07:32:00'}
|
||||
ldt1 := toml_doc.value('ldt1')
|
||||
assert ldt1.string() == '1979-05-27T07:32:00' // Running this before yielded a double free segfault
|
||||
assert ldt1.datetime() == odt_time
|
||||
assert ldt1.string() == '1979-05-27T07:32:00' // Used to yield a memory corruption
|
||||
|
||||
// ld1 test section
|
||||
od_time := toml.Date{'1979-05-27'}
|
||||
ld1 := toml_doc.value('ld1')
|
||||
assert ld1.date() == od_time
|
||||
assert ld1.string() == '1979-05-27' // Used to yield a memory corruption
|
||||
|
||||
// lt1 test section
|
||||
mut ot_time := toml.Time{'07:32:00'}
|
||||
lt1 := toml_doc.value('lt1')
|
||||
assert lt1.time() == ot_time
|
||||
assert lt1.string() == '07:32:00' // Used to yield a memory corruption
|
||||
|
||||
// lt2 test section
|
||||
ot_time = toml.Time{'00:32:00.999999'}
|
||||
lt2 := toml_doc.value('lt2')
|
||||
assert lt2.time() == ot_time
|
||||
assert lt2.string() == '00:32:00.999999' // Used to yield a memory corruption
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
import os
|
||||
import toml
|
||||
import toml.ast
|
||||
import x.json2
|
||||
|
||||
const hide_oks = os.getenv('VTEST_HIDE_OK') == '1'
|
||||
const no_jq = os.getenv('VNO_JQ') == '1'
|
||||
|
||||
// Instructions for developers:
|
||||
// The actual tests and data can be obtained by doing:
|
||||
// `git clone -n https://github.com/toml-rs/toml.git vlib/toml/tests/testdata/toml_rs`
|
||||
// `git -C vlib/toml/tests/testdata/toml_rs reset --hard 9bd454c`
|
||||
// See also the CI toml tests
|
||||
// Kept for easier handling of future updates to the tests
|
||||
const valid_exceptions = [
|
||||
'valid/example-v0.3.0.toml',
|
||||
'valid/example-v0.4.0.toml',
|
||||
'valid/datetime-truncate.toml', // Not considered valid since RFC 3339 doesn't permit > 6 ms digits ??
|
||||
]
|
||||
const invalid_exceptions = []string{}
|
||||
|
||||
const valid_value_exceptions = [
|
||||
// These have correct values, and should've passed, but the format of arrays is *mixed* in the JSON ??
|
||||
'valid/datetime-truncate.toml',
|
||||
'valid/example2.toml',
|
||||
'valid/example-v0.4.0.toml',
|
||||
'valid/example-v0.3.0.toml',
|
||||
]
|
||||
|
||||
// These have correct values, and should've passed as-is, but the format of arrays changes in the JSON ??
|
||||
// We account for that here
|
||||
const use_type_2_arrays = [
|
||||
'valid/table-array-implicit.toml',
|
||||
'valid/table-array-many.toml',
|
||||
'valid/table-array-one.toml',
|
||||
'valid/table-array-nest.toml',
|
||||
'valid/table-array-nest-no-keys.toml',
|
||||
]
|
||||
const tests_folder = os.join_path('crates', 'test-suite', 'tests')
|
||||
const jq = os.find_abs_path_of_executable('jq') or { '' }
|
||||
const compare_work_dir_root = os.join_path(os.vtmp_dir(), 'toml_toml_rs')
|
||||
// From: https://stackoverflow.com/a/38266731/1904615
|
||||
const jq_normalize = r'# Apply f to composite entities recursively using keys[], and to atoms
|
||||
def sorted_walk(f):
|
||||
. as $in
|
||||
| if type == "object" then
|
||||
reduce keys[] as $key
|
||||
( {}; . + { ($key): ($in[$key] | sorted_walk(f)) } ) | f
|
||||
elif type == "array" then map( sorted_walk(f) ) | f
|
||||
else f
|
||||
end;
|
||||
|
||||
def normalize: sorted_walk(if type == "array" then sort else . end);
|
||||
|
||||
normalize'
|
||||
|
||||
fn run(args []string) !string {
|
||||
res := os.execute(args.join(' '))
|
||||
if res.exit_code != 0 {
|
||||
return error('${args[0]} failed with return code ${res.exit_code}.\n${res.output}')
|
||||
}
|
||||
return res.output
|
||||
}
|
||||
|
||||
// test_toml_rs_toml_rs run though 'testdata/toml_rs/crates/test-suite/tests/*' if found.
|
||||
fn test_toml_rs_toml_rs() {
|
||||
eprintln('> running ${@LOCATION}')
|
||||
test_root := '${@VEXEROOT}/vlib/toml/tests/testdata/toml_rs/crates/test-suite/tests'
|
||||
if os.is_dir(test_root) {
|
||||
valid_test_files := os.walk_ext(os.join_path(test_root, 'valid'), '.toml')
|
||||
invalid_test_files := os.walk_ext(os.join_path(test_root, 'invalid'), '.toml')
|
||||
println('Testing ${valid_test_files.len} valid TOML files...')
|
||||
mut valid := 0
|
||||
mut e := 0
|
||||
for i, valid_test_file in valid_test_files {
|
||||
mut relative := valid_test_file.all_after(tests_folder).trim_left(os.path_separator)
|
||||
$if windows {
|
||||
relative = relative.replace('/', '\\')
|
||||
}
|
||||
|
||||
if relative in valid_exceptions {
|
||||
e++
|
||||
idx := valid_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${valid_test_files.len}] "${valid_test_file}" VALID EXCEPTION [${idx}/${valid_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
if !hide_oks {
|
||||
println('OK [${i + 1}/${valid_test_files.len}] "${valid_test_file}"...')
|
||||
}
|
||||
toml_doc := toml.parse_file(valid_test_file)!
|
||||
valid++
|
||||
}
|
||||
println('${valid}/${valid_test_files.len} TOML files were parsed correctly')
|
||||
if valid_exceptions.len > 0 {
|
||||
println('TODO Skipped parsing of ${e} valid TOML files...')
|
||||
}
|
||||
|
||||
// If the command-line tool `jq` is installed, value tests can be run as well.
|
||||
if jq != '' && !no_jq {
|
||||
println('Testing value output of ${valid_test_files.len} valid TOML files using "${jq}"...')
|
||||
|
||||
if os.exists(compare_work_dir_root) {
|
||||
os.rmdir_all(compare_work_dir_root)!
|
||||
}
|
||||
os.mkdir_all(compare_work_dir_root)!
|
||||
|
||||
jq_normalize_path := os.join_path(compare_work_dir_root, 'normalize.jq')
|
||||
os.write_file(jq_normalize_path, jq_normalize)!
|
||||
|
||||
valid = 0
|
||||
e = 0
|
||||
for i, valid_test_file in valid_test_files {
|
||||
mut relative := valid_test_file.all_after(tests_folder).trim_left(os.path_separator)
|
||||
$if windows {
|
||||
relative = relative.replace('/', '\\')
|
||||
}
|
||||
if !os.exists(valid_test_file.all_before_last('.') + '.json') {
|
||||
println('N/A [${i + 1}/${valid_test_files.len}] "${valid_test_file}"...')
|
||||
continue
|
||||
}
|
||||
// Skip the file if we know it can't be parsed or we know that the value retrieval needs work.
|
||||
if relative in valid_exceptions {
|
||||
e++
|
||||
idx := valid_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${valid_test_files.len}] "${valid_test_file}" VALID EXCEPTION [${idx}/${valid_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
if relative in valid_value_exceptions {
|
||||
e++
|
||||
idx := valid_value_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${valid_test_files.len}] "${valid_test_file}" VALID VALUE EXCEPTION [${idx}/${valid_value_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
|
||||
if !hide_oks {
|
||||
println('OK [${i + 1}/${valid_test_files.len}] "${valid_test_file}"...')
|
||||
}
|
||||
toml_doc := toml.parse_file(valid_test_file)!
|
||||
|
||||
v_toml_json_path := os.join_path(compare_work_dir_root,
|
||||
|
||||
os.file_name(valid_test_file).all_before_last('.') + '.v.json')
|
||||
toml_rs_toml_json_path := os.join_path(compare_work_dir_root,
|
||||
|
||||
os.file_name(valid_test_file).all_before_last('.') + '.json')
|
||||
|
||||
mut array_type := 1
|
||||
if relative in use_type_2_arrays {
|
||||
array_type = 2
|
||||
}
|
||||
|
||||
os.write_file(v_toml_json_path, to_toml_rs(toml_doc.ast.table, array_type))!
|
||||
|
||||
toml_rs_json := os.read_file(valid_test_file.all_before_last('.') + '.json')!
|
||||
|
||||
os.write_file(toml_rs_toml_json_path, toml_rs_json)!
|
||||
|
||||
v_normalized_json := run([jq, '-S', '-f "${jq_normalize_path}"', v_toml_json_path]) or {
|
||||
contents := os.read_file(v_toml_json_path)!
|
||||
panic(err.msg() + '\n${contents}')
|
||||
}
|
||||
toml_rs_normalized_json := run([jq, '-S', '-f "${jq_normalize_path}"',
|
||||
toml_rs_toml_json_path]) or {
|
||||
contents := os.read_file(v_toml_json_path)!
|
||||
panic(err.msg() + '\n${contents}')
|
||||
}
|
||||
|
||||
assert toml_rs_normalized_json == v_normalized_json
|
||||
|
||||
valid++
|
||||
}
|
||||
println('${valid}/${valid_test_files.len} TOML files were parsed correctly and value checked')
|
||||
if valid_value_exceptions.len > 0 {
|
||||
println('TODO Skipped value checks of ${e} valid TOML files...')
|
||||
}
|
||||
}
|
||||
|
||||
println('Testing ${invalid_test_files.len} invalid TOML files...')
|
||||
mut invalid := 0
|
||||
e = 0
|
||||
for i, invalid_test_file in invalid_test_files {
|
||||
mut relative := invalid_test_file.all_after(tests_folder).trim_left(os.path_separator)
|
||||
$if windows {
|
||||
relative = relative.replace('/', '\\')
|
||||
}
|
||||
if relative in invalid_exceptions {
|
||||
e++
|
||||
idx := invalid_exceptions.index(relative) + 1
|
||||
println('SKIP [${i + 1}/${invalid_test_files.len}] "${invalid_test_file}" INVALID EXCEPTION [${idx}/${invalid_exceptions.len}]...')
|
||||
continue
|
||||
}
|
||||
|
||||
if !hide_oks {
|
||||
println('OK [${i + 1}/${invalid_test_files.len}] "${invalid_test_file}"...')
|
||||
}
|
||||
if toml_doc := toml.parse_file(invalid_test_file) {
|
||||
content_that_should_have_failed := os.read_file(invalid_test_file)!
|
||||
println(' This TOML should have failed:\n${'-'.repeat(40)}\n${content_that_should_have_failed}\n${'-'.repeat(40)}')
|
||||
assert false
|
||||
} else {
|
||||
if !hide_oks {
|
||||
println(' ${err.msg()}')
|
||||
}
|
||||
assert true
|
||||
}
|
||||
invalid++
|
||||
}
|
||||
println('${invalid}/${invalid_test_files.len} TOML files were parsed correctly')
|
||||
if invalid_exceptions.len > 0 {
|
||||
println('TODO Skipped parsing of ${invalid_exceptions.len} invalid TOML files...')
|
||||
}
|
||||
} else {
|
||||
println('No test data directory found in "${test_root}"')
|
||||
assert true
|
||||
}
|
||||
}
|
||||
|
||||
// to_toml_rs_time
|
||||
fn to_toml_rs_time(time_str string) string {
|
||||
if time_str.contains('.') {
|
||||
date_and_time := time_str.all_before('.')
|
||||
mut ms := time_str.all_after('.')
|
||||
z := if ms.contains('Z') { 'Z' } else { '' }
|
||||
ms = ms.replace('Z', '')
|
||||
if ms.len > 3 {
|
||||
ms = ms[..3]
|
||||
}
|
||||
return date_and_time + '.' + ms + z
|
||||
} else {
|
||||
return time_str
|
||||
}
|
||||
}
|
||||
|
||||
// to_toml_rs returns an toml_rs compatible json string converted from the `value` ast.Value.
|
||||
fn to_toml_rs(value ast.Value, array_type int) string {
|
||||
match value {
|
||||
ast.Quoted {
|
||||
json_text := json2.Any(value.text).json_str()
|
||||
return '{ "type": "string", "value": ${json_text} }'
|
||||
}
|
||||
ast.DateTime {
|
||||
// Normalization for json
|
||||
mut json_text := json2.Any(value.text).json_str().to_upper().replace(' ', 'T')
|
||||
typ := if json_text.ends_with('Z"') || json_text.all_after('T').contains('-')
|
||||
|| json_text.all_after('T').contains('+') {
|
||||
'datetime'
|
||||
} else {
|
||||
'datetime-local'
|
||||
}
|
||||
// NOTE test suite inconsistency.
|
||||
// It seems it's implementation specific how time and
|
||||
// date-time values are represented in detail. For now we follow the toml-lang format
|
||||
// that expands to 6 digits which is also a valid RFC 3339 representation.
|
||||
json_text = to_toml_rs_time(json_text[1..json_text.len - 1])
|
||||
return '{ "type": "${typ}", "value": "${json_text}" }'
|
||||
}
|
||||
ast.Date {
|
||||
json_text := json2.Any(value.text).json_str()
|
||||
return '{ "type": "date", "value": ${json_text} }'
|
||||
}
|
||||
ast.Time {
|
||||
mut json_text := json2.Any(value.text).json_str()
|
||||
json_text = to_toml_rs_time(json_text[1..json_text.len - 1])
|
||||
return '{ "type": "time", "value": "${json_text}" }'
|
||||
}
|
||||
ast.Bool {
|
||||
json_text := json2.Any(value.text.bool()).json_str()
|
||||
return '{ "type": "bool", "value": "${json_text}" }'
|
||||
}
|
||||
ast.Null {
|
||||
json_text := json2.Any(value.text).json_str()
|
||||
return '{ "type": "null", "value": ${json_text} }'
|
||||
}
|
||||
ast.Number {
|
||||
text := value.text
|
||||
if text.contains('inf') || text.contains('nan') {
|
||||
return '{ "type": "float", "value": ${value.text} }'
|
||||
}
|
||||
if !text.starts_with('0x') && (text.contains('.') || text.to_lower().contains('e')) {
|
||||
mut val := ''
|
||||
if text.to_lower().contains('e') && !text.contains('-') {
|
||||
val = '${value.f64():.1f}'
|
||||
} else {
|
||||
val = '${value.f64()}'
|
||||
}
|
||||
return '{ "type": "float", "value": "${val}" }'
|
||||
}
|
||||
return '{ "type": "integer", "value": "${value.i64()}" }'
|
||||
}
|
||||
map[string]ast.Value {
|
||||
mut str := '{ '
|
||||
for key, val in value {
|
||||
json_key := json2.Any(key).json_str()
|
||||
str += ' ${json_key}: ${to_toml_rs(val, array_type)},'
|
||||
}
|
||||
str = str.trim_right(',')
|
||||
str += ' }'
|
||||
return str
|
||||
}
|
||||
[]ast.Value {
|
||||
mut str := ''
|
||||
if array_type == 1 {
|
||||
str = '{ "type": "array", "value": [ '
|
||||
} else {
|
||||
str = '[ '
|
||||
}
|
||||
for val in value {
|
||||
str += ' ${to_toml_rs(val, array_type)},'
|
||||
}
|
||||
str = str.trim_right(',')
|
||||
if array_type == 1 {
|
||||
str += ' ] }\n'
|
||||
} else {
|
||||
str += ' ]\n'
|
||||
}
|
||||
return str
|
||||
}
|
||||
}
|
||||
|
||||
return '<error>'
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import os
|
||||
import toml
|
||||
import toml.to
|
||||
|
||||
const toml_text = os.read_file(
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.toml') or { panic(err) }
|
||||
|
||||
fn test_toml() {
|
||||
// File containing the complete text from the example in the official TOML project README.md:
|
||||
// https://github.com/toml-lang/toml/blob/3b11f6921da7b6f5db37af039aa021fee450c091/README.md#Example
|
||||
toml_doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
toml_json := to.json(toml_doc)
|
||||
|
||||
// NOTE Kept for easier debugging:
|
||||
// dump(toml_doc.ast)
|
||||
// println(toml_json)
|
||||
// assert false
|
||||
|
||||
assert toml_json == os.read_file(
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.out') or { panic(err) }
|
||||
|
||||
title := toml_doc.value('title')
|
||||
assert title == toml.Any('TOML Example')
|
||||
assert title as string == 'TOML Example'
|
||||
|
||||
database := toml_doc.value('database') as map[string]toml.Any
|
||||
db_serv := database['server'] or {
|
||||
panic('could not access "server" index in "database" variable')
|
||||
}
|
||||
assert db_serv as string == '192.168.1.1'
|
||||
|
||||
assert toml_doc.value('owner.name') as string == 'Tom Preston-Werner'
|
||||
|
||||
assert toml_doc.value('database.server') as string == '192.168.1.1'
|
||||
|
||||
database_ports := toml_doc.value('database.ports') as []toml.Any
|
||||
assert database_ports[0] as i64 == 8000
|
||||
assert database_ports[1] as i64 == 8001
|
||||
assert database_ports[2] as i64 == 8002
|
||||
assert database_ports[0].int() == 8000
|
||||
assert database_ports[1].int() == 8001
|
||||
assert database_ports[2].int() == 8002
|
||||
|
||||
assert toml_doc.value('database.connection_max') as i64 == 5000
|
||||
assert toml_doc.value('database.enabled') as bool == true
|
||||
|
||||
assert toml_doc.value('servers.alpha.ip').string() == '10.0.0.1'
|
||||
assert toml_doc.value('servers.alpha.dc').string() == 'eqdc10'
|
||||
|
||||
assert toml_doc.value('servers.beta.ip').string() == '10.0.0.2'
|
||||
assert toml_doc.value('servers.beta.dc').string() == 'eqdc10'
|
||||
|
||||
clients_data := (toml_doc.value('clients.data') as []toml.Any)
|
||||
// dump(clients_data)
|
||||
// assert false
|
||||
gamma_delta_array := clients_data[0] as []toml.Any
|
||||
digits_array := clients_data[1] as []toml.Any
|
||||
assert gamma_delta_array[0].string() == 'gamma'
|
||||
assert gamma_delta_array[1].string() == 'delta'
|
||||
assert digits_array[0].int() == 1
|
||||
assert digits_array[1].int() == 2
|
||||
|
||||
clients_hosts := (toml_doc.value('clients.hosts') as []toml.Any).as_strings()
|
||||
assert clients_hosts[0] == 'alpha'
|
||||
assert clients_hosts[1] == 'omega'
|
||||
}
|
||||
|
||||
fn test_toml_file() {
|
||||
out_path := os.join_path(os.vtmp_dir(), 'toml_tests')
|
||||
test_file := os.join_path(out_path, 'toml_example.toml')
|
||||
os.mkdir_all(out_path) or { assert false }
|
||||
defer {
|
||||
os.rmdir_all(out_path) or {}
|
||||
}
|
||||
os.write_file(test_file, toml_text) or { assert false }
|
||||
toml_doc := toml.parse_file(test_file) or { panic(err) }
|
||||
|
||||
toml_json := to.json(toml_doc)
|
||||
|
||||
// NOTE Kept for easier debugging:
|
||||
// dump(toml_doc.ast)
|
||||
// println(toml_json)
|
||||
// assert false
|
||||
|
||||
assert toml_json == os.read_file(
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.out') or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_toml_parse_text() {
|
||||
toml_doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
toml_json := to.json(toml_doc)
|
||||
assert toml_json == os.read_file(
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.out') or { panic(err) }
|
||||
}
|
||||
|
||||
fn test_toml_parse() {
|
||||
toml_doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
toml_json := to.json(toml_doc)
|
||||
assert toml_json == os.read_file(
|
||||
os.real_path(os.join_path(os.dir(@FILE), 'testdata', os.file_name(@FILE).all_before_last('.'))) +
|
||||
'.out') or { panic(err) }
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import toml
|
||||
import strconv
|
||||
|
||||
fn test_string() {
|
||||
str_value := 'test string'
|
||||
toml_txt := 'string = "test string"'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
|
||||
value := toml_doc.value('string')
|
||||
assert value == toml.Any(str_value)
|
||||
assert value as string == str_value
|
||||
assert value.string() == str_value
|
||||
}
|
||||
|
||||
fn test_i64() {
|
||||
toml_txt := 'i64 = 120'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
|
||||
value := toml_doc.value('i64')
|
||||
assert value == toml.Any(i64(120))
|
||||
assert value as i64 == 120
|
||||
assert value.i64() == i64(120)
|
||||
}
|
||||
|
||||
fn test_bool() {
|
||||
toml_txt := '
|
||||
bool_true = true
|
||||
bool_false = false'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
|
||||
value_true := toml_doc.value('bool_true')
|
||||
assert value_true == toml.Any(true)
|
||||
assert value_true as bool == true
|
||||
assert value_true != toml.Any(false)
|
||||
assert value_true as bool != false
|
||||
assert value_true.bool() == true
|
||||
|
||||
value_false := toml_doc.value('bool_false')
|
||||
assert value_false == toml.Any(false)
|
||||
assert value_false as bool == false
|
||||
assert value_false != toml.Any(true)
|
||||
assert value_false as bool != true
|
||||
assert value_false.bool() == false
|
||||
}
|
||||
|
||||
fn test_bool_key_is_not_value() {
|
||||
toml_txt := 'true = true
|
||||
false = false'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
|
||||
value_true := toml_doc.value('true')
|
||||
assert value_true == toml.Any(true)
|
||||
assert value_true as bool == true
|
||||
assert value_true != toml.Any(false)
|
||||
assert value_true as bool != false
|
||||
|
||||
value_false := toml_doc.value('false')
|
||||
assert value_false == toml.Any(false)
|
||||
assert value_false as bool == false
|
||||
assert value_false != toml.Any(true)
|
||||
assert value_false as bool != true
|
||||
}
|
||||
|
||||
fn test_single_letter_key() {
|
||||
toml_txt := '[v]
|
||||
open_sourced = "Jun 22 2019 20:20:28"'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
|
||||
value := toml_doc.value('v.open_sourced').string()
|
||||
assert value == 'Jun 22 2019 20:20:28'
|
||||
}
|
||||
|
||||
fn test_hex_values() {
|
||||
// Regression test
|
||||
// '0xb' is carefully chosen to include the 'b' character that also denotes binary via 0b prefix.
|
||||
toml_txt := 'hex = 0xb'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
|
||||
value := toml_doc.value('hex')
|
||||
assert value as i64 == 11
|
||||
assert value.i64() == 11
|
||||
}
|
||||
|
||||
fn test_comment_as_last_value() {
|
||||
toml_txt := '
|
||||
test = 42
|
||||
# this line has comment as last thing'
|
||||
toml_doc := toml.parse_text(toml_txt) or { panic(err) }
|
||||
|
||||
value := toml_doc.value('test')
|
||||
assert value as i64 == 42
|
||||
assert value.i64() == 42
|
||||
}
|
||||
|
||||
fn test_nan_and_inf_values() {
|
||||
mut toml_doc := toml.parse_text('nan = nan') or { panic(err) }
|
||||
mut value := toml_doc.value('nan')
|
||||
assert value.string() == 'nan'
|
||||
|
||||
toml_doc = toml.parse_text('nan = nan#comment') or { panic(err) }
|
||||
value = toml_doc.value('nan')
|
||||
assert value.string() == 'nan'
|
||||
|
||||
toml_doc = toml.parse_text('nan = -nan') or { panic(err) }
|
||||
value = toml_doc.value('nan')
|
||||
assert value.string() == 'nan'
|
||||
|
||||
toml_doc = toml.parse_text('nan = +nan') or { panic(err) }
|
||||
value = toml_doc.value('nan')
|
||||
assert value.string() == 'nan'
|
||||
|
||||
toml_doc = toml.parse_text('inf = inf') or { panic(err) }
|
||||
value = toml_doc.value('inf')
|
||||
assert value.u64() == strconv.double_plus_infinity
|
||||
|
||||
toml_doc = toml.parse_text('inf = +inf') or { panic(err) }
|
||||
value = toml_doc.value('inf')
|
||||
assert value.u64() == strconv.double_plus_infinity
|
||||
|
||||
toml_doc = toml.parse_text('inf = -inf') or { panic(err) }
|
||||
value = toml_doc.value('inf')
|
||||
assert value.u64() == strconv.double_minus_infinity
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
import toml
|
||||
import strconv
|
||||
|
||||
const toml_text = '
|
||||
modules = [ "ui", "toml" ]
|
||||
errors = []
|
||||
|
||||
[[themes]]
|
||||
name = "Dracula"
|
||||
colors = [ "red", "black", "white" ]
|
||||
|
||||
[[themes]]
|
||||
name = "Lemon"
|
||||
colors = [
|
||||
"green",
|
||||
"yellow",
|
||||
[ "transparent" ]
|
||||
]
|
||||
|
||||
[[tests]]
|
||||
id = 1
|
||||
|
||||
[[tests]]
|
||||
id = 2
|
||||
|
||||
[values]
|
||||
nan = nan
|
||||
inf = inf
|
||||
test = 2
|
||||
minus-inf = -inf
|
||||
|
||||
[[themes]]
|
||||
name = "Ice"
|
||||
colors = [
|
||||
"blue",
|
||||
"white"
|
||||
]
|
||||
'
|
||||
|
||||
const toml_text_2 = "
|
||||
[defaults]
|
||||
run.flags = ['-f 1']
|
||||
|
||||
[[defaults.env]]
|
||||
'RUN_PATH' = '\$OUT_PATH'
|
||||
'RUN_TIME' = 5
|
||||
'TEST_PATH' = '/tmp/test'
|
||||
"
|
||||
|
||||
const toml_text_3 = '
|
||||
[[foo.bar]]
|
||||
baz = 1
|
||||
bzz = 1
|
||||
|
||||
[[foo.bar]]
|
||||
baz = 2
|
||||
bzz = 2
|
||||
'
|
||||
|
||||
fn test_value_query_in_array() {
|
||||
toml_doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
mut value := toml_doc.value('themes[0].colors[1]').string()
|
||||
assert value == 'black'
|
||||
value = toml_doc.value('themes[1].colors[0]').string()
|
||||
assert value == 'green'
|
||||
value = toml_doc.value('themes[1].colors[2].[0]').string()
|
||||
assert value == 'transparent'
|
||||
value = toml_doc.value('modules[1]').string()
|
||||
assert value == 'toml'
|
||||
value = toml_doc.value('errors[11]').default_to('<none>').string()
|
||||
assert value == '<none>'
|
||||
|
||||
value = toml_doc.value('themes[2].colors[0]').string()
|
||||
assert value == 'blue'
|
||||
}
|
||||
|
||||
fn test_any_value_query() {
|
||||
toml_doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
themes := toml_doc.value('themes')
|
||||
assert themes.value('[0].colors[0]').string() == 'red'
|
||||
|
||||
themes_arr := toml_doc.value('themes') as []toml.Any
|
||||
assert themes_arr[0].value('colors[0]').string() == 'red'
|
||||
|
||||
mut any := themes
|
||||
assert any.value('[1].name').string() == 'Lemon'
|
||||
any = any.value('[1]')
|
||||
assert any.value('name').string() == 'Lemon'
|
||||
|
||||
any = toml_doc.value('themes').value('[1].colors').value('[1]')
|
||||
assert any.string() == 'yellow'
|
||||
|
||||
any = toml_doc.value('themes[1]').value('colors[1]')
|
||||
assert any.string() == 'yellow'
|
||||
|
||||
any = toml_doc.value('themes[1].colors[0]')
|
||||
assert any.string() == 'green'
|
||||
|
||||
any = toml_doc.value('values')
|
||||
any = any.value('test')
|
||||
assert any.int() == 2
|
||||
}
|
||||
|
||||
fn test_inf_and_nan_query() {
|
||||
toml_doc := toml.parse_text(toml_text) or { panic(err) }
|
||||
|
||||
value := toml_doc.value('values.nan').string()
|
||||
assert value == 'nan'
|
||||
|
||||
mut value_u64 := toml_doc.value('values.inf').u64()
|
||||
assert value_u64 == strconv.double_plus_infinity
|
||||
value_u64 = toml_doc.value('values.minus-inf').u64()
|
||||
assert value_u64 == strconv.double_minus_infinity
|
||||
}
|
||||
|
||||
fn test_any_value_query_2() {
|
||||
toml_doc := toml.parse_text(toml_text_2) or { panic(err) }
|
||||
defaults := toml_doc.value('defaults')
|
||||
assert defaults.value('run.flags[0]').string() == '-f 1'
|
||||
assert defaults.value('env[0].RUN_TIME').int() == 5
|
||||
}
|
||||
|
||||
fn test_any_value_query_for_nested_tables_array() {
|
||||
toml_doc := toml.parse_text(toml_text_3) or { panic(err) }
|
||||
items := toml_doc.value('foo.bar').array()
|
||||
assert items.len == 2
|
||||
assert items[0].value('baz').int() == 1
|
||||
assert items[0].value('bzz').int() == 1
|
||||
assert items[1].value('baz').int() == 2
|
||||
assert items[1].value('bzz').int() == 2
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module to
|
||||
|
||||
import toml
|
||||
import x.json2
|
||||
|
||||
type DocOrAny = toml.Any | toml.Doc
|
||||
|
||||
// json returns `doa` as a JSON encoded string.
|
||||
pub fn json(doa DocOrAny) string {
|
||||
match doa {
|
||||
toml.Doc {
|
||||
return doa.ast.table.str()
|
||||
}
|
||||
toml.Any {
|
||||
return any_to_json(doa)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// json returns `a` as a JSON encoded string.
|
||||
fn any_to_json(a toml.Any) string {
|
||||
match a {
|
||||
toml.Null {
|
||||
return 'null'
|
||||
}
|
||||
toml.DateTime {
|
||||
return json2.Any(toml.Any(a).string()).json_str()
|
||||
}
|
||||
toml.Date {
|
||||
return json2.Any(toml.Any(a).string()).json_str()
|
||||
}
|
||||
toml.Time {
|
||||
return json2.Any(toml.Any(a).string()).json_str()
|
||||
}
|
||||
string {
|
||||
return json2.Any(toml.Any(a).string()).json_str()
|
||||
}
|
||||
bool {
|
||||
return json2.Any(toml.Any(a).bool()).json_str()
|
||||
}
|
||||
f32 {
|
||||
return json2.Any(toml.Any(a).f32()).json_str()
|
||||
}
|
||||
f64 {
|
||||
return json2.Any(toml.Any(a).f64()).json_str()
|
||||
}
|
||||
i64 {
|
||||
return json2.Any(toml.Any(a).i64()).json_str()
|
||||
}
|
||||
int {
|
||||
return json2.Any(toml.Any(a).int()).json_str()
|
||||
}
|
||||
u64 {
|
||||
return json2.Any(toml.Any(a).u64()).json_str()
|
||||
}
|
||||
map[string]toml.Any {
|
||||
mut str := '{'
|
||||
for key, val in a {
|
||||
json_key := json2.Any(key)
|
||||
str += ' ${json_key.json_str()}: ${any_to_json(val)},'
|
||||
}
|
||||
str = str.trim_right(',')
|
||||
str += ' }'
|
||||
return str
|
||||
}
|
||||
[]toml.Any {
|
||||
mut str := '['
|
||||
for val in a {
|
||||
str += ' ${any_to_json(val)},'
|
||||
}
|
||||
str = str.trim_right(',')
|
||||
str += ' ]'
|
||||
return str
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// json_any returns `Any` as a `x.json2.Any` type.
|
||||
pub fn json_any(a toml.Any) json2.Any {
|
||||
match a {
|
||||
toml.Null {
|
||||
return json2.Null{}
|
||||
}
|
||||
toml.DateTime {
|
||||
return json2.Any(a.str())
|
||||
}
|
||||
toml.Date {
|
||||
return json2.Any(a.str())
|
||||
}
|
||||
toml.Time {
|
||||
return json2.Any(a.str())
|
||||
}
|
||||
string {
|
||||
return json2.Any(a.str())
|
||||
}
|
||||
bool {
|
||||
return json2.Any(bool(a))
|
||||
}
|
||||
int {
|
||||
return json2.Any(int(a))
|
||||
}
|
||||
f32 {
|
||||
return json2.Any(f32(a))
|
||||
}
|
||||
f64 {
|
||||
return json2.Any(f64(a))
|
||||
}
|
||||
i64 {
|
||||
return json2.Any(i64(a))
|
||||
}
|
||||
u64 {
|
||||
return json2.Any(u64(a))
|
||||
}
|
||||
map[string]toml.Any {
|
||||
mut jmap := map[string]json2.Any{}
|
||||
for key, val in a {
|
||||
jmap[key] = json_any(val)
|
||||
}
|
||||
return jmap
|
||||
}
|
||||
[]toml.Any {
|
||||
mut jarr := []json2.Any{}
|
||||
|
||||
for val in a {
|
||||
jarr << json_any(val)
|
||||
}
|
||||
|
||||
return jarr
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module token
|
||||
|
||||
// Position represents a position in a TOML document.
|
||||
pub struct Pos {
|
||||
pub:
|
||||
len int // length of the literal in the source
|
||||
line_nr int // the line number in the source where the token occurred
|
||||
pos int // the position of the token in scanner text
|
||||
col int // the column in the source where the token occurred
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module token
|
||||
|
||||
// Token holds information about the current scan of bytes.
|
||||
pub struct Token {
|
||||
pub:
|
||||
kind Kind // the token number/enum; for quick comparisons
|
||||
lit string // literal representation of the token
|
||||
col int // the column in the source where the token occurred
|
||||
line_nr int // the line number in the source where the token occurred
|
||||
pos int // the position of the token in scanner text
|
||||
len int // length of the literal
|
||||
}
|
||||
|
||||
// Kind represents a logical type of entity found in any given TOML document.
|
||||
pub enum Kind {
|
||||
unknown
|
||||
eof
|
||||
bare // user
|
||||
boolean // true or false
|
||||
number // 123
|
||||
quoted // 'foo', "foo", """foo""" or '''foo'''
|
||||
plus // +
|
||||
minus // -
|
||||
underscore // _
|
||||
comma // ,
|
||||
colon // :
|
||||
hash // # comment
|
||||
assign // =
|
||||
lcbr // {
|
||||
rcbr // }
|
||||
lsbr // [
|
||||
rsbr // ]
|
||||
nl // \n linefeed / newline character
|
||||
cr // \r carriage return
|
||||
tab // \t character
|
||||
whitespace // ` `
|
||||
period // .
|
||||
_end_
|
||||
}
|
||||
|
||||
// pos returns the exact position of a token in the input.
|
||||
@[inline]
|
||||
pub fn (tok &Token) pos() Pos {
|
||||
return Pos{
|
||||
len: tok.len
|
||||
line_nr: tok.line_nr - 1
|
||||
pos: tok.pos
|
||||
col: tok.col - 1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,594 @@
|
||||
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module toml
|
||||
|
||||
import toml.ast
|
||||
import toml.input
|
||||
import toml.scanner
|
||||
import toml.parser
|
||||
|
||||
// Null is used in sumtype checks as a "default" value when nothing else is possible.
|
||||
pub struct Null {
|
||||
}
|
||||
|
||||
// decode decodes a TOML `string` into the target type `T`.
|
||||
// If `T` has a custom `.from_toml()` method, it will be used instead of the default.
|
||||
pub fn decode[T](toml_txt string) !T {
|
||||
doc := parse_text(toml_txt)!
|
||||
mut typ := T{}
|
||||
$for method in T.methods {
|
||||
$if method.name == 'from_toml' {
|
||||
typ.$method(doc.to_any())
|
||||
return typ
|
||||
}
|
||||
}
|
||||
$if T !is $struct {
|
||||
return error('toml.decode: expected struct, found ${T.name}')
|
||||
}
|
||||
decode_struct[T](doc.to_any(), mut typ)
|
||||
return typ
|
||||
}
|
||||
|
||||
fn decode_struct[T](doc Any, mut typ T) {
|
||||
$for field in T.fields {
|
||||
mut field_name := field.name
|
||||
mut skip := false
|
||||
for attr in field.attrs {
|
||||
if attr == 'skip' {
|
||||
skip = true
|
||||
break
|
||||
}
|
||||
if attr.starts_with('toml:') {
|
||||
field_name = attr.all_after(':').trim_space()
|
||||
}
|
||||
}
|
||||
value := doc.value(field_name)
|
||||
// only set the field's value when value != null and !skip, else field got it's default value
|
||||
if !skip && value != null {
|
||||
$if field.is_enum {
|
||||
typ.$(field.name) = value.int()
|
||||
} $else $if field.typ is string {
|
||||
typ.$(field.name) = value.string()
|
||||
} $else $if field.typ is bool {
|
||||
typ.$(field.name) = value.bool()
|
||||
} $else $if field.typ is int {
|
||||
typ.$(field.name) = value.int()
|
||||
} $else $if field.typ is i64 {
|
||||
typ.$(field.name) = value.i64()
|
||||
} $else $if field.typ is u64 {
|
||||
typ.$(field.name) = value.u64()
|
||||
} $else $if field.typ is f32 {
|
||||
typ.$(field.name) = value.f32()
|
||||
} $else $if field.typ is f64 {
|
||||
typ.$(field.name) = value.f64()
|
||||
} $else $if field.typ is DateTime {
|
||||
typ.$(field.name) = value.datetime()
|
||||
} $else $if field.typ is Date {
|
||||
typ.$(field.name) = value.date()
|
||||
} $else $if field.typ is Time {
|
||||
typ.$(field.name) = value.time()
|
||||
} $else $if field.typ is Any {
|
||||
typ.$(field.name) = value
|
||||
} $else $if field.is_array {
|
||||
typ.$(field.name) = decode_array(typ.$(field.name), value.array())
|
||||
} $else $if field.is_map {
|
||||
typ.$(field.name) = decode_map(typ.$(field.name), value.as_map())
|
||||
} $else $if field.is_struct {
|
||||
mut s := typ.$(field.name)
|
||||
decode_struct(value, mut s)
|
||||
typ.$(field.name) = s
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_array[T](current []T, values []Any) []T {
|
||||
$if T is string {
|
||||
return values.map(it.string())
|
||||
} $else $if T is bool {
|
||||
return values.map(it.bool())
|
||||
} $else $if T is int {
|
||||
return values.map(it.int())
|
||||
} $else $if T is i64 {
|
||||
return values.map(it.i64())
|
||||
} $else $if T is u64 {
|
||||
return values.map(it.u64())
|
||||
} $else $if T is f32 {
|
||||
return values.map(it.f32())
|
||||
} $else $if T is f64 {
|
||||
return values.map(it.f64())
|
||||
} $else $if T is DateTime {
|
||||
return values.map(it.datetime())
|
||||
} $else $if T is Date {
|
||||
return values.map(it.date())
|
||||
} $else $if T is Time {
|
||||
return values.map(it.time())
|
||||
} $else $if T is Any {
|
||||
return values
|
||||
} $else $if T is $struct {
|
||||
mut decoded := []T{cap: values.len}
|
||||
for value in values {
|
||||
if value is map[string]Any {
|
||||
mut item := T{}
|
||||
decode_struct(value, mut item)
|
||||
decoded << item
|
||||
}
|
||||
}
|
||||
return decoded
|
||||
} $else {
|
||||
return current
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_map[T](current map[string]T, values map[string]Any) map[string]T {
|
||||
$if T is string {
|
||||
mut decoded := map[string]T{}
|
||||
for key, value in values {
|
||||
decoded[key] = value.string()
|
||||
}
|
||||
return decoded
|
||||
} $else $if T is bool {
|
||||
mut decoded := map[string]T{}
|
||||
for key, value in values {
|
||||
decoded[key] = value.bool()
|
||||
}
|
||||
return decoded
|
||||
} $else $if T is int {
|
||||
mut decoded := map[string]T{}
|
||||
for key, value in values {
|
||||
decoded[key] = value.int()
|
||||
}
|
||||
return decoded
|
||||
} $else $if T is i64 {
|
||||
mut decoded := map[string]T{}
|
||||
for key, value in values {
|
||||
decoded[key] = value.i64()
|
||||
}
|
||||
return decoded
|
||||
} $else $if T is u64 {
|
||||
mut decoded := map[string]T{}
|
||||
for key, value in values {
|
||||
decoded[key] = value.u64()
|
||||
}
|
||||
return decoded
|
||||
} $else $if T is f32 {
|
||||
mut decoded := map[string]T{}
|
||||
for key, value in values {
|
||||
decoded[key] = value.f32()
|
||||
}
|
||||
return decoded
|
||||
} $else $if T is f64 {
|
||||
mut decoded := map[string]T{}
|
||||
for key, value in values {
|
||||
decoded[key] = value.f64()
|
||||
}
|
||||
return decoded
|
||||
} $else $if T is DateTime {
|
||||
mut decoded := map[string]T{}
|
||||
for key, value in values {
|
||||
decoded[key] = value.datetime()
|
||||
}
|
||||
return decoded
|
||||
} $else $if T is Date {
|
||||
mut decoded := map[string]T{}
|
||||
for key, value in values {
|
||||
decoded[key] = value.date()
|
||||
}
|
||||
return decoded
|
||||
} $else $if T is Time {
|
||||
mut decoded := map[string]T{}
|
||||
for key, value in values {
|
||||
decoded[key] = value.time()
|
||||
}
|
||||
return decoded
|
||||
} $else $if T is Any {
|
||||
return values.clone()
|
||||
} $else $if T is $struct {
|
||||
mut decoded := map[string]T{}
|
||||
for key, value in values {
|
||||
if value is map[string]Any {
|
||||
mut item := T{}
|
||||
decode_struct(value, mut item)
|
||||
decoded[key] = item
|
||||
}
|
||||
}
|
||||
return decoded
|
||||
} $else {
|
||||
return current
|
||||
}
|
||||
}
|
||||
|
||||
// encode encodes the type `T` into a TOML string.
|
||||
// If `T` has a custom `.to_toml()` method, it will be used instead of the default.
|
||||
pub fn encode[T](typ T) string {
|
||||
$if T is $struct {
|
||||
$for method in T.methods {
|
||||
$if method.name == 'to_toml' {
|
||||
return typ.$method()
|
||||
}
|
||||
}
|
||||
mp := encode_struct[T](typ)
|
||||
return mp.to_toml()
|
||||
} $else {
|
||||
$compile_error('Currently only type `struct` is supported for `T` to encode as TOML')
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
fn encode_struct[T](typ T) map[string]Any {
|
||||
mut mp := map[string]Any{}
|
||||
$for field in T.fields {
|
||||
mut skip := false
|
||||
mut field_name := field.name
|
||||
for attr in field.attrs {
|
||||
if attr == 'skip' {
|
||||
skip = true
|
||||
break
|
||||
}
|
||||
if attr.starts_with('toml:') {
|
||||
field_name = attr.all_after(':').trim_space()
|
||||
}
|
||||
}
|
||||
if !skip {
|
||||
mp[field_name] = to_any(typ.$(field.name))
|
||||
}
|
||||
}
|
||||
return mp
|
||||
}
|
||||
|
||||
fn voidptr_to_toml_string[T](value T) string {
|
||||
ptr := unsafe { voidptr(&value) }
|
||||
return unsafe { '0x${ptr_str(*(&voidptr(ptr)))}' }
|
||||
}
|
||||
|
||||
fn to_any[T](value T) Any {
|
||||
$if T is $enum {
|
||||
return Any(int(value))
|
||||
} $else $if T is Date {
|
||||
return Any(value)
|
||||
} $else $if T is Time {
|
||||
return Any(value)
|
||||
} $else $if T is Null {
|
||||
return Any(value)
|
||||
} $else $if T is bool {
|
||||
return Any(value)
|
||||
} $else $if T is f32 {
|
||||
return Any(value)
|
||||
} $else $if T is f64 {
|
||||
return Any(value)
|
||||
} $else $if T is i64 {
|
||||
return Any(value)
|
||||
} $else $if T is int {
|
||||
return Any(value)
|
||||
} $else $if T is u64 {
|
||||
return Any(value)
|
||||
} $else $if T is DateTime {
|
||||
return Any(value)
|
||||
} $else $if T is Any {
|
||||
return value
|
||||
} $else $if T is $struct {
|
||||
$for method in T.methods {
|
||||
$if method.name == 'to_toml' {
|
||||
return Any(value.$method())
|
||||
}
|
||||
}
|
||||
return encode_struct(value)
|
||||
} $else $if T is $array {
|
||||
mut arr := []Any{cap: value.len}
|
||||
for v in value {
|
||||
arr << to_any(v)
|
||||
}
|
||||
return arr
|
||||
} $else $if T is $map {
|
||||
mut mmap := map[string]Any{}
|
||||
for key, val in value {
|
||||
mmap['${key}'] = to_any(val)
|
||||
}
|
||||
return mmap
|
||||
} $else {
|
||||
if typeof(value).name == 'voidptr' {
|
||||
return Any(voidptr_to_toml_string(value))
|
||||
}
|
||||
return Any('${value}')
|
||||
}
|
||||
}
|
||||
|
||||
// DateTime is the representation of an RFC 3339 datetime string.
|
||||
pub struct DateTime {
|
||||
pub:
|
||||
datetime string
|
||||
}
|
||||
|
||||
// str returns the RFC 3339 string representation of the datetime.
|
||||
pub fn (dt DateTime) str() string {
|
||||
return dt.datetime
|
||||
}
|
||||
|
||||
// Date is the representation of an RFC 3339 date-only string.
|
||||
pub struct Date {
|
||||
pub:
|
||||
date string
|
||||
}
|
||||
|
||||
// str returns the RFC 3339 date-only string representation.
|
||||
pub fn (d Date) str() string {
|
||||
return d.date
|
||||
}
|
||||
|
||||
// Time is the representation of an RFC 3339 time-only string.
|
||||
pub struct Time {
|
||||
pub:
|
||||
time string
|
||||
}
|
||||
|
||||
// str returns the RFC 3339 time-only string representation.
|
||||
pub fn (t Time) str() string {
|
||||
return t.time
|
||||
}
|
||||
|
||||
// Config is used to configure the toml parser.
|
||||
// Only one of the fields `text` or `file_path`, is allowed to be set at time of configuration.
|
||||
pub struct Config {
|
||||
pub:
|
||||
text string // TOML text
|
||||
file_path string // '/path/to/file.toml'
|
||||
parse_comments bool
|
||||
}
|
||||
|
||||
// Doc is a representation of a TOML document.
|
||||
// A document can be constructed from a `string` buffer or from a file path
|
||||
pub struct Doc {
|
||||
pub:
|
||||
ast &ast.Root = unsafe { nil }
|
||||
}
|
||||
|
||||
// parse_file parses the TOML file in `path`.
|
||||
pub fn parse_file(path string) !Doc {
|
||||
input_config := input.Config{
|
||||
file_path: path
|
||||
}
|
||||
scanner_config := scanner.Config{
|
||||
input: input_config
|
||||
}
|
||||
parser_config := parser.Config{
|
||||
scanner: scanner.new_scanner(scanner_config)!
|
||||
}
|
||||
mut p := parser.new_parser(parser_config)
|
||||
ast_ := p.parse()!
|
||||
return Doc{
|
||||
ast: ast_
|
||||
}
|
||||
}
|
||||
|
||||
// parse_text parses the TOML document provided in `text`.
|
||||
pub fn parse_text(text string) !Doc {
|
||||
input_config := input.Config{
|
||||
text: text
|
||||
}
|
||||
scanner_config := scanner.Config{
|
||||
input: input_config
|
||||
}
|
||||
parser_config := parser.Config{
|
||||
scanner: scanner.new_scanner(scanner_config)!
|
||||
}
|
||||
mut p := parser.new_parser(parser_config)
|
||||
ast_ := p.parse()!
|
||||
return Doc{
|
||||
ast: ast_
|
||||
}
|
||||
}
|
||||
|
||||
// parse_dotted_key converts `key` string to an array of strings.
|
||||
// parse_dotted_key preserves strings delimited by both `"` and `'`.
|
||||
pub fn parse_dotted_key(key string) ![]string {
|
||||
mut out := []string{}
|
||||
mut buf := ''
|
||||
mut in_string := false
|
||||
mut delim := u8(` `)
|
||||
for ch in key {
|
||||
if ch in [`"`, `'`] {
|
||||
if !in_string {
|
||||
delim = ch
|
||||
}
|
||||
in_string = !in_string && ch == delim
|
||||
if !in_string {
|
||||
if buf != '' && buf != ' ' {
|
||||
out << buf
|
||||
}
|
||||
buf = ''
|
||||
delim = ` `
|
||||
}
|
||||
continue
|
||||
}
|
||||
buf += ch.ascii_str()
|
||||
if !in_string && ch == `.` {
|
||||
if buf != '' && buf != ' ' {
|
||||
buf = buf[..buf.len - 1]
|
||||
if buf != '' && buf != ' ' {
|
||||
out << buf
|
||||
}
|
||||
}
|
||||
buf = ''
|
||||
continue
|
||||
}
|
||||
}
|
||||
if buf != '' && buf != ' ' {
|
||||
out << buf
|
||||
}
|
||||
if in_string {
|
||||
return error(@FN +
|
||||
': could not parse key, missing closing string delimiter `${delim.ascii_str()}`')
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parse_array_key converts `key` string to a key and index part.
|
||||
fn parse_array_key(key string) (string, int) {
|
||||
mut index := -1
|
||||
mut k := key
|
||||
if k.contains('[') {
|
||||
index = k.all_after('[').all_before(']').int()
|
||||
if k.starts_with('[') {
|
||||
k = '' // k.all_after(']')
|
||||
} else {
|
||||
k = k.all_before('[')
|
||||
}
|
||||
}
|
||||
return k, index
|
||||
}
|
||||
|
||||
// decode decodes a TOML `string` into the target struct type `T`.
|
||||
pub fn (d Doc) decode[T]() !T {
|
||||
$if T !is $struct {
|
||||
return error('Doc.decode: expected struct, found ${T.name}')
|
||||
}
|
||||
mut typ := T{}
|
||||
decode_struct(d.to_any(), mut typ)
|
||||
return typ
|
||||
}
|
||||
|
||||
// to_any converts the `Doc` to toml.Any type.
|
||||
pub fn (d Doc) to_any() Any {
|
||||
return ast_to_any(d.ast.table)
|
||||
}
|
||||
|
||||
// reflect returns `T` with `T.<field>`'s value set to the
|
||||
// value of any 1st level TOML key by the same name.
|
||||
pub fn (d Doc) reflect[T]() T {
|
||||
return d.to_any().reflect[T]()
|
||||
}
|
||||
|
||||
// value queries a value from the TOML document.
|
||||
// `key` supports a small query syntax scheme:
|
||||
// Maps can be queried in "dotted" form e.g. `a.b.c`.
|
||||
// quoted keys are supported as `a."b.c"` or `a.'b.c'`.
|
||||
// Arrays can be queried with `a[0].b[1].[2]`.
|
||||
pub fn (d Doc) value(key string) Any {
|
||||
key_split := parse_dotted_key(key) or { return null }
|
||||
return d.value_(d.ast.table, key_split)
|
||||
}
|
||||
|
||||
pub const null = Any(Null{})
|
||||
|
||||
// value_opt queries a value from the TOML document. Returns an error if the
|
||||
// key is not valid or there is no value for the key.
|
||||
pub fn (d Doc) value_opt(key string) !Any {
|
||||
key_split := parse_dotted_key(key) or { return error('invalid dotted key') }
|
||||
x := d.value_(d.ast.table, key_split)
|
||||
if x is Null {
|
||||
return error('no value for key')
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
// value_ returns the value found at `key` in the map `values` as `Any` type.
|
||||
fn (d Doc) value_(value ast.Value, key []string) Any {
|
||||
if key.len == 0 {
|
||||
return null
|
||||
}
|
||||
mut ast_value := ast.Value(ast.Null{})
|
||||
k, index := parse_array_key(key[0])
|
||||
|
||||
if k == '' {
|
||||
a := value as []ast.Value
|
||||
ast_value = a[index] or { return null }
|
||||
}
|
||||
|
||||
if value is map[string]ast.Value {
|
||||
ast_value = value[k] or { return null }
|
||||
if index > -1 {
|
||||
a := ast_value as []ast.Value
|
||||
ast_value = a[index] or { return null }
|
||||
}
|
||||
}
|
||||
|
||||
if key.len <= 1 {
|
||||
return ast_to_any(ast_value)
|
||||
}
|
||||
match ast_value {
|
||||
map[string]ast.Value, []ast.Value {
|
||||
return d.value_(ast_value, key[1..])
|
||||
}
|
||||
else {
|
||||
return ast_to_any(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ast_to_any converts `from` ast.Value to toml.Any value.
|
||||
pub fn ast_to_any(value ast.Value) Any {
|
||||
return ast_to_any_(value)
|
||||
}
|
||||
|
||||
fn ast_to_any_(value ast.Value) Any {
|
||||
match value {
|
||||
ast.Date {
|
||||
return Any(Date{value.text.clone()})
|
||||
}
|
||||
ast.Time {
|
||||
return Any(Time{value.text.clone()})
|
||||
}
|
||||
ast.DateTime {
|
||||
return Any(DateTime{value.text.clone()})
|
||||
}
|
||||
ast.Quoted {
|
||||
return Any(value.text.clone())
|
||||
}
|
||||
ast.Number {
|
||||
val_text := value.text
|
||||
if val_text == 'inf' || val_text == '+inf' || val_text == '-inf' {
|
||||
// NOTE values taken from strconv
|
||||
if !val_text.starts_with('-') {
|
||||
// strconv.double_plus_infinity
|
||||
return Any(u64(0x7FF0000000000000))
|
||||
} else {
|
||||
// strconv.double_minus_infinity
|
||||
return Any(u64(0xFFF0000000000000))
|
||||
}
|
||||
}
|
||||
if val_text == 'nan' || val_text == '+nan' || val_text == '-nan' {
|
||||
return Any('nan')
|
||||
}
|
||||
if !val_text.starts_with('0x')
|
||||
&& (val_text.contains('.') || val_text.to_lower().contains('e')) {
|
||||
return Any(value.f64())
|
||||
}
|
||||
return Any(value.i64())
|
||||
}
|
||||
ast.Bool {
|
||||
str := value.text
|
||||
if str == 'true' {
|
||||
return Any(true)
|
||||
}
|
||||
return Any(false)
|
||||
}
|
||||
map[string]ast.Value {
|
||||
m := (value as map[string]ast.Value)
|
||||
mut am := map[string]Any{}
|
||||
for k, v in m {
|
||||
converted := ast_to_any_(v)
|
||||
am[k] = converted
|
||||
}
|
||||
return am
|
||||
// return d.get_map_value(m, key_split[1..].join('.'))
|
||||
}
|
||||
[]ast.Value {
|
||||
a := (value as []ast.Value)
|
||||
mut aa := []Any{cap: a.len}
|
||||
for val in a {
|
||||
converted := ast_to_any_(val)
|
||||
aa << converted
|
||||
}
|
||||
return aa
|
||||
}
|
||||
else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
// TODO: decide this
|
||||
// panic(@MOD + '.' + @STRUCT + '.' + @FN + ' can\'t convert "${value}"')
|
||||
// return Any('')
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
// Copyright (c) 2021 Lars Pontoppidan. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module util
|
||||
|
||||
// is_key_char returns true if the given u8 is a valid key character.
|
||||
@[inline]
|
||||
pub fn is_key_char(c u8) bool {
|
||||
return c.is_letter() // || c == `_` || c == `-` <- these are identified when tokenizing
|
||||
}
|
||||
|
||||
// is_ascii_control_character returns true if `byte_char` is an ASCII control character.
|
||||
@[inline]
|
||||
pub fn is_ascii_control_character(byte_char u8) bool {
|
||||
return (byte_char >= 0 && byte_char <= 0x1f) || byte_char == 0x7f
|
||||
}
|
||||
|
||||
// is_illegal_ascii_control_character returns true if a `byte_char` ASCII control character
|
||||
// is considered "illegal" in TOML .
|
||||
@[inline]
|
||||
pub fn is_illegal_ascii_control_character(byte_char u8) bool {
|
||||
return byte_char != 0x09 && is_ascii_control_character(byte_char)
|
||||
}
|
||||
|
||||
// printdbg is a utility function for displaying a key:pair error message
|
||||
// when `-d trace_toml` is passed to the compiler.
|
||||
@[if trace_toml ?]
|
||||
pub fn printdbg(id string, message string) {
|
||||
eprintln(id + ' ' + message)
|
||||
}
|
||||
Reference in new issue
Block a user