Initial project import

This commit is contained in:
Agila Forge Importer committed 2026-07-13 15:20:32 +00:00
commit 23f2d29e32
11957 files changed
+1754577

No files matched your search

+247
View File
@@ -0,0 +1,247 @@
module base64
// encode_in_buffer base64 encodes the `[]u8` passed in `data` into `buffer`.
// encode_in_buffer returns the size of the encoded data in the buffer.
// Please note: The buffer should be large enough (i.e. 4/3 of the data.len, or larger) to hold the encoded data.
// Please note: The function does NOT allocate new memory, and is suitable for handling very large strings.
pub fn encode_in_buffer(data []u8, buffer &u8) int {
return encode_from_buffer(buffer, data.data, data.len)
}
// encode_from_buffer will perform encoding from any type of src buffer
// and write the bytes into `dest`.
// Please note: The `dest` buffer should be large enough (i.e. 4/3 of the src_len, or larger) to hold the encoded data.
// Please note: This function is for internal base64 encoding
fn encode_from_buffer(dest &u8, src &u8, src_len int) int {
if src_len == 0 {
return 0
}
output_length := 4 * ((src_len + 2) / 3)
mut d := unsafe { src }
mut b := unsafe { dest }
etable := enc_table.str
mut di := 0
mut si := 0
n := (src_len / 3) * 3
for si < n {
// Convert 3x 8bit source bytes into 4 bytes
unsafe {
val := u32(d[si + 0]) << 16 | u32(d[si + 1]) << 8 | u32(d[si + 2])
b[di + 0] = etable[(val >> 18) & 0x3F]
b[di + 1] = etable[(val >> 12) & 0x3F]
b[di + 2] = etable[(val >> 6) & 0x3F]
b[di + 3] = etable[val & 0x3F]
}
si += 3
di += 4
}
remain := src_len - si
if remain == 0 {
return output_length
}
// Add the remaining small block and padding
unsafe {
mut val := u32(d[si + 0]) << 16
if remain == 2 {
val |= u32(d[si + 1]) << 8
}
b[di + 0] = etable[(val >> 18) & 0x3F]
b[di + 1] = etable[(val >> 12) & 0x3F]
match remain {
2 {
b[di + 2] = etable[(val >> 6) & 0x3F]
b[di + 3] = u8(`=`)
}
1 {
b[di + 2] = u8(`=`)
b[di + 3] = u8(`=`)
}
else {
panic('base64: This case should never occur.')
}
}
}
return output_length
}
// decode_in_buffer decodes the base64 encoded `string` reference passed in `data` into `buffer`.
// decode_in_buffer returns the size of the decoded data in the buffer.
// Please note: The `buffer` should be large enough (i.e. 3/4 of the data.len, or larger)
// to hold the decoded data.
// Please note: This function does NOT allocate new memory, and is thus suitable for handling very large strings.
pub fn decode_in_buffer(data &string, buffer &u8) int {
return decode_from_buffer(buffer, data.str, data.len)
}
// decode_from_buffer decodes the base64 encoded ASCII bytes from `data` into `buffer`.
// decode_from_buffer returns the size of the decoded data in the buffer.
// Please note: The `buffer` should be large enough (i.e. 3/4 of the data.len, or larger)
// to hold the decoded data.
// Please note: This function does NOT allocate new memory, and is thus suitable for handling very large strings.
pub fn decode_in_buffer_bytes(data []u8, buffer &u8) int {
return decode_from_buffer(buffer, data.data, data.len)
}
// decode_from_buffer decodes the base64 encoded ASCII bytes from `src` into `dest`.
// decode_from_buffer returns the size of the decoded data in the buffer.
// Please note: The `dest` buffer should be large enough (i.e. 3/4 of the `src_len`, or larger)
// to hold the decoded data.
// Please note: This function does NOT allocate new memory, and is thus suitable for handling very large strings.
// Please note: This function is for internal base64 decoding
fn decode_from_buffer(dest &u8, src &u8, src_len int) int {
if src_len < 4 {
return 0
}
mut padding := 0
if unsafe { src[src_len - 1] == `=` } {
if unsafe { src[src_len - 2] == `=` } {
padding = 2
} else {
padding = 1
}
}
mut d := unsafe { src }
mut b := unsafe { dest }
unsafe {
mut n_decoded_bytes := 0 // padding bytes are also counted towards this.
mut si := 0
mut datablock_64 := B64_64_datablock{
data: 0
}
mut datablock_32 := B64_32_datablock{
data: 0
}
for src_len - si >= 8 {
// Converting 8 bytes of input into 6 bytes of output. Storing these in the upper bytes of an u64.
datablock_64.data = assemble64(u8(index[d[si + 0]]), u8(index[d[si + 1]]), u8(index[d[
si + 2]]), u8(index[d[si + 3]]), u8(index[d[si + 4]]), u8(index[d[si + 5]]), u8(index[d[
si + 6]]), u8(index[d[si + 7]]))
// Reading out the individual bytes from the u64. Watch out with endianness.
$if little_endian {
b[n_decoded_bytes + 0] = datablock_64.data_byte[7]
b[n_decoded_bytes + 1] = datablock_64.data_byte[6]
b[n_decoded_bytes + 2] = datablock_64.data_byte[5]
b[n_decoded_bytes + 3] = datablock_64.data_byte[4]
b[n_decoded_bytes + 4] = datablock_64.data_byte[3]
b[n_decoded_bytes + 5] = datablock_64.data_byte[2]
} $else {
b[n_decoded_bytes + 0] = datablock_64.data_byte[0]
b[n_decoded_bytes + 1] = datablock_64.data_byte[1]
b[n_decoded_bytes + 2] = datablock_64.data_byte[2]
b[n_decoded_bytes + 3] = datablock_64.data_byte[3]
b[n_decoded_bytes + 4] = datablock_64.data_byte[4]
b[n_decoded_bytes + 5] = datablock_64.data_byte[5]
}
n_decoded_bytes += 6
si += 8
}
for src_len - si >= 4 {
datablock_32.data = assemble32(u8(index[d[si + 0]]), u8(index[d[si + 1]]), u8(index[d[
si + 2]]), u8(index[d[si + 3]]))
$if little_endian {
b[n_decoded_bytes + 0] = datablock_32.data_byte[3]
b[n_decoded_bytes + 1] = datablock_32.data_byte[2]
b[n_decoded_bytes + 2] = datablock_32.data_byte[1]
b[n_decoded_bytes + 3] = datablock_32.data_byte[0]
} $else {
b[n_decoded_bytes + 0] = datablock_32.data_byte[0]
b[n_decoded_bytes + 1] = datablock_32.data_byte[1]
b[n_decoded_bytes + 2] = datablock_32.data_byte[2]
b[n_decoded_bytes + 3] = datablock_32.data_byte[3]
}
n_decoded_bytes += 3
si += 4
}
return n_decoded_bytes - padding
}
}
union B64_64_datablock {
mut:
data u64
data_byte [8]u8
}
union B64_32_datablock {
mut:
data u32
data_byte [4]u8
}
// decode decodes the base64 encoded `string` value passed in `data`.
// Please note: If you need to decode many strings repeatedly, take a look at `decode_in_buffer`.
// Example: assert base64.decode('ViBpbiBiYXNlIDY0') == 'V in base 64'.bytes()
pub fn decode(data string) []u8 {
mut size := i64(data.len) * 3 / 4
if size <= 0 || data.len % 4 != 0 {
return []
}
size = (size + 3) & ~0x03 // round to the next multiple of 4 (the decoding loop writes multiples of 4 bytes)
unsafe {
buffer := malloc(int(size))
n := decode_in_buffer(data, buffer)
return buffer.vbytes(n)
}
}
// decode_str is the string variant of decode
pub fn decode_str(data string) string {
size := data.len * 3 / 4
if size <= 0 || data.len % 4 != 0 {
return ''
}
unsafe {
buffer := malloc_noscan(size + 1)
buffer[size] = 0
blen := decode_in_buffer(data, buffer)
return tos(buffer, blen)
}
}
// encode encodes the `[]u8` value passed in `data` to base64.
// Please note: base64 encoding returns a `string` that is ~ 4/3 larger than the input.
// Please note: If you need to encode many strings repeatedly, take a look at `encode_in_buffer`.
// Example: assert base64.encode('V in base 64'.bytes()) == 'ViBpbiBiYXNlIDY0'
pub fn encode(data []u8) string {
return alloc_and_encode(data.data, data.len)
}
// encode_str is the string variant of encode
pub fn encode_str(data string) string {
return alloc_and_encode(data.str, data.len)
}
// alloc_and_encode is a private function that allocates and encodes data into a string
// Used by encode and encode_str
fn alloc_and_encode(src &u8, len int) string {
if len == 0 {
return ''
}
size := 4 * ((len + 2) / 3)
if size <= 0 {
return ''
}
unsafe {
buffer := malloc_noscan(size + 1)
buffer[size] = 0
blen := encode_from_buffer(buffer, src, len)
return tos(buffer, blen)
}
}
+66
View File
@@ -0,0 +1,66 @@
// Copyright (c) 2019-2024 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
// Based off: https://github.com/golang/go/blob/master/src/encoding/base64/base64.go
// Last commit: https://github.com/golang/go/commit/9a93baf4d7d13d7d5c67388c93960d78abc8e11e
module base64
const index = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 63, 62, 62, 63, 52, 53, 54, 55, 56, 57,
58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 63, 0, 26, 27, 28, 29, 30, 31, 32, 33,
34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51]!
const ending_table = [0, 2, 1]!
const enc_table = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
// url_decode returns a decoded URL `string` version of
// the a base64 url encoded `string` passed in `data`.
pub fn url_decode(data string) []u8 {
mut result := data.replace_each(['-', '+', '_', '/'])
match result.len % 4 {
// Pad with trailing '='s
2 { result += '==' } // 2 pad chars
3 { result += '=' } // 1 pad char
else {} // no padding
}
return decode(result)
}
// url_decode_str is the string variant of url_decode
pub fn url_decode_str(data string) string {
mut result := data.replace_each(['-', '+', '_', '/'])
match result.len % 4 {
// Pad with trailing '='s
2 { result += '==' } // 2 pad chars
3 { result += '=' } // 1 pad char
else {} // no padding
}
return decode_str(result)
}
// url_encode returns a base64 URL encoded `string` version
// of the value passed in `data`.
pub fn url_encode(data []u8) string {
return encode(data).replace_each(['+', '-', '/', '_', '=', ''])
}
// url_encode_str is the string variant of url_encode
pub fn url_encode_str(data string) string {
return encode_str(data).replace_each(['+', '-', '/', '_', '=', ''])
}
// assemble64 assembles 8 base64 digits into 6 bytes.
// Each digit comes from the decode map.
// Please note: Invalid base64 digits are not expected and not handled.
fn assemble64(n1 u8, n2 u8, n3 u8, n4 u8, n5 u8, n6 u8, n7 u8, n8 u8) u64 {
return u64(n1) << 58 | u64(n2) << 52 | u64(n3) << 46 | u64(n4) << 40 | u64(n5) << 34 | u64(n6) << 28 | u64(n7) << 22 | u64(n8) << 16
}
// assemble32 assembles 4 base64 digits into 3 bytes.
// Each digit comes from the decode map.
// Please note: Invalid base64 digits are not expected and not handled.
fn assemble32(n1 u8, n2 u8, n3 u8, n4 u8) u32 {
return u32(n1) << 26 | u32(n2) << 20 | u32(n3) << 14 | u32(n4) << 8
}
@@ -0,0 +1,59 @@
import encoding.base64
fn test_long_encoding() {
repeats := 1000
input_size := 3000
s_original := []u8{len: input_size, init: `a`}
s_encoded := base64.encode(s_original)
s_encoded_bytes := s_encoded.bytes()
s_decoded := base64.decode(s_encoded)
assert s_encoded.len > s_original.len
assert s_original == s_decoded
ebuffer := unsafe { malloc(s_encoded.len) }
dbuffer := unsafe { malloc(s_decoded.len) }
defer {
unsafe { free(ebuffer) }
unsafe { free(dbuffer) }
}
encoded_size := base64.encode_in_buffer(s_original, ebuffer)
mut encoded_in_buf := []u8{len: encoded_size}
unsafe { vmemcpy(encoded_in_buf.data, ebuffer, encoded_size) }
assert input_size * 4 / 3 == encoded_size
assert encoded_in_buf[0] == `Y`
assert encoded_in_buf[1] == `W`
assert encoded_in_buf[2] == `F`
assert encoded_in_buf[3] == `h`
assert encoded_in_buf[encoded_size - 4] == `Y`
assert encoded_in_buf[encoded_size - 3] == `W`
assert encoded_in_buf[encoded_size - 2] == `F`
assert encoded_in_buf[encoded_size - 1] == `h`
assert encoded_in_buf == s_encoded_bytes
decoded_size := base64.decode_in_buffer(s_encoded, dbuffer)
assert decoded_size == input_size
mut decoded_in_buf := []u8{len: decoded_size}
unsafe { vmemcpy(decoded_in_buf.data, dbuffer, decoded_size) }
assert decoded_in_buf == s_original
mut s := 0
for _ in 0 .. repeats {
resultsize := base64.encode_in_buffer(s_original, ebuffer)
s += resultsize
assert resultsize == s_encoded.len
}
for _ in 0 .. repeats {
resultsize := base64.decode_in_buffer(s_encoded, dbuffer)
s += resultsize
assert resultsize == s_decoded.len
}
println('Final s: ${s}')
// assert s == 39147008
}
+148
View File
@@ -0,0 +1,148 @@
import encoding.base64
struct TestPair {
decoded string
encoded string
}
const pairs = [
// RFC 3548 examples
TestPair{'\x14\xfb\x9c\x03\xd9\x7e', 'FPucA9l+'},
TestPair{'\x14\xfb\x9c\x03\xd9', 'FPucA9k='},
TestPair{'\x14\xfb\x9c\x03', 'FPucAw=='},
// RFC 4648 examples
TestPair{'', ''},
TestPair{'f', 'Zg=='},
TestPair{'fo', 'Zm8='},
TestPair{'foo', 'Zm9v'},
TestPair{'foob', 'Zm9vYg=='},
TestPair{'fooba', 'Zm9vYmE='},
TestPair{'foobar', 'Zm9vYmFy'},
// Wikipedia examples
TestPair{'sure.', 'c3VyZS4='},
TestPair{'sure', 'c3VyZQ=='},
TestPair{'sur', 'c3Vy'},
TestPair{'su', 'c3U='},
TestPair{'leasure.', 'bGVhc3VyZS4='},
TestPair{'easure.', 'ZWFzdXJlLg=='},
TestPair{'asure.', 'YXN1cmUu'},
TestPair{'sure.', 'c3VyZS4='},
]
const man_pair = TestPair{'Man is distinguished, not only by his reason, but by this singular passion from other animals, which is a lust of the mind, that by a perseverance of delight in the continued and indefatigable generation of knowledge, exceeds the short vehemence of any carnal pleasure.', 'TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0aGlzIHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1c3Qgb2YgdGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0aGUgY29udGludWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdlLCBleGNlZWRzIHRoZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4='}
fn test_decode() {
assert base64.decode(man_pair.encoded) == man_pair.decoded.bytes()
// Test for incorrect padding.
assert base64.decode('aGk') == ''.bytes()
assert base64.decode('aGk=') == 'hi'.bytes()
assert base64.decode('aGk==') == ''.bytes()
for i, p in pairs {
got := base64.decode(p.encoded)
if got != p.decoded.bytes() {
eprintln('pairs[${i}]: expected = ${p.decoded}, got = ${got}')
assert false
}
}
}
fn test_decode_str() {
assert base64.decode_str(man_pair.encoded) == man_pair.decoded
// Test for incorrect padding.
assert base64.decode_str('aGk') == ''
assert base64.decode_str('aGk=') == 'hi'
assert base64.decode_str('aGk==') == ''
for i, p in pairs {
got := base64.decode_str(p.encoded)
if got != p.decoded {
eprintln('pairs[${i}]: expected = ${p.decoded}, got = ${got}')
assert false
}
}
}
fn test_encode() {
assert base64.encode(man_pair.decoded.bytes()) == man_pair.encoded
for i, p in pairs {
got := base64.encode(p.decoded.bytes())
if got != p.encoded {
eprintln('pairs[${i}]: expected = ${p.encoded}, got = ${got}')
assert false
}
}
}
fn test_encode_str() {
assert base64.encode_str(man_pair.decoded) == man_pair.encoded
for i, p in pairs {
got := base64.encode_str(p.decoded)
if got != p.encoded {
eprintln('pairs[${i}]: expected = ${p.encoded}, got = ${got}')
assert false
}
}
}
fn test_url_encode() {
test := base64.url_encode('Hello Base64Url encoding!'.bytes())
assert test == 'SGVsbG8gQmFzZTY0VXJsIGVuY29kaW5nIQ'
}
fn test_url_encode_str() {
test := base64.url_encode_str('Hello Base64Url encoding!')
assert test == 'SGVsbG8gQmFzZTY0VXJsIGVuY29kaW5nIQ'
}
fn test_url_decode() {
test := base64.url_decode('SGVsbG8gQmFzZTY0VXJsIGVuY29kaW5nIQ')
assert test == 'Hello Base64Url encoding!'.bytes()
}
fn test_url_decode_str() {
test := base64.url_decode_str('SGVsbG8gQmFzZTY0VXJsIGVuY29kaW5nIQ')
assert test == 'Hello Base64Url encoding!'
}
fn test_encode_null_u8() {
assert base64.encode([u8(`A`), 0, `C`]) == 'QQBD'
}
fn test_encode_null_byte_str() {
// While this works, bytestr() does a memcpy
s := [u8(`A`), 0, `C`].bytestr()
assert base64.encode_str(s) == 'QQBD'
}
fn test_decode_null_u8() {
assert base64.decode('QQBD') == [u8(`A`), 0, `C`]
}
fn test_decode_null_byte_str() {
// While this works, bytestr() does a memcpy
s := [u8(`A`), 0, `C`].bytestr()
assert base64.decode_str('QQBD') == s
}
fn test_decode_in_buffer_bytes() {
rfc4648_pairs := [
TestPair{'foob', 'Zm9vYg=='},
TestPair{'fooba', 'Zm9vYmE='},
TestPair{'foobar', 'Zm9vYmFy'},
]
mut src_dec_buf := []u8{len: 8}
mut src_enc_buf := []u8{len: 8}
mut out_buf := []u8{len: 8}
for p in rfc4648_pairs {
src_dec_buf = p.decoded.bytes()
src_enc_buf = p.encoded.bytes()
n := base64.decode_in_buffer_bytes(src_enc_buf, out_buf.data)
assert src_dec_buf == out_buf[..n]
}
}