Initial project import

This commit is contained in:
Agila Forge Importer committed 2026-07-13 15:20:32 +00:00
commit 23f2d29e32
11957 files changed
+1754577

No files matched your search

+109
View File
@@ -0,0 +1,109 @@
`net.s3` is an S3-compatible client written in pure V.
It speaks the AWS Signature Version 4 protocol on top of `crypto.hmac` and
`crypto.sha256`, with no third-party dependencies. The same client targets any
S3-compatible endpoint (hosted or self-hosted) by configuring the right
`endpoint`, `region` and credentials.
## Quick start
```v ignore
import net.s3
c := s3.new_client(s3.Credentials{
endpoint: 'https://s3.example.com'
access_key_id: '...'
secret_access_key: '...'
bucket: 'my-bucket'
})
c.put('hello.txt', 'Hi from V!'.bytes())!
text := c.get_string('hello.txt')!
url := c.presign('hello.txt', expires_in: 3600)!
```
## Credentials from the environment
`Credentials.from_env()` resolves each field from the first non-empty
provider-specific environment variable, so the same code works against AWS,
self-hosted and managed S3 services without reconfiguration:
```v ignore
import net.s3
c := s3.new_client(s3.Credentials.from_env())
```
Supported variables (first non-empty wins per field):
* key id: `S3_ACCESS_KEY_ID`, `AWS_ACCESS_KEY_ID`, `CELLAR_ADDON_KEY_ID`,
`SCW_ACCESS_KEY`, `B2_APPLICATION_KEY_ID`, `R2_ACCESS_KEY_ID`, `SPACES_KEY`
* secret: `S3_SECRET_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`,
`CELLAR_ADDON_KEY_SECRET`, `SCW_SECRET_KEY`, `B2_APPLICATION_KEY`,
`R2_SECRET_ACCESS_KEY`, `SPACES_SECRET`
* session token: `S3_SESSION_TOKEN`, `AWS_SESSION_TOKEN`
* region: `S3_REGION`, `AWS_REGION`, `AWS_DEFAULT_REGION`,
`SCW_DEFAULT_REGION`
* bucket: `S3_BUCKET`
* endpoint: `S3_ENDPOINT`, `AWS_ENDPOINT`, `AWS_ENDPOINT_URL`,
`CELLAR_ADDON_HOST`, `B2_ENDPOINT`, `R2_ENDPOINT`, `SPACES_ENDPOINT`
## Multipart upload
`upload_file` automatically picks single-shot or multipart based on file size.
For finer control, `start_multipart` returns a stateful `MultipartUploader`
that streams chunks generated on the fly:
```v ignore
import net.s3
c := s3.new_client(s3.Credentials.from_env())
c.upload_file('big.bin', '/path/to/big.bin', s3.PutOptions{
content_type: 'application/octet-stream'
})!
```
## `s3://` URLs
Importing `net.s3` registers an `s3://` scheme handler with `net.http`, so the
generic `http.fetch(url: 's3://...')` route works out of the box. There is
also a direct `s3.fetch` helper:
```v ignore
import net.s3
resp := s3.fetch('s3://my-bucket/hello.txt')!
println(resp.body.bytestr())
```
## File handle
`Client.file(key)` returns a small `File` reference for ergonomic call sites:
```v ignore
import net.s3
c := s3.new_client(s3.Credentials.from_env())
f := c.file('hello.txt')
text := f.text()!
url := f.presign(expires_in: 3600)!
```
## Tests
The unit suite is offline and runs by default:
```
v test vlib/net/s3/
```
The integration suite is gated on `S3_INTEGRATION=1` and exercises the
client against a live endpoint:
```
S3_INTEGRATION=1 \
S3_HOST=https://s3.example.com \
S3_KEY_ID=... S3_KEY_SECRET=... \
S3_BUCKET=v-s3-tests \
v test vlib/net/s3/integration_test.v
```
+192
View File
@@ -0,0 +1,192 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
// BucketOptions configures bucket-level operations. `region_constraint`
// lets `create_bucket` pin the bucket to a specific region (S3 sends a
// `<CreateBucketConfiguration>` body when this is non-empty).
@[params]
pub struct BucketOptions {
pub:
bucket string
acl Acl
region_constraint string
}
// create_bucket creates a new bucket. Returns:
// - nil on success (HTTP 200)
// - S3Error("BucketAlreadyOwnedByYou") if you already own this bucket
// - S3Error("BucketAlreadyExists") if someone else owns it
// - S3Error("InvalidBucketName") for non-conformant names
//
// The S3 wire response for these states is HTTP 409, parsed from the
// returned XML body.
pub fn (c &Client) create_bucket(opts BucketOptions) ! {
bucket := pick_bucket(c.credentials, opts.bucket)!
validate_bucket_name(bucket)!
creds := c.creds_for(bucket)
path := bucket_path(creds, bucket)
body := if opts.region_constraint != '' {
'<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><LocationConstraint>${escape_xml(opts.region_constraint)}</LocationConstraint></CreateBucketConfiguration>'
} else {
''
}
mut headers := map[string]string{}
if opts.acl != .unset {
headers['x-amz-acl'] = opts.acl.to_header_value()
}
if body != '' {
headers['content-type'] = 'application/xml'
}
signed := sign_request(creds, SignRequest{
method: 'PUT'
path: path
payload_hash: if body == '' { empty_sha256 } else { sha256_hex(body.bytes()) }
extra_headers: headers
})!
resp := c.do_http(signed, body)!
if resp.status_code !in [200, 204] {
return new_http_error(resp.status_code, bucket, resp.body)
}
}
// delete_bucket removes an empty bucket. S3 returns 409 BucketNotEmpty if it
// still has keys — caller is expected to clean up first or handle the error.
pub fn (c &Client) delete_bucket(opts BucketOptions) ! {
bucket := pick_bucket(c.credentials, opts.bucket)!
creds := c.creds_for(bucket)
path := bucket_path(creds, bucket)
signed := sign_request(creds, SignRequest{
method: 'DELETE'
path: path
payload_hash: empty_sha256
})!
resp := c.do_http(signed, '')!
if resp.status_code !in [200, 204] {
return new_http_error(resp.status_code, bucket, resp.body)
}
}
// bucket_exists checks bucket existence/access. Returns true if accessible (200),
// false on 404 / 403 (no such bucket or no read permission), error on other
// statuses. Uses HEAD under the hood — no body is fetched.
pub fn (c &Client) bucket_exists(opts BucketOptions) !bool {
bucket := pick_bucket(c.credentials, opts.bucket)!
creds := c.creds_for(bucket)
path := bucket_path(creds, bucket)
signed := sign_request(creds, SignRequest{
method: 'HEAD'
path: path
payload_hash: empty_sha256
})!
resp := c.do_http(signed, '')!
if resp.status_code == 404 || resp.status_code == 403 {
return false
}
if resp.status_code != 200 {
return new_http_error(resp.status_code, bucket, resp.body)
}
return true
}
// validate_bucket_name applies the S3 bucket naming rules honoured by most
// providers:
// - 3..63 chars
// - lowercase letters, digits, dots, hyphens only
// - must start/end with letter or digit
// - no consecutive dots, no `.-` / `-.`
// - cannot look like an IPv4 address
//
// Provider-specific reservations (e.g. `xn--`, `sthree-`) are intentionally
// not checked here — they vary, so the server-side error is authoritative.
pub fn validate_bucket_name(name string) ! {
if name.len < 3 || name.len > 63 {
return new_error('InvalidBucketName', 'Bucket name must be 3..63 characters: ${name}')
}
first := name[0]
last := name[name.len - 1]
if !is_lower_alnum(first) || !is_lower_alnum(last) {
return new_error('InvalidBucketName',
'Bucket name must start and end with a lowercase letter or digit: ${name}')
}
mut prev := u8(0)
for b in name.bytes() {
if !is_bucket_char(b) {
return new_error('InvalidBucketName', 'Bucket name "${name}" contains invalid character: ${[
b,
].bytestr()}')
}
if prev == `.` && b == `.` {
return new_error('InvalidBucketName', 'Bucket name "${name}" contains consecutive dots')
}
if (prev == `.` && b == `-`) || (prev == `-` && b == `.`) {
return new_error('InvalidBucketName',
'Bucket name "${name}" contains adjacent dot/hyphen')
}
prev = b
}
if looks_like_ipv4(name) {
return new_error('InvalidBucketName', 'Bucket name "${name}" looks like an IP address')
}
}
fn is_lower_alnum(b u8) bool {
return (b >= `a` && b <= `z`) || (b >= `0` && b <= `9`)
}
fn is_bucket_char(b u8) bool {
return is_lower_alnum(b) || b == `-` || b == `.`
}
fn looks_like_ipv4(s string) bool {
parts := s.split('.')
if parts.len != 4 {
return false
}
for p in parts {
if p.len == 0 || p.len > 3 {
return false
}
for b in p.bytes() {
if b < `0` || b > `9` {
return false
}
}
}
return true
}
fn pick_bucket(c Credentials, override string) !string {
b := if override != '' { override } else { c.bucket }
if b == '' {
return new_error('InvalidPath',
'No bucket given (set Credentials.bucket or pass bucket via options)')
}
return b
}
fn bucket_path(creds Credentials, bucket string) string {
extra := creds.extra_path()
if creds.virtual_hosted_style {
return if extra == '' { '/' } else { extra }
}
encoded := uri_encode_path(strip_slashes(bucket))
return '${extra}/${encoded}'
}
// escape_xml does the minimal XML special-char escape we need for body content.
fn escape_xml(s string) string {
mut out := []u8{cap: s.len}
for b in s.bytes() {
match b {
`<` { out << '&lt;'.bytes() }
`>` { out << '&gt;'.bytes() }
`&` { out << '&amp;'.bytes() }
`"` { out << '&quot;'.bytes() }
`'` { out << '&apos;'.bytes() }
else { out << b }
}
}
return out.bytestr()
}
+45
View File
@@ -0,0 +1,45 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
fn test_validate_bucket_name_accepts_valid() {
validate_bucket_name('my-bucket') or { assert false, 'should pass' }
validate_bucket_name('a.b.c') or { assert false, 'should pass' }
validate_bucket_name('abc') or { assert false, 'should pass' }
validate_bucket_name('a' + 'b'.repeat(61) + 'c') or { assert false, 'should pass' }
}
fn test_validate_bucket_name_rejects_invalid() {
if _ := validate_bucket_name('Bad-UPPER') {
assert false, 'uppercase must be rejected'
}
if _ := validate_bucket_name('ab') {
assert false, 'shorter than 3 chars must be rejected'
}
if _ := validate_bucket_name('-leading-hyphen') {
assert false, 'leading hyphen must be rejected'
}
if _ := validate_bucket_name('trailing-hyphen-') {
assert false, 'trailing hyphen must be rejected'
}
if _ := validate_bucket_name('192.168.0.1') {
assert false, 'IPv4-shaped name must be rejected'
}
if _ := validate_bucket_name('a..b') {
assert false, 'consecutive dots must be rejected'
}
if _ := validate_bucket_name('a.-b') {
assert false, 'adjacent dot/hyphen must be rejected'
}
if _ := validate_bucket_name('a-.b') {
assert false, 'adjacent hyphen/dot must be rejected'
}
if _ := validate_bucket_name('') {
assert false, 'empty name must be rejected'
}
long := 'a'.repeat(64)
if _ := validate_bucket_name(long) {
assert false, 'longer than 63 chars must be rejected'
}
}
+405
View File
@@ -0,0 +1,405 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
import net.http
import time
// Client is the entry point for S3 operations. It carries default credentials
// and tuneable HTTP behaviour; per-call overrides go through the option params
// of each method. Instantiate once, reuse for many objects.
@[heap]
pub struct Client {
pub:
credentials Credentials
// part_size is the multipart-upload chunk size in bytes (default 5 MiB,
// the S3 minimum).
part_size i64 = 5 * 1024 * 1024
// queue_size is the intended parallel-upload concurrency for multipart
// (currently sequential; reserved).
queue_size int = 5
// retry is the number of retry attempts for failed uploads.
retry int = 3
// read_timeout / write_timeout map onto V's net.http settings.
// Defaults are generous because parts can be 5 MiB+ on slow links.
read_timeout i64 = 5 * 60 * time.second
write_timeout i64 = 5 * 60 * time.second
}
// new_client builds a Client. Pass an empty Credentials to fall back to env vars.
pub fn new_client(creds Credentials) Client {
resolved := if creds.access_key_id == '' && creds.secret_access_key == '' {
Credentials.from_env().merge(creds)
} else {
creds
}
return Client{
credentials: resolved
}
}
// PutOptions configures a put / write call.
@[params]
pub struct PutOptions {
pub:
bucket string
content_type string
content_disposition string
content_encoding string
cache_control string
acl Acl
storage_class StorageClass
request_payer bool
// hash_payload, when true, computes SHA-256 of the body before signing
// instead of using `UNSIGNED-PAYLOAD`. Slightly stronger integrity guarantee
// at the cost of one full body scan.
hash_payload bool
}
// GetOptions configures a get / read call.
@[params]
pub struct GetOptions {
pub:
bucket string
range string // e.g. 'bytes=0-1023' for partial downloads
version_id string
request_payer bool
}
// StatOptions configures stat / size / exists checks.
@[params]
pub struct StatOptions {
pub:
bucket string
request_payer bool
}
// put uploads `data` to `key`. Use `upload_file` / `upload_bytes_multipart`
// for streaming or for files larger than ~100 MiB; `put` keeps everything in
// memory.
pub fn (c &Client) put(key string, data []u8, opts PutOptions) ! {
creds := c.creds_for(opts.bucket)
host := canonical_host(creds, opts.bucket)
if host == '' {
return new_error('InvalidEndpoint', 'cannot determine S3 host (set endpoint or bucket)')
}
path := build_object_path(creds, opts.bucket, key)!
payload_hash := if opts.hash_payload { sha256_hex(data) } else { unsigned_payload }
signed := sign_request(creds, SignRequest{
method: 'PUT'
path: path
payload_hash: payload_hash
extra_headers: put_object_headers(opts)
})!
resp := c.do_http(signed, data.bytestr())!
if resp.status_code !in [200, 204] {
return new_http_error(resp.status_code, key, resp.body)
}
}
// put_object_headers maps `PutOptions` to the wire headers used by both the
// single-shot `put` and the multipart `initiate_multipart` paths. Keeping the
// mapping in one place ensures the two upload modes accept the exact same set
// of options without drift.
fn put_object_headers(opts PutOptions) map[string]string {
mut headers := map[string]string{}
if opts.content_type != '' {
headers['content-type'] = opts.content_type
}
if opts.content_disposition != '' {
headers['content-disposition'] = opts.content_disposition
}
if opts.content_encoding != '' {
headers['content-encoding'] = opts.content_encoding
}
if opts.cache_control != '' {
headers['cache-control'] = opts.cache_control
}
if opts.acl != .unset {
headers['x-amz-acl'] = opts.acl.to_header_value()
}
if opts.storage_class != .unset {
headers['x-amz-storage-class'] = opts.storage_class.to_header_value()
}
if opts.request_payer {
headers['x-amz-request-payer'] = 'requester'
}
return headers
}
// get downloads the entire object body into memory. For large files set a
// `range` and assemble the result yourself, or use a presigned URL with an
// HTTP streaming client.
pub fn (c &Client) get(key string, opts GetOptions) ![]u8 {
creds := c.creds_for(opts.bucket)
path := build_object_path(creds, opts.bucket, key)!
mut headers := map[string]string{}
if opts.range != '' {
headers['range'] = opts.range
}
if opts.request_payer {
headers['x-amz-request-payer'] = 'requester'
}
mut query := ''
if opts.version_id != '' {
query = 'versionId=' + uri_encode_query(opts.version_id)
}
signed := sign_request(creds, SignRequest{
method: 'GET'
path: path
query: query
payload_hash: empty_sha256
extra_headers: headers
})!
resp := c.do_http(signed, '')!
if resp.status_code !in [200, 206] {
return new_http_error(resp.status_code, key, resp.body)
}
return resp.body.bytes()
}
// get_string is a convenience wrapper around `get` that returns the body as a
// V `string`.
pub fn (c &Client) get_string(key string, opts GetOptions) !string {
bytes := c.get(key, opts)!
return bytes.bytestr()
}
// stat returns the object metadata (size / last_modified / etag / content_type).
// Returns an `S3Error` with code `NoSuchKey` when the object doesn't exist.
pub fn (c &Client) stat(key string, opts StatOptions) !Stat {
creds := c.creds_for(opts.bucket)
path := build_object_path(creds, opts.bucket, key)!
mut headers := map[string]string{}
if opts.request_payer {
headers['x-amz-request-payer'] = 'requester'
}
signed := sign_request(creds, SignRequest{
method: 'HEAD'
path: path
payload_hash: empty_sha256
extra_headers: headers
})!
resp := c.do_http(signed, '')!
if resp.status_code == 404 {
return new_error('NoSuchKey', 'No such key: ${key}')
}
if resp.status_code !in [200, 204] {
return new_http_error(resp.status_code, key, resp.body)
}
return Stat{
size: resp.header.get(.content_length) or { '0' }.i64()
last_modified: resp.header.get(.last_modified) or { '' }
etag: (resp.header.get(.etag) or { '' }).trim('"')
content_type: resp.header.get(.content_type) or { '' }
}
}
// exists is `stat` + boolean: true on 200, false on 404, error otherwise.
pub fn (c &Client) exists(key string, opts StatOptions) !bool {
c.stat(key, opts) or {
if err is S3Error && err.code == 'NoSuchKey' {
return false
}
return err
}
return true
}
// size returns just the Content-Length of the object.
pub fn (c &Client) size(key string, opts StatOptions) !i64 {
st := c.stat(key, opts)!
return st.size
}
// delete removes a single object. S3 returns 204 on success, 204 on
// already-absent (idempotent) — we surface success in both cases.
pub fn (c &Client) delete(key string, opts StatOptions) ! {
creds := c.creds_for(opts.bucket)
path := build_object_path(creds, opts.bucket, key)!
mut headers := map[string]string{}
if opts.request_payer {
headers['x-amz-request-payer'] = 'requester'
}
signed := sign_request(creds, SignRequest{
method: 'DELETE'
path: path
payload_hash: empty_sha256
extra_headers: headers
})!
resp := c.do_http(signed, '')!
if resp.status_code !in [200, 204] {
return new_http_error(resp.status_code, key, resp.body)
}
}
// presign generates a presigned URL — see PresignOptions for tunables.
pub fn (c &Client) presign(key string, opts PresignOptions) !string {
creds := c.creds_for(opts.bucket)
path := build_object_path(creds, opts.bucket, key)!
mut extra := map[string]string{}
if opts.acl != .unset {
extra['X-Amz-Acl'] = opts.acl.to_header_value()
}
if opts.storage_class != .unset {
extra['x-amz-storage-class'] = opts.storage_class.to_header_value()
}
if opts.content_type != '' {
extra['response-content-type'] = opts.content_type
}
if opts.content_disposition != '' {
extra['response-content-disposition'] = opts.content_disposition
}
if opts.request_payer {
extra['x-amz-request-payer'] = 'requester'
}
return presign_url(creds, PresignRequest{
method: http_method_to_string(opts.method)
path: path
expires_in: opts.expires_in
extra_query: extra
})
}
// file returns a File reference for the given key, bound to this client.
// `path` may be `<bucket>/<key>` if the client has no default bucket and
// `bucket` here is empty.
pub fn (c &Client) file(key string, opts FileOptions) File {
return File{
client: c
key: key
bucket: opts.bucket
}
}
// creds_for returns credentials with `bucket` overridden if the caller passed one.
fn (c &Client) creds_for(bucket string) Credentials {
if bucket == '' {
return c.credentials
}
mut dup := c.credentials
dup = Credentials{
...dup
bucket: bucket
}
return dup
}
// build_object_path produces the canonical URI path for a key.
// Path style: `/<bucket>/<encoded-key>[+endpoint extra path]`
// Virtual hosted: `/<encoded-key>`
// Returns an error if neither bucket nor key is provided.
// The key is forwarded byte-exact (only percent-encoded): S3 treats keys as
// opaque identifiers, so `folder/`, `/x` and `a//b` all designate distinct
// objects and must reach the wire as written.
pub fn build_object_path(creds Credentials, bucket_override string, key string) !string {
bucket := if bucket_override != '' { bucket_override } else { creds.bucket }
if key == '' {
return new_error('InvalidPath', 'Empty object key')
}
encoded_key := uri_encode_path(key)
extra := creds.extra_path()
if creds.virtual_hosted_style {
return '${extra}/${encoded_key}'
}
if bucket == '' {
return new_error('InvalidPath',
'No bucket given (set Credentials.bucket or pass bucket via options)')
}
encoded_bucket := uri_encode_path(strip_slashes(bucket))
return '${extra}/${encoded_bucket}/${encoded_key}'
}
// HttpResponse is the small subset of an HTTP response we surface to callers
// of the lower-level helpers. `body` is the raw response body for non-stream
// requests; `header` keeps V's typed Header for parsing.
pub struct HttpResponse {
pub:
status_code int
body string
header http.Header
}
// do_http fires off the signed request via V's `net.http`. The HTTP method is
// taken from `signed.method` (set by `sign_request`) so callers cannot drift
// between what was signed and what is sent on the wire.
fn (c &Client) do_http(signed SignedRequest, body string) !HttpResponse {
mut header := http.new_header()
for k, v in signed.headers {
// `Host` is set automatically by V's http client; skip to avoid duplicates.
if k.to_lower() == 'host' {
continue
}
header.add_custom(canonical_header_name(k), v)!
}
method_enum := parse_http_method(signed.method)!
req := http.new_request(method_enum, signed.url, body)
mut req_mut := http.Request{
...req
header: header
read_timeout: c.read_timeout
write_timeout: c.write_timeout
}
resp := req_mut.do() or {
return new_error('NetworkError', 'HTTP request failed: ${err.msg()}')
}
$if s3_debug ? {
eprintln('[s3] ${signed.method} ${signed.url}')
for k, v in signed.headers {
eprintln('[s3] > ${k}: ${redacted_value(k, v)}')
}
eprintln('[s3] -> ${resp.status_code}')
eprintln('[s3] body: ${resp.body}')
}
return HttpResponse{
status_code: resp.status_code
body: resp.body
header: resp.header
}
}
// redacted_value masks credentials so they cannot leak via s3_debug logs.
fn redacted_value(name string, value string) string {
low := name.to_lower()
if low == 'authorization' || low.starts_with('x-amz-security-token') {
return '<redacted>'
}
return value
}
// canonical_header_name turns 'x-amz-content-sha256' into 'X-Amz-Content-Sha256'
// for the wire. HTTP/1.1 says headers are case-insensitive, but some
// intermediaries / mocks aren't, so we emit the canonical Title-Case form.
fn canonical_header_name(s string) string {
mut out := []u8{cap: s.len}
mut upper_next := true
for b in s.bytes() {
if b == `-` {
out << b
upper_next = true
continue
}
if upper_next && b >= `a` && b <= `z` {
out << b - 32
} else {
out << b
}
upper_next = false
}
return out.bytestr()
}
// parse_http_method maps a SigV4 canonical method string to V's `http.Method`.
// The signer enforces the supported set up-front, so anything unexpected here
// indicates a programming bug rather than user input.
fn parse_http_method(s string) !http.Method {
return match s.to_upper() {
'GET' { http.Method.get }
'PUT' { http.Method.put }
'POST' { http.Method.post }
'DELETE' { http.Method.delete }
'HEAD' { http.Method.head }
else { new_error('InvalidMethod', 'Unsupported HTTP method: ${s}') }
}
}
+204
View File
@@ -0,0 +1,204 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
import os
// Credentials carries the authentication material plus endpoint and addressing
// preferences. It is intentionally kept small; defaults (region, endpoint, etc.)
// are derived only when the request is signed, never stored implicitly, so the
// same Credentials value can be reused across regions/endpoints.
//
// Field naming matches V conventions (snake_case). The `from_env` helper
// recognises several provider conventions — see `from_env`.
pub struct Credentials {
pub:
access_key_id string
secret_access_key string
session_token string
region string
bucket string
endpoint string // 'https://s3.fr-par.scw.cloud' or 'host:port' — host part is what gets signed
virtual_hosted_style bool // when true, '<bucket>.<endpoint-host>' addressing
insecure_http bool // permit `http://` endpoints (false by default — never silently downgrades)
}
// Credentials.from_env reads credentials from environment variables, trying several
// provider conventions in order so the same code works against many hosts
// without reconfiguration. Each field is resolved independently; the first
// non-empty value wins.
//
// Lookup order per field:
// key id : S3_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID,
// CELLAR_ADDON_KEY_ID, SCW_ACCESS_KEY,
// B2_APPLICATION_KEY_ID, R2_ACCESS_KEY_ID, SPACES_KEY
// secret : S3_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY,
// CELLAR_ADDON_KEY_SECRET, SCW_SECRET_KEY,
// B2_APPLICATION_KEY, R2_SECRET_ACCESS_KEY, SPACES_SECRET
// session : S3_SESSION_TOKEN, AWS_SESSION_TOKEN
// region : S3_REGION, AWS_REGION, AWS_DEFAULT_REGION, SCW_DEFAULT_REGION
// bucket : S3_BUCKET
// endpoint : S3_ENDPOINT, AWS_ENDPOINT, AWS_ENDPOINT_URL,
// CELLAR_ADDON_HOST, B2_ENDPOINT, R2_ENDPOINT, SPACES_ENDPOINT
pub fn Credentials.from_env() Credentials {
endpoint := env_first('S3_ENDPOINT', 'AWS_ENDPOINT', 'AWS_ENDPOINT_URL', 'CELLAR_ADDON_HOST',
'B2_ENDPOINT', 'R2_ENDPOINT', 'SPACES_ENDPOINT')
insecure := endpoint.starts_with('http://')
return Credentials{
access_key_id: env_first('S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID',
'CELLAR_ADDON_KEY_ID', 'SCW_ACCESS_KEY', 'B2_APPLICATION_KEY_ID', 'R2_ACCESS_KEY_ID',
'SPACES_KEY')
secret_access_key: env_first('S3_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY',
'CELLAR_ADDON_KEY_SECRET', 'SCW_SECRET_KEY', 'B2_APPLICATION_KEY',
'R2_SECRET_ACCESS_KEY', 'SPACES_SECRET')
session_token: env_first('S3_SESSION_TOKEN', 'AWS_SESSION_TOKEN')
region: env_first('S3_REGION', 'AWS_REGION', 'AWS_DEFAULT_REGION',
'SCW_DEFAULT_REGION')
bucket: env_first('S3_BUCKET')
endpoint: endpoint
virtual_hosted_style: false
insecure_http: insecure
}
}
// merge produces a copy of `c` with non-empty fields from `other` overriding.
// Useful when callers pass per-call overrides while keeping a default Client.
pub fn (c Credentials) merge(other Credentials) Credentials {
return Credentials{
access_key_id: if other.access_key_id != '' {
other.access_key_id
} else {
c.access_key_id
}
secret_access_key: if other.secret_access_key != '' {
other.secret_access_key
} else {
c.secret_access_key
}
session_token: if other.session_token != '' {
other.session_token
} else {
c.session_token
}
region: if other.region != '' { other.region } else { c.region }
bucket: if other.bucket != '' { other.bucket } else { c.bucket }
endpoint: if other.endpoint != '' { other.endpoint } else { c.endpoint }
virtual_hosted_style: c.virtual_hosted_style || other.virtual_hosted_style
insecure_http: c.insecure_http || other.insecure_http
}
}
// resolved_region returns the region to use for signing. Order:
// 1. explicit `c.region`
// 2. parsed from `c.endpoint` when it follows the `s3.<region>.amazonaws.com` pattern
// 3. `'auto'` for Cloudflare R2
// 4. `'us-east-1'` (S3 historical default) when no endpoint is set
pub fn (c Credentials) resolved_region() string {
if c.region != '' {
return c.region
}
return guess_region(c.endpoint)
}
// validate ensures the credentials carry the minimum needed to sign a
// request. Also rejects credentials / region / bucket / endpoint values that
// contain CR or LF — those would let an attacker who controls *any* config
// field smuggle headers into the Authorization line.
pub fn (c Credentials) validate() ! {
if c.access_key_id == '' || c.secret_access_key == '' {
return new_error('MissingCredentials',
"Missing S3 credentials. 'access_key_id' and 'secret_access_key' are required.")
}
for name, value in {
'access_key_id': c.access_key_id
'secret_access_key': c.secret_access_key
'session_token': c.session_token
'region': c.region
'bucket': c.bucket
'endpoint': c.endpoint
} {
if contains_crlf(value) {
return new_error('InvalidCredentials',
'Credential field "${name}" contains CR or LF — refused (header-injection guard)')
}
}
}
// host_only returns the bare host[:port] from `c.endpoint`, stripping any
// scheme and trailing path. Returned value is what gets signed in the
// `host` header for SigV4.
pub fn (c Credentials) host_only() string {
mut ep := c.endpoint
if i := ep.index('://') {
ep = ep[i + 3..]
}
if j := ep.index('/') {
ep = ep[..j]
}
return ep
}
// extra_path returns the path component of `c.endpoint`, including any
// leading '/'. Useful for proxies that mount S3 under a sub-path.
pub fn (c Credentials) extra_path() string {
mut ep := c.endpoint
if i := ep.index('://') {
ep = ep[i + 3..]
}
if j := ep.index('/') {
return ep[j..]
}
return ''
}
// scheme returns 'http' or 'https' based on the endpoint's explicit scheme
// when present, falling back to `insecure_http`. So all three of these work:
// endpoint: 'https://s3.example.com' → https
// endpoint: 's3.example.com' → https (default)
// endpoint: 'http://localhost:9000' → http (auto-detected)
// endpoint: 'localhost:9000', insecure_http: true → http
pub fn (c Credentials) scheme() string {
if c.endpoint.starts_with('http://') {
return 'http'
}
if c.endpoint.starts_with('https://') {
return 'https'
}
return if c.insecure_http { 'http' } else { 'https' }
}
// guess_region derives the SigV4 region from an endpoint URL. Public so it
// can be reused by the higher-level Client for log / inspect output.
pub fn guess_region(endpoint string) string {
if endpoint == '' {
return 'us-east-1'
}
if endpoint.ends_with('.r2.cloudflarestorage.com') {
return 'auto'
}
if amz_end := endpoint.index('.amazonaws.com') {
if s3_pos := endpoint.index('s3.') {
start := s3_pos + 3
if start < amz_end {
return endpoint[start..amz_end]
}
}
// AWS global endpoint (`s3.amazonaws.com`) and legacy forms
// (`s3-external-1.amazonaws.com`) sign with `us-east-1`. SigV4
// rejects `auto` against AWS, so fall back here instead.
return 'us-east-1'
}
return 'auto'
}
// env_first returns the first non-empty env var among the names provided.
fn env_first(names ...string) string {
for n in names {
v := os.getenv(n)
if v != '' {
return v
}
}
return ''
}
+104
View File
@@ -0,0 +1,104 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
fn test_credentials_validate_rejects_crlf_injection() {
c := Credentials{
access_key_id: 'KEY'
secret_access_key: 'secret\r\nInjected: header'
}
if _ := c.validate() {
assert false, 'should have rejected CRLF'
}
}
fn test_credentials_from_env_returns_empty_when_unset() {
// Cannot safely set env vars from a test without polluting the parent
// shell, so just verify the call returns a value of the right shape.
c := Credentials.from_env()
_ = c
}
fn test_endpoint_scheme_is_derived_when_explicit() {
c1 := Credentials{
access_key_id: 'K'
secret_access_key: 'S'
endpoint: 'https://s3.example.com'
}
assert c1.scheme() == 'https'
c2 := Credentials{
access_key_id: 'K'
secret_access_key: 'S'
endpoint: 'http://localhost:9000'
}
assert c2.scheme() == 'http', 'http:// endpoint must yield http even when insecure_http is unset'
c3 := Credentials{
access_key_id: 'K'
secret_access_key: 'S'
endpoint: 'http://localhost:9000'
insecure_http: true
}
assert c3.scheme() == 'http'
// Bare host: defaults to https unless insecure_http is set.
c4 := Credentials{
access_key_id: 'K'
secret_access_key: 'S'
endpoint: 'localhost:9000'
}
assert c4.scheme() == 'https'
c5 := Credentials{
access_key_id: 'K'
secret_access_key: 'S'
endpoint: 'localhost:9000'
insecure_http: true
}
assert c5.scheme() == 'http'
}
fn test_endpoint_host_only_strips_scheme_and_path() {
c := Credentials{
access_key_id: 'K'
secret_access_key: 'S'
endpoint: 'https://proxy.example.com:8443/s3'
}
assert c.host_only() == 'proxy.example.com:8443'
assert c.extra_path() == '/s3'
}
fn test_guess_region_paths() {
assert guess_region('') == 'us-east-1'
assert guess_region('https://s3.eu-west-3.amazonaws.com') == 'eu-west-3'
assert guess_region('https://my-acct.r2.cloudflarestorage.com') == 'auto'
assert guess_region('https://s3.example.com') == 'auto'
// AWS global endpoints sign with us-east-1, not 'auto'.
assert guess_region('https://s3.amazonaws.com') == 'us-east-1'
assert guess_region('s3.amazonaws.com') == 'us-east-1'
assert guess_region('https://s3-external-1.amazonaws.com') == 'us-east-1'
}
fn test_credentials_merge_overrides_only_non_empty() {
base := Credentials{
access_key_id: 'BASE'
secret_access_key: 'basesecret'
region: 'eu-west-1'
bucket: 'b1'
}
other := Credentials{
bucket: 'b2'
}
got := base.merge(other)
assert got.access_key_id == 'BASE'
assert got.region == 'eu-west-1'
assert got.bucket == 'b2'
}
fn test_resolved_region_prefers_explicit() {
c := Credentials{
access_key_id: 'K'
secret_access_key: 'S'
region: 'eu-central-1'
endpoint: 'https://s3.us-west-2.amazonaws.com'
}
assert c.resolved_region() == 'eu-central-1'
}
+103
View File
@@ -0,0 +1,103 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
// uri_encode performs RFC 3986 percent-encoding as required by Signature V4.
// Only the unreserved set A–Z / a–z / 0–9 / '-' / '_' / '.' / '~' is preserved.
// When `encode_slash` is false (used for object keys), '/' is left intact and
// backslashes are normalized to '/' so Windows-style paths produce the same
// canonical key. All other bytes are emitted as %XX with uppercase hex digits.
pub fn uri_encode(input string, encode_slash bool) string {
mut out := []u8{cap: input.len + (input.len >> 2)} // 25% headroom
for b in input.bytes() {
match b {
`A`...`Z`, `a`...`z`, `0`...`9`, `-`, `_`, `.`, `~` {
out << b
}
`/`, `\\` {
if encode_slash {
append_percent(mut out, b)
} else {
out << if b == `\\` { u8(`/`) } else { b }
}
}
else {
append_percent(mut out, b)
}
}
}
return out.bytestr()
}
// uri_encode_path encodes an S3 object key segment-aware: '/' is preserved
// because S3 paths use it as the segment separator.
@[inline]
pub fn uri_encode_path(path string) string {
return uri_encode(path, false)
}
// uri_encode_query encodes a value that will appear inside a query string.
// Slashes must be percent-encoded.
@[inline]
pub fn uri_encode_query(value string) string {
return uri_encode(value, true)
}
// strip_slashes removes leading and trailing '/' or '\\' separators. S3
// canonical paths must contain a single leading slash, no trailing one.
pub fn strip_slashes(s string) string {
if s == '' {
return s
}
mut start := 0
mut end := s.len
for start < end && (s[start] == `/` || s[start] == `\\`) {
start++
}
for end > start && (s[end - 1] == `/` || s[end - 1] == `\\`) {
end--
}
return s[start..end]
}
// contains_crlf returns true if `value` contains a CR or LF byte. Header
// values that pass user-provided strings (ACL, content-type, …) MUST be
// checked to prevent HTTP header injection (CRLF smuggling).
@[inline]
pub fn contains_crlf(value string) bool {
for b in value.bytes() {
if b == `\r` || b == `\n` {
return true
}
}
return false
}
// to_hex_lower formats raw bytes as their lowercase hex string. Used for
// SHA-256 digests inside SigV4 (the spec requires lowercase).
pub fn to_hex_lower(data []u8) string {
mut out := []u8{len: data.len * 2}
for i, b in data {
out[i * 2] = hex_lower_nibble(b >> 4)
out[i * 2 + 1] = hex_lower_nibble(b & 0x0F)
}
return out.bytestr()
}
@[inline]
fn append_percent(mut out []u8, b u8) {
out << `%`
out << hex_upper_nibble(b >> 4)
out << hex_upper_nibble(b & 0x0F)
}
@[inline]
fn hex_upper_nibble(n u8) u8 {
return if n < 10 { n + `0` } else { n - 10 + `A` }
}
@[inline]
fn hex_lower_nibble(n u8) u8 {
return if n < 10 { n + `0` } else { n - 10 + `a` }
}
+49
View File
@@ -0,0 +1,49 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
fn test_uri_encode_path_keeps_slashes() {
assert uri_encode_path('/foo/bar baz/test.txt') == '/foo/bar%20baz/test.txt'
assert uri_encode_path('/folder/file with %.txt') == '/folder/file%20with%20%25.txt'
}
fn test_uri_encode_path_normalises_backslashes() {
// Windows-style separators must collapse to '/' in the canonical key.
assert uri_encode_path('foo\\bar\\baz.txt') == 'foo/bar/baz.txt'
}
fn test_uri_encode_query_encodes_slashes() {
assert uri_encode_query('a/b') == 'a%2Fb'
assert uri_encode_query('hello world') == 'hello%20world'
}
fn test_uri_encode_unreserved_passthrough() {
// RFC 3986 unreserved set must stay intact.
assert uri_encode('AZaz09-_.~', false) == 'AZaz09-_.~'
}
fn test_uri_encode_percent_uppercase() {
// AWS SigV4 mandates uppercase hex digits.
assert uri_encode('é', true) == '%C3%A9'
}
fn test_strip_slashes_trims_both_ends() {
assert strip_slashes('/foo/bar/') == 'foo/bar'
assert strip_slashes('//foo//') == 'foo'
assert strip_slashes('foo') == 'foo'
assert strip_slashes('') == ''
assert strip_slashes('///') == ''
}
fn test_contains_crlf() {
assert contains_crlf('hello\r\nworld') == true
assert contains_crlf('hello\rworld') == true
assert contains_crlf('hello\nworld') == true
assert contains_crlf('plain') == false
}
fn test_to_hex_lower() {
assert to_hex_lower([u8(0x00), 0x0f, 0xff, 0xa0]) == '000fffa0'
assert to_hex_lower([]) == ''
}
+206
View File
@@ -0,0 +1,206 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
// S3Error is the structured error returned for every signing or service
// failure. `code` is stable (e.g. `NoSuchKey`, `MissingCredentials`),
// `message` is human-readable, `status` is the HTTP status (0 for
// client-side errors), and `path` is the offending object key when known.
//
// V's `IError` interface only requires `msg()` and `code()`, so this type can
// be returned via `!T` and inspected with type-assertion / `as`.
pub struct S3Error {
pub:
code string
message string
status int
path string
resource string // S3's <Resource> field, when present
request_id string // S3 RequestId, useful for support tickets
}
// msg renders the error in a single line. Includes the S3 error code so users
// can switch on it without parsing the prose. Path is appended when known.
pub fn (e &S3Error) msg() string {
mut buf := '[${e.code}] ${e.message}'
if e.status != 0 {
buf += ' (HTTP ${e.status})'
}
if e.path != '' {
buf += ' — ${e.path}'
}
return buf
}
// code returns a stable numeric code so callers can use `if err.code() == ...`.
// We map a few well-known S3 codes; everything else returns 0 so callers fall
// back to string comparison on `e.code`.
pub fn (e &S3Error) code() int {
return match e.code {
'NoSuchKey' {
404
}
'NoSuchBucket' {
404
}
'BucketAlreadyExists', 'BucketAlreadyOwnedByYou' {
409
}
'AccessDenied', 'SignatureDoesNotMatch' {
403
}
'MissingCredentials', 'InvalidEndpoint', 'InvalidPath', 'InvalidMethod',
'InvalidSessionToken' {
400
}
else {
e.status
}
}
}
// new_error builds an S3Error from a code + message. Use this for client-side
// validation failures (missing creds, invalid path, etc.).
pub fn new_error(code string, message string) IError {
return &S3Error{
code: code
message: message
}
}
// new_http_error wraps an HTTP-level failure. Body is the raw response body
// — `parse_xml_error` is responsible for digging out the structured
// `<Error>` envelope when the server returns one.
pub fn new_http_error(status int, path string, body string) IError {
parsed := parse_xml_error(body)
if parsed.code != '' {
return &S3Error{
code: parsed.code
message: parsed.message
status: status
path: path
resource: parsed.resource
request_id: parsed.request_id
}
}
// Body was empty or unparseable — fall back to a generic code keyed by status.
return &S3Error{
code: fallback_code_for(status)
message: if body.len > 0 { body } else { 'HTTP ${status}' }
status: status
path: path
}
}
// XmlErrorFields is what we pull out of an `<Error>` XML envelope.
struct XmlErrorFields {
code string
message string
resource string
request_id string
}
// parse_xml_error extracts the standard S3 error XML:
//
// <Error>
// <Code>...</Code>
// <Message>...</Message>
// <Resource>...</Resource>
// <RequestId>...</RequestId>
// </Error>
//
// We use a tiny tag scanner (no DOM) — the format is rigid enough that this
// stays correct, faster than spinning up the full XML parser, and there is no
// attribute parsing to worry about.
fn parse_xml_error(body string) XmlErrorFields {
return XmlErrorFields{
code: extract_xml_tag(body, 'Code')
message: extract_xml_tag(body, 'Message')
resource: extract_xml_tag(body, 'Resource')
request_id: extract_xml_tag(body, 'RequestId')
}
}
// extract_xml_tag returns the inner text of `<tag>...</tag>` (first match,
// case-sensitive) with the five predefined XML entities decoded, or '' if
// the tag is absent.
pub fn extract_xml_tag(body string, tag string) string {
open := '<' + tag + '>'
close := '</' + tag + '>'
start := body.index(open) or { return '' }
rest_off := start + open.len
end := body.index_after(close, rest_off) or { return '' }
return decode_xml_entities(body[rest_off..end])
}
// decode_xml_entities decodes the five predefined XML entities. We don't try
// to handle arbitrary `&#nn;` sequences because S3 only ever uses these five.
pub fn decode_xml_entities(s string) string {
if !s.contains('&') {
return s
}
mut out := []u8{cap: s.len}
mut i := 0
for i < s.len {
if s[i] != `&` {
out << s[i]
i++
continue
}
matched := match true {
s.len - i >= 5 && s[i..i + 5] == '&amp;' {
out << `&`
i += 5
true
}
s.len - i >= 4 && s[i..i + 4] == '&lt;' {
out << `<`
i += 4
true
}
s.len - i >= 4 && s[i..i + 4] == '&gt;' {
out << `>`
i += 4
true
}
s.len - i >= 6 && s[i..i + 6] == '&quot;' {
out << `"`
i += 6
true
}
s.len - i >= 6 && s[i..i + 6] == '&apos;' {
out << `'`
i += 6
true
}
else {
false
}
}
if !matched {
out << s[i]
i++
}
}
return out.bytestr()
}
fn fallback_code_for(status int) string {
return match status {
301, 307 { 'PermanentRedirect' }
400 { 'BadRequest' }
401, 403 { 'AccessDenied' }
404 { 'NotFound' }
405 { 'MethodNotAllowed' }
409 { 'Conflict' }
411 { 'LengthRequired' }
412 { 'PreconditionFailed' }
416 { 'InvalidRange' }
429 { 'TooManyRequests' }
500 { 'InternalError' }
503 { 'ServiceUnavailable' }
else { 'HTTPError' }
}
}
+82
View File
@@ -0,0 +1,82 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
fn test_extract_xml_tag_basic() {
body := '<?xml version="1.0"?><Error><Code>NoSuchKey</Code><Message>Key not found</Message><RequestId>R1</RequestId></Error>'
assert extract_xml_tag(body, 'Code') == 'NoSuchKey'
assert extract_xml_tag(body, 'Message') == 'Key not found'
assert extract_xml_tag(body, 'RequestId') == 'R1'
assert extract_xml_tag(body, 'Missing') == ''
}
fn test_extract_xml_tag_decodes_entities() {
body := '<Message>a &amp; b &lt;tag&gt;</Message>'
assert extract_xml_tag(body, 'Message') == 'a & b <tag>'
}
fn test_decode_xml_entities() {
assert decode_xml_entities('&quot;hello&quot;') == '"hello"'
assert decode_xml_entities('a &amp; b &lt; c &gt; d &apos;e&apos;') == "a & b < c > d 'e'"
assert decode_xml_entities('no entities here') == 'no entities here'
}
fn test_s3error_msg_contains_code_and_path() {
e := S3Error{
code: 'NoSuchKey'
message: 'object missing'
status: 404
path: 'foo/bar.txt'
}
rendered := e.msg()
assert rendered.contains('NoSuchKey')
assert rendered.contains('object missing')
assert rendered.contains('404')
assert rendered.contains('foo/bar.txt')
}
fn test_s3error_code_maps_known_strings() {
assert (&S3Error{
code: 'NoSuchKey'
}).code() == 404
assert (&S3Error{
code: 'NoSuchBucket'
}).code() == 404
assert (&S3Error{
code: 'AccessDenied'
}).code() == 403
assert (&S3Error{
code: 'BucketAlreadyExists'
}).code() == 409
// Unknown code falls back to the wire status.
assert (&S3Error{
code: 'WeirdCode'
status: 418
}).code() == 418
}
fn test_new_http_error_parses_xml_body() {
body := '<?xml version="1.0"?><Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><Resource>/bucket/missing.txt</Resource><RequestId>ABC123</RequestId></Error>'
err := new_http_error(404, 'missing.txt', body)
if err is S3Error {
assert err.code == 'NoSuchKey'
assert err.message == 'The specified key does not exist.'
assert err.resource == '/bucket/missing.txt'
assert err.request_id == 'ABC123'
assert err.status == 404
assert err.path == 'missing.txt'
} else {
assert false, 'expected S3Error'
}
}
fn test_new_http_error_falls_back_when_body_empty() {
err := new_http_error(503, 'foo', '')
if err is S3Error {
assert err.code == 'ServiceUnavailable'
assert err.status == 503
} else {
assert false, 'expected S3Error'
}
}
+162
View File
@@ -0,0 +1,162 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
import net.http
// FetchOptions overlays an S3 endpoint over the `fetch` call. All fields
// are optional; bucket and key are taken from the URL.
@[params]
pub struct FetchOptions {
pub:
method http.Method = .get
body []u8
credentials Credentials
// content_type, acl, etc. are forwarded as-is when method is PUT/POST.
content_type string
content_disposition string
content_encoding string
cache_control string
acl Acl
storage_class StorageClass
request_payer bool
range string
hash_payload bool
}
// FetchResponse is the simplified return type of `fetch`. It's intentionally
// flat (no streaming yet) — easier to consume than V's `http.Response` and
// surfaces the most useful fields.
pub struct FetchResponse {
pub:
status_code int
body []u8
headers map[string]string
etag string
content_type string
content_length i64
}
// fetch is a `fetch('s3://bucket/key', { ... })`-style helper.
//
// Examples:
//
// resp := s3.fetch('s3://my-bucket/path/to/file.txt')!
// resp := s3.fetch('s3://my-bucket/key', method: .put, body: 'hello'.bytes())!
// resp := s3.fetch('s3://key', method: .get, credentials: s3.Credentials{ bucket: 'b', ... })!
//
// The URL must use the `s3://` scheme. Anything else is rejected outright
// (avoids accidentally calling a real HTTP endpoint with S3 credentials).
pub fn fetch(url string, opts FetchOptions) !FetchResponse {
if !url.starts_with('s3://') {
return new_error('InvalidURL', 's3.fetch only accepts s3:// URLs (got: ${redact_url(url)})')
}
bucket, key := parse_s3_url(url)!
mut creds := if opts.credentials.access_key_id == '' {
Credentials.from_env()
} else {
opts.credentials
}
if bucket != '' {
creds = Credentials{
...creds
bucket: bucket
}
}
creds.validate()!
c := Client{
credentials: creds
}
match opts.method {
.get {
data := c.get(key,
bucket: bucket
range: opts.range
request_payer: opts.request_payer
)!
return FetchResponse{
status_code: 200
body: data
content_length: i64(data.len)
}
}
.head {
st := c.stat(key,
bucket: bucket
request_payer: opts.request_payer
)!
return FetchResponse{
status_code: 200
etag: st.etag
content_type: st.content_type
content_length: st.size
}
}
.put {
c.put(key, opts.body,
bucket: bucket
content_type: opts.content_type
content_disposition: opts.content_disposition
content_encoding: opts.content_encoding
cache_control: opts.cache_control
acl: opts.acl
storage_class: opts.storage_class
request_payer: opts.request_payer
hash_payload: opts.hash_payload
)!
return FetchResponse{
status_code: 200
}
}
.delete {
c.delete(key,
bucket: bucket
request_payer: opts.request_payer
)!
return FetchResponse{
status_code: 204
}
}
else {
return new_error('InvalidMethod', 's3.fetch: unsupported method ${opts.method}')
}
}
return new_error('InvalidMethod', 's3.fetch: unreachable')
}
// parse_s3_url splits `s3://bucket/key/with/slashes` into (bucket, key).
//
// Special case: `s3://key` (no second path component) returns ('', 'key') —
// the caller is then expected to provide the bucket via credentials.
pub fn parse_s3_url(url string) !(string, string) {
if !url.starts_with('s3://') {
return new_error('InvalidURL', 'Not an s3:// URL')
}
rest := url[5..] // strip 's3://'
if rest == '' {
return new_error('InvalidURL', 'Empty s3:// URL')
}
if i := rest.index('/') {
bucket := rest[..i]
key := rest[i + 1..]
if key == '' {
return new_error('InvalidURL', 'Empty key in s3:// URL')
}
return bucket, key
}
// no '/' separator — treat the whole rest as the key
return '', rest
}
// redact_url strips query strings before logging. Used in error messages so
// presigned-URL-style query params (which can contain credentials) aren't
// leaked into logs.
pub fn redact_url(url string) string {
if i := url.index('?') {
return url[..i] + '?<redacted>'
}
return url
}
+79
View File
@@ -0,0 +1,79 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
fn test_parse_s3_url_full() {
bucket, key := parse_s3_url('s3://my-bucket/path/to/key.txt') or { panic(err) }
assert bucket == 'my-bucket'
assert key == 'path/to/key.txt'
}
fn test_parse_s3_url_only_key() {
bucket, key := parse_s3_url('s3://standalone-key') or { panic(err) }
assert bucket == ''
assert key == 'standalone-key'
}
fn test_parse_s3_url_rejects_other_schemes() {
if _, _ := parse_s3_url('https://example.com/x') {
assert false, 'should have errored'
}
if _, _ := parse_s3_url('http://x') {
assert false, 'should have errored'
}
if _, _ := parse_s3_url('') {
assert false, 'should have errored'
}
}
fn test_parse_s3_url_rejects_empty_key() {
if _, _ := parse_s3_url('s3://my-bucket/') {
assert false, 'should have errored'
}
if _, _ := parse_s3_url('s3://') {
assert false, 'should have errored'
}
}
fn test_redact_url_strips_query() {
assert redact_url('https://x.y/path?X-Amz-Signature=abc') == 'https://x.y/path?<redacted>'
assert redact_url('https://x.y/path') == 'https://x.y/path'
}
fn test_fetch_rejects_non_s3_scheme() {
if _ := fetch('https://example.com/x') {
assert false
}
if _ := fetch('http://example.com/x') {
assert false
}
if _ := fetch('file:///etc/passwd') {
assert false
}
}
fn test_fetch_rejects_post_method() {
// POST is not part of the s3.fetch contract: object writes use PUT,
// and POST in S3 is reserved for multipart-control endpoints which
// take an XML body that fetch does not generate. Reject explicitly
// rather than silently rewriting to PUT.
if _ := fetch('s3://my-bucket/key.txt', method: .post, body: 'x'.bytes()) {
assert false, 'POST should be rejected'
}
}
fn test_build_object_path_preserves_leading_and_trailing_slashes() {
creds := Credentials{
access_key_id: 'K'
secret_access_key: 'S'
bucket: 'b'
}
// S3 keys are byte-exact: `folder/`, `/x` and `a//b` must round-trip
// untouched (only percent-encoded). Stripping slashes would address a
// different object than requested.
assert build_object_path(creds, '', 'folder/')! == '/b/folder/'
assert build_object_path(creds, '', '/x')! == '/b//x'
assert build_object_path(creds, '', 'a//b')! == '/b/a//b'
assert build_object_path(creds, '', 'plain.txt')! == '/b/plain.txt'
}
+88
View File
@@ -0,0 +1,88 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
// FileOptions configures a File handle.
@[params]
pub struct FileOptions {
pub:
bucket string
}
// File is a reference to one S3 object. It holds no buffer; every method
// round-trips to S3 (or generates a presigned URL for `presign`).
//
// Construct via `Client.file(...)` or directly with `File{ client: &c, key: '...' }`.
pub struct File {
pub:
client &Client
bucket string
key string
}
// read returns the full object body. For range reads, see `read_range`.
pub fn (f &File) read() ![]u8 {
return f.client.get(f.key, bucket: f.bucket)
}
// text is a UTF-8 convenience over `read`.
pub fn (f &File) text() !string {
return f.client.get_string(f.key, bucket: f.bucket)
}
// read_range fetches `bytes=<begin>-<end_inclusive>` (HTTP Range semantics).
// Use `end < 0` to read to end-of-file.
pub fn (f &File) read_range(begin i64, end i64) ![]u8 {
range := if end < 0 { 'bytes=${begin}-' } else { 'bytes=${begin}-${end}' }
return f.client.get(f.key, bucket: f.bucket, range: range)
}
// write uploads `data` as the entire object body.
pub fn (f &File) write(data []u8, opts PutOptions) ! {
mut o := opts
if o.bucket == '' {
o = PutOptions{
...opts
bucket: f.bucket
}
}
f.client.put(f.key, data, o)!
}
// write_string is a UTF-8 convenience over `write`.
pub fn (f &File) write_string(s string, opts PutOptions) ! {
f.write(s.bytes(), opts)!
}
// stat returns object metadata (size / etag / last-modified / content-type).
pub fn (f &File) stat() !Stat {
return f.client.stat(f.key, bucket: f.bucket)
}
// exists is a HEAD that converts 404 into `false`.
pub fn (f &File) exists() !bool {
return f.client.exists(f.key, bucket: f.bucket)
}
// size returns the Content-Length, in bytes.
pub fn (f &File) size() !i64 {
return f.client.size(f.key, bucket: f.bucket)
}
// delete removes the object. Idempotent: no error when the object is absent.
pub fn (f &File) delete() ! {
f.client.delete(f.key, bucket: f.bucket)!
}
// presign returns a presigned URL for this object. See `PresignOptions`.
pub fn (f &File) presign(opts PresignOptions) !string {
mut o := opts
if o.bucket == '' && f.bucket != '' {
o = PresignOptions{
...opts
bucket: f.bucket
}
}
return f.client.presign(f.key, o)
}
+51
View File
@@ -0,0 +1,51 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
import net.http
// init registers an `s3://` scheme handler with `net.http` so that
// `http.fetch(url: 's3://...')` and friends route here transparently. Only
// active when this module is imported.
fn init() {
http.register_scheme('s3', http_fetch_handler)
}
// http_fetch_handler bridges `http.fetch` into `s3.fetch`. Credentials are
// pulled from the environment (`S3_*`, `AWS_*`, `CELLAR_ADDON_*`, …); for
// explicit credentials, callers should use `s3.fetch` directly.
fn http_fetch_handler(config http.FetchConfig) !http.Response {
resp := fetch(config.url,
method: config.method
body: config.data.bytes()
credentials: Credentials.from_env()
content_type: config.header.get(.content_type) or { '' }
)!
mut header := http.new_header()
for k, v in resp.headers {
header.add_custom(k, v) or {}
}
if resp.content_type != '' {
header.set(.content_type, resp.content_type)
}
if resp.etag != '' {
header.set(.etag, resp.etag)
}
return http.Response{
status_code: resp.status_code
body: resp.body.bytestr()
header: header
}
}
fn http_method_to_string(m http.Method) string {
return match m {
.get { 'GET' }
.put { 'PUT' }
.post { 'POST' }
.delete { 'DELETE' }
.head { 'HEAD' }
else { 'GET' }
}
}
+287
View File
@@ -0,0 +1,287 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
// Integration tests against a live S3-compatible endpoint. They are skipped
// unless the `S3_INTEGRATION` env var is set, and the `S3_HOST`, `S3_KEY_ID`
// and `S3_KEY_SECRET` env vars are populated. `v test .` stays fast and
// offline by default.
//
// Run with:
// S3_INTEGRATION=1 \
// S3_HOST=s3.example.com \
// S3_KEY_ID=... S3_KEY_SECRET=... \
// S3_BUCKET=vs3-tests \
// v test .
import os
import rand
import time
import net.http
// http_dispatch_creds_present returns true when the env exposes the AWS-style
// names that http.fetch's bridge will pick up via `Credentials.from_env()`.
// The integration test below relies on this — `S3_HOST` / `S3_KEY_ID` /
// `S3_KEY_SECRET` (used by the rest of this file) aren't recognised by
// `from_env()`, so we map them on the fly when needed.
fn ensure_aws_env_for_bridge() {
if os.getenv('S3_ACCESS_KEY_ID') == '' {
os.setenv('S3_ACCESS_KEY_ID', os.getenv('S3_KEY_ID'), true)
}
if os.getenv('S3_SECRET_ACCESS_KEY') == '' {
os.setenv('S3_SECRET_ACCESS_KEY', os.getenv('S3_KEY_SECRET'), true)
}
if os.getenv('S3_ENDPOINT') == '' {
host := os.getenv('S3_HOST')
endpoint := if host.contains('://') { host } else { 'https://${host}' }
os.setenv('S3_ENDPOINT', endpoint, true)
}
}
fn integration_enabled() bool {
return os.getenv('S3_INTEGRATION') != ''
}
fn live_client() ?Client {
if !integration_enabled() {
return none
}
host := os.getenv('S3_HOST')
key_id := os.getenv('S3_KEY_ID')
secret := os.getenv('S3_KEY_SECRET')
if host == '' || key_id == '' || secret == '' {
return none
}
endpoint := if host.contains('://') { host } else { 'https://${host}' }
bucket := os.getenv_opt('S3_BUCKET') or { 'vs3-tests' }
return Client{
credentials: Credentials{
endpoint: endpoint
access_key_id: key_id
secret_access_key: secret
bucket: bucket
region: 'us-east-1'
}
}
}
fn rand_key(prefix string) string {
return '${prefix}/${time.now().unix()}-${rand.u64()}.txt'
}
fn test_integration_roundtrip() {
c := live_client() or { return }
key := rand_key('it/roundtrip')
defer {
c.delete(key) or {}
}
c.put(key, 'hi'.bytes(), content_type: 'text/plain') or { panic(err) }
got := c.get_string(key) or { panic(err) }
assert got == 'hi'
stat := c.stat(key) or { panic(err) }
assert stat.size == 2
assert stat.content_type.starts_with('text/plain')
}
fn test_integration_range_read() {
c := live_client() or { return }
key := rand_key('it/range')
defer {
c.delete(key) or {}
}
c.put(key, 'abcdefghijklmnop'.bytes()) or { panic(err) }
first := c.get(key, range: 'bytes=0-4') or { panic(err) }
assert first.bytestr() == 'abcde', 'got: ${first.bytestr()}'
tail := c.get(key, range: 'bytes=10-') or { panic(err) }
assert tail.bytestr() == 'klmnop', 'got: ${tail.bytestr()}'
}
fn test_integration_presign_get() {
c := live_client() or { return }
key := rand_key('it/presign')
defer {
c.delete(key) or {}
}
c.put(key, 'PRESIGN_OK'.bytes()) or { panic(err) }
url := c.presign(key, expires_in: 60) or { panic(err) }
body := fetch_url(url) or { panic(err) }
assert body.contains('PRESIGN_OK'), 'got: ${body}'
}
fn test_integration_list_with_prefix() {
c := live_client() or { return }
prefix := 'it/list-${rand.u64()}/'
defer {
// Best-effort cleanup
if res := c.list(prefix: prefix, max_keys: 100) {
for o in res.objects {
c.delete(o.key) or {}
}
}
}
for i in 0 .. 3 {
c.put('${prefix}item-${i}.txt', 'x'.bytes()) or { panic(err) }
}
res := c.list(prefix: prefix, fetch_owner: false) or { panic(err) }
assert res.objects.len == 3
for o in res.objects {
assert o.key.starts_with(prefix)
assert o.size == 1
}
}
fn test_integration_exists_and_delete() {
c := live_client() or { return }
key := rand_key('it/exists')
c.put(key, 'present'.bytes()) or { panic(err) }
assert c.exists(key) or { panic(err) }
c.delete(key) or { panic(err) }
assert !(c.exists(key) or { panic(err) })
}
fn test_integration_fetch_helper() {
c := live_client() or { return }
key := rand_key('it/fetch')
url := 's3://${c.credentials.bucket}/${key}'
defer {
c.delete(key) or {}
}
put := fetch(url,
method: .put
body: 'fetch_works'.bytes()
credentials: c.credentials
) or { panic(err) }
assert put.status_code == 200
get := fetch(url, credentials: c.credentials) or { panic(err) }
assert get.body.bytestr() == 'fetch_works'
}
fn test_integration_multipart_upload_bytes() {
c := live_client() or { return }
// 6 MiB triggers multipart (one full part + a tail).
mut data := []u8{len: 6 * 1024 * 1024}
for i in 0 .. data.len {
data[i] = u8(i & 0xFF)
}
key := rand_key('it/multipart')
defer {
c.delete(key) or {}
}
c.upload_bytes_multipart(key, data) or { panic(err) }
stat := c.stat(key) or { panic(err) }
assert stat.size == data.len
got := c.get(key) or { panic(err) }
assert got.len == data.len
for i in 0 .. data.len {
if got[i] != data[i] {
assert false, 'byte mismatch at ${i}'
break
}
}
}
fn test_integration_multipart_parallel_many_parts() {
c := live_client() or { return }
// Force 8 parts at the 5 MiB minimum + tail. With queue_size=5 (default)
// we get genuine parallel dispatch and out-of-order completion that the
// dispatcher must re-order before CompleteMultipartUpload.
mut data := []u8{len: 8 * 5 * 1024 * 1024 + 1024}
for i in 0 .. data.len {
data[i] = u8((i * 31 + 7) & 0xFF)
}
key := rand_key('it/parallel')
defer {
c.delete(key) or {}
}
c.upload_bytes_multipart(key, data) or { panic(err) }
stat := c.stat(key) or { panic(err) }
assert stat.size == data.len, 'size mismatch: got ${stat.size}'
// Verify a tail range, where corruption from out-of-order parts would land.
tail := c.get(key, range: 'bytes=${data.len - 1024}-') or { panic(err) }
for i in 0 .. tail.len {
if tail[i] != data[data.len - 1024 + i] {
assert false, 'tail byte mismatch at offset ${i}'
break
}
}
}
fn test_integration_bucket_lifecycle() {
c := live_client() or { return }
// Random suffix keeps reruns idempotent.
name := 's3v-it-${rand.u64():08x}'.to_lower()
c.create_bucket(bucket: name) or {
// Some providers require region constraint or have different defaults
// — surface but don't fail the suite for that one provider quirk.
eprintln('skip: create_bucket(${name}) failed: ${err}')
return
}
defer {
c.delete_bucket(bucket: name) or {}
}
exists := c.bucket_exists(bucket: name) or { panic(err) }
assert exists, 'bucket ${name} should exist after create'
// Delete once explicitly so we can assert it disappeared.
c.delete_bucket(bucket: name) or { panic(err) }
gone := c.bucket_exists(bucket: name) or { panic(err) }
assert !gone, 'bucket ${name} should be gone after delete'
}
fn test_integration_invalid_credentials_yields_clean_error() {
if !integration_enabled() {
return
}
host := os.getenv('S3_HOST')
if host == '' {
return
}
c := Client{
credentials: Credentials{
endpoint: 'https://${host}'
access_key_id: 'AKIADOESNOTEXIST'
secret_access_key: 'badbadbadbadbadbadbadbadbadbadbadbadbadbad'
bucket: os.getenv_opt('S3_BUCKET') or { 'vs3-tests' }
}
}
if _ := c.stat('any-key') {
assert false, 'expected error'
} else {
assert err is S3Error, 'got non-S3 error: ${typeof(err).name}'
}
}
// fetch_url is a tiny `http.get`-style helper used only in the presign test.
fn fetch_url(url string) !string {
resp := http.get(url)!
return resp.body
}
// Exercises the `http.fetch(url: 's3://...')` bridge end-to-end. The s3
// module's `init()` registers itself with net.http; this test goes through
// that path (no direct `s3.fetch` call) to prove the dispatch works.
fn test_integration_http_fetch_dispatch_to_s3() {
c := live_client() or { return }
ensure_aws_env_for_bridge()
bucket := c.credentials.bucket
key := rand_key('it/http-bridge')
url := 's3://${bucket}/${key}'
put := http.fetch(url: url, method: .put, data: 'via-http.fetch') or {
assert false, 'PUT through bridge failed: ${err}'
return
}
assert put.status_code == 200, 'unexpected PUT status: ${put.status_code}'
get := http.fetch(url: url) or {
assert false, 'GET through bridge failed: ${err}'
return
}
assert get.status_code == 200, 'unexpected GET status: ${get.status_code}'
assert get.body == 'via-http.fetch', 'body mismatch: ${get.body}'
del := http.fetch(url: url, method: .delete) or {
assert false, 'DELETE through bridge failed: ${err}'
return
}
assert del.status_code in [200, 204], 'unexpected DELETE status: ${del.status_code}'
}
+121
View File
@@ -0,0 +1,121 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
// list returns up to ~1000 objects matching `opts`. Use the returned
// `next_continuation_token` to page through more.
//
// Speaks the ListObjectsV2 protocol.
pub fn (c &Client) list(opts ListOptions) !ListResult {
bucket := pick_bucket(c.credentials, opts.bucket)!
creds := c.creds_for(bucket)
path := bucket_path(creds, bucket)
mut q := map[string]string{}
q['list-type'] = '2'
if opts.prefix != '' {
q['prefix'] = opts.prefix
}
if opts.continuation_token != '' {
q['continuation-token'] = opts.continuation_token
}
if opts.delimiter != '' {
q['delimiter'] = opts.delimiter
}
if opts.max_keys > 0 {
q['max-keys'] = opts.max_keys.str()
}
if opts.start_after != '' {
q['start-after'] = opts.start_after
}
if opts.encoding_type != '' {
q['encoding-type'] = opts.encoding_type
}
if opts.fetch_owner {
q['fetch-owner'] = 'true'
}
query := canonical_query_string(q)
signed := sign_request(creds, SignRequest{
method: 'GET'
path: path
query: query
payload_hash: empty_sha256
})!
resp := c.do_http(signed, '')!
if resp.status_code != 200 {
return new_http_error(resp.status_code, bucket, resp.body)
}
return parse_list_response(resp.body)
}
// parse_list_response is hand-rolled XML extraction. ListObjectsV2 uses a
// rigid structure with no attributes we care about, so a tag scanner beats
// pulling in the full XML decoder both in code size and predictability
// against malformed input.
pub fn parse_list_response(body string) !ListResult {
objects := parse_contents(body)
common := parse_common_prefixes(body)
max_keys := extract_xml_tag(body, 'MaxKeys').int()
key_count := extract_xml_tag(body, 'KeyCount').int()
is_truncated := extract_xml_tag(body, 'IsTruncated') == 'true'
return ListResult{
name: extract_xml_tag(body, 'Name')
prefix: extract_xml_tag(body, 'Prefix')
delimiter: extract_xml_tag(body, 'Delimiter')
start_after: extract_xml_tag(body, 'StartAfter')
max_keys: max_keys
key_count: key_count
is_truncated: is_truncated
continuation_token: extract_xml_tag(body, 'ContinuationToken')
next_continuation_token: extract_xml_tag(body, 'NextContinuationToken')
objects: objects
common_prefixes: common
}
}
fn parse_contents(body string) []ObjectInfo {
mut out := []ObjectInfo{}
mut start := 0
for {
open := body.index_after('<Contents>', start) or { break }
close := body.index_after('</Contents>', open) or { break }
seg := body[open + '<Contents>'.len..close]
mut owner := ?Owner(none)
owner_open := seg.index('<Owner>') or { -1 }
if owner_open >= 0 {
owner_close := seg.index('</Owner>') or { -1 }
if owner_close > owner_open {
owner_seg := seg[owner_open + '<Owner>'.len..owner_close]
owner = Owner{
id: extract_xml_tag(owner_seg, 'ID')
display_name: extract_xml_tag(owner_seg, 'DisplayName')
}
}
}
out << ObjectInfo{
key: extract_xml_tag(seg, 'Key')
last_modified: extract_xml_tag(seg, 'LastModified')
etag: extract_xml_tag(seg, 'ETag').trim('"')
size: extract_xml_tag(seg, 'Size').i64()
storage_class: extract_xml_tag(seg, 'StorageClass')
owner: owner
}
start = close + '</Contents>'.len
}
return out
}
fn parse_common_prefixes(body string) []CommonPrefix {
mut out := []CommonPrefix{}
mut start := 0
for {
open := body.index_after('<CommonPrefixes>', start) or { break }
close := body.index_after('</CommonPrefixes>', open) or { break }
seg := body[open + '<CommonPrefixes>'.len..close]
out << CommonPrefix{
prefix: extract_xml_tag(seg, 'Prefix')
}
start = close + '</CommonPrefixes>'.len
}
return out
}
+57
View File
@@ -0,0 +1,57 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
// Captured from a real ListObjectsV2 response (S3-compatible endpoint),
// trimmed to the fields we parse.
const list_response_xml = '<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Name>test-bucket</Name>
<Prefix>foo/</Prefix>
<KeyCount>2</KeyCount>
<MaxKeys>1000</MaxKeys>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>foo/a.txt</Key>
<Size>10</Size>
<ETag>&quot;d41d8cd98f00b204e9800998ecf8427e&quot;</ETag>
<LastModified>2024-01-15T00:00:00.000Z</LastModified>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>foo/b.txt</Key>
<Size>20</Size>
<ETag>&quot;9b8b80d22b1d56a3f9b2c25d6e3e74c5&quot;</ETag>
<LastModified>2024-01-16T00:00:00.000Z</LastModified>
</Contents>
</ListBucketResult>'
fn test_parse_list_response_basic() {
r := parse_list_response(list_response_xml) or { panic(err) }
assert r.name == 'test-bucket'
assert r.prefix == 'foo/'
assert r.key_count == 2
assert !r.is_truncated
assert r.objects.len == 2
assert r.objects[0].key == 'foo/a.txt'
assert r.objects[0].size == 10
assert r.objects[0].etag == 'd41d8cd98f00b204e9800998ecf8427e'
assert r.objects[1].key == 'foo/b.txt'
}
fn test_parse_list_response_truncated() {
body := '<?xml version="1.0"?><ListBucketResult><Name>b</Name><IsTruncated>true</IsTruncated><NextContinuationToken>opaqueToken==</NextContinuationToken><KeyCount>1</KeyCount><Contents><Key>only.txt</Key><Size>1</Size><ETag>&quot;abc&quot;</ETag></Contents></ListBucketResult>'
r := parse_list_response(body) or { panic(err) }
assert r.is_truncated
assert r.next_continuation_token == 'opaqueToken=='
}
fn test_parse_list_response_common_prefixes() {
body := '<?xml version="1.0"?><ListBucketResult><Name>b</Name><Delimiter>/</Delimiter><CommonPrefixes><Prefix>folder1/</Prefix></CommonPrefixes><CommonPrefixes><Prefix>folder2/</Prefix></CommonPrefixes><KeyCount>0</KeyCount></ListBucketResult>'
r := parse_list_response(body) or { panic(err) }
assert r.delimiter == '/'
assert r.common_prefixes.len == 2
assert r.common_prefixes[0].prefix == 'folder1/'
assert r.common_prefixes[1].prefix == 'folder2/'
}
+410
View File
@@ -0,0 +1,410 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
import os
import strings
import time
// Multipart upload constants — defaults aligned with the S3 protocol limits.
pub const min_part_size = i64(5 * 1024 * 1024) // 5 MiB
pub const max_part_size = i64(5 * 1024 * 1024 * 1024) // 5 GiB
pub const max_parts = 10000 // hard S3 limit
// upload_file streams a local file to S3, choosing single-part or multipart
// based on size. Use this for files larger than ~50 MiB or anything you
// don't want to slurp into memory.
//
// `key` is the destination object key. The local file is read in
// `client.part_size` chunks (default 5 MiB).
pub fn (c &Client) upload_file(key string, local_path string, opts PutOptions) ! {
stat_local := os.stat(local_path) or {
return new_error('LocalFileNotFound',
'Cannot stat local file: ${local_path} (${err.msg()})')
}
size := i64(stat_local.size)
if size <= min_part_size {
data := os.read_bytes(local_path) or {
return new_error('LocalReadFailed', 'Cannot read ${local_path}: ${err.msg()}')
}
return c.put(key, data, opts)
}
c.upload_file_multipart(key, local_path, size, opts)!
}
// MultipartUploader is a stateful handle to an in-flight multipart upload,
// produced by `Client.start_multipart`. Use it when you want to push parts
// generated on-the-fly (network sources, decompressors, anything you don't
// want to materialise on disk):
//
// mut up := c.start_multipart('key', s3.PutOptions{ content_type: 'application/octet-stream' })!
// for chunk in chunks { up.upload(chunk)! }
// up.complete()!
//
// On error, `complete()` / `upload()` return without aborting. The caller
// must invoke `abort()` themselves so that `defer { up.abort() or {} }`
// remains an explicit, visible cleanup hook.
pub struct MultipartUploader {
mut:
client &Client
key string
upload_id string
opts PutOptions
parts []PartRef
part_number int
completed bool
aborted bool
}
// start_multipart begins a multipart upload and returns a MultipartUploader.
// Memory cost: zero — each `upload(chunk)` call streams the chunk to S3 and
// returns when it is acknowledged.
pub fn (c &Client) start_multipart(key string, opts PutOptions) !MultipartUploader {
upload_id := c.initiate_multipart(key, opts)!
return MultipartUploader{
// `unsafe { c }` is required to store the receiver's pointer in the
// returned struct — V's borrow checker forbids this implicitly, even
// though `Client` is `@[heap]` so the lifetime is safe.
client: unsafe { c }
key: key
upload_id: upload_id
opts: opts
parts: []PartRef{}
part_number: 1
}
}
// upload pushes one chunk as the next part. Each chunk MUST be at least
// `min_part_size` (5 MiB) except the last one — that's an S3 invariant
// the server enforces; we do not buffer for you.
pub fn (mut u MultipartUploader) upload(data []u8) ! {
if u.completed || u.aborted {
return new_error('InvalidState', 'multipart upload is already finalised')
}
if u.part_number > max_parts {
return new_error('TooManyParts', 'Exceeded ${max_parts} parts — increase part_size')
}
etag := u.client.upload_part(u.key, u.upload_id, u.part_number, data, u.opts)!
u.parts << PartRef{
part_number: u.part_number
etag: etag
}
u.part_number++
}
// complete finalises the upload. After this call the object is visible.
pub fn (mut u MultipartUploader) complete() ! {
if u.completed {
return
}
u.client.complete_multipart(u.key, u.upload_id, u.parts, u.opts)!
u.completed = true
}
// abort cancels the in-flight upload. Idempotent.
pub fn (mut u MultipartUploader) abort() ! {
if u.aborted || u.completed {
return
}
u.client.abort_multipart(u.key, u.upload_id, u.opts)!
u.aborted = true
}
// upload_bytes_multipart uploads an in-memory byte buffer using multipart.
// Useful for large generated payloads (test fixtures up to 5 GiB). Uses up
// to `Client.queue_size` parallel part uploads.
pub fn (c &Client) upload_bytes_multipart(key string, data []u8, opts PutOptions) ! {
upload_id := c.initiate_multipart(key, opts)!
parts := c.run_parallel_parts(key, upload_id, opts, fn [data] (offset i64, want int) ![]u8 {
end := if offset + i64(want) > i64(data.len) { i64(data.len) } else { offset + i64(want) }
return data[offset..end]
}, i64(data.len)) or {
c.abort_multipart(key, upload_id, opts) or {}
return err
}
c.complete_multipart(key, upload_id, parts, opts)!
}
// upload_file_multipart streams a local file part-by-part with up to
// `Client.queue_size` concurrent uploads. Peak memory is roughly
// `queue_size * part_size`.
pub fn (c &Client) upload_file_multipart(key string, local_path string, size i64, opts PutOptions) ! {
mut f := os.open(local_path) or {
return new_error('LocalReadFailed', 'open ${local_path}: ${err.msg()}')
}
defer {
f.close()
}
upload_id := c.initiate_multipart(key, opts)!
parts := c.run_parallel_parts(key, upload_id, opts, fn [mut f] (offset i64, want int) ![]u8 {
mut buf := []u8{len: want}
n := f.read_bytes_into(u64(offset), mut buf) or {
return new_error('LocalReadFailed', 'read_bytes_into: ${err.msg()}')
}
if n <= 0 {
return new_error('LocalReadFailed', 'unexpected EOF at offset ${offset}')
}
return buf[..n]
}, size) or {
c.abort_multipart(key, upload_id, opts) or {}
return err
}
c.complete_multipart(key, upload_id, parts, opts)!
}
// PartJob is a single upload-part task pushed onto the worker queue.
struct PartJob {
part_number int
data []u8
}
// PartResult carries the worker outcome back to the dispatcher.
struct PartResult {
part_number int
etag string
err ?IError
}
// part_worker pulls jobs off `jobs` and pushes results onto `results`. Exits
// when `jobs` is closed.
fn (c &Client) part_worker(key string, upload_id string, opts PutOptions, jobs chan PartJob, results chan PartResult) {
for {
job := <-jobs or { return }
etag := c.upload_part(key, upload_id, job.part_number, job.data, opts) or {
results <- PartResult{
part_number: job.part_number
err: err
}
continue
}
results <- PartResult{
part_number: job.part_number
etag: etag
}
}
}
// run_parallel_parts dispatches `producer` chunks across a worker pool of
// `Client.queue_size` goroutines and gathers the resulting PartRefs. The
// caller is responsible for issuing `initiate_multipart` / `abort_multipart`
// / `complete_multipart` around this call. `producer(offset, want)` must
// return a freshly-allocated chunk of length ≤ `want` bytes (workers may
// hold the slice past the next call). Returns parts in completion order;
// `complete_multipart` re-sorts by part number before sending the manifest.
fn (c &Client) run_parallel_parts(key string, upload_id string, opts PutOptions, producer fn (offset i64, want int) ![]u8, total i64) ![]PartRef {
workers := if c.queue_size < 1 { 1 } else { c.queue_size }
part_size := if c.part_size < min_part_size { min_part_size } else { c.part_size }
jobs := chan PartJob{cap: workers}
results := chan PartResult{cap: workers}
for _ in 0 .. workers {
spawn c.part_worker(key, upload_id, opts, jobs, results)
}
mut parts := []PartRef{}
mut first_err := ?IError(none)
mut sent := 0
mut received := 0
mut offset := i64(0)
mut part_number := 1
for offset < total && first_err == none {
if part_number > max_parts {
first_err = new_error('TooManyParts',
'Exceeded ${max_parts} parts — increase part_size')
break
}
want := if offset + part_size > total { int(total - offset) } else { int(part_size) }
chunk := producer(offset, want) or {
first_err = err
break
}
if chunk.len == 0 {
break
}
jobs <- PartJob{
part_number: part_number
data: chunk
}
sent++
offset += i64(chunk.len)
part_number++
// Drain ready results without blocking — surfaces errors early so we
// can stop dispatching new parts.
for {
select {
r := <-results {
received++
if e := r.err {
if first_err == none {
first_err = e
}
} else {
parts << PartRef{
part_number: r.part_number
etag: r.etag
}
}
}
else {
break
}
}
}
}
jobs.close()
for received < sent {
r := <-results
received++
if e := r.err {
if first_err == none {
first_err = e
}
} else {
parts << PartRef{
part_number: r.part_number
etag: r.etag
}
}
}
results.close()
if e := first_err {
return e
}
return parts
}
// PartRef is one ETag/PartNumber pair recorded during multipart upload.
pub struct PartRef {
pub:
part_number int
etag string
}
// initiate_multipart starts an upload and returns the UploadId. ACL,
// content-type, and friends from `opts` are sent here — they apply to the
// final object.
pub fn (c &Client) initiate_multipart(key string, opts PutOptions) !string {
creds := c.creds_for(opts.bucket)
path := build_object_path(creds, opts.bucket, key)!
signed := sign_request(creds, SignRequest{
method: 'POST'
path: path
query: 'uploads='
payload_hash: empty_sha256
extra_headers: put_object_headers(opts)
})!
resp := c.do_http(signed, '')!
if resp.status_code != 200 {
return new_http_error(resp.status_code, key, resp.body)
}
upload_id := extract_xml_tag(resp.body, 'UploadId')
if upload_id == '' {
return new_error('InvalidResponse', 'CreateMultipartUploadResult missing UploadId')
}
return upload_id
}
// upload_part uploads a single chunk and returns the server-side ETag. Up to
// `Client.retry` attempts with exponential backoff (200ms, 400ms, 800ms, …)
// on transient failures.
//
// The payload is always SHA-256-signed (not `UNSIGNED-PAYLOAD`) so the
// service validates byte-for-byte integrity end-to-end. Multipart uploads
// don't carry a Content-MD5 header by default; without payload signing a
// flipped bit in transit would silently produce a corrupt object that
// passes the multipart ETag check.
pub fn (c &Client) upload_part(key string, upload_id string, part_number int, data []u8, opts PutOptions) !string {
creds := c.creds_for(opts.bucket)
path := build_object_path(creds, opts.bucket, key)!
query := canonical_query_string({
'partNumber': part_number.str()
'uploadId': upload_id
})
// Hash the payload once — multi-MiB chunks should not be re-hashed per attempt.
// The signature itself still has to be recomputed on each retry because the
// `x-amz-date` header advances.
payload_hash := sha256_hex(data)
body := data.bytestr()
max_attempts := if c.retry < 1 { 1 } else { c.retry }
mut last_err := IError(new_error('NetworkError', 'upload_part: no attempt was made'))
for attempt in 0 .. max_attempts {
signed := sign_request(creds, SignRequest{
method: 'PUT'
path: path
query: query
payload_hash: payload_hash
})!
resp := c.do_http(signed, body) or {
last_err = err
if attempt + 1 < max_attempts {
time.sleep((1 << attempt) * 200 * time.millisecond)
}
continue
}
if resp.status_code == 200 {
etag := resp.header.get(.etag) or { '' }
if etag == '' {
return new_error('InvalidResponse', 'UploadPart returned no ETag')
}
return etag.trim('"')
}
last_err = new_http_error(resp.status_code, key, resp.body)
if attempt + 1 < max_attempts {
time.sleep((1 << attempt) * 200 * time.millisecond)
}
}
return last_err
}
// complete_multipart finalizes a multipart upload. Parts must be in ascending
// `part_number` order; we sort defensively.
pub fn (c &Client) complete_multipart(key string, upload_id string, parts []PartRef, opts PutOptions) ! {
mut sorted := parts.clone()
sorted.sort(a.part_number < b.part_number)
mut body := strings.new_builder(1024)
body.write_string('<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/">')
for p in sorted {
body.write_string('<Part><PartNumber>${p.part_number}</PartNumber><ETag>"${p.etag}"</ETag></Part>')
}
body.write_string('</CompleteMultipartUpload>')
xml := body.str()
creds := c.creds_for(opts.bucket)
path := build_object_path(creds, opts.bucket, key)!
q := 'uploadId=' + uri_encode_query(upload_id)
signed := sign_request(creds, SignRequest{
method: 'POST'
path: path
query: q
payload_hash: sha256_hex(xml.bytes())
extra_headers: {
'content-type': 'application/xml'
}
})!
resp := c.do_http(signed, xml)!
if resp.status_code != 200 {
return new_http_error(resp.status_code, key, resp.body)
}
// CompleteMultipartUpload may return HTTP 200 with an <Error> body — surface those.
if resp.body.contains('<Error>') {
return new_http_error(200, key, resp.body)
}
}
// abort_multipart cancels an in-flight upload. Best-effort — callers usually
// invoke it inside an error path and don't care about the result.
pub fn (c &Client) abort_multipart(key string, upload_id string, opts PutOptions) ! {
creds := c.creds_for(opts.bucket)
path := build_object_path(creds, opts.bucket, key)!
q := 'uploadId=' + uri_encode_query(upload_id)
signed := sign_request(creds, SignRequest{
method: 'DELETE'
path: path
query: q
payload_hash: empty_sha256
})!
resp := c.do_http(signed, '')!
if resp.status_code !in [204, 200, 404] {
return new_http_error(resp.status_code, key, resp.body)
}
}
+40
View File
@@ -0,0 +1,40 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
// Module s3 is an S3-compatible client for V.
//
// Quick start:
//
// import s3
//
// client := s3.new_client(s3.Credentials{
// endpoint: 'https://s3.example.com'
// access_key_id: '...'
// secret_access_key: '...'
// bucket: 'my-bucket'
// })
// client.put('hello.txt', 'Hi from V!'.bytes())!
// text := client.get_string('hello.txt')!
// url := client.presign('hello.txt', expires_in: 3600)!
//
// // s3:// fetch helper:
// resp := s3.fetch('s3://my-bucket/hello.txt')!
//
// Construct credentials from the environment (S3_*, AWS_*, CELLAR_ADDON_*,
// SCW_*, B2_*, R2_*, SPACES_* — first non-empty wins per field):
//
// client := s3.new_client(s3.Credentials.from_env())
//
// Multipart for large files:
//
// client.upload_file('big.bin', '/path/to/big.bin', s3.PutOptions{ content_type: 'application/octet-stream' })!
module s3
// version is the module version, kept in sync with v.mod.
pub const version = '0.1.0'
// info_string returns a one-line build-time identification, useful in
// User-Agent strings or `--version` output.
pub fn info_string() string {
return 's3/${version}'
}
+294
View File
@@ -0,0 +1,294 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
import crypto.hmac
import crypto.sha256
import time
// service_name is the SigV4 service identifier baked into the signing key.
pub const service_name = 's3'
// algo is the SigV4 algorithm marker S3 expects.
pub const algo = 'AWS4-HMAC-SHA256'
// unsigned_payload is the magic string used in `x-amz-content-sha256` when
// the payload is not pre-hashed. Used by the single-shot put path to avoid
// buffering / re-scanning the entire body just to sign it.
pub const unsigned_payload = 'UNSIGNED-PAYLOAD'
// empty_sha256 is `sha256("")` precomputed. Used for HEAD/GET/DELETE where
// there is no body and we want a real hash for stricter S3 endpoints.
pub const empty_sha256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
// SignRequest describes the request to sign. The signer is intentionally
// agnostic of HTTP option semantics (ACL, storage class, …) — the caller
// pre-fills `extra_headers` with whatever it intends to send. That keeps the
// signer pure and easy to test against published SigV4 reference vectors.
pub struct SignRequest {
pub:
method string @[required] // GET, PUT, POST, DELETE, HEAD
path string @[required] // canonical URI (already URI-encoded, kept as-is)
query string // canonical query string (sorted, encoded; no leading '?')
payload_hash string // hex SHA-256 of body or `unsigned_payload`
extra_headers map[string]string // lowercase keys, raw values — will be added to canonical/signed set
sign_time time.Time // when omitted (Time{}) we use time.utc()
}
// SignedRequest is the output of header signing.
pub struct SignedRequest {
pub:
method string // canonical HTTP method that was signed (GET, PUT, …)
url string // scheme://host<extra_path>/<bucket>/<key>?<query>
host string // host[:port], suitable for the Host header
amz_date string // YYYYMMDDTHHMMSSZ
authorization string // ready-to-send Authorization header value
headers map[string]string // ALL headers that MUST be sent for the signature to verify
}
// sign_request builds the SigV4 Authorization header for an HTTP request.
//
// The returned `SignedRequest.headers` is the full set the caller must send
// (excluding `Content-Length`, which the HTTP client adds itself). Adding,
// removing, or mutating a header after signing will break the signature.
pub fn sign_request(creds Credentials, req SignRequest) !SignedRequest {
creds.validate()!
method := normalize_method(req.method) or {
return new_error('InvalidMethod',
'Method must be GET, PUT, POST, DELETE or HEAD; got: ${req.method}')
}
host := canonical_host(creds, '')
if host == '' {
return new_error('InvalidEndpoint',
'No endpoint and no bucket provided — cannot determine host')
}
region := creds.resolved_region()
now := if req.sign_time.year == 0 { time.utc() } else { req.sign_time }
amz_date := format_amz_date(now)
amz_day := amz_date[..8]
payload := if req.payload_hash == '' { unsigned_payload } else { req.payload_hash }
// Build the headers set. We always sign host + amz date + content sha
// because S3 requires them. Reject CRLF before they reach the wire.
mut headers := map[string]string{}
for k, v in req.extra_headers {
if contains_crlf(v) || contains_crlf(k) {
return new_error('InvalidHeader',
'Header "${k}" contains CR/LF — refused (header injection guard)')
}
headers[k.to_lower()] = v
}
headers['host'] = host
headers['x-amz-content-sha256'] = payload
headers['x-amz-date'] = amz_date
if creds.session_token != '' {
headers['x-amz-security-token'] = creds.session_token
}
signed_header_names := sorted_keys(headers).join(';')
canonical := build_canonical_request(method, req.path, req.query, headers, signed_header_names,
payload)
credential_scope := '${amz_day}/${region}/${service_name}/aws4_request'
string_to_sign := '${algo}\n${amz_date}\n${credential_scope}\n${sha256_hex(canonical.bytes())}'
signature := to_hex_lower(hmac_sha256(derive_signing_key(creds.secret_access_key, amz_day,
region, service_name), string_to_sign.bytes()))
authorization := '${algo} Credential=${creds.access_key_id}/${credential_scope}, SignedHeaders=${signed_header_names}, Signature=${signature}'
headers['authorization'] = authorization
scheme := creds.scheme()
mut url := '${scheme}://${host}${req.path}'
if req.query != '' {
url += '?' + req.query
}
return SignedRequest{
method: method
url: url
host: host
amz_date: amz_date
authorization: authorization
headers: headers
}
}
// PresignRequest describes a presigned URL to generate. The output is a
// self-contained URL — no extra headers are required at request time.
pub struct PresignRequest {
pub:
method string @[required]
path string @[required] // canonical URI (already URI-encoded)
expires_in int = 86400 // 1..604800 seconds (SigV4 hard limit is 7 days)
extra_query map[string]string // additional signed query params (e.g. response-content-type, x-amz-acl)
sign_time time.Time
}
// presign_url returns a fully-formed `https://...` URL signed via SigV4
// query-string parameters. The URL is valid until `now + expires_in`.
pub fn presign_url(creds Credentials, req PresignRequest) !string {
creds.validate()!
method := normalize_method(req.method) or {
return new_error('InvalidMethod',
'Method must be GET, PUT, POST, DELETE or HEAD; got: ${req.method}')
}
if req.expires_in < 1 || req.expires_in > 604800 {
return new_error('InvalidExpiry', 'expires_in must be between 1 and 604800 seconds')
}
host := canonical_host(creds, '')
if host == '' {
return new_error('InvalidEndpoint',
'No endpoint and no bucket provided — cannot determine host')
}
region := creds.resolved_region()
now := if req.sign_time.year == 0 { time.utc() } else { req.sign_time }
amz_date := format_amz_date(now)
amz_day := amz_date[..8]
credential := '${creds.access_key_id}/${amz_day}/${region}/${service_name}/aws4_request'
// Required signed query parameters; we then merge in extras from the caller.
mut params := map[string]string{}
params['X-Amz-Algorithm'] = algo
params['X-Amz-Credential'] = credential
params['X-Amz-Date'] = amz_date
params['X-Amz-Expires'] = req.expires_in.str()
params['X-Amz-SignedHeaders'] = 'host'
if creds.session_token != '' {
params['X-Amz-Security-Token'] = creds.session_token
}
for k, v in req.extra_query {
if contains_crlf(k) || contains_crlf(v) {
return new_error('InvalidQueryParam', 'Query param "${k}" contains CR/LF — refused')
}
params[k] = v
}
canonical_query := canonical_query_string(params)
headers := {
'host': host
}
canonical := build_canonical_request(method, req.path, canonical_query, headers, 'host',
unsigned_payload)
credential_scope := '${amz_day}/${region}/${service_name}/aws4_request'
string_to_sign := '${algo}\n${amz_date}\n${credential_scope}\n${sha256_hex(canonical.bytes())}'
signature := to_hex_lower(hmac_sha256(derive_signing_key(creds.secret_access_key, amz_day,
region, service_name), string_to_sign.bytes()))
scheme := creds.scheme()
return '${scheme}://${host}${req.path}?${canonical_query}&X-Amz-Signature=${signature}'
}
// build_canonical_request assembles the canonical request string per
// SigV4 §3.2. `path` and `query` MUST already be URI-encoded.
pub fn build_canonical_request(method string, path string, query string, headers map[string]string,
signed_headers string, payload_hash string) string {
sorted := sorted_keys(headers)
mut lines := []string{cap: sorted.len}
for k in sorted {
lines << '${k}:${normalize_header_value(headers[k])}'
}
return '${method}\n${path}\n${query}\n${lines.join('\n')}\n\n${signed_headers}\n${payload_hash}'
}
// canonical_query_string sorts query params by key (then by value if the same
// key appears twice — we don't here) and joins them as `k=v&k=v` with values
// already URI-encoded.
pub fn canonical_query_string(params map[string]string) string {
keys := sorted_keys(params)
mut parts := []string{cap: keys.len}
for k in keys {
parts << '${uri_encode_query(k)}=${uri_encode_query(params[k])}'
}
return parts.join('&')
}
// derive_signing_key implements the four-step HMAC chain that produces the
// SigV4 signing key. The result is cacheable per (secret, date, region,
// service) tuple — left to the caller to memoize if needed.
pub fn derive_signing_key(secret string, date string, region string, service string) []u8 {
k_date := hmac_sha256(('AWS4' + secret).bytes(), date.bytes())
k_region := hmac_sha256(k_date, region.bytes())
k_service := hmac_sha256(k_region, service.bytes())
return hmac_sha256(k_service, 'aws4_request'.bytes())
}
// format_amz_date returns the basic ISO-8601 timestamp used by SigV4
// (`YYYYMMDDTHHMMSSZ`). `t` is assumed to be in UTC.
pub fn format_amz_date(t time.Time) string {
return '${t.year:04d}${t.month:02d}${t.day:02d}T${t.hour:02d}${t.minute:02d}${t.second:02d}Z'
}
// canonical_host resolves the on-the-wire host. Path-style addressing is
// the default; virtual-hosted style uses `<bucket>.<endpoint-host>`. If no
// endpoint is configured we fall back to `s3.<region>.amazonaws.com` (the
// canonical default for clients pointed at AWS S3 itself).
pub fn canonical_host(creds Credentials, bucket_override string) string {
bucket := if bucket_override != '' { bucket_override } else { creds.bucket }
host := creds.host_only()
if host == '' {
region := creds.resolved_region()
if creds.virtual_hosted_style {
if bucket == '' {
return ''
}
return '${bucket}.s3.${region}.amazonaws.com'
}
return 's3.${region}.amazonaws.com'
}
if creds.virtual_hosted_style && bucket != '' {
return '${bucket}.${host}'
}
return host
}
// normalize_method uppercases and validates the HTTP method.
pub fn normalize_method(m string) ?string {
up := m.to_upper()
return match up {
'GET', 'PUT', 'POST', 'DELETE', 'HEAD' { up }
else { none }
}
}
// normalize_header_value collapses runs of internal whitespace to a single
// space and trims leading/trailing whitespace, per SigV4 §3.2.2 step 3.
fn normalize_header_value(v string) string {
if v == '' {
return v
}
mut out := []u8{cap: v.len}
mut prev_space := false
mut started := false
for b in v.bytes() {
if b == ` ` || b == `\t` {
if started {
prev_space = true
}
continue
}
if prev_space {
out << ` `
prev_space = false
}
out << b
started = true
}
return out.bytestr()
}
// sha256_hex returns the lowercase hex digest of `data`.
@[inline]
pub fn sha256_hex(data []u8) string {
return to_hex_lower(sha256.sum256(data))
}
// hmac_sha256 wraps `crypto.hmac.new` for HMAC-SHA-256 with the V stdlib's
// type signature (a hash function that returns `[]u8`).
@[inline]
pub fn hmac_sha256(key []u8, data []u8) []u8 {
return hmac.new(key, data, sha256.sum256, sha256.block_size)
}
// sorted_keys returns the keys of `m` in lexical order.
fn sorted_keys(m map[string]string) []string {
mut keys := m.keys()
keys.sort()
return keys
}
+224
View File
@@ -0,0 +1,224 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
import time
// Test vectors come from the AWS-published Signature V4 reference test
// suite ("Examples of the complete version 4 signing process").
// These secret/key pairs are the documented example values, not real
// credentials.
const aws_secret = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
const aws_key_id = 'AKIAIOSFODNN7EXAMPLE'
// SigV4 vector for `s3-get-object` (examplebucket / test.txt with a Range
// header). Source: AWS docs (Signature V4 examples).
//
// GET /test.txt HTTP/1.1
// Host: examplebucket.s3.amazonaws.com
// Range: bytes=0-9
// X-Amz-Date: 20130524T000000Z
// X-Amz-Content-SHA256: <empty-sha>
fn test_aws_sigv4_get_object_vector() {
t := time.parse_iso8601('2013-05-24T00:00:00.000Z') or { panic(err) }
creds := Credentials{
access_key_id: aws_key_id
secret_access_key: aws_secret
region: 'us-east-1'
endpoint: 'examplebucket.s3.amazonaws.com'
virtual_hosted_style: false // host already carries the bucket
}
req := SignRequest{
method: 'GET'
path: '/test.txt'
query: ''
payload_hash: empty_sha256
extra_headers: {
'range': 'bytes=0-9'
}
sign_time: t
}
signed := sign_request(creds, req) or { panic(err) }
expected := 'AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41'
assert signed.authorization == expected, 'Got:\n${signed.authorization}\nExpected:\n${expected}'
}
// SigV4 vector for `s3-put-object` (examplebucket / test$file.text with body
// "Welcome to Amazon S3.").
//
// PUT /test%24file.text HTTP/1.1
// Host: examplebucket.s3.amazonaws.com
// Date: Fri, 24 May 2013 00:00:00 GMT
// x-amz-date: 20130524T000000Z
// x-amz-storage-class: REDUCED_REDUNDANCY
fn test_aws_sigv4_put_object_vector() {
t := time.parse_iso8601('2013-05-24T00:00:00.000Z') or { panic(err) }
creds := Credentials{
access_key_id: aws_key_id
secret_access_key: aws_secret
region: 'us-east-1'
endpoint: 'examplebucket.s3.amazonaws.com'
virtual_hosted_style: false
}
body := 'Welcome to Amazon S3.'
body_hash := sha256_hex(body.bytes())
req := SignRequest{
method: 'PUT'
path: '/test%24file.text'
query: ''
payload_hash: body_hash
extra_headers: {
'date': 'Fri, 24 May 2013 00:00:00 GMT'
'x-amz-storage-class': 'REDUCED_REDUNDANCY'
}
sign_time: t
}
signed := sign_request(creds, req) or { panic(err) }
expected := 'AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd'
assert signed.authorization == expected, 'Got:\n${signed.authorization}\nExpected:\n${expected}'
}
// Asserts the canonical request string for a ListObjectsV2 GET. The
// canonical request is the most stable interop point — Authorization line
// formatting has wobbled across AWS doc revisions, but the canonical hash
// is always the same.
fn test_aws_sigv4_list_objects_canonical_request() {
headers := {
'host': 'examplebucket.s3.amazonaws.com'
'x-amz-content-sha256': empty_sha256
'x-amz-date': '20130524T000000Z'
}
canonical := build_canonical_request('GET', '/', 'list-type=2&prefix=foo', headers,
'host;x-amz-content-sha256;x-amz-date', empty_sha256)
expected := 'GET\n/\nlist-type=2&prefix=foo\nhost:examplebucket.s3.amazonaws.com\nx-amz-content-sha256:${empty_sha256}\nx-amz-date:20130524T000000Z\n\nhost;x-amz-content-sha256;x-amz-date\n${empty_sha256}'
assert canonical == expected, 'canonical mismatch:\n${canonical}'
}
// Cross-checked manually with openssl on the AWS docs example
// (date 20120215, region us-east-1, service iam):
//
// echo -n 'aws4_request' | openssl dgst -sha256 -mac HMAC -macopt hexkey:<kService>
fn test_signing_key_chain() {
key := derive_signing_key(aws_secret, '20120215', 'us-east-1', 'iam')
assert to_hex_lower(key) == '004aa806e13dae88b9032d9261bcb04c67d023afadd221e6b0d206e1760e0b5e', 'Got: ${to_hex_lower(key)}'
}
fn test_normalize_header_value_collapses_whitespace() {
assert normalize_header_value(' hello world ') == 'hello world'
assert normalize_header_value('a\tb\tc') == 'a b c'
assert normalize_header_value('') == ''
}
fn test_canonical_query_string_sorted_and_encoded() {
q := canonical_query_string({
'b': '2'
'a': '1'
'c': 'hello world'
})
assert q == 'a=1&b=2&c=hello%20world'
}
fn test_format_amz_date() {
t := time.parse_iso8601('2024-01-15T12:34:56.000Z') or { panic(err) }
assert format_amz_date(t) == '20240115T123456Z'
}
fn test_signer_rejects_unsupported_method() {
creds := Credentials{
access_key_id: 'KEY'
secret_access_key: 'secret'
region: 'us-east-1'
endpoint: 'examplebucket.s3.amazonaws.com'
}
if _ := sign_request(creds, SignRequest{ method: 'PATCH', path: '/x', payload_hash: empty_sha256 }) {
assert false
}
}
fn test_signer_rejects_crlf_in_extra_headers() {
creds := Credentials{
access_key_id: 'KEY'
secret_access_key: 'secret'
region: 'us-east-1'
endpoint: 'examplebucket.s3.amazonaws.com'
}
bad := SignRequest{
method: 'GET'
path: '/foo'
payload_hash: empty_sha256
extra_headers: {
'x-evil': 'value\r\nInjected: yes'
}
}
if _ := sign_request(creds, bad) {
assert false, 'expected CRLF rejection'
} else {
if err is S3Error {
assert err.code == 'InvalidHeader' || err.code == 'InvalidCredentials', 'unexpected code: ${err.code}'
} else {
assert false, 'wrong error type'
}
}
}
fn test_presign_rejects_crlf_in_extra_query() {
creds := Credentials{
access_key_id: 'KEY'
secret_access_key: 'secret'
region: 'us-east-1'
endpoint: 'examplebucket.s3.amazonaws.com'
}
bad := PresignRequest{
method: 'GET'
path: '/foo'
extra_query: {
'X-Evil': 'value\r\nInjected: yes'
}
}
if _ := presign_url(creds, bad) {
assert false, 'expected CRLF rejection'
} else {
assert err is S3Error
}
}
fn test_presign_rejects_invalid_expiry() {
creds := Credentials{
access_key_id: 'KEY'
secret_access_key: 'secret'
region: 'us-east-1'
endpoint: 'examplebucket.s3.amazonaws.com'
}
if _ := presign_url(creds, PresignRequest{ method: 'GET', path: '/x', expires_in: 0 }) {
assert false, 'expires_in=0 must be rejected'
}
if _ := presign_url(creds, PresignRequest{
method: 'GET'
path: '/x'
expires_in: 700_000
})
{
assert false, 'expires_in > 7 days must be rejected'
}
}
fn test_presign_url_contains_required_query_params() {
creds := Credentials{
access_key_id: 'KEY'
secret_access_key: 'secret'
region: 'us-east-1'
endpoint: 'examplebucket.s3.amazonaws.com'
}
url := presign_url(creds, PresignRequest{
method: 'GET'
path: '/foo.txt'
expires_in: 300
}) or { panic(err) }
assert url.contains('X-Amz-Algorithm=AWS4-HMAC-SHA256')
assert url.contains('X-Amz-Credential=')
assert url.contains('X-Amz-Date=')
assert url.contains('X-Amz-Expires=300')
assert url.contains('X-Amz-SignedHeaders=host')
assert url.contains('X-Amz-Signature=')
}
+169
View File
@@ -0,0 +1,169 @@
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module s3
import net.http
// ACL is the canned Access Control List applied to a stored object or bucket.
// Values match the S3 wire protocol.
pub enum Acl {
unset
private
public_read
public_read_write
aws_exec_read
authenticated_read
bucket_owner_read
bucket_owner_full_control
log_delivery_write
}
// to_header_value renders the ACL as the canonical S3 string used in headers
// and presigned query parameters. `.unset` returns an empty string so callers
// can skip the header.
pub fn (a Acl) to_header_value() string {
return match a {
.unset { '' }
.private { 'private' }
.public_read { 'public-read' }
.public_read_write { 'public-read-write' }
.aws_exec_read { 'aws-exec-read' }
.authenticated_read { 'authenticated-read' }
.bucket_owner_read { 'bucket-owner-read' }
.bucket_owner_full_control { 'bucket-owner-full-control' }
.log_delivery_write { 'log-delivery-write' }
}
}
// parse_acl returns the matching Acl from a wire string.
// Returns `.unset` when the input is empty or unknown so the caller can ignore it.
pub fn parse_acl(s string) Acl {
return match s {
'private' { Acl.private }
'public-read' { Acl.public_read }
'public-read-write' { Acl.public_read_write }
'aws-exec-read' { Acl.aws_exec_read }
'authenticated-read' { Acl.authenticated_read }
'bucket-owner-read' { Acl.bucket_owner_read }
'bucket-owner-full-control' { Acl.bucket_owner_full_control }
'log-delivery-write' { Acl.log_delivery_write }
else { Acl.unset }
}
}
// StorageClass enumerates the standard S3 storage tiers. `.unset` means the
// server default (typically STANDARD) and produces no `x-amz-storage-class`
// header. Providers vary on which tiers they actually honour.
pub enum StorageClass {
unset
standard
deep_archive
express_onezone
glacier
glacier_ir
intelligent_tiering
onezone_ia
outposts
reduced_redundancy
snow
standard_ia
}
// to_header_value renders the storage class as the on-wire string.
pub fn (sc StorageClass) to_header_value() string {
return match sc {
.unset { '' }
.standard { 'STANDARD' }
.deep_archive { 'DEEP_ARCHIVE' }
.express_onezone { 'EXPRESS_ONEZONE' }
.glacier { 'GLACIER' }
.glacier_ir { 'GLACIER_IR' }
.intelligent_tiering { 'INTELLIGENT_TIERING' }
.onezone_ia { 'ONEZONE_IA' }
.outposts { 'OUTPOSTS' }
.reduced_redundancy { 'REDUCED_REDUNDANCY' }
.snow { 'SNOW' }
.standard_ia { 'STANDARD_IA' }
}
}
// Stat is the result of HEAD-ing an object.
pub struct Stat {
pub:
size i64 // Content-Length in bytes
last_modified string // RFC 1123 date as returned by S3
etag string // unquoted ETag (server returns it wrapped in quotes; we strip them)
content_type string // MIME type
}
// Owner identifies an S3 object owner — only populated when `fetch_owner` is true.
pub struct Owner {
pub:
id string
display_name string
}
// ObjectInfo is one entry from a ListObjectsV2 response.
pub struct ObjectInfo {
pub:
key string
last_modified string
etag string
size i64
storage_class string
owner ?Owner
}
// CommonPrefix represents a "directory-like" prefix in a list result when a
// delimiter is provided.
pub struct CommonPrefix {
pub:
prefix string
}
// ListResult aggregates a ListObjectsV2 response.
// NextContinuationToken should be passed back as `continuation_token` to fetch the next page.
pub struct ListResult {
pub:
name string
prefix string
delimiter string
start_after string
max_keys int
key_count int
is_truncated bool
continuation_token string
next_continuation_token string
objects []ObjectInfo
common_prefixes []CommonPrefix
}
// ListOptions configures a ListObjectsV2 call. `bucket` overrides the client's
// default; leave empty to use the client's bound bucket.
@[params]
pub struct ListOptions {
pub:
bucket string
prefix string
continuation_token string
delimiter string
max_keys int // 0 means default (server picks 1000)
start_after string
encoding_type string // 'url' or empty
fetch_owner bool
}
// PresignOptions controls presigned URL generation.
@[params]
pub struct PresignOptions {
pub:
bucket string // overrides the client's default bucket for this call
method http.Method = .get
expires_in int = 86400 // seconds, 1..604800
acl Acl
storage_class StorageClass
content_type string
content_disposition string
request_payer bool
}