Initial project import
This commit is contained in:
commit
23f2d29e32
11957 files changed
+1754577
No files matched your search
@@ -0,0 +1,109 @@
|
||||
`net.s3` is an S3-compatible client written in pure V.
|
||||
|
||||
It speaks the AWS Signature Version 4 protocol on top of `crypto.hmac` and
|
||||
`crypto.sha256`, with no third-party dependencies. The same client targets any
|
||||
S3-compatible endpoint (hosted or self-hosted) by configuring the right
|
||||
`endpoint`, `region` and credentials.
|
||||
|
||||
## Quick start
|
||||
|
||||
```v ignore
|
||||
import net.s3
|
||||
|
||||
c := s3.new_client(s3.Credentials{
|
||||
endpoint: 'https://s3.example.com'
|
||||
access_key_id: '...'
|
||||
secret_access_key: '...'
|
||||
bucket: 'my-bucket'
|
||||
})
|
||||
|
||||
c.put('hello.txt', 'Hi from V!'.bytes())!
|
||||
text := c.get_string('hello.txt')!
|
||||
url := c.presign('hello.txt', expires_in: 3600)!
|
||||
```
|
||||
|
||||
## Credentials from the environment
|
||||
|
||||
`Credentials.from_env()` resolves each field from the first non-empty
|
||||
provider-specific environment variable, so the same code works against AWS,
|
||||
self-hosted and managed S3 services without reconfiguration:
|
||||
|
||||
```v ignore
|
||||
import net.s3
|
||||
|
||||
c := s3.new_client(s3.Credentials.from_env())
|
||||
```
|
||||
|
||||
Supported variables (first non-empty wins per field):
|
||||
|
||||
* key id: `S3_ACCESS_KEY_ID`, `AWS_ACCESS_KEY_ID`, `CELLAR_ADDON_KEY_ID`,
|
||||
`SCW_ACCESS_KEY`, `B2_APPLICATION_KEY_ID`, `R2_ACCESS_KEY_ID`, `SPACES_KEY`
|
||||
* secret: `S3_SECRET_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`,
|
||||
`CELLAR_ADDON_KEY_SECRET`, `SCW_SECRET_KEY`, `B2_APPLICATION_KEY`,
|
||||
`R2_SECRET_ACCESS_KEY`, `SPACES_SECRET`
|
||||
* session token: `S3_SESSION_TOKEN`, `AWS_SESSION_TOKEN`
|
||||
* region: `S3_REGION`, `AWS_REGION`, `AWS_DEFAULT_REGION`,
|
||||
`SCW_DEFAULT_REGION`
|
||||
* bucket: `S3_BUCKET`
|
||||
* endpoint: `S3_ENDPOINT`, `AWS_ENDPOINT`, `AWS_ENDPOINT_URL`,
|
||||
`CELLAR_ADDON_HOST`, `B2_ENDPOINT`, `R2_ENDPOINT`, `SPACES_ENDPOINT`
|
||||
|
||||
## Multipart upload
|
||||
|
||||
`upload_file` automatically picks single-shot or multipart based on file size.
|
||||
For finer control, `start_multipart` returns a stateful `MultipartUploader`
|
||||
that streams chunks generated on the fly:
|
||||
|
||||
```v ignore
|
||||
import net.s3
|
||||
|
||||
c := s3.new_client(s3.Credentials.from_env())
|
||||
c.upload_file('big.bin', '/path/to/big.bin', s3.PutOptions{
|
||||
content_type: 'application/octet-stream'
|
||||
})!
|
||||
```
|
||||
|
||||
## `s3://` URLs
|
||||
|
||||
Importing `net.s3` registers an `s3://` scheme handler with `net.http`, so the
|
||||
generic `http.fetch(url: 's3://...')` route works out of the box. There is
|
||||
also a direct `s3.fetch` helper:
|
||||
|
||||
```v ignore
|
||||
import net.s3
|
||||
|
||||
resp := s3.fetch('s3://my-bucket/hello.txt')!
|
||||
println(resp.body.bytestr())
|
||||
```
|
||||
|
||||
## File handle
|
||||
|
||||
`Client.file(key)` returns a small `File` reference for ergonomic call sites:
|
||||
|
||||
```v ignore
|
||||
import net.s3
|
||||
|
||||
c := s3.new_client(s3.Credentials.from_env())
|
||||
f := c.file('hello.txt')
|
||||
text := f.text()!
|
||||
url := f.presign(expires_in: 3600)!
|
||||
```
|
||||
|
||||
## Tests
|
||||
|
||||
The unit suite is offline and runs by default:
|
||||
|
||||
```
|
||||
v test vlib/net/s3/
|
||||
```
|
||||
|
||||
The integration suite is gated on `S3_INTEGRATION=1` and exercises the
|
||||
client against a live endpoint:
|
||||
|
||||
```
|
||||
S3_INTEGRATION=1 \
|
||||
S3_HOST=https://s3.example.com \
|
||||
S3_KEY_ID=... S3_KEY_SECRET=... \
|
||||
S3_BUCKET=v-s3-tests \
|
||||
v test vlib/net/s3/integration_test.v
|
||||
```
|
||||
@@ -0,0 +1,192 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
// BucketOptions configures bucket-level operations. `region_constraint`
|
||||
// lets `create_bucket` pin the bucket to a specific region (S3 sends a
|
||||
// `<CreateBucketConfiguration>` body when this is non-empty).
|
||||
@[params]
|
||||
pub struct BucketOptions {
|
||||
pub:
|
||||
bucket string
|
||||
acl Acl
|
||||
region_constraint string
|
||||
}
|
||||
|
||||
// create_bucket creates a new bucket. Returns:
|
||||
// - nil on success (HTTP 200)
|
||||
// - S3Error("BucketAlreadyOwnedByYou") if you already own this bucket
|
||||
// - S3Error("BucketAlreadyExists") if someone else owns it
|
||||
// - S3Error("InvalidBucketName") for non-conformant names
|
||||
//
|
||||
// The S3 wire response for these states is HTTP 409, parsed from the
|
||||
// returned XML body.
|
||||
pub fn (c &Client) create_bucket(opts BucketOptions) ! {
|
||||
bucket := pick_bucket(c.credentials, opts.bucket)!
|
||||
validate_bucket_name(bucket)!
|
||||
creds := c.creds_for(bucket)
|
||||
path := bucket_path(creds, bucket)
|
||||
body := if opts.region_constraint != '' {
|
||||
'<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><LocationConstraint>${escape_xml(opts.region_constraint)}</LocationConstraint></CreateBucketConfiguration>'
|
||||
} else {
|
||||
''
|
||||
}
|
||||
mut headers := map[string]string{}
|
||||
if opts.acl != .unset {
|
||||
headers['x-amz-acl'] = opts.acl.to_header_value()
|
||||
}
|
||||
if body != '' {
|
||||
headers['content-type'] = 'application/xml'
|
||||
}
|
||||
signed := sign_request(creds, SignRequest{
|
||||
method: 'PUT'
|
||||
path: path
|
||||
payload_hash: if body == '' { empty_sha256 } else { sha256_hex(body.bytes()) }
|
||||
extra_headers: headers
|
||||
})!
|
||||
resp := c.do_http(signed, body)!
|
||||
if resp.status_code !in [200, 204] {
|
||||
return new_http_error(resp.status_code, bucket, resp.body)
|
||||
}
|
||||
}
|
||||
|
||||
// delete_bucket removes an empty bucket. S3 returns 409 BucketNotEmpty if it
|
||||
// still has keys — caller is expected to clean up first or handle the error.
|
||||
pub fn (c &Client) delete_bucket(opts BucketOptions) ! {
|
||||
bucket := pick_bucket(c.credentials, opts.bucket)!
|
||||
creds := c.creds_for(bucket)
|
||||
path := bucket_path(creds, bucket)
|
||||
signed := sign_request(creds, SignRequest{
|
||||
method: 'DELETE'
|
||||
path: path
|
||||
payload_hash: empty_sha256
|
||||
})!
|
||||
resp := c.do_http(signed, '')!
|
||||
if resp.status_code !in [200, 204] {
|
||||
return new_http_error(resp.status_code, bucket, resp.body)
|
||||
}
|
||||
}
|
||||
|
||||
// bucket_exists checks bucket existence/access. Returns true if accessible (200),
|
||||
// false on 404 / 403 (no such bucket or no read permission), error on other
|
||||
// statuses. Uses HEAD under the hood — no body is fetched.
|
||||
pub fn (c &Client) bucket_exists(opts BucketOptions) !bool {
|
||||
bucket := pick_bucket(c.credentials, opts.bucket)!
|
||||
creds := c.creds_for(bucket)
|
||||
path := bucket_path(creds, bucket)
|
||||
signed := sign_request(creds, SignRequest{
|
||||
method: 'HEAD'
|
||||
path: path
|
||||
payload_hash: empty_sha256
|
||||
})!
|
||||
resp := c.do_http(signed, '')!
|
||||
if resp.status_code == 404 || resp.status_code == 403 {
|
||||
return false
|
||||
}
|
||||
if resp.status_code != 200 {
|
||||
return new_http_error(resp.status_code, bucket, resp.body)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// validate_bucket_name applies the S3 bucket naming rules honoured by most
|
||||
// providers:
|
||||
// - 3..63 chars
|
||||
// - lowercase letters, digits, dots, hyphens only
|
||||
// - must start/end with letter or digit
|
||||
// - no consecutive dots, no `.-` / `-.`
|
||||
// - cannot look like an IPv4 address
|
||||
//
|
||||
// Provider-specific reservations (e.g. `xn--`, `sthree-`) are intentionally
|
||||
// not checked here — they vary, so the server-side error is authoritative.
|
||||
pub fn validate_bucket_name(name string) ! {
|
||||
if name.len < 3 || name.len > 63 {
|
||||
return new_error('InvalidBucketName', 'Bucket name must be 3..63 characters: ${name}')
|
||||
}
|
||||
first := name[0]
|
||||
last := name[name.len - 1]
|
||||
if !is_lower_alnum(first) || !is_lower_alnum(last) {
|
||||
return new_error('InvalidBucketName',
|
||||
'Bucket name must start and end with a lowercase letter or digit: ${name}')
|
||||
}
|
||||
mut prev := u8(0)
|
||||
for b in name.bytes() {
|
||||
if !is_bucket_char(b) {
|
||||
return new_error('InvalidBucketName', 'Bucket name "${name}" contains invalid character: ${[
|
||||
b,
|
||||
].bytestr()}')
|
||||
}
|
||||
if prev == `.` && b == `.` {
|
||||
return new_error('InvalidBucketName', 'Bucket name "${name}" contains consecutive dots')
|
||||
}
|
||||
if (prev == `.` && b == `-`) || (prev == `-` && b == `.`) {
|
||||
return new_error('InvalidBucketName',
|
||||
'Bucket name "${name}" contains adjacent dot/hyphen')
|
||||
}
|
||||
prev = b
|
||||
}
|
||||
if looks_like_ipv4(name) {
|
||||
return new_error('InvalidBucketName', 'Bucket name "${name}" looks like an IP address')
|
||||
}
|
||||
}
|
||||
|
||||
fn is_lower_alnum(b u8) bool {
|
||||
return (b >= `a` && b <= `z`) || (b >= `0` && b <= `9`)
|
||||
}
|
||||
|
||||
fn is_bucket_char(b u8) bool {
|
||||
return is_lower_alnum(b) || b == `-` || b == `.`
|
||||
}
|
||||
|
||||
fn looks_like_ipv4(s string) bool {
|
||||
parts := s.split('.')
|
||||
if parts.len != 4 {
|
||||
return false
|
||||
}
|
||||
for p in parts {
|
||||
if p.len == 0 || p.len > 3 {
|
||||
return false
|
||||
}
|
||||
for b in p.bytes() {
|
||||
if b < `0` || b > `9` {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
fn pick_bucket(c Credentials, override string) !string {
|
||||
b := if override != '' { override } else { c.bucket }
|
||||
if b == '' {
|
||||
return new_error('InvalidPath',
|
||||
'No bucket given (set Credentials.bucket or pass bucket via options)')
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
fn bucket_path(creds Credentials, bucket string) string {
|
||||
extra := creds.extra_path()
|
||||
if creds.virtual_hosted_style {
|
||||
return if extra == '' { '/' } else { extra }
|
||||
}
|
||||
encoded := uri_encode_path(strip_slashes(bucket))
|
||||
return '${extra}/${encoded}'
|
||||
}
|
||||
|
||||
// escape_xml does the minimal XML special-char escape we need for body content.
|
||||
fn escape_xml(s string) string {
|
||||
mut out := []u8{cap: s.len}
|
||||
for b in s.bytes() {
|
||||
match b {
|
||||
`<` { out << '<'.bytes() }
|
||||
`>` { out << '>'.bytes() }
|
||||
`&` { out << '&'.bytes() }
|
||||
`"` { out << '"'.bytes() }
|
||||
`'` { out << '''.bytes() }
|
||||
else { out << b }
|
||||
}
|
||||
}
|
||||
return out.bytestr()
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
fn test_validate_bucket_name_accepts_valid() {
|
||||
validate_bucket_name('my-bucket') or { assert false, 'should pass' }
|
||||
validate_bucket_name('a.b.c') or { assert false, 'should pass' }
|
||||
validate_bucket_name('abc') or { assert false, 'should pass' }
|
||||
validate_bucket_name('a' + 'b'.repeat(61) + 'c') or { assert false, 'should pass' }
|
||||
}
|
||||
|
||||
fn test_validate_bucket_name_rejects_invalid() {
|
||||
if _ := validate_bucket_name('Bad-UPPER') {
|
||||
assert false, 'uppercase must be rejected'
|
||||
}
|
||||
if _ := validate_bucket_name('ab') {
|
||||
assert false, 'shorter than 3 chars must be rejected'
|
||||
}
|
||||
if _ := validate_bucket_name('-leading-hyphen') {
|
||||
assert false, 'leading hyphen must be rejected'
|
||||
}
|
||||
if _ := validate_bucket_name('trailing-hyphen-') {
|
||||
assert false, 'trailing hyphen must be rejected'
|
||||
}
|
||||
if _ := validate_bucket_name('192.168.0.1') {
|
||||
assert false, 'IPv4-shaped name must be rejected'
|
||||
}
|
||||
if _ := validate_bucket_name('a..b') {
|
||||
assert false, 'consecutive dots must be rejected'
|
||||
}
|
||||
if _ := validate_bucket_name('a.-b') {
|
||||
assert false, 'adjacent dot/hyphen must be rejected'
|
||||
}
|
||||
if _ := validate_bucket_name('a-.b') {
|
||||
assert false, 'adjacent hyphen/dot must be rejected'
|
||||
}
|
||||
if _ := validate_bucket_name('') {
|
||||
assert false, 'empty name must be rejected'
|
||||
}
|
||||
long := 'a'.repeat(64)
|
||||
if _ := validate_bucket_name(long) {
|
||||
assert false, 'longer than 63 chars must be rejected'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,405 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
import net.http
|
||||
import time
|
||||
|
||||
// Client is the entry point for S3 operations. It carries default credentials
|
||||
// and tuneable HTTP behaviour; per-call overrides go through the option params
|
||||
// of each method. Instantiate once, reuse for many objects.
|
||||
@[heap]
|
||||
pub struct Client {
|
||||
pub:
|
||||
credentials Credentials
|
||||
// part_size is the multipart-upload chunk size in bytes (default 5 MiB,
|
||||
// the S3 minimum).
|
||||
part_size i64 = 5 * 1024 * 1024
|
||||
// queue_size is the intended parallel-upload concurrency for multipart
|
||||
// (currently sequential; reserved).
|
||||
queue_size int = 5
|
||||
// retry is the number of retry attempts for failed uploads.
|
||||
retry int = 3
|
||||
// read_timeout / write_timeout map onto V's net.http settings.
|
||||
// Defaults are generous because parts can be 5 MiB+ on slow links.
|
||||
read_timeout i64 = 5 * 60 * time.second
|
||||
write_timeout i64 = 5 * 60 * time.second
|
||||
}
|
||||
|
||||
// new_client builds a Client. Pass an empty Credentials to fall back to env vars.
|
||||
pub fn new_client(creds Credentials) Client {
|
||||
resolved := if creds.access_key_id == '' && creds.secret_access_key == '' {
|
||||
Credentials.from_env().merge(creds)
|
||||
} else {
|
||||
creds
|
||||
}
|
||||
return Client{
|
||||
credentials: resolved
|
||||
}
|
||||
}
|
||||
|
||||
// PutOptions configures a put / write call.
|
||||
@[params]
|
||||
pub struct PutOptions {
|
||||
pub:
|
||||
bucket string
|
||||
content_type string
|
||||
content_disposition string
|
||||
content_encoding string
|
||||
cache_control string
|
||||
acl Acl
|
||||
storage_class StorageClass
|
||||
request_payer bool
|
||||
// hash_payload, when true, computes SHA-256 of the body before signing
|
||||
// instead of using `UNSIGNED-PAYLOAD`. Slightly stronger integrity guarantee
|
||||
// at the cost of one full body scan.
|
||||
hash_payload bool
|
||||
}
|
||||
|
||||
// GetOptions configures a get / read call.
|
||||
@[params]
|
||||
pub struct GetOptions {
|
||||
pub:
|
||||
bucket string
|
||||
range string // e.g. 'bytes=0-1023' for partial downloads
|
||||
version_id string
|
||||
request_payer bool
|
||||
}
|
||||
|
||||
// StatOptions configures stat / size / exists checks.
|
||||
@[params]
|
||||
pub struct StatOptions {
|
||||
pub:
|
||||
bucket string
|
||||
request_payer bool
|
||||
}
|
||||
|
||||
// put uploads `data` to `key`. Use `upload_file` / `upload_bytes_multipart`
|
||||
// for streaming or for files larger than ~100 MiB; `put` keeps everything in
|
||||
// memory.
|
||||
pub fn (c &Client) put(key string, data []u8, opts PutOptions) ! {
|
||||
creds := c.creds_for(opts.bucket)
|
||||
host := canonical_host(creds, opts.bucket)
|
||||
if host == '' {
|
||||
return new_error('InvalidEndpoint', 'cannot determine S3 host (set endpoint or bucket)')
|
||||
}
|
||||
path := build_object_path(creds, opts.bucket, key)!
|
||||
payload_hash := if opts.hash_payload { sha256_hex(data) } else { unsigned_payload }
|
||||
signed := sign_request(creds, SignRequest{
|
||||
method: 'PUT'
|
||||
path: path
|
||||
payload_hash: payload_hash
|
||||
extra_headers: put_object_headers(opts)
|
||||
})!
|
||||
resp := c.do_http(signed, data.bytestr())!
|
||||
if resp.status_code !in [200, 204] {
|
||||
return new_http_error(resp.status_code, key, resp.body)
|
||||
}
|
||||
}
|
||||
|
||||
// put_object_headers maps `PutOptions` to the wire headers used by both the
|
||||
// single-shot `put` and the multipart `initiate_multipart` paths. Keeping the
|
||||
// mapping in one place ensures the two upload modes accept the exact same set
|
||||
// of options without drift.
|
||||
fn put_object_headers(opts PutOptions) map[string]string {
|
||||
mut headers := map[string]string{}
|
||||
if opts.content_type != '' {
|
||||
headers['content-type'] = opts.content_type
|
||||
}
|
||||
if opts.content_disposition != '' {
|
||||
headers['content-disposition'] = opts.content_disposition
|
||||
}
|
||||
if opts.content_encoding != '' {
|
||||
headers['content-encoding'] = opts.content_encoding
|
||||
}
|
||||
if opts.cache_control != '' {
|
||||
headers['cache-control'] = opts.cache_control
|
||||
}
|
||||
if opts.acl != .unset {
|
||||
headers['x-amz-acl'] = opts.acl.to_header_value()
|
||||
}
|
||||
if opts.storage_class != .unset {
|
||||
headers['x-amz-storage-class'] = opts.storage_class.to_header_value()
|
||||
}
|
||||
if opts.request_payer {
|
||||
headers['x-amz-request-payer'] = 'requester'
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
// get downloads the entire object body into memory. For large files set a
|
||||
// `range` and assemble the result yourself, or use a presigned URL with an
|
||||
// HTTP streaming client.
|
||||
pub fn (c &Client) get(key string, opts GetOptions) ![]u8 {
|
||||
creds := c.creds_for(opts.bucket)
|
||||
path := build_object_path(creds, opts.bucket, key)!
|
||||
mut headers := map[string]string{}
|
||||
if opts.range != '' {
|
||||
headers['range'] = opts.range
|
||||
}
|
||||
if opts.request_payer {
|
||||
headers['x-amz-request-payer'] = 'requester'
|
||||
}
|
||||
mut query := ''
|
||||
if opts.version_id != '' {
|
||||
query = 'versionId=' + uri_encode_query(opts.version_id)
|
||||
}
|
||||
signed := sign_request(creds, SignRequest{
|
||||
method: 'GET'
|
||||
path: path
|
||||
query: query
|
||||
payload_hash: empty_sha256
|
||||
extra_headers: headers
|
||||
})!
|
||||
resp := c.do_http(signed, '')!
|
||||
if resp.status_code !in [200, 206] {
|
||||
return new_http_error(resp.status_code, key, resp.body)
|
||||
}
|
||||
return resp.body.bytes()
|
||||
}
|
||||
|
||||
// get_string is a convenience wrapper around `get` that returns the body as a
|
||||
// V `string`.
|
||||
pub fn (c &Client) get_string(key string, opts GetOptions) !string {
|
||||
bytes := c.get(key, opts)!
|
||||
return bytes.bytestr()
|
||||
}
|
||||
|
||||
// stat returns the object metadata (size / last_modified / etag / content_type).
|
||||
// Returns an `S3Error` with code `NoSuchKey` when the object doesn't exist.
|
||||
pub fn (c &Client) stat(key string, opts StatOptions) !Stat {
|
||||
creds := c.creds_for(opts.bucket)
|
||||
path := build_object_path(creds, opts.bucket, key)!
|
||||
mut headers := map[string]string{}
|
||||
if opts.request_payer {
|
||||
headers['x-amz-request-payer'] = 'requester'
|
||||
}
|
||||
signed := sign_request(creds, SignRequest{
|
||||
method: 'HEAD'
|
||||
path: path
|
||||
payload_hash: empty_sha256
|
||||
extra_headers: headers
|
||||
})!
|
||||
resp := c.do_http(signed, '')!
|
||||
if resp.status_code == 404 {
|
||||
return new_error('NoSuchKey', 'No such key: ${key}')
|
||||
}
|
||||
if resp.status_code !in [200, 204] {
|
||||
return new_http_error(resp.status_code, key, resp.body)
|
||||
}
|
||||
return Stat{
|
||||
size: resp.header.get(.content_length) or { '0' }.i64()
|
||||
last_modified: resp.header.get(.last_modified) or { '' }
|
||||
etag: (resp.header.get(.etag) or { '' }).trim('"')
|
||||
content_type: resp.header.get(.content_type) or { '' }
|
||||
}
|
||||
}
|
||||
|
||||
// exists is `stat` + boolean: true on 200, false on 404, error otherwise.
|
||||
pub fn (c &Client) exists(key string, opts StatOptions) !bool {
|
||||
c.stat(key, opts) or {
|
||||
if err is S3Error && err.code == 'NoSuchKey' {
|
||||
return false
|
||||
}
|
||||
return err
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// size returns just the Content-Length of the object.
|
||||
pub fn (c &Client) size(key string, opts StatOptions) !i64 {
|
||||
st := c.stat(key, opts)!
|
||||
return st.size
|
||||
}
|
||||
|
||||
// delete removes a single object. S3 returns 204 on success, 204 on
|
||||
// already-absent (idempotent) — we surface success in both cases.
|
||||
pub fn (c &Client) delete(key string, opts StatOptions) ! {
|
||||
creds := c.creds_for(opts.bucket)
|
||||
path := build_object_path(creds, opts.bucket, key)!
|
||||
mut headers := map[string]string{}
|
||||
if opts.request_payer {
|
||||
headers['x-amz-request-payer'] = 'requester'
|
||||
}
|
||||
signed := sign_request(creds, SignRequest{
|
||||
method: 'DELETE'
|
||||
path: path
|
||||
payload_hash: empty_sha256
|
||||
extra_headers: headers
|
||||
})!
|
||||
resp := c.do_http(signed, '')!
|
||||
if resp.status_code !in [200, 204] {
|
||||
return new_http_error(resp.status_code, key, resp.body)
|
||||
}
|
||||
}
|
||||
|
||||
// presign generates a presigned URL — see PresignOptions for tunables.
|
||||
pub fn (c &Client) presign(key string, opts PresignOptions) !string {
|
||||
creds := c.creds_for(opts.bucket)
|
||||
path := build_object_path(creds, opts.bucket, key)!
|
||||
mut extra := map[string]string{}
|
||||
if opts.acl != .unset {
|
||||
extra['X-Amz-Acl'] = opts.acl.to_header_value()
|
||||
}
|
||||
if opts.storage_class != .unset {
|
||||
extra['x-amz-storage-class'] = opts.storage_class.to_header_value()
|
||||
}
|
||||
if opts.content_type != '' {
|
||||
extra['response-content-type'] = opts.content_type
|
||||
}
|
||||
if opts.content_disposition != '' {
|
||||
extra['response-content-disposition'] = opts.content_disposition
|
||||
}
|
||||
if opts.request_payer {
|
||||
extra['x-amz-request-payer'] = 'requester'
|
||||
}
|
||||
return presign_url(creds, PresignRequest{
|
||||
method: http_method_to_string(opts.method)
|
||||
path: path
|
||||
expires_in: opts.expires_in
|
||||
extra_query: extra
|
||||
})
|
||||
}
|
||||
|
||||
// file returns a File reference for the given key, bound to this client.
|
||||
// `path` may be `<bucket>/<key>` if the client has no default bucket and
|
||||
// `bucket` here is empty.
|
||||
pub fn (c &Client) file(key string, opts FileOptions) File {
|
||||
return File{
|
||||
client: c
|
||||
key: key
|
||||
bucket: opts.bucket
|
||||
}
|
||||
}
|
||||
|
||||
// creds_for returns credentials with `bucket` overridden if the caller passed one.
|
||||
fn (c &Client) creds_for(bucket string) Credentials {
|
||||
if bucket == '' {
|
||||
return c.credentials
|
||||
}
|
||||
mut dup := c.credentials
|
||||
dup = Credentials{
|
||||
...dup
|
||||
bucket: bucket
|
||||
}
|
||||
return dup
|
||||
}
|
||||
|
||||
// build_object_path produces the canonical URI path for a key.
|
||||
// Path style: `/<bucket>/<encoded-key>[+endpoint extra path]`
|
||||
// Virtual hosted: `/<encoded-key>`
|
||||
// Returns an error if neither bucket nor key is provided.
|
||||
// The key is forwarded byte-exact (only percent-encoded): S3 treats keys as
|
||||
// opaque identifiers, so `folder/`, `/x` and `a//b` all designate distinct
|
||||
// objects and must reach the wire as written.
|
||||
pub fn build_object_path(creds Credentials, bucket_override string, key string) !string {
|
||||
bucket := if bucket_override != '' { bucket_override } else { creds.bucket }
|
||||
if key == '' {
|
||||
return new_error('InvalidPath', 'Empty object key')
|
||||
}
|
||||
encoded_key := uri_encode_path(key)
|
||||
extra := creds.extra_path()
|
||||
if creds.virtual_hosted_style {
|
||||
return '${extra}/${encoded_key}'
|
||||
}
|
||||
if bucket == '' {
|
||||
return new_error('InvalidPath',
|
||||
'No bucket given (set Credentials.bucket or pass bucket via options)')
|
||||
}
|
||||
encoded_bucket := uri_encode_path(strip_slashes(bucket))
|
||||
return '${extra}/${encoded_bucket}/${encoded_key}'
|
||||
}
|
||||
|
||||
// HttpResponse is the small subset of an HTTP response we surface to callers
|
||||
// of the lower-level helpers. `body` is the raw response body for non-stream
|
||||
// requests; `header` keeps V's typed Header for parsing.
|
||||
pub struct HttpResponse {
|
||||
pub:
|
||||
status_code int
|
||||
body string
|
||||
header http.Header
|
||||
}
|
||||
|
||||
// do_http fires off the signed request via V's `net.http`. The HTTP method is
|
||||
// taken from `signed.method` (set by `sign_request`) so callers cannot drift
|
||||
// between what was signed and what is sent on the wire.
|
||||
fn (c &Client) do_http(signed SignedRequest, body string) !HttpResponse {
|
||||
mut header := http.new_header()
|
||||
for k, v in signed.headers {
|
||||
// `Host` is set automatically by V's http client; skip to avoid duplicates.
|
||||
if k.to_lower() == 'host' {
|
||||
continue
|
||||
}
|
||||
header.add_custom(canonical_header_name(k), v)!
|
||||
}
|
||||
method_enum := parse_http_method(signed.method)!
|
||||
req := http.new_request(method_enum, signed.url, body)
|
||||
mut req_mut := http.Request{
|
||||
...req
|
||||
header: header
|
||||
read_timeout: c.read_timeout
|
||||
write_timeout: c.write_timeout
|
||||
}
|
||||
resp := req_mut.do() or {
|
||||
return new_error('NetworkError', 'HTTP request failed: ${err.msg()}')
|
||||
}
|
||||
$if s3_debug ? {
|
||||
eprintln('[s3] ${signed.method} ${signed.url}')
|
||||
for k, v in signed.headers {
|
||||
eprintln('[s3] > ${k}: ${redacted_value(k, v)}')
|
||||
}
|
||||
eprintln('[s3] -> ${resp.status_code}')
|
||||
eprintln('[s3] body: ${resp.body}')
|
||||
}
|
||||
return HttpResponse{
|
||||
status_code: resp.status_code
|
||||
body: resp.body
|
||||
header: resp.header
|
||||
}
|
||||
}
|
||||
|
||||
// redacted_value masks credentials so they cannot leak via s3_debug logs.
|
||||
fn redacted_value(name string, value string) string {
|
||||
low := name.to_lower()
|
||||
if low == 'authorization' || low.starts_with('x-amz-security-token') {
|
||||
return '<redacted>'
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// canonical_header_name turns 'x-amz-content-sha256' into 'X-Amz-Content-Sha256'
|
||||
// for the wire. HTTP/1.1 says headers are case-insensitive, but some
|
||||
// intermediaries / mocks aren't, so we emit the canonical Title-Case form.
|
||||
fn canonical_header_name(s string) string {
|
||||
mut out := []u8{cap: s.len}
|
||||
mut upper_next := true
|
||||
for b in s.bytes() {
|
||||
if b == `-` {
|
||||
out << b
|
||||
upper_next = true
|
||||
continue
|
||||
}
|
||||
if upper_next && b >= `a` && b <= `z` {
|
||||
out << b - 32
|
||||
} else {
|
||||
out << b
|
||||
}
|
||||
upper_next = false
|
||||
}
|
||||
return out.bytestr()
|
||||
}
|
||||
|
||||
// parse_http_method maps a SigV4 canonical method string to V's `http.Method`.
|
||||
// The signer enforces the supported set up-front, so anything unexpected here
|
||||
// indicates a programming bug rather than user input.
|
||||
fn parse_http_method(s string) !http.Method {
|
||||
return match s.to_upper() {
|
||||
'GET' { http.Method.get }
|
||||
'PUT' { http.Method.put }
|
||||
'POST' { http.Method.post }
|
||||
'DELETE' { http.Method.delete }
|
||||
'HEAD' { http.Method.head }
|
||||
else { new_error('InvalidMethod', 'Unsupported HTTP method: ${s}') }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
import os
|
||||
|
||||
// Credentials carries the authentication material plus endpoint and addressing
|
||||
// preferences. It is intentionally kept small; defaults (region, endpoint, etc.)
|
||||
// are derived only when the request is signed, never stored implicitly, so the
|
||||
// same Credentials value can be reused across regions/endpoints.
|
||||
//
|
||||
// Field naming matches V conventions (snake_case). The `from_env` helper
|
||||
// recognises several provider conventions — see `from_env`.
|
||||
pub struct Credentials {
|
||||
pub:
|
||||
access_key_id string
|
||||
secret_access_key string
|
||||
session_token string
|
||||
region string
|
||||
bucket string
|
||||
endpoint string // 'https://s3.fr-par.scw.cloud' or 'host:port' — host part is what gets signed
|
||||
virtual_hosted_style bool // when true, '<bucket>.<endpoint-host>' addressing
|
||||
insecure_http bool // permit `http://` endpoints (false by default — never silently downgrades)
|
||||
}
|
||||
|
||||
// Credentials.from_env reads credentials from environment variables, trying several
|
||||
// provider conventions in order so the same code works against many hosts
|
||||
// without reconfiguration. Each field is resolved independently; the first
|
||||
// non-empty value wins.
|
||||
//
|
||||
// Lookup order per field:
|
||||
// key id : S3_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID,
|
||||
// CELLAR_ADDON_KEY_ID, SCW_ACCESS_KEY,
|
||||
// B2_APPLICATION_KEY_ID, R2_ACCESS_KEY_ID, SPACES_KEY
|
||||
// secret : S3_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY,
|
||||
// CELLAR_ADDON_KEY_SECRET, SCW_SECRET_KEY,
|
||||
// B2_APPLICATION_KEY, R2_SECRET_ACCESS_KEY, SPACES_SECRET
|
||||
// session : S3_SESSION_TOKEN, AWS_SESSION_TOKEN
|
||||
// region : S3_REGION, AWS_REGION, AWS_DEFAULT_REGION, SCW_DEFAULT_REGION
|
||||
// bucket : S3_BUCKET
|
||||
// endpoint : S3_ENDPOINT, AWS_ENDPOINT, AWS_ENDPOINT_URL,
|
||||
// CELLAR_ADDON_HOST, B2_ENDPOINT, R2_ENDPOINT, SPACES_ENDPOINT
|
||||
pub fn Credentials.from_env() Credentials {
|
||||
endpoint := env_first('S3_ENDPOINT', 'AWS_ENDPOINT', 'AWS_ENDPOINT_URL', 'CELLAR_ADDON_HOST',
|
||||
'B2_ENDPOINT', 'R2_ENDPOINT', 'SPACES_ENDPOINT')
|
||||
insecure := endpoint.starts_with('http://')
|
||||
return Credentials{
|
||||
access_key_id: env_first('S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID',
|
||||
'CELLAR_ADDON_KEY_ID', 'SCW_ACCESS_KEY', 'B2_APPLICATION_KEY_ID', 'R2_ACCESS_KEY_ID',
|
||||
'SPACES_KEY')
|
||||
secret_access_key: env_first('S3_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY',
|
||||
'CELLAR_ADDON_KEY_SECRET', 'SCW_SECRET_KEY', 'B2_APPLICATION_KEY',
|
||||
'R2_SECRET_ACCESS_KEY', 'SPACES_SECRET')
|
||||
session_token: env_first('S3_SESSION_TOKEN', 'AWS_SESSION_TOKEN')
|
||||
region: env_first('S3_REGION', 'AWS_REGION', 'AWS_DEFAULT_REGION',
|
||||
'SCW_DEFAULT_REGION')
|
||||
bucket: env_first('S3_BUCKET')
|
||||
endpoint: endpoint
|
||||
virtual_hosted_style: false
|
||||
insecure_http: insecure
|
||||
}
|
||||
}
|
||||
|
||||
// merge produces a copy of `c` with non-empty fields from `other` overriding.
|
||||
// Useful when callers pass per-call overrides while keeping a default Client.
|
||||
pub fn (c Credentials) merge(other Credentials) Credentials {
|
||||
return Credentials{
|
||||
access_key_id: if other.access_key_id != '' {
|
||||
other.access_key_id
|
||||
} else {
|
||||
c.access_key_id
|
||||
}
|
||||
secret_access_key: if other.secret_access_key != '' {
|
||||
other.secret_access_key
|
||||
} else {
|
||||
c.secret_access_key
|
||||
}
|
||||
session_token: if other.session_token != '' {
|
||||
other.session_token
|
||||
} else {
|
||||
c.session_token
|
||||
}
|
||||
region: if other.region != '' { other.region } else { c.region }
|
||||
bucket: if other.bucket != '' { other.bucket } else { c.bucket }
|
||||
endpoint: if other.endpoint != '' { other.endpoint } else { c.endpoint }
|
||||
virtual_hosted_style: c.virtual_hosted_style || other.virtual_hosted_style
|
||||
insecure_http: c.insecure_http || other.insecure_http
|
||||
}
|
||||
}
|
||||
|
||||
// resolved_region returns the region to use for signing. Order:
|
||||
// 1. explicit `c.region`
|
||||
// 2. parsed from `c.endpoint` when it follows the `s3.<region>.amazonaws.com` pattern
|
||||
// 3. `'auto'` for Cloudflare R2
|
||||
// 4. `'us-east-1'` (S3 historical default) when no endpoint is set
|
||||
pub fn (c Credentials) resolved_region() string {
|
||||
if c.region != '' {
|
||||
return c.region
|
||||
}
|
||||
return guess_region(c.endpoint)
|
||||
}
|
||||
|
||||
// validate ensures the credentials carry the minimum needed to sign a
|
||||
// request. Also rejects credentials / region / bucket / endpoint values that
|
||||
// contain CR or LF — those would let an attacker who controls *any* config
|
||||
// field smuggle headers into the Authorization line.
|
||||
pub fn (c Credentials) validate() ! {
|
||||
if c.access_key_id == '' || c.secret_access_key == '' {
|
||||
return new_error('MissingCredentials',
|
||||
"Missing S3 credentials. 'access_key_id' and 'secret_access_key' are required.")
|
||||
}
|
||||
for name, value in {
|
||||
'access_key_id': c.access_key_id
|
||||
'secret_access_key': c.secret_access_key
|
||||
'session_token': c.session_token
|
||||
'region': c.region
|
||||
'bucket': c.bucket
|
||||
'endpoint': c.endpoint
|
||||
} {
|
||||
if contains_crlf(value) {
|
||||
return new_error('InvalidCredentials',
|
||||
'Credential field "${name}" contains CR or LF — refused (header-injection guard)')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// host_only returns the bare host[:port] from `c.endpoint`, stripping any
|
||||
// scheme and trailing path. Returned value is what gets signed in the
|
||||
// `host` header for SigV4.
|
||||
pub fn (c Credentials) host_only() string {
|
||||
mut ep := c.endpoint
|
||||
if i := ep.index('://') {
|
||||
ep = ep[i + 3..]
|
||||
}
|
||||
if j := ep.index('/') {
|
||||
ep = ep[..j]
|
||||
}
|
||||
return ep
|
||||
}
|
||||
|
||||
// extra_path returns the path component of `c.endpoint`, including any
|
||||
// leading '/'. Useful for proxies that mount S3 under a sub-path.
|
||||
pub fn (c Credentials) extra_path() string {
|
||||
mut ep := c.endpoint
|
||||
if i := ep.index('://') {
|
||||
ep = ep[i + 3..]
|
||||
}
|
||||
if j := ep.index('/') {
|
||||
return ep[j..]
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
// scheme returns 'http' or 'https' based on the endpoint's explicit scheme
|
||||
// when present, falling back to `insecure_http`. So all three of these work:
|
||||
// endpoint: 'https://s3.example.com' → https
|
||||
// endpoint: 's3.example.com' → https (default)
|
||||
// endpoint: 'http://localhost:9000' → http (auto-detected)
|
||||
// endpoint: 'localhost:9000', insecure_http: true → http
|
||||
pub fn (c Credentials) scheme() string {
|
||||
if c.endpoint.starts_with('http://') {
|
||||
return 'http'
|
||||
}
|
||||
if c.endpoint.starts_with('https://') {
|
||||
return 'https'
|
||||
}
|
||||
return if c.insecure_http { 'http' } else { 'https' }
|
||||
}
|
||||
|
||||
// guess_region derives the SigV4 region from an endpoint URL. Public so it
|
||||
// can be reused by the higher-level Client for log / inspect output.
|
||||
pub fn guess_region(endpoint string) string {
|
||||
if endpoint == '' {
|
||||
return 'us-east-1'
|
||||
}
|
||||
if endpoint.ends_with('.r2.cloudflarestorage.com') {
|
||||
return 'auto'
|
||||
}
|
||||
if amz_end := endpoint.index('.amazonaws.com') {
|
||||
if s3_pos := endpoint.index('s3.') {
|
||||
start := s3_pos + 3
|
||||
if start < amz_end {
|
||||
return endpoint[start..amz_end]
|
||||
}
|
||||
}
|
||||
// AWS global endpoint (`s3.amazonaws.com`) and legacy forms
|
||||
// (`s3-external-1.amazonaws.com`) sign with `us-east-1`. SigV4
|
||||
// rejects `auto` against AWS, so fall back here instead.
|
||||
return 'us-east-1'
|
||||
}
|
||||
return 'auto'
|
||||
}
|
||||
|
||||
// env_first returns the first non-empty env var among the names provided.
|
||||
fn env_first(names ...string) string {
|
||||
for n in names {
|
||||
v := os.getenv(n)
|
||||
if v != '' {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ''
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
fn test_credentials_validate_rejects_crlf_injection() {
|
||||
c := Credentials{
|
||||
access_key_id: 'KEY'
|
||||
secret_access_key: 'secret\r\nInjected: header'
|
||||
}
|
||||
if _ := c.validate() {
|
||||
assert false, 'should have rejected CRLF'
|
||||
}
|
||||
}
|
||||
|
||||
fn test_credentials_from_env_returns_empty_when_unset() {
|
||||
// Cannot safely set env vars from a test without polluting the parent
|
||||
// shell, so just verify the call returns a value of the right shape.
|
||||
c := Credentials.from_env()
|
||||
_ = c
|
||||
}
|
||||
|
||||
fn test_endpoint_scheme_is_derived_when_explicit() {
|
||||
c1 := Credentials{
|
||||
access_key_id: 'K'
|
||||
secret_access_key: 'S'
|
||||
endpoint: 'https://s3.example.com'
|
||||
}
|
||||
assert c1.scheme() == 'https'
|
||||
c2 := Credentials{
|
||||
access_key_id: 'K'
|
||||
secret_access_key: 'S'
|
||||
endpoint: 'http://localhost:9000'
|
||||
}
|
||||
assert c2.scheme() == 'http', 'http:// endpoint must yield http even when insecure_http is unset'
|
||||
c3 := Credentials{
|
||||
access_key_id: 'K'
|
||||
secret_access_key: 'S'
|
||||
endpoint: 'http://localhost:9000'
|
||||
insecure_http: true
|
||||
}
|
||||
assert c3.scheme() == 'http'
|
||||
// Bare host: defaults to https unless insecure_http is set.
|
||||
c4 := Credentials{
|
||||
access_key_id: 'K'
|
||||
secret_access_key: 'S'
|
||||
endpoint: 'localhost:9000'
|
||||
}
|
||||
assert c4.scheme() == 'https'
|
||||
c5 := Credentials{
|
||||
access_key_id: 'K'
|
||||
secret_access_key: 'S'
|
||||
endpoint: 'localhost:9000'
|
||||
insecure_http: true
|
||||
}
|
||||
assert c5.scheme() == 'http'
|
||||
}
|
||||
|
||||
fn test_endpoint_host_only_strips_scheme_and_path() {
|
||||
c := Credentials{
|
||||
access_key_id: 'K'
|
||||
secret_access_key: 'S'
|
||||
endpoint: 'https://proxy.example.com:8443/s3'
|
||||
}
|
||||
assert c.host_only() == 'proxy.example.com:8443'
|
||||
assert c.extra_path() == '/s3'
|
||||
}
|
||||
|
||||
fn test_guess_region_paths() {
|
||||
assert guess_region('') == 'us-east-1'
|
||||
assert guess_region('https://s3.eu-west-3.amazonaws.com') == 'eu-west-3'
|
||||
assert guess_region('https://my-acct.r2.cloudflarestorage.com') == 'auto'
|
||||
assert guess_region('https://s3.example.com') == 'auto'
|
||||
// AWS global endpoints sign with us-east-1, not 'auto'.
|
||||
assert guess_region('https://s3.amazonaws.com') == 'us-east-1'
|
||||
assert guess_region('s3.amazonaws.com') == 'us-east-1'
|
||||
assert guess_region('https://s3-external-1.amazonaws.com') == 'us-east-1'
|
||||
}
|
||||
|
||||
fn test_credentials_merge_overrides_only_non_empty() {
|
||||
base := Credentials{
|
||||
access_key_id: 'BASE'
|
||||
secret_access_key: 'basesecret'
|
||||
region: 'eu-west-1'
|
||||
bucket: 'b1'
|
||||
}
|
||||
other := Credentials{
|
||||
bucket: 'b2'
|
||||
}
|
||||
got := base.merge(other)
|
||||
assert got.access_key_id == 'BASE'
|
||||
assert got.region == 'eu-west-1'
|
||||
assert got.bucket == 'b2'
|
||||
}
|
||||
|
||||
fn test_resolved_region_prefers_explicit() {
|
||||
c := Credentials{
|
||||
access_key_id: 'K'
|
||||
secret_access_key: 'S'
|
||||
region: 'eu-central-1'
|
||||
endpoint: 'https://s3.us-west-2.amazonaws.com'
|
||||
}
|
||||
assert c.resolved_region() == 'eu-central-1'
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
// uri_encode performs RFC 3986 percent-encoding as required by Signature V4.
|
||||
// Only the unreserved set A–Z / a–z / 0–9 / '-' / '_' / '.' / '~' is preserved.
|
||||
// When `encode_slash` is false (used for object keys), '/' is left intact and
|
||||
// backslashes are normalized to '/' so Windows-style paths produce the same
|
||||
// canonical key. All other bytes are emitted as %XX with uppercase hex digits.
|
||||
pub fn uri_encode(input string, encode_slash bool) string {
|
||||
mut out := []u8{cap: input.len + (input.len >> 2)} // 25% headroom
|
||||
for b in input.bytes() {
|
||||
match b {
|
||||
`A`...`Z`, `a`...`z`, `0`...`9`, `-`, `_`, `.`, `~` {
|
||||
out << b
|
||||
}
|
||||
`/`, `\\` {
|
||||
if encode_slash {
|
||||
append_percent(mut out, b)
|
||||
} else {
|
||||
out << if b == `\\` { u8(`/`) } else { b }
|
||||
}
|
||||
}
|
||||
else {
|
||||
append_percent(mut out, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out.bytestr()
|
||||
}
|
||||
|
||||
// uri_encode_path encodes an S3 object key segment-aware: '/' is preserved
|
||||
// because S3 paths use it as the segment separator.
|
||||
@[inline]
|
||||
pub fn uri_encode_path(path string) string {
|
||||
return uri_encode(path, false)
|
||||
}
|
||||
|
||||
// uri_encode_query encodes a value that will appear inside a query string.
|
||||
// Slashes must be percent-encoded.
|
||||
@[inline]
|
||||
pub fn uri_encode_query(value string) string {
|
||||
return uri_encode(value, true)
|
||||
}
|
||||
|
||||
// strip_slashes removes leading and trailing '/' or '\\' separators. S3
|
||||
// canonical paths must contain a single leading slash, no trailing one.
|
||||
pub fn strip_slashes(s string) string {
|
||||
if s == '' {
|
||||
return s
|
||||
}
|
||||
mut start := 0
|
||||
mut end := s.len
|
||||
for start < end && (s[start] == `/` || s[start] == `\\`) {
|
||||
start++
|
||||
}
|
||||
for end > start && (s[end - 1] == `/` || s[end - 1] == `\\`) {
|
||||
end--
|
||||
}
|
||||
return s[start..end]
|
||||
}
|
||||
|
||||
// contains_crlf returns true if `value` contains a CR or LF byte. Header
|
||||
// values that pass user-provided strings (ACL, content-type, …) MUST be
|
||||
// checked to prevent HTTP header injection (CRLF smuggling).
|
||||
@[inline]
|
||||
pub fn contains_crlf(value string) bool {
|
||||
for b in value.bytes() {
|
||||
if b == `\r` || b == `\n` {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// to_hex_lower formats raw bytes as their lowercase hex string. Used for
|
||||
// SHA-256 digests inside SigV4 (the spec requires lowercase).
|
||||
pub fn to_hex_lower(data []u8) string {
|
||||
mut out := []u8{len: data.len * 2}
|
||||
for i, b in data {
|
||||
out[i * 2] = hex_lower_nibble(b >> 4)
|
||||
out[i * 2 + 1] = hex_lower_nibble(b & 0x0F)
|
||||
}
|
||||
return out.bytestr()
|
||||
}
|
||||
|
||||
@[inline]
|
||||
fn append_percent(mut out []u8, b u8) {
|
||||
out << `%`
|
||||
out << hex_upper_nibble(b >> 4)
|
||||
out << hex_upper_nibble(b & 0x0F)
|
||||
}
|
||||
|
||||
@[inline]
|
||||
fn hex_upper_nibble(n u8) u8 {
|
||||
return if n < 10 { n + `0` } else { n - 10 + `A` }
|
||||
}
|
||||
|
||||
@[inline]
|
||||
fn hex_lower_nibble(n u8) u8 {
|
||||
return if n < 10 { n + `0` } else { n - 10 + `a` }
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
fn test_uri_encode_path_keeps_slashes() {
|
||||
assert uri_encode_path('/foo/bar baz/test.txt') == '/foo/bar%20baz/test.txt'
|
||||
assert uri_encode_path('/folder/file with %.txt') == '/folder/file%20with%20%25.txt'
|
||||
}
|
||||
|
||||
fn test_uri_encode_path_normalises_backslashes() {
|
||||
// Windows-style separators must collapse to '/' in the canonical key.
|
||||
assert uri_encode_path('foo\\bar\\baz.txt') == 'foo/bar/baz.txt'
|
||||
}
|
||||
|
||||
fn test_uri_encode_query_encodes_slashes() {
|
||||
assert uri_encode_query('a/b') == 'a%2Fb'
|
||||
assert uri_encode_query('hello world') == 'hello%20world'
|
||||
}
|
||||
|
||||
fn test_uri_encode_unreserved_passthrough() {
|
||||
// RFC 3986 unreserved set must stay intact.
|
||||
assert uri_encode('AZaz09-_.~', false) == 'AZaz09-_.~'
|
||||
}
|
||||
|
||||
fn test_uri_encode_percent_uppercase() {
|
||||
// AWS SigV4 mandates uppercase hex digits.
|
||||
assert uri_encode('é', true) == '%C3%A9'
|
||||
}
|
||||
|
||||
fn test_strip_slashes_trims_both_ends() {
|
||||
assert strip_slashes('/foo/bar/') == 'foo/bar'
|
||||
assert strip_slashes('//foo//') == 'foo'
|
||||
assert strip_slashes('foo') == 'foo'
|
||||
assert strip_slashes('') == ''
|
||||
assert strip_slashes('///') == ''
|
||||
}
|
||||
|
||||
fn test_contains_crlf() {
|
||||
assert contains_crlf('hello\r\nworld') == true
|
||||
assert contains_crlf('hello\rworld') == true
|
||||
assert contains_crlf('hello\nworld') == true
|
||||
assert contains_crlf('plain') == false
|
||||
}
|
||||
|
||||
fn test_to_hex_lower() {
|
||||
assert to_hex_lower([u8(0x00), 0x0f, 0xff, 0xa0]) == '000fffa0'
|
||||
assert to_hex_lower([]) == ''
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
// S3Error is the structured error returned for every signing or service
|
||||
// failure. `code` is stable (e.g. `NoSuchKey`, `MissingCredentials`),
|
||||
// `message` is human-readable, `status` is the HTTP status (0 for
|
||||
// client-side errors), and `path` is the offending object key when known.
|
||||
//
|
||||
// V's `IError` interface only requires `msg()` and `code()`, so this type can
|
||||
// be returned via `!T` and inspected with type-assertion / `as`.
|
||||
pub struct S3Error {
|
||||
pub:
|
||||
code string
|
||||
message string
|
||||
status int
|
||||
path string
|
||||
resource string // S3's <Resource> field, when present
|
||||
request_id string // S3 RequestId, useful for support tickets
|
||||
}
|
||||
|
||||
// msg renders the error in a single line. Includes the S3 error code so users
|
||||
// can switch on it without parsing the prose. Path is appended when known.
|
||||
pub fn (e &S3Error) msg() string {
|
||||
mut buf := '[${e.code}] ${e.message}'
|
||||
if e.status != 0 {
|
||||
buf += ' (HTTP ${e.status})'
|
||||
}
|
||||
if e.path != '' {
|
||||
buf += ' — ${e.path}'
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
// code returns a stable numeric code so callers can use `if err.code() == ...`.
|
||||
// We map a few well-known S3 codes; everything else returns 0 so callers fall
|
||||
// back to string comparison on `e.code`.
|
||||
pub fn (e &S3Error) code() int {
|
||||
return match e.code {
|
||||
'NoSuchKey' {
|
||||
404
|
||||
}
|
||||
'NoSuchBucket' {
|
||||
404
|
||||
}
|
||||
'BucketAlreadyExists', 'BucketAlreadyOwnedByYou' {
|
||||
409
|
||||
}
|
||||
'AccessDenied', 'SignatureDoesNotMatch' {
|
||||
403
|
||||
}
|
||||
'MissingCredentials', 'InvalidEndpoint', 'InvalidPath', 'InvalidMethod',
|
||||
'InvalidSessionToken' {
|
||||
400
|
||||
}
|
||||
else {
|
||||
e.status
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// new_error builds an S3Error from a code + message. Use this for client-side
|
||||
// validation failures (missing creds, invalid path, etc.).
|
||||
pub fn new_error(code string, message string) IError {
|
||||
return &S3Error{
|
||||
code: code
|
||||
message: message
|
||||
}
|
||||
}
|
||||
|
||||
// new_http_error wraps an HTTP-level failure. Body is the raw response body
|
||||
// — `parse_xml_error` is responsible for digging out the structured
|
||||
// `<Error>` envelope when the server returns one.
|
||||
pub fn new_http_error(status int, path string, body string) IError {
|
||||
parsed := parse_xml_error(body)
|
||||
if parsed.code != '' {
|
||||
return &S3Error{
|
||||
code: parsed.code
|
||||
message: parsed.message
|
||||
status: status
|
||||
path: path
|
||||
resource: parsed.resource
|
||||
request_id: parsed.request_id
|
||||
}
|
||||
}
|
||||
// Body was empty or unparseable — fall back to a generic code keyed by status.
|
||||
return &S3Error{
|
||||
code: fallback_code_for(status)
|
||||
message: if body.len > 0 { body } else { 'HTTP ${status}' }
|
||||
status: status
|
||||
path: path
|
||||
}
|
||||
}
|
||||
|
||||
// XmlErrorFields is what we pull out of an `<Error>` XML envelope.
|
||||
struct XmlErrorFields {
|
||||
code string
|
||||
message string
|
||||
resource string
|
||||
request_id string
|
||||
}
|
||||
|
||||
// parse_xml_error extracts the standard S3 error XML:
|
||||
//
|
||||
// <Error>
|
||||
// <Code>...</Code>
|
||||
// <Message>...</Message>
|
||||
// <Resource>...</Resource>
|
||||
// <RequestId>...</RequestId>
|
||||
// </Error>
|
||||
//
|
||||
// We use a tiny tag scanner (no DOM) — the format is rigid enough that this
|
||||
// stays correct, faster than spinning up the full XML parser, and there is no
|
||||
// attribute parsing to worry about.
|
||||
fn parse_xml_error(body string) XmlErrorFields {
|
||||
return XmlErrorFields{
|
||||
code: extract_xml_tag(body, 'Code')
|
||||
message: extract_xml_tag(body, 'Message')
|
||||
resource: extract_xml_tag(body, 'Resource')
|
||||
request_id: extract_xml_tag(body, 'RequestId')
|
||||
}
|
||||
}
|
||||
|
||||
// extract_xml_tag returns the inner text of `<tag>...</tag>` (first match,
|
||||
// case-sensitive) with the five predefined XML entities decoded, or '' if
|
||||
// the tag is absent.
|
||||
pub fn extract_xml_tag(body string, tag string) string {
|
||||
open := '<' + tag + '>'
|
||||
close := '</' + tag + '>'
|
||||
start := body.index(open) or { return '' }
|
||||
rest_off := start + open.len
|
||||
end := body.index_after(close, rest_off) or { return '' }
|
||||
return decode_xml_entities(body[rest_off..end])
|
||||
}
|
||||
|
||||
// decode_xml_entities decodes the five predefined XML entities. We don't try
|
||||
// to handle arbitrary `&#nn;` sequences because S3 only ever uses these five.
|
||||
pub fn decode_xml_entities(s string) string {
|
||||
if !s.contains('&') {
|
||||
return s
|
||||
}
|
||||
mut out := []u8{cap: s.len}
|
||||
mut i := 0
|
||||
for i < s.len {
|
||||
if s[i] != `&` {
|
||||
out << s[i]
|
||||
i++
|
||||
continue
|
||||
}
|
||||
matched := match true {
|
||||
s.len - i >= 5 && s[i..i + 5] == '&' {
|
||||
out << `&`
|
||||
i += 5
|
||||
true
|
||||
}
|
||||
s.len - i >= 4 && s[i..i + 4] == '<' {
|
||||
out << `<`
|
||||
i += 4
|
||||
true
|
||||
}
|
||||
s.len - i >= 4 && s[i..i + 4] == '>' {
|
||||
out << `>`
|
||||
i += 4
|
||||
true
|
||||
}
|
||||
s.len - i >= 6 && s[i..i + 6] == '"' {
|
||||
out << `"`
|
||||
i += 6
|
||||
true
|
||||
}
|
||||
s.len - i >= 6 && s[i..i + 6] == ''' {
|
||||
out << `'`
|
||||
i += 6
|
||||
true
|
||||
}
|
||||
else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
if !matched {
|
||||
out << s[i]
|
||||
i++
|
||||
}
|
||||
}
|
||||
return out.bytestr()
|
||||
}
|
||||
|
||||
fn fallback_code_for(status int) string {
|
||||
return match status {
|
||||
301, 307 { 'PermanentRedirect' }
|
||||
400 { 'BadRequest' }
|
||||
401, 403 { 'AccessDenied' }
|
||||
404 { 'NotFound' }
|
||||
405 { 'MethodNotAllowed' }
|
||||
409 { 'Conflict' }
|
||||
411 { 'LengthRequired' }
|
||||
412 { 'PreconditionFailed' }
|
||||
416 { 'InvalidRange' }
|
||||
429 { 'TooManyRequests' }
|
||||
500 { 'InternalError' }
|
||||
503 { 'ServiceUnavailable' }
|
||||
else { 'HTTPError' }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
fn test_extract_xml_tag_basic() {
|
||||
body := '<?xml version="1.0"?><Error><Code>NoSuchKey</Code><Message>Key not found</Message><RequestId>R1</RequestId></Error>'
|
||||
assert extract_xml_tag(body, 'Code') == 'NoSuchKey'
|
||||
assert extract_xml_tag(body, 'Message') == 'Key not found'
|
||||
assert extract_xml_tag(body, 'RequestId') == 'R1'
|
||||
assert extract_xml_tag(body, 'Missing') == ''
|
||||
}
|
||||
|
||||
fn test_extract_xml_tag_decodes_entities() {
|
||||
body := '<Message>a & b <tag></Message>'
|
||||
assert extract_xml_tag(body, 'Message') == 'a & b <tag>'
|
||||
}
|
||||
|
||||
fn test_decode_xml_entities() {
|
||||
assert decode_xml_entities('"hello"') == '"hello"'
|
||||
assert decode_xml_entities('a & b < c > d 'e'') == "a & b < c > d 'e'"
|
||||
assert decode_xml_entities('no entities here') == 'no entities here'
|
||||
}
|
||||
|
||||
fn test_s3error_msg_contains_code_and_path() {
|
||||
e := S3Error{
|
||||
code: 'NoSuchKey'
|
||||
message: 'object missing'
|
||||
status: 404
|
||||
path: 'foo/bar.txt'
|
||||
}
|
||||
rendered := e.msg()
|
||||
assert rendered.contains('NoSuchKey')
|
||||
assert rendered.contains('object missing')
|
||||
assert rendered.contains('404')
|
||||
assert rendered.contains('foo/bar.txt')
|
||||
}
|
||||
|
||||
fn test_s3error_code_maps_known_strings() {
|
||||
assert (&S3Error{
|
||||
code: 'NoSuchKey'
|
||||
}).code() == 404
|
||||
assert (&S3Error{
|
||||
code: 'NoSuchBucket'
|
||||
}).code() == 404
|
||||
assert (&S3Error{
|
||||
code: 'AccessDenied'
|
||||
}).code() == 403
|
||||
assert (&S3Error{
|
||||
code: 'BucketAlreadyExists'
|
||||
}).code() == 409
|
||||
// Unknown code falls back to the wire status.
|
||||
assert (&S3Error{
|
||||
code: 'WeirdCode'
|
||||
status: 418
|
||||
}).code() == 418
|
||||
}
|
||||
|
||||
fn test_new_http_error_parses_xml_body() {
|
||||
body := '<?xml version="1.0"?><Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><Resource>/bucket/missing.txt</Resource><RequestId>ABC123</RequestId></Error>'
|
||||
err := new_http_error(404, 'missing.txt', body)
|
||||
if err is S3Error {
|
||||
assert err.code == 'NoSuchKey'
|
||||
assert err.message == 'The specified key does not exist.'
|
||||
assert err.resource == '/bucket/missing.txt'
|
||||
assert err.request_id == 'ABC123'
|
||||
assert err.status == 404
|
||||
assert err.path == 'missing.txt'
|
||||
} else {
|
||||
assert false, 'expected S3Error'
|
||||
}
|
||||
}
|
||||
|
||||
fn test_new_http_error_falls_back_when_body_empty() {
|
||||
err := new_http_error(503, 'foo', '')
|
||||
if err is S3Error {
|
||||
assert err.code == 'ServiceUnavailable'
|
||||
assert err.status == 503
|
||||
} else {
|
||||
assert false, 'expected S3Error'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
import net.http
|
||||
|
||||
// FetchOptions overlays an S3 endpoint over the `fetch` call. All fields
|
||||
// are optional; bucket and key are taken from the URL.
|
||||
@[params]
|
||||
pub struct FetchOptions {
|
||||
pub:
|
||||
method http.Method = .get
|
||||
body []u8
|
||||
credentials Credentials
|
||||
// content_type, acl, etc. are forwarded as-is when method is PUT/POST.
|
||||
content_type string
|
||||
content_disposition string
|
||||
content_encoding string
|
||||
cache_control string
|
||||
acl Acl
|
||||
storage_class StorageClass
|
||||
request_payer bool
|
||||
range string
|
||||
hash_payload bool
|
||||
}
|
||||
|
||||
// FetchResponse is the simplified return type of `fetch`. It's intentionally
|
||||
// flat (no streaming yet) — easier to consume than V's `http.Response` and
|
||||
// surfaces the most useful fields.
|
||||
pub struct FetchResponse {
|
||||
pub:
|
||||
status_code int
|
||||
body []u8
|
||||
headers map[string]string
|
||||
etag string
|
||||
content_type string
|
||||
content_length i64
|
||||
}
|
||||
|
||||
// fetch is a `fetch('s3://bucket/key', { ... })`-style helper.
|
||||
//
|
||||
// Examples:
|
||||
//
|
||||
// resp := s3.fetch('s3://my-bucket/path/to/file.txt')!
|
||||
// resp := s3.fetch('s3://my-bucket/key', method: .put, body: 'hello'.bytes())!
|
||||
// resp := s3.fetch('s3://key', method: .get, credentials: s3.Credentials{ bucket: 'b', ... })!
|
||||
//
|
||||
// The URL must use the `s3://` scheme. Anything else is rejected outright
|
||||
// (avoids accidentally calling a real HTTP endpoint with S3 credentials).
|
||||
pub fn fetch(url string, opts FetchOptions) !FetchResponse {
|
||||
if !url.starts_with('s3://') {
|
||||
return new_error('InvalidURL', 's3.fetch only accepts s3:// URLs (got: ${redact_url(url)})')
|
||||
}
|
||||
bucket, key := parse_s3_url(url)!
|
||||
mut creds := if opts.credentials.access_key_id == '' {
|
||||
Credentials.from_env()
|
||||
} else {
|
||||
opts.credentials
|
||||
}
|
||||
if bucket != '' {
|
||||
creds = Credentials{
|
||||
...creds
|
||||
bucket: bucket
|
||||
}
|
||||
}
|
||||
creds.validate()!
|
||||
|
||||
c := Client{
|
||||
credentials: creds
|
||||
}
|
||||
match opts.method {
|
||||
.get {
|
||||
data := c.get(key,
|
||||
bucket: bucket
|
||||
range: opts.range
|
||||
request_payer: opts.request_payer
|
||||
)!
|
||||
return FetchResponse{
|
||||
status_code: 200
|
||||
body: data
|
||||
content_length: i64(data.len)
|
||||
}
|
||||
}
|
||||
.head {
|
||||
st := c.stat(key,
|
||||
bucket: bucket
|
||||
request_payer: opts.request_payer
|
||||
)!
|
||||
return FetchResponse{
|
||||
status_code: 200
|
||||
etag: st.etag
|
||||
content_type: st.content_type
|
||||
content_length: st.size
|
||||
}
|
||||
}
|
||||
.put {
|
||||
c.put(key, opts.body,
|
||||
bucket: bucket
|
||||
content_type: opts.content_type
|
||||
content_disposition: opts.content_disposition
|
||||
content_encoding: opts.content_encoding
|
||||
cache_control: opts.cache_control
|
||||
acl: opts.acl
|
||||
storage_class: opts.storage_class
|
||||
request_payer: opts.request_payer
|
||||
hash_payload: opts.hash_payload
|
||||
)!
|
||||
return FetchResponse{
|
||||
status_code: 200
|
||||
}
|
||||
}
|
||||
.delete {
|
||||
c.delete(key,
|
||||
bucket: bucket
|
||||
request_payer: opts.request_payer
|
||||
)!
|
||||
return FetchResponse{
|
||||
status_code: 204
|
||||
}
|
||||
}
|
||||
else {
|
||||
return new_error('InvalidMethod', 's3.fetch: unsupported method ${opts.method}')
|
||||
}
|
||||
}
|
||||
|
||||
return new_error('InvalidMethod', 's3.fetch: unreachable')
|
||||
}
|
||||
|
||||
// parse_s3_url splits `s3://bucket/key/with/slashes` into (bucket, key).
|
||||
//
|
||||
// Special case: `s3://key` (no second path component) returns ('', 'key') —
|
||||
// the caller is then expected to provide the bucket via credentials.
|
||||
pub fn parse_s3_url(url string) !(string, string) {
|
||||
if !url.starts_with('s3://') {
|
||||
return new_error('InvalidURL', 'Not an s3:// URL')
|
||||
}
|
||||
rest := url[5..] // strip 's3://'
|
||||
if rest == '' {
|
||||
return new_error('InvalidURL', 'Empty s3:// URL')
|
||||
}
|
||||
if i := rest.index('/') {
|
||||
bucket := rest[..i]
|
||||
key := rest[i + 1..]
|
||||
if key == '' {
|
||||
return new_error('InvalidURL', 'Empty key in s3:// URL')
|
||||
}
|
||||
return bucket, key
|
||||
}
|
||||
// no '/' separator — treat the whole rest as the key
|
||||
return '', rest
|
||||
}
|
||||
|
||||
// redact_url strips query strings before logging. Used in error messages so
|
||||
// presigned-URL-style query params (which can contain credentials) aren't
|
||||
// leaked into logs.
|
||||
pub fn redact_url(url string) string {
|
||||
if i := url.index('?') {
|
||||
return url[..i] + '?<redacted>'
|
||||
}
|
||||
return url
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
fn test_parse_s3_url_full() {
|
||||
bucket, key := parse_s3_url('s3://my-bucket/path/to/key.txt') or { panic(err) }
|
||||
assert bucket == 'my-bucket'
|
||||
assert key == 'path/to/key.txt'
|
||||
}
|
||||
|
||||
fn test_parse_s3_url_only_key() {
|
||||
bucket, key := parse_s3_url('s3://standalone-key') or { panic(err) }
|
||||
assert bucket == ''
|
||||
assert key == 'standalone-key'
|
||||
}
|
||||
|
||||
fn test_parse_s3_url_rejects_other_schemes() {
|
||||
if _, _ := parse_s3_url('https://example.com/x') {
|
||||
assert false, 'should have errored'
|
||||
}
|
||||
if _, _ := parse_s3_url('http://x') {
|
||||
assert false, 'should have errored'
|
||||
}
|
||||
if _, _ := parse_s3_url('') {
|
||||
assert false, 'should have errored'
|
||||
}
|
||||
}
|
||||
|
||||
fn test_parse_s3_url_rejects_empty_key() {
|
||||
if _, _ := parse_s3_url('s3://my-bucket/') {
|
||||
assert false, 'should have errored'
|
||||
}
|
||||
if _, _ := parse_s3_url('s3://') {
|
||||
assert false, 'should have errored'
|
||||
}
|
||||
}
|
||||
|
||||
fn test_redact_url_strips_query() {
|
||||
assert redact_url('https://x.y/path?X-Amz-Signature=abc') == 'https://x.y/path?<redacted>'
|
||||
assert redact_url('https://x.y/path') == 'https://x.y/path'
|
||||
}
|
||||
|
||||
fn test_fetch_rejects_non_s3_scheme() {
|
||||
if _ := fetch('https://example.com/x') {
|
||||
assert false
|
||||
}
|
||||
if _ := fetch('http://example.com/x') {
|
||||
assert false
|
||||
}
|
||||
if _ := fetch('file:///etc/passwd') {
|
||||
assert false
|
||||
}
|
||||
}
|
||||
|
||||
fn test_fetch_rejects_post_method() {
|
||||
// POST is not part of the s3.fetch contract: object writes use PUT,
|
||||
// and POST in S3 is reserved for multipart-control endpoints which
|
||||
// take an XML body that fetch does not generate. Reject explicitly
|
||||
// rather than silently rewriting to PUT.
|
||||
if _ := fetch('s3://my-bucket/key.txt', method: .post, body: 'x'.bytes()) {
|
||||
assert false, 'POST should be rejected'
|
||||
}
|
||||
}
|
||||
|
||||
fn test_build_object_path_preserves_leading_and_trailing_slashes() {
|
||||
creds := Credentials{
|
||||
access_key_id: 'K'
|
||||
secret_access_key: 'S'
|
||||
bucket: 'b'
|
||||
}
|
||||
// S3 keys are byte-exact: `folder/`, `/x` and `a//b` must round-trip
|
||||
// untouched (only percent-encoded). Stripping slashes would address a
|
||||
// different object than requested.
|
||||
assert build_object_path(creds, '', 'folder/')! == '/b/folder/'
|
||||
assert build_object_path(creds, '', '/x')! == '/b//x'
|
||||
assert build_object_path(creds, '', 'a//b')! == '/b/a//b'
|
||||
assert build_object_path(creds, '', 'plain.txt')! == '/b/plain.txt'
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
// FileOptions configures a File handle.
|
||||
@[params]
|
||||
pub struct FileOptions {
|
||||
pub:
|
||||
bucket string
|
||||
}
|
||||
|
||||
// File is a reference to one S3 object. It holds no buffer; every method
|
||||
// round-trips to S3 (or generates a presigned URL for `presign`).
|
||||
//
|
||||
// Construct via `Client.file(...)` or directly with `File{ client: &c, key: '...' }`.
|
||||
pub struct File {
|
||||
pub:
|
||||
client &Client
|
||||
bucket string
|
||||
key string
|
||||
}
|
||||
|
||||
// read returns the full object body. For range reads, see `read_range`.
|
||||
pub fn (f &File) read() ![]u8 {
|
||||
return f.client.get(f.key, bucket: f.bucket)
|
||||
}
|
||||
|
||||
// text is a UTF-8 convenience over `read`.
|
||||
pub fn (f &File) text() !string {
|
||||
return f.client.get_string(f.key, bucket: f.bucket)
|
||||
}
|
||||
|
||||
// read_range fetches `bytes=<begin>-<end_inclusive>` (HTTP Range semantics).
|
||||
// Use `end < 0` to read to end-of-file.
|
||||
pub fn (f &File) read_range(begin i64, end i64) ![]u8 {
|
||||
range := if end < 0 { 'bytes=${begin}-' } else { 'bytes=${begin}-${end}' }
|
||||
return f.client.get(f.key, bucket: f.bucket, range: range)
|
||||
}
|
||||
|
||||
// write uploads `data` as the entire object body.
|
||||
pub fn (f &File) write(data []u8, opts PutOptions) ! {
|
||||
mut o := opts
|
||||
if o.bucket == '' {
|
||||
o = PutOptions{
|
||||
...opts
|
||||
bucket: f.bucket
|
||||
}
|
||||
}
|
||||
f.client.put(f.key, data, o)!
|
||||
}
|
||||
|
||||
// write_string is a UTF-8 convenience over `write`.
|
||||
pub fn (f &File) write_string(s string, opts PutOptions) ! {
|
||||
f.write(s.bytes(), opts)!
|
||||
}
|
||||
|
||||
// stat returns object metadata (size / etag / last-modified / content-type).
|
||||
pub fn (f &File) stat() !Stat {
|
||||
return f.client.stat(f.key, bucket: f.bucket)
|
||||
}
|
||||
|
||||
// exists is a HEAD that converts 404 into `false`.
|
||||
pub fn (f &File) exists() !bool {
|
||||
return f.client.exists(f.key, bucket: f.bucket)
|
||||
}
|
||||
|
||||
// size returns the Content-Length, in bytes.
|
||||
pub fn (f &File) size() !i64 {
|
||||
return f.client.size(f.key, bucket: f.bucket)
|
||||
}
|
||||
|
||||
// delete removes the object. Idempotent: no error when the object is absent.
|
||||
pub fn (f &File) delete() ! {
|
||||
f.client.delete(f.key, bucket: f.bucket)!
|
||||
}
|
||||
|
||||
// presign returns a presigned URL for this object. See `PresignOptions`.
|
||||
pub fn (f &File) presign(opts PresignOptions) !string {
|
||||
mut o := opts
|
||||
if o.bucket == '' && f.bucket != '' {
|
||||
o = PresignOptions{
|
||||
...opts
|
||||
bucket: f.bucket
|
||||
}
|
||||
}
|
||||
return f.client.presign(f.key, o)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
import net.http
|
||||
|
||||
// init registers an `s3://` scheme handler with `net.http` so that
|
||||
// `http.fetch(url: 's3://...')` and friends route here transparently. Only
|
||||
// active when this module is imported.
|
||||
fn init() {
|
||||
http.register_scheme('s3', http_fetch_handler)
|
||||
}
|
||||
|
||||
// http_fetch_handler bridges `http.fetch` into `s3.fetch`. Credentials are
|
||||
// pulled from the environment (`S3_*`, `AWS_*`, `CELLAR_ADDON_*`, …); for
|
||||
// explicit credentials, callers should use `s3.fetch` directly.
|
||||
fn http_fetch_handler(config http.FetchConfig) !http.Response {
|
||||
resp := fetch(config.url,
|
||||
method: config.method
|
||||
body: config.data.bytes()
|
||||
credentials: Credentials.from_env()
|
||||
content_type: config.header.get(.content_type) or { '' }
|
||||
)!
|
||||
mut header := http.new_header()
|
||||
for k, v in resp.headers {
|
||||
header.add_custom(k, v) or {}
|
||||
}
|
||||
if resp.content_type != '' {
|
||||
header.set(.content_type, resp.content_type)
|
||||
}
|
||||
if resp.etag != '' {
|
||||
header.set(.etag, resp.etag)
|
||||
}
|
||||
return http.Response{
|
||||
status_code: resp.status_code
|
||||
body: resp.body.bytestr()
|
||||
header: header
|
||||
}
|
||||
}
|
||||
|
||||
fn http_method_to_string(m http.Method) string {
|
||||
return match m {
|
||||
.get { 'GET' }
|
||||
.put { 'PUT' }
|
||||
.post { 'POST' }
|
||||
.delete { 'DELETE' }
|
||||
.head { 'HEAD' }
|
||||
else { 'GET' }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
// Integration tests against a live S3-compatible endpoint. They are skipped
|
||||
// unless the `S3_INTEGRATION` env var is set, and the `S3_HOST`, `S3_KEY_ID`
|
||||
// and `S3_KEY_SECRET` env vars are populated. `v test .` stays fast and
|
||||
// offline by default.
|
||||
//
|
||||
// Run with:
|
||||
// S3_INTEGRATION=1 \
|
||||
// S3_HOST=s3.example.com \
|
||||
// S3_KEY_ID=... S3_KEY_SECRET=... \
|
||||
// S3_BUCKET=vs3-tests \
|
||||
// v test .
|
||||
import os
|
||||
import rand
|
||||
import time
|
||||
import net.http
|
||||
|
||||
// http_dispatch_creds_present returns true when the env exposes the AWS-style
|
||||
// names that http.fetch's bridge will pick up via `Credentials.from_env()`.
|
||||
// The integration test below relies on this — `S3_HOST` / `S3_KEY_ID` /
|
||||
// `S3_KEY_SECRET` (used by the rest of this file) aren't recognised by
|
||||
// `from_env()`, so we map them on the fly when needed.
|
||||
fn ensure_aws_env_for_bridge() {
|
||||
if os.getenv('S3_ACCESS_KEY_ID') == '' {
|
||||
os.setenv('S3_ACCESS_KEY_ID', os.getenv('S3_KEY_ID'), true)
|
||||
}
|
||||
if os.getenv('S3_SECRET_ACCESS_KEY') == '' {
|
||||
os.setenv('S3_SECRET_ACCESS_KEY', os.getenv('S3_KEY_SECRET'), true)
|
||||
}
|
||||
if os.getenv('S3_ENDPOINT') == '' {
|
||||
host := os.getenv('S3_HOST')
|
||||
endpoint := if host.contains('://') { host } else { 'https://${host}' }
|
||||
os.setenv('S3_ENDPOINT', endpoint, true)
|
||||
}
|
||||
}
|
||||
|
||||
fn integration_enabled() bool {
|
||||
return os.getenv('S3_INTEGRATION') != ''
|
||||
}
|
||||
|
||||
fn live_client() ?Client {
|
||||
if !integration_enabled() {
|
||||
return none
|
||||
}
|
||||
host := os.getenv('S3_HOST')
|
||||
key_id := os.getenv('S3_KEY_ID')
|
||||
secret := os.getenv('S3_KEY_SECRET')
|
||||
if host == '' || key_id == '' || secret == '' {
|
||||
return none
|
||||
}
|
||||
endpoint := if host.contains('://') { host } else { 'https://${host}' }
|
||||
bucket := os.getenv_opt('S3_BUCKET') or { 'vs3-tests' }
|
||||
return Client{
|
||||
credentials: Credentials{
|
||||
endpoint: endpoint
|
||||
access_key_id: key_id
|
||||
secret_access_key: secret
|
||||
bucket: bucket
|
||||
region: 'us-east-1'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn rand_key(prefix string) string {
|
||||
return '${prefix}/${time.now().unix()}-${rand.u64()}.txt'
|
||||
}
|
||||
|
||||
fn test_integration_roundtrip() {
|
||||
c := live_client() or { return }
|
||||
key := rand_key('it/roundtrip')
|
||||
defer {
|
||||
c.delete(key) or {}
|
||||
}
|
||||
c.put(key, 'hi'.bytes(), content_type: 'text/plain') or { panic(err) }
|
||||
got := c.get_string(key) or { panic(err) }
|
||||
assert got == 'hi'
|
||||
stat := c.stat(key) or { panic(err) }
|
||||
assert stat.size == 2
|
||||
assert stat.content_type.starts_with('text/plain')
|
||||
}
|
||||
|
||||
fn test_integration_range_read() {
|
||||
c := live_client() or { return }
|
||||
key := rand_key('it/range')
|
||||
defer {
|
||||
c.delete(key) or {}
|
||||
}
|
||||
c.put(key, 'abcdefghijklmnop'.bytes()) or { panic(err) }
|
||||
first := c.get(key, range: 'bytes=0-4') or { panic(err) }
|
||||
assert first.bytestr() == 'abcde', 'got: ${first.bytestr()}'
|
||||
tail := c.get(key, range: 'bytes=10-') or { panic(err) }
|
||||
assert tail.bytestr() == 'klmnop', 'got: ${tail.bytestr()}'
|
||||
}
|
||||
|
||||
fn test_integration_presign_get() {
|
||||
c := live_client() or { return }
|
||||
key := rand_key('it/presign')
|
||||
defer {
|
||||
c.delete(key) or {}
|
||||
}
|
||||
c.put(key, 'PRESIGN_OK'.bytes()) or { panic(err) }
|
||||
url := c.presign(key, expires_in: 60) or { panic(err) }
|
||||
body := fetch_url(url) or { panic(err) }
|
||||
assert body.contains('PRESIGN_OK'), 'got: ${body}'
|
||||
}
|
||||
|
||||
fn test_integration_list_with_prefix() {
|
||||
c := live_client() or { return }
|
||||
prefix := 'it/list-${rand.u64()}/'
|
||||
defer {
|
||||
// Best-effort cleanup
|
||||
if res := c.list(prefix: prefix, max_keys: 100) {
|
||||
for o in res.objects {
|
||||
c.delete(o.key) or {}
|
||||
}
|
||||
}
|
||||
}
|
||||
for i in 0 .. 3 {
|
||||
c.put('${prefix}item-${i}.txt', 'x'.bytes()) or { panic(err) }
|
||||
}
|
||||
res := c.list(prefix: prefix, fetch_owner: false) or { panic(err) }
|
||||
assert res.objects.len == 3
|
||||
for o in res.objects {
|
||||
assert o.key.starts_with(prefix)
|
||||
assert o.size == 1
|
||||
}
|
||||
}
|
||||
|
||||
fn test_integration_exists_and_delete() {
|
||||
c := live_client() or { return }
|
||||
key := rand_key('it/exists')
|
||||
c.put(key, 'present'.bytes()) or { panic(err) }
|
||||
assert c.exists(key) or { panic(err) }
|
||||
c.delete(key) or { panic(err) }
|
||||
assert !(c.exists(key) or { panic(err) })
|
||||
}
|
||||
|
||||
fn test_integration_fetch_helper() {
|
||||
c := live_client() or { return }
|
||||
key := rand_key('it/fetch')
|
||||
url := 's3://${c.credentials.bucket}/${key}'
|
||||
defer {
|
||||
c.delete(key) or {}
|
||||
}
|
||||
put := fetch(url,
|
||||
method: .put
|
||||
body: 'fetch_works'.bytes()
|
||||
credentials: c.credentials
|
||||
) or { panic(err) }
|
||||
assert put.status_code == 200
|
||||
get := fetch(url, credentials: c.credentials) or { panic(err) }
|
||||
assert get.body.bytestr() == 'fetch_works'
|
||||
}
|
||||
|
||||
fn test_integration_multipart_upload_bytes() {
|
||||
c := live_client() or { return }
|
||||
// 6 MiB triggers multipart (one full part + a tail).
|
||||
mut data := []u8{len: 6 * 1024 * 1024}
|
||||
for i in 0 .. data.len {
|
||||
data[i] = u8(i & 0xFF)
|
||||
}
|
||||
key := rand_key('it/multipart')
|
||||
defer {
|
||||
c.delete(key) or {}
|
||||
}
|
||||
c.upload_bytes_multipart(key, data) or { panic(err) }
|
||||
stat := c.stat(key) or { panic(err) }
|
||||
assert stat.size == data.len
|
||||
got := c.get(key) or { panic(err) }
|
||||
assert got.len == data.len
|
||||
for i in 0 .. data.len {
|
||||
if got[i] != data[i] {
|
||||
assert false, 'byte mismatch at ${i}'
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn test_integration_multipart_parallel_many_parts() {
|
||||
c := live_client() or { return }
|
||||
// Force 8 parts at the 5 MiB minimum + tail. With queue_size=5 (default)
|
||||
// we get genuine parallel dispatch and out-of-order completion that the
|
||||
// dispatcher must re-order before CompleteMultipartUpload.
|
||||
mut data := []u8{len: 8 * 5 * 1024 * 1024 + 1024}
|
||||
for i in 0 .. data.len {
|
||||
data[i] = u8((i * 31 + 7) & 0xFF)
|
||||
}
|
||||
key := rand_key('it/parallel')
|
||||
defer {
|
||||
c.delete(key) or {}
|
||||
}
|
||||
c.upload_bytes_multipart(key, data) or { panic(err) }
|
||||
stat := c.stat(key) or { panic(err) }
|
||||
assert stat.size == data.len, 'size mismatch: got ${stat.size}'
|
||||
// Verify a tail range, where corruption from out-of-order parts would land.
|
||||
tail := c.get(key, range: 'bytes=${data.len - 1024}-') or { panic(err) }
|
||||
for i in 0 .. tail.len {
|
||||
if tail[i] != data[data.len - 1024 + i] {
|
||||
assert false, 'tail byte mismatch at offset ${i}'
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn test_integration_bucket_lifecycle() {
|
||||
c := live_client() or { return }
|
||||
// Random suffix keeps reruns idempotent.
|
||||
name := 's3v-it-${rand.u64():08x}'.to_lower()
|
||||
c.create_bucket(bucket: name) or {
|
||||
// Some providers require region constraint or have different defaults
|
||||
// — surface but don't fail the suite for that one provider quirk.
|
||||
eprintln('skip: create_bucket(${name}) failed: ${err}')
|
||||
return
|
||||
}
|
||||
defer {
|
||||
c.delete_bucket(bucket: name) or {}
|
||||
}
|
||||
exists := c.bucket_exists(bucket: name) or { panic(err) }
|
||||
assert exists, 'bucket ${name} should exist after create'
|
||||
// Delete once explicitly so we can assert it disappeared.
|
||||
c.delete_bucket(bucket: name) or { panic(err) }
|
||||
gone := c.bucket_exists(bucket: name) or { panic(err) }
|
||||
assert !gone, 'bucket ${name} should be gone after delete'
|
||||
}
|
||||
|
||||
fn test_integration_invalid_credentials_yields_clean_error() {
|
||||
if !integration_enabled() {
|
||||
return
|
||||
}
|
||||
host := os.getenv('S3_HOST')
|
||||
if host == '' {
|
||||
return
|
||||
}
|
||||
c := Client{
|
||||
credentials: Credentials{
|
||||
endpoint: 'https://${host}'
|
||||
access_key_id: 'AKIADOESNOTEXIST'
|
||||
secret_access_key: 'badbadbadbadbadbadbadbadbadbadbadbadbadbad'
|
||||
bucket: os.getenv_opt('S3_BUCKET') or { 'vs3-tests' }
|
||||
}
|
||||
}
|
||||
if _ := c.stat('any-key') {
|
||||
assert false, 'expected error'
|
||||
} else {
|
||||
assert err is S3Error, 'got non-S3 error: ${typeof(err).name}'
|
||||
}
|
||||
}
|
||||
|
||||
// fetch_url is a tiny `http.get`-style helper used only in the presign test.
|
||||
fn fetch_url(url string) !string {
|
||||
resp := http.get(url)!
|
||||
return resp.body
|
||||
}
|
||||
|
||||
// Exercises the `http.fetch(url: 's3://...')` bridge end-to-end. The s3
|
||||
// module's `init()` registers itself with net.http; this test goes through
|
||||
// that path (no direct `s3.fetch` call) to prove the dispatch works.
|
||||
fn test_integration_http_fetch_dispatch_to_s3() {
|
||||
c := live_client() or { return }
|
||||
ensure_aws_env_for_bridge()
|
||||
bucket := c.credentials.bucket
|
||||
key := rand_key('it/http-bridge')
|
||||
url := 's3://${bucket}/${key}'
|
||||
|
||||
put := http.fetch(url: url, method: .put, data: 'via-http.fetch') or {
|
||||
assert false, 'PUT through bridge failed: ${err}'
|
||||
return
|
||||
}
|
||||
assert put.status_code == 200, 'unexpected PUT status: ${put.status_code}'
|
||||
|
||||
get := http.fetch(url: url) or {
|
||||
assert false, 'GET through bridge failed: ${err}'
|
||||
return
|
||||
}
|
||||
assert get.status_code == 200, 'unexpected GET status: ${get.status_code}'
|
||||
assert get.body == 'via-http.fetch', 'body mismatch: ${get.body}'
|
||||
|
||||
del := http.fetch(url: url, method: .delete) or {
|
||||
assert false, 'DELETE through bridge failed: ${err}'
|
||||
return
|
||||
}
|
||||
assert del.status_code in [200, 204], 'unexpected DELETE status: ${del.status_code}'
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
// list returns up to ~1000 objects matching `opts`. Use the returned
|
||||
// `next_continuation_token` to page through more.
|
||||
//
|
||||
// Speaks the ListObjectsV2 protocol.
|
||||
pub fn (c &Client) list(opts ListOptions) !ListResult {
|
||||
bucket := pick_bucket(c.credentials, opts.bucket)!
|
||||
creds := c.creds_for(bucket)
|
||||
path := bucket_path(creds, bucket)
|
||||
mut q := map[string]string{}
|
||||
q['list-type'] = '2'
|
||||
if opts.prefix != '' {
|
||||
q['prefix'] = opts.prefix
|
||||
}
|
||||
if opts.continuation_token != '' {
|
||||
q['continuation-token'] = opts.continuation_token
|
||||
}
|
||||
if opts.delimiter != '' {
|
||||
q['delimiter'] = opts.delimiter
|
||||
}
|
||||
if opts.max_keys > 0 {
|
||||
q['max-keys'] = opts.max_keys.str()
|
||||
}
|
||||
if opts.start_after != '' {
|
||||
q['start-after'] = opts.start_after
|
||||
}
|
||||
if opts.encoding_type != '' {
|
||||
q['encoding-type'] = opts.encoding_type
|
||||
}
|
||||
if opts.fetch_owner {
|
||||
q['fetch-owner'] = 'true'
|
||||
}
|
||||
query := canonical_query_string(q)
|
||||
signed := sign_request(creds, SignRequest{
|
||||
method: 'GET'
|
||||
path: path
|
||||
query: query
|
||||
payload_hash: empty_sha256
|
||||
})!
|
||||
resp := c.do_http(signed, '')!
|
||||
if resp.status_code != 200 {
|
||||
return new_http_error(resp.status_code, bucket, resp.body)
|
||||
}
|
||||
return parse_list_response(resp.body)
|
||||
}
|
||||
|
||||
// parse_list_response is hand-rolled XML extraction. ListObjectsV2 uses a
|
||||
// rigid structure with no attributes we care about, so a tag scanner beats
|
||||
// pulling in the full XML decoder both in code size and predictability
|
||||
// against malformed input.
|
||||
pub fn parse_list_response(body string) !ListResult {
|
||||
objects := parse_contents(body)
|
||||
common := parse_common_prefixes(body)
|
||||
max_keys := extract_xml_tag(body, 'MaxKeys').int()
|
||||
key_count := extract_xml_tag(body, 'KeyCount').int()
|
||||
is_truncated := extract_xml_tag(body, 'IsTruncated') == 'true'
|
||||
return ListResult{
|
||||
name: extract_xml_tag(body, 'Name')
|
||||
prefix: extract_xml_tag(body, 'Prefix')
|
||||
delimiter: extract_xml_tag(body, 'Delimiter')
|
||||
start_after: extract_xml_tag(body, 'StartAfter')
|
||||
max_keys: max_keys
|
||||
key_count: key_count
|
||||
is_truncated: is_truncated
|
||||
continuation_token: extract_xml_tag(body, 'ContinuationToken')
|
||||
next_continuation_token: extract_xml_tag(body, 'NextContinuationToken')
|
||||
objects: objects
|
||||
common_prefixes: common
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_contents(body string) []ObjectInfo {
|
||||
mut out := []ObjectInfo{}
|
||||
mut start := 0
|
||||
for {
|
||||
open := body.index_after('<Contents>', start) or { break }
|
||||
close := body.index_after('</Contents>', open) or { break }
|
||||
seg := body[open + '<Contents>'.len..close]
|
||||
mut owner := ?Owner(none)
|
||||
owner_open := seg.index('<Owner>') or { -1 }
|
||||
if owner_open >= 0 {
|
||||
owner_close := seg.index('</Owner>') or { -1 }
|
||||
if owner_close > owner_open {
|
||||
owner_seg := seg[owner_open + '<Owner>'.len..owner_close]
|
||||
owner = Owner{
|
||||
id: extract_xml_tag(owner_seg, 'ID')
|
||||
display_name: extract_xml_tag(owner_seg, 'DisplayName')
|
||||
}
|
||||
}
|
||||
}
|
||||
out << ObjectInfo{
|
||||
key: extract_xml_tag(seg, 'Key')
|
||||
last_modified: extract_xml_tag(seg, 'LastModified')
|
||||
etag: extract_xml_tag(seg, 'ETag').trim('"')
|
||||
size: extract_xml_tag(seg, 'Size').i64()
|
||||
storage_class: extract_xml_tag(seg, 'StorageClass')
|
||||
owner: owner
|
||||
}
|
||||
start = close + '</Contents>'.len
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
fn parse_common_prefixes(body string) []CommonPrefix {
|
||||
mut out := []CommonPrefix{}
|
||||
mut start := 0
|
||||
for {
|
||||
open := body.index_after('<CommonPrefixes>', start) or { break }
|
||||
close := body.index_after('</CommonPrefixes>', open) or { break }
|
||||
seg := body[open + '<CommonPrefixes>'.len..close]
|
||||
out << CommonPrefix{
|
||||
prefix: extract_xml_tag(seg, 'Prefix')
|
||||
}
|
||||
start = close + '</CommonPrefixes>'.len
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
// Captured from a real ListObjectsV2 response (S3-compatible endpoint),
|
||||
// trimmed to the fields we parse.
|
||||
const list_response_xml = '<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult>
|
||||
<Name>test-bucket</Name>
|
||||
<Prefix>foo/</Prefix>
|
||||
<KeyCount>2</KeyCount>
|
||||
<MaxKeys>1000</MaxKeys>
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<Contents>
|
||||
<Key>foo/a.txt</Key>
|
||||
<Size>10</Size>
|
||||
<ETag>"d41d8cd98f00b204e9800998ecf8427e"</ETag>
|
||||
<LastModified>2024-01-15T00:00:00.000Z</LastModified>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>
|
||||
<Contents>
|
||||
<Key>foo/b.txt</Key>
|
||||
<Size>20</Size>
|
||||
<ETag>"9b8b80d22b1d56a3f9b2c25d6e3e74c5"</ETag>
|
||||
<LastModified>2024-01-16T00:00:00.000Z</LastModified>
|
||||
</Contents>
|
||||
</ListBucketResult>'
|
||||
|
||||
fn test_parse_list_response_basic() {
|
||||
r := parse_list_response(list_response_xml) or { panic(err) }
|
||||
assert r.name == 'test-bucket'
|
||||
assert r.prefix == 'foo/'
|
||||
assert r.key_count == 2
|
||||
assert !r.is_truncated
|
||||
assert r.objects.len == 2
|
||||
assert r.objects[0].key == 'foo/a.txt'
|
||||
assert r.objects[0].size == 10
|
||||
assert r.objects[0].etag == 'd41d8cd98f00b204e9800998ecf8427e'
|
||||
assert r.objects[1].key == 'foo/b.txt'
|
||||
}
|
||||
|
||||
fn test_parse_list_response_truncated() {
|
||||
body := '<?xml version="1.0"?><ListBucketResult><Name>b</Name><IsTruncated>true</IsTruncated><NextContinuationToken>opaqueToken==</NextContinuationToken><KeyCount>1</KeyCount><Contents><Key>only.txt</Key><Size>1</Size><ETag>"abc"</ETag></Contents></ListBucketResult>'
|
||||
r := parse_list_response(body) or { panic(err) }
|
||||
assert r.is_truncated
|
||||
assert r.next_continuation_token == 'opaqueToken=='
|
||||
}
|
||||
|
||||
fn test_parse_list_response_common_prefixes() {
|
||||
body := '<?xml version="1.0"?><ListBucketResult><Name>b</Name><Delimiter>/</Delimiter><CommonPrefixes><Prefix>folder1/</Prefix></CommonPrefixes><CommonPrefixes><Prefix>folder2/</Prefix></CommonPrefixes><KeyCount>0</KeyCount></ListBucketResult>'
|
||||
r := parse_list_response(body) or { panic(err) }
|
||||
assert r.delimiter == '/'
|
||||
assert r.common_prefixes.len == 2
|
||||
assert r.common_prefixes[0].prefix == 'folder1/'
|
||||
assert r.common_prefixes[1].prefix == 'folder2/'
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
import os
|
||||
import strings
|
||||
import time
|
||||
|
||||
// Multipart upload constants — defaults aligned with the S3 protocol limits.
|
||||
pub const min_part_size = i64(5 * 1024 * 1024) // 5 MiB
|
||||
pub const max_part_size = i64(5 * 1024 * 1024 * 1024) // 5 GiB
|
||||
pub const max_parts = 10000 // hard S3 limit
|
||||
|
||||
// upload_file streams a local file to S3, choosing single-part or multipart
|
||||
// based on size. Use this for files larger than ~50 MiB or anything you
|
||||
// don't want to slurp into memory.
|
||||
//
|
||||
// `key` is the destination object key. The local file is read in
|
||||
// `client.part_size` chunks (default 5 MiB).
|
||||
pub fn (c &Client) upload_file(key string, local_path string, opts PutOptions) ! {
|
||||
stat_local := os.stat(local_path) or {
|
||||
return new_error('LocalFileNotFound',
|
||||
'Cannot stat local file: ${local_path} (${err.msg()})')
|
||||
}
|
||||
size := i64(stat_local.size)
|
||||
if size <= min_part_size {
|
||||
data := os.read_bytes(local_path) or {
|
||||
return new_error('LocalReadFailed', 'Cannot read ${local_path}: ${err.msg()}')
|
||||
}
|
||||
return c.put(key, data, opts)
|
||||
}
|
||||
c.upload_file_multipart(key, local_path, size, opts)!
|
||||
}
|
||||
|
||||
// MultipartUploader is a stateful handle to an in-flight multipart upload,
|
||||
// produced by `Client.start_multipart`. Use it when you want to push parts
|
||||
// generated on-the-fly (network sources, decompressors, anything you don't
|
||||
// want to materialise on disk):
|
||||
//
|
||||
// mut up := c.start_multipart('key', s3.PutOptions{ content_type: 'application/octet-stream' })!
|
||||
// for chunk in chunks { up.upload(chunk)! }
|
||||
// up.complete()!
|
||||
//
|
||||
// On error, `complete()` / `upload()` return without aborting. The caller
|
||||
// must invoke `abort()` themselves so that `defer { up.abort() or {} }`
|
||||
// remains an explicit, visible cleanup hook.
|
||||
pub struct MultipartUploader {
|
||||
mut:
|
||||
client &Client
|
||||
key string
|
||||
upload_id string
|
||||
opts PutOptions
|
||||
parts []PartRef
|
||||
part_number int
|
||||
completed bool
|
||||
aborted bool
|
||||
}
|
||||
|
||||
// start_multipart begins a multipart upload and returns a MultipartUploader.
|
||||
// Memory cost: zero — each `upload(chunk)` call streams the chunk to S3 and
|
||||
// returns when it is acknowledged.
|
||||
pub fn (c &Client) start_multipart(key string, opts PutOptions) !MultipartUploader {
|
||||
upload_id := c.initiate_multipart(key, opts)!
|
||||
return MultipartUploader{
|
||||
// `unsafe { c }` is required to store the receiver's pointer in the
|
||||
// returned struct — V's borrow checker forbids this implicitly, even
|
||||
// though `Client` is `@[heap]` so the lifetime is safe.
|
||||
client: unsafe { c }
|
||||
key: key
|
||||
upload_id: upload_id
|
||||
opts: opts
|
||||
parts: []PartRef{}
|
||||
part_number: 1
|
||||
}
|
||||
}
|
||||
|
||||
// upload pushes one chunk as the next part. Each chunk MUST be at least
|
||||
// `min_part_size` (5 MiB) except the last one — that's an S3 invariant
|
||||
// the server enforces; we do not buffer for you.
|
||||
pub fn (mut u MultipartUploader) upload(data []u8) ! {
|
||||
if u.completed || u.aborted {
|
||||
return new_error('InvalidState', 'multipart upload is already finalised')
|
||||
}
|
||||
if u.part_number > max_parts {
|
||||
return new_error('TooManyParts', 'Exceeded ${max_parts} parts — increase part_size')
|
||||
}
|
||||
etag := u.client.upload_part(u.key, u.upload_id, u.part_number, data, u.opts)!
|
||||
u.parts << PartRef{
|
||||
part_number: u.part_number
|
||||
etag: etag
|
||||
}
|
||||
u.part_number++
|
||||
}
|
||||
|
||||
// complete finalises the upload. After this call the object is visible.
|
||||
pub fn (mut u MultipartUploader) complete() ! {
|
||||
if u.completed {
|
||||
return
|
||||
}
|
||||
u.client.complete_multipart(u.key, u.upload_id, u.parts, u.opts)!
|
||||
u.completed = true
|
||||
}
|
||||
|
||||
// abort cancels the in-flight upload. Idempotent.
|
||||
pub fn (mut u MultipartUploader) abort() ! {
|
||||
if u.aborted || u.completed {
|
||||
return
|
||||
}
|
||||
u.client.abort_multipart(u.key, u.upload_id, u.opts)!
|
||||
u.aborted = true
|
||||
}
|
||||
|
||||
// upload_bytes_multipart uploads an in-memory byte buffer using multipart.
|
||||
// Useful for large generated payloads (test fixtures up to 5 GiB). Uses up
|
||||
// to `Client.queue_size` parallel part uploads.
|
||||
pub fn (c &Client) upload_bytes_multipart(key string, data []u8, opts PutOptions) ! {
|
||||
upload_id := c.initiate_multipart(key, opts)!
|
||||
parts := c.run_parallel_parts(key, upload_id, opts, fn [data] (offset i64, want int) ![]u8 {
|
||||
end := if offset + i64(want) > i64(data.len) { i64(data.len) } else { offset + i64(want) }
|
||||
return data[offset..end]
|
||||
}, i64(data.len)) or {
|
||||
c.abort_multipart(key, upload_id, opts) or {}
|
||||
return err
|
||||
}
|
||||
c.complete_multipart(key, upload_id, parts, opts)!
|
||||
}
|
||||
|
||||
// upload_file_multipart streams a local file part-by-part with up to
|
||||
// `Client.queue_size` concurrent uploads. Peak memory is roughly
|
||||
// `queue_size * part_size`.
|
||||
pub fn (c &Client) upload_file_multipart(key string, local_path string, size i64, opts PutOptions) ! {
|
||||
mut f := os.open(local_path) or {
|
||||
return new_error('LocalReadFailed', 'open ${local_path}: ${err.msg()}')
|
||||
}
|
||||
defer {
|
||||
f.close()
|
||||
}
|
||||
upload_id := c.initiate_multipart(key, opts)!
|
||||
parts := c.run_parallel_parts(key, upload_id, opts, fn [mut f] (offset i64, want int) ![]u8 {
|
||||
mut buf := []u8{len: want}
|
||||
n := f.read_bytes_into(u64(offset), mut buf) or {
|
||||
return new_error('LocalReadFailed', 'read_bytes_into: ${err.msg()}')
|
||||
}
|
||||
if n <= 0 {
|
||||
return new_error('LocalReadFailed', 'unexpected EOF at offset ${offset}')
|
||||
}
|
||||
return buf[..n]
|
||||
}, size) or {
|
||||
c.abort_multipart(key, upload_id, opts) or {}
|
||||
return err
|
||||
}
|
||||
c.complete_multipart(key, upload_id, parts, opts)!
|
||||
}
|
||||
|
||||
// PartJob is a single upload-part task pushed onto the worker queue.
|
||||
struct PartJob {
|
||||
part_number int
|
||||
data []u8
|
||||
}
|
||||
|
||||
// PartResult carries the worker outcome back to the dispatcher.
|
||||
struct PartResult {
|
||||
part_number int
|
||||
etag string
|
||||
err ?IError
|
||||
}
|
||||
|
||||
// part_worker pulls jobs off `jobs` and pushes results onto `results`. Exits
|
||||
// when `jobs` is closed.
|
||||
fn (c &Client) part_worker(key string, upload_id string, opts PutOptions, jobs chan PartJob, results chan PartResult) {
|
||||
for {
|
||||
job := <-jobs or { return }
|
||||
etag := c.upload_part(key, upload_id, job.part_number, job.data, opts) or {
|
||||
results <- PartResult{
|
||||
part_number: job.part_number
|
||||
err: err
|
||||
}
|
||||
continue
|
||||
}
|
||||
results <- PartResult{
|
||||
part_number: job.part_number
|
||||
etag: etag
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// run_parallel_parts dispatches `producer` chunks across a worker pool of
|
||||
// `Client.queue_size` goroutines and gathers the resulting PartRefs. The
|
||||
// caller is responsible for issuing `initiate_multipart` / `abort_multipart`
|
||||
// / `complete_multipart` around this call. `producer(offset, want)` must
|
||||
// return a freshly-allocated chunk of length ≤ `want` bytes (workers may
|
||||
// hold the slice past the next call). Returns parts in completion order;
|
||||
// `complete_multipart` re-sorts by part number before sending the manifest.
|
||||
fn (c &Client) run_parallel_parts(key string, upload_id string, opts PutOptions, producer fn (offset i64, want int) ![]u8, total i64) ![]PartRef {
|
||||
workers := if c.queue_size < 1 { 1 } else { c.queue_size }
|
||||
part_size := if c.part_size < min_part_size { min_part_size } else { c.part_size }
|
||||
jobs := chan PartJob{cap: workers}
|
||||
results := chan PartResult{cap: workers}
|
||||
for _ in 0 .. workers {
|
||||
spawn c.part_worker(key, upload_id, opts, jobs, results)
|
||||
}
|
||||
|
||||
mut parts := []PartRef{}
|
||||
mut first_err := ?IError(none)
|
||||
mut sent := 0
|
||||
mut received := 0
|
||||
mut offset := i64(0)
|
||||
mut part_number := 1
|
||||
|
||||
for offset < total && first_err == none {
|
||||
if part_number > max_parts {
|
||||
first_err = new_error('TooManyParts',
|
||||
'Exceeded ${max_parts} parts — increase part_size')
|
||||
break
|
||||
}
|
||||
want := if offset + part_size > total { int(total - offset) } else { int(part_size) }
|
||||
chunk := producer(offset, want) or {
|
||||
first_err = err
|
||||
break
|
||||
}
|
||||
if chunk.len == 0 {
|
||||
break
|
||||
}
|
||||
jobs <- PartJob{
|
||||
part_number: part_number
|
||||
data: chunk
|
||||
}
|
||||
sent++
|
||||
offset += i64(chunk.len)
|
||||
part_number++
|
||||
// Drain ready results without blocking — surfaces errors early so we
|
||||
// can stop dispatching new parts.
|
||||
for {
|
||||
select {
|
||||
r := <-results {
|
||||
received++
|
||||
if e := r.err {
|
||||
if first_err == none {
|
||||
first_err = e
|
||||
}
|
||||
} else {
|
||||
parts << PartRef{
|
||||
part_number: r.part_number
|
||||
etag: r.etag
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
jobs.close()
|
||||
for received < sent {
|
||||
r := <-results
|
||||
received++
|
||||
if e := r.err {
|
||||
if first_err == none {
|
||||
first_err = e
|
||||
}
|
||||
} else {
|
||||
parts << PartRef{
|
||||
part_number: r.part_number
|
||||
etag: r.etag
|
||||
}
|
||||
}
|
||||
}
|
||||
results.close()
|
||||
if e := first_err {
|
||||
return e
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
// PartRef is one ETag/PartNumber pair recorded during multipart upload.
|
||||
pub struct PartRef {
|
||||
pub:
|
||||
part_number int
|
||||
etag string
|
||||
}
|
||||
|
||||
// initiate_multipart starts an upload and returns the UploadId. ACL,
|
||||
// content-type, and friends from `opts` are sent here — they apply to the
|
||||
// final object.
|
||||
pub fn (c &Client) initiate_multipart(key string, opts PutOptions) !string {
|
||||
creds := c.creds_for(opts.bucket)
|
||||
path := build_object_path(creds, opts.bucket, key)!
|
||||
signed := sign_request(creds, SignRequest{
|
||||
method: 'POST'
|
||||
path: path
|
||||
query: 'uploads='
|
||||
payload_hash: empty_sha256
|
||||
extra_headers: put_object_headers(opts)
|
||||
})!
|
||||
resp := c.do_http(signed, '')!
|
||||
if resp.status_code != 200 {
|
||||
return new_http_error(resp.status_code, key, resp.body)
|
||||
}
|
||||
upload_id := extract_xml_tag(resp.body, 'UploadId')
|
||||
if upload_id == '' {
|
||||
return new_error('InvalidResponse', 'CreateMultipartUploadResult missing UploadId')
|
||||
}
|
||||
return upload_id
|
||||
}
|
||||
|
||||
// upload_part uploads a single chunk and returns the server-side ETag. Up to
|
||||
// `Client.retry` attempts with exponential backoff (200ms, 400ms, 800ms, …)
|
||||
// on transient failures.
|
||||
//
|
||||
// The payload is always SHA-256-signed (not `UNSIGNED-PAYLOAD`) so the
|
||||
// service validates byte-for-byte integrity end-to-end. Multipart uploads
|
||||
// don't carry a Content-MD5 header by default; without payload signing a
|
||||
// flipped bit in transit would silently produce a corrupt object that
|
||||
// passes the multipart ETag check.
|
||||
pub fn (c &Client) upload_part(key string, upload_id string, part_number int, data []u8, opts PutOptions) !string {
|
||||
creds := c.creds_for(opts.bucket)
|
||||
path := build_object_path(creds, opts.bucket, key)!
|
||||
query := canonical_query_string({
|
||||
'partNumber': part_number.str()
|
||||
'uploadId': upload_id
|
||||
})
|
||||
// Hash the payload once — multi-MiB chunks should not be re-hashed per attempt.
|
||||
// The signature itself still has to be recomputed on each retry because the
|
||||
// `x-amz-date` header advances.
|
||||
payload_hash := sha256_hex(data)
|
||||
body := data.bytestr()
|
||||
max_attempts := if c.retry < 1 { 1 } else { c.retry }
|
||||
mut last_err := IError(new_error('NetworkError', 'upload_part: no attempt was made'))
|
||||
for attempt in 0 .. max_attempts {
|
||||
signed := sign_request(creds, SignRequest{
|
||||
method: 'PUT'
|
||||
path: path
|
||||
query: query
|
||||
payload_hash: payload_hash
|
||||
})!
|
||||
resp := c.do_http(signed, body) or {
|
||||
last_err = err
|
||||
if attempt + 1 < max_attempts {
|
||||
time.sleep((1 << attempt) * 200 * time.millisecond)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if resp.status_code == 200 {
|
||||
etag := resp.header.get(.etag) or { '' }
|
||||
if etag == '' {
|
||||
return new_error('InvalidResponse', 'UploadPart returned no ETag')
|
||||
}
|
||||
return etag.trim('"')
|
||||
}
|
||||
last_err = new_http_error(resp.status_code, key, resp.body)
|
||||
if attempt + 1 < max_attempts {
|
||||
time.sleep((1 << attempt) * 200 * time.millisecond)
|
||||
}
|
||||
}
|
||||
return last_err
|
||||
}
|
||||
|
||||
// complete_multipart finalizes a multipart upload. Parts must be in ascending
|
||||
// `part_number` order; we sort defensively.
|
||||
pub fn (c &Client) complete_multipart(key string, upload_id string, parts []PartRef, opts PutOptions) ! {
|
||||
mut sorted := parts.clone()
|
||||
sorted.sort(a.part_number < b.part_number)
|
||||
mut body := strings.new_builder(1024)
|
||||
body.write_string('<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/">')
|
||||
for p in sorted {
|
||||
body.write_string('<Part><PartNumber>${p.part_number}</PartNumber><ETag>"${p.etag}"</ETag></Part>')
|
||||
}
|
||||
body.write_string('</CompleteMultipartUpload>')
|
||||
xml := body.str()
|
||||
creds := c.creds_for(opts.bucket)
|
||||
path := build_object_path(creds, opts.bucket, key)!
|
||||
q := 'uploadId=' + uri_encode_query(upload_id)
|
||||
signed := sign_request(creds, SignRequest{
|
||||
method: 'POST'
|
||||
path: path
|
||||
query: q
|
||||
payload_hash: sha256_hex(xml.bytes())
|
||||
extra_headers: {
|
||||
'content-type': 'application/xml'
|
||||
}
|
||||
})!
|
||||
resp := c.do_http(signed, xml)!
|
||||
if resp.status_code != 200 {
|
||||
return new_http_error(resp.status_code, key, resp.body)
|
||||
}
|
||||
// CompleteMultipartUpload may return HTTP 200 with an <Error> body — surface those.
|
||||
if resp.body.contains('<Error>') {
|
||||
return new_http_error(200, key, resp.body)
|
||||
}
|
||||
}
|
||||
|
||||
// abort_multipart cancels an in-flight upload. Best-effort — callers usually
|
||||
// invoke it inside an error path and don't care about the result.
|
||||
pub fn (c &Client) abort_multipart(key string, upload_id string, opts PutOptions) ! {
|
||||
creds := c.creds_for(opts.bucket)
|
||||
path := build_object_path(creds, opts.bucket, key)!
|
||||
q := 'uploadId=' + uri_encode_query(upload_id)
|
||||
signed := sign_request(creds, SignRequest{
|
||||
method: 'DELETE'
|
||||
path: path
|
||||
query: q
|
||||
payload_hash: empty_sha256
|
||||
})!
|
||||
resp := c.do_http(signed, '')!
|
||||
if resp.status_code !in [204, 200, 404] {
|
||||
return new_http_error(resp.status_code, key, resp.body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
// Module s3 is an S3-compatible client for V.
|
||||
//
|
||||
// Quick start:
|
||||
//
|
||||
// import s3
|
||||
//
|
||||
// client := s3.new_client(s3.Credentials{
|
||||
// endpoint: 'https://s3.example.com'
|
||||
// access_key_id: '...'
|
||||
// secret_access_key: '...'
|
||||
// bucket: 'my-bucket'
|
||||
// })
|
||||
// client.put('hello.txt', 'Hi from V!'.bytes())!
|
||||
// text := client.get_string('hello.txt')!
|
||||
// url := client.presign('hello.txt', expires_in: 3600)!
|
||||
//
|
||||
// // s3:// fetch helper:
|
||||
// resp := s3.fetch('s3://my-bucket/hello.txt')!
|
||||
//
|
||||
// Construct credentials from the environment (S3_*, AWS_*, CELLAR_ADDON_*,
|
||||
// SCW_*, B2_*, R2_*, SPACES_* — first non-empty wins per field):
|
||||
//
|
||||
// client := s3.new_client(s3.Credentials.from_env())
|
||||
//
|
||||
// Multipart for large files:
|
||||
//
|
||||
// client.upload_file('big.bin', '/path/to/big.bin', s3.PutOptions{ content_type: 'application/octet-stream' })!
|
||||
module s3
|
||||
|
||||
// version is the module version, kept in sync with v.mod.
|
||||
pub const version = '0.1.0'
|
||||
|
||||
// info_string returns a one-line build-time identification, useful in
|
||||
// User-Agent strings or `--version` output.
|
||||
pub fn info_string() string {
|
||||
return 's3/${version}'
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
import crypto.hmac
|
||||
import crypto.sha256
|
||||
import time
|
||||
|
||||
// service_name is the SigV4 service identifier baked into the signing key.
|
||||
pub const service_name = 's3'
|
||||
|
||||
// algo is the SigV4 algorithm marker S3 expects.
|
||||
pub const algo = 'AWS4-HMAC-SHA256'
|
||||
|
||||
// unsigned_payload is the magic string used in `x-amz-content-sha256` when
|
||||
// the payload is not pre-hashed. Used by the single-shot put path to avoid
|
||||
// buffering / re-scanning the entire body just to sign it.
|
||||
pub const unsigned_payload = 'UNSIGNED-PAYLOAD'
|
||||
|
||||
// empty_sha256 is `sha256("")` precomputed. Used for HEAD/GET/DELETE where
|
||||
// there is no body and we want a real hash for stricter S3 endpoints.
|
||||
pub const empty_sha256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
|
||||
|
||||
// SignRequest describes the request to sign. The signer is intentionally
|
||||
// agnostic of HTTP option semantics (ACL, storage class, …) — the caller
|
||||
// pre-fills `extra_headers` with whatever it intends to send. That keeps the
|
||||
// signer pure and easy to test against published SigV4 reference vectors.
|
||||
pub struct SignRequest {
|
||||
pub:
|
||||
method string @[required] // GET, PUT, POST, DELETE, HEAD
|
||||
path string @[required] // canonical URI (already URI-encoded, kept as-is)
|
||||
query string // canonical query string (sorted, encoded; no leading '?')
|
||||
payload_hash string // hex SHA-256 of body or `unsigned_payload`
|
||||
extra_headers map[string]string // lowercase keys, raw values — will be added to canonical/signed set
|
||||
sign_time time.Time // when omitted (Time{}) we use time.utc()
|
||||
}
|
||||
|
||||
// SignedRequest is the output of header signing.
|
||||
pub struct SignedRequest {
|
||||
pub:
|
||||
method string // canonical HTTP method that was signed (GET, PUT, …)
|
||||
url string // scheme://host<extra_path>/<bucket>/<key>?<query>
|
||||
host string // host[:port], suitable for the Host header
|
||||
amz_date string // YYYYMMDDTHHMMSSZ
|
||||
authorization string // ready-to-send Authorization header value
|
||||
headers map[string]string // ALL headers that MUST be sent for the signature to verify
|
||||
}
|
||||
|
||||
// sign_request builds the SigV4 Authorization header for an HTTP request.
|
||||
//
|
||||
// The returned `SignedRequest.headers` is the full set the caller must send
|
||||
// (excluding `Content-Length`, which the HTTP client adds itself). Adding,
|
||||
// removing, or mutating a header after signing will break the signature.
|
||||
pub fn sign_request(creds Credentials, req SignRequest) !SignedRequest {
|
||||
creds.validate()!
|
||||
method := normalize_method(req.method) or {
|
||||
return new_error('InvalidMethod',
|
||||
'Method must be GET, PUT, POST, DELETE or HEAD; got: ${req.method}')
|
||||
}
|
||||
host := canonical_host(creds, '')
|
||||
if host == '' {
|
||||
return new_error('InvalidEndpoint',
|
||||
'No endpoint and no bucket provided — cannot determine host')
|
||||
}
|
||||
region := creds.resolved_region()
|
||||
now := if req.sign_time.year == 0 { time.utc() } else { req.sign_time }
|
||||
amz_date := format_amz_date(now)
|
||||
amz_day := amz_date[..8]
|
||||
payload := if req.payload_hash == '' { unsigned_payload } else { req.payload_hash }
|
||||
|
||||
// Build the headers set. We always sign host + amz date + content sha
|
||||
// because S3 requires them. Reject CRLF before they reach the wire.
|
||||
mut headers := map[string]string{}
|
||||
for k, v in req.extra_headers {
|
||||
if contains_crlf(v) || contains_crlf(k) {
|
||||
return new_error('InvalidHeader',
|
||||
'Header "${k}" contains CR/LF — refused (header injection guard)')
|
||||
}
|
||||
headers[k.to_lower()] = v
|
||||
}
|
||||
headers['host'] = host
|
||||
headers['x-amz-content-sha256'] = payload
|
||||
headers['x-amz-date'] = amz_date
|
||||
if creds.session_token != '' {
|
||||
headers['x-amz-security-token'] = creds.session_token
|
||||
}
|
||||
|
||||
signed_header_names := sorted_keys(headers).join(';')
|
||||
canonical := build_canonical_request(method, req.path, req.query, headers, signed_header_names,
|
||||
payload)
|
||||
credential_scope := '${amz_day}/${region}/${service_name}/aws4_request'
|
||||
string_to_sign := '${algo}\n${amz_date}\n${credential_scope}\n${sha256_hex(canonical.bytes())}'
|
||||
signature := to_hex_lower(hmac_sha256(derive_signing_key(creds.secret_access_key, amz_day,
|
||||
region, service_name), string_to_sign.bytes()))
|
||||
authorization := '${algo} Credential=${creds.access_key_id}/${credential_scope}, SignedHeaders=${signed_header_names}, Signature=${signature}'
|
||||
headers['authorization'] = authorization
|
||||
|
||||
scheme := creds.scheme()
|
||||
mut url := '${scheme}://${host}${req.path}'
|
||||
if req.query != '' {
|
||||
url += '?' + req.query
|
||||
}
|
||||
return SignedRequest{
|
||||
method: method
|
||||
url: url
|
||||
host: host
|
||||
amz_date: amz_date
|
||||
authorization: authorization
|
||||
headers: headers
|
||||
}
|
||||
}
|
||||
|
||||
// PresignRequest describes a presigned URL to generate. The output is a
|
||||
// self-contained URL — no extra headers are required at request time.
|
||||
pub struct PresignRequest {
|
||||
pub:
|
||||
method string @[required]
|
||||
path string @[required] // canonical URI (already URI-encoded)
|
||||
expires_in int = 86400 // 1..604800 seconds (SigV4 hard limit is 7 days)
|
||||
extra_query map[string]string // additional signed query params (e.g. response-content-type, x-amz-acl)
|
||||
sign_time time.Time
|
||||
}
|
||||
|
||||
// presign_url returns a fully-formed `https://...` URL signed via SigV4
|
||||
// query-string parameters. The URL is valid until `now + expires_in`.
|
||||
pub fn presign_url(creds Credentials, req PresignRequest) !string {
|
||||
creds.validate()!
|
||||
method := normalize_method(req.method) or {
|
||||
return new_error('InvalidMethod',
|
||||
'Method must be GET, PUT, POST, DELETE or HEAD; got: ${req.method}')
|
||||
}
|
||||
if req.expires_in < 1 || req.expires_in > 604800 {
|
||||
return new_error('InvalidExpiry', 'expires_in must be between 1 and 604800 seconds')
|
||||
}
|
||||
host := canonical_host(creds, '')
|
||||
if host == '' {
|
||||
return new_error('InvalidEndpoint',
|
||||
'No endpoint and no bucket provided — cannot determine host')
|
||||
}
|
||||
region := creds.resolved_region()
|
||||
now := if req.sign_time.year == 0 { time.utc() } else { req.sign_time }
|
||||
amz_date := format_amz_date(now)
|
||||
amz_day := amz_date[..8]
|
||||
credential := '${creds.access_key_id}/${amz_day}/${region}/${service_name}/aws4_request'
|
||||
|
||||
// Required signed query parameters; we then merge in extras from the caller.
|
||||
mut params := map[string]string{}
|
||||
params['X-Amz-Algorithm'] = algo
|
||||
params['X-Amz-Credential'] = credential
|
||||
params['X-Amz-Date'] = amz_date
|
||||
params['X-Amz-Expires'] = req.expires_in.str()
|
||||
params['X-Amz-SignedHeaders'] = 'host'
|
||||
if creds.session_token != '' {
|
||||
params['X-Amz-Security-Token'] = creds.session_token
|
||||
}
|
||||
for k, v in req.extra_query {
|
||||
if contains_crlf(k) || contains_crlf(v) {
|
||||
return new_error('InvalidQueryParam', 'Query param "${k}" contains CR/LF — refused')
|
||||
}
|
||||
params[k] = v
|
||||
}
|
||||
|
||||
canonical_query := canonical_query_string(params)
|
||||
headers := {
|
||||
'host': host
|
||||
}
|
||||
canonical := build_canonical_request(method, req.path, canonical_query, headers, 'host',
|
||||
unsigned_payload)
|
||||
credential_scope := '${amz_day}/${region}/${service_name}/aws4_request'
|
||||
string_to_sign := '${algo}\n${amz_date}\n${credential_scope}\n${sha256_hex(canonical.bytes())}'
|
||||
signature := to_hex_lower(hmac_sha256(derive_signing_key(creds.secret_access_key, amz_day,
|
||||
region, service_name), string_to_sign.bytes()))
|
||||
scheme := creds.scheme()
|
||||
return '${scheme}://${host}${req.path}?${canonical_query}&X-Amz-Signature=${signature}'
|
||||
}
|
||||
|
||||
// build_canonical_request assembles the canonical request string per
|
||||
// SigV4 §3.2. `path` and `query` MUST already be URI-encoded.
|
||||
pub fn build_canonical_request(method string, path string, query string, headers map[string]string,
|
||||
signed_headers string, payload_hash string) string {
|
||||
sorted := sorted_keys(headers)
|
||||
mut lines := []string{cap: sorted.len}
|
||||
for k in sorted {
|
||||
lines << '${k}:${normalize_header_value(headers[k])}'
|
||||
}
|
||||
return '${method}\n${path}\n${query}\n${lines.join('\n')}\n\n${signed_headers}\n${payload_hash}'
|
||||
}
|
||||
|
||||
// canonical_query_string sorts query params by key (then by value if the same
|
||||
// key appears twice — we don't here) and joins them as `k=v&k=v` with values
|
||||
// already URI-encoded.
|
||||
pub fn canonical_query_string(params map[string]string) string {
|
||||
keys := sorted_keys(params)
|
||||
mut parts := []string{cap: keys.len}
|
||||
for k in keys {
|
||||
parts << '${uri_encode_query(k)}=${uri_encode_query(params[k])}'
|
||||
}
|
||||
return parts.join('&')
|
||||
}
|
||||
|
||||
// derive_signing_key implements the four-step HMAC chain that produces the
|
||||
// SigV4 signing key. The result is cacheable per (secret, date, region,
|
||||
// service) tuple — left to the caller to memoize if needed.
|
||||
pub fn derive_signing_key(secret string, date string, region string, service string) []u8 {
|
||||
k_date := hmac_sha256(('AWS4' + secret).bytes(), date.bytes())
|
||||
k_region := hmac_sha256(k_date, region.bytes())
|
||||
k_service := hmac_sha256(k_region, service.bytes())
|
||||
return hmac_sha256(k_service, 'aws4_request'.bytes())
|
||||
}
|
||||
|
||||
// format_amz_date returns the basic ISO-8601 timestamp used by SigV4
|
||||
// (`YYYYMMDDTHHMMSSZ`). `t` is assumed to be in UTC.
|
||||
pub fn format_amz_date(t time.Time) string {
|
||||
return '${t.year:04d}${t.month:02d}${t.day:02d}T${t.hour:02d}${t.minute:02d}${t.second:02d}Z'
|
||||
}
|
||||
|
||||
// canonical_host resolves the on-the-wire host. Path-style addressing is
|
||||
// the default; virtual-hosted style uses `<bucket>.<endpoint-host>`. If no
|
||||
// endpoint is configured we fall back to `s3.<region>.amazonaws.com` (the
|
||||
// canonical default for clients pointed at AWS S3 itself).
|
||||
pub fn canonical_host(creds Credentials, bucket_override string) string {
|
||||
bucket := if bucket_override != '' { bucket_override } else { creds.bucket }
|
||||
host := creds.host_only()
|
||||
if host == '' {
|
||||
region := creds.resolved_region()
|
||||
if creds.virtual_hosted_style {
|
||||
if bucket == '' {
|
||||
return ''
|
||||
}
|
||||
return '${bucket}.s3.${region}.amazonaws.com'
|
||||
}
|
||||
return 's3.${region}.amazonaws.com'
|
||||
}
|
||||
if creds.virtual_hosted_style && bucket != '' {
|
||||
return '${bucket}.${host}'
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
// normalize_method uppercases and validates the HTTP method.
|
||||
pub fn normalize_method(m string) ?string {
|
||||
up := m.to_upper()
|
||||
return match up {
|
||||
'GET', 'PUT', 'POST', 'DELETE', 'HEAD' { up }
|
||||
else { none }
|
||||
}
|
||||
}
|
||||
|
||||
// normalize_header_value collapses runs of internal whitespace to a single
|
||||
// space and trims leading/trailing whitespace, per SigV4 §3.2.2 step 3.
|
||||
fn normalize_header_value(v string) string {
|
||||
if v == '' {
|
||||
return v
|
||||
}
|
||||
mut out := []u8{cap: v.len}
|
||||
mut prev_space := false
|
||||
mut started := false
|
||||
for b in v.bytes() {
|
||||
if b == ` ` || b == `\t` {
|
||||
if started {
|
||||
prev_space = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if prev_space {
|
||||
out << ` `
|
||||
prev_space = false
|
||||
}
|
||||
out << b
|
||||
started = true
|
||||
}
|
||||
return out.bytestr()
|
||||
}
|
||||
|
||||
// sha256_hex returns the lowercase hex digest of `data`.
|
||||
@[inline]
|
||||
pub fn sha256_hex(data []u8) string {
|
||||
return to_hex_lower(sha256.sum256(data))
|
||||
}
|
||||
|
||||
// hmac_sha256 wraps `crypto.hmac.new` for HMAC-SHA-256 with the V stdlib's
|
||||
// type signature (a hash function that returns `[]u8`).
|
||||
@[inline]
|
||||
pub fn hmac_sha256(key []u8, data []u8) []u8 {
|
||||
return hmac.new(key, data, sha256.sum256, sha256.block_size)
|
||||
}
|
||||
|
||||
// sorted_keys returns the keys of `m` in lexical order.
|
||||
fn sorted_keys(m map[string]string) []string {
|
||||
mut keys := m.keys()
|
||||
keys.sort()
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
import time
|
||||
|
||||
// Test vectors come from the AWS-published Signature V4 reference test
|
||||
// suite ("Examples of the complete version 4 signing process").
|
||||
// These secret/key pairs are the documented example values, not real
|
||||
// credentials.
|
||||
const aws_secret = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
|
||||
const aws_key_id = 'AKIAIOSFODNN7EXAMPLE'
|
||||
|
||||
// SigV4 vector for `s3-get-object` (examplebucket / test.txt with a Range
|
||||
// header). Source: AWS docs (Signature V4 examples).
|
||||
//
|
||||
// GET /test.txt HTTP/1.1
|
||||
// Host: examplebucket.s3.amazonaws.com
|
||||
// Range: bytes=0-9
|
||||
// X-Amz-Date: 20130524T000000Z
|
||||
// X-Amz-Content-SHA256: <empty-sha>
|
||||
fn test_aws_sigv4_get_object_vector() {
|
||||
t := time.parse_iso8601('2013-05-24T00:00:00.000Z') or { panic(err) }
|
||||
creds := Credentials{
|
||||
access_key_id: aws_key_id
|
||||
secret_access_key: aws_secret
|
||||
region: 'us-east-1'
|
||||
endpoint: 'examplebucket.s3.amazonaws.com'
|
||||
virtual_hosted_style: false // host already carries the bucket
|
||||
}
|
||||
req := SignRequest{
|
||||
method: 'GET'
|
||||
path: '/test.txt'
|
||||
query: ''
|
||||
payload_hash: empty_sha256
|
||||
extra_headers: {
|
||||
'range': 'bytes=0-9'
|
||||
}
|
||||
sign_time: t
|
||||
}
|
||||
signed := sign_request(creds, req) or { panic(err) }
|
||||
expected := 'AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41'
|
||||
assert signed.authorization == expected, 'Got:\n${signed.authorization}\nExpected:\n${expected}'
|
||||
}
|
||||
|
||||
// SigV4 vector for `s3-put-object` (examplebucket / test$file.text with body
|
||||
// "Welcome to Amazon S3.").
|
||||
//
|
||||
// PUT /test%24file.text HTTP/1.1
|
||||
// Host: examplebucket.s3.amazonaws.com
|
||||
// Date: Fri, 24 May 2013 00:00:00 GMT
|
||||
// x-amz-date: 20130524T000000Z
|
||||
// x-amz-storage-class: REDUCED_REDUNDANCY
|
||||
fn test_aws_sigv4_put_object_vector() {
|
||||
t := time.parse_iso8601('2013-05-24T00:00:00.000Z') or { panic(err) }
|
||||
creds := Credentials{
|
||||
access_key_id: aws_key_id
|
||||
secret_access_key: aws_secret
|
||||
region: 'us-east-1'
|
||||
endpoint: 'examplebucket.s3.amazonaws.com'
|
||||
virtual_hosted_style: false
|
||||
}
|
||||
body := 'Welcome to Amazon S3.'
|
||||
body_hash := sha256_hex(body.bytes())
|
||||
req := SignRequest{
|
||||
method: 'PUT'
|
||||
path: '/test%24file.text'
|
||||
query: ''
|
||||
payload_hash: body_hash
|
||||
extra_headers: {
|
||||
'date': 'Fri, 24 May 2013 00:00:00 GMT'
|
||||
'x-amz-storage-class': 'REDUCED_REDUNDANCY'
|
||||
}
|
||||
sign_time: t
|
||||
}
|
||||
signed := sign_request(creds, req) or { panic(err) }
|
||||
expected := 'AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd'
|
||||
assert signed.authorization == expected, 'Got:\n${signed.authorization}\nExpected:\n${expected}'
|
||||
}
|
||||
|
||||
// Asserts the canonical request string for a ListObjectsV2 GET. The
|
||||
// canonical request is the most stable interop point — Authorization line
|
||||
// formatting has wobbled across AWS doc revisions, but the canonical hash
|
||||
// is always the same.
|
||||
fn test_aws_sigv4_list_objects_canonical_request() {
|
||||
headers := {
|
||||
'host': 'examplebucket.s3.amazonaws.com'
|
||||
'x-amz-content-sha256': empty_sha256
|
||||
'x-amz-date': '20130524T000000Z'
|
||||
}
|
||||
canonical := build_canonical_request('GET', '/', 'list-type=2&prefix=foo', headers,
|
||||
'host;x-amz-content-sha256;x-amz-date', empty_sha256)
|
||||
expected := 'GET\n/\nlist-type=2&prefix=foo\nhost:examplebucket.s3.amazonaws.com\nx-amz-content-sha256:${empty_sha256}\nx-amz-date:20130524T000000Z\n\nhost;x-amz-content-sha256;x-amz-date\n${empty_sha256}'
|
||||
assert canonical == expected, 'canonical mismatch:\n${canonical}'
|
||||
}
|
||||
|
||||
// Cross-checked manually with openssl on the AWS docs example
|
||||
// (date 20120215, region us-east-1, service iam):
|
||||
//
|
||||
// echo -n 'aws4_request' | openssl dgst -sha256 -mac HMAC -macopt hexkey:<kService>
|
||||
fn test_signing_key_chain() {
|
||||
key := derive_signing_key(aws_secret, '20120215', 'us-east-1', 'iam')
|
||||
assert to_hex_lower(key) == '004aa806e13dae88b9032d9261bcb04c67d023afadd221e6b0d206e1760e0b5e', 'Got: ${to_hex_lower(key)}'
|
||||
}
|
||||
|
||||
fn test_normalize_header_value_collapses_whitespace() {
|
||||
assert normalize_header_value(' hello world ') == 'hello world'
|
||||
assert normalize_header_value('a\tb\tc') == 'a b c'
|
||||
assert normalize_header_value('') == ''
|
||||
}
|
||||
|
||||
fn test_canonical_query_string_sorted_and_encoded() {
|
||||
q := canonical_query_string({
|
||||
'b': '2'
|
||||
'a': '1'
|
||||
'c': 'hello world'
|
||||
})
|
||||
assert q == 'a=1&b=2&c=hello%20world'
|
||||
}
|
||||
|
||||
fn test_format_amz_date() {
|
||||
t := time.parse_iso8601('2024-01-15T12:34:56.000Z') or { panic(err) }
|
||||
assert format_amz_date(t) == '20240115T123456Z'
|
||||
}
|
||||
|
||||
fn test_signer_rejects_unsupported_method() {
|
||||
creds := Credentials{
|
||||
access_key_id: 'KEY'
|
||||
secret_access_key: 'secret'
|
||||
region: 'us-east-1'
|
||||
endpoint: 'examplebucket.s3.amazonaws.com'
|
||||
}
|
||||
if _ := sign_request(creds, SignRequest{ method: 'PATCH', path: '/x', payload_hash: empty_sha256 }) {
|
||||
assert false
|
||||
}
|
||||
}
|
||||
|
||||
fn test_signer_rejects_crlf_in_extra_headers() {
|
||||
creds := Credentials{
|
||||
access_key_id: 'KEY'
|
||||
secret_access_key: 'secret'
|
||||
region: 'us-east-1'
|
||||
endpoint: 'examplebucket.s3.amazonaws.com'
|
||||
}
|
||||
bad := SignRequest{
|
||||
method: 'GET'
|
||||
path: '/foo'
|
||||
payload_hash: empty_sha256
|
||||
extra_headers: {
|
||||
'x-evil': 'value\r\nInjected: yes'
|
||||
}
|
||||
}
|
||||
if _ := sign_request(creds, bad) {
|
||||
assert false, 'expected CRLF rejection'
|
||||
} else {
|
||||
if err is S3Error {
|
||||
assert err.code == 'InvalidHeader' || err.code == 'InvalidCredentials', 'unexpected code: ${err.code}'
|
||||
} else {
|
||||
assert false, 'wrong error type'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn test_presign_rejects_crlf_in_extra_query() {
|
||||
creds := Credentials{
|
||||
access_key_id: 'KEY'
|
||||
secret_access_key: 'secret'
|
||||
region: 'us-east-1'
|
||||
endpoint: 'examplebucket.s3.amazonaws.com'
|
||||
}
|
||||
bad := PresignRequest{
|
||||
method: 'GET'
|
||||
path: '/foo'
|
||||
extra_query: {
|
||||
'X-Evil': 'value\r\nInjected: yes'
|
||||
}
|
||||
}
|
||||
if _ := presign_url(creds, bad) {
|
||||
assert false, 'expected CRLF rejection'
|
||||
} else {
|
||||
assert err is S3Error
|
||||
}
|
||||
}
|
||||
|
||||
fn test_presign_rejects_invalid_expiry() {
|
||||
creds := Credentials{
|
||||
access_key_id: 'KEY'
|
||||
secret_access_key: 'secret'
|
||||
region: 'us-east-1'
|
||||
endpoint: 'examplebucket.s3.amazonaws.com'
|
||||
}
|
||||
if _ := presign_url(creds, PresignRequest{ method: 'GET', path: '/x', expires_in: 0 }) {
|
||||
assert false, 'expires_in=0 must be rejected'
|
||||
}
|
||||
if _ := presign_url(creds, PresignRequest{
|
||||
method: 'GET'
|
||||
path: '/x'
|
||||
expires_in: 700_000
|
||||
})
|
||||
{
|
||||
assert false, 'expires_in > 7 days must be rejected'
|
||||
}
|
||||
}
|
||||
|
||||
fn test_presign_url_contains_required_query_params() {
|
||||
creds := Credentials{
|
||||
access_key_id: 'KEY'
|
||||
secret_access_key: 'secret'
|
||||
region: 'us-east-1'
|
||||
endpoint: 'examplebucket.s3.amazonaws.com'
|
||||
}
|
||||
url := presign_url(creds, PresignRequest{
|
||||
method: 'GET'
|
||||
path: '/foo.txt'
|
||||
expires_in: 300
|
||||
}) or { panic(err) }
|
||||
assert url.contains('X-Amz-Algorithm=AWS4-HMAC-SHA256')
|
||||
assert url.contains('X-Amz-Credential=')
|
||||
assert url.contains('X-Amz-Date=')
|
||||
assert url.contains('X-Amz-Expires=300')
|
||||
assert url.contains('X-Amz-SignedHeaders=host')
|
||||
assert url.contains('X-Amz-Signature=')
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
// Copyright (c) 2019-2026 Alexander Medvednikov. All rights reserved.
|
||||
// Use of this source code is governed by an MIT license
|
||||
// that can be found in the LICENSE file.
|
||||
module s3
|
||||
|
||||
import net.http
|
||||
|
||||
// ACL is the canned Access Control List applied to a stored object or bucket.
|
||||
// Values match the S3 wire protocol.
|
||||
pub enum Acl {
|
||||
unset
|
||||
private
|
||||
public_read
|
||||
public_read_write
|
||||
aws_exec_read
|
||||
authenticated_read
|
||||
bucket_owner_read
|
||||
bucket_owner_full_control
|
||||
log_delivery_write
|
||||
}
|
||||
|
||||
// to_header_value renders the ACL as the canonical S3 string used in headers
|
||||
// and presigned query parameters. `.unset` returns an empty string so callers
|
||||
// can skip the header.
|
||||
pub fn (a Acl) to_header_value() string {
|
||||
return match a {
|
||||
.unset { '' }
|
||||
.private { 'private' }
|
||||
.public_read { 'public-read' }
|
||||
.public_read_write { 'public-read-write' }
|
||||
.aws_exec_read { 'aws-exec-read' }
|
||||
.authenticated_read { 'authenticated-read' }
|
||||
.bucket_owner_read { 'bucket-owner-read' }
|
||||
.bucket_owner_full_control { 'bucket-owner-full-control' }
|
||||
.log_delivery_write { 'log-delivery-write' }
|
||||
}
|
||||
}
|
||||
|
||||
// parse_acl returns the matching Acl from a wire string.
|
||||
// Returns `.unset` when the input is empty or unknown so the caller can ignore it.
|
||||
pub fn parse_acl(s string) Acl {
|
||||
return match s {
|
||||
'private' { Acl.private }
|
||||
'public-read' { Acl.public_read }
|
||||
'public-read-write' { Acl.public_read_write }
|
||||
'aws-exec-read' { Acl.aws_exec_read }
|
||||
'authenticated-read' { Acl.authenticated_read }
|
||||
'bucket-owner-read' { Acl.bucket_owner_read }
|
||||
'bucket-owner-full-control' { Acl.bucket_owner_full_control }
|
||||
'log-delivery-write' { Acl.log_delivery_write }
|
||||
else { Acl.unset }
|
||||
}
|
||||
}
|
||||
|
||||
// StorageClass enumerates the standard S3 storage tiers. `.unset` means the
|
||||
// server default (typically STANDARD) and produces no `x-amz-storage-class`
|
||||
// header. Providers vary on which tiers they actually honour.
|
||||
pub enum StorageClass {
|
||||
unset
|
||||
standard
|
||||
deep_archive
|
||||
express_onezone
|
||||
glacier
|
||||
glacier_ir
|
||||
intelligent_tiering
|
||||
onezone_ia
|
||||
outposts
|
||||
reduced_redundancy
|
||||
snow
|
||||
standard_ia
|
||||
}
|
||||
|
||||
// to_header_value renders the storage class as the on-wire string.
|
||||
pub fn (sc StorageClass) to_header_value() string {
|
||||
return match sc {
|
||||
.unset { '' }
|
||||
.standard { 'STANDARD' }
|
||||
.deep_archive { 'DEEP_ARCHIVE' }
|
||||
.express_onezone { 'EXPRESS_ONEZONE' }
|
||||
.glacier { 'GLACIER' }
|
||||
.glacier_ir { 'GLACIER_IR' }
|
||||
.intelligent_tiering { 'INTELLIGENT_TIERING' }
|
||||
.onezone_ia { 'ONEZONE_IA' }
|
||||
.outposts { 'OUTPOSTS' }
|
||||
.reduced_redundancy { 'REDUCED_REDUNDANCY' }
|
||||
.snow { 'SNOW' }
|
||||
.standard_ia { 'STANDARD_IA' }
|
||||
}
|
||||
}
|
||||
|
||||
// Stat is the result of HEAD-ing an object.
|
||||
pub struct Stat {
|
||||
pub:
|
||||
size i64 // Content-Length in bytes
|
||||
last_modified string // RFC 1123 date as returned by S3
|
||||
etag string // unquoted ETag (server returns it wrapped in quotes; we strip them)
|
||||
content_type string // MIME type
|
||||
}
|
||||
|
||||
// Owner identifies an S3 object owner — only populated when `fetch_owner` is true.
|
||||
pub struct Owner {
|
||||
pub:
|
||||
id string
|
||||
display_name string
|
||||
}
|
||||
|
||||
// ObjectInfo is one entry from a ListObjectsV2 response.
|
||||
pub struct ObjectInfo {
|
||||
pub:
|
||||
key string
|
||||
last_modified string
|
||||
etag string
|
||||
size i64
|
||||
storage_class string
|
||||
owner ?Owner
|
||||
}
|
||||
|
||||
// CommonPrefix represents a "directory-like" prefix in a list result when a
|
||||
// delimiter is provided.
|
||||
pub struct CommonPrefix {
|
||||
pub:
|
||||
prefix string
|
||||
}
|
||||
|
||||
// ListResult aggregates a ListObjectsV2 response.
|
||||
// NextContinuationToken should be passed back as `continuation_token` to fetch the next page.
|
||||
pub struct ListResult {
|
||||
pub:
|
||||
name string
|
||||
prefix string
|
||||
delimiter string
|
||||
start_after string
|
||||
max_keys int
|
||||
key_count int
|
||||
is_truncated bool
|
||||
continuation_token string
|
||||
next_continuation_token string
|
||||
objects []ObjectInfo
|
||||
common_prefixes []CommonPrefix
|
||||
}
|
||||
|
||||
// ListOptions configures a ListObjectsV2 call. `bucket` overrides the client's
|
||||
// default; leave empty to use the client's bound bucket.
|
||||
@[params]
|
||||
pub struct ListOptions {
|
||||
pub:
|
||||
bucket string
|
||||
prefix string
|
||||
continuation_token string
|
||||
delimiter string
|
||||
max_keys int // 0 means default (server picks 1000)
|
||||
start_after string
|
||||
encoding_type string // 'url' or empty
|
||||
fetch_owner bool
|
||||
}
|
||||
|
||||
// PresignOptions controls presigned URL generation.
|
||||
@[params]
|
||||
pub struct PresignOptions {
|
||||
pub:
|
||||
bucket string // overrides the client's default bucket for this call
|
||||
method http.Method = .get
|
||||
expires_in int = 86400 // seconds, 1..604800
|
||||
acl Acl
|
||||
storage_class StorageClass
|
||||
content_type string
|
||||
content_disposition string
|
||||
request_payer bool
|
||||
}
|
||||
Reference in new issue
Block a user