1236 lines
36 KiB
V
1236 lines
36 KiB
V
// Copyright (c) 2019-2024 Alexander Medvednikov. All rights reserved.
|
|
// Use of this source code is governed by an MIT license
|
|
// that can be found in the LICENSE file.
|
|
module http
|
|
|
|
import io
|
|
import net
|
|
import net.urllib
|
|
import rand
|
|
import strings
|
|
import time
|
|
|
|
pub type RequestRedirectFn = fn (request &Request, nredirects int, new_url string) !
|
|
|
|
pub type RequestProgressFn = fn (request &Request, chunk []u8, read_so_far u64) !
|
|
|
|
pub type RequestProgressBodyFn = fn (request &Request, chunk []u8, body_read_so_far u64, body_expected_size u64, status_code int) !
|
|
|
|
pub type RequestFinishFn = fn (request &Request, final_size u64) !
|
|
|
|
// Request holds information about an HTTP request (either received by
|
|
// a server or to be sent by a client)
|
|
pub struct Request {
|
|
mut:
|
|
cookies map[string]string
|
|
pub mut:
|
|
version Version = .v1_1
|
|
method Method = .get
|
|
header Header
|
|
host string
|
|
data string
|
|
url string
|
|
user_agent string = 'v.http'
|
|
verbose bool
|
|
user_ptr voidptr
|
|
proxy &HttpProxy = unsafe { nil }
|
|
// NOT implemented for ssl connections
|
|
// time = -1 for no timeout
|
|
read_timeout i64 = 30 * time.second
|
|
write_timeout i64 = 30 * time.second
|
|
|
|
validate bool // when true, certificate failures will stop further processing
|
|
verify string
|
|
cert string
|
|
cert_key string
|
|
in_memory_verification bool // if true, verify, cert, and cert_key are read from memory, not from a file
|
|
allow_redirect bool = true // whether to allow redirect
|
|
max_retries int = 5 // maximum number of retries required when an underlying socket error occurs
|
|
enable_http2 bool = true // when true (the default) and the URL is https, advertise ALPN `h2, http/1.1` and use HTTP/2 if the server selects it; set to false to force HTTP/1.1. Ignored for plain http://, and for the Windows SChannel backend which has no ALPN yet (see vlang/v#27383). on_progress / on_progress_body / stop_copying_limit / stop_receiving_limit are honored on the HTTP/2 path; on_progress fires per DATA frame payload rather than per raw network read.
|
|
disable_connection_reuse bool // opt out of the shared connection pool: open a fresh connection for this request, send `Connection: close`, and close the connection after the response (the pre-pooling behavior)
|
|
// callbacks to allow custom reporting code to run, while the request is running, and to implement streaming
|
|
on_redirect RequestRedirectFn = unsafe { nil }
|
|
on_progress RequestProgressFn = unsafe { nil }
|
|
on_progress_body RequestProgressBodyFn = unsafe { nil }
|
|
on_finish RequestFinishFn = unsafe { nil }
|
|
|
|
stop_copying_limit i64 = -1 // after this many bytes are received, stop copying to the response. Note that on_progress and on_progress_body callbacks, will continue to fire normally, until the full response is read, which allows you to implement streaming downloads, without keeping the whole big response in memory
|
|
stop_receiving_limit i64 = -1 // after this many bytes are received, break out of the loop that reads the response, effectively stopping the request early. No more on_progress callbacks will be fired. The on_finish callback will fire.
|
|
}
|
|
|
|
@[manualfree]
|
|
fn (mut req Request) free() {
|
|
mut freed_ptrs := map[u64]bool{}
|
|
unsafe {
|
|
req.cookies.free()
|
|
for i := 0; i < req.header.cur_pos; i++ {
|
|
mut key := req.header.data[i].key
|
|
mut value := req.header.data[i].value
|
|
key_ptr := u64(usize(key.str))
|
|
if key_ptr !in freed_ptrs {
|
|
key.free()
|
|
freed_ptrs[key_ptr] = true
|
|
}
|
|
value_ptr := u64(usize(value.str))
|
|
if value_ptr !in freed_ptrs {
|
|
value.free()
|
|
freed_ptrs[value_ptr] = true
|
|
}
|
|
}
|
|
mut host := req.host
|
|
host_ptr := u64(usize(host.str))
|
|
if host_ptr !in freed_ptrs {
|
|
host.free()
|
|
freed_ptrs[host_ptr] = true
|
|
}
|
|
mut data := req.data
|
|
data_ptr := u64(usize(data.str))
|
|
if data_ptr !in freed_ptrs {
|
|
data.free()
|
|
freed_ptrs[data_ptr] = true
|
|
}
|
|
mut url := req.url
|
|
url_ptr := u64(usize(url.str))
|
|
if url_ptr !in freed_ptrs {
|
|
url.free()
|
|
freed_ptrs[url_ptr] = true
|
|
}
|
|
mut user_agent := req.user_agent
|
|
user_agent_ptr := u64(usize(user_agent.str))
|
|
if user_agent_ptr !in freed_ptrs {
|
|
user_agent.free()
|
|
freed_ptrs[user_agent_ptr] = true
|
|
}
|
|
mut verify := req.verify
|
|
verify_ptr := u64(usize(verify.str))
|
|
if verify_ptr !in freed_ptrs {
|
|
verify.free()
|
|
freed_ptrs[verify_ptr] = true
|
|
}
|
|
mut cert := req.cert
|
|
cert_ptr := u64(usize(cert.str))
|
|
if cert_ptr !in freed_ptrs {
|
|
cert.free()
|
|
freed_ptrs[cert_ptr] = true
|
|
}
|
|
mut cert_key := req.cert_key
|
|
cert_key_ptr := u64(usize(cert_key.str))
|
|
if cert_key_ptr !in freed_ptrs {
|
|
cert_key.free()
|
|
freed_ptrs[cert_key_ptr] = true
|
|
}
|
|
}
|
|
}
|
|
|
|
// reset frees request-owned data and resets the request to default values.
|
|
@[manualfree]
|
|
pub fn (mut req Request) reset() {
|
|
req.free()
|
|
req = Request{}
|
|
}
|
|
|
|
// add_header adds the key and value of an HTTP request header
|
|
// To add a custom header, use add_custom_header
|
|
pub fn (mut req Request) add_header(key CommonHeader, val string) {
|
|
req.header.add(key, val)
|
|
}
|
|
|
|
// add_custom_header adds the key and value of an HTTP request header
|
|
// This method may fail if the key contains characters that are not permitted
|
|
pub fn (mut req Request) add_custom_header(key string, val string) ! {
|
|
return req.header.add_custom(key, val)
|
|
}
|
|
|
|
// add_cookie adds a cookie to the request.
|
|
pub fn (mut req Request) add_cookie(c Cookie) {
|
|
req.cookies[c.name] = c.value
|
|
}
|
|
|
|
// cookie returns the named cookie provided in the request or `none` if not found.
|
|
// If multiple cookies match the given name, only one cookie will be returned.
|
|
pub fn (req &Request) cookie(name string) ?Cookie {
|
|
// TODO(alex) this should work once Cookie is used
|
|
// return req.cookies[name] or { none }
|
|
|
|
if value := req.cookies[name] {
|
|
return Cookie{
|
|
name: name
|
|
value: value
|
|
}
|
|
}
|
|
return none
|
|
}
|
|
|
|
// do will send the HTTP request and returns `http.Response` as soon as the response is received
|
|
pub fn (req &Request) do() !Response {
|
|
mut rurl := urllib.parse(req.url) or { return error('http.Request.do: invalid url ${req.url}') }
|
|
mut resp := Response{}
|
|
mut method := req.method
|
|
mut data := req.data
|
|
mut header := req.header
|
|
mut nredirects := 0
|
|
for {
|
|
if nredirects == max_redirects {
|
|
return error('http.request.do: maximum number of redirects reached (${max_redirects})')
|
|
}
|
|
qresp := req.method_and_url_to_response(method, rurl, data, header)!
|
|
resp = qresp
|
|
if !req.allow_redirect {
|
|
break
|
|
}
|
|
status := resp.status()
|
|
if status !in [.moved_permanently, .found, .see_other, .temporary_redirect,
|
|
.permanent_redirect] {
|
|
break
|
|
}
|
|
// follow any redirects
|
|
mut redirect_url := resp.header.get(.location) or { '' }
|
|
mut qrurl := urllib.URL{}
|
|
if redirect_url.len > 0 && redirect_url[0] == `/` {
|
|
qrurl = rurl.parse(redirect_url) or {
|
|
return error('http.request.do: invalid URL in redirect "${redirect_url}"')
|
|
}
|
|
redirect_url = qrurl.str()
|
|
} else {
|
|
qrurl = urllib.parse(redirect_url) or {
|
|
return error('http.request.do: invalid URL in redirect "${redirect_url}"')
|
|
}
|
|
}
|
|
if req.on_redirect != unsafe { nil } {
|
|
req.on_redirect(req, nredirects, redirect_url)!
|
|
}
|
|
method, data, header = redirected_request_parts(method, status, data, header)
|
|
rurl = qrurl
|
|
nredirects++
|
|
}
|
|
return resp
|
|
}
|
|
|
|
fn redirected_request_parts(method Method, status Status, data string, header Header) (Method, string, Header) {
|
|
next_method := redirected_method(method, status)
|
|
if next_method == method {
|
|
return method, data, header
|
|
}
|
|
mut next_header := header
|
|
next_header.delete(.content_length)
|
|
next_header.delete(.content_type)
|
|
next_header.delete(.transfer_encoding)
|
|
return next_method, '', next_header
|
|
}
|
|
|
|
fn redirected_method(method Method, status Status) Method {
|
|
return match status {
|
|
.see_other {
|
|
if method == Method.head {
|
|
Method.head
|
|
} else {
|
|
Method.get
|
|
}
|
|
}
|
|
.moved_permanently, .found {
|
|
if method == Method.post {
|
|
Method.get
|
|
} else {
|
|
method
|
|
}
|
|
}
|
|
else {
|
|
method
|
|
}
|
|
}
|
|
}
|
|
|
|
fn (req &Request) method_and_url_to_response(method Method, url urllib.URL, data string, header Header) !Response {
|
|
host_name := url.hostname()
|
|
scheme := url.scheme
|
|
p := url.escaped_path().trim_left('/')
|
|
path := if url.query().len > 0 { '/${p}?${url.query().encode()}' } else { '/${p}' }
|
|
mut nport := url.port().int()
|
|
if nport == 0 {
|
|
if scheme == 'http' {
|
|
nport = 80
|
|
}
|
|
if scheme == 'https' {
|
|
nport = 443
|
|
}
|
|
}
|
|
// println('fetch ${method}, ${scheme}, ${host_name}, ${nport}, ${path} ')
|
|
if req.proxy == unsafe { nil } && !req.disable_connection_reuse
|
|
&& (scheme == 'http' || scheme == 'https') {
|
|
// Default path: route through the shared connection-pooling Transport,
|
|
// which reuses keep-alive connections across requests to the same origin.
|
|
mut transport := default_transport()
|
|
for i in 0 .. req.max_retries {
|
|
res := transport.round_trip(req, method, scheme, host_name, nport, path, data, header) or {
|
|
if i == req.max_retries - 1 || is_no_need_retry_error(err.code()) {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
return res
|
|
}
|
|
return error('http.request.method_and_url_to_response: exhausted retries')
|
|
}
|
|
if scheme == 'https' && req.proxy == unsafe { nil } {
|
|
// println('ssl_do( ${nport}, ${method}, ${host_name}, ${path} )')
|
|
for i in 0 .. req.max_retries {
|
|
res := req.ssl_do(nport, method, host_name, path, data, header) or {
|
|
if i == req.max_retries - 1 || is_no_need_retry_error(err.code()) {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
return res
|
|
}
|
|
} else if scheme == 'http' && req.proxy == unsafe { nil } {
|
|
// println('http_do( ${nport}, ${method}, ${host_name}, ${path} )')
|
|
for i in 0 .. req.max_retries {
|
|
res := req.http_do('${host_name}:${nport}', method, path, data, header) or {
|
|
if i == req.max_retries - 1 || is_no_need_retry_error(err.code()) {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
return res
|
|
}
|
|
} else if req.proxy != unsafe { nil } {
|
|
for i in 0 .. req.max_retries {
|
|
res := req.proxy.http_do(url, method, path, req, data, header) or {
|
|
if i == req.max_retries - 1 || is_no_need_retry_error(err.code()) {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
return res
|
|
}
|
|
}
|
|
return error('http.request.method_and_url_to_response: unsupported scheme: "${scheme}"')
|
|
}
|
|
|
|
fn (req &Request) build_request_headers(method Method, host_name string, port int, path string) string {
|
|
return req.build_request_headers_with(method, host_name, port, path, req.data, req.header)
|
|
}
|
|
|
|
fn (req &Request) build_request_headers_with(method Method, host_name string, port int, path string, data string, header Header) string {
|
|
return req.build_request_headers_opts(method, host_name, port, path, data, header, true)
|
|
}
|
|
|
|
// build_request_headers_opts builds the raw HTTP/1.x request. With
|
|
// `connection_close` true it appends `Connection: close` (the historical
|
|
// one-shot behavior); the pooled keep-alive path passes false and emits no
|
|
// Connection header, leaving the HTTP/1.1 default (keep-alive) in effect and
|
|
// respecting any Connection header the caller set themselves.
|
|
fn (req &Request) build_request_headers_opts(method Method, host_name string, port int, path string, data string, header Header, connection_close bool) string {
|
|
mut sb := strings.new_builder(4096)
|
|
version := if req.version == .unknown { Version.v1_1 } else { req.version }
|
|
sb.write_string(method.str())
|
|
sb.write_string(' ')
|
|
sb.write_string(path)
|
|
sb.write_string(' ')
|
|
sb.write_string(version.str())
|
|
sb.write_string('\r\n')
|
|
if !header.contains(.host) {
|
|
sb.write_string('Host: ')
|
|
if port != 80 && port != 443 && port != 0 {
|
|
sb.write_string('${host_name}:${port}')
|
|
} else {
|
|
sb.write_string(host_name)
|
|
}
|
|
sb.write_string('\r\n')
|
|
}
|
|
if !header.contains(.user_agent) {
|
|
ua := req.user_agent
|
|
sb.write_string('User-Agent: ')
|
|
sb.write_string(ua)
|
|
sb.write_string('\r\n')
|
|
}
|
|
if !header.contains(.content_length) {
|
|
// Write Content-Length: 0 even if there's no content, since some APIs
|
|
// stop working without this header.
|
|
sb.write_string('Content-Length: ')
|
|
sb.write_string(data.len.str())
|
|
sb.write_string('\r\n')
|
|
}
|
|
chkey := CommonHeader.cookie.str()
|
|
for key in header.keys() {
|
|
if key == chkey {
|
|
continue
|
|
}
|
|
val := header.custom_values(key).join('; ')
|
|
sb.write_string(key)
|
|
sb.write_string(': ')
|
|
sb.write_string(val)
|
|
sb.write_string('\r\n')
|
|
}
|
|
sb.write_string(req.build_request_cookies_header_with_header(header))
|
|
if connection_close {
|
|
sb.write_string('Connection: close\r\n')
|
|
}
|
|
sb.write_string('\r\n')
|
|
sb.write_string(data)
|
|
return sb.str()
|
|
}
|
|
|
|
fn (req &Request) build_request_cookies_header() string {
|
|
return req.build_request_cookies_header_with_header(req.header)
|
|
}
|
|
|
|
fn (req &Request) build_request_cookies_header_with_header(header Header) string {
|
|
if req.cookies.len < 1 {
|
|
return ''
|
|
}
|
|
mut sb_cookie := strings.new_builder(1024)
|
|
hvcookies := header.values(.cookie)
|
|
total_cookies := req.cookies.len + hvcookies.len
|
|
sb_cookie.write_string('Cookie: ')
|
|
mut idx := 0
|
|
for key, val in req.cookies {
|
|
sb_cookie.write_string(key)
|
|
sb_cookie.write_string('=')
|
|
sb_cookie.write_string(val)
|
|
if idx < total_cookies - 1 {
|
|
sb_cookie.write_string('; ')
|
|
}
|
|
idx++
|
|
}
|
|
for c in hvcookies {
|
|
sb_cookie.write_string(c)
|
|
if idx < total_cookies - 1 {
|
|
sb_cookie.write_string('; ')
|
|
}
|
|
idx++
|
|
}
|
|
sb_cookie.write_string('\r\n')
|
|
return sb_cookie.str()
|
|
}
|
|
|
|
fn (req &Request) http_do(host string, method Method, path string, data string, header Header) !Response {
|
|
host_name, port := net.split_address(host)!
|
|
s := req.build_request_headers_with(method, host_name, port, path, data, header)
|
|
mut client := net.dial_tcp(host)!
|
|
client.set_read_timeout(req.read_timeout)
|
|
client.set_write_timeout(req.write_timeout)
|
|
// TODO: this really needs to be exposed somehow
|
|
client.write(s.bytes())!
|
|
$if trace_http_request ? {
|
|
eprint('> ')
|
|
eprint(s)
|
|
eprintln('')
|
|
}
|
|
response_data := req.read_all_from_client_connection(client)!
|
|
client.close()!
|
|
response_text := response_data.data.bytestr()
|
|
$if trace_http_response ? {
|
|
eprint('< ')
|
|
eprint(response_text)
|
|
eprintln('')
|
|
}
|
|
if req.on_finish != unsafe { nil } {
|
|
req.on_finish(req, u64(response_text.len))!
|
|
}
|
|
return parse_received_response(response_text, response_data.info)
|
|
}
|
|
|
|
// h1_exchange_tcp sends an already-built HTTP/1.x request over an open TCP
|
|
// connection and reads one response, leaving the connection open. The bool
|
|
// result reports whether the response was precisely framed, so the connection
|
|
// can safely carry another request (see ReceivedResponseInfo.reusable).
|
|
fn (req &Request) h1_exchange_tcp(mut client net.TcpConn, raw string) !(Response, bool) {
|
|
client.write(raw.bytes()) or {
|
|
return error('http.transport: connection write failed: ${err.msg()}')
|
|
}
|
|
response_data := req.read_all_from_client_connection(client)!
|
|
response_text := response_data.data.bytestr()
|
|
$if trace_http_response ? {
|
|
eprint('< ')
|
|
eprint(response_text)
|
|
eprintln('')
|
|
}
|
|
if req.on_finish != unsafe { nil } {
|
|
req.on_finish(req, u64(response_text.len))!
|
|
}
|
|
resp := parse_received_response(response_text, response_data.info)!
|
|
return resp, response_data.info.reusable
|
|
}
|
|
|
|
// abstract over reading the whole content from TCP or SSL connections:
|
|
type FnReceiveChunk = fn (con voidptr, buf &u8, bufsize int) !int
|
|
|
|
enum ChunkedBodyTrackerState {
|
|
chunk_size
|
|
chunk_data
|
|
chunk_data_crlf_start
|
|
chunk_data_crlf_end
|
|
trailer_line
|
|
}
|
|
|
|
struct ChunkedBodyTracker {
|
|
mut:
|
|
state ChunkedBodyTrackerState = .chunk_size
|
|
line_buf []u8
|
|
chunk_left u64
|
|
decoded_len u64
|
|
complete bool
|
|
invalid bool
|
|
}
|
|
|
|
fn (mut tracker ChunkedBodyTracker) advance(data []u8, mut decoded []u8) bool {
|
|
if tracker.complete || tracker.invalid || data.len == 0 {
|
|
return tracker.complete
|
|
}
|
|
mut i := 0
|
|
for i < data.len {
|
|
match tracker.state {
|
|
.chunk_size {
|
|
ch := data[i]
|
|
i++
|
|
if ch == `\r` {
|
|
continue
|
|
}
|
|
if ch != `\n` {
|
|
tracker.line_buf << ch
|
|
continue
|
|
}
|
|
chunk_size := parse_chunked_size_line(tracker.line_buf) or {
|
|
tracker.invalid = true
|
|
return false
|
|
}
|
|
tracker.line_buf.clear()
|
|
if chunk_size == 0 {
|
|
tracker.state = .trailer_line
|
|
continue
|
|
}
|
|
tracker.chunk_left = chunk_size
|
|
tracker.state = .chunk_data
|
|
}
|
|
.chunk_data {
|
|
available := data.len - i
|
|
if tracker.chunk_left < u64(available) {
|
|
decoded << data[i..i + int(tracker.chunk_left)]
|
|
tracker.decoded_len += tracker.chunk_left
|
|
i += int(tracker.chunk_left)
|
|
tracker.chunk_left = 0
|
|
} else {
|
|
decoded << data[i..]
|
|
tracker.decoded_len += u64(available)
|
|
tracker.chunk_left -= u64(available)
|
|
i = data.len
|
|
}
|
|
if tracker.chunk_left == 0 {
|
|
tracker.state = .chunk_data_crlf_start
|
|
}
|
|
}
|
|
.chunk_data_crlf_start {
|
|
if data[i] != `\r` {
|
|
tracker.invalid = true
|
|
return false
|
|
}
|
|
i++
|
|
tracker.state = .chunk_data_crlf_end
|
|
}
|
|
.chunk_data_crlf_end {
|
|
if data[i] != `\n` {
|
|
tracker.invalid = true
|
|
return false
|
|
}
|
|
i++
|
|
tracker.state = .chunk_size
|
|
}
|
|
.trailer_line {
|
|
ch := data[i]
|
|
i++
|
|
if ch == `\r` {
|
|
continue
|
|
}
|
|
if ch != `\n` {
|
|
tracker.line_buf << ch
|
|
continue
|
|
}
|
|
if tracker.line_buf.len == 0 {
|
|
tracker.complete = true
|
|
return true
|
|
}
|
|
tracker.line_buf.clear()
|
|
}
|
|
}
|
|
}
|
|
return tracker.complete
|
|
}
|
|
|
|
fn parse_chunked_size_line(line []u8) !u64 {
|
|
mut size := u64(0)
|
|
mut has_digit := false
|
|
for ch in line {
|
|
if ch == `;` {
|
|
break
|
|
}
|
|
if !ch.is_hex_digit() {
|
|
return error('invalid chunk size')
|
|
}
|
|
has_digit = true
|
|
size = (size << 4) | u64(chunked_hex_value(ch))
|
|
}
|
|
if !has_digit {
|
|
return error('invalid chunk size')
|
|
}
|
|
return size
|
|
}
|
|
|
|
fn chunked_hex_value(ch u8) u8 {
|
|
if `0` <= ch && ch <= `9` {
|
|
return ch - `0`
|
|
}
|
|
if `a` <= ch && ch <= `f` {
|
|
return ch - `a` + 10
|
|
}
|
|
if `A` <= ch && ch <= `F` {
|
|
return ch - `A` + 10
|
|
}
|
|
return 0
|
|
}
|
|
|
|
struct ReceivedResponseInfo {
|
|
headers_end int = -1
|
|
is_chunked_transfer bool
|
|
has_truncated_body bool
|
|
// reusable is true when the read loop terminated via precise response
|
|
// framing (Content-Length satisfied, chunked transfer complete, or a
|
|
// no-body status), meaning the connection holds no response leftovers and
|
|
// can safely carry another request. EOF-terminated or truncated reads
|
|
// leave it false.
|
|
reusable bool
|
|
}
|
|
|
|
fn parse_received_response(response_text string, info ReceivedResponseInfo) !Response {
|
|
if info.is_chunked_transfer && info.has_truncated_body && info.headers_end > 0
|
|
&& info.headers_end <= response_text.len {
|
|
return parse_response(response_text[..info.headers_end])
|
|
}
|
|
return parse_response(response_text)
|
|
}
|
|
|
|
// response_has_no_body returns true when the HTTP method or status code
|
|
// guarantees that no response body is sent (HEAD requests, 1xx informational,
|
|
// 204 No Content, 304 Not Modified). For these, a `Content-Length` header
|
|
// describes the body that *would* have been sent for a GET, so it must not
|
|
// drive read termination or completion validation. (RFC 7230 §3.3.3)
|
|
fn response_has_no_body(method Method, status_code int) bool {
|
|
if method == .head {
|
|
return true
|
|
}
|
|
return status_code in [101, 102, 103, 204, 304]
|
|
}
|
|
|
|
fn validate_received_response_completion(has_content_length bool, expected_size u64, body_so_far u64, is_chunked_transfer bool, chunked_complete bool) ! {
|
|
if has_content_length && body_so_far < expected_size {
|
|
return error('http.request: response body ended early: received ${body_so_far} of ${expected_size} bytes')
|
|
}
|
|
if is_chunked_transfer && !chunked_complete {
|
|
return error('http.request: incomplete chunked response')
|
|
}
|
|
}
|
|
|
|
fn (req &Request) receive_all_data_from_cb_in_builder(mut content strings.Builder, con voidptr, receive_chunk_cb FnReceiveChunk) !ReceivedResponseInfo {
|
|
mut buff := [bufsize]u8{}
|
|
bp := unsafe { &buff[0] }
|
|
mut readcounter := 0
|
|
mut body_pos := u64(0)
|
|
mut headers_end := -1
|
|
mut expected_size := u64(0)
|
|
mut has_content_length := false
|
|
mut is_chunked_transfer := false
|
|
mut chunked_body_tracker := ChunkedBodyTracker{}
|
|
mut header_buf := strings.new_builder(1024)
|
|
mut old_len := u64(0)
|
|
mut new_len := u64(0)
|
|
mut status_code := -1
|
|
mut has_truncated_body := false
|
|
mut framed_complete := false
|
|
for {
|
|
readcounter++
|
|
len := receive_chunk_cb(con, bp, bufsize) or {
|
|
if err is io.Eof {
|
|
body_so_far := if headers_end >= 0 && old_len > body_pos {
|
|
old_len - body_pos
|
|
} else {
|
|
u64(0)
|
|
}
|
|
if !response_has_no_body(req.method, status_code) {
|
|
validate_received_response_completion(has_content_length, expected_size,
|
|
body_so_far, is_chunked_transfer, chunked_body_tracker.complete)!
|
|
}
|
|
break
|
|
}
|
|
return err
|
|
}
|
|
$if debug_http ? {
|
|
eprintln('ssl_do, read ${readcounter:4d} | len: ${len}')
|
|
eprintln('-'.repeat(20))
|
|
eprintln(unsafe { tos(bp, len) })
|
|
eprintln('-'.repeat(20))
|
|
}
|
|
if len <= 0 {
|
|
body_so_far := if headers_end >= 0 && old_len > body_pos {
|
|
old_len - body_pos
|
|
} else {
|
|
u64(0)
|
|
}
|
|
if !response_has_no_body(req.method, status_code) {
|
|
validate_received_response_completion(has_content_length, expected_size,
|
|
body_so_far, is_chunked_transfer, chunked_body_tracker.complete)!
|
|
}
|
|
break
|
|
}
|
|
new_len = old_len + u64(len)
|
|
// Note: `schunk` and `bchunk` are used as convenient stack located views to the currently filled part of `buff`:
|
|
schunk := unsafe { bp.vstring_literal_with_len(len) }
|
|
mut bchunk := unsafe { bp.vbytes(len) }
|
|
if readcounter == 1 {
|
|
http_line := schunk.all_before('\r\n')
|
|
status_code = http_line.all_after(' ').all_before(' ').int()
|
|
}
|
|
if req.on_progress != unsafe { nil } {
|
|
req.on_progress(req, bchunk, u64(new_len))!
|
|
}
|
|
if headers_end < 0 {
|
|
unsafe { header_buf.write_ptr(bp, len) }
|
|
if header_buf.len >= headers_body_boundary.len {
|
|
header_str := header_buf.bytestr()
|
|
hidx := header_str.index_(headers_body_boundary)
|
|
if hidx >= 0 {
|
|
headers_end = hidx + headers_body_boundary.len
|
|
body_pos = u64(headers_end)
|
|
for line in header_str[..hidx].split('\r\n') {
|
|
if line.len == 0 {
|
|
continue
|
|
}
|
|
low := line.to_lower()
|
|
if low.starts_with('content-length:') {
|
|
raw_cl := line.all_after(':').trim_space()
|
|
mut valid_cl := raw_cl.len > 0
|
|
for ch in raw_cl {
|
|
if !ch.is_digit() {
|
|
valid_cl = false
|
|
break
|
|
}
|
|
}
|
|
if valid_cl {
|
|
expected_size = raw_cl.u64()
|
|
has_content_length = true
|
|
}
|
|
} else if low.starts_with('transfer-encoding:')
|
|
&& has_header_token(line.all_after(':').trim_space(), 'chunked') {
|
|
is_chunked_transfer = true
|
|
}
|
|
}
|
|
if is_chunked_transfer {
|
|
has_content_length = false
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if headers_end >= 0 && old_len < u64(headers_end) {
|
|
header_bytes_in_chunk := int(u64(headers_end) - old_len)
|
|
if header_bytes_in_chunk >= len {
|
|
bchunk = []u8{}
|
|
} else {
|
|
bchunk = unsafe { (&u8(bchunk.data) + header_bytes_in_chunk).vbytes(len - header_bytes_in_chunk) }
|
|
}
|
|
}
|
|
mut body_so_far := u64(0)
|
|
if headers_end >= 0 && new_len > body_pos {
|
|
body_so_far = u64(new_len) - body_pos
|
|
}
|
|
mut progress_body_so_far := body_so_far
|
|
mut chunked_complete := false
|
|
if is_chunked_transfer {
|
|
mut dechunked := []u8{}
|
|
chunked_complete = chunked_body_tracker.advance(bchunk, mut dechunked)
|
|
progress_body_so_far = chunked_body_tracker.decoded_len
|
|
if req.on_progress_body != unsafe { nil } && dechunked.len > 0 {
|
|
req.on_progress_body(req, dechunked, progress_body_so_far, expected_size,
|
|
status_code)!
|
|
}
|
|
} else if req.on_progress_body != unsafe { nil } {
|
|
req.on_progress_body(req, bchunk, progress_body_so_far, expected_size, status_code)!
|
|
}
|
|
if !(req.stop_copying_limit > 0 && new_len > req.stop_copying_limit) {
|
|
unsafe { content.write_ptr(bp, len) }
|
|
} else if headers_end >= 0 && new_len > body_pos {
|
|
has_truncated_body = true
|
|
}
|
|
if is_chunked_transfer && chunked_complete {
|
|
framed_complete = true
|
|
break
|
|
}
|
|
if headers_end >= 0 && response_has_no_body(req.method, status_code) {
|
|
// HEAD / 1xx / 204 / 304: response body is forbidden by the spec, so
|
|
// stop as soon as the headers terminator is in. Any `Content-Length`
|
|
// describes a body that will never be sent.
|
|
framed_complete = true
|
|
break
|
|
}
|
|
if has_content_length && body_so_far >= expected_size {
|
|
// The framing is satisfied even when expected_size == 0: on a
|
|
// keep-alive connection the server sends nothing further, so waiting
|
|
// for EOF here would stall until the read timeout.
|
|
// A response carrying MORE body bytes than its declared
|
|
// Content-Length is malformed: the stream position is no longer
|
|
// trustworthy, so such a connection must never be reused.
|
|
framed_complete = body_so_far == expected_size
|
|
break
|
|
}
|
|
if req.stop_receiving_limit > 0 && new_len > req.stop_receiving_limit {
|
|
break
|
|
}
|
|
old_len = new_len
|
|
}
|
|
return ReceivedResponseInfo{
|
|
headers_end: headers_end
|
|
is_chunked_transfer: is_chunked_transfer
|
|
has_truncated_body: has_truncated_body
|
|
reusable: framed_complete
|
|
}
|
|
}
|
|
|
|
fn read_from_tcp_connection_cb(con voidptr, buf &u8, bufsize int) !int {
|
|
mut r := unsafe { &net.TcpConn(con) }
|
|
return r.read_ptr(buf, bufsize)
|
|
}
|
|
|
|
struct ReceivedResponseBuffer {
|
|
data []u8
|
|
info ReceivedResponseInfo
|
|
}
|
|
|
|
fn (req &Request) read_all_from_client_connection(r &net.TcpConn) !ReceivedResponseBuffer {
|
|
mut content := strings.new_builder(4096)
|
|
info := req.receive_all_data_from_cb_in_builder(mut content, voidptr(r),
|
|
read_from_tcp_connection_cb)!
|
|
return ReceivedResponseBuffer{
|
|
data: content
|
|
info: info
|
|
}
|
|
}
|
|
|
|
// referer returns 'Referer' header value of the given request
|
|
pub fn (req &Request) referer() string {
|
|
return req.header.get(.referer) or { '' }
|
|
}
|
|
|
|
// parse_request parses a raw HTTP request into a Request object.
|
|
// See also: `parse_request_head`, which parses only the headers.
|
|
pub fn parse_request(mut reader io.BufferedReader) !Request {
|
|
mut request := parse_request_head(mut reader)!
|
|
|
|
// body
|
|
mut body := []u8{}
|
|
if length := request.header.get(.content_length) {
|
|
n := length.int()
|
|
if n > 0 {
|
|
body = []u8{len: n}
|
|
mut count := 0
|
|
for count < body.len {
|
|
count += reader.read(mut body[count..]) or { break }
|
|
}
|
|
}
|
|
}
|
|
|
|
request.data = body.bytestr()
|
|
return request
|
|
}
|
|
|
|
// parse_request_head parses *only* the header of a raw HTTP request into a Request object
|
|
pub fn parse_request_head(mut reader io.BufferedReader) !Request {
|
|
// request line
|
|
mut line := reader.read_line()!
|
|
method, target, version := parse_request_line(line)!
|
|
|
|
// headers
|
|
mut header := new_header()
|
|
line = reader.read_line()!
|
|
for line != '' {
|
|
// key, value := parse_header(line)!
|
|
mut pos := parse_header_fast(line)!
|
|
key := line[..pos]
|
|
for pos < line.len - 1 && line[pos + 1].is_space() {
|
|
// Skip space or tab in value name
|
|
pos++
|
|
}
|
|
if pos + 1 < line.len {
|
|
value := line[pos + 1..]
|
|
_, _ = key, value
|
|
// println('key,value=${key},${value}')
|
|
header.add_custom(key, value)!
|
|
}
|
|
line = reader.read_line()!
|
|
}
|
|
// header.coerce(canonicalize: true)
|
|
|
|
mut request_cookies := map[string]string{}
|
|
for _, cookie in read_cookies(header, '') {
|
|
request_cookies[cookie.name] = cookie.value
|
|
}
|
|
|
|
return Request{
|
|
method: method
|
|
url: target.str()
|
|
header: header
|
|
host: (header.get(.host) or { '' }).clone()
|
|
version: version
|
|
cookies: request_cookies
|
|
}
|
|
}
|
|
|
|
// parse_request_head parses *only* the header of a raw HTTP request into a Request object
|
|
pub fn parse_request_head_str(s string) !Request {
|
|
pos0 := s.index_('\n')
|
|
if pos0 == -1 {
|
|
return error('malformed request: no request line found')
|
|
}
|
|
line0 := s[..pos0].trim_space()
|
|
method, target, version := parse_request_line_fast(line0)!
|
|
|
|
// headers
|
|
mut header := new_header()
|
|
mut line_start := pos0 + 1
|
|
for line_start < s.len {
|
|
mut line_end := s.index_after_('\n', line_start)
|
|
if line_end == -1 {
|
|
line_end = s.len
|
|
}
|
|
mut line := s[line_start..line_end]
|
|
if line.len > 0 && line[line.len - 1] == `\r` {
|
|
line = line[..line.len - 1]
|
|
}
|
|
// IMPORTANT: HTTP headers end at the first empty line.
|
|
// If we hit this, we are now at the body, so we stop parsing headers.
|
|
if line == '' {
|
|
break
|
|
}
|
|
mut pos := parse_header_fast(line) or {
|
|
line_start = line_end + 1
|
|
continue
|
|
}
|
|
key := line[..pos]
|
|
|
|
// Skip space or tab after the colon
|
|
mut val_start := pos + 1
|
|
for val_start < line.len && line[val_start].is_space() {
|
|
val_start++
|
|
}
|
|
|
|
if val_start < line.len {
|
|
value := line[val_start..]
|
|
header.add_custom(key, value)!
|
|
}
|
|
line_start = line_end + 1
|
|
}
|
|
|
|
mut request_cookies := map[string]string{}
|
|
for _, cookie in read_cookies(header, '') {
|
|
request_cookies[cookie.name] = cookie.value
|
|
}
|
|
|
|
return Request{
|
|
method: method
|
|
url: target
|
|
header: header
|
|
host: (header.get(.host) or { '' }).clone()
|
|
version: version
|
|
cookies: request_cookies
|
|
}
|
|
}
|
|
|
|
fn parse_request_line_fast(line string) !(Method, string, Version) {
|
|
space1 := line.index_u8(` `)
|
|
if space1 <= 0 {
|
|
return error('bad request header')
|
|
}
|
|
space2_rel := line.index_after_(' ', space1 + 1)
|
|
if space2_rel == -1 || space2_rel == space1 + 1 || space2_rel >= line.len - 1 {
|
|
return error('bad request header')
|
|
}
|
|
method := method_from_str(line[..space1])
|
|
version := version_from_str(line[space2_rel + 1..])
|
|
if version == .unknown {
|
|
return error('unsupported version')
|
|
}
|
|
return method, line[space1 + 1..space2_rel], version
|
|
}
|
|
|
|
const headers_body_boundary = '\r\n\r\n'
|
|
|
|
// parse_request_str parses a raw HTTP request string into a Request object.
|
|
pub fn parse_request_str(s string) !Request {
|
|
mut request := parse_request_head_str(s)!
|
|
body_pos := s.index_(headers_body_boundary)
|
|
if body_pos != -1 {
|
|
request.data = s[body_pos + headers_body_boundary.len..]
|
|
}
|
|
return request
|
|
}
|
|
|
|
fn parse_request_line(line string) !(Method, urllib.URL, Version) {
|
|
// println('S=${s}')
|
|
words := line.split(' ')
|
|
// println('words=')
|
|
// println(words)
|
|
if words.len != 3 {
|
|
return error('bad request header')
|
|
}
|
|
method_str, target_str, version_str := words[0], words[1], words[2]
|
|
|
|
/*
|
|
space1, space2 := fast_request_words(line)
|
|
// if words.len != 3 {
|
|
if space1 == 0 || space2 == 0 {
|
|
return error('malformed request line')
|
|
}
|
|
method_str := s.substr_unsafe(0, space1)
|
|
target_str := s.substr_unsafe(space1 + 1, space2)
|
|
version_str := s.substr_unsafe(space2 + 1, s.len)
|
|
*/
|
|
// println('${method_str}!${target_str}!${version_str}')
|
|
// method := method_from_str(words[0])
|
|
// target := urllib.parse(words[1])!
|
|
// version := version_from_str(words[2])
|
|
method := method_from_str(method_str)
|
|
target := urllib.parse_request_uri(target_str)!
|
|
// println('before version_str="${version_str}"')
|
|
version := version_from_str(version_str)
|
|
// println('VERSION="${version}"')
|
|
if version == .unknown {
|
|
return error('unsupported version')
|
|
}
|
|
return method, target, version
|
|
}
|
|
|
|
// Parse URL encoded key=value&key=value forms
|
|
//
|
|
// FIXME: Some servers can require the
|
|
// parameter in a specific order.
|
|
//
|
|
// a possible solution is to use the a list of QueryValue
|
|
pub fn parse_form(body string) map[string]string {
|
|
mut form := map[string]string{}
|
|
|
|
if body.match_glob('{*}') {
|
|
form['json'] = body
|
|
} else {
|
|
words := body.split('&')
|
|
|
|
for word in words {
|
|
kv := word.split_nth('=', 2)
|
|
if kv.len != 2 {
|
|
continue
|
|
}
|
|
key := urllib.query_unescape(kv[0]) or { continue }
|
|
val := urllib.query_unescape(kv[1]) or { continue }
|
|
form[key] = val
|
|
}
|
|
}
|
|
return form
|
|
// }
|
|
// todo: parse form-data and application/json
|
|
// ...
|
|
}
|
|
|
|
pub struct FileData {
|
|
pub:
|
|
filename string
|
|
content_type string
|
|
data string
|
|
}
|
|
|
|
pub struct UnexpectedExtraAttributeError {
|
|
Error
|
|
pub:
|
|
attributes []string
|
|
}
|
|
|
|
pub fn (err UnexpectedExtraAttributeError) msg() string {
|
|
return 'Encountered unexpected extra attributes: ${err.attributes}'
|
|
}
|
|
|
|
pub struct MultiplePathAttributesError {
|
|
Error
|
|
}
|
|
|
|
pub fn (err MultiplePathAttributesError) msg() string {
|
|
return 'Expected at most one path attribute'
|
|
}
|
|
|
|
// multipart_form_body converts form and file data into a multipart/form
|
|
// HTTP request body. It is the inverse of parse_multipart_form. Returns
|
|
// (body, boundary).
|
|
// Note: Form keys should not contain quotes
|
|
fn multipart_form_body(form map[string]string, files map[string][]FileData) (string, string) {
|
|
rboundary := rand.ulid()
|
|
mut sb := strings.new_builder(1024)
|
|
for name, value in form {
|
|
sb.write_string('\r\n--')
|
|
sb.write_string(rboundary)
|
|
sb.write_string('\r\nContent-Disposition: form-data; name="')
|
|
sb.write_string(name)
|
|
sb.write_string('"\r\n\r\n')
|
|
sb.write_string(value)
|
|
}
|
|
for name, fs in files {
|
|
for f in fs {
|
|
sb.write_string('\r\n--')
|
|
sb.write_string(rboundary)
|
|
sb.write_string('\r\nContent-Disposition: form-data; name="')
|
|
sb.write_string(name)
|
|
sb.write_string('"; filename="')
|
|
sb.write_string(f.filename)
|
|
sb.write_string('"\r\nContent-Type: ')
|
|
sb.write_string(f.content_type)
|
|
sb.write_string('\r\n\r\n')
|
|
sb.write_string(f.data)
|
|
}
|
|
}
|
|
sb.write_string('\r\n--')
|
|
sb.write_string(rboundary)
|
|
sb.write_string('--')
|
|
return sb.str(), rboundary
|
|
}
|
|
|
|
struct LineSegmentIndexes {
|
|
mut:
|
|
start int
|
|
end int
|
|
}
|
|
|
|
// parse_multipart_form parses an http request body, given a boundary string
|
|
// For more details about multipart forms, see:
|
|
// https://datatracker.ietf.org/doc/html/rfc2183
|
|
// https://datatracker.ietf.org/doc/html/rfc2388
|
|
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
|
|
pub fn parse_multipart_form(body string, boundary string) (map[string]string, map[string][]FileData) {
|
|
// dump(body)
|
|
// dump(boundary)
|
|
mut form := map[string]string{}
|
|
mut files := map[string][]FileData{}
|
|
if body.len == 0 || boundary.len == 0 || boundary.len > body.len {
|
|
return form, files
|
|
}
|
|
mut field_start := body.index_after_(boundary, 0)
|
|
if field_start == -1 {
|
|
return form, files
|
|
}
|
|
field_start += boundary.len
|
|
mut line_segments := []LineSegmentIndexes{cap: 100}
|
|
for {
|
|
if field_start > body.len - boundary.len {
|
|
break
|
|
}
|
|
field_end := body.index_after_(boundary, field_start)
|
|
if field_end == -1 {
|
|
break
|
|
}
|
|
line_segments.clear()
|
|
mut line_idx, mut line_start := 0, field_start
|
|
for cidx := field_start; cidx < field_end; cidx++ {
|
|
if line_idx >= 6 {
|
|
// no need to scan further
|
|
break
|
|
}
|
|
if body[cidx] == `\n` {
|
|
line_segments << LineSegmentIndexes{line_start, cidx}
|
|
line_start = cidx + 1
|
|
line_idx++
|
|
}
|
|
}
|
|
line_segments << LineSegmentIndexes{line_start, field_end}
|
|
field_start = field_end + boundary.len
|
|
if line_segments.len < 2 {
|
|
continue
|
|
}
|
|
line1 := body#[line_segments[1].start..line_segments[1].end]
|
|
line2 := if line_segments.len == 2 {
|
|
''
|
|
} else {
|
|
body#[line_segments[2].start..line_segments[2].end]
|
|
}
|
|
disposition := parse_disposition(line1.trim_space())
|
|
// Grab everything between the double quotes
|
|
name := disposition['name'] or { continue }
|
|
// Parse files
|
|
// TODO: handle `filename*`, see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
|
|
if filename := disposition['filename'] {
|
|
// reject early broken content
|
|
if line_segments.len < 5 {
|
|
continue
|
|
}
|
|
// reject early non Content-Type headers
|
|
if !line2.to_lower().starts_with('content-type:') {
|
|
continue
|
|
}
|
|
content_type := line2.split_nth(':', 2)[1].trim_space()
|
|
// line1: Content-Disposition: form-data; name="upfile"; filename="photo123.jpg"
|
|
// line2: Content-Type: image/jpeg
|
|
// line3:
|
|
// line4: DATA
|
|
// ...
|
|
// lineX: --
|
|
data_end := field_end - 4 // each multipart field ends with \r\n--
|
|
if data_end < line_segments[4].start {
|
|
continue
|
|
}
|
|
data := body[line_segments[4].start..data_end]
|
|
// dump(data.limit(20).bytes())
|
|
// dump(data.len)
|
|
if name !in files {
|
|
files[name] = []FileData{}
|
|
}
|
|
files[name] << FileData{
|
|
filename: filename
|
|
content_type: content_type
|
|
data: data
|
|
}
|
|
continue
|
|
}
|
|
if line_segments.len < 4 {
|
|
continue
|
|
}
|
|
data_end := field_end - 4
|
|
if data_end < line_segments[3].start {
|
|
continue
|
|
}
|
|
form[name] = body[line_segments[3].start..data_end]
|
|
}
|
|
// dump(form)
|
|
return form, files
|
|
}
|
|
|
|
// parse_disposition parses the Content-Disposition header of a multipart form.
|
|
// Returns a map of the key="value" pairs
|
|
// Example: assert parse_disposition('Content-Disposition: form-data; name="a"; filename="b"') == {'name': 'a', 'filename': 'b'}
|
|
fn parse_disposition(line string) map[string]string {
|
|
mut data := map[string]string{}
|
|
for word in line.split(';') {
|
|
kv := word.split_nth('=', 2)
|
|
if kv.len != 2 {
|
|
continue
|
|
}
|
|
key, value := kv[0].to_lower().trim_left(' \t'), kv[1]
|
|
if value.starts_with('"') && value.ends_with('"') {
|
|
data[key] = value[1..value.len - 1]
|
|
} else {
|
|
data[key] = value
|
|
}
|
|
}
|
|
return data
|
|
}
|
|
|
|
fn is_no_need_retry_error(err_code int) bool {
|
|
return err_code in [
|
|
net.err_port_out_of_range.code(),
|
|
net.err_no_udp_remote.code(),
|
|
net.err_connect_timed_out.code(),
|
|
net.err_timed_out_code,
|
|
transport_err_unsafe_retry,
|
|
]
|
|
}
|