Add TLS client connections over Tcp and trust options for Tcp and Http

Cargo.toml

@@ -27,15 +27,13 @@ libsqlite3-sys = { version = "=0.33.0", features = ["bundled"] }
 # backend, and WebPKI roots keep the static host self-contained for Roc linking.
 hyper = { version = "=1.6.0", default-features = false, features = ["http1", "client"] }
 hyper-util = { version = "=0.1.12", default-features = false, features = ["client", "client-legacy", "http1", "tokio"] }
-hyper-rustls = { version = "=0.27.6", default-features = false, features = ["http1", "tls12", "webpki-tokio", "ring"] }
+hyper-rustls = { version = "=0.27.6", default-features = false, features = ["http1", "tls12", "ring"] }
 http-body-util = "=0.1.3"
 tokio = { version = "=1.45.0", default-features = false, features = ["rt", "net", "time", "process", "sync", "io-util", "macros"] }
 bytes = "=1.12.1"
 tower-service = "=0.3.3"
-
-[dev-dependencies]
-# Exercise the production ring provider against a local TLS server.
 rustls = { version = "=0.23.46", default-features = false, features = ["std", "ring", "tls12"] }
+webpki-roots = "=1.0.8"
 
 # Keep freed small-allocation pages on the statically linked Linux hosts.
 # Generated Roc ABI helpers already allocate through Rust's global allocator.

README.md

@@ -7,7 +7,7 @@
 
 A Roc [platform](https://github.com/roc-lang/roc/wiki/Roc-concepts-explained#platform) for command-line programs.
 
-`basic-cli` supports command execution, directories, environment variables, files, HTTP, locales, paths, random seeds, sleeping, SQLite, standard input/output/error, TCP, terminal raw mode, and UTC time.
+`basic-cli` supports command execution, directories, environment variables, files, HTTP, locales, paths, random seeds, sleeping, SQLite, standard input/output/error, TCP (including TLS client connections), terminal raw mode, and UTC time.
 
 ## Supported targets
 
@@ -35,10 +35,15 @@ These host behaviors may affect application design:
 
 - HTTP and HTTPS use HTTP/1 only. [Issue #455](https://github.com/roc-lang/basic-cli/issues/455)
   tracks research into HTTP/2 support and its tradeoffs.
-- HTTPS certificates are validated against bundled WebPKI roots, not the
-  operating system trust store. A certificate trusted only through a locally
-  installed OS root will therefore be rejected. [Issue #454](https://github.com/roc-lang/basic-cli/issues/454)
-  tracks research into the appropriate trust configuration for basic-cli.
+- HTTPS and `Tcp.connect_tls!` validate certificates against bundled WebPKI
+  roots by default, not the operating system trust store (`SSL_CERT_FILE` is
+  not read either). A certificate trusted only through a locally installed OS
+  root will therefore be rejected. An application can add its own roots from
+  PEM bytes with `tls: { trust: AddPem(bytes) }`, or accept any certificate
+  while developing with `tls: { trust: DangerAcceptInvalidCerts }`, in the
+  options of `Http.send!`, `Http.send_streaming!` or `Tcp.connect_tls!`. The
+  trust applies to every redirect hop. [Issue #454](https://github.com/roc-lang/basic-cli/issues/454)
+  tracks research into the appropriate default trust source for basic-cli.
 - The timeout configured on an HTTP `Request` is the deadline for the whole
   exchange, response body included; `NoTimeout` leaves it unbounded.
   `Http.send!` and the functions built on it buffer the response completely
@@ -86,8 +91,9 @@ These host behaviors may affect application design:
   creating a new regular file. Existing files retain their access grants;
   use `RequireNew` with `OwnerOnly` for private files. Explicit permissions
   replace ACL grants and fail if the filesystem cannot enforce the policy.
-- Several network and TLS failures currently use a generic HTTP transport
-  error ([issue #438](https://github.com/roc-lang/basic-cli/issues/438)).
+- A rejected certificate or failed TLS handshake is `HttpErr(TlsErr(err))`,
+  naming the cause. Several other network failures currently use a generic
+  HTTP transport error ([issue #438](https://github.com/roc-lang/basic-cli/issues/438)).
 - TCP reads and writes always have a timeout: a `Tcp.Connection` holds the
   `io_timeout` it was created with, which bounds each read or write as a
   whole, and `with_timeout` returns the same connection with another. The wait
@@ -96,6 +102,13 @@ These host behaviors may affect application design:
   `TcpErr(TimedOut)`.
 - A TCP read that times out or is interrupted consumes nothing: the bytes that had arrived stay
   buffered for the next read. A write that times out may have sent any prefix.
+- `Tcp.connect_tls!` returns an ordinary `Tcp.Connection`: its
+  `connect_timeout` covers resolution, connecting and the handshake, and reads
+  and writes keep every TCP guarantee above. A bad PEM in `AddPem` is
+  `TlsErr(BadPem(_))`, reported before any network activity. A TLS failure
+  after the handshake is `TcpErr(Other(message))`, and so is a peer that closes
+  without TLS `close_notify`, rather than a clean end of input. Releasing the
+  connection sends `close_notify` without waiting.
 - `Tcp.Connection.read_until!`, `read_line!` and `lines` stop at the
   connection's line limit (1 MiB unless changed with `with_line_limit`),
   preventing an untrusted peer from causing unbounded buffering. The rest of

examples/tcp-tls.roc

@@ -0,0 +1,57 @@
+## Exchange lines with a TLS echo server whose certificate is signed by a
+## private certificate authority, read from a PEM file.
+app [main!] { pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.27.0/9yjxqdGTP4xSpSEmDazwTmRsehfRZzA19pwEKkR6fj1g.tar.zst" }
+
+import pf.Duration
+import pf.OsStr
+import pf.Path
+import pf.Stdout
+import pf.Tcp
+
+# The test runner starts a local TLS echo server on port 8443 and passes the
+# certificate it trusts. To try it by hand, run scripts/tls_test_server.py in
+# another terminal, then:
+#
+#     $ roc examples/tcp-tls.roc -- tests/tls/localhost-cert.pem
+#
+# Without an argument only the bundled public roots are trusted, so the same
+# server is rejected with `TlsErr(UnknownIssuer)`.
+main! : List(OsStr) => Try({}, _)
+main! = |args| {
+	trust = match args {
+		[ca_file] => AddPem(Path.from_os_str(ca_file).read_bytes!()?)
+		_ => Default
+	}
+
+	# The handshake verifies the certificate against the host name. It is part
+	# of the connect, so `connect_timeout` bounds it as well.
+	conn = Tcp.connect_tls!("localhost", 8443, { connect_timeout: Duration.seconds(5), io_timeout: Duration.seconds(5), tls: { trust } })?
+	Stdout.line!("Connected over TLS")?
+
+	# From here on it is an ordinary `Tcp.Connection`.
+	conn.write_utf8!("hello\nover tls\n")?
+	var $pending = 2
+	for! item in conn.lines() {
+		match item {
+			Ok(line) => Stdout.line!("< ${line}")?
+			# These two concern one line; the stream carries on with the next.
+			Err(BadUtf8({ index, .. })) => Stdout.line!("< (invalid UTF-8 at byte ${index.to_str()})")?
+			Err(ReadLimitExceeded(limit)) => Stdout.line!("< (a line longer than ${limit.to_str()} bytes)")?
+			# The connection failed or timed out, and this is the last item.
+			Err(TcpErr(err)) => {
+				return Err(TcpErr(err))
+			}
+			Err(Interrupted) => {
+				return Err(Interrupted)
+			}
+		}
+		# The echo server keeps the connection open, so stop at the last reply
+		# instead of waiting for the end of input.
+		$pending = $pending - 1
+		if $pending == 0 {
+			break
+		}
+	}
+
+	Ok({})
+}

platform/Host.roc

@@ -3,6 +3,7 @@ import InternalHttp
 import InternalSqlite
 import Resource
 import System
+import Tls
 
 ## Declare the hosted effects and ABI-safe data exchanged with the native host.
 Host :: [].{
@@ -166,7 +167,7 @@ Host :: [].{
 	# reads from the one cursor the response's streams share: `Chunk` is the
 	# next non-empty chunk of at most 64 KiB, `ToEnd(max)` is everything unread,
 	# or `ReadLimitExceeded` once more than `max` bytes of it have arrived.
-	http_send_streaming! : InternalHttp.RequestToAndFromHost, [NoTimeout, After(U64)], [NoTimeout, At(U64)] => Try(InternalHttp.StreamingResponseFromHost, InternalHttp.TransportErr)
+	http_send_streaming! : InternalHttp.RequestToAndFromHost, [NoTimeout, After(U64)], [NoTimeout, At(U64)], Tls.Trust => Try(InternalHttp.StreamingResponseFromHost, InternalHttp.TransportErr)
 	http_read_body! : Resource.HttpResponse, [Chunk, ToEnd(U64)] => Try(List(U8), [EndOfBody, ReadLimitExceeded, HttpErr(InternalHttp.TransportErr)])
 	signal_catch! : List([Interrupt, Terminate(System.Unix), User1(System.Unix), User2(System.Unix), WindowResize(System.Unix)]) => Try({}, [SignalErr(IOErr)])
 	signal_poll! : () => List([Interrupt, Terminate, User1, User2, WindowResize])
@@ -188,4 +189,8 @@ Host :: [].{
 	unix_socket_listen! : NativePath => Try(Resource.UnixSocketListener, [UnixSocketErr(IOErr)])
 	unix_socket_accept! : Resource.UnixSocketListener, [NoTimeout, After(U64)] => Try(Resource.UnixSocketConnection, [Closed, UnixSocketErr(IOErr)])
 	unix_socket_listener_close! : Resource.UnixSocketListener => Try({}, [UnixSocketErr(IOErr)])
+
+	# TLS over TCP. The timeout, in nanoseconds, bounds resolution, connecting
+	# and the handshake. `TlsErr` is a failed handshake or unusable trust roots.
+	tcp_connect_tls! : Str, U16, U64, Tls.Trust => Try(Resource.TcpConnection, [TcpErr(IOErr), TlsErr(Tls.Err)])
 }

platform/Http.roc

@@ -2,6 +2,7 @@ import Duration
 import Host
 import InternalHttp
 import Resource
+import Tls
 import Url
 import http.Method
 import http.Header
@@ -24,17 +25,21 @@ import http.Response
 ## adds no content type. Explicit content types are always preserved.
 ##
 ## See the [host runtime behavior](https://github.com/roc-lang/basic-cli#host-runtime-behavior)
-## for HTTP protocol, TLS trust-store, and timeout details.
+## for HTTP protocol, TLS trust, and timeout details.
 Http :: [].{
 
 	## Failures while sending a request, reading its body or following redirects.
-	TransportErr : [Timeout, NetworkError, BadBody, Other(List(U8)), RedirectLimitExceeded(U64), InvalidRedirectUrl(Url.ParseErr), AmbiguousRedirectLocation, InsecureRedirect]
+	TransportErr : [Timeout, NetworkError, BadBody, Other(List(U8)), RedirectLimitExceeded(U64), InvalidRedirectUrl(Url.ParseErr), AmbiguousRedirectLocation, InsecureRedirect, TlsErr(Tls.Err)]
 
 	## Redirects are opt-in; `{}` returns the first response unchanged.
 	## A positive maximum follows 301/302/303/307/308, using one request
 	## deadline across every hop and the final body. Cross-origin requests
 	## discard credentials; HTTPS-to-HTTP redirects are rejected.
-	SendOptions := { max_redirects : U64 ?? 0 }
+	##
+	## `tls` chooses which server certificates HTTPS accepts, on the first
+	## request and on every redirect hop; `{}` keeps the bundled roots. A
+	## cross-origin redirect is checked with the same trust.
+	SendOptions := { max_redirects : U64 ?? 0, tls : Tls.Options ?? {} }
 
 	## Validate and send an HTTP request.
 	##
@@ -48,7 +53,7 @@ Http :: [].{
 	## ```
 	send! : Request, SendOptions => Try(Response, [Interrupted, InvalidUrl(Url.ParseErr), HttpErr(TransportErr)])
 	send! = |request, options| {
-		response = send_streaming!(request, { idle_timeout: NoTimeout, max_redirects: options.max_redirects })?
+		response = send_streaming!(request, { idle_timeout: NoTimeout, max_redirects: options.max_redirects, tls: options.tls })?
 		body = match response.read_to_end!(NoLimit) {
 			Ok(bytes) => bytes
 			Err(ReadLimitExceeded(_)) => {
@@ -145,10 +150,11 @@ Http :: [].{
 	## leave the request at `NoTimeout` and rely on `idle_timeout`.
 	##
 	## `max_redirects` has the same meaning as the `max_redirects` field of [SendOptions].
+	## `tls` has the same meaning as in [SendOptions].
 	## A caught pending signal returns bare `Interrupted` from a wait. Body
 	## reads preserve their cursor and collected bytes after interruption;
 	## acknowledge the signal before reading again. The deadline keeps running.
-	StreamingOptions := { idle_timeout : [NoTimeout, After(Duration)] ?? After(Duration.seconds(30)), max_redirects : U64 ?? 0 }
+	StreamingOptions := { idle_timeout : [NoTimeout, After(Duration)] ?? After(Duration.seconds(30)), max_redirects : U64 ?? 0, tls : Tls.Options ?? {} }
 
 	## Validate and send an HTTP request, returning once the status and
 	## headers have arrived. The body is read afterwards, through the
@@ -308,6 +314,7 @@ transport_error = |error|
 		NetworkError => HttpErr(NetworkError)
 		BadBody => HttpErr(BadBody)
 		Other(message) => HttpErr(Other(message))
+		TlsErr(err) => HttpErr(TlsErr(err))
 	}
 
 ## The same driver serves buffered and streamed responses. The deadline is
@@ -319,7 +326,7 @@ send_redirected! = |request, url, options, remaining, deadline| {
 		After(duration) => After(duration.to_nanos())
 	}
 	hop = request.with_headers(sanitize_headers(Request.headers(request), False, False))
-	raw = match Host.http_send_streaming!(InternalHttp.to_host_request(hop), idle, deadline) {
+	raw = match Host.http_send_streaming!(InternalHttp.to_host_request(hop), idle, deadline, options.tls.trust) {
 		Ok(value) => value
 		Err(error) => {
 			return Err(transport_error(error))
@@ -420,6 +427,8 @@ sanitize_headers = |headers, cross_origin, drop_body| {
 
 expect Http.SendOptions.{}.max_redirects == 0
 expect Http.StreamingOptions.{}.max_redirects == 0
+expect Http.SendOptions.{}.tls.trust == Default
+expect Http.StreamingOptions.{ tls: { trust: DangerAcceptInvalidCerts } }.tls.trust == DangerAcceptInvalidCerts
 expect redirect_url("https://example.com/a/b", "../c#part") == Url.parse("https://example.com/c")
 expect redirect_url("https://example.com/a", "//other.example/b") == Url.parse("https://other.example/b")
 expect same_origin("https://EXAMPLE.com:443/a", "https://example.com/b")

platform/InternalHttp.roc

@@ -3,14 +3,15 @@ import http.Method
 import http.Request
 import http.Response
 import Resource
+import Tls
 
 ## Define host-ABI HTTP types and convert them to the shared HTTP package types.
 ## These records map directly to the generated Rust glue types.
 InternalHttp :: [].{
 
 	## Errors raised by the host while sending a request, before a real HTTP
 	## response is available.
-	TransportErr : [Interrupted, Timeout, NetworkError, BadBody, Other(List(U8))]
+	TransportErr : [Interrupted, Timeout, NetworkError, BadBody, Other(List(U8)), TlsErr(Tls.Err)]
 
 	# Generated Rust glue uses tuple headers at the host ABI boundary.
 	HostHeaderTuple : (Str, Str)

platform/Tcp.roc

@@ -2,6 +2,7 @@ import Duration
 import Host
 import IOErr
 import Resource
+import Tls
 
 ## The line limit a new connection holds: 1 MiB.
 default_line_limit : U64
@@ -24,6 +25,7 @@ accept_wait = |accept_timeout|
 ## results compare with `==`: `result == Err(TcpErr(ConnectionRefused))`.
 ## A caught, pending signal returns bare `Interrupted` from waits. A read that
 ## is interrupted keeps its unread bytes; a write may have sent a prefix.
+## `connect_tls!` adds `TlsErr(Tls.Err)` for a failed handshake.
 ##
 ## Timeouts are [Duration]s. The wait to connect or to accept bounds that one
 ## call. The connection it produces holds its own I/O timeout, which bounds
@@ -372,6 +374,42 @@ Tcp :: [].{
 			Err(TcpErr(Interrupted)) => Err(Interrupted)
 			Err(TcpErr(err)) => Err(TcpErr(err))
 		}
+
+	## Options for [connect_tls!]: the two timeouts of [connect!], and TLS
+	## settings that default to `{}`.
+	ConnectTlsOptions := { connect_timeout : Duration, io_timeout : Duration, tls : Tls.Options ?? {} }
+
+	## Open a TLS client connection.
+	##
+	## The handshake verifies the server's certificate for `host`, which is also
+	## sent as the server name (SNI) unless it is an IP address. An IP address is
+	## verified against the certificate's IP address entries.
+	##
+	## `connect_timeout` bounds this call: hostname resolution, connecting, and
+	## the handshake. `io_timeout` is the timeout the new [Connection] holds, as
+	## for [connect!]. The connection behaves exactly like one from [connect!]:
+	## reads that time out or are interrupted consume nothing, and the line
+	## limit applies.
+	##
+	## A failed handshake is `TlsErr(err)`, with [Tls.Err] naming the cause.
+	## `BadPem` is reported before any network activity. A TLS failure later,
+	## while reading or writing, is `TcpErr(Other(message))`. So is a peer
+	## that closes without TLS's `close_notify`: data could have been cut off.
+	## Releasing the connection sends `close_notify` without waiting.
+	##
+	## ```roc
+	## conn = Tcp.connect_tls!("example.com", 443, { connect_timeout: Duration.seconds(5), io_timeout: Duration.seconds(30) })?
+	## conn.write_utf8!("HEAD / HTTP/1.1\r\nHost: example.com\r\nConnection: close\r\n\r\n")?
+	## status_line = conn.read_line!()?
+	## ```
+	connect_tls! : Str, U16, ConnectTlsOptions => Try(Connection, [Interrupted, TlsErr(Tls.Err), TcpErr(IOErr)])
+	connect_tls! = |host, port, options|
+		match Host.tcp_connect_tls!(host, port, options.connect_timeout.to_nanos(), options.tls.trust) {
+			Ok(handle) => Ok(Connection.{ host: handle, timeout: options.io_timeout, line_limit: default_line_limit })
+			Err(TcpErr(Interrupted)) => Err(Interrupted)
+			Err(TcpErr(err)) => Err(TcpErr(err))
+			Err(TlsErr(err)) => Err(TlsErr(err))
+		}
 }
 
 expect default_line_limit == 1_048_576

platform/Tls.roc

@@ -0,0 +1,66 @@
+## Choose which server certificates a TLS client accepts, for [Tcp.connect_tls!]
+## and for HTTPS through [Http.send!] and [Http.send_streaming!].
+##
+## See the [host runtime behavior](https://github.com/roc-lang/basic-cli#host-runtime-behavior)
+## for where the default roots come from.
+Tls :: [].{
+
+	## Which server certificates a client accepts.
+	##
+	## - `Default`: certificates that chain to the bundled WebPKI (Mozilla) roots.
+	##   The operating system's trust store is not consulted.
+	## - `AddPem(bytes)`: the bundled roots plus every `CERTIFICATE` in the PEM
+	##   bytes, such as a private certificate authority. Other PEM sections are
+	##   ignored. Bytes without any certificate are `BadPem`, before anything is
+	##   sent.
+	## - `DangerAcceptInvalidCerts`: any certificate, for any name, expired or
+	##   not. The connection is still encrypted and the server must still hold
+	##   the key of the certificate it presents, but anyone on the network path
+	##   can present their own. Only for development against a server with a
+	##   self-signed certificate.
+	Trust : [Default, AddPem(List(U8)), DangerAcceptInvalidCerts]
+
+	## TLS settings. `{}` selects the defaults.
+	##
+	## ```roc
+	## ca = Path.read_bytes!("ca.pem")?
+	## conn = Tcp.connect_tls!("db.internal", 6380, { connect_timeout: Duration.seconds(5), io_timeout: Duration.seconds(30), tls: { trust: AddPem(ca) } })?
+	## ```
+	Options := { trust : Trust ?? Default }
+
+	## Why a TLS handshake failed, or why the settings were rejected before
+	## connecting.
+	##
+	## - `UnknownIssuer`: the certificate does not chain to a trusted root.
+	## - `Expired`: the certificate, or one it chains to, has expired.
+	## - `NameMismatch`: the certificate is not valid for the host name or IP
+	##   address that was connected to.
+	## - `BadPem(message)`: the `AddPem` bytes hold no usable certificate.
+	## - `Other(message)`: any other handshake failure, such as no common
+	##   protocol version or an alert from the server.
+	Err := [UnknownIssuer, Expired, NameMismatch, BadPem(Str), Other(Str)].{
+
+		## Compare two TLS errors, so results that contain one work with `==`.
+		is_eq : _
+
+		## Describe the error for display.
+		to_str : Err -> Str
+		to_str = |err|
+			match err {
+				UnknownIssuer => "the server certificate is not issued by a trusted certificate authority"
+				Expired => "the server certificate has expired"
+				NameMismatch => "the server certificate is not valid for the requested host"
+				BadPem(message) => "the PEM trust roots cannot be used: ${message}"
+				Other(message) => message
+			}
+	}
+}
+
+expect Tls.Options.{}.trust == Default
+expect Tls.Options.{ trust: AddPem([1]) }.trust == AddPem([1])
+expect Tls.Err.to_str(Expired) == "the server certificate has expired"
+expect {
+	result : Try({}, [TlsErr(Tls.Err)])
+	result = Err(TlsErr(NameMismatch))
+	result == Err(TlsErr(NameMismatch)) and result != Err(TlsErr(Expired))
+}