diff --git a/CHANGELOG.md b/CHANGELOG.md index 42a2ccf..2081849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ part carries no `Content-Type` header (previously `text/plain`). ### Added +- **HTTP `Range` requests on static files.** The static-file server answers a + single byte range (`bytes=A-B`, `bytes=A-`, `bytes=-N`) with `206 Partial + Content` and a `Content-Range` header, seeking directly via `sendfile(2)`, so + clients can resume interrupted downloads and seek within large assets. An + unsatisfiable range gets `416 Range Not Satisfiable`; every static response + now advertises `Accept-Ranges: bytes`. Multiple ranges and malformed specs + fall back to the full `200`. - **`App.request-timeout` (60s)** bounds first byte to complete request, so a slow-dripping body can no longer hold a connection open. - **`Expect: 100-continue` is answered** with the interim response once the diff --git a/test/smoke-server.carp b/test/smoke-server.carp index 13f7f40..3defb2a 100644 --- a/test/smoke-server.carp +++ b/test/smoke-server.carp @@ -14,7 +14,8 @@ (if (Request.chunked? r) "chunked" "plain") &(match (Request.header r "Content-Length") (Maybe.Just v) v - (Maybe.Nothing) @"none")))))) + (Maybe.Nothing) @"none"))))) + (StaticFile.mount @"/static" @"test/static-fixtures")) bh (the (Array (Fn [&Request &(Map String String)] (Maybe Response))) []) ah (the (Array (Fn [&Request &(Map String String) Response] Response)) diff --git a/test/smoke.sh b/test/smoke.sh index c770c9d..1fa4fc5 100755 --- a/test/smoke.sh +++ b/test/smoke.sh @@ -89,4 +89,22 @@ CODE=$(curl -s --max-time 10 -o /dev/null -w '%{http_code}' \ -H 'Transfer-Encoding: gzip' --data-binary 'x' "$BASE/echo") [ "$CODE" = "400" ] || fail "non-chunked Transfer-Encoding not rejected (got: $CODE)" +# 8. a byte range yields 206 Partial Content with the requested bytes +check "byte range" +RH=$(curl -s --max-time 10 -D - -o "$WORK/range" -r 0-4 "$BASE/static/index.html") +echo "$RH" | grep -qi '206 Partial Content' || fail "range request not 206" +echo "$RH" | grep -qi 'Content-Range: bytes 0-4/11' || fail "wrong Content-Range" +[ "$(cat "$WORK/range")" = "root " ] || fail "range body wrong (got: $(cat "$WORK/range"))" + +# 9. an open-ended range reassembles to the whole file +check "open range" +[ "$(curl -s --max-time 10 -r 0- "$BASE/static/index.html")" \ + = "$(cat test/static-fixtures/index.html)" ] || fail "open range mismatch" + +# 10. a range past the end is rejected with 416 +check "unsatisfiable range" +CODE=$(curl -s --max-time 10 -o /dev/null -w '%{http_code}' \ + -r 100-200 "$BASE/static/index.html") +[ "$CODE" = "416" ] || fail "unsatisfiable range not 416 (got: $CODE)" + echo "smoke: all checks passed" diff --git a/test/web.carp b/test/web.carp index 23fd1f4..fe01a37 100644 --- a/test/web.carp +++ b/test/web.carp @@ -29,6 +29,39 @@ (set! cursor @(Pair.b &p))))) status)) +(defn rr-tag [r] + (match-ref r + (RangeResolution.Full) 0 + (RangeResolution.Partial _ _) 1 + (RangeResolution.Unsatisfiable) 2)) + +(defn rr-start [r] + (match-ref r + (RangeResolution.Partial s _) @s + (RangeResolution.Full) -1l + (RangeResolution.Unsatisfiable) -1l)) + +(defn rr-end [r] + (match-ref r + (RangeResolution.Partial _ e) @e + (RangeResolution.Full) -1l + (RangeResolution.Unsatisfiable) -1l)) + +; builds the response for the raw request `raw` against a static-file app +; rooted at test/static-fixtures (index.html is 11 bytes) +(defn static-resp [raw] + (let [app (StaticFile.mount (App.create) @"/static" @"test/static-fixtures") + bh (the (Array (Fn [&Request &(Map String String)] (Maybe Response))) []) + ah (the + (Array (Fn [&Request &(Map String String) Response] Response)) + []) + pair (web-build-response &app &bh &ah &(String.to-bytes raw))] + @(Pair.a &pair))) + +(defn header-val [resp name] + @(Array.unsafe-first + &(Map.get-with-default (Response.headers resp) name &[@""]))) + (deftest test ; -- Response.text -- (assert-equal test @@ -655,6 +688,169 @@ (String.empty? (Response.body (Pair.a &pair)))) "HEAD + chunked has empty body") + ; -- Range resolution: bytes=A-B -- + (assert-equal test + 1 + (rr-tag &(web-resolve-range "bytes=0-499" 1000l)) + "bytes=0-499 is a partial range") + (assert-equal test + 0l + (rr-start &(web-resolve-range "bytes=0-499" 1000l)) + "bytes=0-499 starts at 0") + (assert-equal test + 499l + (rr-end &(web-resolve-range "bytes=0-499" 1000l)) + "bytes=0-499 ends at 499") + + ; -- Range resolution: bytes=A- (open-ended) -- + (assert-equal test + 500l + (rr-start &(web-resolve-range "bytes=500-" 1000l)) + "bytes=500- starts at 500") + (assert-equal test + 999l + (rr-end &(web-resolve-range "bytes=500-" 1000l)) + "bytes=500- runs to EOF") + + ; -- Range resolution: bytes=-N (suffix) -- + (assert-equal test + 500l + (rr-start &(web-resolve-range "bytes=-500" 1000l)) + "bytes=-500 starts 500 bytes before EOF") + (assert-equal test + 999l + (rr-end &(web-resolve-range "bytes=-500" 1000l)) + "bytes=-500 ends at EOF") + + ; -- Range resolution: bytes=0- covers the whole file -- + (assert-equal test + 0l + (rr-start &(web-resolve-range "bytes=0-" 1000l)) + "bytes=0- starts at 0") + (assert-equal test + 999l + (rr-end &(web-resolve-range "bytes=0-" 1000l)) + "bytes=0- ends at EOF") + + ; -- Range resolution: end past EOF is clamped -- + (assert-equal test + 999l + (rr-end &(web-resolve-range "bytes=0-100000" 1000l)) + "bytes=0-100000 clamps end to EOF") + (assert-equal test + 1 + (rr-tag &(web-resolve-range "bytes=0-100000" 1000l)) + "bytes=0-100000 stays satisfiable after clamping") + + ; -- Range resolution: start at/past size is unsatisfiable -- + (assert-equal test + 2 + (rr-tag &(web-resolve-range "bytes=2000-3000" 1000l)) + "bytes=2000-3000 past EOF is unsatisfiable") + (assert-equal test + 2 + (rr-tag &(web-resolve-range "bytes=1000-1000" 1000l)) + "start equal to size is unsatisfiable") + + ; -- Range resolution: malformed specs fall back to full -- + (assert-equal test + 0 + (rr-tag &(web-resolve-range "bytes=abc" 1000l)) + "bytes=abc falls back to full") + (assert-equal test + 0 + (rr-tag &(web-resolve-range "bytes=" 1000l)) + "bytes= falls back to full") + (assert-equal test + 0 + (rr-tag &(web-resolve-range "0-499" 1000l)) + "missing bytes= falls back to full") + (assert-equal test + 0 + (rr-tag &(web-resolve-range "bytes=1-2-3" 1000l)) + "extra dash falls back to full") + + ; -- Range resolution: multiple ranges fall back to full -- + (assert-equal test + 0 + (rr-tag &(web-resolve-range "bytes=0-499,600-700" 1000l)) + "multiple ranges fall back to full") + + ; -- Range resolution: zero-length suffix and empty files -- + (assert-equal test + 2 + (rr-tag &(web-resolve-range "bytes=-0" 1000l)) + "bytes=-0 is unsatisfiable") + (assert-equal test + 2 + (rr-tag &(web-resolve-range "bytes=0-" 0l)) + "any range on an empty file is unsatisfiable") + + ; -- Range resolution: oversized bounds do not overflow -- + (assert-equal test + 2 + (rr-tag &(web-resolve-range "bytes=999999999999-" 1000l)) + "oversized start saturates to unsatisfiable") + (assert-equal test + 999l + (rr-end &(web-resolve-range "bytes=-999999999999" 1000l)) + "oversized suffix saturates to the whole file") + + ; -- HEAD with a satisfiable range answers 206 -- + (assert-equal test + 206 + @(Response.code + &(static-resp + "HEAD /static/index.html HTTP/1.1\r\nRange: bytes=0-4\r\n\r\n")) + "HEAD range request returns 206") + (assert-equal test + "bytes 0-4/11" + &(header-val + &(static-resp + "HEAD /static/index.html HTTP/1.1\r\nRange: bytes=0-4\r\n\r\n") + "Content-Range") + "HEAD range request sets Content-Range") + (assert-equal test + "5" + &(header-val + &(static-resp + "HEAD /static/index.html HTTP/1.1\r\nRange: bytes=0-4\r\n\r\n") + "Content-Length") + "HEAD range request sets range-length Content-Length") + (assert-true test + (String.empty? + (Response.body + &(static-resp + "HEAD /static/index.html HTTP/1.1\r\nRange: bytes=0-4\r\n\r\n"))) + "HEAD range request has no body") + + ; -- HEAD with an unsatisfiable range answers 416 -- + (assert-equal test + 416 + @(Response.code + &(static-resp + "HEAD /static/index.html HTTP/1.1\r\nRange: bytes=100-200\r\n\r\n")) + "HEAD unsatisfiable range returns 416") + (assert-equal test + "bytes */11" + &(header-val + &(static-resp + "HEAD /static/index.html HTTP/1.1\r\nRange: bytes=100-200\r\n\r\n") + "Content-Range") + "HEAD 416 sets Content-Range with the total size") + + ; -- normal static responses advertise range support -- + (assert-equal test + "bytes" + &(header-val + &(static-resp "HEAD /static/index.html HTTP/1.1\r\n\r\n") + "Accept-Ranges") + "static HEAD advertises Accept-Ranges") + (assert-equal test + 200 + @(Response.code &(static-resp "HEAD /static/index.html HTTP/1.1\r\n\r\n")) + "static HEAD without a range stays 200") + ; -- chunked request body reaches the handler decoded -- (assert-equal test "hello world" diff --git a/web.carp b/web.carp index 3e83d94..fa52f8e 100644 --- a/web.carp +++ b/web.carp @@ -423,7 +423,11 @@ server does not detect the `X-Sendfile` header. The `Content-Type` is inferred from the extension. The server sets `Content-Length` from the file size. An `ETag` header is computed from the file's modification time and size, avoiding a full read of the -file contents.") +file contents. + +The server advertises `Accept-Ranges: bytes` and honours a single +`Range` request against the transfer, answering `206 Partial Content` +with a `Content-Range` header (or `416` when unsatisfiable).") (defn sendfile [path] (let [fd (IO.Raw.open path IO.Raw.O-RDONLY)] (if (< fd 0) @@ -1377,6 +1381,177 @@ responses. The `protocols` field lists the subprotocols that this route supports (set! r (~(Array.unsafe-nth hooks i) req params r))) r)) +(deftype RangeResolution + (Full []) + (Partial [Long Long]) + (Unsatisfiable [])) + +; Parse a decimal byte-range bound, saturating at total+1 once the value +; exceeds the file size so oversized specs cannot overflow Long. +(hidden web-parse-bound) +(defn web-parse-bound [s total] + (let-do [bytes (String.to-bytes s) + n (Array.length &bytes) + acc 0l + ok (> n 0) + over false] + (for [i 0 n] + (let [b (Byte.to-int @(Array.unsafe-nth &bytes i))] + (if (and (>= b 48) (<= b 57)) + (unless over + (let [d (Long.from-int (- b 48))] + (if (or (> acc (/ total 10l)) (> d (- total (* acc 10l)))) + (do (set! over true) (set! acc (+ total 1l))) + (set! acc (+ (* acc 10l) d))))) + (set! ok false)))) + (if ok (Maybe.Just acc) (Maybe.Nothing)))) + +; Resolve a single byte-range spec against a file of `total` bytes into Full +; (serve the whole file; also the fallback for malformed/multi-range specs), +; Partial with an inclusive [start end], or Unsatisfiable (start at/past EOF). +(defn web-resolve-range [spec total] + (if (not (String.starts-with? spec "bytes=")) + (RangeResolution.Full) + (let [rest &(String.suffix spec 6)] + (if (String.contains? rest \,) + (RangeResolution.Full) + (let [dash (String.index-of rest \-)] + (if (< dash 0) + (RangeResolution.Full) + (let [start-s &(String.prefix rest dash) + end-s &(String.suffix rest (+ dash 1))] + (cond + (String.empty? start-s) + (match (web-parse-bound end-s total) + (Maybe.Nothing) (RangeResolution.Full) + (Maybe.Just n) + (cond + (<= n 0l) (RangeResolution.Unsatisfiable) + (<= total 0l) (RangeResolution.Unsatisfiable) + (>= n total) (RangeResolution.Partial 0l (- total 1l)) + (RangeResolution.Partial (- total n) (- total 1l)))) + (String.empty? end-s) + (match (web-parse-bound start-s total) + (Maybe.Nothing) (RangeResolution.Full) + (Maybe.Just a) + (if (>= a total) + (RangeResolution.Unsatisfiable) + (RangeResolution.Partial a (- total 1l)))) + (match (web-parse-bound start-s total) + (Maybe.Nothing) (RangeResolution.Full) + (Maybe.Just a) + (match (web-parse-bound end-s total) + (Maybe.Nothing) (RangeResolution.Full) + (Maybe.Just b) + (cond + (> a b) (RangeResolution.Full) + (>= a total) (RangeResolution.Unsatisfiable) + (RangeResolution.Partial a + (if (< b (- total 1l)) + b + (- total 1l)))))))))))))) + +; Read the internal X-Sendfile-Range marker carrying the request's raw Range. +(hidden web-range-header) +(defn web-range-header [resp] + (match (Map.get-maybe (Response.headers resp) "X-Sendfile-Range") + (Maybe.Just vals) + (if (> (Array.length &vals) 0) + (Maybe.Just @(Array.unsafe-first &vals)) + (Maybe.Nothing)) + (Maybe.Nothing) (Maybe.Nothing))) + +(hidden web-strip-range-marker) +(defn web-strip-range-marker [resp] + (Response.set-headers resp + (Map.remove @(Response.headers &resp) + &@"X-Sendfile-Range"))) + +; Attach the request's Range header to a sendfile response as an internal +; marker, resolved against the file size at transfer setup. +(hidden web-mark-range) +(defn web-mark-range [req resp] + (match (web-sendfile-path &resp) + (Maybe.Nothing) resp + (Maybe.Just _) + (match (Request.header req "Range") + (Maybe.Nothing) resp + (Maybe.Just rv) + (let [k @"X-Sendfile-Range" + v [rv]] + (Response.set-headers resp (Map.put @(Response.headers &resp) &k &v)))))) + +; Resolve the marker attached by web-mark-range against a known file size. +(hidden web-resolve-marker) +(defn web-resolve-marker [resp total] + (match (web-range-header resp) + (Maybe.Nothing) (RangeResolution.Full) + (Maybe.Just rh) (web-resolve-range &rh total))) + +(hidden web-content-range) +(defn web-content-range [start end total] + (fmt "bytes %s-%s/%s" &(Long.str start) &(Long.str end) &(Long.str total))) + +(hidden web-accept-ranges) +(defn web-accept-ranges [resp] + (let [k @"Accept-Ranges" + v [@"bytes"]] + (Response.set-headers resp (Map.put @(Response.headers &resp) &k &v)))) + +; A 416 Range Not Satisfiable response with Content-Range: bytes */total. +(hidden web-range-not-satisfiable) +(defn web-range-not-satisfiable [total] + (let [cr-k @"Content-Range" + cr-v [(fmt "bytes */%s" &(Long.str total))] + ar-k @"Accept-Ranges" + ar-v [@"bytes"] + hdrs (Map.put (Map.put {} &cr-k &cr-v) &ar-k &ar-v)] + (Response.init 416 @"Range Not Satisfiable" @"HTTP/1.1" [] hdrs @""))) + +; Turn a full sendfile response into a 206 with Content-Range and +; Accept-Ranges. Content-Length and body are set by web-serialize-response. +(hidden web-partial-headers) +(defn web-partial-headers [resp start end total] + (let [cr-k @"Content-Range" + cr-v [(web-content-range start end total)] + ar-k @"Accept-Ranges" + ar-v [@"bytes"] + hdrs (Map.put + (Map.put @(Response.headers &resp) &cr-k &cr-v) + &ar-k + &ar-v)] + (Response.set-headers + (Response.with-status resp 206 @"Partial Content") + hdrs))) + +; HEAD variant: full-file sendfile response, body stripped, Accept-Ranges and +; Content-Length set from the file size. +(hidden web-head-full) +(defn web-head-full [resp sz] + (let [hdrs (Map.remove @(Response.headers &resp) &@"X-Sendfile") + ar-k @"Accept-Ranges" + ar-v [@"bytes"] + cl-k @"Content-Length" + cl-v [(Long.str sz)] + h (Map.put (Map.put hdrs &ar-k &ar-v) &cl-k &cl-v)] + (-> resp (Response.set-body @"") (Response.set-headers h)))) + +; HEAD variant: 206 with Content-Range and range-length Content-Length, no body. +(hidden web-head-partial) +(defn web-head-partial [resp start end total] + (let [hdrs (Map.remove @(Response.headers &resp) &@"X-Sendfile") + cr-k @"Content-Range" + cr-v [(web-content-range start end total)] + ar-k @"Accept-Ranges" + ar-v [@"bytes"] + cl-k @"Content-Length" + cl-v [(Long.str (+ (- end start) 1l))] + h (Map.put (Map.put (Map.put hdrs &cr-k &cr-v) &ar-k &ar-v) &cl-k &cl-v)] + (-> + (Response.with-status resp 206 @"Partial Content") + (Response.set-body @"") + (Response.set-headers h)))) + ; Strip the response body for HEAD requests while preserving Content-Length. ; For sendfile responses, computes Content-Length from the file and removes ; X-Sendfile so the server does not attempt a file transfer. @@ -1393,12 +1568,12 @@ responses. The `protocols` field lists the subprotocols that this route supports (ignore (IO.Raw.close-fd ffd)) (if (< sz 0l) (Response.not-found) - (let [hdrs (Map.remove @(Response.headers &resp) &@"X-Sendfile") - cl-k @"Content-Length" - cl-v [(Long.str sz)]] - (-> resp - (Response.set-body @"") - (Response.set-headers (Map.put hdrs &cl-k &cl-v)))))))) + (match (web-resolve-marker &resp sz) + (RangeResolution.Full) + (web-head-full (web-strip-range-marker resp) sz) + (RangeResolution.Partial start end) + (web-head-partial (web-strip-range-marker resp) start end sz) + (RangeResolution.Unsatisfiable) (web-range-not-satisfiable sz)))))) (Maybe.Nothing) (if (Map.contains? (Response.headers &resp) "Transfer-Encoding") (Response.set-body resp @"") @@ -1527,10 +1702,11 @@ responses. The `protocols` field lists the subprotocols that this route supports (let [r (Array.unsafe-nth (App.routes app) handler-idx)] (~(Route.handler r) &req ¶ms)))) after-resp (web-run-after after-hooks &req ¶ms resp) + ranged (web-mark-range &req after-resp) ; ETag conditional: return 304 if If-None-Match matches - cond-resp (if (web-etag-match? &req &after-resp) - (web-not-modified after-resp) - after-resp) + cond-resp (if (web-etag-match? &req &ranged) + (web-not-modified ranged) + ranged) ; HEAD: strip body, preserve Content-Length final (if is-head (web-strip-head-body cond-resp) cond-resp)] (Pair.init final ka))))) @@ -1712,6 +1888,51 @@ fallback.") (Map.remove! (ConnState.sf-sizes cs) &fd) (conn-done-writing cs poll fd))) + ; Open a sendfile response's target, resolve an optional byte range against + ; its size, and return the serialized head. A satisfiable range seeds the fd's + ; offset/size so the existing transfer loop sends exactly [start, end]. + (hidden setup-sendfile) + (defn setup-sendfile [cs fd resp ka] + (match (web-sendfile-path &resp) + (Maybe.Nothing) (web-serialize-response resp ka -1l) + (Maybe.Just p) + (let [ffd (IO.Raw.open &p IO.Raw.O-RDONLY)] + (if (< ffd 0) + (web-serialize-response (Response.not-found) ka -1l) + (let [sz (IO.Raw.fstat-size ffd)] + (if (< sz 0l) + (do + (ignore (IO.Raw.close-fd ffd)) + (web-serialize-response (Response.not-found) ka -1l)) + (match (web-resolve-marker &resp sz) + (RangeResolution.Full) + (do + (Map.put! (ConnState.sf-fds cs) &fd &ffd) + (Map.put! (ConnState.sf-offsets cs) &fd &0l) + (Map.put! (ConnState.sf-sizes cs) &fd &sz) + (web-serialize-response + (web-accept-ranges (web-strip-range-marker resp)) + ka + sz)) + (RangeResolution.Partial start end) + (do + (Map.put! (ConnState.sf-fds cs) &fd &ffd) + (Map.put! (ConnState.sf-offsets cs) &fd &start) + (Map.put! (ConnState.sf-sizes cs) &fd &(+ end 1l)) + (web-serialize-response + (web-partial-headers (web-strip-range-marker resp) + start + end + sz) + ka + (+ (- end start) 1l))) + (RangeResolution.Unsatisfiable) + (do + (ignore (IO.Raw.close-fd ffd)) + (web-serialize-response (web-range-not-satisfiable sz) + ka + -1l))))))))) + (hidden conn-cleanup) (defn conn-cleanup [cs poll fd] (do @@ -2331,37 +2552,7 @@ fallback.") false)) resp @(Pair.a &pair) ka @(Pair.b &pair) - sf-path (web-sendfile-path &resp) - sf-result (match-ref &sf-path - (Maybe.Just p) - (let [ffd (IO.Raw.open p - IO.Raw.O-RDONLY)] - (if (< ffd 0) - -1l - (let [sz (IO.Raw.fstat-size ffd)] - (if (< sz 0l) - (do - (ignore (IO.Raw.close-fd ffd)) - -1l) - (do - (Map.put! (ConnState.sf-fds cs) - &fd - &ffd) - (Map.put! (ConnState.sf-offsets cs) - &fd - &0l) - (Map.put! (ConnState.sf-sizes cs) - &fd - &sz) - sz))))) - (Maybe.Nothing) -1l) - actual-resp (if (and (Maybe.just? &sf-path) - (< sf-result 0l)) - (Response.not-found) - resp) - wbuf (web-serialize-response actual-resp - ka - sf-result)] + wbuf (setup-sendfile cs fd resp ka)] (Map.put! (ConnState.write-bufs cs) &fd &wbuf)