[4.0] ac94f16 Now really make chunked req.body work with pass.
Poul-Henning Kamp
phk at FreeBSD.org
Tue Jun 24 11:31:57 CEST 2014
commit ac94f1662761f9d9b7e3f527a08476a5cddb7dfc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date: Mon Jun 23 12:29:33 2014 +0000
Now really make chunked req.body work with pass.
Please test this ASAP (before 4.0.1)
diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 30fcaea..cfb2238 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -60,12 +60,6 @@ sub vcl_recv {
return (pipe);
}
- /* We don't support chunked uploads, except when piping. */
- if ((req.method == "POST" || req.method == "PUT") &&
- req.http.transfer-encoding ~ "chunked") {
- return(pipe);
- }
-
if (req.method != "GET" && req.method != "HEAD") {
/* We only deal with GET and HEAD by default */
return (pass);
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index aa009f2..b6d72dd 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -192,6 +192,31 @@ V1F_Setup_Fetch(struct busyobj *bo)
}
/*--------------------------------------------------------------------
+ * Pass the request body to the backend with chunks
+ */
+
+static int __match_proto__(req_body_iter_f)
+vbf_iter_req_body_chunked(struct req *req, void *priv, void *ptr, size_t l)
+{
+ struct worker *wrk;
+ char buf[20];
+
+ CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+ CAST_OBJ_NOTNULL(wrk, priv, WORKER_MAGIC);
+
+ if (l > 0) {
+ bprintf(buf, "%jx\r\n", (uintmax_t)l);
+ VSLb(req->vsl, SLT_Debug, "WWWW: %s", buf);
+ (void)WRW_Write(wrk, buf, strlen(buf));
+ (void)WRW_Write(wrk, ptr, l);
+ (void)WRW_Write(wrk, "\r\n", 2);
+ if (WRW_Flush(wrk))
+ return (-1);
+ }
+ return (0);
+}
+
+/*--------------------------------------------------------------------
* Pass the request body to the backend
*/
@@ -231,6 +256,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
int i, j, first;
struct http_conn *htc;
ssize_t hdrbytes;
+ int do_chunked = 0;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
CHECK_OBJ_ORNULL(req, REQ_MAGIC);
@@ -262,6 +288,11 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
if (!http_GetHdr(bo->bereq, H_Host, NULL))
VDI_AddHostHeader(bo->bereq, vc);
+ if (req != NULL && req->req_body_status == REQ_BODY_CHUNKED) {
+ http_PrintfHeader(hp, "Transfer-Encoding: chunked");
+ do_chunked = 1;
+ }
+
(void)VTCP_blocking(vc->fd); /* XXX: we should timeout instead */
WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_prev);
hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req);
@@ -270,10 +301,16 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
i = 0;
if (req != NULL) {
- i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk);
- if (req->req_body_status == REQ_BODY_DONE)
+ if (do_chunked) {
+ i = HTTP1_IterateReqBody(req,
+ vbf_iter_req_body_chunked, wrk);
+ (void)WRW_Write(wrk, "0\r\n\r\n", 5);
+ } else {
+ i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk);
+ }
+ if (req->req_body_status == REQ_BODY_DONE) {
retry = -1;
- if (req->req_body_status == REQ_BODY_FAIL) {
+ } else if (req->req_body_status == REQ_BODY_FAIL) {
VSLb(bo->vsl, SLT_FetchError,
"req.body read error: %d (%s)",
errno, strerror(errno));
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index dac34e0..454ec28 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -285,8 +285,10 @@ http1_req_body_status(struct req *req)
req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
return (REQ_BODY_PRESENT);
}
- if (http_HdrIs(req->http, H_Transfer_Encoding, "chunked"))
+ if (http_HdrIs(req->http, H_Transfer_Encoding, "chunked")) {
+ req->chunk_ctr = -1;
return (REQ_BODY_CHUNKED);
+ }
if (http_GetHdr(req->http, H_Transfer_Encoding, NULL))
return (REQ_BODY_FAIL);
return (REQ_BODY_NONE);
@@ -512,6 +514,7 @@ http1_iter_req_body(struct req *req, enum req_body_state_e bs,
switch (HTTP1_Chunked(req->htc, &req->chunk_ctr, &err,
&req->acct.req_bodybytes, buf, &len)) {
case H1CR_ERROR:
+ VSLb(req->vsl, SLT_Debug, "CHUNKERR: %s", err);
return (-1);
case H1CR_MORE:
return (len);
@@ -639,7 +642,7 @@ int
HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
{
struct storage *st;
- ssize_t l;
+ ssize_t l, l2;
CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -661,6 +664,7 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
req->req_body_status = REQ_BODY_FAIL;
return (-1);
}
+ l2 = 0;
st = NULL;
do {
@@ -690,13 +694,28 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
break;
}
if (l > 0) {
+ l2 += l;
st->len += l;
if (st->space == st->len)
st = NULL;
}
} while (l > 0);
- if (l == 0)
+ if (l == 0) {
+ req->req_bodybytes = l2;
+ /* We must update also the "pristine" req.* copy */
+
+ http_Unset(req->http0, H_Content_Length);
+ http_Unset(req->http0, H_Transfer_Encoding);
+ http_PrintfHeader(req->http0, "Content-Length: %ju",
+ req->req_bodybytes);
+
+ http_Unset(req->http, H_Content_Length);
+ http_Unset(req->http, H_Transfer_Encoding);
+ http_PrintfHeader(req->http, "Content-Length: %ju",
+ req->req_bodybytes);
+
req->req_body_status = REQ_BODY_CACHED;
+ }
VSLb_ts_req(req, "ReqBody", VTIM_real());
return (l);
}
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index a3e2005..f0a1abc 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -196,7 +196,7 @@ HTTP1_Read(struct http_conn *htc, void *d, size_t len)
{
size_t l;
unsigned char *p;
- ssize_t i;
+ ssize_t i = 0;
CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
l = 0;
@@ -212,12 +212,12 @@ HTTP1_Read(struct http_conn *htc, void *d, size_t len)
if (htc->pipeline.b == htc->pipeline.e)
htc->pipeline.b = htc->pipeline.e = NULL;
}
- if (len == 0)
- return (l);
- i = read(htc->fd, p, len);
- if (i < 0) {
- VSLb(htc->vsl, SLT_FetchError, "%s", strerror(errno));
- return (i);
+ if (len > 0) {
+ i = read(htc->fd, p, len);
+ if (i < 0) {
+ VSLb(htc->vsl, SLT_FetchError, "%s", strerror(errno));
+ return (i);
+ }
}
return (i + l);
}
diff --git a/bin/varnishtest/tests/c00055.vtc b/bin/varnishtest/tests/c00055.vtc
index e02bc47..0344d02 100644
--- a/bin/varnishtest/tests/c00055.vtc
+++ b/bin/varnishtest/tests/c00055.vtc
@@ -25,7 +25,7 @@ varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend {
varnish v1 -cliok "param.set debug +syncvsl"
client c1 {
- txreq -body "FOO"
+ txreq -req "POST" -body "FOO"
rxresp
expect resp.http.Foo == "Foo"
expect resp.bodylen == 2
diff --git a/bin/varnishtest/tests/c00067.vtc b/bin/varnishtest/tests/c00067.vtc
index b57f904..73bcb34 100644
--- a/bin/varnishtest/tests/c00067.vtc
+++ b/bin/varnishtest/tests/c00067.vtc
@@ -4,11 +4,16 @@ server s1 {
rxreq
expect req.bodylen == 106
txresp -body "ABCD"
+ rxreq
+ expect req.bodylen == 108
+ txresp -body "ABCDE"
} -start
varnish v1 -vcl+backend {
} -start
+varnish v1 -cliok "param.set debug +syncvsl"
+
client c1 {
txreq -req POST -nolen -hdr "Transfer-encoding: chunked"
chunked {BLA}
@@ -22,3 +27,25 @@ client c1 {
expect resp.status == 200
expect resp.bodylen == 4
} -run
+
+delay .2
+
+varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend {
+ sub vcl_recv {
+ C{ VRT_CacheReqBody(ctx, 1000); }C
+ }
+}
+
+client c1 {
+ txreq -req POST -nolen -hdr "Transfer-encoding: chunked"
+ chunked {BLAS}
+ delay .2
+ chunkedlen 100
+ delay .2
+ chunked {TFOO}
+ delay .2
+ chunkedlen 0
+ rxresp
+ expect resp.status == 200
+ expect resp.bodylen == 5
+} -run
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 966f043..28b7340 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -458,7 +458,7 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
return;
}
p = http_find_header(hh, "transfer-encoding");
- if (p != NULL && !strcmp(p, "chunked")) {
+ if (p != NULL && !strcasecmp(p, "chunked")) {
while (http_rxchunk(hp) != 0)
continue;
vtc_dump(hp->vl, 4, "body", hp->body, ll);
More information about the varnish-commit
mailing list