[master] 35febe9 Yah! Enable gzip-file stitching and pass the first trivial ESI+GZIP testcase
Poul-Henning Kamp
phk at project.varnish-software.com
Fri Jan 21 19:14:47 CET 2011
commit 35febe95ae77ba13c4da970d3cc26468abb1e517
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date: Fri Jan 21 18:13:50 2011 +0000
Yah! Enable gzip-file stitching and pass the first trivial
ESI+GZIP testcase
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index b0497fb..9ee361d 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -98,6 +98,7 @@ struct ban;
struct SHA256Context;
struct vsc_lck;
struct waitinglist;
+struct vef_priv;
struct lock { void *priv; }; // Opaque
@@ -280,7 +281,7 @@ struct worker {
struct storage *storage;
struct vfp *vfp;
struct vgz *vgz_rx;
- struct vgz *vgz_esi;
+ struct vef_priv *vef_priv;
unsigned do_esi;
unsigned do_gzip;
unsigned is_gzip;
diff --git a/bin/varnishd/cache_esi.h b/bin/varnishd/cache_esi.h
index b57cda9..4110687 100644
--- a/bin/varnishd/cache_esi.h
+++ b/bin/varnishd/cache_esi.h
@@ -27,6 +27,7 @@
*
*/
+#define VEC_GZ (0x21)
#define VEC_V1 (0x40 + 1)
#define VEC_V2 (0x40 + 2)
#define VEC_V8 (0x40 + 8)
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 6a344ec..a17d101 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -174,8 +174,10 @@ ESI_Deliver(struct sess *sp)
struct storage *st;
uint8_t *p, *e, *q, *r;
unsigned off;
- ssize_t l, l_crc;
- uint32_t crc, crc_ref;
+ ssize_t l, l_icrc, l_crc;
+ uint32_t crc, icrc;
+ uint8_t tailbuf[8 + 5];
+ int dogzip;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
st = sp->obj->esidata;
@@ -183,6 +185,14 @@ ESI_Deliver(struct sess *sp)
p = st->ptr;
e = st->ptr + st->len;
+ if (*p == VEC_GZ) {
+ p++;
+ dogzip = 1;
+ crc = crc32(0L, Z_NULL, 0);
+ l_crc = 0;
+ } else
+ dogzip = 0;
+
st = VTAILQ_FIRST(&sp->obj->store);
off = 0;
@@ -192,15 +202,19 @@ ESI_Deliver(struct sess *sp)
case VEC_V2:
case VEC_V8:
l = ved_decode_len(&p);
- assert(*p == VEC_C1 || *p == VEC_C2 || *p == VEC_C8);
- l_crc = ved_decode_len(&p);
- crc = vbe32dec(p);
- p += 4;
+ if (dogzip) {
+ assert(*p == VEC_C1 || *p == VEC_C2 ||
+ *p == VEC_C8);
+ l_icrc = ved_decode_len(&p);
+ icrc = vbe32dec(p);
+ p += 4;
+ }
q = (void*)strchr((const char*)p, '\0');
assert (q > p);
- crc_ref = crc32(0L, Z_NULL, 0);
- crc_ref = crc32(crc_ref, st->ptr + off, l);
- xxxassert(crc_ref == crc);
+ if (dogzip) {
+ crc = crc32_combine(crc, icrc, l_icrc);
+ l_crc += l_icrc;
+ }
ved_sendchunk(sp, p, q - p, st->ptr + off, l);
off += l;
p = q + 1;
@@ -229,6 +243,22 @@ ESI_Deliver(struct sess *sp)
INCOMPL();
}
}
+ if (dogzip) {
+ /* Emit a gzip literal block with finish bit set */
+ tailbuf[0] = 0x01;
+ tailbuf[1] = 0x00;
+ tailbuf[2] = 0x00;
+ tailbuf[3] = 0xff;
+ tailbuf[4] = 0xff;
+
+ /* Emit CRC32 */
+ vle32enc(tailbuf + 5, crc);
+
+ /* MOD(2^32) length */
+ vle32enc(tailbuf + 9, l_crc);
+
+ ved_sendchunk(sp, "d\r\n", 3, tailbuf, 13);
+ }
(void)WRW_Flush(sp->wrk);
}
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 690f41f..68643d2 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -25,7 +25,7 @@
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
- * VEP Varnish Esi Parsing
+ * VEF Varnish Esi Fetching
*/
#include "config.h"
@@ -95,16 +95,6 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
}
/*---------------------------------------------------------------------
- * We receive a ungzip'ed object, and want to store it gzip'ed.
- */
-
-static int __match_proto__()
-vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
-{
- return (vfp_esi_bytes_uu(sp, htc, bytes));
-}
-
-/*---------------------------------------------------------------------
* We receive a gzip'ed object, and want to store it ungzip'ed.
*/
@@ -160,6 +150,123 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
}
/*---------------------------------------------------------------------
+ */
+
+struct vef_priv {
+ unsigned magic;
+#define VEF_MAGIC 0xf104b51f
+ struct vgz *vgz;
+
+ char *bufp;
+ ssize_t tot;
+ int error;
+};
+
+/*---------------------------------------------------------------------
+ * We receive a ungzip'ed object, and want to store it gzip'ed.
+ */
+
+#include "vend.h"
+
+static ssize_t
+vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
+{
+ struct vef_priv *vef;
+ struct storage *st;
+ size_t dl;
+ const void *dp;
+ int i;
+ char *p;
+
+printf("ZCB(%jd, %d)\n", l, flg);
+
+ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ vef = sp->wrk->vef_priv;
+ CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+ assert(l >= 0);
+
+ if (vef->error) {
+ vef->tot += l;
+ return (vef->tot);
+ }
+
+ VGZ_Ibuf(vef->vgz, vef->bufp, l);
+ do {
+ if (sp->wrk->storage == NULL) {
+ l = params->fetch_chunksize * 1024LL;
+ sp->wrk->storage = STV_alloc(sp, l);
+ }
+ if (sp->wrk->storage == NULL) {
+ vef->error = ENOMEM;
+ vef->tot += l;
+ return (vef->tot);
+ }
+
+ st = sp->wrk->storage;
+ VGZ_Obuf(vef->vgz, st->ptr + st->len, st->space - st->len);
+ p = (void*)(st->ptr + st->len);
+ i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
+printf("GZI = %d %jd\n", i, dl);
+ vef->tot += dl;
+ st->len += dl;
+ if (st->len == st->space) {
+ VTAILQ_INSERT_TAIL(&sp->obj->store,
+ sp->wrk->storage, list);
+ sp->wrk->storage = NULL;
+ st = NULL;
+ }
+ } while (!VGZ_IbufEmpty(vef->vgz));
+ vef->bufp += l;
+if (flg == VGZ_FINISH)
+ assert(i == 1); /* XXX */
+else
+ assert(i == 0); /* XXX */
+printf("ZCB = %jd\n", vef->tot);
+fflush(stdout);
+usleep(100);
+ return (vef->tot);
+}
+
+static int __match_proto__()
+vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+ ssize_t l, w;
+ char ibuf[1024 * params->gzip_stack_buffer];
+ struct vef_priv *vef;
+
+ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ vef = sp->wrk->vef_priv;
+ CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+
+ while (bytes > 0) {
+ if (sp->wrk->storage == NULL) {
+ l = params->fetch_chunksize * 1024LL;
+ sp->wrk->storage = STV_alloc(sp, l);
+ }
+ if (sp->wrk->storage == NULL) {
+ errno = ENOMEM;
+ return (-1);
+ }
+ l = sizeof ibuf;
+ if (l > bytes)
+ l = bytes;
+ w = HTC_Read(htc, ibuf, l);
+ if (w <= 0) {
+printf("RT %jd\n", w);
+ return (w);
+}
+ bytes -= w;
+ vef->bufp = ibuf;
+ VEP_parse(sp, ibuf, l);
+ if (vef->error) {
+ // errno = vef->error;
+ return (-1);
+ }
+ }
+ return (1);
+}
+
+/*---------------------------------------------------------------------
* We receive a gzip'ed object, and want to store it gzip'ed.
*/
@@ -175,14 +282,26 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
static void __match_proto__()
vfp_esi_begin(struct sess *sp, size_t estimate)
{
+ struct vef_priv *vef;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
/* XXX: snapshot WS's ? We'll need the space */
- VEP_Init(sp, NULL);
-
- if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
+ if (sp->wrk->is_gzip && sp->wrk->do_gunzip) {
sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
+ VEP_Init(sp, NULL);
+ } else if (sp->wrk->is_gunzip && sp->wrk->do_gzip) {
+ VEP_Init(sp, vfp_vep_callback);
+ vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
+ AN(vef);
+ memset(vef, 0, sizeof *vef);
+ vef->magic = VEF_MAGIC;
+ vef->vgz = VGZ_NewGzip(sp, sp->ws);
+ AZ(sp->wrk->vef_priv);
+ sp->wrk->vef_priv = vef;
+ } else {
+ VEP_Init(sp, NULL);
+ }
(void)estimate;
}
@@ -193,6 +312,7 @@ vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
int i;
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+printf("BYTES = %jd\n", bytes);
if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
i = vfp_esi_bytes_gu(sp, htc, bytes);
else if (sp->wrk->is_gunzip && sp->wrk->do_gzip)
@@ -201,6 +321,7 @@ vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
i = vfp_esi_bytes_gg(sp, htc, bytes);
else
i = vfp_esi_bytes_uu(sp, htc, bytes);
+printf("BYTES = %d\n", i);
return (i);
}
@@ -209,8 +330,10 @@ vfp_esi_end(struct sess *sp)
{
struct storage *st;
struct vsb *vsb;
+ struct vef_priv *vef;
ssize_t l;
+printf("END\n");
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
VGZ_Destroy(&sp->wrk->vgz_rx);
@@ -230,6 +353,16 @@ vfp_esi_end(struct sess *sp)
st = sp->wrk->storage;
sp->wrk->storage = NULL;
+
+ if (sp->wrk->vef_priv != NULL) {
+ vef = sp->wrk->vef_priv;
+ sp->wrk->vef_priv = NULL;
+ CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+ XXXAZ(vef->error);
+printf("TOT %jd\n", vef->tot);
+sp->obj->len = vef->tot;
+ }
+
if (st == NULL)
return (0);
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 0be7faa..5e85eff 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -64,6 +64,7 @@ struct vep_state {
struct vsb *vsb;
const struct sess *sp;
+ int dogzip;
vep_callback_t *cb;
/* Internal Counter for default call-back function */
@@ -285,9 +286,12 @@ vep_emit_verbatim(const struct vep_state *vep, ssize_t l, ssize_t l_crc)
Debug("---> VERBATIM(%jd)\n", (intmax_t)l);
}
vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V8);
- vep_emit_len(vep, l_crc, VEC_C1, VEC_C2, VEC_C8);
- vbe32enc(buf, vep->crc);
- vsb_bcat(vep->vsb, buf, sizeof buf);
+ if (vep->dogzip) {
+ vep_emit_len(vep, l_crc, VEC_C1, VEC_C2, VEC_C8);
+ vbe32enc(buf, vep->crc);
+ vsb_bcat(vep->vsb, buf, sizeof buf);
+ }
+ /* Emit Chunked header */
vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
}
@@ -992,14 +996,19 @@ VEP_Init(const struct sess *sp, vep_callback_t *cb)
memset(vep, 0, sizeof *vep);
vep->magic = VEP_MAGIC;
vep->sp = sp;
- if (cb != NULL)
+ vep->vsb = vsb_newauto();
+ AN(vep->vsb);
+
+ if (cb != NULL) {
+ vep->dogzip = 1;
+ /* XXX */
+ vsb_printf(vep->vsb, "%c", VEC_GZ);
vep->cb = cb;
- else
+ } else {
vep->cb = vep_default_cb;
+ }
vep->state = VEP_START;
- vep->vsb = vsb_newauto();
- AN(vep->vsb);
vep->crc = crc32(0L, Z_NULL, 0);
vep->crcp = crc32(0L, Z_NULL, 0);
}
@@ -1020,8 +1029,9 @@ VEP_Finish(const struct sess *sp)
if (vep->o_pending)
vep_mark_common(vep, vep->ver_p, vep->last_mark);
if (vep->o_wait > 0) {
- lcb = vep->cb(vep->sp, 0, VGZ_FINISH);
+ lcb = vep->cb(vep->sp, 0, VGZ_ALIGN);
vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
+ (void)vep->cb(vep->sp, 0, VGZ_FINISH);
}
sp->wrk->vep = NULL;
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 7de0fb7..8f113f2 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -204,8 +204,8 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
i = sp->wrk->vfp->bytes(sp, htc, cl);
if (i <= 0) {
- WSP(sp, SLT_FetchError, "straight read_error: %d (%s)",
- errno, strerror(errno));
+ WSP(sp, SLT_FetchError, "straight read_error: %d %d (%s)",
+ i, errno, strerror(errno));
return (-1);
}
return (0);
diff --git a/bin/varnishtest/tests/e00021.vtc b/bin/varnishtest/tests/e00021.vtc
new file mode 100644
index 0000000..1debf38
--- /dev/null
+++ b/bin/varnishtest/tests/e00021.vtc
@@ -0,0 +1,41 @@
+# $Id$
+
+test "ESI ability to stitch gzip files together"
+
+
+server s1 {
+ rxreq
+ txresp -body {
+ <esi:remove>
+ This is a test: Unseen University
+ <esi:include src="trick question">
+ <!--esi XXX -->
+ </esi:remove>
+ <esX>This is a test: Hello world
+ }
+} -start
+
+varnish v1 -vcl+backend {
+ sub vcl_recv {
+ set req.esi = true;
+ }
+ sub vcl_fetch {
+ set beresp.do_esi = true;
+ set beresp.do_gzip = true;
+ }
+} -start
+
+varnish v1 -cliok "param.set esi_syntax 4"
+varnish v1 -cliok "param.set http_gzip_support true"
+
+client c1 {
+ txreq -hdr "Accept-Encoding: gzip"
+ rxresp
+ expect resp.http.content-encoding == gzip
+ gunzip
+ expect resp.status == 200
+ expect resp.bodylen == 40
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 2
More information about the varnish-commit
mailing list