r5715 - in trunk/varnish-cache/bin: varnishd varnishtest/tests
phk at varnish-cache.org
phk at varnish-cache.org
Tue Jan 11 14:12:34 CET 2011
Author: phk
Date: 2011-01-11 14:12:34 +0100 (Tue, 11 Jan 2011)
New Revision: 5715
Added:
trunk/varnish-cache/bin/varnishtest/tests/g00003.vtc
Modified:
trunk/varnish-cache/bin/varnishd/cache_center.c
trunk/varnish-cache/bin/varnishd/cache_fetch.c
trunk/varnish-cache/bin/varnishd/cache_gzip.c
trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Flesh out the gzip and gunzip fetch processors.
Now you can get varnish to gzip or gunzip objects as they are
received from the backend by setting
sub vcl_fetch {
beresp.do_gzip = true;
beresp.do_guzip = true;
}
They will only do something if the object is/isn't gzip'ed already
(according to the beresp.http.Content-Encoding header) so the above
silly-ish example will compress al uncompressed objects and decompress
all compressed objects.
NB: Lots of errorchecks not implemented yet.
Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c 2011-01-11 12:00:42 UTC (rev 5714)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c 2011-01-11 13:12:34 UTC (rev 5715)
@@ -585,6 +585,22 @@
AZ(sp->objcore);
}
+ AZ(sp->wrk->vfp);
+ /* XXX: precedence, also: do_esi */
+
+ if (sp->wrk->do_gunzip &&
+ http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
+ http_Unset(sp->wrk->beresp, H_Content_Encoding);
+ sp->wrk->vfp = &vfp_gunzip;
+ }
+
+ if (sp->wrk->do_gzip &&
+ !http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
+ http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->beresp,
+ "Content-Encoding: %s", "gzip");
+ sp->wrk->vfp = &vfp_gzip;
+ }
+
l = http_EstimateWS(sp->wrk->beresp,
sp->pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
@@ -642,12 +658,6 @@
else
sp->obj->last_modified = sp->wrk->entered;
- AZ(sp->wrk->vfp);
- /* XXX: precedence, also: do_esi */
- if (sp->wrk->do_gunzip)
- sp->wrk->vfp = &vfp_gunzip;
- else if (sp->wrk->do_gzip)
- sp->wrk->vfp = &vfp_gzip;
i = FetchBody(sp);
sp->wrk->vfp = NULL;
AZ(sp->wrk->wfd);
Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2011-01-11 12:00:42 UTC (rev 5714)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2011-01-11 13:12:34 UTC (rev 5715)
@@ -60,7 +60,7 @@
* 'estimate' is the estimate of the number of bytes we expect to receive,
* as seen on the socket, or zero if unknown.
*/
-static void
+static void __match_proto__()
vfp_nop_begin(struct sess *sp, size_t estimate)
{
@@ -84,7 +84,7 @@
* Return 1 when 'bytes' have been processed.
*/
-static int
+static int __match_proto__()
vfp_nop_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
{
ssize_t l, w;
@@ -130,7 +130,7 @@
* Return 0 for OK
*/
-static int
+static int __match_proto__()
vfp_nop_end(struct sess *sp)
{
struct storage *st;
Modified: trunk/varnish-cache/bin/varnishd/cache_gzip.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_gzip.c 2011-01-11 12:00:42 UTC (rev 5714)
+++ trunk/varnish-cache/bin/varnishd/cache_gzip.c 2011-01-11 13:12:34 UTC (rev 5715)
@@ -41,6 +41,7 @@
SVNID("$Id$")
#include "cache.h"
+#include "stevedore.h"
#include "zlib.h"
@@ -52,7 +53,7 @@
char *tmp_snapshot;
struct ws *buf;
- unsigned bufsiz;
+ size_t bufsiz;
z_stream vz;
};
@@ -173,31 +174,86 @@
* A VFP for gunzip'ing an object as we receive it from the backend
*/
-static void
+static void __match_proto__()
vfp_gunzip_begin(struct sess *sp, size_t estimate)
{
(void)estimate;
sp->wrk->vfp_private = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
}
-static int
+static int __match_proto__()
vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
{
- struct vgz *vgz;
+ struct vgz *vg;
+ struct storage *st;
+ ssize_t l, w;
+ int i = -100;
- CAST_OBJ_NOTNULL(vgz, sp->wrk->vfp_private, VGZ_MAGIC);
- (void)htc;
- (void)bytes;
+ CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+ AZ(vg->vz.avail_in);
+ while (bytes > 0 || vg->vz.avail_in > 0) {
+ if (sp->wrk->storage == NULL)
+ sp->wrk->storage = STV_alloc(sp,
+ params->fetch_chunksize * 1024LL);
+ if (sp->wrk->storage == NULL) {
+ errno = ENOMEM;
+ return (-1);
+ }
+ st = sp->wrk->storage;
+
+ vg->vz.next_out = st->ptr + st->len;
+ vg->vz.avail_out = st->space - st->len;
+
+ if (vg->vz.avail_in == 0 && bytes > 0) {
+ l = vg->bufsiz;
+ if (l > bytes)
+ l = bytes;
+ w = HTC_Read(htc, vg->buf->f, l);
+ if (w <= 0)
+ return (w);
+ vg->vz.next_in = (void*)vg->buf->f;
+ vg->vz.avail_in = w;
+ bytes -= w;
+ }
+
+ i = inflate(&vg->vz, 0);
+ assert(i == Z_OK || i == Z_STREAM_END);
+ st->len = st->space - vg->vz.avail_out;
+ if (st->len == st->space) {
+ VTAILQ_INSERT_TAIL(&sp->obj->store,
+ sp->wrk->storage, list);
+ sp->obj->len += st->len;
+ sp->wrk->storage = NULL;
+ }
+ }
+ if (i == Z_STREAM_END)
+ return (1);
return (-1);
}
-static int
+static int __match_proto__()
vfp_gunzip_end(struct sess *sp)
{
- struct vgz *vgz;
+ struct vgz *vg;
+ struct storage *st;
- CAST_OBJ_NOTNULL(vgz, sp->wrk->vfp_private, VGZ_MAGIC);
- return (-1);
+ CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+ VGZ_Destroy(&vg);
+
+ st = sp->wrk->storage;
+ sp->wrk->storage = NULL;
+ if (st == NULL)
+ return (0);
+
+ if (st->len == 0) {
+ STV_free(st);
+ return (0);
+ }
+ if (st->len < st->space)
+ STV_trim(st, st->len);
+ sp->obj->len += st->len;
+ VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+ return (0);
}
struct vfp vfp_gunzip = {
@@ -213,27 +269,98 @@
* A VFP for gzip'ing an object as we receive it from the backend
*/
-static void
+static void __match_proto__()
vfp_gzip_begin(struct sess *sp, size_t estimate)
{
- (void)sp;
+ struct vgz *vg;
(void)estimate;
+
+ vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
+ /* XXX: hack */
+ memset(&vg->vz, 0, sizeof vg->vz);
+ assert(Z_OK == deflateInit2(&vg->vz,
+ 0,
+ Z_DEFLATED,
+ 31,
+ 9,
+ Z_DEFAULT_STRATEGY));
+
+ sp->wrk->vfp_private = vg;
}
-static int
+static int __match_proto__()
vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
{
- (void)sp;
- (void)htc;
- (void)bytes;
+ struct vgz *vg;
+ struct storage *st;
+ ssize_t l, w;
+ int i = -100;
+
+ CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+ AZ(vg->vz.avail_in);
+ while (bytes > 0 || vg->vz.avail_in > 0) {
+ if (sp->wrk->storage == NULL)
+ sp->wrk->storage = STV_alloc(sp,
+ params->fetch_chunksize * 1024LL);
+ if (sp->wrk->storage == NULL) {
+ errno = ENOMEM;
+ return (-1);
+ }
+ st = sp->wrk->storage;
+
+ vg->vz.next_out = st->ptr + st->len;
+ vg->vz.avail_out = st->space - st->len;
+
+ if (vg->vz.avail_in == 0 && bytes > 0) {
+ l = vg->bufsiz;
+ if (l > bytes)
+ l = bytes;
+ w = HTC_Read(htc, vg->buf->f, l);
+ if (w <= 0)
+ return (w);
+ vg->vz.next_in = (void*)vg->buf->f;
+ vg->vz.avail_in = w;
+ bytes -= w;
+ }
+
+ i = deflate(&vg->vz, bytes == 0 ? Z_FINISH : 0);
+ assert(i == Z_OK || i == Z_STREAM_END);
+ st->len = st->space - vg->vz.avail_out;
+ if (st->len == st->space) {
+ VTAILQ_INSERT_TAIL(&sp->obj->store,
+ sp->wrk->storage, list);
+ sp->obj->len += st->len;
+ sp->wrk->storage = NULL;
+ }
+ }
+ if (i == Z_STREAM_END)
+ return (1);
return (-1);
}
-static int
+static int __match_proto__()
vfp_gzip_end(struct sess *sp)
{
- (void)sp;
- return (-1);
+ struct vgz *vg;
+ struct storage *st;
+
+ CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+ VGZ_Destroy(&vg);
+
+ st = sp->wrk->storage;
+ sp->wrk->storage = NULL;
+ if (st == NULL)
+ return (0);
+
+ if (st->len == 0) {
+ STV_free(st);
+ return (0);
+ }
+ if (st->len < st->space)
+ STV_trim(st, st->len);
+ sp->obj->len += st->len;
+ VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+ return (0);
}
struct vfp vfp_gzip = {
@@ -241,4 +368,3 @@
.bytes = vfp_gzip_bytes,
.end = vfp_gzip_end,
};
-
Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c 2011-01-11 12:00:42 UTC (rev 5714)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c 2011-01-11 13:12:34 UTC (rev 5715)
@@ -236,6 +236,7 @@
/*--------------------------------------------------------------------
* We have a gzip'ed object and need to ungzip it for a client which
* does not understand gzip.
+ * XXX: handle invalid gzip data better (how ?)
*/
static void
Added: trunk/varnish-cache/bin/varnishtest/tests/g00003.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/g00003.vtc (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/g00003.vtc 2011-01-11 13:12:34 UTC (rev 5715)
@@ -0,0 +1,54 @@
+# $Id$
+
+test "test gunzip on fetch"
+
+server s1 {
+ rxreq
+ expect req.url == "/foo"
+ expect req.http.accept-encoding == "gzip"
+ txresp -gziplen 41
+
+ rxreq
+ expect req.url == "/bar"
+ expect req.http.accept-encoding == "gzip"
+ txresp -bodylen 42
+
+ rxreq
+ expect req.url == "/foobar"
+ expect req.http.accept-encoding == "gzip"
+ txresp -bodylen 43
+} -start
+
+varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend {
+
+ sub vcl_fetch {
+ set beresp.do_gunzip = true;
+ if (req.url == "/foobar") {
+ set beresp.do_gzip = true;
+ }
+ }
+} -start
+
+client c1 {
+ txreq -url /foo -hdr "Accept-Encoding: gzip"
+ rxresp
+ expect resp.http.content-encoding == "resp.http.content-encoding"
+ expect resp.bodylen == 41
+
+ txreq -url /bar -hdr "Accept-Encoding: gzip"
+ rxresp
+ expect resp.http.content-encoding == "resp.http.content-encoding"
+ expect resp.bodylen == 42
+
+ txreq -url /foobar -hdr "Accept-Encoding: gzip"
+ rxresp
+ expect resp.http.content-encoding == "gzip"
+ gunzip
+ expect resp.bodylen == 43
+
+ txreq -url /foobar
+ rxresp
+ expect resp.http.content-encoding == "resp.http.content-encoding"
+ expect resp.bodylen == 43
+} -run
+
More information about the varnish-commit
mailing list