cache_req

From guillaume at varnish-software.com Fri Feb 1 19:06:09 2019 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Fri, 1 Feb 2019 19:06:09 +0000 (UTC) Subject: [master] fd9e40dba stricter magic checks Message-ID: <20190201190609.1D747AD81E@lists.varnish-cache.org> commit fd9e40dba9470c8e4242428a2c5ce774f754ccea Author: Guillaume Quintard Date: Thu Jan 31 14:22:20 2019 -0800 stricter magic checks diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 9b4f5fedb..76adf6ef6 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -429,6 +429,10 @@ VRT_r_obj_storage(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); + AN(ctx->req->objcore->stobj); + CHECK_OBJ_NOTNULL(ctx->req->objcore->stobj->stevedore, + STEVEDORE_MAGIC); return (ctx->req->objcore->stobj->stevedore); } @@ -673,6 +677,8 @@ VRT_r_req_xid(VRT_CTX) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(ctx->req->http, HTTP_MAGIC); + AN(ctx->req->vsl); return (WS_Printf(ctx->req->http->ws, "%u", VXID(ctx->req->vsl->wid))); @@ -684,6 +690,8 @@ VRT_r_bereq_xid(VRT_CTX) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo->bereq, HTTP_MAGIC); + AN(ctx->bo->vsl); return (WS_Printf(ctx->bo->bereq->ws, "%u", VXID(ctx->bo->vsl->wid))); @@ -692,15 +700,26 @@ VRT_r_bereq_xid(VRT_CTX) VCL_STRING VRT_r_sess_xid(VRT_CTX) { + struct sess *sp; + struct http *http; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (VALID_OBJ(ctx->req, REQ_MAGIC)) - return (WS_Printf(ctx->req->http->ws, "%u", - VXID(ctx->req->sp->vxid))); + if (ctx->req) { + CHECK_OBJ(ctx->req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(ctx->req->http, HTTP_MAGIC); + sp = ctx->req->sp; + http = ctx->req->http; + } else { + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo->bereq, HTTP_MAGIC); + sp = ctx->bo->sp; + http = ctx->bo->bereq; + } - CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); - return (WS_Printf(ctx->bo->bereq->ws, "%u", - VXID(ctx->bo->sp->vxid))); + CHECK_OBJ_NOTNULL(http, HTTP_MAGIC); + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + return (WS_Printf(http->ws, "%u", VXID(sp->vxid))); } /*-------------------------------------------------------------------- @@ -763,9 +782,10 @@ VRT_r_local_##var(VRT_CTX) \ struct sess *sp; \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ - if (VALID_OBJ(ctx->req, REQ_MAGIC)) \ + if (ctx->req) { \ + CHECK_OBJ(ctx->req, REQ_MAGIC); \ sp = ctx->req->sp; \ - else { \ + } else { \ CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); \ sp = ctx->bo->sp; \ } \ diff --git a/lib/libvmod_unix/vmod_unix.c b/lib/libvmod_unix/vmod_unix.c index 59bbb2d8e..4388d58d4 100644 --- a/lib/libvmod_unix/vmod_unix.c +++ b/lib/libvmod_unix/vmod_unix.c @@ -67,9 +67,12 @@ get_sp(VRT_CTX) { struct sess *sp; - if (VALID_OBJ(ctx->req, REQ_MAGIC)) + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + if (ctx->req) { + CHECK_OBJ(ctx->req, REQ_MAGIC); sp = ctx->req->sp; - else { + } else { CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); sp = ctx->bo->sp; } From slink at schokola.de Sat Feb 2 13:28:23 2019 From: slink at schokola.de (Nils Goroll) Date: Sat, 2 Feb 2019 14:28:23 +0100 Subject: [master] fd9e40dba stricter magic checks In-Reply-To: <20190201190609.1D747AD81E@lists.varnish-cache.org> References: <20190201190609.1D747AD81E@lists.varnish-cache.org> Message-ID: On 01/02/2019 20:06, Guillaume Quintard wrote: > > commit fd9e40dba9470c8e4242428a2c5ce774f754ccea > Author: Guillaume Quintard > Date: Thu Jan 31 14:22:20 2019 -0800 > ... > @@ -692,15 +700,26 @@ VRT_r_bereq_xid(VRT_CTX) > VCL_STRING > VRT_r_sess_xid(VRT_CTX) > { > + struct sess *sp; > + struct http *http; > + > CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); > > - if (VALID_OBJ(ctx->req, REQ_MAGIC)) > - return (WS_Printf(ctx->req->http->ws, "%u", > - VXID(ctx->req->sp->vxid))); > + if (ctx->req) { > + CHECK_OBJ(ctx->req, REQ_MAGIC); > + CHECK_OBJ_NOTNULL(ctx->req->http, HTTP_MAGIC); > + sp = ctx->req->sp; > + http = ctx->req->http; > + } else { > + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); > + CHECK_OBJ_NOTNULL(ctx->bo->bereq, HTTP_MAGIC); > + sp = ctx->bo->sp; > + http = ctx->bo->bereq; > + } > > - CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); > - return (WS_Printf(ctx->bo->bereq->ws, "%u", > - VXID(ctx->bo->sp->vxid))); > + CHECK_OBJ_NOTNULL(http, HTTP_MAGIC); > + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); > + return (WS_Printf(http->ws, "%u", VXID(sp->vxid))); > } Why the de-tour via http->ws and not just use ctx->ws ? Nils From nils.goroll at uplex.de Sat Feb 2 13:50:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 2 Feb 2019 13:50:09 +0000 (UTC) Subject: [master] 406c701b5 sort the current changes into the usual sections Message-ID: <20190202135009.B887995A82@lists.varnish-cache.org> commit 406c701b5a0aa443666df1705a8827e5012a7159 Author: Nils Goroll Date: Sat Feb 2 14:49:19 2019 +0100 sort the current changes into the usual sections diff --git a/doc/changes.rst b/doc/changes.rst index 37b7dfbba..740906e32 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -32,10 +32,6 @@ Varnish Cache trunk (ongoing) * Extend JSON support in the CLI (2783_) -* fixed ``varnishhist`` display error (2780_) - -* ``libvarnish``: ``VRT_VSA_GetPtr`` renamed to ``VSA_GetPtr`` - * Improve accuracy of statistics (VSC) * In ``Error: out of workspace`` log entries, the workspace name is @@ -48,24 +44,43 @@ Varnish Cache trunk (ongoing) * fix ``varnishd -I`` (2782_) -* fix ``varnishstat -f`` in curses mode (interactively, without - ``-1``, 2787_) - -* Handle an out-of-workspace condition in HTTP/2 delivery more - gracefully (2589_) - * added a thread pool watchdog which will restart the worker process if scheduling tasks onto worker threads appears stuck. The new parameter ``thread_pool_watchdog`` configures it. (2418_) +* Changed ``ExpKill`` log tags to emit microsecond-precision + timestamps instead of nanoseconds (2792_) + +* Changed the default of the ``thread_pool_watchdog`` parameter + to 60 seconds to match the ``cli_timeout`` default + +* VSB quoted output has been unified to three-digit octal, + VSB_QUOTE_ESCHEX has been added to prefer hex over octal quoting + +* retired long deprecated parameters (VIP16_). Replacement mapping is: + ``shm_reclen`` -> ``vsl_reclen`` + ``vcl_dir`` -> ``vcl_path`` + ``vmod_dir`` -> ``vmod_path`` + +VCL +--- + +* added ``req.is_hitmiss`` and ``req.is_hitpass`` (2743_) + + +bundled tools +------------- + * Improved varnish log client performance (2788_) -* Fixed regression introduced just before 6.1.0 release which caused - an unnecessary incompatibility with VSL files written by previous - versions. (2790_) +* For ``varnishtest -L``, also keep VCL C source files -* Changed ``ExpKill`` log tags to emit microsecond-precision - timestamps instead of nanoseconds (2792_) +* Add ``param.reset`` command to ``varnishadm`` + +C APIs (for vmod and utility authors) +------------------------------------- + +* ``libvarnish``: ``VRT_VSA_GetPtr`` renamed to ``VSA_GetPtr`` * Included ``vtree.h`` in the distribution for vmods and renamed the red/black tree macros from ``VRB_*`` to ``VRBT_*`` @@ -75,42 +90,42 @@ Varnish Cache trunk (ongoing) ``PRIV_TOP`` from a list to a red/black tree) for performance. (2813_) -* Changed the default of the ``thread_pool_watchdog`` parameter - to 60 seconds to match the ``cli_timeout`` default - -* Fix warmup/rampup of the shard director (2823_) +* Vmod developers are advised that anything returned by a vmod + function/method is assumed to be immutable. In other words, a vmod + `must not` modify any data which was previously returned. * Added ``vstrerror()`` as a safe wrapper for ``strerror()`` to avoid a NULL pointer dereference under rare conditions where the latter could return NULL. (2815_) -* Fix VRT_priv_task for calls from vcl_pipe {} (2820_) +* Varnish-based tools using the VUT interface should now consider + using the ``VUT_Usage()`` function for consistency -* For ``varnishtest -L``, also keep VCL C source files -* Fix assinging == (2809_) +Fixed bugs +---------- -* Fix vmod object constructor documentation in the ``vmodtool.py`` - - generated RST files +* fixed ``varnishhist`` display error (2780_) -* Vmod developers are advised that anything returned by a vmod - function/method is assumed to be immutable. In other words, a vmod - `must not` modify any data which was previously returned. +* fix ``varnishstat -f`` in curses mode (interactively, without + ``-1``, 2787_) -* VSB quoted output has been unified to three-digit octal, - VSB_QUOTE_ESCHEX has been added to prefer hex over octal quoting +* Handle an out-of-workspace condition in HTTP/2 delivery more + gracefully (2589_) -* Varnish-based tools using the VUT interface should now consider - using the ``VUT_Usage()`` function for consistency +* Fixed regression introduced just before 6.1.0 release which caused + an unnecessary incompatibility with VSL files written by previous + versions. (2790_) -* retired long deprecated parameters (VIP16_). Replacement mapping is: - ``shm_reclen`` -> ``vsl_reclen`` - ``vcl_dir`` -> ``vcl_path`` - ``vmod_dir`` -> ``vmod_path`` +* Fix warmup/rampup of the shard director (2823_) -* added ``req.is_hitmiss`` and ``req.is_hitpass`` (2743_) +* Fix VRT_priv_task for calls from vcl_pipe {} (2820_) + +* Fix assinging == (2809_) + +* Fix vmod object constructor documentation in the ``vmodtool.py`` - + generated RST files -* Add ``param.reset`` command to ``varnishadm`` .. _2809: https://github.com/varnishcache/varnish-cache/issues/2809 .. _2820: https://github.com/varnishcache/varnish-cache/issues/2820 From nils.goroll at uplex.de Sat Feb 2 13:53:06 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 2 Feb 2019 13:53:06 +0000 (UTC) Subject: [master] 7767fc32a document event_function change Message-ID: <20190202135306.E6E0795D4D@lists.varnish-cache.org> commit 7767fc32a368439915086f26ef99cf6aae578484 Author: Nils Goroll Date: Sat Feb 2 14:51:44 2019 +0100 document event_function change Ref d8ae26b5a0b4a1102360a61a145e4730e52d68bc diff --git a/doc/changes.rst b/doc/changes.rst index 740906e32..c7c690b1c 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -101,6 +101,10 @@ C APIs (for vmod and utility authors) * Varnish-based tools using the VUT interface should now consider using the ``VUT_Usage()`` function for consistency +* The `event_function` callback for VCL events in vmods used to have + the fixed name ``event_function`` and has now been renamed to + `$Prefix`\ ``event_function`` if `$Prefix` is defined in the + ``.vcc`` files or ``vmod_event_function`` by default Fixed bugs ---------- From nils.goroll at uplex.de Sat Feb 2 14:01:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 2 Feb 2019 14:01:07 +0000 (UTC) Subject: [master] ae98a60fd correctly document event_function change Message-ID: <20190202140107.8853996162@lists.varnish-cache.org> commit ae98a60fd09c83e85d76c0f2eaadc705816f57d9 Author: Nils Goroll Date: Sat Feb 2 14:59:29 2019 +0100 correctly document event_function change my previous commit 7767fc32a368439915086f26ef99cf6aae578484 was incaccurate Ref d8ae26b5a0b4a1102360a61a145e4730e52d68bc diff --git a/doc/changes.rst b/doc/changes.rst index c7c690b1c..fca592c34 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -101,10 +101,13 @@ C APIs (for vmod and utility authors) * Varnish-based tools using the VUT interface should now consider using the ``VUT_Usage()`` function for consistency -* The `event_function` callback for VCL events in vmods used to have - the fixed name ``event_function`` and has now been renamed to - `$Prefix`\ ``event_function`` if `$Prefix` is defined in the - ``.vcc`` files or ``vmod_event_function`` by default +* The name of the `event_function` callback for VCL events in vmods is + now prefixed by `$Prefix`\ ``_``\ ` if `$Prefix` is defined in the + ``.vcc`` file, or ``vmod_`` by default. + + So, for example, with ``$Event foo`` and no `$Prefix`, the event + function will be called ``vmod_foo`` and with ``$Prefix bar`` it + will be called ``bar_foo``. Fixed bugs ---------- From nils.goroll at uplex.de Sat Feb 2 14:30:12 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 2 Feb 2019 14:30:12 +0000 (UTC) Subject: [master] 9023b9924 Document vmod rst anchor name change Message-ID: <20190202143012.D1F7396CA1@lists.varnish-cache.org> commit 9023b99241650baba5987b075d5e054528c30989 Author: Nils Goroll Date: Sat Feb 2 15:28:08 2019 +0100 Document vmod rst anchor name change Ref c09032032700f91fcf06e3018d206cbb730d1772 diff --git a/doc/changes.rst b/doc/changes.rst index fca592c34..a702698f5 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -109,6 +109,18 @@ C APIs (for vmod and utility authors) function will be called ``vmod_foo`` and with ``$Prefix bar`` it will be called ``bar_foo``. +* in the `vmodtool`\ -generated ReStructuredText documentation, + anchors have been renamed + + * from ``obj_``\ `class` to `vmodname`\ ``.``\ `class` for + constructors and + * from ``func_``\ `class` to `vmodname`\ ``.``\ `function` for functions and + * from ``func_``\ `class` to `vmodname`\ ``.``\ `class`\ ``.``\ + `method` for methods, + + repsectively. In short, the anchor is now named equal to VCL syntax + for constructors and functions and similarly to VCL syntax for methods. + Fixed bugs ---------- From guillaume at varnish-software.com Sat Feb 2 17:27:06 2019 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Sat, 2 Feb 2019 17:27:06 +0000 (UTC) Subject: [master] f3f8aa163 Directly use ctx->ws (thanks Nils) Message-ID: <20190202172706.E6DE2A08F0@lists.varnish-cache.org> commit f3f8aa1632d8caf6cb555088294816c86f31f2a9 Author: Guillaume Quintard Date: Sat Feb 2 09:25:29 2019 -0800 Directly use ctx->ws (thanks Nils) diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 76adf6ef6..7cfa27eeb 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -690,10 +690,9 @@ VRT_r_bereq_xid(VRT_CTX) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); - CHECK_OBJ_NOTNULL(ctx->bo->bereq, HTTP_MAGIC); AN(ctx->bo->vsl); - return (WS_Printf(ctx->bo->bereq->ws, "%u", + return (WS_Printf(ctx->ws, "%u", VXID(ctx->bo->vsl->wid))); } @@ -701,25 +700,19 @@ VCL_STRING VRT_r_sess_xid(VRT_CTX) { struct sess *sp; - struct http *http; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (ctx->req) { CHECK_OBJ(ctx->req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req->http, HTTP_MAGIC); sp = ctx->req->sp; - http = ctx->req->http; } else { CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); - CHECK_OBJ_NOTNULL(ctx->bo->bereq, HTTP_MAGIC); sp = ctx->bo->sp; - http = ctx->bo->bereq; } - CHECK_OBJ_NOTNULL(http, HTTP_MAGIC); CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - return (WS_Printf(http->ws, "%u", VXID(sp->vxid))); + return (WS_Printf(ctx->ws, "%u", VXID(sp->vxid))); } /*-------------------------------------------------------------------- From phk at FreeBSD.org Mon Feb 4 13:44:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 4 Feb 2019 13:44:09 +0000 (UTC) Subject: [master] d4180b6a3 Make the VMOD.file_id depend only on the $... lines, so people can freely update comments and documentation. Message-ID: <20190204134409.2932AAF282@lists.varnish-cache.org> commit d4180b6a39adedd90fb2c58d8e394f987237e15d Author: Poul-Henning Kamp Date: Mon Feb 4 09:14:20 2019 +0000 Make the VMOD.file_id depend only on the $... lines, so people can freely update comments and documentation. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 1e92145e4..1f02696d2 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -850,13 +850,14 @@ class vcc(object): global inputline b = open(self.inputfile, "rb").read() a = "\n" + b.decode("utf-8") - self.file_id = hashlib.sha256(b).hexdigest() + h = hashlib.sha256() s = a.split("\n$") self.copyright = s.pop(0).strip() while s: ss = re.split('\n([^\t ])', s.pop(0), maxsplit=1) toks = self.tokenize(ss[0]) inputline = '$' + ' '.join(toks) + h.update((inputline + '\n').encode('utf-8')) docstr = "".join(ss[1:]) stanzaclass = DISPATCH.get(toks[0]) if stanzaclass is None: @@ -864,6 +865,7 @@ class vcc(object): stanzaclass(self, toks, docstr) inputline = None self.csn = "Vmod_%s%s_Func" % (self.sympfx, self.modname) + self.file_id = h.hexdigest() def tokenize(self, txt, seps=None, quotes=None): if seps is None: From phk at FreeBSD.org Mon Feb 4 13:44:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 4 Feb 2019 13:44:09 +0000 (UTC) Subject: [master] 0fca8adf1 Add VARGS() macro for vmod argument structures. Message-ID: <20190204134409.436A5AF286@lists.varnish-cache.org> commit 0fca8adf15a0a024fc03d40eeb61534187fb623d Author: Poul-Henning Kamp Date: Mon Feb 4 13:41:55 2019 +0000 Add VARGS() macro for vmod argument structures. Change name for enums and args to have "enum_" and "arg_" prefixes. Minimal changes to the vmods to react to these changes. diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 4664df7ff..88e89b36d 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -67,6 +67,12 @@ -emacro(845, HTTPH) // Info 845: The left argument to operator '&&' is certain to be 0 -esym(773, PCRE_DATE) // Expression-like macro '___' not parenthesized + +////////////// +// Macros defined differently in each VMOD +-esym(767, VPFX) // macro '___' was defined differently in another module +-esym(767, VARGS) // macro '___' was defined differently in another module +-esym(767, VENUM) // macro '___' was defined differently in another module ////////////// -efunc(1791, pdiff) // return last on line ////////////// diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 1f02696d2..310f4a43f 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -31,6 +31,7 @@ Read the vmod.vcc file (inputvcc) and produce: vmod_if.h -- Prototypes for the implementation vmod_if.c -- Magic glue & datastructures to make things a VMOD. vmod_${name}.rst -- Extracted documentation + vmod_${name}.man.rst -- Extracted documentation (rst2man input) """ # This script should work with both Python 2 and Python 3. @@ -129,10 +130,14 @@ def unquote(txt): def fmt_cstruct(fo, a, b): ''' Output line in vmod struct ''' - t = "\t%s\t" % a + t = '\t%s' % a + if len(t.expandtabs()) > 40: + t += '\n\t\t\t\t\t' + else: + t += '\t' while len(t.expandtabs()) < 40: - t += "\t" - fo.write("%s%s\n" % (t, b)) + t += '\t' + fo.write('%s%s\n' % (t, b)) ####################################################################### @@ -430,7 +435,7 @@ class ProtoType(object): return "typedef " + self.proto(args, name=self.typedef_name()) def argstructname(self): - return "struct %s_arg" % self.cname(True) + return "struct VARGS(%s)" % self.cname(False) def argstructure(self): s = "\n" + self.argstructname() + " {\n" @@ -471,7 +476,8 @@ class ProtoType(object): self.retval.jsonproto(ll) ll.append('%s.%s' % (self.st.vcc.csn, cfunc)) if self.argstruct: - ll.append(self.argstructname()) + # We cannot use VARGS() here, we are after the #undef + ll.append('struct arg_%s' % self.cname(True)) else: ll.append("") for i in self.args: @@ -662,8 +668,7 @@ class EventStanza(Stanza): def cstuff(self, fo, where): if where == 'h': - fo.write("vmod_event_f %s%s;\n" % - (self.vcc.sympfx, self.event_func)) + fo.write("vmod_event_f VPFX(%s);\n" % self.event_func) def cstruct(self, fo, define): if define: @@ -741,7 +746,7 @@ class ObjectStanza(Stanza): fo.write(' :ref:`%s`\n \n' % i.rstlbl) def cstuff(self, fo, w): - sn = self.vcc.sympfx + self.vcc.modname + "_" + self.proto.name + sn = 'VPFX(' + self.vcc.modname + '_' + self.proto.name + ')' fo.write("struct %s;\n" % sn) fo.write(self.init.cproto( @@ -952,8 +957,17 @@ class vcc(object): fo.write("#endif\n") fo.write("\n") + fo.write('#define VPFX(a) %s##a\n' % self.sympfx) + fo.write('#define VARGS(a) arg_%s##a\n' % self.sympfx) + fo.write('#define VENUM(a) enum_%s##a\n' % self.sympfx) + fo.write('\n') + for j in sorted(self.enums): - fo.write("extern VCL_ENUM %senum_%s;\n" % (self.sympfx, j)) + fo.write("extern VCL_ENUM VENUM(%s);\n" % j) + fo.write("\n") + for j in sorted(self.enums): + fo.write("//lint -esym(759, enum_%s%s)\n" % (self.sympfx, j)) + fo.write("//lint -esym(765, enum_%s%s)\n" % (self.sympfx, j)) fo.write("\n") for j in self.contents: @@ -974,7 +988,7 @@ class vcc(object): j.cstruct(fo, False) fo.write("\n") for j in sorted(self.enums): - fo.write("\t&%senum_%s,\n" % (self.sympfx, j)) + fmt_cstruct(fo, '.enum_%s =' % j, '&VENUM(%s),' % j) fo.write("};\n") def json(self, fo): @@ -1026,6 +1040,11 @@ class vcc(object): write_c_file_warning(fo) + fx.write('#define VPFX(a) %s##a\n' % self.sympfx) + fx.write('#define VARGS(a) arg_%s##a\n' % self.sympfx) + fx.write('#define VENUM(a) enum_%s##a\n' % self.sympfx) + fx.write('\n') + fo.write('#include "config.h"\n') fo.write('#include \n') for i in ["vdef", "vrt", self.pfx, "vmod_abi"]: @@ -1033,7 +1052,7 @@ class vcc(object): fo.write("\n") for j in sorted(self.enums): - fo.write('VCL_ENUM %senum_%s = "%s";\n' % (self.sympfx, j, j)) + fo.write('VCL_ENUM VENUM(%s) = "%s";\n' % (j, j)) fo.write("\n") for i in self.contents: @@ -1053,6 +1072,10 @@ class vcc(object): fo.write("\n/*lint -esym(754, " + self.csn + "::*) */\n") self.cstruct_init(fo) + fx.write('#undef VPFX\n') + fx.write('#undef VARGS\n') + fx.write('#undef VENUM\n') + fx.close() fo.write("\nstatic const char Vmod_Proto[] =\n") diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index 66b9f4e24..d032a439b 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -118,7 +118,7 @@ static const struct vrt_blob null_blob[1] = {{ static enum encoding parse_encoding(VCL_ENUM e) { -#define VMODENUM(n) if (e == vmod_enum_ ## n) return(n); +#define VMODENUM(n) if (e == VENUM(n)) return(n); #include "tbl_encodings.h" WRONG("illegal encoding enum"); } @@ -126,7 +126,7 @@ parse_encoding(VCL_ENUM e) static enum case_e parse_case(VCL_ENUM e) { -#define VMODENUM(n) if (e == vmod_enum_ ## n) return(n); +#define VMODENUM(n) if (e == VENUM(n)) return(n); #include "tbl_case.h" WRONG("illegal case enum"); } diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 43e203262..c62b3e85e 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -94,16 +94,16 @@ xyzzy_author(VRT_CTX, VCL_ENUM person, VCL_ENUM someone) (void)someone; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (person == xyzzy_enum_phk) + if (person == VENUM(phk)) return ("Poul-Henning"); assert(strcmp(person, "phk")); - if (person == xyzzy_enum_des) + if (person == VENUM(des)) return ("Dag-Erling"); assert(strcmp(person, "des")); - if (person == xyzzy_enum_kristian) + if (person == VENUM(kristian)) return ("Kristian"); assert(strcmp(person, "kristian")); - if (person == xyzzy_enum_mithrandir) + if (person == VENUM(mithrandir)) return ("Tollef"); assert(strcmp(person, "mithrandir")); WRONG("Illegal VMOD enum"); @@ -194,7 +194,7 @@ xyzzy_rot52(VRT_CTX, VCL_HTTP hp) } VCL_STRING v_matchproto_(td_debug_argtest) -xyzzy_argtest(VRT_CTX, struct xyzzy_argtest_arg *arg) +xyzzy_argtest(VRT_CTX, struct VARGS(argtest) *arg) { char buf[100]; diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c index 4e716dae4..0dea592a3 100644 --- a/lib/libvmod_debug/vmod_debug_obj.c +++ b/lib/libvmod_debug/vmod_debug_obj.c @@ -157,17 +157,17 @@ xyzzy_obj_test_priv_top(VRT_CTX, * obj_opt (optional arguments and privs) */ struct xyzzy_debug_obj_opt { - unsigned magic; -#define VMOD_DEBUG_OBJ_OPT_MAGIC 0xccbd9b78 - char *name; - struct xyzzy_obj_opt_meth_opt_arg args; - void *freeptr; + unsigned magic; +#define VMOD_DEBUG_OBJ_OPT_MAGIC 0xccbd9b78 + char *name; + struct VARGS(obj_opt_meth_opt) args; + void *freeptr; }; VCL_VOID v_matchproto_() xyzzy_obj_opt__init(VRT_CTX, struct xyzzy_debug_obj_opt **op, const char *vcl_name, - struct xyzzy_obj_opt__init_arg *args) + struct VARGS(obj_opt__init) *args) { struct xyzzy_debug_obj_opt *o; @@ -221,7 +221,7 @@ xyzzy_obj_opt__fini(struct xyzzy_debug_obj_opt **op) VCL_STRING v_matchproto_() xyzzy_obj_opt_meth_opt(VRT_CTX, struct xyzzy_debug_obj_opt *o, - struct xyzzy_obj_opt_meth_opt_arg *args) + struct VARGS(obj_opt_meth_opt) *args) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(o, VMOD_DEBUG_OBJ_OPT_MAGIC); diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 9e8abcbc0..4495474d3 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -182,7 +182,7 @@ struct vmod_directors_shard { static enum by_e parse_by_e(VCL_ENUM e) { -#define VMODENUM(n) if (e == vmod_enum_ ## n) return(BY_ ## n); +#define VMODENUM(n) if (e == VENUM(n)) return(BY_ ## n); #include "tbl_by.h" WRONG("illegal by enum"); } @@ -190,7 +190,7 @@ parse_by_e(VCL_ENUM e) static enum healthy_e parse_healthy_e(VCL_ENUM e) { -#define VMODENUM(n) if (e == vmod_enum_ ## n) return(n); +#define VMODENUM(n) if (e == VENUM(n)) return(n); #include "tbl_healthy.h" WRONG("illegal healthy enum"); } @@ -198,7 +198,7 @@ parse_healthy_e(VCL_ENUM e) static enum resolve_e parse_resolve_e(VCL_ENUM e) { -#define VMODENUM(n) if (e == vmod_enum_ ## n) return(n); +#define VMODENUM(n) if (e == VENUM(n)) return(n); #include "tbl_resolve.h" WRONG("illegal resolve enum"); } @@ -343,7 +343,7 @@ vmod_shard_associate(VRT_CTX, VCL_BOOL v_matchproto_(td_directors_shard_add_backend) vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard, - struct vmod_shard_add_backend_arg *args) + struct VARGS(shard_add_backend) *args) { CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); @@ -361,7 +361,7 @@ vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard, VCL_BOOL v_matchproto_(td_directors_shard_remove_backend) vmod_shard_remove_backend(VRT_CTX, struct vmod_directors_shard *vshard, - struct vmod_shard_remove_backend_arg *args) + struct VARGS(shard_remove_backend) *args) { VCL_BACKEND be = args->valid_backend ? args->backend : NULL; VCL_STRING ident = args->valid_ident ? args->ident : NULL; @@ -498,7 +498,7 @@ shard_blob_key(VCL_BLOB key_blob) #define tobit(args, name) ((args)->valid_##name ? arg_##name : 0) static uint32_t -shard_backend_arg_mask(const struct vmod_shard_backend_arg * const a) +shard_backend_arg_mask(const struct VARGS(shard_backend) * const a) { return (tobit(a, by) | tobit(a, key) | @@ -511,7 +511,7 @@ shard_backend_arg_mask(const struct vmod_shard_backend_arg * const a) tobit(a, resolve)); } static uint32_t -shard_param_set_mask(const struct vmod_shard_param_set_arg * const a) +shard_param_set_mask(const struct VARGS(shard_param_set) * const a) { return (tobit(a, by) | tobit(a, key) | @@ -640,7 +640,7 @@ shard_param_args(VRT_CTX, VCL_BACKEND v_matchproto_(td_directors_shard_backend) vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, - struct vmod_shard_backend_arg *a) + struct VARGS(shard_backend) *a) { struct vmod_directors_shard_param pstk; struct vmod_directors_shard_param *pp = NULL; @@ -884,7 +884,7 @@ shard_param_prep(VRT_CTX, struct vmod_directors_shard_param *p, VCL_VOID v_matchproto_(td_directors_shard_param_set) vmod_shard_param_set(VRT_CTX, struct vmod_directors_shard_param *p, - struct vmod_shard_param_set_arg *a) + struct VARGS(shard_param_set) *a) { uint32_t args = shard_param_set_mask(a); From nils.goroll at uplex.de Mon Feb 4 13:52:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Feb 2019 13:52:09 +0000 (UTC) Subject: [master] 0c041817c changelog tlc Message-ID: <20190204135209.6F855AF7E3@lists.varnish-cache.org> commit 0c041817c7e45b4258b657f33cd806a045951b89 Author: Nils Goroll Date: Mon Feb 4 14:51:08 2019 +0100 changelog tlc diff --git a/doc/changes.rst b/doc/changes.rst index a702698f5..7d1b59252 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -121,6 +121,33 @@ C APIs (for vmod and utility authors) repsectively. In short, the anchor is now named equal to VCL syntax for constructors and functions and similarly to VCL syntax for methods. +* VRT API has been updated to 9.0 + + * ``HTTP_Copy()`` was removed, ``HTTP_Dup()`` and ``HTTP_Clone()`` were added + + * Previously, ``VCL_BLOB`` was implemented as ``struct vmod_priv``, + which had the following shortcomings: + + * blobs are immutable, but that was not reflected by the ``priv`` + pointer + + * the existence of a free pointer suggested automatic memory + management, which did never and will not exist for blobs. + + The ``VCL_BLOB`` type is now implemented as ``struct vrt_blob``, + with the ``blob`` member replacing the former ``priv`` pointer and + the ``free`` pointer removed. + + A ``type`` member was added for lightweight type checking similar + to the miniobject ``magic`` member, but in contrast to it, + ``type`` should never be asserted upon. + + ``VRT_blob()`` was updated accordingly. + + * ``req->req_bodybytes`` was removed. Replacement code snippet:: + + AZ(ObjGetU64(req->wrk, req->body_oc, OA_LEN, &u)); + Fixed bugs ---------- From nils.goroll at uplex.de Mon Feb 4 14:11:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Feb 2019 14:11:09 +0000 (UTC) Subject: [master] 03f1a2148 Output correct director health in CLI Message-ID: <20190204141109.6B296AFEA5@lists.varnish-cache.org> commit 03f1a214828c7c250fa4a4edfe9011bcfc864860 Author: Nils Goroll Date: Fri Jan 25 17:47:20 2019 +0100 Output correct director health in CLI As VRT_Healthy needs a VRT_CTX, we move vcl_get_ctx and vcl_rel_ctx to cache_varnishd.h scope as VCL_Get_CliCtx and VCL_Rel_CliCtx. It has been discussed before if all varnishd cli commands should have a VRT_CTX, so the natural solution would seem to add one in or close to VCLS_Poll(), yet CLI commands need specific ctxes (e.g. vcl.load needs VCL_MET_INIT), so (for now) it appears sensible to limit the ctx' scope to individual backend method calls. Fixes #2887 diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index ed1309db3..9bb13a444 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -277,6 +277,20 @@ struct list_args { const char *jsep; }; +static const char * +cli_health(struct director *d) +{ + struct vrt_ctx *ctx; + VCL_BOOL healthy; + + ctx = VCL_Get_CliCtx(0,0); + healthy = VRT_Healthy(ctx, d, NULL); + VCL_Rel_CliCtx(&ctx); + AZ(ctx); + + return (healthy ? "healthy" : "sick"); +} + static int v_matchproto_(vcl_be_func) do_list(struct cli *cli, struct director *d, void *priv) { @@ -295,7 +309,7 @@ do_list(struct cli *cli, struct director *d, void *priv) if (d->vdir->methods->list != NULL) d->vdir->methods->list(d, cli->sb, 0, 0, 0); else - VCLI_Out(cli, "%-10s", d->sick ? "sick" : "healthy"); + VCLI_Out(cli, "%-10s", cli_health(d)); VTIM_format(d->vdir->health_changed, time_str); VCLI_Out(cli, " %s", time_str); @@ -327,7 +341,7 @@ do_list_json(struct cli *cli, struct director *d, void *priv) if (d->vdir->methods->list != NULL) d->vdir->methods->list(d, cli->sb, 0, 0, 1); else - VCLI_Out(cli, "\"%s\"", d->sick ? "sick" : "healthy"); + VCLI_Out(cli, "\"%s\"", cli_health(d)); VCLI_Out(cli, ",\n"); if ((la->p || la->v) && d->vdir->methods->list != NULL) { diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 38ca918a7..07fd45969 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -389,6 +389,8 @@ const char *VCL_Return_Name(unsigned); const char *VCL_Method_Name(unsigned); void VCL_Bo2Ctx(struct vrt_ctx *, struct busyobj *); void VCL_Req2Ctx(struct vrt_ctx *, struct req *); +struct vrt_ctx *VCL_Get_CliCtx(unsigned, int); +void VCL_Rel_CliCtx(struct vrt_ctx **); #define VCL_MET_MAC(l,u,t,b) \ void VCL_##l##_method(struct vcl *, struct worker *, struct req *, \ diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 270452132..ed89868d7 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -106,8 +106,8 @@ VCL_Req2Ctx(struct vrt_ctx *ctx, struct req *req) /*--------------------------------------------------------------------*/ -static struct vrt_ctx * -vcl_get_ctx(unsigned method, int msg) +struct vrt_ctx * +VCL_Get_CliCtx(unsigned method, int msg) { ASSERT_CLI(); @@ -126,8 +126,8 @@ vcl_get_ctx(unsigned method, int msg) return (&ctx_cli); } -static void -vcl_rel_ctx(struct vrt_ctx **ctx) +void +VCL_Rel_CliCtx(struct vrt_ctx **ctx) { ASSERT_CLI(); @@ -614,7 +614,7 @@ VCL_Poll(void) struct vcl *vcl, *vcl2; ASSERT_CLI(); - ctx = vcl_get_ctx(0, 0); + ctx = VCL_Get_CliCtx(0, 0); VTAILQ_FOREACH_SAFE(vcl, &vcl_head, list, vcl2) { if (vcl->temp == VCL_TEMP_BUSY || vcl->temp == VCL_TEMP_COOLING) { @@ -639,7 +639,7 @@ VCL_Poll(void) VSC_C_main->n_vcl_discard--; } } - vcl_rel_ctx(&ctx); + VCL_Rel_CliCtx(&ctx); } /*--------------------------------------------------------------------*/ @@ -730,9 +730,9 @@ vcl_cli_load(struct cli *cli, const char * const *av, void *priv) AZ(priv); ASSERT_CLI(); - ctx = vcl_get_ctx(VCL_MET_INIT, 1); + ctx = VCL_Get_CliCtx(VCL_MET_INIT, 1); vcl_load(cli, ctx, av[2], av[3], av[4]); - vcl_rel_ctx(&ctx); + VCL_Rel_CliCtx(&ctx); } static void v_matchproto_(cli_func_t) @@ -744,7 +744,7 @@ vcl_cli_state(struct cli *cli, const char * const *av, void *priv) ASSERT_CLI(); AN(av[2]); AN(av[3]); - ctx = vcl_get_ctx(0, 1); + ctx = VCL_Get_CliCtx(0, 1); ctx->vcl = vcl_find(av[2]); AN(ctx->vcl); // MGT ensures this if (vcl_set_state(ctx, av[3])) { @@ -755,7 +755,7 @@ vcl_cli_state(struct cli *cli, const char * const *av, void *priv) if (VSB_len(ctx->msg)) VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(ctx->msg)); } - vcl_rel_ctx(&ctx); + VCL_Rel_CliCtx(&ctx); } static void v_matchproto_(cli_func_t) From nils.goroll at uplex.de Mon Feb 4 14:38:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Feb 2019 14:38:09 +0000 (UTC) Subject: [master] 7bc370950 improve backend.list documentation Message-ID: <20190204143809.871E4B0831@lists.varnish-cache.org> commit 7bc37095037539eec9570e72f2c4fb13518434a9 Author: Nils Goroll Date: Mon Feb 4 15:31:05 2019 +0100 improve backend.list documentation * move -p documentation from CLI "help backend.list" to sphinx * Add note on the fact that the health state as seen from VCL may differ diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h index 57efeaab6..6c020f072 100644 --- a/include/tbl/cli_cmds.h +++ b/include/tbl/cli_cmds.h @@ -257,8 +257,13 @@ CLI_CMD(DEBUG_LISTEN_ADDRESS, CLI_CMD(BACKEND_LIST, "backend.list", "backend.list [-j] [-p] []", - "List backends. -p also shows probe status.", - "``-j`` specifies JSON output.", + "List backends.\n", + " ``-p`` also shows probe status.\n\n" + " ``-j`` specifies JSON output.\n\n" + " The health state reported here is generic. A backend's health " + "may also depend on the context it is being used in (e.g. " + "the object's hash), so the actual health state as visible " + "from VCL (e.g. using std.healthy()) may differ.", 0, 2 ) From nils.goroll at uplex.de Mon Feb 4 14:38:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Feb 2019 14:38:09 +0000 (UTC) Subject: [master] 820bafa42 fix varnish-cli(7) indentation of cli command docs Message-ID: <20190204143809.9AD7FB0834@lists.varnish-cache.org> commit 820bafa4200d7a80b2d48023573631ab41c16fb4 Author: Nils Goroll Date: Mon Feb 4 15:36:32 2019 +0100 fix varnish-cli(7) indentation of cli command docs diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h index 6c020f072..5c28ed5c8 100644 --- a/include/tbl/cli_cmds.h +++ b/include/tbl/cli_cmds.h @@ -43,7 +43,7 @@ CLI_CMD(BAN, "ban", "ban [&& ...]", "Mark obsolete all objects where all the conditions match.", - "See :ref:`vcl(7)_ban` for details", + " See :ref:`vcl(7)_ban` for details", 3, -1 ) @@ -52,7 +52,7 @@ CLI_CMD(BAN_LIST, "ban.list [-j]", "List the active bans.", - " Unless ``-j`` is specified (for JSON output), " + " Unless ``-j`` is specified (for JSON output), " " the output format is:\n\n" " * Time the ban was issued.\n\n" " * Objects referencing this ban.\n\n" @@ -105,7 +105,7 @@ CLI_CMD(VCL_LIST, "vcl.list", "vcl.list [-j]", "List all loaded configuration.", - "``-j`` specifies JSON output.", + " ``-j`` specifies JSON output.", 0, 0 ) @@ -146,7 +146,7 @@ CLI_CMD(PARAM_SHOW, "param.show [-l|-j] [|changed]", "Show parameters and their values.", - "The long form with ``-l`` shows additional information, including" + " The long form with ``-l`` shows additional information, including" " documentation and minimum, maximum and default values, if defined" " for the parameter. JSON output is specified with ``-j``, in which" " the information for the long form is included; only one of ``-l`` or" @@ -184,7 +184,7 @@ CLI_CMD(PING, "ping", "ping [-j] []", "Keep connection alive.", - "The response is formatted as JSON if ``-j`` is specified.", + " The response is formatted as JSON if ``-j`` is specified.", 0, 1 ) @@ -192,7 +192,7 @@ CLI_CMD(HELP, "help", "help [-j] []", "Show command/protocol help.", - "``-j`` specifies JSON output.", + " ``-j`` specifies JSON output.", 0, 1 ) @@ -208,7 +208,7 @@ CLI_CMD(SERVER_STATUS, "status", "status [-j]", "Check status of Varnish cache process.", - "``-j`` specifies JSON output.", + " ``-j`` specifies JSON output.", 0, 0 ) @@ -232,7 +232,7 @@ CLI_CMD(PANIC_SHOW, "panic.show", "panic.show [-j]", "Return the last panic, if any.", - "``-j`` specifies JSON output -- the panic message is returned as an" + " ``-j`` specifies JSON output -- the panic message is returned as an" " unstructured JSON string.", 0, 0 ) @@ -339,7 +339,7 @@ CLI_CMD(STORAGE_LIST, "storage.list", "storage.list [-j]", "List storage devices.", - "``-j`` specifies JSON output.", + " ``-j`` specifies JSON output.", 0, 0 ) From phk at FreeBSD.org Mon Feb 4 15:32:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 4 Feb 2019 15:32:10 +0000 (UTC) Subject: [master] 0d6803d29 Quench flexelint noise Message-ID: <20190204153210.46BD2B19CA@lists.varnish-cache.org> commit 0d6803d293420dc655c14a9b06450e26f3e73476 Author: Poul-Henning Kamp Date: Mon Feb 4 15:26:31 2019 +0000 Quench flexelint noise diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 310f4a43f..94a9f5a71 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -960,6 +960,9 @@ class vcc(object): fo.write('#define VPFX(a) %s##a\n' % self.sympfx) fo.write('#define VARGS(a) arg_%s##a\n' % self.sympfx) fo.write('#define VENUM(a) enum_%s##a\n' % self.sympfx) + fo.write('//lint -esym(755, VPFX)\n') + fo.write('//lint -esym(755, VARGS)\n') + fo.write('//lint -esym(755, VENUM)\n') fo.write('\n') for j in sorted(self.enums): From nils.goroll at uplex.de Mon Feb 4 18:39:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Feb 2019 18:39:08 +0000 (UTC) Subject: [master] bc5f754f2 Also add a ctx to vdi_list_f (director list callback) Message-ID: <20190204183908.49DD563A9@lists.varnish-cache.org> commit bc5f754f2c153650ac3b3f10ec7397c14bfbb203 Author: Nils Goroll Date: Mon Feb 4 19:24:26 2019 +0100 Also add a ctx to vdi_list_f (director list callback) ... to bring "more nuance" (quote phk) to the director health state output, the list callback needs to be able to query VRT_Health() of the director's backends, so, in turn, we also need a CTX here. Ref https://github.com/varnishcache/varnish-cache/pull/2888#issuecomment-460258147 Of the following out-of-tree vmods, none uses the .list callback, so I expect minimal vmod author uprise: - saintmode (varnish-modules) - all_healthy - dynamic - cluster - weightadjust (someome(tm) needs to update to vdi_methods) diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index f1426f618..2c8ce70f8 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -445,12 +445,14 @@ vbe_panic(const struct director *d, struct vsb *vsb) /*-------------------------------------------------------------------- */ -static void -vbe_list(const struct director *d, struct vsb *vsb, int vflag, int pflag, - int jflag) +static void v_matchproto_(vdi_list_f) +vbe_list(VRT_CTX, const struct director *d, struct vsb *vsb, int vflag, + int pflag, int jflag) { struct backend *bp; + (void)ctx; + CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC); diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index 9bb13a444..152418d55 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -278,15 +278,9 @@ struct list_args { }; static const char * -cli_health(struct director *d) +cli_health(VRT_CTX, struct director *d) { - struct vrt_ctx *ctx; - VCL_BOOL healthy; - - ctx = VCL_Get_CliCtx(0,0); - healthy = VRT_Healthy(ctx, d, NULL); - VCL_Rel_CliCtx(&ctx); - AZ(ctx); + VCL_BOOL healthy = VRT_Healthy(ctx, d, NULL); return (healthy ? "healthy" : "sick"); } @@ -296,6 +290,7 @@ do_list(struct cli *cli, struct director *d, void *priv) { char time_str[VTIM_FORMAT_SIZE]; struct list_args *la; + struct vrt_ctx *ctx; CAST_OBJ_NOTNULL(la, priv, LIST_ARGS_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); @@ -303,18 +298,24 @@ do_list(struct cli *cli, struct director *d, void *priv) if (d->vdir->admin_health == VDI_AH_DELETED) return (0); + ctx = VCL_Get_CliCtx(0,0); + // XXX admin health "probe" for the no-probe case is confusing VCLI_Out(cli, "\n%-30s %-7s ", d->vdir->cli_name, VDI_Ahealth(d)); if (d->vdir->methods->list != NULL) - d->vdir->methods->list(d, cli->sb, 0, 0, 0); + d->vdir->methods->list(ctx, d, cli->sb, 0, 0, 0); else - VCLI_Out(cli, "%-10s", cli_health(d)); + VCLI_Out(cli, "%-10s", cli_health(ctx, d)); VTIM_format(d->vdir->health_changed, time_str); VCLI_Out(cli, " %s", time_str); if ((la->p || la->v) && d->vdir->methods->list != NULL) - d->vdir->methods->list(d, cli->sb, la->p, la->v, 0); + d->vdir->methods->list(ctx, d, cli->sb, la->p, la->v, 0); + + VCL_Rel_CliCtx(&ctx); + AZ(ctx); + return (0); } @@ -322,6 +323,7 @@ static int v_matchproto_(vcl_be_func) do_list_json(struct cli *cli, struct director *d, void *priv) { struct list_args *la; + struct vrt_ctx *ctx; CAST_OBJ_NOTNULL(la, priv, LIST_ARGS_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); @@ -329,6 +331,8 @@ do_list_json(struct cli *cli, struct director *d, void *priv) if (d->vdir->admin_health == VDI_AH_DELETED) return (0); + ctx = VCL_Get_CliCtx(0,0); + VCLI_Out(cli, "%s", la->jsep); la->jsep = ",\n"; // XXX admin health "probe" for the no-probe case is confusing @@ -339,18 +343,22 @@ do_list_json(struct cli *cli, struct director *d, void *priv) VCLI_Out(cli, "\"admin_health\": \"%s\",\n", VDI_Ahealth(d)); VCLI_Out(cli, "\"probe_message\": "); if (d->vdir->methods->list != NULL) - d->vdir->methods->list(d, cli->sb, 0, 0, 1); + d->vdir->methods->list(ctx, d, cli->sb, 0, 0, 1); else - VCLI_Out(cli, "\"%s\"", cli_health(d)); + VCLI_Out(cli, "\"%s\"", cli_health(ctx, d)); VCLI_Out(cli, ",\n"); if ((la->p || la->v) && d->vdir->methods->list != NULL) { VCLI_Out(cli, "\"probe_details\": "); - d->vdir->methods->list(d, cli->sb, la->p, la->v, 1); + d->vdir->methods->list(ctx, d, cli->sb, la->p, la->v, 1); } VCLI_Out(cli, "\"last_change\": %.3f\n", d->vdir->health_changed); VSB_indent(cli->sb, -2); VCLI_Out(cli, "}"); + + VCL_Rel_CliCtx(&ctx); + AZ(ctx); + return (0); } diff --git a/include/vrt.h b/include/vrt.h index 9cbf6a6d7..4b547e839 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -60,6 +60,7 @@ * changed VRT_blob() * req->req_bodybytes removed * use: AZ(ObjGetU64(req->wrk, req->body_oc, OA_LEN, &u)); + * struct vdi_methods .list callback signature changed * 8.0 (2018-09-15) * VRT_Strands() added * VRT_StrandsWS() added @@ -476,7 +477,7 @@ typedef enum sess_close vdi_http1pipe_f(VRT_CTX, VCL_BACKEND); typedef void vdi_event_f(VCL_BACKEND, enum vcl_event_e); typedef void vdi_destroy_f(VCL_BACKEND); typedef void vdi_panic_f(VCL_BACKEND, struct vsb *); -typedef void vdi_list_f(VCL_BACKEND, struct vsb *, int, int, int); +typedef void vdi_list_f(VRT_CTX, VCL_BACKEND, struct vsb *, int, int, int); struct vdi_methods { unsigned magic; From phk at FreeBSD.org Tue Feb 5 06:36:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Feb 2019 06:36:10 +0000 (UTC) Subject: [master] 9b0b9fa39 Constify Message-ID: <20190205063610.7751597F2E@lists.varnish-cache.org> commit 9b0b9fa3956e2010759f1161bd22e2beb12dd52c Author: Poul-Henning Kamp Date: Tue Feb 5 06:32:23 2019 +0000 Constify diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index 152418d55..a94c36ab5 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -278,7 +278,7 @@ struct list_args { }; static const char * -cli_health(VRT_CTX, struct director *d) +cli_health(VRT_CTX, const struct director *d) { VCL_BOOL healthy = VRT_Healthy(ctx, d, NULL); From phk at FreeBSD.org Tue Feb 5 06:36:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Feb 2019 06:36:10 +0000 (UTC) Subject: [master] 40460fe90 Change the 'len' field in vmod_priv and vrt_blob to a 'long' Message-ID: <20190205063610.8C6FB97F32@lists.varnish-cache.org> commit 40460fe90aa27b3aea93e7bd560c3ebe11f3f9ec Author: Poul-Henning Kamp Date: Tue Feb 5 06:33:34 2019 +0000 Change the 'len' field in vmod_priv and vrt_blob to a 'long' We have code which relies on it being able to contain negative values as error indications, but 'ssize_t' from unistd.h is a bridge too far for vrt.h which is meant to be _incredibly_ portable to avoid problems at compile time. diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 13ab454e2..fe9da5d77 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -68,7 +68,7 @@ pan_privs(struct vsb *vsb, const struct vrt_privs *privs) VRBT_FOREACH(vp, vrt_priv_tree, &privs->privs) { PAN_CheckMagic(vsb, vp, VRT_PRIV_MAGIC); VSB_printf(vsb, - "priv {p %p l %d f %p} vmod %jx\n", + "priv {p %p l %ld f %p} vmod %jx\n", vp->priv->priv, vp->priv->len, vp->priv->free, diff --git a/include/vrt.h b/include/vrt.h index 4b547e839..2e4ed2301 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -53,6 +53,7 @@ * * * 9.0 (scheduled for 2019-03-15) + * Make 'len' in vmod_priv and vrt_blob a 'long' * HTTP_Copy() removed * HTTP_Dup() added * HTTP_Clone() added @@ -177,7 +178,7 @@ struct strands { struct vrt_blob { unsigned type; - size_t len; + long len; const void *blob; }; @@ -531,7 +532,7 @@ typedef int vmod_event_f(VRT_CTX, struct vmod_priv *, enum vcl_event_e); typedef void vmod_priv_free_f(void *); struct vmod_priv { void *priv; - int len; + long len; vmod_priv_free_f *free; }; From phk at FreeBSD.org Tue Feb 5 06:40:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Feb 2019 06:40:08 +0000 (UTC) Subject: [master] 731c2182a Adjust printf to suit previous commit Message-ID: <20190205064008.741739B304@lists.varnish-cache.org> commit 731c2182a7afe20eb22931ff224041c4b96a6a82 Author: Poul-Henning Kamp Date: Tue Feb 5 06:39:39 2019 +0000 Adjust printf to suit previous commit diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index d032a439b..75e691794 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -527,7 +527,7 @@ vmod_sub(VRT_CTX, VCL_BLOB b, VCL_BYTES n, VCL_BYTES off) if (off + n > b->len) { VERR(ctx, "size %jd from offset %jd requires more bytes than " - "blob length %zd in blob.sub()", + "blob length %ld in blob.sub()", (intmax_t)n, (intmax_t)off, b->len); return NULL; } From phk at FreeBSD.org Tue Feb 5 07:08:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Feb 2019 07:08:07 +0000 (UTC) Subject: [master] ebbbad964 Flexelinting Message-ID: <20190205070807.D22F29BD35@lists.varnish-cache.org> commit ebbbad96484f2ec6823911f460ad5a1107c8a9c3 Author: Poul-Henning Kamp Date: Tue Feb 5 07:07:42 2019 +0000 Flexelinting diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index db16284f6..19e3463b3 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -73,7 +73,7 @@ } while (0) -static double timeout = 5; // XXX should be settable by arg ? +static const double timeout = 5; // XXX should be settable by arg ? static void cli_write(int sock, const char *s) diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h index e12463981..41b7781a5 100644 --- a/bin/varnishd/cache/cache_objhead.h +++ b/bin/varnishd/cache/cache_objhead.h @@ -66,7 +66,6 @@ void HSH_DeleteObjHead(const struct worker *, struct objhead *); int HSH_DerefObjCore(struct worker *, struct objcore **, int rushmax); #define HSH_RUSH_POLICY -1 -#define HSH_RUSH_ALL INT_MAX enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **); void HSH_Ref(struct objcore *o); diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index ed89868d7..a524a7981 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -50,7 +50,9 @@ const char * const VCL_TEMP_COLD = "cold"; const char * const VCL_TEMP_WARM = "warm"; const char * const VCL_TEMP_BUSY = "busy"; const char * const VCL_TEMP_COOLING = "cooling"; -const char * const VCL_TEMP_LABEL = "label"; + +// not really a state +static const char * const VCL_TEMP_LABEL = "label"; /* * XXX: Presently all modifications to this list happen from the diff --git a/bin/varnishd/cache/cache_vcl.h b/bin/varnishd/cache/cache_vcl.h index e9d99a710..aa530cb5b 100644 --- a/bin/varnishd/cache/cache_vcl.h +++ b/bin/varnishd/cache/cache_vcl.h @@ -75,7 +75,6 @@ extern const char * const VCL_TEMP_COLD; extern const char * const VCL_TEMP_WARM; extern const char * const VCL_TEMP_BUSY; extern const char * const VCL_TEMP_COOLING; -extern const char * const VCL_TEMP_LABEL; /* * NB: The COOLING temperature is neither COLD nor WARM. diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 88e89b36d..238dec743 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -72,7 +72,8 @@ // Macros defined differently in each VMOD -esym(767, VPFX) // macro '___' was defined differently in another module -esym(767, VARGS) // macro '___' was defined differently in another module --esym(767, VENUM) // macro '___' was defined differently in another module +-esym(767, VENUM) // macro '___' was defined differently in another module +-esym(14, enum_vmod*) // Symbol '___' previously defined (___) ////////////// -efunc(1791, pdiff) // return last on line ////////////// diff --git a/lib/libvmod_blob/vmod_blob.h b/lib/libvmod_blob/vmod_blob.h index 032f59986..3b027480a 100644 --- a/lib/libvmod_blob/vmod_blob.h +++ b/lib/libvmod_blob/vmod_blob.h @@ -32,13 +32,13 @@ #include enum encoding { - _INVALID = 0, + __INVALID_ENCODING = 0, #define VMODENUM(x) x, #include "tbl_encodings.h" __MAX_ENCODING }; -#define AENC(enc) assert((enc) > _INVALID && (enc) < __MAX_ENCODING) +#define AENC(enc) assert((enc) > __INVALID_ENCODING && (enc) < __MAX_ENCODING) /* * The enums MUST appear in this order, since LOWER and UPPER are used to diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index d8ac65109..154595e8f 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -41,7 +41,7 @@ /*lint -esym(749, shard_change_task_e::*) */ enum shard_change_task_e { - _INVALID = 0, + _SHARD_TASK_E_INVALID = 0, CLEAR, ADD_BE, REMOVE_BE, From phk at FreeBSD.org Tue Feb 5 08:04:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Feb 2019 08:04:07 +0000 (UTC) Subject: [master] 51542bef0 Filter all but the interface symbol out of vmod shlibs. Message-ID: <20190205080407.8D504A0C4A@lists.varnish-cache.org> commit 51542bef0fce998fded3899e53745e075a22bb39 Author: Poul-Henning Kamp Date: Tue Feb 5 08:03:22 2019 +0000 Filter all but the interface symbol out of vmod shlibs. Use boilerplate for vmod_debug diff --git a/.gitignore b/.gitignore index 37b62a9fa..6a6eb8f57 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ Makefile Makefile.in # ... +_* .deps/ .libs/ *.o diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 94a9f5a71..f7886920c 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -66,6 +66,7 @@ libvmod_XXX_la_CFLAGS = \\ \t at SAN_CFLAGS@ libvmod_XXX_la_LDFLAGS = \\ +\t-export-symbols-regex 'Vmod_XXX_Data' \\ \t$(AM_LDFLAGS) \\ \t$(VMOD_LDFLAGS) \\ \t at SAN_LDFLAGS@ diff --git a/lib/libvmod_blob/automake_boilerplate.am b/lib/libvmod_blob/automake_boilerplate.am index e9346968e..5bad5e10a 100644 --- a/lib/libvmod_blob/automake_boilerplate.am +++ b/lib/libvmod_blob/automake_boilerplate.am @@ -18,6 +18,7 @@ libvmod_blob_la_CFLAGS = \ @SAN_CFLAGS@ libvmod_blob_la_LDFLAGS = \ + -export-symbols-regex 'Vmod_blob_Data' \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am index edb074cce..4823e1b4d 100644 --- a/lib/libvmod_debug/Makefile.am +++ b/lib/libvmod_debug/Makefile.am @@ -5,42 +5,10 @@ libvmod_debug_la_SOURCES = \ vmod_debug_obj.c \ vmod_debug_dyn.c -# Boilerplate, do not change +include $(srcdir)/automake_boilerplate.am -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmoddir = $(pkglibdir)/vmods -vmod_srcdir = $(top_srcdir)/lib/libvmod_debug -vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs = - - -noinst_LTLIBRARIES = libvmod_debug.la - -libvmod_debug_la_CFLAGS = \ - @SAN_CFLAGS@ - -libvmod_debug_la_LDFLAGS = \ - $(AM_LDFLAGS) \ - $(VMOD_LDFLAGS) \ - -rpath /nowhere - @SAN_LDFLAGS@ - -nodist_libvmod_debug_la_SOURCES = \ - vcc_if.c \ - vcc_if.h - -$(libvmod_debug_la_OBJECTS): vcc_if.h - -vcc_if.h vmod_debug.rst vmod_debug.man.rst: vcc_if.c - -vcc_if.c: $(vmodtool) $(vmod_srcdir)/vmod.vcc - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(vmod_srcdir)/vmod.vcc +# not --strict +vmodtoolargs = --boilerplate .vsc.c: $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $< @@ -49,14 +17,7 @@ VSC_SRC = VSC_debug.vsc VSC_GEN_C = VSC_debug.c VSC_GEN_H = VSC_debug.h +CLEANFILES += $(VSC_GEN_C) $(VSC_GEN_H) + BUILT_SOURCES = $(VSC_GEN_C) libvmod_debug_la_SOURCES += $(VSC_SRC) - -$(VSC_GEN_C): $(top_srcdir)/lib/libvcc/vsctool.py - -EXTRA_DIST = vmod.vcc - -CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \ - $(builddir)/vmod_debug.rst \ - $(builddir)/vmod_debug.man.rst \ - $(VSC_GEN_C) $(VSC_GEN_H) diff --git a/lib/libvmod_debug/automake_boilerplate.am b/lib/libvmod_debug/automake_boilerplate.am new file mode 100644 index 000000000..fd3aaea3a --- /dev/null +++ b/lib/libvmod_debug/automake_boilerplate.am @@ -0,0 +1,40 @@ + +# Boilerplate generated by vmodtool.py - changes will be overwritten + +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmoddir = $(pkglibdir)/vmods +vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py +vmodtoolargs = --strict --boilerplate + +vmod_LTLIBRARIES = libvmod_debug.la + +libvmod_debug_la_CFLAGS = \ + @SAN_CFLAGS@ + +libvmod_debug_la_LDFLAGS = \ + -export-symbols-regex 'Vmod_debug_Data' \ + $(AM_LDFLAGS) \ + $(VMOD_LDFLAGS) \ + @SAN_LDFLAGS@ + +nodist_libvmod_debug_la_SOURCES = vcc_if.c vcc_if.h + +$(libvmod_debug_la_OBJECTS): vcc_if.h + +vcc_if.h vmod_debug.rst vmod_debug.man.rst: vcc_if.c + +vcc_if.c: $(vmodtool) $(srcdir)/vmod.vcc + @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc + +EXTRA_DIST = vmod.vcc automake_boilerplate.am + +CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \ + $(builddir)/vmod_debug.rst \ + $(builddir)/vmod_debug.man.rst + diff --git a/lib/libvmod_directors/automake_boilerplate.am b/lib/libvmod_directors/automake_boilerplate.am index df54f452c..2c0b9b5b9 100644 --- a/lib/libvmod_directors/automake_boilerplate.am +++ b/lib/libvmod_directors/automake_boilerplate.am @@ -18,6 +18,7 @@ libvmod_directors_la_CFLAGS = \ @SAN_CFLAGS@ libvmod_directors_la_LDFLAGS = \ + -export-symbols-regex 'Vmod_directors_Data' \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_proxy/automake_boilerplate.am b/lib/libvmod_proxy/automake_boilerplate.am index 64f3c5a0a..4ee1e5abd 100644 --- a/lib/libvmod_proxy/automake_boilerplate.am +++ b/lib/libvmod_proxy/automake_boilerplate.am @@ -18,6 +18,7 @@ libvmod_proxy_la_CFLAGS = \ @SAN_CFLAGS@ libvmod_proxy_la_LDFLAGS = \ + -export-symbols-regex 'Vmod_proxy_Data' \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_purge/automake_boilerplate.am b/lib/libvmod_purge/automake_boilerplate.am index b400cdb3c..d107999e3 100644 --- a/lib/libvmod_purge/automake_boilerplate.am +++ b/lib/libvmod_purge/automake_boilerplate.am @@ -18,6 +18,7 @@ libvmod_purge_la_CFLAGS = \ @SAN_CFLAGS@ libvmod_purge_la_LDFLAGS = \ + -export-symbols-regex 'Vmod_purge_Data' \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_std/automake_boilerplate.am b/lib/libvmod_std/automake_boilerplate.am index c1d6e3d42..e744db0c6 100644 --- a/lib/libvmod_std/automake_boilerplate.am +++ b/lib/libvmod_std/automake_boilerplate.am @@ -18,6 +18,7 @@ libvmod_std_la_CFLAGS = \ @SAN_CFLAGS@ libvmod_std_la_LDFLAGS = \ + -export-symbols-regex 'Vmod_std_Data' \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_unix/automake_boilerplate.am b/lib/libvmod_unix/automake_boilerplate.am index b71674a0e..18d06b8cd 100644 --- a/lib/libvmod_unix/automake_boilerplate.am +++ b/lib/libvmod_unix/automake_boilerplate.am @@ -18,6 +18,7 @@ libvmod_unix_la_CFLAGS = \ @SAN_CFLAGS@ libvmod_unix_la_LDFLAGS = \ + -export-symbols-regex 'Vmod_unix_Data' \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_vtc/automake_boilerplate.am b/lib/libvmod_vtc/automake_boilerplate.am index d7bd77ea1..063ab92ba 100644 --- a/lib/libvmod_vtc/automake_boilerplate.am +++ b/lib/libvmod_vtc/automake_boilerplate.am @@ -18,6 +18,7 @@ libvmod_vtc_la_CFLAGS = \ @SAN_CFLAGS@ libvmod_vtc_la_LDFLAGS = \ + -export-symbols-regex 'Vmod_vtc_Data' \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ From phk at FreeBSD.org Tue Feb 5 08:36:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Feb 2019 08:36:08 +0000 (UTC) Subject: [master] 5de84f39d slink assures me that blobs can use size_t for ->len, and that this one <= 0 test is an oversight. Message-ID: <20190205083608.D9291A1824@lists.varnish-cache.org> commit 5de84f39dc715ade7e08d63466294663adc47eee Author: Poul-Henning Kamp Date: Tue Feb 5 08:35:08 2019 +0000 slink assures me that blobs can use size_t for ->len, and that this one <= 0 test is an oversight. diff --git a/include/vrt.h b/include/vrt.h index 2e4ed2301..d3d055945 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -178,7 +178,7 @@ struct strands { struct vrt_blob { unsigned type; - long len; + size_t len; const void *blob; }; diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 4495474d3..e709e9cf0 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -571,7 +571,7 @@ shard_param_args(VRT_CTX, who, p->vcl_name, by_s); return (NULL); } - if (key_blob == NULL || key_blob->len <= 0 || + if (key_blob == NULL || key_blob->len == 0 || key_blob->blob == NULL) { sharddir_err(ctx, SLT_Error, "%s %s: " "by=BLOB but no or empty key_blob " From phk at FreeBSD.org Tue Feb 5 08:41:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Feb 2019 08:41:08 +0000 (UTC) Subject: [master] 2a57aeaf0 Revert printf format to match size_t Message-ID: <20190205084108.68696A1B53@lists.varnish-cache.org> commit 2a57aeaf04f874e68b461a2891177a3dbc99081d Author: Poul-Henning Kamp Date: Tue Feb 5 08:40:39 2019 +0000 Revert printf format to match size_t diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index 75e691794..d032a439b 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -527,7 +527,7 @@ vmod_sub(VRT_CTX, VCL_BLOB b, VCL_BYTES n, VCL_BYTES off) if (off + n > b->len) { VERR(ctx, "size %jd from offset %jd requires more bytes than " - "blob length %ld in blob.sub()", + "blob length %zd in blob.sub()", (intmax_t)n, (intmax_t)off, b->len); return NULL; } From nils.goroll at uplex.de Tue Feb 5 10:04:10 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Feb 2019 10:04:10 +0000 (UTC) Subject: [master] 69dc99d33 fix cli backend.list -p / backend.list -p -j Message-ID: <20190205100410.5797AA52D7@lists.varnish-cache.org> commit 69dc99d33b8073e55c1ed4f26caa1b8fd249499e Author: Nils Goroll Date: Tue Feb 5 11:01:33 2019 +0100 fix cli backend.list -p / backend.list -p -j $ /tmp/bin/varnishadm backend.list -p Backend name Admin Probe Last change boot.a1 probe healthy Tue, 05 Feb 2019 09:58:23 GMT $ /tmp/bin/varnishadm backend.list -pj [ 2, ["backend.list", "-pj"], 1549360935.584, { "boot.a1": { "type": "backend", "admin_health": "probe", "probe_message": "healthy", "probe_details": {}, "last_change": 1549360703.515 }, Fixes #2894 diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 2c8ce70f8..039871fc0 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -458,8 +458,12 @@ vbe_list(VRT_CTX, const struct director *d, struct vsb *vsb, int vflag, if (bp->probe != NULL) VBP_Status(vsb, bp, vflag | pflag, jflag); - else if ((vflag | pflag) == 0 && jflag) + else if (jflag && (vflag | pflag)) + VSB_printf(vsb, "{},\n"); + else if (jflag) VSB_printf(vsb, "\"%s\"", d->sick ? "sick" : "healthy"); + else if (vflag | pflag) + return; else VSB_printf(vsb, "%-10s", d->sick ? "sick" : "healthy"); } From nils.goroll at uplex.de Tue Feb 5 10:09:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Feb 2019 10:09:09 +0000 (UTC) Subject: [master] 2669a2f6a sync vrt changelog Message-ID: <20190205100909.185F9A55CB@lists.varnish-cache.org> commit 2669a2f6a9c23697d211b5e51cf9eba15e78efa4 Author: Nils Goroll Date: Tue Feb 5 11:08:25 2019 +0100 sync vrt changelog diff --git a/include/vrt.h b/include/vrt.h index d3d055945..24d0373c3 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -53,7 +53,7 @@ * * * 9.0 (scheduled for 2019-03-15) - * Make 'len' in vmod_priv and vrt_blob a 'long' + * Make 'len' in vmod_priv 'long' * HTTP_Copy() removed * HTTP_Dup() added * HTTP_Clone() added From nils.goroll at uplex.de Tue Feb 5 10:39:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Feb 2019 10:39:08 +0000 (UTC) Subject: [master] b71ceaa65 centralize definition of backend.list fields width Message-ID: <20190205103908.38C3CA610E@lists.varnish-cache.org> commit b71ceaa65886e06b1987d1ed5425b8d82377fee6 Author: Nils Goroll Date: Tue Feb 5 11:28:29 2019 +0100 centralize definition of backend.list fields width Yes, this is fugly, yet formatting of the backend.list output is spread all over the place (including vmods via the vdi_list_f callback), so until we decide it is worth spending time on separating formatting from content, I see no other option than making the field widths known to all vrt. The route ahead will likely be to only generate json and make varnishadm reponsible for formatting plain text output. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 039871fc0..36f036fdc 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -465,7 +465,8 @@ vbe_list(VRT_CTX, const struct director *d, struct vsb *vsb, int vflag, else if (vflag | pflag) return; else - VSB_printf(vsb, "%-10s", d->sick ? "sick" : "healthy"); + VSB_printf(vsb, "%-*s", VDI_LIST_W_PROBE, + d->sick ? "sick" : "healthy"); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 3f478db38..19719fca3 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -515,7 +515,7 @@ void VBP_Status(struct vsb *vsb, const struct backend *be, int details, int json) { struct vbp_target *vt; - char buf[12]; + char buf[VDI_LIST_W_PROBE + 2]; CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); vt = be->probe; @@ -527,7 +527,7 @@ VBP_Status(struct vsb *vsb, const struct backend *be, int details, int json) if (json) VSB_printf(vsb, "\"%s\"", buf); else - VSB_printf(vsb, "%-10s", buf); + VSB_printf(vsb, "%-*s", VDI_LIST_W_PROBE, buf); return; } diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index a94c36ab5..3b21903f4 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -301,12 +301,14 @@ do_list(struct cli *cli, struct director *d, void *priv) ctx = VCL_Get_CliCtx(0,0); // XXX admin health "probe" for the no-probe case is confusing - VCLI_Out(cli, "\n%-30s %-7s ", d->vdir->cli_name, VDI_Ahealth(d)); + VCLI_Out(cli, "\n%-*s %-*s ", + VDI_LIST_W_NAME, d->vdir->cli_name, + VDI_LIST_W_ADMIN, VDI_Ahealth(d)); if (d->vdir->methods->list != NULL) d->vdir->methods->list(ctx, d, cli->sb, 0, 0, 0); else - VCLI_Out(cli, "%-10s", cli_health(ctx, d)); + VCLI_Out(cli, "%-*s", VDI_LIST_W_PROBE, cli_health(ctx, d)); VTIM_format(d->vdir->health_changed, time_str); VCLI_Out(cli, " %s", time_str); @@ -402,8 +404,11 @@ cli_backend_list(struct cli *cli, const char * const *av, void *priv) VCLI_Out(cli, "}"); VCLI_JSON_end(cli); } else { - VCLI_Out(cli, "%-30s %-7s %-10s %s", - "Backend name", "Admin", "Probe", "Last change"); + VCLI_Out(cli, "%-*s %-*s %-*s %s", + VDI_LIST_W_NAME, "Backend name", + VDI_LIST_W_ADMIN, "Admin", + VDI_LIST_W_PROBE, "Probe", + "Last change"); (void)VCL_IterDirector(cli, av[i], do_list, la); } } diff --git a/include/vrt.h b/include/vrt.h index 24d0373c3..b4c77462b 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -496,6 +496,11 @@ struct vdi_methods { vdi_list_f *list; }; +/* width of fields in "backend.list" */ +#define VDI_LIST_W_NAME 30 +#define VDI_LIST_W_ADMIN 7 +#define VDI_LIST_W_PROBE 10 + struct vcldir; struct director { From nils.goroll at uplex.de Tue Feb 5 10:47:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Feb 2019 10:47:08 +0000 (UTC) Subject: [master] 5cb568af8 widen the Probe field of the backend.list CLI command Message-ID: <20190205104708.722B3A6588@lists.varnish-cache.org> commit 5cb568af895d22229bebbc8731f8c5869db75e37 Author: Nils Goroll Date: Tue Feb 5 11:45:51 2019 +0100 widen the Probe field of the backend.list CLI command diff --git a/doc/changes.rst b/doc/changes.rst index 7d1b59252..a83fe8e84 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -62,6 +62,12 @@ Varnish Cache trunk (ongoing) ``vcl_dir`` -> ``vcl_path`` ``vmod_dir`` -> ``vmod_path`` +* Changed the width of the `Probe` column of the ``backend.list`` + cli command from 10 to 14 characters + + For best forward compatibility, we recommend that scripts parse JSON + output as obtained using the ``-j`` option. + VCL --- diff --git a/include/vrt.h b/include/vrt.h index b4c77462b..091508dda 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -499,7 +499,7 @@ struct vdi_methods { /* width of fields in "backend.list" */ #define VDI_LIST_W_NAME 30 #define VDI_LIST_W_ADMIN 7 -#define VDI_LIST_W_PROBE 10 +#define VDI_LIST_W_PROBE 14 struct vcldir; From nils.goroll at uplex.de Tue Feb 5 10:52:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Feb 2019 10:52:09 +0000 (UTC) Subject: [master] 70693bee0 do we want to retire backend.list -v ? Message-ID: <20190205105209.13FE3A72E4@lists.varnish-cache.org> commit 70693bee053043f696b221cb5daba8f040b9b06e Author: Nils Goroll Date: Tue Feb 5 11:51:34 2019 +0100 do we want to retire backend.list -v ? diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index 3b21903f4..bff135a1f 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -371,6 +371,10 @@ cli_backend_list(struct cli *cli, const char * const *av, void *priv) struct list_args la[1]; int i; + /* + * XXX for all cases in varnish-cache, -v is synonymous to + * -p and -v is not documented. Retire? + */ (void)priv; ASSERT_CLI(); INIT_OBJ(la, LIST_ARGS_MAGIC); From phk at FreeBSD.org Tue Feb 5 11:00:16 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Feb 2019 11:00:16 +0000 (UTC) Subject: [master] 9ceeed8aa Fail VCL loading if VMOD objects are left uninitialized. Message-ID: <20190205110016.D4320A7913@lists.varnish-cache.org> commit 9ceeed8aa880bdb3eb0378aee1ace3d2dc6fca0f Author: Poul-Henning Kamp Date: Tue Feb 5 10:58:36 2019 +0000 Fail VCL loading if VMOD objects are left uninitialized. Allow VMOD writers to permit with NULL_OK flag. Only call object destructor on initialized objects. Fixes #2839 diff --git a/bin/varnishtest/tests/r02839.vtc b/bin/varnishtest/tests/r02839.vtc new file mode 100644 index 000000000..16525a833 --- /dev/null +++ b/bin/varnishtest/tests/r02839.vtc @@ -0,0 +1,32 @@ +varnishtest "Test uninitialized vmod objects" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend "" -start + +varnish v1 -errvcl "Object vo_null not initialized" { + import debug; + + backend default { .host = "127.0.0.1"; } + + sub vcl_init { + if (false) { + new null = debug.obj(); + } + } +} + +varnish v1 -vcl { + import debug; + + backend default { .host = "127.0.0.1"; } + + sub vcl_init { + if (false) { + new null = debug.obj_opt(); + } + } +} diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index d9bc4f84b..09c1d4704 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -293,6 +293,7 @@ EmitInitFini(const struct vcc *tl) { struct inifin *p, *q = NULL; unsigned has_event = 0; + struct symbol *sy; Fh(tl, 0, "\n"); Fh(tl, 0, "static unsigned vgc_inistep;\n"); @@ -317,6 +318,13 @@ EmitInitFini(const struct vcc *tl) if (VSB_len(p->event)) has_event = 1; } + VTAILQ_FOREACH(sy, &tl->sym_objects, sideways) { + Fc(tl, 0, "\tif (!%s) {\n", sy->rname); + Fc(tl, 0, "\t\tVRT_fail(ctx, " + "\"Object %s not initialized\");\n" , sy->rname); + Fc(tl, 0, "\t\treturn(1);\n"); + Fc(tl, 0, "\t}\n"); + } Fc(tl, 0, "\treturn(0);\n"); Fc(tl, 0, "}\n"); @@ -744,6 +752,7 @@ VCC_New(void) VTAILQ_INIT(&tl->tokens); VTAILQ_INIT(&tl->sources); VTAILQ_INIT(&tl->procs); + VTAILQ_INIT(&tl->sym_objects); tl->nsources = 0; diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 4e7e94c05..ece13c662 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -134,6 +134,7 @@ struct symbol { unsigned magic; #define SYMBOL_MAGIC 0x3368c9fb VTAILQ_ENTRY(symbol) list; + VTAILQ_ENTRY(symbol) sideways; VTAILQ_HEAD(,symbol) children; char *name; @@ -255,6 +256,8 @@ struct vcc { const char *default_director; const char *default_probe; + VTAILQ_HEAD(, symbol) sym_objects; + unsigned unique; unsigned vmod_count; }; diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 678821429..483411010 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -353,6 +353,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) struct vsb *buf; const struct vjsn_val *vv, *vf; const char *p; + int null_ok = 0; (void)sym; ExpectErr(tl, ID); @@ -380,8 +381,19 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) } CAST_OBJ_NOTNULL(vv, sy2->eval_priv, VJSN_VAL_MAGIC); + // vv = object name vv = VTAILQ_NEXT(vv, list); + // vv = flags + assert(vv->type == VJSN_OBJECT); + VTAILQ_FOREACH(vf, &vv->children, list) + if (!strcmp(vf->name, "NULL_OK") && vf->type == VJSN_TRUE) + null_ok = 1; + if (!null_ok) + VTAILQ_INSERT_TAIL(&tl->sym_objects, sy1, sideways); + + vv = VTAILQ_NEXT(vv, list); + // vv = struct name Fh(tl, 0, "static %s *%s;\n\n", vv->value, sy1->rname); vv = VTAILQ_NEXT(vv, list); @@ -412,7 +424,8 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) vf = VTAILQ_FIRST(&vf->children); vf = VTAILQ_NEXT(vf, list); ifp = New_IniFin(tl); - VSB_printf(ifp->fin, "\t\t%s(&%s);", vf->value, sy1->rname); + VSB_printf(ifp->fin, "\t\tif (%s)\n", sy1->rname); + VSB_printf(ifp->fin, "\t\t\t\t%s(&%s);", vf->value, sy1->rname); /* Instantiate symbols for the methods */ buf = VSB_new_auto(); diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index f7886920c..3ad485273 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -502,6 +502,7 @@ class Stanza(object): self.doc = doc self.vcc = vcc self.rstlbl = None + self.null_ok = False self.methods = None self.proto = None self.parse() @@ -708,6 +709,9 @@ class ObjectStanza(Stanza): ''' $Object TYPE class ( ARGUMENTS ) ''' def parse(self): + if self.toks[1] == "NULL_OK": + self.toks.pop(1) + self.null_ok = True self.proto = ProtoType(self, retval=False) self.proto.obj = "x" + self.proto.name @@ -768,6 +772,7 @@ class ObjectStanza(Stanza): ll = [ "$OBJ", self.proto.name, + { "NULL_OK": self.null_ok }, "struct %s%s_%s" % (self.vcc.sympfx, self.vcc.modname, self.proto.name), ] diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 65464fd32..016bca407 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -218,7 +218,7 @@ in nanoseconds. For comparable results, a higher size run should called first and discarded. -$Object obj_opt(PRIV_CALL, PRIV_VCL, PRIV_TASK, +$Object NULL_OK obj_opt(PRIV_CALL, PRIV_VCL, PRIV_TASK, [STRING s], [BOOL b]) Test object constructor with all the fancy stuff. diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c index 0dea592a3..34eacca75 100644 --- a/lib/libvmod_debug/vmod_debug_obj.c +++ b/lib/libvmod_debug/vmod_debug_obj.c @@ -68,7 +68,6 @@ xyzzy_obj__fini(struct xyzzy_debug_obj **op) AN(op); AN(*op); FREE_OBJ(*op); - *op = NULL; } VCL_VOID v_matchproto_() @@ -204,8 +203,7 @@ xyzzy_obj_opt__fini(struct xyzzy_debug_obj_opt **op) struct xyzzy_debug_obj_opt *o; AN(op); - if (*op == NULL) - return; /* init has failed */ + AN(*op); TAKE_OBJ_NOTNULL(o, op, VMOD_DEBUG_OBJ_OPT_MAGIC); From phk at FreeBSD.org Tue Feb 5 14:10:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Feb 2019 14:10:11 +0000 (UTC) Subject: [master] 8471010e3 Report the VCL name of the uninitialized object. Message-ID: <20190205141011.6FF54AF499@lists.varnish-cache.org> commit 8471010e394f2d45ff2b2afbb6b3216734265587 Author: Poul-Henning Kamp Date: Tue Feb 5 14:09:36 2019 +0000 Report the VCL name of the uninitialized object. diff --git a/bin/varnishtest/tests/r02839.vtc b/bin/varnishtest/tests/r02839.vtc index 16525a833..275d934ff 100644 --- a/bin/varnishtest/tests/r02839.vtc +++ b/bin/varnishtest/tests/r02839.vtc @@ -7,14 +7,14 @@ server s1 { varnish v1 -vcl+backend "" -start -varnish v1 -errvcl "Object vo_null not initialized" { +varnish v1 -errvcl "Object nullobj not initialized" { import debug; backend default { .host = "127.0.0.1"; } sub vcl_init { if (false) { - new null = debug.obj(); + new nullobj = debug.obj(); } } } diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 09c1d4704..13994eb6b 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -321,7 +321,7 @@ EmitInitFini(const struct vcc *tl) VTAILQ_FOREACH(sy, &tl->sym_objects, sideways) { Fc(tl, 0, "\tif (!%s) {\n", sy->rname); Fc(tl, 0, "\t\tVRT_fail(ctx, " - "\"Object %s not initialized\");\n" , sy->rname); + "\"Object %s not initialized\");\n" , sy->name); Fc(tl, 0, "\t\treturn(1);\n"); Fc(tl, 0, "\t}\n"); } From nils.goroll at uplex.de Tue Feb 5 14:23:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Feb 2019 14:23:08 +0000 (UTC) Subject: [master] 03047de64 fix vmodtool generated RST headers Message-ID: <20190205142308.ECFECAF9BB@lists.varnish-cache.org> commit 03047de644276f13fdf4c10fa65f0828b97f4bed Author: Nils Goroll Date: Tue Feb 5 15:19:41 2019 +0100 fix vmodtool generated RST headers Neither the vmod_ default $Prefix not an author-supplied $Prefix has got anything to do with the actual name of the vmod. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 3ad485273..e31da1f7b 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -585,7 +585,7 @@ class ModuleStanza(Stanza): def rsthead(self, fo, man): if man: - write_rst_hdr(fo, self.vcc.sympfx + self.vcc.modname, "=", "=") + write_rst_hdr(fo, "VMOD " + self.vcc.modname, "=", "=") write_rst_hdr(fo, self.vcc.moddesc, "-", "-") fo.write("\n") fo.write(":Manual section: " + self.vcc.mansection + "\n") @@ -593,7 +593,7 @@ class ModuleStanza(Stanza): if self.rstlbl: fo.write('\n.. _' + self.rstlbl + ':\n') write_rst_hdr(fo, - self.vcc.sympfx + self.vcc.modname + + "VMOD " + self.vcc.modname + ' - ' + self.vcc.moddesc, "=", "=") From nils.goroll at uplex.de Tue Feb 5 15:02:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Feb 2019 15:02:08 +0000 (UTC) Subject: [master] 6412b84c3 test that structural tags are included in a (-w)ritten log Message-ID: <20190205150208.E4956B0661@lists.varnish-cache.org> commit 6412b84c3d37ef887c58262b4a7346d415a41771 Author: Nils Goroll Date: Tue Feb 5 16:00:21 2019 +0100 test that structural tags are included in a (-w)ritten log Tests #2852 Sorry, not the big overhaul of u6 yet... diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index 55b2fe909..b263af36c 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -12,7 +12,8 @@ process p1 { } -start shell { exec varnishlog -n ${v1_name} -D -P ${tmpdir}/vlog.pid \ - -w ${tmpdir}/vlog.bin -R 10/s + -w ${tmpdir}/vlog.bin -R 10/s \ + -i RespStatus -i ReqURL -i BereqURL } shell -match "Usage: .*varnishlog " \ From nils.goroll at uplex.de Tue Feb 5 15:11:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Feb 2019 15:11:09 +0000 (UTC) Subject: [master] bd70a2039 Test that director with sick backends only is shown as sick Message-ID: <20190205151109.6E193B0FBD@lists.varnish-cache.org> commit bd70a20395e9df742a5526f255da8d5ad26422cf Author: Nils Goroll Date: Tue Feb 5 16:10:10 2019 +0100 Test that director with sick backends only is shown as sick Tests #2887 diff --git a/bin/varnishtest/tests/r02887.vtc b/bin/varnishtest/tests/r02887.vtc new file mode 100644 index 000000000..52fbf7a81 --- /dev/null +++ b/bin/varnishtest/tests/r02887.vtc @@ -0,0 +1,20 @@ +varnishtest "Test that director with sick backends only is shown as sick" + +varnish v1 -vcl { + + import directors; + + probe p { .url = "/foo"; } + backend b1 { .host = "${bad_backend}"; .probe = p;} + backend b2 { .host = "${bad_backend}"; .probe = p;} + backend b3 { .host = "${bad_backend}"; .probe = p;} + + sub vcl_init { + new foo = directors.random(); + foo.add_backend(b1, 1); + foo.add_backend(b2, 1); + foo.add_backend(b3, 1); + } +} -start + +varnish v1 -cliexpect "foo.*sick" "backend.list" From nils.goroll at uplex.de Tue Feb 5 15:28:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Feb 2019 15:28:09 +0000 (UTC) Subject: [master] 69574c5af doc tlc Message-ID: <20190205152809.E903FB28FD@lists.varnish-cache.org> commit 69574c5aff714d34904837a5eae188797ff78961 Author: Nils Goroll Date: Tue Feb 5 16:27:29 2019 +0100 doc tlc diff --git a/doc/changes.rst b/doc/changes.rst index a83fe8e84..d8fb2404c 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -154,6 +154,20 @@ C APIs (for vmod and utility authors) AZ(ObjGetU64(req->wrk, req->body_oc, OA_LEN, &u)); +* vmodtool has been changed significantly to avoid various name + clashes. Rather than using literal prefixes/suffixes, vmod authors + should now (and might have to for making existing code continue to + compile) use the following macros + + * ``VPFX(name)`` to prepend the vmod prefix (``vmod_`` by default) + + * ``VARGS(name)`` as the name of a function/method's argument + struct, e.g.:: + + VCL_VOID vmod_test(VRT_CTX, struct VARGS(test) *args) { ... + + * ``VENUM(name)`` to access the enum by the name `name` + Fixed bugs ---------- diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index 36ef2b51c..574da32dc 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -324,6 +324,10 @@ ENUM Allows values from a set of constant strings. `Note` that the C-type is a string, not a C enum. + Enums will be passed as fixed pointers, so instead of string + comparisons, also pointer comparisons with ``VENUM(name)`` are + possible. + HEADER C-type: ``const struct gethdr_s *`` From dridi.boukelmoune at gmail.com Tue Feb 5 16:30:15 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Feb 2019 16:30:15 +0000 (UTC) Subject: [6.0] 7cf202583 promote vtree.h to vmod include Message-ID: <20190205163015.DB85EBEC30@lists.varnish-cache.org> commit 7cf202583c4e012405766e945042a35af76f8089 Author: Nils Goroll Date: Tue Nov 6 11:47:11 2018 +0100 promote vtree.h to vmod include Now that we include it in cache.h, it is required by vmods Ref #2813 diff --git a/include/Makefile.am b/include/Makefile.am index 98050fd78..01755e09d 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -71,6 +71,7 @@ nobase_pkginclude_HEADERS += \ vsha256.h \ vtcp.h \ vtim.h \ + vtree.h \ vrnd.h # Private headers @@ -98,7 +99,6 @@ nobase_noinst_HEADERS = \ vsub.h \ vss.h \ vtcp.h \ - vtree.h \ vus.h ## keep in sync with lib/libvcc/Makefile.am From dridi.boukelmoune at gmail.com Tue Feb 5 16:30:16 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Feb 2019 16:30:16 +0000 (UTC) Subject: [6.0] d2ef8217e lost commit from ee9e0f6e7259580f17bcba83158c2cc9b79c13f3 Message-ID: <20190205163016.0D4E7BEC33@lists.varnish-cache.org> commit d2ef8217eaa1a6b884d73658e34d28006bf4cd55 Author: Nils Goroll Date: Tue Nov 6 11:54:50 2018 +0100 lost commit from ee9e0f6e7259580f17bcba83158c2cc9b79c13f3 diff --git a/tools/include_wash.py b/tools/include_wash.py index 4df6eeaf3..973d5d6ad 100644 --- a/tools/include_wash.py +++ b/tools/include_wash.py @@ -33,7 +33,7 @@ def check(fn): if i in l: for i in ("stddef.h", "stdint.h", "vrt.h", "math.h", "pthread.h", "stdarg.h", "sys/types.h", - "vdef.h", "miniobj.h", "vas.h", "vqueue.h"): + "vdef.h", "miniobj.h", "vas.h", "vqueue.h", "vtree.h"): if i in l: print(fn, i + " included with cache.h") From dridi.boukelmoune at gmail.com Tue Feb 5 16:30:16 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Feb 2019 16:30:16 +0000 (UTC) Subject: [6.0] 0e3bcf7d9 changelog tlc Message-ID: <20190205163016.2FECABEC37@lists.varnish-cache.org> commit 0e3bcf7d9d2d464ceb599644c4e124dfe0af9979 Author: Nils Goroll Date: Tue Nov 6 12:25:00 2018 +0100 changelog tlc will continue looking backwards from 79687f1344387a26d5e3b0a3b0454ac917db76de Conflicts: doc/changes.rst diff --git a/doc/changes.rst b/doc/changes.rst index 624d4a43c..5ac9da18f 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -26,6 +26,14 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via individual releases. These documents are updated as part of the release process. +================================ +Varnish Cache 6.0.3 (unreleased) +================================ + +* Included ``vtree.h`` in the distribution for vmods and + renamed the red/black tree macros from ``VRB_*`` to ``VRBT_*`` + to disambiguate from the acronym for Varnish Request Body. + ================================ Varnish Cache 6.0.2 (2018-11-07) ================================ @@ -70,6 +78,28 @@ Varnish Cache 6.0.2 (2018-11-07) * Fix ``varnishstat -f`` in curses mode (interactively, without ``-1``, 2787_) +* Changed the default of the ``thread_pool_watchdog`` parameter + to 60 seconds to match the ``cli_timeout`` default + +* Fix warmup/rampup of the shard director (2823_) + +* Fix VRT_priv_task for calls from vcl_pipe {} (2820_) + +* Fix vmod object constructor documentation in the ``vmodtool.py`` - + generated RST files + +* Vmod developers are advised that anything returned by a vmod + function/method is assumed to be immutable. In other words, a vmod + `must not` modify any data which was previously returned. + +* ``Content-Length`` header is not rewritten in response to a HEAD + request, allows responses to HEAD requests to be cached + independently from GET responses. + +* ``return(fail("mumble"))`` can have a string argument that is + emitted by VCC as an error message if the VCL load fails due to the + return. (2694_) + * Handle an out-of-workspace condition in HTTP/2 delivery more gracefully (2589_) @@ -104,6 +134,7 @@ Varnish Cache 6.0.2 (2018-11-07) .. _2694: https://github.com/varnishcache/varnish-cache/issues/2694 .. _2696: https://github.com/varnishcache/varnish-cache/issues/2696 .. _2708: https://github.com/varnishcache/varnish-cache/issues/2708 +.. _2713: https://github.com/varnishcache/varnish-cache/issues/2713 .. _2744: https://github.com/varnishcache/varnish-cache/pull/2744 .. _2745: https://github.com/varnishcache/varnish-cache/issues/2745 .. _2746: https://github.com/varnishcache/varnish-cache/issues/2746 From dridi.boukelmoune at gmail.com Tue Feb 5 16:30:16 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Feb 2019 16:30:16 +0000 (UTC) Subject: [6.0] 5fdbd0957 polish blob.same and blob.equal Message-ID: <20190205163016.4B8A2BEC3B@lists.varnish-cache.org> commit 5fdbd0957eae0accf1dc10657db30e3766260654 Author: Nils Goroll Date: Mon Nov 12 17:37:24 2018 +0100 polish blob.same and blob.equal If two blob pointers compare equal, the blobs are both the same and equal - both being NULL is implied. diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index 632580e91..08fd2ddf8 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -527,7 +527,7 @@ vmod_same(VRT_CTX, VCL_BLOB b1, VCL_BLOB b2) { (void) ctx; - if (b1 == NULL && b2 == NULL) + if (b1 == b2) return 1; if (b1 == NULL || b2 == NULL) return 0; @@ -539,7 +539,7 @@ vmod_equal(VRT_CTX, VCL_BLOB b1, VCL_BLOB b2) { (void) ctx; - if (b1 == NULL && b2 == NULL) + if (b1 == b2) return 1; if (b1 == NULL || b2 == NULL) return 0; From dridi.boukelmoune at gmail.com Tue Feb 5 16:30:16 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Feb 2019 16:30:16 +0000 (UTC) Subject: [6.0] 06425c23a Improve argstruct test coverage Message-ID: <20190205163016.7025BBEC3F@lists.varnish-cache.org> commit 06425c23a46440e7deb14e4d13a6d1d55c8e4c47 Author: Nils Goroll Date: Tue Nov 13 12:35:53 2018 +0100 Improve argstruct test coverage Related to #2810 Conflicts: lib/libvmod_debug/vmod.vcc diff --git a/bin/varnishtest/tests/m00019.vtc b/bin/varnishtest/tests/m00019.vtc index b2c9d666f..edc0e3ab7 100644 --- a/bin/varnishtest/tests/m00019.vtc +++ b/bin/varnishtest/tests/m00019.vtc @@ -15,6 +15,10 @@ varnish v1 -vcl+backend { new obj3 = debug.obj(string="s_two", number=two); new obj4 = debug.obj(number=three, string="s_three"); new obj5 = debug.obj(number=three); + + new oo0 = debug.obj_opt(); + new oo1 = debug.obj_opt(s="string"); + new oo2 = debug.obj_opt(b=true); } sub vcl_deliver { @@ -32,6 +36,10 @@ varnish v1 -vcl+backend { set resp.http.obj3 = obj3.string() + ", " + obj3.number(); set resp.http.obj4 = obj4.string() + ", " + obj4.number(); set resp.http.obj5 = obj5.string() + ", " + obj5.number(); + + set resp.http.oo0 = oo0.meth_opt(s="s1"); + set resp.http.oo1 = oo1.meth_opt(b=false); + set resp.http.oo2 = oo2.meth_opt(); } } -start @@ -53,6 +61,11 @@ client c1 { expect resp.http.obj3 == "s_two, two" expect resp.http.obj4 == "s_three, three" expect resp.http.obj5 == "default, three" + + expect resp.http.oo0 == "obj oo0 obj_s *undef* obj_b *undef* met_s s1 met_b *undef*" + expect resp.http.oo1 == "obj oo1 obj_s string obj_b *undef* met_s *undef* met_b false" + expect resp.http.oo2 == "obj oo2 obj_s *undef* obj_b true met_s *undef* met_b *undef*" + } -run delay .1 diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 2b56f2e35..27f601ffa 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -178,3 +178,13 @@ Update counter $Function VOID vsc_destroy() Remove a vsc + +$Object obj_opt(PRIV_CALL, PRIV_VCL, PRIV_TASK, + [STRING s], [BOOL b]) + +Test object constructor with all the fancy stuff. + +$Method STRING .meth_opt(PRIV_CALL, PRIV_VCL, PRIV_TASK, + [STRING s], [BOOL b]) + +Test object method with all the fancy stuff. diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c index fcf74a740..9038bcb81 100644 --- a/lib/libvmod_debug/vmod_debug_obj.c +++ b/lib/libvmod_debug/vmod_debug_obj.c @@ -152,3 +152,94 @@ xyzzy_obj_test_priv_top(VRT_CTX, (void)o; return (xyzzy_test_priv_top(ctx, priv, s)); } + +/* ---------------------------------------------------------------------------- + * obj_opt (optional arguments and privs) + */ +struct xyzzy_debug_obj_opt { + unsigned magic; +#define VMOD_DEBUG_OBJ_OPT_MAGIC 0xccbd9b78 + char *name; + struct xyzzy_obj_opt_meth_opt_arg args; + void *freeptr; +}; + +VCL_VOID v_matchproto_() +xyzzy_obj_opt__init(VRT_CTX, + struct xyzzy_debug_obj_opt **op, const char *vcl_name, + struct xyzzy_obj_opt__init_arg *args) +{ + struct xyzzy_debug_obj_opt *o; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(args); + + AN(args->arg1); // priv_call + AN(args->arg2); // priv_vcl + AN(args->arg3); // priv_task + assert(args->arg1 != args->arg2); + assert(args->arg2 != args->arg3); + + if (args->valid_s) + AN(args->s); + + AN(op); + AZ(*op); + ALLOC_OBJ(o, VMOD_DEBUG_OBJ_OPT_MAGIC); + AN(o); + *op = o; + REPLACE(o->name, vcl_name); + memcpy(&o->args, args, sizeof o->args); + if (args->valid_s) { + REPLACE(o->freeptr, args->s); + o->args.s = o->freeptr; + } +} + +VCL_VOID v_matchproto_() +xyzzy_obj_opt__fini(struct xyzzy_debug_obj_opt **op) +{ + struct xyzzy_debug_obj_opt *o; + + AN(op); + if (*op == NULL) + return; /* init has failed */ + + TAKE_OBJ_NOTNULL(o, op, VMOD_DEBUG_OBJ_OPT_MAGIC); + + REPLACE(o->name, NULL); + if (o->freeptr) { + AN(o->args.valid_s); + REPLACE(o->freeptr, NULL); + } + FREE_OBJ(o); + AZ(o); +} + +VCL_STRING v_matchproto_() +xyzzy_obj_opt_meth_opt(VRT_CTX, + struct xyzzy_debug_obj_opt *o, + struct xyzzy_obj_opt_meth_opt_arg *args) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(o, VMOD_DEBUG_OBJ_OPT_MAGIC); + + AN(args); + AN(args->arg1); // priv_call + AN(args->arg2); // priv_vcl + AN(args->arg3); // priv_task + assert(args->arg1 != args->arg2); + assert(args->arg2 != args->arg3); + + return (VRT_CollectString(ctx, + "obj ", o->name, + " obj_s ", (o->args.valid_s ? o->args.s : "*undef*"), + " obj_b ", (o->args.valid_b + ? (o->args.b ? "true" : "false" ) + : "*undef*"), + " met_s ", (args->valid_s ? args->s : "*undef*"), + " met_b ", (args->valid_b + ? (args->b ? "true" : "false" ) + : "*undef*"), + vrt_magic_string_end)); +} From dridi.boukelmoune at gmail.com Tue Feb 5 16:30:16 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Feb 2019 16:30:16 +0000 (UTC) Subject: [6.0] 16033c289 Allow BOOL [!=]= BOOL comparisons Message-ID: <20190205163016.8D069BEC43@lists.varnish-cache.org> commit 16033c289e47507870ee48cf9935f9f22aa10230 Author: Poul-Henning Kamp Date: Mon Oct 29 11:59:44 2018 +0000 Allow BOOL [!=]= BOOL comparisons Fixes: #2809 diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 1f0699e19..780befa0c 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1111,6 +1111,8 @@ static const struct cmps vcc_cmps[] = { IDENT_REL(SUB), IDENT_REL(INSTANCE), + {BOOL, T_EQ, cmp_simple, "((!(\v1)) == (!(\v2)))" }, + {BOOL, T_NEQ, cmp_simple, "((!(\v1)) != (!(\v2)))" }, {IP, T_EQ, cmp_simple, "!VRT_ipcmp(\v1, \v2)" }, {IP, T_NEQ, cmp_simple, "VRT_ipcmp(\v1, \v2)" }, @@ -1144,9 +1146,6 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, vcc_type_t fmt) ERRCHK(tl); tk = tl->t; - if ((*e)->fmt == BOOL) - return; - for (cp = vcc_cmps; cp->fmt != VOID; cp++) { if ((*e)->fmt == cp->fmt && tl->t->tok == cp->token) { AN(cp->func); From dridi.boukelmoune at gmail.com Tue Feb 5 16:30:16 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Feb 2019 16:30:16 +0000 (UTC) Subject: [6.0] d4ff5c553 improve busyobj panic Message-ID: <20190205163016.AB83BBEC47@lists.varnish-cache.org> commit d4ff5c553a5fbf0dd816969c80e04a9e0bbd3289 Author: Nils Goroll Date: Tue Nov 13 17:43:41 2018 +0100 improve busyobj panic For backend requests, we do not have the thread local storage of the client thread available, so trq.req will normally be (nil). Dump the busyobj back pointers to req/sess/wrk to maximize the usefulness of panics. Also reorder the panic by struct busyobj and add some more members which may or may not be useful (but when we find out, it is likely too late). Also move the vmod list dump to the top level. Noticed when trying to understand if #2667 was also a stack overflow in the context of #2817 Conflicts: bin/varnishd/cache/cache_panic.c diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 8282ed814..d75e0b23f 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -61,6 +61,7 @@ static struct vsb pan_vsb_storage, *pan_vsb; static pthread_mutex_t panicstr_mtx; static void pan_sess(struct vsb *, const struct sess *); +static void pan_req(struct vsb *, const struct req *); /*--------------------------------------------------------------------*/ @@ -400,8 +401,37 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) return; VSB_indent(vsb, 2); PAN_CheckMagic(vsb, bo, BUSYOBJ_MAGIC); + VSB_printf(vsb, "end = %p,\n", bo->end); + VSB_printf(vsb, "retries = %d,\n", bo->retries); + + if (bo->req != NULL) + pan_req(vsb, bo->req); + if (bo->sp != NULL) + pan_sess(vsb, bo->sp); + if (bo->wrk != NULL) + pan_wrk(vsb, bo->wrk); + + if (bo->vfc != NULL) + pan_vfp(vsb, bo->vfc); + pan_ws(vsb, bo->ws); - VSB_printf(vsb, "retries = %d, ", bo->retries); + VSB_printf(vsb, "ws_bo = %p,\n", (void *)bo->ws_bo); + + // bereq0 left out + if (bo->bereq != NULL && bo->bereq->ws != NULL) + pan_http(vsb, "bereq", bo->bereq); + if (bo->beresp != NULL && bo->beresp->ws != NULL) + pan_http(vsb, "beresp", bo->beresp); + if (bo->stale_oc) + pan_objcore(vsb, "stale_oc", bo->stale_oc); + if (bo->fetch_objcore) + pan_objcore(vsb, "fetch", bo->fetch_objcore); + + if (VALID_OBJ(bo->htc, HTTP_CONN_MAGIC)) + pan_htc(vsb, bo->htc); + + // fetch_task left out + VSB_printf(vsb, "flags = {"); p = ""; /*lint -save -esym(438,p) -e539 */ @@ -411,27 +441,14 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) /*lint -restore */ VSB_printf(vsb, "},\n"); - if (VALID_OBJ(bo->htc, HTTP_CONN_MAGIC)) - pan_htc(vsb, bo->htc); - - if (bo->vfc) - pan_vfp(vsb, bo->vfc); + // timeouts/timers/acct/storage left out VDI_Panic(bo->director_req, vsb, "director_req"); if (bo->director_resp == bo->director_req) VSB_printf(vsb, "director_resp = director_req,\n"); else VDI_Panic(bo->director_resp, vsb, "director_resp"); - if (bo->bereq != NULL && bo->bereq->ws != NULL) - pan_http(vsb, "bereq", bo->bereq); - if (bo->beresp != NULL && bo->beresp->ws != NULL) - pan_http(vsb, "beresp", bo->beresp); - if (bo->fetch_objcore) - pan_objcore(vsb, "fetch", bo->fetch_objcore); - if (bo->stale_oc) - pan_objcore(vsb, "ims", bo->stale_oc); VCL_Panic(vsb, bo->vcl); - VMOD_Panic(vsb); VSB_indent(vsb, -2); VSB_printf(vsb, "},\n"); } @@ -496,7 +513,6 @@ pan_req(struct vsb *vsb, const struct req *req) pan_http(vsb, "resp", req->resp); VCL_Panic(vsb, req->vcl); - VMOD_Panic(vsb); if (req->body_oc != NULL) pan_objcore(vsb, "BODY", req->body_oc); @@ -684,6 +700,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond, pan_busyobj(pan_vsb, bo); if (bo != NULL) VSL_Flush(bo->vsl, 0); + VMOD_Panic(pan_vsb); } else { VSB_printf(pan_vsb, "Feature short panic supressed details.\n"); From dridi.boukelmoune at gmail.com Tue Feb 5 16:30:16 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Feb 2019 16:30:16 +0000 (UTC) Subject: [6.0] 33fcdd966 Add missing include Message-ID: <20190205163016.DA63CBEC4C@lists.varnish-cache.org> commit 33fcdd966aa11ea4f933b9415141e19048459790 Author: Federico G. Schwindt Date: Wed Nov 14 06:55:09 2018 +0900 Add missing include diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index 69f319509..750f5ddf7 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include From dridi.boukelmoune at gmail.com Tue Feb 5 16:30:16 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Feb 2019 16:30:16 +0000 (UTC) Subject: [6.0] 63ae1b0bd Make FlexeLint a little bit happier Message-ID: <20190205163016.F236EBEC4F@lists.varnish-cache.org> commit 63ae1b0bde381184a20078b032f9cfdcb8964e5e Author: Poul-Henning Kamp Date: Wed Nov 14 08:35:15 2018 +0000 Make FlexeLint a little bit happier diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c index 9038bcb81..3ecf33d0c 100644 --- a/lib/libvmod_debug/vmod_debug_obj.c +++ b/lib/libvmod_debug/vmod_debug_obj.c @@ -183,6 +183,9 @@ xyzzy_obj_opt__init(VRT_CTX, if (args->valid_s) AN(args->s); + if (args->valid_b) + AN(args->b); + AN(op); AZ(*op); ALLOC_OBJ(o, VMOD_DEBUG_OBJ_OPT_MAGIC); From phk at FreeBSD.org Tue Feb 5 18:08:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Feb 2019 18:08:09 +0000 (UTC) Subject: [master] 9b39b5bfa Properly prefix the arg structures. Message-ID: <20190205180809.6704D66F2@lists.varnish-cache.org> commit 9b39b5bfa753efa06d8034898652cef6a817c80e Author: Poul-Henning Kamp Date: Tue Feb 5 18:06:35 2019 +0000 Properly prefix the arg structures. Spotted by: Nils diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index e31da1f7b..5a80fdaac 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -478,7 +478,8 @@ class ProtoType(object): ll.append('%s.%s' % (self.st.vcc.csn, cfunc)) if self.argstruct: # We cannot use VARGS() here, we are after the #undef - ll.append('struct arg_%s' % self.cname(True)) + ll.append('struct arg_%s%s_%s' % + (self.st.vcc.sympfx, self.st.vcc.modname, self.cname(False))) else: ll.append("") for i in self.args: @@ -950,6 +951,17 @@ class vcc(object): fo.write(AMBOILERPLATE.replace("XXX", self.modname)) fo.close() + def mkdefs(self, fo): + fo.write('#define VPFX(a) %s##a\n' % self.sympfx) + fo.write('#define VARGS(a) arg_%s%s_##a\n' % + (self.sympfx, self.modname)) + fo.write('#define VENUM(a) enum_%s%s_##a\n' % + (self.sympfx, self.modname)) + fo.write('//lint -esym(755, VPFX)\n') + fo.write('//lint -esym(755, VARGS)\n') + fo.write('//lint -esym(755, VENUM)\n') + fo.write('\n') + def mkhfile(self): ''' Produce vcc_if.h file ''' fn = self.pfx + ".h" @@ -963,13 +975,7 @@ class vcc(object): fo.write("#endif\n") fo.write("\n") - fo.write('#define VPFX(a) %s##a\n' % self.sympfx) - fo.write('#define VARGS(a) arg_%s##a\n' % self.sympfx) - fo.write('#define VENUM(a) enum_%s##a\n' % self.sympfx) - fo.write('//lint -esym(755, VPFX)\n') - fo.write('//lint -esym(755, VARGS)\n') - fo.write('//lint -esym(755, VENUM)\n') - fo.write('\n') + self.mkdefs(fo); for j in sorted(self.enums): fo.write("extern VCL_ENUM VENUM(%s);\n" % j) @@ -1049,10 +1055,7 @@ class vcc(object): write_c_file_warning(fo) - fx.write('#define VPFX(a) %s##a\n' % self.sympfx) - fx.write('#define VARGS(a) arg_%s##a\n' % self.sympfx) - fx.write('#define VENUM(a) enum_%s##a\n' % self.sympfx) - fx.write('\n') + self.mkdefs(fx); fo.write('#include "config.h"\n') fo.write('#include \n') From phk at FreeBSD.org Tue Feb 5 19:01:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Feb 2019 19:01:08 +0000 (UTC) Subject: [master] 0203b3d37 Flexelinting Message-ID: <20190205190109.03B127A4B@lists.varnish-cache.org> commit 0203b3d377f9ace1f735f47b299b07d7809eb320 Author: Poul-Henning Kamp Date: Tue Feb 5 19:00:02 2019 +0000 Flexelinting diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 238dec743..42a40290f 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -70,10 +70,9 @@ ////////////// // Macros defined differently in each VMOD --esym(767, VPFX) // macro '___' was defined differently in another module --esym(767, VARGS) // macro '___' was defined differently in another module --esym(767, VENUM) // macro '___' was defined differently in another module --esym(14, enum_vmod*) // Symbol '___' previously defined (___) +// -esym(767, VPFX) // macro '___' was defined differently in another module +// -esym(767, VARGS) // macro '___' was defined differently in another module +// -esym(767, VENUM) // macro '___' was defined differently in another module ////////////// -efunc(1791, pdiff) // return last on line ////////////// diff --git a/flint.lnt b/flint.lnt index 1a0b88caf..6c98c5c94 100644 --- a/flint.lnt +++ b/flint.lnt @@ -91,9 +91,9 @@ /////////////////////////////////////////////////////////////////////// // Vmod/vmodtool.py --esym(14, vmod_enum_*) // Symbol '___' previously defined (___) --esym(759, vmod_enum_*) // header declaration for symbol '___' defined at (___) --esym(765, vmod_enum_*) // external '___' (___) could be made static +//-esym(14, vmod_enum_*) // Symbol '___' previously defined (___) +//-esym(759, vmod_enum_*) // header declaration for symbol '___' defined at (___) +//-esym(765, vmod_enum_*) // external '___' (___) could be made static /////////////////////////////////////////////////////////////////////// // diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 5a80fdaac..2a88c2ab4 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -957,7 +957,9 @@ class vcc(object): (self.sympfx, self.modname)) fo.write('#define VENUM(a) enum_%s%s_##a\n' % (self.sympfx, self.modname)) - fo.write('//lint -esym(755, VPFX)\n') + for a in ('VPFX', 'VARGS', 'VENUM'): + for b in (755, 767): + fo.write('//lint -esym(%d, %s)\n' % (b, a)) fo.write('//lint -esym(755, VARGS)\n') fo.write('//lint -esym(755, VENUM)\n') fo.write('\n') @@ -981,8 +983,12 @@ class vcc(object): fo.write("extern VCL_ENUM VENUM(%s);\n" % j) fo.write("\n") for j in sorted(self.enums): - fo.write("//lint -esym(759, enum_%s%s)\n" % (self.sympfx, j)) - fo.write("//lint -esym(765, enum_%s%s)\n" % (self.sympfx, j)) + fo.write("//lint -esym(14, enum_%s%s_%s)\n" % + (self.sympfx, self.modname, j)) + fo.write("//lint -esym(759, enum_%s%s_%s)\n" % + (self.sympfx, self.modname, j)) + fo.write("//lint -esym(765, enum_%s%s_%s)\n" % + (self.sympfx, self.modname, j)) fo.write("\n") for j in self.contents: From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:12 +0000 (UTC) Subject: [6.0] 767cb7a1c Try to straighten out resp.body processing. Message-ID: <20190206084012.87B409BDBB@lists.varnish-cache.org> commit 767cb7a1c3bed127138aebed1323b918af9f55e4 Author: Poul-Henning Kamp Date: Wed Nov 14 11:01:58 2018 +0000 Try to straighten out resp.body processing. Conflicts: bin/varnishd/cache/cache_req_fsm.c diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index b29f484e1..f81891207 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -350,70 +350,66 @@ cnt_transmit(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req->transport, TRANSPORT_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); AZ(req->stale_oc); + AZ(req->res_mode); - /* Grab a ref to the bo if there is one */ + /* Grab a ref to the bo if there is one (=streaming) */ boc = HSH_RefBoc(req->objcore); - clval = http_GetContentLength(req->resp); + /* RFC 7230, 3.3.3 */ + status = http_GetStatus(req->resp); + head = !strcmp(req->http0->hd[HTTP_HDR_METHOD].b, "HEAD"); + err = 0; + if (boc != NULL) req->resp_len = clval; else req->resp_len = ObjGetLen(req->wrk, req->objcore); - req->res_mode = 0; - - /* RFC 7230, 3.3.3 */ - status = http_GetStatus(req->resp); - head = !strcmp(req->http0->hd[HTTP_HDR_METHOD].b, "HEAD"); - if (head) { - if (req->objcore->flags & OC_F_PASS) - sendbody = -1; - else - sendbody = 0; - } else if (status < 200 || status == 204 || status == 304) { - req->resp_len = -1; - sendbody = 0; - } else + if (head || status < 200 || status == 204 || status == 304) + sendbody = 0; /* rfc7230,l,1748,1752 */ + else sendbody = 1; - err = 0; - if (sendbody >= 0) { - if (!req->disable_esi && req->resp_len != 0 && - ObjHasAttr(wrk, req->objcore, OA_ESIDATA) && - VDP_push(req, &VDP_esi, NULL, 0) < 0) - err++; - - if (cache_param->http_gzip_support && - ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) && - !RFC2616_Req_Gzip(req->http) && - VDP_push(req, &VDP_gunzip, NULL, 1) < 0) - err++; - - if (cache_param->http_range_support && - http_IsStatus(req->resp, 200)) { - http_ForceHeader(req->resp, H_Accept_Ranges, "bytes"); - if (sendbody && http_GetHdr(req->http, H_Range, &r)) - VRG_dorange(req, r); - } - } + if (!req->disable_esi && req->resp_len != 0 && + ObjHasAttr(wrk, req->objcore, OA_ESIDATA) && + VDP_push(req, &VDP_esi, NULL, 0) < 0) + err++; - if (sendbody < 0 || head) { - /* Don't touch HEAD C-L */ - sendbody = 0; - } else if (clval >= 0 && clval == req->resp_len) { - /* Reuse C-L header */ - } else { - http_Unset(req->resp, H_Content_Length); - if (req->resp_len >= 0 && sendbody) - http_PrintfHeader(req->resp, - "Content-Length: %jd", req->resp_len); + if (cache_param->http_gzip_support && + ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) && + !RFC2616_Req_Gzip(req->http) && + VDP_push(req, &VDP_gunzip, NULL, 1) < 0) + err++; + + if (cache_param->http_range_support && status == 200) { + http_ForceHeader(req->resp, H_Accept_Ranges, "bytes"); + if (http_GetHdr(req->http, H_Range, &r)) + VRG_dorange(req, r); } - if (err == 0) - req->transport->deliver(req, boc, sendbody); - else { + if (err) { VSLb(req->vsl, SLT_Error, "Failure to push processors"); req->doclose = SC_OVERLOAD; + } else { + if (status < 200 || status == 204) { + // rfc7230,l,1691,1695 + http_Unset(req->resp, H_Content_Length); + } else if (status == 304) { + // rfc7230,l,1675,1677 + http_Unset(req->resp, H_Content_Length); + } else if (clval >= 0 && clval == req->resp_len) { + /* Reuse C-L header */ + } else if (head && req->objcore->flags & OC_F_PASS) { + /* Don't touch C-L header */ + } else { + http_Unset(req->resp, H_Content_Length); + if (req->resp_len >= 0) + http_PrintfHeader(req->resp, + "Content-Length: %jd", req->resp_len); + } + if (req->resp_len == 0) + sendbody = 0; + req->transport->deliver(req, boc, sendbody); } VSLb_ts_req(req, "Resp", W_TIM_real(wrk)); @@ -434,6 +430,7 @@ cnt_transmit(struct worker *wrk, struct req *req) (void)HSH_DerefObjCore(wrk, &req->objcore, HSH_RUSH_POLICY); http_Teardown(req->resp); + req->res_mode = 0; return (REQ_FSM_DONE); } diff --git a/bin/varnishtest/tests/b00065.vtc b/bin/varnishtest/tests/b00065.vtc index 5eb86d8d9..daf030659 100644 --- a/bin/varnishtest/tests/b00065.vtc +++ b/bin/varnishtest/tests/b00065.vtc @@ -1,20 +1,33 @@ -varnishtest "Test that C-L on HEAD request aren't nuked" +varnishtest "Check that HEAD+pass returns Content-Length if backend provides it" server s1 { + rxreq + expect req.method == "GET" + txresp -bodylen 5 rxreq expect req.method == "HEAD" - txresp -nolen -hdr "content-length: 17" + txresp -nolen -hdr "Content-Length: 6" } -start varnish v1 -vcl+backend { - sub vcl_backend_fetch { - set bereq.method = "HEAD"; + sub vcl_recv { + if (req.url == "/2") { + return (pass); + } + } + sub vcl_backend_response { + set beresp.do_stream = false; } } -start client c1 { - txreq -req HEAD -hdr "connection: close" - rxresphdrs - expect resp.http.content-length == 17 - expect_close + txreq -req HEAD + rxresp -no_obj + expect resp.http.content-length == 5 +} -run + +client c1 { + txreq -req HEAD -url /2 + rxresp -no_obj + expect resp.http.content-length == 6 } -run diff --git a/bin/varnishtest/tests/r00730.vtc b/bin/varnishtest/tests/r00730.vtc deleted file mode 100644 index e28c8c912..000000000 --- a/bin/varnishtest/tests/r00730.vtc +++ /dev/null @@ -1,31 +0,0 @@ -varnishtest "Check that HEAD returns Content-Length if backend provides it" - -server s1 { - rxreq - txresp -bodylen 5 - rxreq - txresp -nolen -hdr "Content-Length: 6" -} -start - -varnish v1 -vcl+backend { - sub vcl_recv { - if (req.url == "/2") { - return (pass); - } - } - sub vcl_backend_response { - set beresp.do_stream = false; - } -} -start - -client c1 { - txreq -req HEAD - rxresp -no_obj - expect resp.http.content-length == 5 -} -run - -client c1 { - txreq -req HEAD -url /2 - rxresp -no_obj - expect resp.http.content-length == 6 -} -run From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:12 +0000 (UTC) Subject: [6.0] 24a589604 Get the RFC comment right Message-ID: <20190206084012.9B9589BDBD@lists.varnish-cache.org> commit 24a5896047fc5345f21f4fa0f587283da4219d01 Author: Poul-Henning Kamp Date: Wed Nov 14 11:05:50 2018 +0000 Get the RFC comment right diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index f81891207..864a4eb26 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -365,10 +365,12 @@ cnt_transmit(struct worker *wrk, struct req *req) else req->resp_len = ObjGetLen(req->wrk, req->objcore); - if (head || status < 200 || status == 204 || status == 304) - sendbody = 0; /* rfc7230,l,1748,1752 */ - else + if (head || status < 200 || status == 204 || status == 304) { + // rfc7230,l,1748,1752 + sendbody = 0; + } else { sendbody = 1; + } if (!req->disable_esi && req->resp_len != 0 && ObjHasAttr(wrk, req->objcore, OA_ESIDATA) && From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:12 +0000 (UTC) Subject: [6.0] 78886a10a Revert "Make FlexeLint a little bit happier" Message-ID: <20190206084012.BBC5B9BDC2@lists.varnish-cache.org> commit 78886a10ab88ef7e5d62168fb000d6112ffa26f9 Author: Nils Goroll Date: Wed Nov 14 12:43:41 2018 +0100 Revert "Make FlexeLint a little bit happier" The fact that a boolean argument is present does not imply that it's true. This reverts commit 57cd74cf51b63a54df68600a573eb0e20e2a6c7a. I promise to look after this once I got my Flexelint config back, which, unfortunately, I lost during a reinstall. Yes, I do have a backup, but cannot access it easily at the moment. diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c index 3ecf33d0c..9038bcb81 100644 --- a/lib/libvmod_debug/vmod_debug_obj.c +++ b/lib/libvmod_debug/vmod_debug_obj.c @@ -183,9 +183,6 @@ xyzzy_obj_opt__init(VRT_CTX, if (args->valid_s) AN(args->s); - if (args->valid_b) - AN(args->b); - AN(op); AZ(*op); ALLOC_OBJ(o, VMOD_DEBUG_OBJ_OPT_MAGIC); From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:12 +0000 (UTC) Subject: [6.0] 2d133bc7a work around older libedit which fails to append rl_completion_append_character Message-ID: <20190206084012.D5D509BDD2@lists.varnish-cache.org> commit 2d133bc7ab472881640623d8cc8fee6c43f98cb6 Author: Nils Goroll Date: Thu Nov 15 16:25:04 2018 +0100 work around older libedit which fails to append rl_completion_append_character I failed to find any issue in varnish and failed to find a workaround for the code, so change the test to add an additional space after the tab-completion. If anyone has a better solution, please go head. Otherwise I feel this is not worth any more time. The test change, though not being ideal, should not cause any harm at least. Closes #2833 diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc index d6e8098dd..b53a9ba7b 100644 --- a/bin/varnishtest/tests/u00011.vtc +++ b/bin/varnishtest/tests/u00011.vtc @@ -26,7 +26,7 @@ process p1 -write "vcl.li\t\r" process p1 -expect-text 0 1 "active auto/warm" -process p1 -write "vcl.s\t\th\tvcl1\r" +process p1 -write "vcl.s\t\th\t vcl1\r" process p1 -expect-text 0 1 "backend s1" From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:12 +0000 (UTC) Subject: [6.0] 104679274 vtc: array not needed for token expansion Message-ID: <20190206084012.EEF1E9BDD9@lists.varnish-cache.org> commit 10467927428b0c913220e76ef0cbeec002a5d988 Author: Nils Goroll Date: Sat Nov 17 12:15:44 2018 +0100 vtc: array not needed for token expansion diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 2b98596a8..9d4a899d2 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -302,7 +302,7 @@ parse_string(const char *spec, const struct cmds *cmd, void *priv, struct vtclog *vl) { char *token_s[MAX_TOKENS], *token_e[MAX_TOKENS]; - struct vsb *token_exp[MAX_TOKENS]; + struct vsb *token_exp; char *e, *p, *q, *f, *buf; int nest_brace; int tn; @@ -401,14 +401,13 @@ parse_string(const char *spec, const struct cmds *cmd, void *priv, assert(tn < MAX_TOKENS); token_s[tn] = NULL; for (tn = 0; token_s[tn] != NULL; tn++) { - token_exp[tn] = NULL; AN(token_e[tn]); /*lint !e771 */ *token_e[tn] = '\0'; /*lint !e771 */ if (NULL != strstr(token_s[tn], "${")) { - token_exp[tn] = macro_expand(vl, token_s[tn]); + token_exp = macro_expand(vl, token_s[tn]); if (vtc_error) return; - token_s[tn] = VSB_data(token_exp[tn]); + token_s[tn] = VSB_data(token_exp); token_e[tn] = strchr(token_s[tn], '\0'); } } From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:13 +0000 (UTC) Subject: [6.0] c1a72221c vtc: zero the token arrays for each command Message-ID: <20190206084013.144209BDE1@lists.varnish-cache.org> commit c1a72221cbc5d3e0a32385f8fa19dede6cc53fa4 Author: Nils Goroll Date: Sat Nov 17 12:17:18 2018 +0100 vtc: zero the token arrays for each command arguments could leak from one vtc command to the next where NULL arguments do not necessarily denote the end of arguments, eg. in varnish_expect diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 9d4a899d2..c7350df0a 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -339,6 +339,8 @@ parse_string(const char *spec, const struct cmds *cmd, void *priv, vtc_log(vl, 2, "=== %.*s", (int)(q - p), p); /* First content on line, collect tokens */ + memset(token_s, 0, sizeof token_s); + memset(token_e, 0, sizeof token_e); tn = 0; f = p; while (p < e) { From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:13 +0000 (UTC) Subject: [6.0] 72f249e42 fix vcl temperature after failing warmup event Message-ID: <20190206084013.2FC249BDE6@lists.varnish-cache.org> commit 72f249e427a5e02e3750a95988ed8276211e31ae Author: Nils Goroll Date: Sat Nov 17 15:06:41 2018 +0100 fix vcl temperature after failing warmup event Fixes #2835 diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index ae846e0a6..1d3583612 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -504,10 +504,12 @@ vcl_set_state(VRT_CTX, const char *state) else { vcl->temp = VCL_TEMP_WARM; i = vcl_send_event(ctx, VCL_EVENT_WARM); - if (i == 0) + if (i == 0) { vcl_BackendEvent(vcl, VCL_EVENT_WARM); - else - AZ(vcl->conf->event_vcl(ctx, VCL_EVENT_COLD)); + break; + } + AZ(vcl->conf->event_vcl(ctx, VCL_EVENT_COLD)); + vcl->temp = VCL_TEMP_COLD; } break; default: diff --git a/bin/varnishtest/tests/v00003.vtc b/bin/varnishtest/tests/v00003.vtc index ff218f28c..6236f28f1 100644 --- a/bin/varnishtest/tests/v00003.vtc +++ b/bin/varnishtest/tests/v00003.vtc @@ -88,6 +88,8 @@ varnish v1 -expect !VBE.vcl1.default.happy varnish v1 -cliok "param.set max_esi_depth 42" varnish v1 -clierr 300 "vcl.state vcl1 warm" +varnish v1 -cliexpect "available *cold/cold *[0-9]+ *vcl1\\s+active *warm/warm *[0-9]+ *vcl2" "vcl.list" + # A warm-up failure can also fail a child start varnish v1 -cliok stop varnish v1 -cliok "vcl.state vcl1 warm" From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:13 +0000 (UTC) Subject: [6.0] 0f8e21002 centralize vcl state symbols / strings in the manager Message-ID: <20190206084013.4CD269BE07@lists.varnish-cache.org> commit 0f8e21002c086a485dc978e7e8240940ab835388 Author: Nils Goroll Date: Wed Nov 14 18:37:27 2018 +0100 centralize vcl state symbols / strings in the manager The actual benefit of this will be in follow-up commits Conflicts: bin/varnishd/mgt/mgt_vcl.c diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 8c05c5215..0cb29e44c 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -48,9 +48,10 @@ #include "vev.h" #include "vtim.h" -static const char * const VCL_STATE_COLD = "cold"; -static const char * const VCL_STATE_WARM = "warm"; -static const char * const VCL_STATE_AUTO = "auto"; +#define VCL_STATE(sym, str) \ + static const char * const VCL_STATE_ ## sym = str; +#include "tbl/vcl_states.h" + static const char * const VCL_STATE_LABEL = "label"; struct vclprog; @@ -106,6 +107,18 @@ static int mgt_vcl_setstate(struct cli *, struct vclprog *, const char *); /*--------------------------------------------------------------------*/ +static const char * +mcf_vcl_parse_state(const char *s) +{ + if (s == NULL || *s == '\0') + return (NULL); +#define VCL_STATE(sym, str) \ + if (!strcmp(s, VCL_STATE_ ## sym)) \ + return (VCL_STATE_ ## sym); +#include "tbl/vcl_states.h" + return (NULL); +} + static struct vclprog * mcf_vcl_byname(const char *name) { @@ -171,7 +184,7 @@ mcf_find_no_vcl(struct cli *cli, const char *name) static int mcf_is_label(const struct vclprog *vp) { - return (!strcmp(vp->state, VCL_STATE_LABEL)); + return (vp->state == VCL_STATE_LABEL); } /*--------------------------------------------------------------------*/ @@ -469,13 +482,10 @@ mgt_new_vcl(struct cli *cli, const char *vclname, const char *vclsrc, if (state == NULL) state = VCL_STATE_AUTO; - else if (!strcmp(state, VCL_STATE_AUTO)) - state = VCL_STATE_AUTO; - else if (!strcmp(state, VCL_STATE_COLD)) - state = VCL_STATE_COLD; - else if (!strcmp(state, VCL_STATE_WARM)) - state = VCL_STATE_WARM; - else { + else + state = mcf_vcl_parse_state(state); + + if (state == NULL) { VCLI_Out(cli, "State must be one of auto, cold or warm."); VCLI_SetResult(cli, CLIS_PARAM); return; @@ -507,7 +517,7 @@ mgt_new_vcl(struct cli *cli, const char *vclname, const char *vclsrc, } free(p); - if (vp->warm && !strcmp(vp->state, "auto")) + if (vp->warm && vp->state == VCL_STATE_AUTO) vp->go_cold = VTIM_mono(); } @@ -626,6 +636,7 @@ mcf_vcl_load(struct cli *cli, const char * const *av, void *priv) static void v_matchproto_(cli_func_t) mcf_vcl_state(struct cli *cli, const char * const *av, void *priv) { + const char *state; struct vclprog *vp; (void)priv; @@ -638,22 +649,30 @@ mcf_vcl_state(struct cli *cli, const char * const *av, void *priv) VCLI_SetResult(cli, CLIS_PARAM); return; } + + state = mcf_vcl_parse_state(av[3]); + if (state == NULL) { + VCLI_Out(cli, "State must be one of auto, cold or warm."); + VCLI_SetResult(cli, CLIS_PARAM); + return; + } + if (!VTAILQ_EMPTY(&vp->dto)) { - AN(strcmp(vp->state, "cold")); - if (!strcmp(av[3], "cold")) { + assert(vp->state != VCL_STATE_COLD); + if (state == VCL_STATE_COLD) { VCLI_Out(cli, "A labeled VCL cannot be set cold"); VCLI_SetResult(cli, CLIS_CANT); return; } } - if (!strcmp(vp->state, av[3])) + if (vp->state == state) return; - if (!strcmp(av[3], VCL_STATE_AUTO)) { + if (state == VCL_STATE_AUTO) { vp->state = VCL_STATE_AUTO; (void)mgt_vcl_setstate(cli, vp, VCL_STATE_AUTO); - } else if (!strcmp(av[3], VCL_STATE_COLD)) { + } else if (state == VCL_STATE_COLD) { if (vp == active_vcl) { VCLI_Out(cli, "Cannot set the active VCL cold."); VCLI_SetResult(cli, CLIS_CANT); @@ -661,7 +680,7 @@ mcf_vcl_state(struct cli *cli, const char * const *av, void *priv) } vp->state = VCL_STATE_AUTO; (void)mgt_vcl_setstate(cli, vp, VCL_STATE_COLD); - } else if (!strcmp(av[3], VCL_STATE_WARM)) { + } else if (state == VCL_STATE_WARM) { if (mgt_vcl_setstate(cli, vp, VCL_STATE_WARM) == 0) vp->state = VCL_STATE_WARM; } else { diff --git a/include/tbl/vcl_states.h b/include/tbl/vcl_states.h new file mode 100644 index 000000000..6fe74dad1 --- /dev/null +++ b/include/tbl/vcl_states.h @@ -0,0 +1,8 @@ +/*lint -save -e525 -e539 */ +VCL_STATE(COLD, "cold") +VCL_STATE(WARM, "warm") +VCL_STATE(AUTO, "auto") +#undef VCL_STATE +// LABEL is private to mgt_vcl.c + +/*lint -restore */ From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:13 +0000 (UTC) Subject: [6.0] 63ec3aec3 centralize updating the vcl->state in vcl_set_state Message-ID: <20190206084013.6D5919BE27@lists.varnish-cache.org> commit 63ec3aec31971c950a7a2d31dae83d258de4e156 Author: Nils Goroll Date: Sat Nov 17 14:13:51 2018 +0100 centralize updating the vcl->state in vcl_set_state Allow calling it for this purpose only (stop asserting if the temperature has not changed) diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 1d3583612..fd364dcc5 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -476,9 +476,9 @@ vcl_set_state(VRT_CTX, const char *state) switch (state[0]) { case '0': - assert(vcl->temp != VCL_TEMP_COLD); + if (vcl->temp == VCL_TEMP_COLD) + break; if (vcl->busy == 0 && VCL_WARM(vcl)) { - vcl->temp = VTAILQ_EMPTY(&vcl->ref_list) ? VCL_TEMP_COLD : VCL_TEMP_COOLING; AZ(vcl_send_event(ctx, VCL_EVENT_COLD)); @@ -492,7 +492,8 @@ vcl_set_state(VRT_CTX, const char *state) vcl->temp = VCL_TEMP_COOLING; break; case '1': - assert(vcl->temp != VCL_TEMP_WARM); + if (vcl->temp == VCL_TEMP_WARM) + break; /* The warm VCL hasn't seen a cold event yet */ if (vcl->temp == VCL_TEMP_BUSY) vcl->temp = VCL_TEMP_WARM; @@ -515,6 +516,8 @@ vcl_set_state(VRT_CTX, const char *state) default: WRONG("Wrong enum state"); } + if (i == 0 && state[1]) + bprintf(vcl->state, "%s", state + 1); AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl)); return (i); @@ -583,7 +586,6 @@ vcl_load(struct cli *cli, struct vrt_ctx *ctx, vcl_cancel_load(ctx, cli, name, "warmup"); return; } - bprintf(vcl->state, "%s", state + 1); VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name); VTAILQ_INSERT_TAIL(&vcl_head, vcl, list); Lck_Lock(&vcl_mtx); @@ -736,9 +738,7 @@ vcl_cli_state(struct cli *cli, const char * const *av, void *priv) ctx = vcl_get_ctx(0, 1); ctx->vcl = vcl_find(av[2]); AN(ctx->vcl); // MGT ensures this - if (vcl_set_state(ctx, av[3]) == 0) { - bprintf(ctx->vcl->state, "%s", av[3] + 1); - } else { + if (vcl_set_state(ctx, av[3])) { AZ(VSB_finish(ctx->msg)); VCLI_SetResult(cli, CLIS_CANT); VCLI_Out(cli, "Failed ", ctx->vcl->loaded_name, From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:13 +0000 (UTC) Subject: [6.0] c8faf7ec8 forgotten file in dist Message-ID: <20190206084013.8AD399BE31@lists.varnish-cache.org> commit c8faf7ec8e451b90fef3cd0e8ac77ef6f4cab7e2 Author: Nils Goroll Date: Sat Nov 17 19:41:44 2018 +0100 forgotten file in dist diff --git a/include/Makefile.am b/include/Makefile.am index 01755e09d..8ac8c2086 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -32,6 +32,7 @@ nobase_pkginclude_HEADERS = \ tbl/symbol_kind.h \ tbl/vcc_types.h \ tbl/vcl_returns.h \ + tbl/vcl_states.h \ tbl/vhd_fsm.h \ tbl/vhd_fsm_funcs.h \ tbl/vhd_return.h \ From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:13 +0000 (UTC) Subject: [6.0] 63ecfe6ac Add error handling for STV_NewObject() in vrb_pull Message-ID: <20190206084013.ACFE89BE42@lists.varnish-cache.org> commit 63ecfe6ac4cc4554a2454dfa90a409377d8ec009 Author: Dag Haavi Finstad Date: Wed Nov 14 13:39:46 2018 +0100 Add error handling for STV_NewObject() in vrb_pull Fixes: #2831 diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index d170bbce6..40f698be3 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -73,7 +73,14 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) req->storage = NULL; - XXXAN(STV_NewObject(req->wrk, req->body_oc, stv, 8)); + if (STV_NewObject(req->wrk, req->body_oc, stv, 8) == 0) { + req->req_body_status = REQ_BODY_FAIL; + HSH_DerefBoc(req->wrk, req->body_oc); + AZ(HSH_DerefObjCore(req->wrk, &req->body_oc, 0)); + (void)VFP_Error(vfc, "Object allocation failed:" + " Ran out of space in %s", stv->vclname); + return (-1); + } vfc->oc = req->body_oc; diff --git a/bin/varnishtest/tests/r02831.vtc b/bin/varnishtest/tests/r02831.vtc new file mode 100644 index 000000000..0a7e85239 --- /dev/null +++ b/bin/varnishtest/tests/r02831.vtc @@ -0,0 +1,57 @@ +varnishtest "#2831: Out of storage in cache_req_body" + +server s1 { + rxreq + expect req.url == "/obj1" + txresp -bodylen 1048400 +} -start + +varnish v1 \ + -arg "-p nuke_limit=0" \ + -arg "-sTransient=default,1m" \ + -syntax 4.0 \ + -vcl+backend { + import std; + sub vcl_recv { + if (req.method == "POST") { + std.cache_req_body(1KB); + } + } + sub vcl_backend_response { + set beresp.do_stream = false; + set beresp.storage = storage.Transient; + # Unset Date header to not change the object sizes + unset beresp.http.Date; + } +} -start + +varnish v1 -cliok "param.set debug +syncvsl" + +delay .1 + +client c1 { + # Fill transient + txreq -url "/obj1" + rxresp + expect resp.status == 200 +} -run + +delay .1 + +varnish v1 -expect SM?.Transient.g_bytes > 1048400 +varnish v1 -expect SM?.Transient.g_space < 100 + +client c1 { + # No space for caching this req.body + txreq -req "POST" -body "foobar" + delay 1 +} -run + +varnish v1 -expect SMA.Transient.c_fail == 1 + +client c1 { + # Check that Varnish is still alive + txreq -url "/obj1" + rxresp + expect resp.status == 200 +} -run From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:13 +0000 (UTC) Subject: [6.0] cab89fbd9 Fix test case: 'SMA'->'SM?' Message-ID: <20190206084013.CD15C9BE62@lists.varnish-cache.org> commit cab89fbd9878060647263de1c77d3b8d06892803 Author: Dag Haavi Finstad Date: Mon Nov 19 13:14:27 2018 +0100 Fix test case: 'SMA'->'SM?' diff --git a/bin/varnishtest/tests/r02831.vtc b/bin/varnishtest/tests/r02831.vtc index 0a7e85239..6f5e87ac5 100644 --- a/bin/varnishtest/tests/r02831.vtc +++ b/bin/varnishtest/tests/r02831.vtc @@ -47,7 +47,7 @@ client c1 { delay 1 } -run -varnish v1 -expect SMA.Transient.c_fail == 1 +varnish v1 -expect SM?.Transient.c_fail == 1 client c1 { # Check that Varnish is still alive From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:13 +0000 (UTC) Subject: [6.0] ff86df680 minor vtim-ification Message-ID: <20190206084013.EB7019BE71@lists.varnish-cache.org> commit ff86df6808dbca2c449fdcdf88c5d1b838814e99 Author: Nils Goroll Date: Mon Nov 19 14:28:14 2018 +0100 minor vtim-ification diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index 6caf931a1..b619199c6 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -110,11 +110,12 @@ VNUM(const char *p) /**********************************************************************/ -double +vtim_dur VNUM_duration(const char *p) { const char *t; - double r, sc = 1.0; + vtim_dur r; + double sc = 1.0; if (p == NULL) return (nan("")); From dridi.boukelmoune at gmail.com Wed Feb 6 08:40:14 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 08:40:14 +0000 (UTC) Subject: [6.0] 033dcd798 Clarify reasoning with comment Message-ID: <20190206084014.12E879BE7C@lists.varnish-cache.org> commit 033dcd798045683c97716e289547155caa9e45f7 Author: Poul-Henning Kamp Date: Mon Nov 19 11:44:12 2018 +0000 Clarify reasoning with comment diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 864a4eb26..0ca6714c1 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -367,7 +367,7 @@ cnt_transmit(struct worker *wrk, struct req *req) if (head || status < 200 || status == 204 || status == 304) { // rfc7230,l,1748,1752 - sendbody = 0; + sendbody = 0; } else { sendbody = 1; } @@ -402,7 +402,14 @@ cnt_transmit(struct worker *wrk, struct req *req) } else if (clval >= 0 && clval == req->resp_len) { /* Reuse C-L header */ } else if (head && req->objcore->flags & OC_F_PASS) { - /* Don't touch C-L header */ + /* + * Don't touch C-L header (debatable) + * + * The only way to do it correctly would be to GET + * to the backend, and discard the body once the + * filters have had a chance to chew on it, but that + * would negate the "pass for huge objects" use case. + */ } else { http_Unset(req->resp, H_Content_Length); if (req->resp_len >= 0) From phk at FreeBSD.org Wed Feb 6 08:52:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Feb 2019 08:52:08 +0000 (UTC) Subject: [master] 32d2c8657 Park issue #2332 in a XXX comment, pending API renovation. Message-ID: <20190206085208.4D0E3A04A1@lists.varnish-cache.org> commit 32d2c86570cbf3bcb8a14c8c12b88d853e54e8cb Author: Poul-Henning Kamp Date: Wed Feb 6 08:51:21 2019 +0000 Park issue #2332 in a XXX comment, pending API renovation. diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index c5ccee3b0..f1b0ac3cb 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -111,6 +111,15 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, i = VRB_Iterate(bo->req, vbf_iter_req_body, bo); if (bo->req->req_body_status == REQ_BODY_FAIL) { + /* + * XXX: (#2332) We should test to see if the backend + * XXX: sent us some headers explaining why. + * XXX: This is hard because of the mistaken API split + * XXX: between cache_backend.c and V1F, and therefore + * XXX: Parked in this comment, pending renovation of + * XXX: the VDI/backend-protocol API to allow non-H1 + * XXX: backends. + */ assert(i < 0); VSLb(bo->vsl, SLT_FetchError, "req.body read error: %d (%s)", From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:09 +0000 (UTC) Subject: [6.0] 02052778b Split INIT/FINI into separate VDP methods Message-ID: <20190206101109.45FE6A214D@lists.varnish-cache.org> commit 02052778b8f87c2c4c2064f5b4b7f387491cee72 Author: Poul-Henning Kamp Date: Wed Oct 10 09:01:11 2018 +0000 Split INIT/FINI into separate VDP methods diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index e3cdcee5c..6d603ad77 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -66,7 +66,7 @@ VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len) assert(act > VDP_NULL || len > 0); /* Call the present layer, while pointing to the next layer down */ - retval = vdpe->vdp->func(req, act, &vdpe->priv, ptr, len); + retval = vdpe->vdp->bytes(req, act, &vdpe->priv, ptr, len); if (retval && (vdc->retval == 0 || retval < vdc->retval)) vdc->retval = retval; /* Latch error value */ vdc->nxt = vdpe; @@ -83,7 +83,7 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) vdc = req->vdc; AN(vdp); AN(vdp->name); - AN(vdp->func); + AN(vdp->bytes); if (vdc->retval) return (vdc->retval); @@ -107,7 +107,8 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) vdc->nxt = VTAILQ_FIRST(&vdc->vdp); AZ(vdc->retval); - vdc->retval = vdpe->vdp->func(req, VDP_INIT, &vdpe->priv, NULL, 0); + if (vdpe->vdp->init != NULL) + vdc->retval = vdpe->vdp->init(req, &vdpe->priv); return (vdc->retval); } @@ -126,8 +127,8 @@ VDP_close(struct req *req) if (vdpe != NULL) { CHECK_OBJ_NOTNULL(vdpe, VDP_ENTRY_MAGIC); VTAILQ_REMOVE(&vdc->vdp, vdpe, list); - AZ(vdpe->vdp->func(req, VDP_FINI, &vdpe->priv, - NULL, 0)); + if (vdpe->vdp->fini != NULL) + AZ(vdpe->vdp->fini(req, &vdpe->priv)); AZ(vdpe->priv); } vdc->nxt = VTAILQ_FIRST(&vdc->vdp); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 1ce7ea4ca..e73663896 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -251,8 +251,38 @@ ved_decode_len(struct req *req, const uint8_t **pp) /*--------------------------------------------------------------------- */ -static int v_matchproto_(vdp_bytes) -ved_vdp(struct req *req, enum vdp_action act, void **priv, +static int v_matchproto_(vdp_init_f) +ved_vdp_esi_init(struct req *req, void **priv) +{ + struct ecx *ecx; + + AZ(*priv); + ALLOC_OBJ(ecx, ECX_MAGIC); + AN(ecx); + assert(sizeof gzip_hdr == 10); + ecx->preq = req; + *priv = ecx; + RFC2616_Weaken_Etag(req->resp); + req->res_mode |= RES_ESI; + if (req->resp_len != 0) + req->resp_len = -1; + return (0); +} + +static int v_matchproto_(vdp_fini_f) +ved_vdp_esi_fini(struct req *req, void **priv) +{ + struct ecx *ecx; + + (void)req; + CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); + FREE_OBJ(ecx); + *priv = NULL; + return (0); +} + +static int v_matchproto_(vdp_bytes_f) +ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { uint8_t *q, *r; @@ -263,25 +293,7 @@ ved_vdp(struct req *req, enum vdp_action act, void **priv, struct ecx *ecx, *pecx = NULL; int retval = 0; - if (act == VDP_INIT) { - AZ(*priv); - ALLOC_OBJ(ecx, ECX_MAGIC); - AN(ecx); - assert(sizeof gzip_hdr == 10); - ecx->preq = req; - *priv = ecx; - RFC2616_Weaken_Etag(req->resp); - req->res_mode |= RES_ESI; - if (req->resp_len != 0) - req->resp_len = -1; - return (0); - } CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); - if (act == VDP_FINI) { - FREE_OBJ(ecx); - *priv = NULL; - return (0); - } pp = ptr; if (req->esi_level > 0) { @@ -435,7 +447,9 @@ ved_vdp(struct req *req, enum vdp_action act, void **priv, const struct vdp VDP_esi = { .name = "esi", - .func = ved_vdp, + .init = ved_vdp_esi_init, + .bytes = ved_vdp_esi_bytes, + .fini = ved_vdp_esi_fini, }; /* @@ -468,8 +482,16 @@ ved_bytes(struct req *req, struct req *preq, enum vdp_action act, * the stream with a bit more overhead. */ -static int v_matchproto_(vdp_bytes) -ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv, +static int v_matchproto_(vdp_fini_f) +ved_pretend_gzip_fini(struct req *req, void **priv) +{ + (void)req; + *priv = NULL; + return (0); +} + +static int v_matchproto_(vdp_bytes_f) +ved_pretend_gzip_bytes(struct req *req, enum vdp_action act, void **priv, const void *pv, ssize_t l) { uint8_t buf1[5], buf2[5]; @@ -483,12 +505,6 @@ ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv, preq = ecx->preq; (void)priv; - if (act == VDP_INIT) - return (0); - if (act == VDP_FINI) { - *priv = NULL; - return (0); - } if (l == 0) return (ved_bytes(req, ecx->preq, act, pv, l)); @@ -527,7 +543,8 @@ ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv, static const struct vdp ved_vdp_pgz = { .name = "PGZ", - .func = ved_pretend_gzip, + .bytes = ved_pretend_gzip_bytes, + .fini = ved_pretend_gzip_fini, }; /*--------------------------------------------------------------------- @@ -769,26 +786,29 @@ ved_stripgzip(struct req *req, const struct boc *boc) /*--------------------------------------------------------------------*/ -static int v_matchproto_(vdp_bytes) +static int v_matchproto_(vdp_fini_f) +ved_vdp_fini(struct req *req, void **priv) +{ + (void)req; + *priv = NULL; + return (0); +} + +static int v_matchproto_(vdp_bytes_f) ved_vdp_bytes(struct req *req, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { struct req *preq; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - if (act == VDP_INIT) - return (0); - if (act == VDP_FINI) { - *priv = NULL; - return (0); - } CAST_OBJ_NOTNULL(preq, *priv, REQ_MAGIC); return (ved_bytes(req, preq, act, ptr, len)); } static const struct vdp ved_ved = { .name = "VED", - .func = ved_vdp_bytes, + .bytes = ved_vdp_bytes, + .fini = ved_vdp_fini, }; /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 0bf90732d..2c31a3c7b 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -92,18 +92,20 @@ enum vfp_status VFP_Error(struct vfp_ctx *, const char *fmt, ...) /* Deliver processors ------------------------------------------------*/ enum vdp_action { - VDP_INIT, /* Happens on VDP_push() */ - VDP_FINI, /* Happens on VDP_pop() */ VDP_NULL, /* Input buffer valid after call */ VDP_FLUSH, /* Input buffer will be invalidated */ }; -typedef int vdp_bytes(struct req *, enum vdp_action, void **priv, +typedef int vdp_init_f(struct req *, void **priv); +typedef int vdp_fini_f(struct req *, void **priv); +typedef int vdp_bytes_f(struct req *, enum vdp_action, void **priv, const void *ptr, ssize_t len); struct vdp { const char *name; - vdp_bytes *func; + vdp_init_f *init; + vdp_bytes_f *bytes; + vdp_fini_f *fini; }; struct vdp_entry { diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index d5d256d56..bfe2f6279 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -284,62 +284,73 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags) * VDP for gunzip'ing */ -static int v_matchproto_(vdp_bytes) -vdp_gunzip(struct req *req, enum vdp_action act, void **priv, - const void *ptr, ssize_t len) +static int v_matchproto_(vdp_init_f) +vdp_gunzip_init(struct req *req, void **priv) { - enum vgzret_e vr; - ssize_t dl; - const void *dp; - struct worker *wrk; struct vgz *vg; const char *p; + ssize_t dl; uint64_t u; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - wrk = req->wrk; - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - - if (act == VDP_INIT) { - vg = VGZ_NewGunzip(req->vsl, "U D -"); - AN(vg); - if (vgz_getmbuf(vg)) { - (void)VGZ_Destroy(&vg); - return (-1); - } + vg = VGZ_NewGunzip(req->vsl, "U D -"); + AN(vg); + if (vgz_getmbuf(vg)) { + (void)VGZ_Destroy(&vg); + return (-1); + } - req->res_mode |= RES_GUNZIP; - VGZ_Obuf(vg, vg->m_buf, vg->m_sz); - *priv = vg; + req->res_mode |= RES_GUNZIP; + VGZ_Obuf(vg, vg->m_buf, vg->m_sz); + *priv = vg; - http_Unset(req->resp, H_Content_Encoding); + http_Unset(req->resp, H_Content_Encoding); - req->resp_len = -1; + req->resp_len = -1; - p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &dl); - if (p != NULL && dl == 32) { - u = vbe64dec(p + 24); - /* - * If the size is non-zero AND we are the top VDP - * (ie: no ESI), we know what size the output will be. - */ - if (u != 0 && - VTAILQ_FIRST(&req->vdc->vdp)->vdp == &VDP_gunzip) - req->resp_len = u; - } - return (0); + p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &dl); + if (p != NULL && dl == 32) { + u = vbe64dec(p + 24); + /* + * If the size is non-zero AND we are the top VDP + * (ie: no ESI), we know what size the output will be. + */ + if (u != 0 && + VTAILQ_FIRST(&req->vdc->vdp)->vdp == &VDP_gunzip) + req->resp_len = u; } + return (0); +} + +static int v_matchproto_(vdp_fini_f) +vdp_gunzip_fini(struct req *req, void **priv) +{ + struct vgz *vg; + (void)req; CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC); AN(vg->m_buf); + (void)VGZ_Destroy(&vg); + *priv = NULL; + return (0); +} - if (act == VDP_FINI) { - /* NB: Gunzip'ing may or may not have completed successfully. */ - AZ(len); - (void)VGZ_Destroy(&vg); - *priv = NULL; - return (0); - } +static int v_matchproto_(vdp_bytes_f) +vdp_gunzip_bytes(struct req *req, enum vdp_action act, void **priv, + const void *ptr, ssize_t len) +{ + enum vgzret_e vr; + ssize_t dl; + const void *dp; + struct worker *wrk; + struct vgz *vg; + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + wrk = req->wrk; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + (void)act; + + CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC); + AN(vg->m_buf); if (len == 0) return (0); @@ -363,7 +374,9 @@ vdp_gunzip(struct req *req, enum vdp_action act, void **priv, const struct vdp VDP_gunzip = { .name = "gunzip", - .func = vdp_gunzip, + .init = vdp_gunzip_init, + .bytes = vdp_gunzip_bytes, + .fini = vdp_gunzip_fini, }; /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index 373d1f676..c6cf1fd3c 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -44,7 +44,20 @@ struct vrg_priv { ssize_t range_off; }; -static int v_matchproto_(vdp_bytes) +static int v_matchproto_(vdp_fini_f) +vrg_range_fini(struct req *req, void **priv) +{ + struct vrg_priv *vrg_priv; + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC); + if (vrg_priv->range_off < vrg_priv->range_high) + Req_Fail(req, SC_RANGE_SHORT); + *priv = NULL; /* struct on ws, no need to free */ + return (0); +} + +static int v_matchproto_(vdp_bytes_f) vrg_range_bytes(struct req *req, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { @@ -54,15 +67,7 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv, struct vrg_priv *vrg_priv; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - if (act == VDP_INIT) - return (0); CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC); - if (act == VDP_FINI) { - if (vrg_priv->range_off < vrg_priv->range_high) - Req_Fail(req, SC_RANGE_SHORT); - *priv = NULL; /* struct on ws, no need to free */ - return (0); - } l = vrg_priv->range_low - vrg_priv->range_off; if (l > 0) { @@ -86,7 +91,8 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv, static const struct vdp vrg_vdp = { .name = "RNG", - .func = vrg_range_bytes, + .bytes = vrg_range_bytes, + .fini = vrg_range_fini, }; /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index 88447b5a6..03c5f3ba8 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -35,7 +35,7 @@ /*--------------------------------------------------------------------*/ -static int v_matchproto_(vdp_bytes) +static int v_matchproto_(vdp_bytes_f) v1d_bytes(struct req *req, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { @@ -43,8 +43,6 @@ v1d_bytes(struct req *req, enum vdp_action act, void **priv, CHECK_OBJ_NOTNULL(req, REQ_MAGIC); (void)priv; - if (act == VDP_INIT || act == VDP_FINI) - return (0); AZ(req->vdc->nxt); /* always at the bottom of the pile */ @@ -59,7 +57,7 @@ v1d_bytes(struct req *req, enum vdp_action act, void **priv, static const struct vdp v1d_vdp = { .name = "V1B", - .func = v1d_bytes, + .bytes = v1d_bytes, }; static void diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 2ec67a60b..60d35bc40 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -72,7 +72,21 @@ V2D_Init(void) /**********************************************************************/ -static int v_matchproto_(vdp_bytes) +static int v_matchproto_(vdp_fini_f) +h2_fini(struct req *req, void **priv) +{ + struct h2_req *r2; + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + (void)priv; + CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC); + H2_Send_Get(req->wrk, r2->h2sess, r2); + H2_Send(req->wrk, r2, H2_F_DATA, H2FF_DATA_END_STREAM, 0, ""); + H2_Send_Rel(r2->h2sess, r2); + return (0); +} + +static int v_matchproto_(vdp_bytes_f) h2_bytes(struct req *req, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { @@ -80,17 +94,13 @@ h2_bytes(struct req *req, enum vdp_action act, void **priv, CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC); + (void)act; (void)priv; - if (act == VDP_INIT) - return (0); - if ((r2->h2sess->error || r2->error) && act != VDP_FINI) + if ((r2->h2sess->error || r2->error)) return (-1); H2_Send_Get(req->wrk, r2->h2sess, r2); - H2_Send(req->wrk, r2, - H2_F_DATA, - act == VDP_FINI ? H2FF_DATA_END_STREAM : H2FF_NONE, - len, ptr); + H2_Send(req->wrk, r2, H2_F_DATA, H2FF_NONE, len, ptr); H2_Send_Rel(r2->h2sess, r2); req->acct.resp_bodybytes += len; return (0); @@ -98,7 +108,8 @@ h2_bytes(struct req *req, enum vdp_action act, void **priv, static const struct vdp h2_vdp = { .name = "H2B", - .func = h2_bytes, + .bytes = h2_bytes, + .fini = h2_fini, }; static inline size_t From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:09 +0000 (UTC) Subject: [6.0] 5882d268f Start registering VDPs Message-ID: <20190206101109.5AD56A2150@lists.varnish-cache.org> commit 5882d268fc481a7a697d6f219ce5feb299b25ab2 Author: Poul-Henning Kamp Date: Thu Oct 11 10:13:04 2018 +0000 Start registering VDPs Conflicts: bin/varnishd/cache/cache_vcl.c bin/varnishd/cache/cache_vcl.h bin/varnishd/cache/cache_vrt_filter.c It was possible to back-port this change without bringing beresp.filters and changing the VRT interface. So we get the benefits of this change and more that will follow and leave the possibility to minor-bump VRT to back-port beresp.filters in the future if that doesn't break. This clearly breaks $ABI strict VMODs, but that's part of the deal. diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 2c31a3c7b..30af823e4 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -128,3 +128,5 @@ struct vdp_ctx { int VDP_bytes(struct req *, enum vdp_action act, const void *ptr, ssize_t len); int VDP_push(struct req *, const struct vdp *, void *priv, int bottom); +void VRT_AddVDP(VRT_CTX, const struct vdp *); +void VRT_RemoveVDP(VRT_CTX, const struct vdp *); diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index fd364dcc5..d7a891802 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -406,6 +406,8 @@ VCL_Close(struct vcl **vclp) CHECK_OBJ_NOTNULL(*vclp, VCL_MAGIC); vcl = *vclp; *vclp = NULL; + assert(VTAILQ_EMPTY(&vcl->vfps)); + assert(VTAILQ_EMPTY(&vcl->vdps)); AZ(dlclose(vcl->dlh)); AZ(errno=pthread_rwlock_destroy(&vcl->temp_rwl)); FREE_OBJ(vcl); @@ -567,6 +569,8 @@ vcl_load(struct cli *cli, struct vrt_ctx *ctx, XXXAN(vcl->loaded_name); VTAILQ_INIT(&vcl->director_list); VTAILQ_INIT(&vcl->ref_list); + VTAILQ_INIT(&vcl->vfps); + VTAILQ_INIT(&vcl->vdps); vcl->temp = VCL_TEMP_INIT; diff --git a/bin/varnishd/cache/cache_vcl.h b/bin/varnishd/cache/cache_vcl.h index 87433afe3..c49f4817e 100644 --- a/bin/varnishd/cache/cache_vcl.h +++ b/bin/varnishd/cache/cache_vcl.h @@ -28,6 +28,11 @@ * */ +struct vfilter; + +VTAILQ_HEAD(vfilter_head, vfilter); + + struct vcl { unsigned magic; #define VCL_MAGIC 0x214188f2 @@ -45,6 +50,8 @@ struct vcl { int nrefs; struct vcl *label; int nlabels; + struct vfilter_head vfps; + struct vfilter_head vdps; }; struct vclref { From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:09 +0000 (UTC) Subject: [6.0] 44464a7fa Allow VDP->init to say "no thanks, but do go on delivering." Message-ID: <20190206101109.74E73A2153@lists.varnish-cache.org> commit 44464a7fa0ecbaa7495f772f94782394954e9215 Author: Poul-Henning Kamp Date: Mon Nov 19 12:21:26 2018 +0000 Allow VDP->init to say "no thanks, but do go on delivering." diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 6d603ad77..f943d3f53 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -78,6 +78,7 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) { struct vdp_entry *vdpe; struct vdp_ctx *vdc; + uintptr_t sn; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); vdc = req->vdc; @@ -91,6 +92,7 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) if (DO_DEBUG(DBG_PROCESSORS)) VSLb(req->vsl, SLT_Debug, "VDP_push(%s)", vdp->name); + sn = WS_Snapshot(req->ws); vdpe = WS_Alloc(req->ws, sizeof *vdpe); if (vdpe == NULL) { AZ(vdc->retval); @@ -109,6 +111,12 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) AZ(vdc->retval); if (vdpe->vdp->init != NULL) vdc->retval = vdpe->vdp->init(req, &vdpe->priv); + if (vdc->retval > 0) { + VTAILQ_REMOVE(&vdc->vdp, vdpe, list); + vdc->nxt = VTAILQ_FIRST(&vdc->vdp); + WS_Reset(req->ws, sn); + vdc->retval = 0; + } return (vdc->retval); } diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 30af823e4..6c2c04b70 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -97,6 +97,13 @@ enum vdp_action { }; typedef int vdp_init_f(struct req *, void **priv); +/* + * Return value: + * negative: Error - abandon delivery + * zero: OK + * positive: Don't push this VDP anyway + */ + typedef int vdp_fini_f(struct req *, void **priv); typedef int vdp_bytes_f(struct req *, enum vdp_action, void **priv, const void *ptr, ssize_t len); From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:09 +0000 (UTC) Subject: [6.0] 55ef1bcdf Remove VDP_push's "bottom" argument, we always build VDP stacks from the top down. Message-ID: <20190206101109.8EEE2A2158@lists.varnish-cache.org> commit 55ef1bcdf4fd776230145f30a25e0f39c600aa3d Author: Poul-Henning Kamp Date: Fri Oct 12 11:45:57 2018 +0000 Remove VDP_push's "bottom" argument, we always build VDP stacks from the top down. Conflicts: bin/varnishd/cache/cache_req_fsm.c The conflict was just the result of cherry-picking changes out of order, nothing to be worried about. diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index f943d3f53..99b2c8318 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -74,7 +74,7 @@ VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len) } int -VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) +VDP_Push(struct req *req, const struct vdp *vdp, void *priv) { struct vdp_entry *vdpe; struct vdp_ctx *vdc; @@ -102,10 +102,7 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) INIT_OBJ(vdpe, VDP_ENTRY_MAGIC); vdpe->vdp = vdp; vdpe->priv = priv; - if (bottom) - VTAILQ_INSERT_TAIL(&vdc->vdp, vdpe, list); - else - VTAILQ_INSERT_HEAD(&vdc->vdp, vdpe, list); + VTAILQ_INSERT_TAIL(&vdc->vdp, vdpe, list); vdc->nxt = VTAILQ_FIRST(&vdc->vdp); AZ(vdc->retval); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index e73663896..bed475be0 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -837,9 +837,9 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) ved_stripgzip(req, boc); } else { if (ecx->isgzip && !i) - (void)VDP_push(req, &ved_vdp_pgz, ecx, 1); + (void)VDP_Push(req, &ved_vdp_pgz, ecx); else - (void)VDP_push(req, &ved_ved, ecx->preq, 1); + (void)VDP_Push(req, &ved_ved, ecx->preq); (void)VDP_DeliverObj(req); (void)VDP_bytes(req, VDP_FLUSH, NULL, 0); } diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 6c2c04b70..778236994 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -134,6 +134,6 @@ struct vdp_ctx { }; int VDP_bytes(struct req *, enum vdp_action act, const void *ptr, ssize_t len); -int VDP_push(struct req *, const struct vdp *, void *priv, int bottom); +int VDP_Push(struct req *, const struct vdp *, void *priv); void VRT_AddVDP(VRT_CTX, const struct vdp *); void VRT_RemoveVDP(VRT_CTX, const struct vdp *); diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index c6cf1fd3c..8bcec96d4 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -181,7 +181,7 @@ vrg_dorange(struct req *req, const char *r) vrg_priv->range_off = 0; vrg_priv->range_low = low; vrg_priv->range_high = high + 1; - if (VDP_push(req, &vrg_vdp, vrg_priv, 1)) + if (VDP_Push(req, &vrg_vdp, vrg_priv)) return ("WS too small"); http_PutResponse(req->resp, "HTTP/1.1", 206, NULL); return (NULL); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 0ca6714c1..cd01c2976 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -374,13 +374,13 @@ cnt_transmit(struct worker *wrk, struct req *req) if (!req->disable_esi && req->resp_len != 0 && ObjHasAttr(wrk, req->objcore, OA_ESIDATA) && - VDP_push(req, &VDP_esi, NULL, 0) < 0) + VDP_Push(req, &VDP_esi, NULL) < 0) err++; if (cache_param->http_gzip_support && ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) && !RFC2616_Req_Gzip(req->http) && - VDP_push(req, &VDP_gunzip, NULL, 1) < 0) + VDP_Push(req, &VDP_gunzip, NULL) < 0) err++; if (cache_param->http_range_support && status == 200) { diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index 03c5f3ba8..12bcaa545 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -127,7 +127,7 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody) if (req->resp_len == 0) sendbody = 0; - if (sendbody && VDP_push(req, &v1d_vdp, NULL, 1)) { + if (sendbody && VDP_Push(req, &v1d_vdp, NULL)) { v1d_error(req, "workspace_thread overflow"); AZ(req->wrk->v1l); return; diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 60d35bc40..c40124e40 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -306,7 +306,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) /* XXX someone into H2 please add appropriate error handling */ if (sendbody) { - if (!VDP_push(req, &h2_vdp, NULL, 1)) + if (!VDP_Push(req, &h2_vdp, NULL)) (void)VDP_DeliverObj(req); } From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:09 +0000 (UTC) Subject: [6.0] 8f56db74c Rearrange range code to ponder life in VDP->init() Message-ID: <20190206101109.A9B58A215C@lists.varnish-cache.org> commit 8f56db74c280633f53ce7bd572fed9790fbe769a Author: Poul-Henning Kamp Date: Mon Nov 19 20:57:47 2018 +0000 Rearrange range code to ponder life in VDP->init() diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index 8bcec96d4..37e4d84e7 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -89,16 +89,10 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv, vrg_priv->range_off >= vrg_priv->range_high ? 1 : 0); } -static const struct vdp vrg_vdp = { - .name = "RNG", - .bytes = vrg_range_bytes, - .fini = vrg_range_fini, -}; - /*--------------------------------------------------------------------*/ static const char * -vrg_dorange(struct req *req, const char *r) +vrg_dorange(struct req *req, const char *r, void **priv) { ssize_t low, high, has_low, has_high, t; struct vrg_priv *vrg_priv; @@ -181,35 +175,47 @@ vrg_dorange(struct req *req, const char *r) vrg_priv->range_off = 0; vrg_priv->range_low = low; vrg_priv->range_high = high + 1; - if (VDP_Push(req, &vrg_vdp, vrg_priv)) - return ("WS too small"); + *priv = vrg_priv; http_PutResponse(req->resp, "HTTP/1.1", 206, NULL); return (NULL); } +static int v_matchproto_(vdp_init_f) +vrg_range_init(struct req *req, void **priv) +{ + const char *r; + const char *err; + + assert(http_GetHdr(req->http, H_Range, &r)); + err = vrg_dorange(req, r, priv); + if (err == NULL) + return (*priv == NULL ? 1 : 0); + + VSLb(req->vsl, SLT_Debug, "RANGE_FAIL %s", err); + if (req->resp_len >= 0) + http_PrintfHeader(req->resp, + "Content-Range: bytes */%jd", + (intmax_t)req->resp_len); + http_PutResponse(req->resp, "HTTP/1.1", 416, NULL); + /* + * XXX: We ought to produce a body explaining things. + * XXX: That really calls for us to hit vcl_synth{} + */ + req->resp_len = 0; + return (1); +} + +static const struct vdp vrg_vdp = { + .name = "range", + .init = vrg_range_init, + .bytes = vrg_range_bytes, + .fini = vrg_range_fini, +}; + void VRG_dorange(struct req *req, const char *r) { - const char *err; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - assert(http_IsStatus(req->resp, 200)); - - /* We must snapshot the length if we're streaming from the backend */ - - err = vrg_dorange(req, r); - if (err != NULL) { - VSLb(req->vsl, SLT_Debug, "RANGE_FAIL %s", err); - if (req->resp_len >= 0) - http_PrintfHeader(req->resp, - "Content-Range: bytes */%jd", - (intmax_t)req->resp_len); - http_PutResponse(req->resp, "HTTP/1.1", 416, NULL); - /* - * XXX: We ought to produce a body explaining things. - * XXX: That really calls for us to hit vcl_synth{} - */ - req->resp_len = 0; - } + (void)r; + AZ(VDP_Push(req, &vrg_vdp, NULL)); } From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:09 +0000 (UTC) Subject: [6.0] ce2c70916 Use vsl_catchup instead of delay .1 Message-ID: <20190206101109.C2BBFA2161@lists.varnish-cache.org> commit ce2c70916db98b7691140a404fa93a12977cb00b Author: Poul-Henning Kamp Date: Mon Nov 19 21:20:00 2018 +0000 Use vsl_catchup instead of delay .1 diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc index 1344fe2a6..3fbe5fefd 100644 --- a/bin/varnishtest/tests/c00034.vtc +++ b/bin/varnishtest/tests/c00034.vtc @@ -22,6 +22,8 @@ client c1 { varnish v1 -expect s_resp_bodybytes == 100 +varnish v1 -vsl_catchup + varnish v1 -cliok "param.set http_range_support on" client c1 { @@ -72,7 +74,8 @@ client c1 { } -run varnish v1 -expect s_resp_bodybytes == 100 -delay .1 + +varnish v1 -vsl_catchup client c1 { # Valid range requests @@ -122,6 +125,8 @@ client c1 { varnish v1 -expect s_resp_bodybytes == 501 +varnish v1 -vsl_catchup + # Test Range streaming with streaming objects with C-L server s1 { @@ -174,7 +179,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { # Closed C-L because we cannot use C-L From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:09 +0000 (UTC) Subject: [6.0] 41c1eeb2a Don't reset the workspace on pointless VDP_Push(), things might still have been happening on the workspace (See: VRG) Message-ID: <20190206101109.DDBABA2166@lists.varnish-cache.org> commit 41c1eeb2a06283edf44487a79b887c5b1abc16fb Author: Poul-Henning Kamp Date: Mon Nov 19 21:26:48 2018 +0000 Don't reset the workspace on pointless VDP_Push(), things might still have been happening on the workspace (See: VRG) diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 99b2c8318..fa3392f5f 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -78,7 +78,6 @@ VDP_Push(struct req *req, const struct vdp *vdp, void *priv) { struct vdp_entry *vdpe; struct vdp_ctx *vdc; - uintptr_t sn; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); vdc = req->vdc; @@ -92,7 +91,6 @@ VDP_Push(struct req *req, const struct vdp *vdp, void *priv) if (DO_DEBUG(DBG_PROCESSORS)) VSLb(req->vsl, SLT_Debug, "VDP_push(%s)", vdp->name); - sn = WS_Snapshot(req->ws); vdpe = WS_Alloc(req->ws, sizeof *vdpe); if (vdpe == NULL) { AZ(vdc->retval); @@ -111,7 +109,6 @@ VDP_Push(struct req *req, const struct vdp *vdp, void *priv) if (vdc->retval > 0) { VTAILQ_REMOVE(&vdc->vdp, vdpe, list); vdc->nxt = VTAILQ_FIRST(&vdc->vdp); - WS_Reset(req->ws, sn); vdc->retval = 0; } return (vdc->retval); From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:10 +0000 (UTC) Subject: [6.0] 0d563d407 gc stray comment Message-ID: <20190206101110.02678A216A@lists.varnish-cache.org> commit 0d563d4076b2a35baffb273b06d37d0c0252696c Author: Nils Goroll Date: Wed Nov 21 19:12:47 2018 +0100 gc stray comment Ref 3c0b8768afda0cc657fb902d1d390bdd0e8b7ce5 diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 5a04eb5aa..dbc12c685 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -413,7 +413,6 @@ VCP_Open(struct conn_pool *cp, double tmo, const void **privp, int *err) h = 0; - /* stats access unprotected */ switch (errno) { case EACCES: case EPERM: From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:10 +0000 (UTC) Subject: [6.0] 03a2102bf assert that -I file processing has finished and closed the fd Message-ID: <20190206101110.1B741A216F@lists.varnish-cache.org> commit 03a2102bf1237c11586fdf0ce82d5ba9d3c596e0 Author: Nils Goroll Date: Thu Nov 22 10:28:43 2018 +0100 assert that -I file processing has finished and closed the fd diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 237fb0f7f..2e0ee14f5 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -870,6 +870,7 @@ main(int argc, char * const *argv) "VEV_Once() = %d", o); } } + assert(I_fd == -1); if (!d_flag && !mgt_has_vcl() && !novcl) MGT_Complain(C_ERR, "No VCL loaded yet"); From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:10 +0000 (UTC) Subject: [6.0] d94c7f8db fix insignificant off-by-one in child file descriptor close code Message-ID: <20190206101110.33A77A2175@lists.varnish-cache.org> commit d94c7f8db1b6ae5692acc52ddeeebf8b4dd876f9 Author: Nils Goroll Date: Thu Nov 22 13:21:24 2018 +0100 fix insignificant off-by-one in child file descriptor close code The name CLOSE_FD_UP_TO implies that the value is included. Also this is the semantics of MCH_TrackHighFd() and relevant for #define CLOSE_FD_UP_TO mgt_max_fd diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 16050b33a..c5487d94e 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -344,7 +344,7 @@ mgt_launch_child(struct cli *cli) */ closelog(); - for (i = STDERR_FILENO + 1; i < CLOSE_FD_UP_TO; i++) { + for (i = STDERR_FILENO + 1; i <= CLOSE_FD_UP_TO; i++) { if (vbit_test(fd_map, i)) continue; if (close(i) == 0) From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:10 +0000 (UTC) Subject: [6.0] 8564c979f Gain more confidence that our file descriptor tracking works Message-ID: <20190206101110.4F68CA217A@lists.varnish-cache.org> commit 8564c979f2379e33ec94162f12046382dd544ccd Author: Nils Goroll Date: Thu Nov 22 13:14:19 2018 +0100 Gain more confidence that our file descriptor tracking works Side note #define CLOSE_FD_UP_TO mgt_max_fd exposes that we are currently missing to track some fds. diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index c5487d94e..3e1abc9f3 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -183,11 +183,15 @@ mch_cli_panic_clear(struct cli *cli, const char * const *av, void *priv) * This is likely to a bit on the low side, as libc and other libraries * has a tendency to cache file descriptors (syslog, resolver, etc.) * so we add a margin of 100 fds. + * + * for added safety, we check that we see no file descriptor open for + * another margin above the limit for which we close by design */ static int mgt_max_fd; #define CLOSE_FD_UP_TO (mgt_max_fd + 100) +#define CHECK_FD_UP_TO (CLOSE_FD_UP_TO + 100) void MCH_TrackHighFd(int fd) @@ -350,6 +354,10 @@ mgt_launch_child(struct cli *cli) if (close(i) == 0) VFIL_null_fd(i); } + for (i = CLOSE_FD_UP_TO + 1; i <= CHECK_FD_UP_TO; i++) { + assert(close(i) == -1); + assert(errno == EBADF); + } mgt_ProcTitle("Child"); From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:10 +0000 (UTC) Subject: [6.0] ca76ef579 fix the missing high fd tracking Message-ID: <20190206101110.68773A217E@lists.varnish-cache.org> commit ca76ef579f34394078ce281955b835958d3d8dfe Author: Nils Goroll Date: Thu Nov 22 13:44:50 2018 +0100 fix the missing high fd tracking diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 3e1abc9f3..8e2494ac8 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -191,6 +191,8 @@ mch_cli_panic_clear(struct cli *cli, const char * const *av, void *priv) static int mgt_max_fd; #define CLOSE_FD_UP_TO (mgt_max_fd + 100) +// XXX should work now - engage? +//#define CLOSE_FD_UP_TO mgt_max_fd #define CHECK_FD_UP_TO (CLOSE_FD_UP_TO + 100) void @@ -215,6 +217,7 @@ MCH_Fd_Inherit(int fd, const char *what) { assert(fd >= 0); + MCH_TrackHighFd(fd); if (fd_map == NULL) fd_map = vbit_new(128); AN(fd_map); From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:10 +0000 (UTC) Subject: [6.0] 1524d95d2 Fix regression from 6bead571da6a2027a518af81e14d87a247f38156 Message-ID: <20190206101110.828FBA218A@lists.varnish-cache.org> commit 1524d95d2f4f8409d7b6507fb93f46b3b4ba46c6 Author: Nils Goroll Date: Thu Nov 22 17:22:19 2018 +0100 Fix regression from 6bead571da6a2027a518af81e14d87a247f38156 Seen once, and failed to reprocuce (yet), needs to be understood better from vmod_blobdigest tests/usage.vtc: **** v1 1.8 CLI RX|No panic to clear *** v1 1.8 debug|Info: manager stopping child *** v1 1.8 debug|Debug: Stopping Child **** v1 1.9 vsl| 0 CLI - EOF on CLI connection, worker stops *** v1 2.8 debug|Info: Child (174227) ended *** v1 2.8 debug|Info: Child (174227) said Child dies *** v1 2.8 debug|Debug: Child cleanup complete *** v1 2.8 debug|Assert error in MCH_TrackHighFd(), mgt/mgt_child.c line 203: *** v1 2.8 debug| Condition(fd > 0) not true. **** v1 2.9 STDOUT poll 0x10 ** v1 2.9 WAIT4 pid=174215 status=0x0086 (user 0.914949 sys 0.567445) * v1 2.9 Expected exit: 0x0 signal: 0 core: 0 ---- v1 2.9 Bad exit status: 0x0086 exit 0x0 signal 6 core 128 * top 2.9 failure during reset On the 6.0 branch the regression appeared in ca76ef579 instead. diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 8e2494ac8..fdcdfd830 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -217,7 +217,9 @@ MCH_Fd_Inherit(int fd, const char *what) { assert(fd >= 0); - MCH_TrackHighFd(fd); + // XXX why? + if (fd > 0) + MCH_TrackHighFd(fd); if (fd_map == NULL) fd_map = vbit_new(128); AN(fd_map); From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:10 +0000 (UTC) Subject: [6.0] e6c5d636f More double to vtim_* conversions Message-ID: <20190206101110.9DD1DA219F@lists.varnish-cache.org> commit e6c5d636f4bc9a8bd901df0b06e85dacb8b2e44f Author: Dridi Boukelmoune Date: Sat Nov 24 01:07:09 2018 +0100 More double to vtim_* conversions This may break out of tree code not respecting include order of vdef.h first (or via cache/cache.h). It's trivial to fix and forces consumers to follow the tracks. Refs #2791 diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index d75e0b23f..ef6234374 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -35,9 +35,6 @@ #include #include -#include "vtim.h" -#include "vcs.h" - #include "cache_varnishd.h" #include "cache_transport.h" @@ -47,6 +44,8 @@ #include "storage/storage.h" #include "vcli_serve.h" +#include "vtim.h" +#include "vcs.h" /* * The panic string is constructed in memory, then copied to the diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index fede8e0ae..b6faa8222 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -255,8 +255,8 @@ enum htc_status_e HTC_RxStuff(struct http_conn *htc, htc_complete_f *func, double *t1, double *t2, double ti, double tn, int maxbytes) { - double tmo; - double now; + vtim_dur tmo; + vtim_real now; enum htc_status_e hs; ssize_t z; @@ -381,7 +381,7 @@ SES_New(struct pool *pp) */ static void v_matchproto_(waiter_handle_f) -ses_handle(struct waited *wp, enum wait_event ev, double now) +ses_handle(struct waited *wp, enum wait_event ev, vtim_real now) { struct sess *sp; struct pool *pp; @@ -518,7 +518,7 @@ SES_Close(struct sess *sp, enum sess_close reason) */ void -SES_Delete(struct sess *sp, enum sess_close reason, double now) +SES_Delete(struct sess *sp, enum sess_close reason, vtim_real now) { CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 011ebafe7..365b821fc 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -75,8 +75,8 @@ struct http_conn { void *priv; /* Timeouts */ - double first_byte_timeout; - double between_bytes_timeout; + vtim_dur first_byte_timeout; + vtim_dur between_bytes_timeout; }; typedef enum htc_status_e htc_complete_f(struct http_conn *); diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 157d0e67c..22f5eb2fe 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -106,7 +106,7 @@ struct haproxy_cli { size_t rxbuf_sz; char *rxbuf; - double timeout; + vtim_dur timeout; }; /********************************************************************** @@ -114,7 +114,7 @@ struct haproxy_cli { */ static int -haproxy_cli_tcp_connect(struct vtclog *vl, const char *addr, double tmo, +haproxy_cli_tcp_connect(struct vtclog *vl, const char *addr, vtim_dur tmo, const char **errp) { int fd; diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 4735472e2..43e300acf 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -116,11 +116,12 @@ #include "vapi/vsm.h" #include "vapi/vsl.h" -#include "vtim.h" -#include "vre.h" #include "vtc.h" +#include "vtim.h" +#include "vre.h" + #define LE_ANY (-1) #define LE_LAST (-2) diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c index f91014c9a..666b82d3c 100644 --- a/bin/varnishtest/vtc_syslog.c +++ b/bin/varnishtest/vtc_syslog.c @@ -37,13 +37,13 @@ #include #include +#include "vtc.h" + #include "vsa.h" #include "vss.h" #include "vtcp.h" #include "vre.h" -#include "vtc.h" - struct syslog_srv { unsigned magic; #define SYSLOG_SRV_MAGIC 0xbf28a692 @@ -63,7 +63,7 @@ struct syslog_srv { ssize_t rxbuf_left; size_t rxbuf_sz; char *rxbuf; - double timeout; + vtim_dur timeout; }; static pthread_mutex_t syslog_mtx; diff --git a/include/vrt.h b/include/vrt.h index 3db7c0438..53c4b773d 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -142,7 +142,7 @@ typedef const struct vmod_priv * VCL_BLOB; typedef const char * VCL_BODY; typedef unsigned VCL_BOOL; typedef int64_t VCL_BYTES; -typedef double VCL_DURATION; +typedef vtim_dur VCL_DURATION; typedef const char * VCL_ENUM; typedef const struct gethdr_s * VCL_HEADER; typedef struct http * VCL_HTTP; @@ -154,7 +154,7 @@ typedef double VCL_REAL; typedef const struct stevedore * VCL_STEVEDORE; typedef const struct strands * VCL_STRANDS; typedef const char * VCL_STRING; -typedef double VCL_TIME; +typedef vtim_real VCL_TIME; typedef struct vcl * VCL_VCL; typedef void VCL_VOID; @@ -197,7 +197,7 @@ struct vrt_ctx { struct http *http_bereq; struct http *http_beresp; - double now; + vtim_real now; /* * method specific argument: @@ -260,9 +260,9 @@ extern const void * const vrt_magic_string_unset; rigid char *port; \ rigid char *path; \ rigid char *hosthdr; \ - double connect_timeout; \ - double first_byte_timeout; \ - double between_bytes_timeout; \ + vtim_dur connect_timeout; \ + vtim_dur first_byte_timeout; \ + vtim_dur between_bytes_timeout; \ unsigned max_connections; \ unsigned proxy_header; @@ -291,8 +291,8 @@ struct vrt_backend { }; #define VRT_BACKEND_PROBE_FIELDS(rigid) \ - double timeout; \ - double interval; \ + vtim_dur timeout; \ + vtim_dur interval; \ unsigned exp_status; \ unsigned window; \ unsigned threshold; \ diff --git a/include/vtcp.h b/include/vtcp.h index 293759392..29f83cb75 100644 --- a/include/vtcp.h +++ b/include/vtcp.h @@ -55,13 +55,13 @@ void VTCP_name(const struct suckaddr *addr, char *abuf, unsigned alen, char *pbuf, unsigned plen); int VTCP_connected(int s); int VTCP_connect(const struct suckaddr *name, int msec); -int VTCP_open(const char *addr, const char *def_port, double timeout, +int VTCP_open(const char *addr, const char *def_port, vtim_dur timeout, const char **err); void VTCP_close(int *s); int VTCP_bind(const struct suckaddr *addr, const char **errp); int VTCP_listen(const struct suckaddr *addr, int depth, const char **errp); int VTCP_listen_on(const char *addr, const char *def_port, int depth, const char **errp); -void VTCP_set_read_timeout(int s, double seconds); -int VTCP_read(int fd, void *ptr, size_t len, double tmo); +void VTCP_set_read_timeout(int s, vtim_dur seconds); +int VTCP_read(int fd, void *ptr, size_t len, vtim_dur tmo); // #endif diff --git a/include/vtim.h b/include/vtim.h index 9c2e6ccfc..1968bc568 100644 --- a/include/vtim.h +++ b/include/vtim.h @@ -33,8 +33,8 @@ extern unsigned VTIM_postel; #define VTIM_FORMAT_SIZE 30 void VTIM_format(double t, char *p); double VTIM_parse(const char *p); -double VTIM_mono(void); -double VTIM_real(void); -void VTIM_sleep(double t); -struct timespec VTIM_timespec(double t); -struct timeval VTIM_timeval(double t); +vtim_mono VTIM_mono(void); +vtim_real VTIM_real(void); +void VTIM_sleep(vtim_dur t); +struct timespec VTIM_timespec(vtim_dur t); +struct timeval VTIM_timeval(vtim_dur t); diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c index 0254a9cf7..771abf88b 100644 --- a/lib/libvarnish/vev.c +++ b/lib/libvarnish/vev.c @@ -354,7 +354,7 @@ VEV_Loop(struct vev_root *evb) /*--------------------------------------------------------------------*/ static int -vev_sched_timeout(struct vev_root *evb, struct vev *e, double t) +vev_sched_timeout(struct vev_root *evb, struct vev *e, vtim_mono t) { int i; diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 93c2fce4a..00f7f63af 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -52,6 +52,7 @@ #include "vsa.h" #include "vss.h" #include "vtcp.h" +#include "vtim.h" /*--------------------------------------------------------------------*/ static void @@ -346,12 +347,10 @@ VTCP_close(int *s) } void -VTCP_set_read_timeout(int s, double seconds) +VTCP_set_read_timeout(int s, vtim_dur seconds) { #ifdef SO_RCVTIMEO_WORKS - struct timeval timeout; - timeout.tv_sec = (int)floor(seconds); - timeout.tv_usec = (int)(1e6 * (seconds - timeout.tv_sec)); + struct timeval timeout = VTIM_timeval(seconds); /* * Solaris bug (present at least in snv_151 and older): If this fails * with EINVAL, the socket is half-closed (SS_CANTSENDMORE) and the @@ -372,13 +371,14 @@ VTCP_set_read_timeout(int s, double seconds) static int v_matchproto_(vss_resolved_f) vtcp_open_callback(void *priv, const struct suckaddr *sa) { + /* XXX: vtim_dur? */ double *p = priv; return (VTCP_connect(sa, (int)floor(*p * 1e3))); } int -VTCP_open(const char *addr, const char *def_port, double timeout, +VTCP_open(const char *addr, const char *def_port, vtim_dur timeout, const char **errp) { int error; @@ -593,7 +593,7 @@ VTCP_Check(int a) */ int -VTCP_read(int fd, void *ptr, size_t len, double tmo) +VTCP_read(int fd, void *ptr, size_t len, vtim_dur tmo) { struct pollfd pfd[1]; int i, j; From dridi.boukelmoune at gmail.com Wed Feb 6 10:11:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 10:11:10 +0000 (UTC) Subject: [6.0] 5ccd4b219 Fix VRT_fail for 'if'/'elseif' conditional expressions Message-ID: <20190206101110.B7A8DA21B8@lists.varnish-cache.org> commit 5ccd4b21925116227e3e5006473108046b3f8712 Author: Dag Haavi Finstad Date: Thu Nov 22 15:43:14 2018 +0100 Fix VRT_fail for 'if'/'elseif' conditional expressions This adds a ctx->handling test at the beginning of any compound statement, to catch the cases where VRT_fail was invoked as part of an if test condition. Fixes: #2840 diff --git a/bin/varnishtest/tests/v00051.vtc b/bin/varnishtest/tests/v00051.vtc index 664af71f7..a12d23ade 100644 --- a/bin/varnishtest/tests/v00051.vtc +++ b/bin/varnishtest/tests/v00051.vtc @@ -425,3 +425,32 @@ varnish v1 -expect sc_vcl_failure == 10 logexpect l1 -wait +####################################################################### +# Fail in vmod call used in an if test + +varnish v1 -vcl+backend { + import debug; + sub vcl_recv { + if (debug.fail2()) { + return (hash); + } + } +} + +logexpect l1 -v v1 -g raw { + expect * 1033 VCL_call "RECV" + expect 0 1033 VCL_Error "Forced failure" + expect 0 1033 VCL_return "fail" +} -start + +client c1 { + txreq + rxresp + expect resp.status == 503 + expect resp.reason == "VCL failed" +} -run + +varnish v1 -expect vcl_fail == 14 +varnish v1 -expect sc_vcl_failure == 11 + +logexpect l1 -wait diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c index ee8830859..2b5075358 100644 --- a/lib/libvcc/vcc_parse.c +++ b/lib/libvcc/vcc_parse.c @@ -149,6 +149,7 @@ vcc_Compound(struct vcc *tl) Fb(tl, 1, "{\n"); tl->indent += INDENT; C(tl, ";"); + Fb(tl, 1, "if (*ctx->handling) return;\n"); while (1) { ERRCHK(tl); t = tl->t; diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 27f601ffa..6398d226d 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -130,6 +130,10 @@ $Function VOID fail() Function to fail vcl code. (See also: RFC748) +$Function BOOL fail2() + +Function to fail vcl code. Always returns true. + $Object dyn(STRING addr, STRING port, PROBE probe=0) Dynamically create a single-backend director, addr and port must not be empty. diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 61a18c9ed..10c9bde96 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -221,6 +221,14 @@ xyzzy_fail(VRT_CTX) VRT_fail(ctx, "Forced failure"); } +VCL_BOOL v_matchproto_(td_debug_fail2) +xyzzy_fail2(VRT_CTX) +{ + + VRT_fail(ctx, "Forced failure"); + return (1); +} + static void v_matchproto_(vmod_priv_free_f) priv_vcl_free(void *priv) { From phk at FreeBSD.org Wed Feb 6 13:48:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Feb 2019 13:48:08 +0000 (UTC) Subject: [master] 23c918ed8 Test that NULL-object handling also copes with 2297 Message-ID: <20190206134808.83F1FA7C0D@lists.varnish-cache.org> commit 23c918ed85a621aad78513d0c6923ed54649bbab Author: Poul-Henning Kamp Date: Wed Feb 6 13:46:22 2019 +0000 Test that NULL-object handling also copes with 2297 Fixes: #2297 diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 355bb3c56..45262850b 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -120,3 +120,12 @@ varnish v1 -errvcl {Expression has type STRING, expected REAL} { set resp.http.who = std.random("foo", "bar"); } } + +varnish v1 -errvcl {Failed initialization} { + import debug; + backend default { .host = "127.0.0.1"; } + sub vcl_init { + return (fail); + new xyz = debug.obj(); + } +} From dridi.boukelmoune at gmail.com Wed Feb 6 13:49:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 13:49:09 +0000 (UTC) Subject: [master] 2ce13cf28 Kill redundant redundant null checks Message-ID: <20190206134909.1DB4AA7DD8@lists.varnish-cache.org> commit 2ce13cf28b099bdc56889de15998c237ba25e267 Author: Dridi Boukelmoune Date: Tue Feb 21 13:58:17 2017 +0100 Kill redundant redundant null checks I caught one during a review and figured I might as well sweep the whole tree, so this patch was created using Coccinelle and the following steps: $ cat >check_obj.cocci <retval >= 0) AN(vdpe); if (vdpe != NULL) { - CHECK_OBJ_NOTNULL(vdpe, VDP_ENTRY_MAGIC); + CHECK_OBJ(vdpe, VDP_ENTRY_MAGIC); VTAILQ_REMOVE(&vdc->vdp, vdpe, list); if (vdpe->vdp->fini != NULL) AZ(vdpe->vdp->fini(req, &vdpe->priv)); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 7cfa27eeb..afd0d475a 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -503,7 +503,7 @@ VRT_u_bereq_body(VRT_CTX) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); if (ctx->bo->req != NULL) { - CHECK_OBJ_NOTNULL(ctx->bo->req, REQ_MAGIC); + CHECK_OBJ(ctx->bo->req, REQ_MAGIC); ctx->bo->req = NULL; ObjSetState(ctx->bo->wrk, ctx->bo->fetch_objcore, BOS_REQ_DONE); diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index c6eb3a927..c17c80e5d 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -436,7 +436,7 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo, CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); INIT_OBJ(&ctx, VRT_CTX_MAGIC); if (req != NULL) { - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_MAGIC); VCL_Req2Ctx(&ctx, req); diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c index e6459bc7d..7869f700f 100644 --- a/lib/libvarnish/vev.c +++ b/lib/libvarnish/vev.c @@ -413,7 +413,7 @@ VEV_Once(struct vev_root *evb) tmo = INFTIM; e = binheap_root(evb->binheap); if (e != NULL) { - CHECK_OBJ_NOTNULL(e, VEV_MAGIC); + CHECK_OBJ(e, VEV_MAGIC); assert(e->__binheap_idx == BINHEAP_NOIDX + 1); t = VTIM_mono(); if (e->__when <= t) diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c index a37faec57..e6db240c5 100644 --- a/lib/libvarnishapi/vxp_parse.c +++ b/lib/libvarnishapi/vxp_parse.c @@ -629,7 +629,7 @@ vex_print(const struct vex *vex, int indent) fprintf(stderr, "%*s%s", indent, "", vxp_tnames[vex->tok]); if (vex->lhs != NULL) { - CHECK_OBJ_NOTNULL(vex->lhs, VEX_LHS_MAGIC); + CHECK_OBJ(vex->lhs, VEX_LHS_MAGIC); AN(vex->lhs->tags); fprintf(stderr, " lhs="); if (vex->lhs->level >= 0) diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index e709e9cf0..5360f3aac 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -402,7 +402,7 @@ shard_get_key(VRT_CTX, const struct vmod_directors_shard_param *p) switch (p->by) { case BY_HASH: if (ctx->bo) { - CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + CHECK_OBJ(ctx->bo, BUSYOBJ_MAGIC); return (vbe32dec(ctx->bo->digest)); } /* FALLTHROUGH */ From phk at FreeBSD.org Wed Feb 6 14:04:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Feb 2019 14:04:09 +0000 (UTC) Subject: [master] 0cf47b5ac Reduce close-margin over highest known filedescriptor. Message-ID: <20190206140409.9805DA8395@lists.varnish-cache.org> commit 0cf47b5acbc42eba48b262fc3c4070c406970d30 Author: Poul-Henning Kamp Date: Wed Feb 6 13:57:45 2019 +0000 Reduce close-margin over highest known filedescriptor. Fixes: #2844 diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index e6fd7c514..92a6313ca 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -180,18 +180,16 @@ mch_cli_panic_clear(struct cli *cli, const char * const *av, void *priv) * * This is likely to a bit on the low side, as libc and other libraries * has a tendency to cache file descriptors (syslog, resolver, etc.) - * so we add a margin of 100 fds. + * so we add a margin of 10 fds. * - * for added safety, we check that we see no file descriptor open for + * For added safety, we check that we see no file descriptor open for * another margin above the limit for which we close by design */ static int mgt_max_fd; -#define CLOSE_FD_UP_TO (mgt_max_fd + 100) -// XXX should work now - engage? -//#define CLOSE_FD_UP_TO mgt_max_fd -#define CHECK_FD_UP_TO (CLOSE_FD_UP_TO + 100) +#define CLOSE_FD_UP_TO (mgt_max_fd + 10) +#define CHECK_FD_UP_TO (CLOSE_FD_UP_TO + 10) void MCH_TrackHighFd(int fd) From dridi.boukelmoune at gmail.com Wed Feb 6 17:25:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 17:25:10 +0000 (UTC) Subject: [master] d6b13ddef Null check Message-ID: <20190206172510.7DF73AE09B@lists.varnish-cache.org> commit d6b13ddef9557e3269f2cc4c014d47487447ef31 Author: Dridi Boukelmoune Date: Wed Feb 6 18:23:36 2019 +0100 Null check diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index c59fe857f..156e50e6f 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -67,6 +67,7 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv, struct vrg_priv *vrg_priv; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AN(priv); CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC); l = vrg_priv->range_low - vrg_priv->range_off; From dridi.boukelmoune at gmail.com Wed Feb 6 17:34:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 17:34:09 +0000 (UTC) Subject: [master] 8a7f5ab50 Null check Message-ID: <20190206173409.24BDDAE505@lists.varnish-cache.org> commit 8a7f5ab5070549681a3c32bd2ad7ee69c38d01f5 Author: Dridi Boukelmoune Date: Wed Feb 6 18:33:21 2019 +0100 Null check diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 1462832a1..22d0b0855 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -254,6 +254,7 @@ ved_vdp_esi_init(struct req *req, void **priv) { struct ecx *ecx; + AN(priv); AZ(*priv); ALLOC_OBJ(ecx, ECX_MAGIC); AN(ecx); From dridi.boukelmoune at gmail.com Wed Feb 6 17:37:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 17:37:08 +0000 (UTC) Subject: [master] e50a4a3cb Null check, hat trick Message-ID: <20190206173708.4128BAE71D@lists.varnish-cache.org> commit e50a4a3cbfa87c5b923c0b18c9ed3fbf88dc3553 Author: Dridi Boukelmoune Date: Wed Feb 6 18:35:43 2019 +0100 Null check, hat trick diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 22d0b0855..59a138bb2 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -282,6 +282,7 @@ ved_vdp_esi_fini(struct req *req, void **priv) struct ecx *ecx; (void)req; + AN(priv); CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); FREE_OBJ(ecx); *priv = NULL; From dridi.boukelmoune at gmail.com Wed Feb 6 17:41:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 6 Feb 2019 17:41:09 +0000 (UTC) Subject: [master] dfc31fda4 Null check Message-ID: <20190206174109.D9C03AEA6E@lists.varnish-cache.org> commit dfc31fda49ff0bcf49d6602d9c0040bc3377073b Author: Dridi Boukelmoune Date: Wed Feb 6 18:39:33 2019 +0100 Null check Apologies for the spam, I didn't think this would be a recurring pattern. diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 59a138bb2..eeb4d1e79 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -301,6 +301,7 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, struct ecx *ecx; int retval = 0; + AN(priv); CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); pp = ptr; From phk at FreeBSD.org Wed Feb 6 21:08:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Feb 2019 21:08:08 +0000 (UTC) Subject: [master] 260d0ecc5 Report what we did get, if we EOF while receiving header. Message-ID: <20190206210808.49EC6B2561@lists.varnish-cache.org> commit 260d0ecc5b4b9702f6d53cd2a6854c373384ea51 Author: Poul-Henning Kamp Date: Wed Feb 6 20:57:49 2019 +0000 Report what we did get, if we EOF while receiving header. diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index a8461530d..815dd8a11 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -633,7 +633,7 @@ http_swallow_body(struct http *hp, char * const *hh, int body) static void http_rxhdr(struct http *hp) { - int i; + int i, j; char *p; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -641,7 +641,7 @@ http_rxhdr(struct http *hp) hp->body = NULL; bprintf(hp->bodylen, "%s", ""); while (1) { - (void)http_rxchar(hp, 1, 0); + j = http_rxchar(hp, 1, 1); p = hp->rxbuf + hp->prxbuf - 1; for (i = 0; p > hp->rxbuf; p--) { if (*p != '\n') @@ -651,12 +651,14 @@ http_rxhdr(struct http *hp) if (++i == 2) break; } - if (i == 2) + if (i == 2 || !j) break; } vtc_dump(hp->vl, 4, "rxhdr", hp->rxbuf, -1); vtc_log(hp->vl, 4, "rxhdrlen = %zd", strlen(hp->rxbuf)); hp->body = hp->rxbuf + hp->prxbuf; + if (!j) + vtc_fatal(hp->vl, "EOF timeout=%d", hp->timeout); } /* SECTION: client-server.spec.rxresp From phk at FreeBSD.org Wed Feb 6 21:08:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Feb 2019 21:08:08 +0000 (UTC) Subject: [master] c21f58dfa Make this test more resistant to machine overload, but also make it fail faster when appropriate Message-ID: <20190206210808.5FEE7B2564@lists.varnish-cache.org> commit c21f58dfa634ae760e77f2f8f47f9b140ff8c1a0 Author: Poul-Henning Kamp Date: Wed Feb 6 21:02:03 2019 +0000 Make this test more resistant to machine overload, but also make it fail faster when appropriate diff --git a/bin/varnishtest/tests/c00094.vtc b/bin/varnishtest/tests/c00094.vtc index 546efb08f..ad7a06a4e 100644 --- a/bin/varnishtest/tests/c00094.vtc +++ b/bin/varnishtest/tests/c00094.vtc @@ -3,7 +3,11 @@ varnishtest "Test Backend Polling with a backend listening at a UDS" barrier b1 cond 2 server s1 -listen "${tmpdir}/s1.sock" { + timeout 3 + fatal + # Probes + loop 8 { rxreq expect req.url == "/" @@ -39,7 +43,7 @@ varnish v1 -vcl { .path = "${s1_sock}"; .probe = { .timeout = 1 s; - .interval = 0.1 s; + .interval = 0.5 s; } } @@ -47,5 +51,5 @@ varnish v1 -vcl { barrier b1 sync -varnish v1 -cliexpect "^CLI RX| -+U+ Good UNIX" "backend.list -p" +varnish v1 -cliexpect "^CLI RX| -+U+-{0,5} Good UNIX" "backend.list -p" varnish v1 -cliexpect "^CLI RX| -+H{10}-{5}H{2}-{0,5} Happy" "backend.list -p" From phk at FreeBSD.org Wed Feb 6 21:59:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Feb 2019 21:59:08 +0000 (UTC) Subject: [master] 72d1d4d77 Allow non-fatal (usually server) rxhdrs to fail silently, also when nothing has been received. Message-ID: <20190206215908.CAE9646F3@lists.varnish-cache.org> commit 72d1d4d77d77913fcb5151bfe2b8203b21d59c19 Author: Poul-Henning Kamp Date: Wed Feb 6 21:57:00 2019 +0000 Allow non-fatal (usually server) rxhdrs to fail silently, also when nothing has been received. diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 815dd8a11..0c136cfd1 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -420,6 +420,10 @@ http_splitheader(struct http *hp, int req) n = 0; p = hp->rxbuf; + if (*p == '\0') { + vtc_log(hp->vl, 4, "No headers"); + return; + } /* REQ/PROTO */ while (vct_islws(*p)) @@ -638,6 +642,7 @@ http_rxhdr(struct http *hp) CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); hp->prxbuf = 0; + hp->rxbuf[0] = '\0'; hp->body = NULL; bprintf(hp->bodylen, "%s", ""); while (1) { @@ -654,11 +659,11 @@ http_rxhdr(struct http *hp) if (i == 2 || !j) break; } - vtc_dump(hp->vl, 4, "rxhdr", hp->rxbuf, -1); - vtc_log(hp->vl, 4, "rxhdrlen = %zd", strlen(hp->rxbuf)); - hp->body = hp->rxbuf + hp->prxbuf; + vtc_dump(hp->vl, 4, "rxhdr", hp->rxbuf, hp->prxbuf); + vtc_log(hp->vl, 4, "rxhdrlen = %d", hp->prxbuf); if (!j) - vtc_fatal(hp->vl, "EOF timeout=%d", hp->timeout); + vtc_log(hp->vl, hp->fatal, "EOF timeout=%d", hp->timeout); + hp->body = hp->rxbuf + hp->prxbuf; } /* SECTION: client-server.spec.rxresp From phk at FreeBSD.org Wed Feb 6 21:59:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Feb 2019 21:59:08 +0000 (UTC) Subject: [master] 009297d52 Stabilize this test under heavy system loads Message-ID: <20190206215908.DFE8846F6@lists.varnish-cache.org> commit 009297d522be4199e70500ce370c29c87c4c8c7c Author: Poul-Henning Kamp Date: Wed Feb 6 21:58:23 2019 +0000 Stabilize this test under heavy system loads diff --git a/bin/varnishtest/tests/c00088.vtc b/bin/varnishtest/tests/c00088.vtc index b39171f9a..d95477eae 100644 --- a/bin/varnishtest/tests/c00088.vtc +++ b/bin/varnishtest/tests/c00088.vtc @@ -1,8 +1,13 @@ varnishtest "Dropping polling of a backend that listens at UDS" -server s1 -listen "${tmpdir}/s1.sock" -repeat 40 { - rxreq - txresp +server s1 -listen "${tmpdir}/s1.sock" { + non_fatal + timeout 3 + loop 40 { + rxreq + txresp + accept + } } -start varnish v1 -vcl { From dridi.boukelmoune at gmail.com Fri Feb 8 09:19:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 09:19:09 +0000 (UTC) Subject: [master] b588404ab Show VDPs if present during a panic Message-ID: <20190208091909.EC262620E@lists.varnish-cache.org> commit b588404ab8da52b2b9a2b7e8f77af091cdaba965 Author: Dridi Boukelmoune Date: Fri Feb 8 10:08:50 2019 +0100 Show VDPs if present during a panic And get a chance to catch vdp->fini() failures with the offending VDP still present in the list of filters. diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 44b0e4668..c956c93a0 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -128,10 +128,10 @@ VDP_close(struct req *req) AN(vdpe); if (vdpe != NULL) { CHECK_OBJ(vdpe, VDP_ENTRY_MAGIC); - VTAILQ_REMOVE(&vdc->vdp, vdpe, list); if (vdpe->vdp->fini != NULL) AZ(vdpe->vdp->fini(req, &vdpe->priv)); AZ(vdpe->priv); + VTAILQ_REMOVE(&vdc->vdp, vdpe, list); } vdc->nxt = VTAILQ_FIRST(&vdc->vdp); } diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index bf4f9dfaf..3a4fae5f4 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -360,6 +360,7 @@ pan_vfp(struct vsb *vsb, const struct vfp_ctx *vfc) VSB_printf(vsb, "vfc = %p {\n", vfc); VSB_indent(vsb, 2); + PAN_CheckMagic(vsb, vfc, VFP_CTX_MAGIC); VSB_printf(vsb, "failed = %d,\n", vfc->failed); VSB_printf(vsb, "req = %p,\n", vfc->req); VSB_printf(vsb, "resp = %p,\n", vfc->resp); @@ -370,8 +371,7 @@ pan_vfp(struct vsb *vsb, const struct vfp_ctx *vfc) VSB_printf(vsb, "filters = {\n"); VSB_indent(vsb, 2); VTAILQ_FOREACH(vfe, &vfc->vfp, list) { - VSB_printf(vsb, "%s = %p {\n", - vfe->vfp->name, vfe); + VSB_printf(vsb, "%s = %p {\n", vfe->vfp->name, vfe); VSB_indent(vsb, 2); VSB_printf(vsb, "priv1 = %p,\n", vfe->priv1); VSB_printf(vsb, "priv2 = %zd,\n", vfe->priv2); @@ -388,6 +388,30 @@ pan_vfp(struct vsb *vsb, const struct vfp_ctx *vfc) VSB_printf(vsb, "},\n"); } +static void +pan_vdp(struct vsb *vsb, const struct vdp_ctx *vdc) +{ + struct vdp_entry *vde; + + VSB_printf(vsb, "vdc = %p {\n", vdc); + VSB_indent(vsb, 2); + PAN_CheckMagic(vsb, vdc, VDP_CTX_MAGIC); + VSB_printf(vsb, "nxt = %p,\n", vdc->nxt); + VSB_printf(vsb, "retval = %d,\n", vdc->retval); + + if (!VTAILQ_EMPTY(&vdc->vdp)) { + VSB_printf(vsb, "filters = {\n"); + VSB_indent(vsb, 2); + VTAILQ_FOREACH(vde, &vdc->vdp, list) + VSB_printf(vsb, "%s = %p { priv = %p }\n", + vde->vdp->name, vde, vde->priv); + VSB_indent(vsb, -2); + VSB_printf(vsb, "},\n"); + } + + VSB_indent(vsb, -2); + VSB_printf(vsb, "},\n"); +} static void pan_busyobj(struct vsb *vsb, const struct busyobj *bo) @@ -511,6 +535,8 @@ pan_req(struct vsb *vsb, const struct req *req) pan_http(vsb, "req", req->http); if (req->resp->ws != NULL) pan_http(vsb, "resp", req->resp); + if (req->vdc != NULL) + pan_vdp(vsb, req->vdc); VCL_Panic(vsb, "vcl", req->vcl); VCL_Panic(vsb, "vcl0", req->vcl0); From dridi.boukelmoune at gmail.com Fri Feb 8 09:23:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 09:23:08 +0000 (UTC) Subject: [master] 832d6da2d Guard all the VDP_Push calls Message-ID: <20190208092308.C889E64F2@lists.varnish-cache.org> commit 832d6da2d732636b42a78920980a84b3e33bda9b Author: Dridi Boukelmoune Date: Fri Feb 8 10:21:41 2019 +0100 Guard all the VDP_Push calls Two of them have no init step, and the third one is fail-safe. diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index eeb4d1e79..9279baaa7 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -833,14 +833,14 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) INIT_OBJ(foo, VED_FOO_MAGIC); foo->ecx = ecx; - (void)VDP_Push(req, &ved_gzgz, foo); + AZ(VDP_Push(req, &ved_gzgz, foo)); } else if (ecx->isgzip && !i) { /* Non-Gzip'ed include in gzip'ed parent */ - (void)VDP_Push(req, &ved_pretend_gz, ecx); + AZ(VDP_Push(req, &ved_pretend_gz, ecx)); } else { /* Anything else goes straight through */ - (void)VDP_Push(req, &ved_ved, ecx); + AZ(VDP_Push(req, &ved_ved, ecx)); } (void)VDP_DeliverObj(req); (void)VDP_bytes(req, VDP_FLUSH, NULL, 0); From phk at FreeBSD.org Fri Feb 8 10:24:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 8 Feb 2019 10:24:08 +0000 (UTC) Subject: [master] bef8b6cf0 Consume less random bits, and don't pace after ramp-up Message-ID: <20190208102408.BE5CF7A2F@lists.varnish-cache.org> commit bef8b6cf0f1a8887484f1bbaba660478444d206e Author: Poul-Henning Kamp Date: Fri Feb 8 10:01:42 2019 +0000 Consume less random bits, and don't pace after ramp-up diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 4b7fe2a4a..ac6c98ec4 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -275,7 +275,6 @@ start_test(void) assert(jp->buf != MAP_FAILED); memset(jp->buf, 0, jp->bufsiz); - VRND_SeedAll(); bprintf(tmpdir, "%s/vtc.%d.%08x", tmppath, (int)getpid(), (unsigned)random()); AZ(mkdir(tmpdir, 0711)); @@ -541,6 +540,7 @@ main(int argc, char * const *argv) { int ch, i; int ntest = 1; /* Run tests this many times */ + int nstart = 0; uintmax_t bufsiz; const char *p; @@ -568,6 +568,7 @@ main(int argc, char * const *argv) if (p != NULL) vtc_maxdur = atoi(p); + VRND_SeedAll(); setbuf(stdout, NULL); setbuf(stderr, NULL); while ((ch = getopt(argc, argv, "b:D:hij:kLln:p:qt:vW")) != -1) { @@ -660,7 +661,7 @@ main(int argc, char * const *argv) start_test(); njob++; /* Stagger ramp-up */ - if (njob < npar) + if (nstart++ < npar) (void)usleep(random() % 100000L); i = 1; continue; From phk at FreeBSD.org Fri Feb 8 10:24:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 8 Feb 2019 10:24:08 +0000 (UTC) Subject: [master] 15c72cf4a Optimize varnishtests central scheduler Message-ID: <20190208102408.D88B77A32@lists.varnish-cache.org> commit 15c72cf4aac1174cddc5a204d2eb3f762178d961 Author: Poul-Henning Kamp Date: Fri Feb 8 10:22:57 2019 +0000 Optimize varnishtests central scheduler diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index ac6c98ec4..a20a9f6fb 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -53,6 +53,17 @@ static const char *argv0; +struct buf { + unsigned magic; +#define BUF_MAGIC 0x39d1258a + VTAILQ_ENTRY(buf) list; + char *buf; + struct vsb *diag; + size_t bufsiz; +}; + +static VTAILQ_HEAD(, buf) free_bufs = VTAILQ_HEAD_INITIALIZER(free_bufs); + struct vtc_tst { unsigned magic; #define TST_MAGIC 0x618d8b88 @@ -69,14 +80,13 @@ struct vtc_job { pid_t child; struct vev *ev; struct vev *evt; - char *buf; + struct buf *bp; char *tmpdir; - unsigned bufsiz; double t0; - struct vsb *diag; int killed; }; + int iflg = 0; unsigned vtc_maxdur = 60; static unsigned vtc_bufsiz = 1024 * 1024; @@ -96,6 +106,40 @@ char *vmod_path = NULL; struct vsb *params_vsb = NULL; int leave_temp; int vtc_witness = 0; +static struct vsb *cbvsb; + +static struct buf * +get_buf(void) +{ + struct buf *bp; + + bp = VTAILQ_FIRST(&free_bufs); + CHECK_OBJ_ORNULL(bp, BUF_MAGIC); + if (bp != NULL) { + VTAILQ_REMOVE(&free_bufs, bp, list); + VSB_clear(bp->diag); + } else { + ALLOC_OBJ(bp, BUF_MAGIC); + AN(bp); + bp->bufsiz = vtc_bufsiz; + bp->buf = mmap(NULL, bp->bufsiz, PROT_READ|PROT_WRITE, + MAP_ANON | MAP_SHARED, -1, 0); + assert(bp->buf != MAP_FAILED); + bp->diag = VSB_new_auto(); + AN(bp->diag); + } + memset(bp->buf, 0, bp->bufsiz); + return (bp); +} + +static void +rel_buf(struct buf **bp) +{ + CHECK_OBJ_NOTNULL(*bp, BUF_MAGIC); + + VTAILQ_INSERT_HEAD(&free_bufs, (*bp), list); + *bp = NULL; +} /********************************************************************** * Parse a -D option argument into a name/val pair, and insert @@ -158,7 +202,6 @@ tst_cb(const struct vev *ve, int what) double t; FILE *f; char *p; - struct vsb *v; CAST_OBJ_NOTNULL(jp, ve->priv, JOB_MAGIC); @@ -173,8 +216,9 @@ tst_cb(const struct vev *ve, int what) *buf = '\0'; i = read(ve->fd, buf, sizeof buf); if (i > 0) - VSB_bcat(jp->diag, buf, i); + VSB_bcat(jp->bp->diag, buf, i); if (i == 0) { + njob--; px = wait4(jp->child, &stx, 0, NULL); assert(px == jp->child); @@ -185,21 +229,20 @@ tst_cb(const struct vev *ve, int what) if (ecode == 0) ecode = WEXITSTATUS(stx); - AZ(VSB_finish(jp->diag)); - v = VSB_new_auto(); - AN(v); - VSB_cat(v, jp->buf); - p = strchr(jp->buf, '\0'); - if (p > jp->buf && p[-1] != '\n') - VSB_putc(v, '\n'); - VSB_quote_pfx(v, "* diag 0.0 ", - VSB_data(jp->diag), -1, VSB_QUOTE_NONL); - AZ(VSB_finish(v)); - VSB_destroy(&jp->diag); - AZ(munmap(jp->buf, jp->bufsiz)); + AZ(VSB_finish(jp->bp->diag)); + + VSB_clear(cbvsb); + VSB_cat(cbvsb, jp->bp->buf); + p = strchr(jp->bp->buf, '\0'); + if (p > jp->bp->buf && p[-1] != '\n') + VSB_putc(cbvsb, '\n'); + VSB_quote_pfx(cbvsb, "* diag 0.0 ", + VSB_data(jp->bp->diag), -1, VSB_QUOTE_NONL); + AZ(VSB_finish(cbvsb)); + rel_buf(&jp->bp); if ((ecode > 1 && vtc_verbosity) || vtc_verbosity > 1) - printf("%s", VSB_data(v)); + printf("%s", VSB_data(cbvsb)); if (!ecode) vtc_good++; @@ -215,11 +258,10 @@ tst_cb(const struct vev *ve, int what) bprintf(buf, "%s/LOG", jp->tmpdir); f = fopen(buf, "w"); AN(f); - (void)fprintf(f, "%s\n", VSB_data(v)); + (void)fprintf(f, "%s\n", VSB_data(cbvsb)); AZ(fclose(f)); } free(jp->tmpdir); - VSB_destroy(&v); if (jp->killed) printf("# top TEST %s TIMED OUT (kill -9)\n", @@ -265,15 +307,7 @@ start_test(void) ALLOC_OBJ(jp, JOB_MAGIC); AN(jp); - jp->diag = VSB_new_auto(); - AN(jp->diag); - - jp->bufsiz = vtc_bufsiz; - - jp->buf = mmap(NULL, jp->bufsiz, PROT_READ|PROT_WRITE, - MAP_ANON | MAP_SHARED, -1, 0); - assert(jp->buf != MAP_FAILED); - memset(jp->buf, 0, jp->bufsiz); + jp->bp = get_buf(); bprintf(tmpdir, "%s/vtc.%d.%08x", tmppath, (int)getpid(), (unsigned)random()); @@ -284,7 +318,7 @@ start_test(void) AN(tp->ntodo); tp->ntodo--; VTAILQ_REMOVE(&tst_head, tp, list); - if (tp->ntodo >0) + if (tp->ntodo > 0) VTAILQ_INSERT_TAIL(&tst_head, tp, list); jp->tst = tp; @@ -304,7 +338,7 @@ start_test(void) assert(dup2(p[1], STDERR_FILENO) == STDERR_FILENO); VSUB_closefrom(STDERR_FILENO + 1); retval = exec_file(jp->tst->filename, jp->tst->script, - jp->tmpdir, jp->buf, jp->bufsiz); + jp->tmpdir, jp->bp->buf, jp->bp->bufsiz); exit(retval); } closefd(&p[1]); @@ -569,6 +603,8 @@ main(int argc, char * const *argv) vtc_maxdur = atoi(p); VRND_SeedAll(); + cbvsb = VSB_new_auto(); + AN(cbvsb); setbuf(stdout, NULL); setbuf(stderr, NULL); while ((ch = getopt(argc, argv, "b:D:hij:kLln:p:qt:vW")) != -1) { From phk at FreeBSD.org Fri Feb 8 12:11:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 8 Feb 2019 12:11:08 +0000 (UTC) Subject: [master] cb61c369e Implement an optional high performance "cleaner" Message-ID: <20190208121108.4CC6B60F3B@lists.varnish-cache.org> commit cb61c369ee6902909c01a92e07b3e8d898176cb4 Author: Poul-Henning Kamp Date: Fri Feb 8 12:08:24 2019 +0000 Implement an optional high performance "cleaner" When running really massive runs, "-j180 -n10000" kind of things, the "rm -rf" of the tmpdir becomes the limiting factor. The new -C option sends that int nice(1)'ed child process. diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index a20a9f6fb..196dd9202 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -108,6 +108,9 @@ int leave_temp; int vtc_witness = 0; static struct vsb *cbvsb; +static int cleaner_fd = -1; +static pid_t cleaner_pid; + static struct buf * get_buf(void) { @@ -172,6 +175,7 @@ usage(void) #define FMT " %-28s # %s\n" fprintf(stderr, FMT, "-b size", "Set internal buffer size (default: 1M)"); + fprintf(stderr, FMT, "-C", "Use cleaner subprocess"); fprintf(stderr, FMT, "-D name=val", "Define macro"); fprintf(stderr, FMT, "-i", "Find varnish binaries in build tree"); fprintf(stderr, FMT, "-j jobs", "Run this many tests in parallel"); @@ -187,6 +191,86 @@ usage(void) exit(1); } +/********************************************************************** + * When running many tests, cleaning the tmpdir with "rm -rf" becomes + * chore which limits our performance. + * When the number of tests are above 100, we spawn a child-process + * to do that for us. + */ + +static void +cleaner_do(const char *dirname) +{ + char buf[BUFSIZ]; + + AZ(memcmp(dirname, tmppath, strlen(tmppath))); + if (cleaner_pid > 0) { + bprintf(buf, "%s\n", dirname); + assert(write(cleaner_fd, buf, strlen(buf)) == strlen(buf)); + return; + } + bprintf(buf, "exec /bin/rm -rf %s\n", dirname); + AZ(system(buf)); +} + +static void +cleaner_setup(void) +{ + int p[2], st; + char buf[BUFSIZ]; + char *q; + pid_t pp; + + AZ(pipe(p)); + assert(p[0] > STDERR_FILENO); + assert(p[1] > STDERR_FILENO); + cleaner_pid = fork(); + assert(cleaner_pid >= 0); + if (cleaner_pid == 0) { + closefd(&p[1]); + AZ(nice(1)); + setbuf(stdin, NULL); + AZ(dup2(p[0], STDIN_FILENO)); + while (fgets(buf, sizeof buf, stdin)) { + AZ(memcmp(buf, tmppath, strlen(tmppath))); + q = buf + strlen(buf); + assert(q > buf); + assert(q[-1] == '\n'); + q[-1] = '\0'; + + /* Dont expend a shell on running /bin/rm */ + pp = fork(); + assert(pp >= 0); + if (pp == 0) + exit(execl("/bin/rm", "rm", "-rf", buf, NULL)); + assert(waitpid(pp, &st, 0) == pp); + AZ(st); + } + exit(0); + } + closefd(&p[0]); + cleaner_fd = p[1]; +} + +static void +cleaner_neuter(void) +{ + if (cleaner_pid > 0) + closefd(&cleaner_fd); +} + +static void +cleaner_finish(void) +{ + int st; + + if (cleaner_pid > 0) { + closefd(&cleaner_fd); + assert(waitpid(cleaner_pid, &st, 0) == cleaner_pid); + AZ(st); + } +} + /********************************************************************** * CallBack */ @@ -252,8 +336,7 @@ tst_cb(const struct vev *ve, int what) vtc_fail++; if (leave_temp == 0 || (leave_temp == 1 && ecode <= 1)) { - bprintf(buf, "rm -rf %s", jp->tmpdir); - AZ(system(buf)); + cleaner_do(jp->tmpdir); } else { bprintf(buf, "%s/LOG", jp->tmpdir); f = fopen(buf, "w"); @@ -332,6 +415,7 @@ start_test(void) jp->child = fork(); assert(jp->child >= 0); if (jp->child == 0) { + cleaner_neuter(); // Too dangerous to have around AZ(setpgid(getpid(), 0)); VFIL_null_fd(STDIN_FILENO); assert(dup2(p[1], STDOUT_FILENO) == STDOUT_FILENO); @@ -575,6 +659,7 @@ main(int argc, char * const *argv) int ch, i; int ntest = 1; /* Run tests this many times */ int nstart = 0; + int use_cleaner = 0; uintmax_t bufsiz; const char *p; @@ -607,7 +692,7 @@ main(int argc, char * const *argv) AN(cbvsb); setbuf(stdout, NULL); setbuf(stderr, NULL); - while ((ch = getopt(argc, argv, "b:D:hij:kLln:p:qt:vW")) != -1) { + while ((ch = getopt(argc, argv, "b:CD:hij:kLln:p:qt:vW")) != -1) { switch (ch) { case 'b': if (VNUM_2bytes(optarg, &bufsiz, 0)) { @@ -622,6 +707,9 @@ main(int argc, char * const *argv) } vtc_bufsiz = (unsigned)bufsiz; break; + case 'C': + use_cleaner = !use_cleaner; + break; case 'D': if (!parse_D_opt(optarg)) { fprintf(stderr, "Cannot parse D opt '%s'\n", @@ -691,6 +779,9 @@ main(int argc, char * const *argv) vb = VEV_New(); + if (use_cleaner) + cleaner_setup(); + i = 0; while (!VTAILQ_EMPTY(&tst_head) || i) { if (!VTAILQ_EMPTY(&tst_head) && njob < npar) { @@ -704,6 +795,7 @@ main(int argc, char * const *argv) } i = VEV_Once(vb); } + cleaner_finish(); if (vtc_continue) fprintf(stderr, "%d tests failed, %d tests skipped, %d tests passed\n", From phk at FreeBSD.org Fri Feb 8 12:18:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 8 Feb 2019 12:18:08 +0000 (UTC) Subject: [master] 5ec2db924 The GCC people are getting ridiculous. Message-ID: <20190208121808.8736561363@lists.varnish-cache.org> commit 5ec2db924defb1c3c53700c457de34e072aee238 Author: Poul-Henning Kamp Date: Fri Feb 8 12:17:04 2019 +0000 The GCC people are getting ridiculous. diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 196dd9202..1a7fa98a2 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -242,7 +242,8 @@ cleaner_setup(void) pp = fork(); assert(pp >= 0); if (pp == 0) - exit(execl("/bin/rm", "rm", "-rf", buf, NULL)); + exit(execl( + "/bin/rm", "rm", "-rf", buf, (char*)0)); assert(waitpid(pp, &st, 0) == pp); AZ(st); } From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:10 +0000 (UTC) Subject: [6.0] 860bddc23 Add VSL rate limiting Message-ID: <20190208123110.4ABA5618C5@lists.varnish-cache.org> commit 860bddc23f7572798d494c15fec8dd46f5689139 Author: Dag Haavi Finstad Date: Tue Nov 20 11:16:51 2018 +0100 Add VSL rate limiting This adds rate limiting to varnishncsa and varnishlog. Rate limiting is done on a per-transaction basis, respective to the grouping mode selected. I.e. for -g request the limit will apply on a per-request basis, -g raw on a per-record basis, etc. Rate limit is specified as -R [/duration]. Default period if not specified is seconds ('s'). diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h index 417b76432..ea0b4c698 100644 --- a/bin/varnishlog/varnishlog_options.h +++ b/bin/varnishlog/varnishlog_options.h @@ -70,6 +70,7 @@ VUT_OPT_n VUT_GLOBAL_OPT_P VUT_OPT_q VUT_OPT_r +VSL_OPT_R VUT_OPT_t VSL_OPT_T VSL_OPT_v diff --git a/bin/varnishncsa/varnishncsa_options.h b/bin/varnishncsa/varnishncsa_options.h index 16bcdb1cd..73765083f 100644 --- a/bin/varnishncsa/varnishncsa_options.h +++ b/bin/varnishncsa/varnishncsa_options.h @@ -88,6 +88,7 @@ VUT_OPT_n VUT_GLOBAL_OPT_P VUT_OPT_q VUT_OPT_r +VSL_OPT_R VUT_OPT_t VUT_GLOBAL_OPT_V NCSA_OPT_w diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h index 3d95c3123..a357abd28 100644 --- a/include/vapi/vapi_options.h +++ b/include/vapi/vapi_options.h @@ -77,6 +77,22 @@ " running queries. Defaults to 1000 transactions." \ ) +#define VSL_OPT_R \ + VOPT("R:", "[-R ]", "Output rate limit", \ + "Restrict the output to the specified limit." \ + " Transactions exceeding the limit will be suppressed." \ + " The limit is specified as the maximum number of" \ + " transactions (with respect to the chosen grouping" \ + " method) and an optional time period. If no duration" \ + " is specified, a default of ``s`` is used. The duration" \ + " field can be formatted as in VCL (e.g. ``-R 10/2m``) or" \ + " as a simple time period without the prefix (e.g." \ + " ``-R 5/m``)." \ + " When in ``-g raw`` grouping mode, this setting can" \ + " not be used alongside ``-i``, ``-I``, ``-x`` or " \ + "``-X``, and we advise using ``-q`` instead." \ + ) + #define VSL_OPT_T \ VOPT("T:", "[-T ]", "Transaction end timeout", \ "Sets the transaction timeout in seconds. This defines the" \ diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h index 0e265c0b1..8817aa8fe 100644 --- a/lib/libvarnishapi/vsl_api.h +++ b/lib/libvarnishapi/vsl_api.h @@ -84,6 +84,8 @@ struct VSL_data { int c_opt; int C_opt; int L_opt; + int R_opt_l; + vtim_dur R_opt_p; double T_opt; int v_opt; }; diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c index b4bef5adb..2929b4ccf 100644 --- a/lib/libvarnishapi/vsl_arg.c +++ b/lib/libvarnishapi/vsl_arg.c @@ -300,6 +300,41 @@ vsl_IX_arg(struct VSL_data *vsl, int opt, const char *arg) return (1); } +static int +vsl_R_arg(struct VSL_data *vsl, const char *arg) +{ + char buf[32] = ""; + char *p; + long l; + + AN(arg); + CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC); + + l = strtol(arg, &p, 0); + if (l <= 0 || l > INT_MAX) + return (vsl_diag(vsl, "-R: Range error")); + vsl->R_opt_l = l; + assert(p != arg); + AN(p); + if (*p == '\0') { + vsl->R_opt_p = 1.0; + return (1); + } + if (*p != '/' || p[1] == '\0') + return (vsl_diag(vsl, "-R: Syntax error")); + p++; + if (strlen(p) > sizeof(buf) - 2) + return (vsl_diag(vsl, "-R: Syntax error")); + if (!isdigit(*p)) + strcat(buf, "1"); + strcat(buf, p); + vsl->R_opt_p = VNUM_duration(buf); + if (isnan(vsl->R_opt_p) || vsl->R_opt_p <= 0.0) + return (vsl_diag(vsl, + "-R: Syntax error: Invalid duration")); + return (1); +} + int VSL_Arg(struct VSL_data *vsl, int opt, const char *arg) { @@ -334,6 +369,8 @@ VSL_Arg(struct VSL_data *vsl, int opt, const char *arg) return (vsl_diag(vsl, "-L: Range error")); vsl->L_opt = (int)l; return (1); + case 'R': + return (vsl_R_arg(vsl, arg)); case 'T': AN(arg); d = VNUM(arg); diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index 6e786d4f7..0075e2322 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -194,6 +194,10 @@ struct VSLQ { VTAILQ_HEAD(,vtx) cache; unsigned n_cache; + /* Rate limiting */ + double credits; + vtim_mono last_use; + /* Raw mode */ struct { struct vslc_raw c; @@ -908,10 +912,33 @@ vtx_force(struct VSLQ *vslq, struct vtx *vtx, const char *reason) AN(vtx->flags & VTX_F_COMPLETE); } +static int +vslq_ratelimit(struct VSLQ *vslq) +{ + vtim_mono now; + vtim_dur delta; + + CHECK_OBJ_NOTNULL(vslq, VSLQ_MAGIC); + CHECK_OBJ_NOTNULL(vslq->vsl, VSL_MAGIC); + + now = VTIM_mono(); + delta = now - vslq->last_use; + vslq->credits += (delta / vslq->vsl->R_opt_p) * vslq->vsl->R_opt_l; + if (vslq->credits > vslq->vsl->R_opt_l) + vslq->credits = vslq->vsl->R_opt_l; + vslq->last_use = now; + + if (vslq->credits < 1.0) + return (0); + + vslq->credits -= 1.0; + return (1); +} + /* Build transaction array, do the query and callback. Returns 0 or the return value from func */ static int -vslq_callback(const struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func, +vslq_callback(struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func, void *priv) { unsigned n = vtx->n_descend + 1; @@ -973,6 +1000,9 @@ vslq_callback(const struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func, if (vslq->query != NULL && !vslq_runquery(vslq->query, ptrans)) return (0); + if (vslq->vsl->R_opt_l != 0 && !vslq_ratelimit(vslq)) + return (0); + /* Callback */ return ((func)(vslq->vsl, ptrans, priv)); } @@ -1078,6 +1108,10 @@ VSLQ_New(struct VSL_data *vsl, struct VSL_cursor **cp, } vslq->grouping = grouping; vslq->query = query; + if (vslq->vsl->R_opt_l != 0) { + vslq->last_use = VTIM_mono(); + vslq->credits = 1; + } /* Setup normal mode */ VRB_INIT(&vslq->tree); @@ -1195,6 +1229,9 @@ vslq_raw(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv) !vslq_runquery(vslq->query, vslq->raw.ptrans)) return (r); + if (vslq->vsl->R_opt_l != 0 && !vslq_ratelimit(vslq)) + return (r); + i = (func)(vslq->vsl, vslq->raw.ptrans, priv); if (i) return (i); From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:10 +0000 (UTC) Subject: [6.0] d30f1a015 Add a real tokenizer (only little use yet) Message-ID: <20190208123110.723C2618C9@lists.varnish-cache.org> commit d30f1a01589aa80dda6dcdfd647c68dad972fbba Author: Poul-Henning Kamp Date: Tue Nov 27 13:21:32 2018 +0000 Add a real tokenizer (only little use yet) Make reporting syntax errors easier. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index e58041266..4989e299b 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -527,7 +527,8 @@ class ProtoType(object): class stanza(object): - def __init__(self, l0, doc, vcc): + def __init__(self, toks, l0, doc, vcc): + self.toks = toks self.line = l0 while doc and doc[0] == '': doc.pop(0) @@ -543,6 +544,12 @@ class stanza(object): def dump(self): print(type(self), self.line) + def syntax(self): + err("Syntax error.\n" + + "\tShould be: " + self.__doc__.strip() + "\n" + + "\tIs: " + " ".join(self.toks) + "\n", + warn=False) + def rstfile(self, fo, man): if self.rstlbl is not None: fo.write(".. _" + self.rstlbl + ":\n\n") @@ -640,17 +647,31 @@ class s_module(stanza): class s_abi(stanza): + + ''' $ABI [strict|vrt] ''' + def parse(self): - if self.line[1] not in ('strict', 'vrt'): + if len(self.toks) != 2: + self.syntax() + valid = { + 'strict': True, + 'vrt': False, + } + self.vcc.strict_abi = valid.get(self.toks[1]) + if self.vcc.strict_abi is None: err("Valid ABI types are 'strict' or 'vrt', got '%s'\n" % - self.line[1]) - self.vcc.strict_abi = self.line[1] == 'strict' + self.toks[1]) self.vcc.contents.append(self) class s_prefix(stanza): + + ''' $Prefix symbol ''' + def parse(self): - self.vcc.sympfx = self.line[1] + "_" + if len(self.toks) != 2: + self.syntax() + self.vcc.sympfx = self.toks[1] + "_" self.vcc.contents.append(self) @@ -832,6 +853,43 @@ DISPATCH = { "Synopsis": s_synopsis, } +def tokenize(str, seps=None, quotes=None): + if seps is None: + seps = "[](){},=" + if quotes is None: + quotes = '"' + "'" + quote = None + out = [] + i = 0 + inside = False + while i < len(str): + c = str[i] + # print("T", [c], quote, inside, i) + i += 1 + if quote is not None and c == quote: + inside = False + quote = None + out[-1] += c + elif quote is not None: + out[-1] += c + elif c.isspace(): + inside = False + elif seps.find(c) >= 0: + inside = False + out.append(c) + elif quotes.find(c) >= 0: + quote = c + out.append(c) + elif inside: + out[-1] += c + else: + out.append(c) + inside = True + #print("TOK", [str]) + #for i in out: + # print("\t", [i]) + return out + class vcc(object): def __init__(self, inputvcc, rstdir, outputprefix): @@ -864,12 +922,13 @@ class vcc(object): self.copyright = s.pop(0).strip() while s: ss = re.split('\n([^\t ])', s.pop(0), maxsplit=1) + toks = tokenize(ss[0]) c = ss[0].split() d = "".join(ss[1:]) - m = DISPATCH.get(c[0]) + m = DISPATCH.get(toks[0]) if m is None: err("Unknown stanze $%s" % ss[:i]) - m([c[0], " ".join(c[1:])], d.split('\n'), self) + m(toks, [c[0], " ".join(c[1:])], d.split('\n'), self) inputline = None def rst_copyright(self, fo): From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:10 +0000 (UTC) Subject: [6.0] af90bd621 More use of tokenlist Message-ID: <20190208123110.8B7E6618CD@lists.varnish-cache.org> commit af90bd6210f9ceab93943ce41954827090282846 Author: Poul-Henning Kamp Date: Tue Nov 27 14:05:09 2018 +0000 More use of tokenlist diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 4989e299b..e9fe75d73 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -589,10 +589,17 @@ class stanza(object): class s_module(stanza): + + ''' $Module modname man_section description ... ''' + def parse(self): + if len(self.toks) < 4: + self.syntax() a = self.line[1].split(None, 2) - self.vcc.modname = a[0] - self.vcc.mansection = a[1] + self.vcc.modname = self.toks[1] + self.vcc.mansection = self.toks[2] + # XXX: Find better solution for moddesc + self.vcc.moddesc = " ".join(self.toks[2:]) self.vcc.moddesc = a[2] self.rstlbl = "vmod_%s(%s)" % ( self.vcc.modname, @@ -676,17 +683,31 @@ class s_prefix(stanza): class s_synopsis(stanza): + + ''' $Synopsis [auto|manual] ''' + def parse(self): - if self.line[1] not in ('auto', 'manual'): + if len(self.toks) != 2: + self.syntax() + valid = { + 'auto': True, + 'manual': False, + } + self.vcc.auto_synopsis = vald.get(self.toks[1]) + if self.vcc.auto_synopsis is None: err("Valid Synopsis values are 'auto' or 'manual', got '%s'\n" % - self.line[1]) - self.vcc.auto_synopsis = self.line[1] == 'auto' + self.toks[1]) self.vcc.contents.append(self) class s_event(stanza): + + ''' $Event function_name ''' + def parse(self): - self.event_func = self.line[1] + if len(self.toks) != 2: + self.syntax() + self.event_func = self.toks[1] self.vcc.contents.append(self) def rstfile(self, fo, man): From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:10 +0000 (UTC) Subject: [6.0] 5a448f21c fix typo Message-ID: <20190208123110.ACBBC618D4@lists.varnish-cache.org> commit 5a448f21c22731407e2eecad9a537c6dda94916a Author: Nils Goroll Date: Tue Nov 27 17:34:50 2018 +0100 fix typo diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index e9fe75d73..72b1c5a6b 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -693,7 +693,7 @@ class s_synopsis(stanza): 'auto': True, 'manual': False, } - self.vcc.auto_synopsis = vald.get(self.toks[1]) + self.vcc.auto_synopsis = valid.get(self.toks[1]) if self.vcc.auto_synopsis is None: err("Valid Synopsis values are 'auto' or 'manual', got '%s'\n" % self.toks[1]) From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:10 +0000 (UTC) Subject: [6.0] 752510121 Implement a better solution: Prefer a single quoted token for $Module description, and suggest people to do it that way. Message-ID: <20190208123110.CB01C618DE@lists.varnish-cache.org> commit 752510121b1dcecf7b6f62d901708c0ad38e3459 Author: Poul-Henning Kamp Date: Tue Nov 27 22:17:39 2018 +0000 Implement a better solution: Prefer a single quoted token for $Module description, and suggest people to do it that way. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 72b1c5a6b..da1657a73 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -120,6 +120,15 @@ CTYPES.update(PRIVS) ####################################################################### +def is_quoted(str): + return len(str) > 2 and str[0] == str[-1] and str[0] in ('"', "'") + +def unquote(str): + assert is_quoted(str) + return str[1:-1] + +####################################################################### + def write_file_warning(fo, a, b, c): fo.write(a + "\n") @@ -598,9 +607,11 @@ class s_module(stanza): a = self.line[1].split(None, 2) self.vcc.modname = self.toks[1] self.vcc.mansection = self.toks[2] - # XXX: Find better solution for moddesc - self.vcc.moddesc = " ".join(self.toks[2:]) - self.vcc.moddesc = a[2] + if len(self.toks) == 4 and is_quoted(self.toks[3]): + self.vcc.moddesc = unquote(self.toks[3]) + else: + print("\nNOTICE: Please put $Module description in quotes.\n") + self.vcc.moddesc = " ".join(self.toks[3:]) self.rstlbl = "vmod_%s(%s)" % ( self.vcc.modname, self.vcc.mansection diff --git a/lib/libvmod_blob/vmod.vcc b/lib/libvmod_blob/vmod.vcc index 0a7e652d5..798989b52 100644 --- a/lib/libvmod_blob/vmod.vcc +++ b/lib/libvmod_blob/vmod.vcc @@ -6,7 +6,7 @@ # Geoffrey Simmons # -$Module blob 3 utilities for the VCL blob type +$Module blob 3 "Utilities for the VCL blob type" $ABI strict diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 6398d226d..f5ab7594a 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -26,7 +26,7 @@ # SUCH DAMAGE. # -$Module debug 3 Development, test and debug +$Module debug 3 "Development, test and debug" $ABI strict $Prefix xyzzy DESCRIPTION diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index 8fecc08e7..54b111fba 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -32,7 +32,7 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -$Module directors 3 Varnish Directors Module +$Module directors 3 "Varnish Directors Module" $ABI strict DESCRIPTION diff --git a/lib/libvmod_proxy/vmod.vcc b/lib/libvmod_proxy/vmod.vcc index d986b5295..34b1199b2 100644 --- a/lib/libvmod_proxy/vmod.vcc +++ b/lib/libvmod_proxy/vmod.vcc @@ -25,7 +25,7 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -$Module proxy 3 Varnish Module to extract TLV attributes from PROXYv2 +$Module proxy 3 "Varnish Module to extract TLV attributes from PROXYv2" $ABI strict DESCRIPTION diff --git a/lib/libvmod_purge/vmod.vcc b/lib/libvmod_purge/vmod.vcc index 25a9cb801..e158aac69 100644 --- a/lib/libvmod_purge/vmod.vcc +++ b/lib/libvmod_purge/vmod.vcc @@ -25,7 +25,7 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -$Module purge 3 Varnish Purge Module +$Module purge 3 "Varnish Purge Module" $ABI strict DESCRIPTION diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index c30172ed1..fc5ef3736 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -25,7 +25,7 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -$Module std 3 Varnish Standard Module +$Module std 3 "Varnish Standard Module" $ABI strict DESCRIPTION diff --git a/lib/libvmod_unix/vmod.vcc b/lib/libvmod_unix/vmod.vcc index e70a092d3..e227e9f99 100644 --- a/lib/libvmod_unix/vmod.vcc +++ b/lib/libvmod_unix/vmod.vcc @@ -5,7 +5,7 @@ # Authors: Geoffrey Simmons # -$Module unix 3 utilities for Unix domain sockets +$Module unix 3 "Utilities for Unix domain sockets" $ABI strict diff --git a/lib/libvmod_vtc/vmod.vcc b/lib/libvmod_vtc/vmod.vcc index c521f37da..94f8f75b6 100644 --- a/lib/libvmod_vtc/vmod.vcc +++ b/lib/libvmod_vtc/vmod.vcc @@ -25,7 +25,7 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -$Module vtc 3 Utility module for varnishtest +$Module vtc 3 "Utility module for varnishtest" $ABI strict DESCRIPTION From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:10 +0000 (UTC) Subject: [6.0] 5e1afbfb5 Ditch the old prototype tokenizer for the new. Message-ID: <20190208123110.EE9C1618FC@lists.varnish-cache.org> commit 5e1afbfb554d81ca3172dcf3254d40ffafb02ea0 Author: Poul-Henning Kamp Date: Tue Nov 27 22:56:20 2018 +0000 Ditch the old prototype tokenizer for the new. Other polish. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index da1657a73..4886c4e26 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -235,10 +235,8 @@ class CType(object): self.spec = [] while True: x = wl.pop(0) - if x[0] == '"' and x[-1] == '"': - x = x[1:-1] - elif x[0] == "'" and x[-1] == "'": - x = x[1:-1] + if is_quoted(x): + x = unquote(x) assert x self.spec.append(x) enums[x] = True @@ -268,6 +266,9 @@ class CType(object): class arg(CType): + + ''' Parse front of word list into argument ''' + def __init__(self, wl, argnames, enums, end): super(arg, self).__init__(wl, enums) @@ -289,10 +290,8 @@ class arg(CType): x = wl.pop(0) if self.vt == "ENUM": - if x[0] == '"' and x[-1] == '"': - x = x[1:-1] - elif x[0] == "'" and x[-1] == "'": - x = x[1:-1] + if is_quoted(x): + x = unquote(x) self.defval = x def json(self, jl): @@ -305,60 +304,13 @@ class arg(CType): ####################################################################### -def lex(l): - wl = [] - s = 0 - assert l - for i in range(len(l)): - c = l[i] - - if s == 0 and re.match('[0-9a-zA-Z_.-]', c): - wl.append(c) - s = 3 - continue - - if s == 3: - if re.match('[0-9a-zA-Z_.-]', c): - wl[-1] += c - continue - s = 0 - - if s == 0 and c in (' ', '\t', '\n', '\r'): - continue - - if s == 0 and c in ('[', '(', '{', '}', ')', ']', ',', '='): - wl.append(c) - elif s == 0 and c in ('"', "'"): - sep = c - s = 1 - wl.append(c) - elif s == 1: - if c == '\\': - s = 2 - else: - wl[-1] += c - if c == sep: - s = 0 - elif s == 2: - wl[-1] += c - s = 1 - else: - err("Syntax error at char %d '%s'" % (i, c), warn=False) - - if s != 0: - err("Syntax error at char %d '%s'" % (i, c), warn=False) - return wl - -####################################################################### - - class ProtoType(object): def __init__(self, st, retval=True, prefix=""): self.st = st self.obj = None self.args = [] self.argstruct = False - wl = lex(st.line[1]) + wl = self.st.toks[1:] if retval: self.retval = CType(wl, st.vcc.enums) @@ -604,7 +556,6 @@ class s_module(stanza): def parse(self): if len(self.toks) < 4: self.syntax() - a = self.line[1].split(None, 2) self.vcc.modname = self.toks[1] self.vcc.mansection = self.toks[2] if len(self.toks) == 4 and is_quoted(self.toks[3]): @@ -885,43 +836,6 @@ DISPATCH = { "Synopsis": s_synopsis, } -def tokenize(str, seps=None, quotes=None): - if seps is None: - seps = "[](){},=" - if quotes is None: - quotes = '"' + "'" - quote = None - out = [] - i = 0 - inside = False - while i < len(str): - c = str[i] - # print("T", [c], quote, inside, i) - i += 1 - if quote is not None and c == quote: - inside = False - quote = None - out[-1] += c - elif quote is not None: - out[-1] += c - elif c.isspace(): - inside = False - elif seps.find(c) >= 0: - inside = False - out.append(c) - elif quotes.find(c) >= 0: - quote = c - out.append(c) - elif inside: - out[-1] += c - else: - out.append(c) - inside = True - #print("TOK", [str]) - #for i in out: - # print("\t", [i]) - return out - class vcc(object): def __init__(self, inputvcc, rstdir, outputprefix): @@ -954,7 +868,8 @@ class vcc(object): self.copyright = s.pop(0).strip() while s: ss = re.split('\n([^\t ])', s.pop(0), maxsplit=1) - toks = tokenize(ss[0]) + toks = self.tokenize(ss[0]) + inputline = '$' + ' '.join(toks) c = ss[0].split() d = "".join(ss[1:]) m = DISPATCH.get(toks[0]) @@ -963,6 +878,44 @@ class vcc(object): m(toks, [c[0], " ".join(c[1:])], d.split('\n'), self) inputline = None + def tokenize(self, str, seps=None, quotes=None): + if seps is None: + seps = "[](){},=" + if quotes is None: + quotes = '"' + "'" + quote = None + out = [] + i = 0 + inside = False + while i < len(str): + c = str[i] + # print("T", [c], quote, inside, i) + i += 1 + if quote is not None and c == quote: + inside = False + quote = None + out[-1] += c + elif quote is not None: + out[-1] += c + elif c.isspace(): + inside = False + elif seps.find(c) >= 0: + inside = False + out.append(c) + elif quotes.find(c) >= 0: + quote = c + out.append(c) + elif inside: + out[-1] += c + else: + out.append(c) + inside = True + #print("TOK", [str]) + #for i in out: + # print("\t", [i]) + return out + + def rst_copyright(self, fo): write_rst_hdr(fo, "COPYRIGHT", "=") fo.write("\n::\n\n") From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:11 +0000 (UTC) Subject: [6.0] 10c8db66f fix vmodtool error message Message-ID: <20190208123111.1961F61905@lists.varnish-cache.org> commit 10c8db66fd6a90ba3fc81139c94d5dfc7052ca27 Author: Nils Goroll Date: Wed Nov 28 13:51:10 2018 +0100 fix vmodtool error message diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 4886c4e26..2ef32e78c 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -874,7 +874,7 @@ class vcc(object): d = "".join(ss[1:]) m = DISPATCH.get(toks[0]) if m is None: - err("Unknown stanze $%s" % ss[:i]) + err("Unknown stanza $%s" % toks[0], warn=False) m(toks, [c[0], " ".join(c[1:])], d.split('\n'), self) inputline = None From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:11 +0000 (UTC) Subject: [6.0] b0249dc88 Test $Synopsis stanza Message-ID: <20190208123111.3B4EB61913@lists.varnish-cache.org> commit b0249dc8892a188e5214da47ab36980ce4b4aacd Author: Nils Goroll Date: Wed Nov 28 13:53:23 2018 +0100 Test $Synopsis stanza this would have caught 7395ecae8fdfaefca9ee5438d35ea6101c7a9f56 diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index f5ab7594a..0409c1f0a 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -29,6 +29,7 @@ $Module debug 3 "Development, test and debug" $ABI strict $Prefix xyzzy +$Synopsis auto DESCRIPTION =========== From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:11 +0000 (UTC) Subject: [6.0] db97d8eaf Add a debug flag to keep VCL C and so files around Message-ID: <20190208123111.5FDEB61924@lists.varnish-cache.org> commit db97d8eaf5b805cd101d5fb7e04a5b31289012b5 Author: Federico G. Schwindt Date: Fri Nov 2 12:02:38 2018 +0000 Add a debug flag to keep VCL C and so files around Enabled when using varnishtest -L. diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 9777584f8..2b476307b 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -380,7 +380,8 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname, } AZ(fclose(fcs)); - (void)unlink(vp.csrcfile); + if (!MGT_DO_DEBUG(DBG_VCL_KEEP)) + (void)unlink(vp.csrcfile); free(vp.csrcfile); free(vp.dir); diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 0cb29e44c..0ed9274f6 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -265,7 +265,8 @@ mgt_vcl_del(struct vclprog *vp) VTAILQ_REMOVE(&vclhead, vp, list); if (vp->fname != NULL) { - AZ(unlink(vp->fname)); + if (!MGT_DO_DEBUG(DBG_VCL_KEEP)) + AZ(unlink(vp->fname)); p = strrchr(vp->fname, '/'); AN(p); *p = '\0'; diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index ac7e17adb..0c7b3f5a3 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -413,8 +413,9 @@ varnish_launch(struct varnish *v) if (vtc_witness) VSB_cat(vsb, " -p debug=+witness"); if (leave_temp) { - VSB_cat(vsb, " -p debug=+vsm_keep"); + VSB_cat(vsb, " -p debug=+vcl_keep"); VSB_cat(vsb, " -p debug=+vmod_so_keep"); + VSB_cat(vsb, " -p debug=+vsm_keep"); } VSB_printf(vsb, " -l 2m"); VSB_printf(vsb, " -p auto_restart=off"); diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h index 4ac3ef60f..8bf099d5d 100644 --- a/include/tbl/debug_bits.h +++ b/include/tbl/debug_bits.h @@ -50,6 +50,7 @@ DEBUG_BIT(H2_NOCHECK, h2_nocheck, "Disable various H2 checks") DEBUG_BIT(VMOD_SO_KEEP, vmod_so_keep, "Keep copied VMOD libraries") DEBUG_BIT(PROCESSORS, processors, "Fetch/Deliver processors") DEBUG_BIT(PROTOCOL, protocol, "Protocol debugging") +DEBUG_BIT(VCL_KEEP, vcl_keep, "Keep VCL C and so files") #undef DEBUG_BIT /*lint -restore */ From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:11 +0000 (UTC) Subject: [6.0] eac2d9f24 Always keep .c and .so files if vcl_keep is set Message-ID: <20190208123111.8748361930@lists.varnish-cache.org> commit eac2d9f24ee11c17c1807ed9656fdda5a9a8030d Author: Federico G. Schwindt Date: Sun Dec 2 17:29:17 2018 +0000 Always keep .c and .so files if vcl_keep is set Add tests for vcl_keep and move vmod_so_keep together. Reported and diagnosed by jw @ irc. diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 2b476307b..c42e889d7 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -346,11 +346,13 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname, VSB_destroy(&sb); if (status || C_flag) { - (void)unlink(vp.csrcfile); + if (!MGT_DO_DEBUG(DBG_VCL_KEEP)) { + (void)unlink(vp.csrcfile); + (void)unlink(vp.libfile); + (void)rmdir(vp.dir); + } free(vp.csrcfile); - (void)unlink(vp.libfile); free(vp.libfile); - (void)rmdir(vp.dir); free(vp.dir); if (status) { VCLI_Out(cli, "VCL compilation failed"); diff --git a/bin/varnishtest/tests/c00095.vtc b/bin/varnishtest/tests/c00095.vtc new file mode 100644 index 000000000..93ce3c8b6 --- /dev/null +++ b/bin/varnishtest/tests/c00095.vtc @@ -0,0 +1,84 @@ +varnishtest "vcl_keep and vmod_so_keep debug bits" + +feature topbuild + +server s1 { +} -start + +varnish v1 -vcl+backend { +} -start + +# Test valid and invalid VCL with vcl_keep unset + +varnish v1 -cliok "param.set debug -vcl_keep" + +varnish v1 -vcl+backend { +} + +shell -err "test -f ./v1/vcl_vcl2.*/vgc.c" + +varnish v1 -errvcl {No backends or directors found} { +} + +shell -err "test -f ./v1/vcl_vcl3.*/vgc.c" + +# Same but with vcl_keep set + +varnish v1 -cliok "param.set debug +vcl_keep" + +varnish v1 -vcl+backend { +} + +shell { + test -f ./v1/vcl_vcl4.*/vgc.c && + test -f ./v1/vcl_vcl4.*/vgc.so +} + +varnish v1 -errvcl {No backends or directors found} { +} + +shell { + test -f ./v1/vcl_vcl5.*/vgc.c && + test -f ./v1/vcl_vcl5.*/vgc.so +} + +# Test vmod with vmod_so_keep set + +varnish v1 -cliok "param.set debug +vmod_so_keep" + +varnish v1 -vcl+backend { + import std; +} + +shell "test -f ./v1/vmod_cache/_vmod_std.*" + +varnish v1 -stop +varnish v1 -cleanup + +# Ensure these are not deleted on exit + +shell { + test -f ./v1/vcl_vcl4.*/vgc.c && + test -f ./v1/vcl_vcl4.*/vgc.so && + test -f ./v1/vcl_vcl5.*/vgc.c && + test -f ./v1/vcl_vcl5.*/vgc.so && + test -f ./v1/vmod_cache/_vmod_std.* +} + +varnish v2 -vcl+backend { +} -start + +# And test vmod with vmod_so_keep unset + +varnish v2 -cliok "param.set debug -vmod_so_keep" + +varnish v2 -vcl+backend { + import std; +} + +shell "test -f ./v2/vmod_cache/_vmod_std.*" + +varnish v2 -stop +varnish v2 -cleanup + +shell -err "test -f ./v2/vmod_cache/_vmod_std.*" diff --git a/bin/varnishtest/tests/m00008.vtc b/bin/varnishtest/tests/m00008.vtc index 7026f578e..a0bc5b2a3 100644 --- a/bin/varnishtest/tests/m00008.vtc +++ b/bin/varnishtest/tests/m00008.vtc @@ -47,14 +47,3 @@ varnish v1 -errvcl {Malformed VMOD std} { backend default { .host = "${s1_sock}"; } import std from "${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so"; } - -# Check creation and cleanup of copied shared objects for VMODs -shell "test -e ./v1/vmod_cache/_vmod_std.*" - -# In case varnishtest was invoked with -L or -l, in which case the -# bit is switched on. -varnish v1 -cliok "param.set debug -vmod_so_keep" - -varnish v1 -stop -varnish v1 -cleanup -shell -err "test -e ./v1/vmod_cache/_vmod_std.*" diff --git a/bin/varnishtest/tests/m00030.vtc b/bin/varnishtest/tests/m00030.vtc deleted file mode 100644 index 4ea2e0ede..000000000 --- a/bin/varnishtest/tests/m00030.vtc +++ /dev/null @@ -1,17 +0,0 @@ -varnishtest "debug bit vmod_so_keep" - -feature topbuild - -varnish v1 -vcl { - import std; - backend b { .host = "${bad_backend}"; } -} -start - -shell "test -e ./v1/vmod_cache/_vmod_std.*" - -varnish v1 -cliok "param.set debug +vmod_so_keep" - -# cf m00008.vtc -varnish v1 -stop -varnish v1 -cleanup -shell "test -e ./v1/vmod_cache/_vmod_std.*" From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:11 +0000 (UTC) Subject: [6.0] ceafe79aa Update osx build and drop sudo: false Message-ID: <20190208123111.A9B1161935@lists.varnish-cache.org> commit ceafe79aa00637efa3fbcbc3cc2606221f2c736f Author: Federico G. Schwindt Date: Sun Dec 2 18:05:19 2018 +0000 Update osx build and drop sudo: false diff --git a/.travis.yml b/.travis.yml index da2bcbb8b..52424fefc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ --- language: c -sudo: false matrix: fast_finish: true include: @@ -16,7 +15,7 @@ matrix: env: CLANG=7 SAN_FLAGS="--enable-asan --enable-ubsan" sudo: required - os: osx - osx_image: xcode10 + osx_image: xcode10.1 compiler: clang allow_failures: - os: osx From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:11 +0000 (UTC) Subject: [6.0] 4fc3914f4 Switch to xenial and tidy up Message-ID: <20190208123111.C60696193B@lists.varnish-cache.org> commit 4fc3914f4717e1c29901da3760320c40c3835ac4 Author: Federico G. Schwindt Date: Sun Dec 2 18:27:02 2018 +0000 Switch to xenial and tidy up diff --git a/.travis.yml b/.travis.yml index 52424fefc..8d31d9e04 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,27 +4,31 @@ matrix: fast_finish: true include: - os: linux - dist: trusty + dist: xenial compiler: gcc - os: linux - dist: trusty + dist: xenial compiler: clang - os: linux - dist: trusty + dist: xenial compiler: clang - env: CLANG=7 SAN_FLAGS="--enable-asan --enable-ubsan" - sudo: required + env: SAN_FLAGS="--enable-asan --enable-ubsan" - os: osx osx_image: xcode10.1 compiler: clang allow_failures: - os: osx - - env: CLANG=7 SAN_FLAGS="--enable-asan --enable-ubsan" + - env: SAN_FLAGS="--enable-asan --enable-ubsan" addons: apt: packages: - python-docutils - python-sphinx + homebrew: + packages: + - docutils + - nghttp2 + - sphinx-doc notifications: irc: channels: @@ -34,19 +38,8 @@ notifications: before_install: - | if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - brew update - brew install docutils sphinx-doc nghttp2 export PATH="/usr/local/opt/sphinx-doc/bin:$PATH" - elif [[ -n "$CLANG" ]]; then - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo apt-add-repository -y \ - "deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-$CLANG main" - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA9EF27F - sudo apt-add-repository -y \ - "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu trusty main" - sudo apt-get update - sudo apt-get install -y clang-$CLANG llvm-$CLANG - export CC=clang-$CLANG + elif [[ -n "$SAN_FLAGS" ]]; then export CONFIGURE_ARGS="--enable-developer-warnings --enable-debugging-symbols --disable-stack-protector --with-persistent-storage ${SAN_FLAGS}" export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0,disable_coredump=0 export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:11 +0000 (UTC) Subject: [6.0] e45fb125d Use upper case for abbreviations. Message-ID: <20190208123111.E0BC461940@lists.varnish-cache.org> commit e45fb125d8c907303d785458e47d9a613f3d668d Author: Lasse Karstensen Date: Sun Dec 2 20:24:17 2018 +0100 Use upper case for abbreviations. Pedantic back-port note: those are acronyms. diff --git a/lib/libvmod_proxy/vmod.vcc b/lib/libvmod_proxy/vmod.vcc index 34b1199b2..79aa4b678 100644 --- a/lib/libvmod_proxy/vmod.vcc +++ b/lib/libvmod_proxy/vmod.vcc @@ -38,14 +38,14 @@ as described in https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt. $Function STRING alpn() Description - Extract alpn attribute. + Extract ALPN attribute. Example set req.http.alpn = proxy.alpn(); $Function STRING authority() Description - Extract authority attribute. This corresponds to sni from a tls + Extract authority attribute. This corresponds to SNI from a TLS connection. Example set req.http.authority = proxy.authority(); @@ -53,7 +53,7 @@ Example $Function BOOL is_ssl() Description - Report if proxy-protocol-v2 has ssl tlv. + Report if proxy-protocol-v2 has SSL TLV. Example | if (proxy.is_ssl()) { @@ -86,7 +86,7 @@ Example $Function STRING ssl_version() Description - Extract ssl version attribute. + Extract SSL version attribute. Example set req.http.ssl-version = proxy.ssl_version(); @@ -100,7 +100,7 @@ Example $Function STRING ssl_cipher() Description - Extract the ssl cipher attribute. + Extract the SSL cipher attribute. Example set req.http.ssl-cipher = proxy.ssl_cipher(); From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:12 +0000 (UTC) Subject: [6.0] 8d85c98a8 Improve docs Message-ID: <20190208123112.092AA61944@lists.varnish-cache.org> commit 8d85c98a8d6312b8e8ebdff8dc221104e57c5088 Author: Federico G. Schwindt Date: Sun Dec 2 20:19:22 2018 +0000 Improve docs Partially addresses #2846. I've avoided mentioning NULL on purpose as it's not really meaningful nor accessible from VCL. ymmv. diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 7aa434da6..f402c2f06 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -93,7 +93,7 @@ Booleans can be either ``true`` or ``false``. In addition, in a boolean context some data types will evaluate to ``true`` or ``false`` depending on their value. -String types will evaluate to ``false`` if they are empty; backend types +String types will evaluate to ``false`` if they are not set; backend types will evaluate to ``false`` if they don't have a backend assigned; integer types will evaluate to ``false`` if their value is zero; duration types will evaluate to ``false`` if their value is equal or less than zero. @@ -135,18 +135,20 @@ include a fractional part, e.g. ``1.5s``. The supported units are: ``y`` years +In string context they return a string with their value rounded to +3 decimal places and excluding the unit, e.g. ``1.500``. + Integers ~~~~~~~~ Certain fields are integers, used as expected. In string context they -return a string. +return a string, e.g. ``1234``. Real numbers ~~~~~~~~~~~~ -VCL understands real numbers. As with integers, when used in a string -context they will return a string. - +VCL understands real numbers. In string context they return a string +with their value rounded to 3 decimal places, e.g. ``3.142``. Regular Expressions ------------------- From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:12 +0000 (UTC) Subject: [6.0] dadb6ebcf More coverage Message-ID: <20190208123112.2B48C6194B@lists.varnish-cache.org> commit dadb6ebcfca40f3ab6a4348236b665e746718483 Author: Federico G. Schwindt Date: Sun Dec 2 21:52:07 2018 +0000 More coverage diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index 8dcb2f920..2f1e41476 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -60,6 +60,25 @@ shell -err -expect "Cannot open output file (No such file or directory)" \ shell -err -expect "Only one of -n and -r options may be used" \ {varnishlog -n ${v1_name} -r ${v1_name}/_.vsm} +shell -err -expect "Expected integer got '}" \ + "varnishlog -q {}" +shell -err -expect "Expected positive integer" \ + "varnishlog -q {-1}" +shell -err -expect "Syntax error in level limit" \ + "varnishlog -q {1a}" +shell -err -expect "Expected VSL tag name got ''" \ + "varnishlog -q ''" +shell -err -expect "Tag name matches zero tags" \ + "varnishlog -q foo" +shell -err -expect "Tag name is ambiguous" \ + "varnishlog -q Resp" +shell -err -expect "Expected string got ''" \ + "varnishlog -q ReqHeader:" +shell -err -expect "Expected integer got ']'" \ + "varnishlog -q ReqHeader:foo[]" +shell -err -expect "Expected positive integer" \ + "varnishlog -q ReqHeader:foo[a]" + process p1 -wait shell {grep -q "0 CLI" ${tmpdir}/vlog} shell -match "0 CLI[ ]+- Wr 200 [0-9]+ PONG" \ From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:12 +0000 (UTC) Subject: [6.0] 964d8b784 Try to get this test running on SunOS Message-ID: <20190208123112.4783C61953@lists.varnish-cache.org> commit 964d8b784d48a54366cb8e1582b076bb196c94f4 Author: Federico G. Schwindt Date: Sun Dec 2 23:58:50 2018 +0000 Try to get this test running on SunOS diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index 2f1e41476..2f648ab8f 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -66,18 +66,18 @@ shell -err -expect "Expected positive integer" \ "varnishlog -q {-1}" shell -err -expect "Syntax error in level limit" \ "varnishlog -q {1a}" -shell -err -expect "Expected VSL tag name got ''" \ - "varnishlog -q ''" +shell -err -expect "Expected VSL tag name got '['" \ + {varnishlog -q "[]"} shell -err -expect "Tag name matches zero tags" \ "varnishlog -q foo" shell -err -expect "Tag name is ambiguous" \ "varnishlog -q Resp" -shell -err -expect "Expected string got ''" \ - "varnishlog -q ReqHeader:" +shell -err -expect "Expected string got '['" \ + {varnishlog -q "ReqHeader:[]"} shell -err -expect "Expected integer got ']'" \ - "varnishlog -q ReqHeader:foo[]" + {varnishlog -q "ReqHeader:foo[]"} shell -err -expect "Expected positive integer" \ - "varnishlog -q ReqHeader:foo[a]" + {varnishlog -q "ReqHeader:foo[a]"} process p1 -wait shell {grep -q "0 CLI" ${tmpdir}/vlog} From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:12 +0000 (UTC) Subject: [6.0] 5458c8b83 fix vmod object constructor documentation Message-ID: <20190208123112.6591B6195C@lists.varnish-cache.org> commit 5458c8b8318c49a272e38a13bcae4d312c817880 Author: Nils Goroll Date: Fri Oct 26 12:34:25 2018 +0200 fix vmod object constructor documentation The vmod name got lost in e7c0497404a5b4aadda71aead3b0e7fe946d6993 example before/after diff: - new xshard_param = shard_param() + new xshard_param = directors.shard_param() diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 2ef32e78c..23f372ce7 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -368,11 +368,13 @@ class ProtoType(object): pfx += pfx s = pfx if type(self.st) == s_object: - s += "new x" + self.bname + " = " + s += "new " + self.obj + " = " elif self.retval is not None: s += self.retval.vcl() + " " - if type(self.st) == s_method: + if type(self.st) == s_object: + s += self.st.vcc.modname + "." + self.name + "(" + elif type(self.st) == s_method: s += self.obj + self.bname + "(" else: s += self.name + "(" From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:12 +0000 (UTC) Subject: [6.0] 0fdc2c9f6 Some pylinting Message-ID: <20190208123112.8269A61962@lists.varnish-cache.org> commit 0fdc2c9f6e89a4bcd4019d3e2fc3cf05fb9c31d6 Author: Poul-Henning Kamp Date: Mon Dec 3 09:53:20 2018 +0000 Some pylinting diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 23f372ce7..92863055f 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -120,12 +120,12 @@ CTYPES.update(PRIVS) ####################################################################### -def is_quoted(str): - return len(str) > 2 and str[0] == str[-1] and str[0] in ('"', "'") +def is_quoted(txt): + return len(txt) > 2 and txt[0] == txt[-1] and txt[0] in ('"', "'") -def unquote(str): - assert is_quoted(str) - return str[1:-1] +def unquote(txt): + assert is_quoted(txt) + return txt[1:-1] ####################################################################### @@ -364,17 +364,17 @@ class ProtoType(object): self.args.append(t) def vcl_proto(self, short, pfx=""): - if type(self.st) == s_method: + if isinstance(self.st, MethodStanza): pfx += pfx s = pfx - if type(self.st) == s_object: + if isinstance(self.st, ObjectStanza): s += "new " + self.obj + " = " elif self.retval is not None: s += self.retval.vcl() + " " - if type(self.st) == s_object: + if isinstance(self.st, ObjectStanza): s += self.st.vcc.modname + "." + self.name + "(" - elif type(self.st) == s_method: + elif isinstance(self.st, MethodStanza): s += self.obj + self.bname + "(" else: s += self.name + "(" @@ -414,7 +414,7 @@ class ProtoType(object): write_rst_hdr(fo, s, '-') fo.write("\n::\n\n" + self.vcl_proto(False, pfx=" ") + "\n") - def synopsis(self, fo, man): + def synopsis(self, fo, unused_man): fo.write(self.vcl_proto(True, pfx=" ") + "\n") fo.write(" \n") @@ -489,7 +489,7 @@ class ProtoType(object): ####################################################################### -class stanza(object): +class Stanza(object): def __init__(self, toks, l0, doc, vcc): self.toks = toks self.line = l0 @@ -504,6 +504,9 @@ class stanza(object): self.proto = None self.parse() + def parse(self): + assert "subclass should have defined" == "parse method" + def dump(self): print(type(self), self.line) @@ -524,34 +527,33 @@ class stanza(object): self.rsttail(fo, man) fo.write("\n") - def rsthead(self, fo, man): - if self.proto is None: - return - self.proto.rsthead(fo) + def rsthead(self, fo, unused_man): + if self.proto is not None: + self.proto.rsthead(fo) - def rstmid(self, fo, man): + def rstmid(self, fo, unused_man): fo.write("\n".join(self.doc) + "\n") - def rsttail(self, fo, man): + def rsttail(self, unused_fo, unused_man): return def synopsis(self, fo, man): if self.proto is not None: self.proto.synopsis(fo, man) - def cstuff(self, fo, where): + def cstuff(self, unused_fo, unused_where): return - def cstruct(self, fo, define): + def cstruct(self, unused_fo, unused_define): return - def json(self, jl): + def json(self, unused_jl): return ####################################################################### -class s_module(stanza): +class ModuleStanza(Stanza): ''' $Module modname man_section description ... ''' @@ -617,7 +619,7 @@ class s_module(stanza): fo.write("\n") -class s_abi(stanza): +class ABIStanza(Stanza): ''' $ABI [strict|vrt] ''' @@ -635,7 +637,7 @@ class s_abi(stanza): self.vcc.contents.append(self) -class s_prefix(stanza): +class PrefixStanza(Stanza): ''' $Prefix symbol ''' @@ -646,7 +648,7 @@ class s_prefix(stanza): self.vcc.contents.append(self) -class s_synopsis(stanza): +class SynopsisStanza(Stanza): ''' $Synopsis [auto|manual] ''' @@ -664,7 +666,7 @@ class s_synopsis(stanza): self.vcc.contents.append(self) -class s_event(stanza): +class EventStanza(Stanza): ''' $Event function_name ''' @@ -696,7 +698,7 @@ class s_event(stanza): ]) -class s_function(stanza): +class FunctionStanza(Stanza): def parse(self): self.proto = ProtoType(self) self.rstlbl = "func_" + self.proto.name @@ -716,7 +718,7 @@ class s_function(stanza): self.proto.json(jl[-1], self.proto.cname()) -class s_object(stanza): +class ObjectStanza(Stanza): def parse(self): self.proto = ProtoType(self, retval=False) self.proto.obj = "x" + self.proto.name @@ -741,7 +743,7 @@ class s_object(stanza): for i in self.methods: i.rstfile(fo, man) - def rstmid(self, fo, man): + def rstmid(self, unused_fo, unused_man): return def synopsis(self, fo, man): @@ -794,17 +796,17 @@ class s_object(stanza): jl.append(ll) def dump(self): - super(s_object, self).dump() + super(ObjectStanza, self).dump() for i in self.methods: i.dump() ####################################################################### -class s_method(stanza): +class MethodStanza(Stanza): def parse(self): p = self.vcc.contents[-1] - assert type(p) == s_object + assert isinstance(p, ObjectStanza) self.pfx = p.proto.name self.proto = ProtoType(self, prefix=self.pfx) if not self.proto.bname.startswith("."): @@ -828,18 +830,21 @@ class s_method(stanza): ####################################################################### DISPATCH = { - "Module": s_module, - "Prefix": s_prefix, - "ABI": s_abi, - "Event": s_event, - "Function": s_function, - "Object": s_object, - "Method": s_method, - "Synopsis": s_synopsis, + "Module": ModuleStanza, + "Prefix": PrefixStanza, + "ABI": ABIStanza, + "Event": EventStanza, + "Function": FunctionStanza, + "Object": ObjectStanza, + "Method": MethodStanza, + "Synopsis": SynopsisStanza, } class vcc(object): + + ''' Processing context for a single .vcc file ''' + def __init__(self, inputvcc, rstdir, outputprefix): self.inputfile = inputvcc self.rstdir = rstdir @@ -880,7 +885,7 @@ class vcc(object): m(toks, [c[0], " ".join(c[1:])], d.split('\n'), self) inputline = None - def tokenize(self, str, seps=None, quotes=None): + def tokenize(self, txt, seps=None, quotes=None): if seps is None: seps = "[](){},=" if quotes is None: @@ -889,8 +894,8 @@ class vcc(object): out = [] i = 0 inside = False - while i < len(str): - c = str[i] + while i < len(txt): + c = txt[i] # print("T", [c], quote, inside, i) i += 1 if quote is not None and c == quote: @@ -1053,13 +1058,13 @@ class vcc(object): fo.write("\n") for i in self.contents: - if type(i) == s_object: + if isinstance(i, ObjectStanza): i.cstuff(fo, 'c') i.cstuff(fx, 'o') fx.write("/* Functions */\n") for i in self.contents: - if type(i) == s_function: + if isinstance(i, FunctionStanza): i.cstuff(fo, 'c') i.cstuff(fx, 'o') From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:12 +0000 (UTC) Subject: [6.0] b0a2afed3 Make it possible to change ncurses update rate. Message-ID: <20190208123112.A11F461967@lists.varnish-cache.org> commit b0a2afed38d217b4c4e8e4108f46ce9f54b1772b Author: Lasse Karstensen Date: Thu Aug 16 09:14:21 2018 +0200 Make it possible to change ncurses update rate. This is a squashed commit including lint removal and formatting suggested by @nigoroll. diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index be9e64c79..0ab37c1ac 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -324,7 +324,7 @@ main(int argc, char * const *argv) AN(VSC_Arg(vsc, 'f', "MAIN.cache_hit")); AN(VSC_Arg(vsc, 'f', "MAIN.cache_miss")); } - do_curses(vd, vsc, 1.0); + do_curses(vd, vsc); } else if (xml) do_xml(vd, vsc); diff --git a/bin/varnishstat/varnishstat.h b/bin/varnishstat/varnishstat.h index 6a73ef238..00099ea02 100644 --- a/bin/varnishstat/varnishstat.h +++ b/bin/varnishstat/varnishstat.h @@ -35,4 +35,4 @@ #include "vas.h" #include "vcs.h" -void do_curses(struct vsm *, struct vsc *, double); +void do_curses(struct vsm *, struct vsc *); diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index ad9c22135..b44ee70d8 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -435,6 +435,8 @@ draw_status(void) mvwprintw(w_status, 1, 0, "Uptime child: "); running(w_status, up_chld, VSM_WRK_RUNNING); + mvwprintw(w_status, 0, 55, "Refresh interval: %.3fs", interval); + if (COLS > 70) { mvwprintw(w_status, 0, getmaxx(w_status) - 37, "Hitrate n: %8u %8u %8u", hitrate.hr_10.n, hitrate.hr_100.n, @@ -899,6 +901,14 @@ handle_keypress(int ch) current = n_ptarray - 1; page_start = (current - l_points) + 1; break; + case '+': + interval += 0.1; + break; + case '-': + interval -= 0.1; + if (interval < 0.1) + interval = 0.1; + break; case 'v': verbosity = VSC_ChangeLevel(verbosity, 1); rebuild = 1; @@ -980,14 +990,12 @@ delpt(void *priv, const struct VSC_point *const vpt) } void -do_curses(struct vsm *vsm, struct vsc *vsc, double delay) +do_curses(struct vsm *vsm, struct vsc *vsc) { long t; int ch; double now; - interval = delay; - verbosity = VSC_ChangeLevel(NULL, 0); initscr(); diff --git a/doc/sphinx/reference/varnishstat.rst b/doc/sphinx/reference/varnishstat.rst index 89f1f2732..023a6d784 100644 --- a/doc/sphinx/reference/varnishstat.rst +++ b/doc/sphinx/reference/varnishstat.rst @@ -109,6 +109,13 @@ The following keys control the interactive display: Sample now. +<+> + Increase refresh interval. + +<-> + Decrease refresh interval. + + OUTPUTS ======= From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:12 +0000 (UTC) Subject: [6.0] ab965b7f4 Add a notification field for temporary messages. Message-ID: <20190208123112.BBA986196D@lists.varnish-cache.org> commit ab965b7f40b5a15d8aec36d09e706c696967fb5d Author: Lasse Karstensen Date: Sun Dec 2 22:09:32 2018 +0100 Add a notification field for temporary messages. diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index b44ee70d8..4ae43643d 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -120,6 +120,10 @@ static double t_sample = 0.; static double interval = 1.; static unsigned vsm_status = 0; +#define NOTIF_MAXLEN 256 +static char notification_message[NOTIF_MAXLEN] = ""; +static double notification_eol = 0.0; + static void init_hitrate(void) { @@ -435,7 +439,8 @@ draw_status(void) mvwprintw(w_status, 1, 0, "Uptime child: "); running(w_status, up_chld, VSM_WRK_RUNNING); - mvwprintw(w_status, 0, 55, "Refresh interval: %.3fs", interval); + if (VTIM_mono() < notification_eol) + mvwaddstr(w_status, 2, 0, notification_message); if (COLS > 70) { mvwprintw(w_status, 0, getmaxx(w_status) - 37, @@ -903,11 +908,19 @@ handle_keypress(int ch) break; case '+': interval += 0.1; + (void)snprintf(notification_message, NOTIF_MAXLEN, + "Refresh interval set to %.1f seconds.", interval); + + notification_eol = VTIM_mono() + 1.25; break; case '-': interval -= 0.1; if (interval < 0.1) interval = 0.1; + (void)snprintf(notification_message, NOTIF_MAXLEN, + "Refresh interval set to %.1f seconds.", interval); + + notification_eol = VTIM_mono() + 1.25; break; case 'v': verbosity = VSC_ChangeLevel(verbosity, 1); From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:12 +0000 (UTC) Subject: [6.0] bc039486e Add test coverage for ncurses update interval. Message-ID: <20190208123112.D85D961978@lists.varnish-cache.org> commit bc039486e4932eed9c31bfd3556f21c9706ae369 Author: Lasse Karstensen Date: Sun Dec 2 22:29:16 2018 +0100 Add test coverage for ncurses update interval. diff --git a/bin/varnishtest/tests/u00008.vtc b/bin/varnishtest/tests/u00008.vtc index 5e1beff3e..0c35fb3fc 100644 --- a/bin/varnishtest/tests/u00008.vtc +++ b/bin/varnishtest/tests/u00008.vtc @@ -1,4 +1,4 @@ -varnishtest "trivial run of varnistat in curses mode" +varnishtest "trivial run of varnishstat in curses mode" server s1 -repeat 4 { rxreq @@ -24,6 +24,14 @@ client c1 { process p1 -expect-text 0 0 "MAIN.s_sess" process p1 -screen_dump +process p1 -write {+} +process p1 -screen_dump +process p1 -expect-text 0 0 "Refresh interval set to 1.1 seconds." + +process p1 -write {-} +process p1 -screen_dump +process p1 -expect-text 0 0 "Refresh interval set to 1.0 seconds." + process p1 -write {vG} process p1 -expect-text 0 0 "VBE.vcl1.s1.req" process p1 -expect-text 0 0 "DIAG" From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:13 +0000 (UTC) Subject: [6.0] 14cfe437d vtim-ification Message-ID: <20190208123113.080DF6197E@lists.varnish-cache.org> commit 14cfe437dd177ef4170f2175ab92e1b6c0a57dd5 Author: Nils Goroll Date: Mon Dec 3 12:30:13 2018 +0100 vtim-ification Inclused re-declaration fixes for some places which I got wrong in the past. Conflicts: bin/varnishd/cache/cache_objhead.h diff --git a/bin/varnishd/cache/cache_obj.h b/bin/varnishd/cache/cache_obj.h index bdb6795bd..5229b7e00 100644 --- a/bin/varnishd/cache/cache_obj.h +++ b/bin/varnishd/cache/cache_obj.h @@ -46,7 +46,7 @@ typedef const void *objgetattr_f(struct worker *, struct objcore *, enum obj_attr attr, ssize_t *len); typedef void *objsetattr_f(struct worker *, struct objcore *, enum obj_attr attr, ssize_t len, const void *ptr); -typedef void objtouch_f(struct worker *, struct objcore *, double now); +typedef void objtouch_f(struct worker *, struct objcore *, vtim_real now); struct obj_methods { objfree_f *objfree; diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h index 6bcb7b301..9e90d5b32 100644 --- a/bin/varnishd/cache/cache_objhead.h +++ b/bin/varnishd/cache/cache_objhead.h @@ -72,7 +72,7 @@ enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **, int always_insert); void HSH_Ref(struct objcore *o); void HSH_AddString(struct req *, void *ctx, const char *str); -unsigned HSH_Purge(struct worker *, struct objhead *, double ttl_now, - double ttl, double grace, double keep); +unsigned HSH_Purge(struct worker *, struct objhead *, vtim_real ttl_now, + vtim_dur ttl, vtim_dur grace, vtim_dur keep); struct objcore *HSH_Private(const struct worker *wrk); void HSH_Abandon(struct objcore *oc); diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index ddecb9c4e..c9341fdd0 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -62,11 +62,11 @@ */ void -RFC2616_Ttl(struct busyobj *bo, double now, double *t_origin, +RFC2616_Ttl(struct busyobj *bo, vtim_real now, vtim_real *t_origin, float *ttl, float *grace, float *keep) { unsigned max_age, age; - double h_date, h_expires; + vtim_real h_date, h_expires; const char *p; const struct http *hp; @@ -258,7 +258,7 @@ int RFC2616_Do_Cond(const struct req *req) { const char *p, *e; - double ims, lm; + vtim_real ims, lm; /* * We MUST ignore If-Modified-Since if we have an If-None-Match diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index b6faa8222..4141dfa21 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -253,7 +253,7 @@ HTC_RxPipeline(struct http_conn *htc, void *p) enum htc_status_e HTC_RxStuff(struct http_conn *htc, htc_complete_f *func, - double *t1, double *t2, double ti, double tn, int maxbytes) + vtim_real *t1, vtim_real *t2, vtim_real ti, vtim_real tn, int maxbytes) { vtim_dur tmo; vtim_real now; diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index e4db8158f..83e22f2f9 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -373,8 +373,8 @@ VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) } void -VSLb_ts(struct vsl_log *vsl, const char *event, double first, double *pprev, - double now) +VSLb_ts(struct vsl_log *vsl, const char *event, vtim_real first, + vtim_real *pprev, vtim_real now) { /* XXX: Make an option to turn off some unnecessary timestamp diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index dbc12c685..9d623ef9c 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -66,7 +66,8 @@ struct pfd { /*-------------------------------------------------------------------- */ -typedef int cp_open_f(const struct conn_pool *, double tmo, const void **privp); +typedef int cp_open_f(const struct conn_pool *, vtim_dur tmo, + const void **privp); typedef void cp_close_f(struct pfd *); typedef int cp_cmp_f(const struct conn_pool *, const void *priv); typedef void cp_name_f(const struct pfd *, char *, unsigned, char *, unsigned); @@ -100,7 +101,7 @@ struct conn_pool { int n_used; - double holddown; + vtim_mono holddown; int holddown_errno; }; @@ -158,7 +159,7 @@ PFD_RemoteName(const struct pfd *p, char *abuf, unsigned alen, char *pbuf, */ static void v_matchproto_(waiter_handle_f) -vcp_handle(struct waited *w, enum wait_event ev, double now) +vcp_handle(struct waited *w, enum wait_event ev, vtim_real now) { struct pfd *pfd; struct conn_pool *cp; @@ -377,10 +378,10 @@ VCP_Recycle(const struct worker *wrk, struct pfd **pfdp) */ static int -VCP_Open(struct conn_pool *cp, double tmo, const void **privp, int *err) +VCP_Open(struct conn_pool *cp, vtim_dur tmo, const void **privp, int *err) { int r; - double h; + vtim_mono h; CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); AN(err); @@ -487,7 +488,7 @@ VCP_Close(struct pfd **pfdp) */ static struct pfd * -VCP_Get(struct conn_pool *cp, double tmo, struct worker *wrk, +VCP_Get(struct conn_pool *cp, vtim_dur tmo, struct worker *wrk, unsigned force_fresh, int *err) { struct pfd *pfd; @@ -579,13 +580,13 @@ struct vtp_cs { }; static inline int -tmo2msec(double tmo) +tmo2msec(vtim_dur tmo) { return ( (int)floor(tmo * 1000.0) ); } static int v_matchproto_(cp_open_f) -vtp_open(const struct conn_pool *cp, double tmo, const void **privp) +vtp_open(const struct conn_pool *cp, vtim_dur tmo, const void **privp) { int s; int msec; @@ -673,7 +674,7 @@ static const struct cp_methods vtp_methods = { */ static int v_matchproto_(cp_open_f) -vus_open(const struct conn_pool *cp, double tmo, const void **privp) +vus_open(const struct conn_pool *cp, vtim_dur tmo, const void **privp) { int s; int msec; @@ -801,7 +802,7 @@ VTP_Rel(struct tcp_pool **tpp) */ int -VTP_Open(struct tcp_pool *tp, double tmo, const void **privp, int *err) +VTP_Open(struct tcp_pool *tp, vtim_dur tmo, const void **privp, int *err) { return (VCP_Open(tp->cp, tmo, privp, err)); } diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index 57564ba96..3c872336f 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -72,7 +72,7 @@ void VTP_Rel(struct tcp_pool **); * the pool is destroyed and all cached connections closed. */ -int VTP_Open(struct tcp_pool *, double tmo, const void **, int*); +int VTP_Open(struct tcp_pool *, vtim_dur tmo, const void **, int*); /* * Open a new connection and return the adress used. * errno will be returned in the last argument. diff --git a/bin/varnishd/cache/cache_transport.h b/bin/varnishd/cache/cache_transport.h index 5de0920a3..5da5e3501 100644 --- a/bin/varnishd/cache/cache_transport.h +++ b/bin/varnishd/cache/cache_transport.h @@ -79,7 +79,7 @@ void VPX_Send_Proxy(int fd, int version, const struct sess *); /* cache_session.c */ struct sess *SES_New(struct pool *); -void SES_Delete(struct sess *, enum sess_close reason, double now); +void SES_Delete(struct sess *, enum sess_close reason, vtim_real now); void SES_Close(struct sess *, enum sess_close reason); void SES_SetTransport(struct worker *, struct sess *, struct req *, const struct transport *); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 365b821fc..45aba733a 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -261,7 +261,7 @@ void ObjSetState(struct worker *, const struct objcore *, enum boc_state_e next); void ObjWaitState(const struct objcore *, enum boc_state_e want); void ObjTrimStore(struct worker *, struct objcore *); -void ObjTouch(struct worker *, struct objcore *, double now); +void ObjTouch(struct worker *, struct objcore *, vtim_real now); void ObjFreeObj(struct worker *, struct objcore *); void ObjSlim(struct worker *, struct objcore *); void *ObjSetAttr(struct worker *, struct objcore *, enum obj_attr, @@ -347,7 +347,7 @@ const char * HTC_Status(enum htc_status_e); void HTC_RxInit(struct http_conn *htc, struct ws *ws); void HTC_RxPipeline(struct http_conn *htc, void *); enum htc_status_e HTC_RxStuff(struct http_conn *, htc_complete_f *, - double *t1, double *t2, double ti, double tn, int maxbytes); + vtim_real *t1, vtim_real *t2, vtim_real ti, vtim_real tn, int maxbytes); #define SESS_ATTR(UP, low, typ, len) \ int SES_Set_##low(const struct sess *sp, const typ *src); \ diff --git a/bin/varnishd/fuzzers/esi_parse_fuzzer.c b/bin/varnishd/fuzzers/esi_parse_fuzzer.c index c09c888ba..972ffa0b1 100644 --- a/bin/varnishd/fuzzers/esi_parse_fuzzer.c +++ b/bin/varnishd/fuzzers/esi_parse_fuzzer.c @@ -56,8 +56,8 @@ VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) } void -VSLb_ts(struct vsl_log *l, const char *event, double first, double *pprev, - double now) +VSLb_ts(struct vsl_log *l, const char *event, vtim_real first, vtim_real *pprev, + vtim_real now) { (void)l; (void)event; diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h index 531f640d7..4a0d8a7d5 100644 --- a/bin/varnishd/waiter/waiter.h +++ b/bin/varnishd/waiter/waiter.h @@ -52,7 +52,7 @@ enum wait_event { WAITER_CLOSE }; -typedef void waiter_handle_f(struct waited *, enum wait_event, double now); +typedef void waiter_handle_f(struct waited *, enum wait_event, vtim_real now); struct waited { unsigned magic; @@ -62,8 +62,8 @@ struct waited { void *priv1; uintptr_t priv2; waiter_handle_f *func; - volatile double *tmo; - double idle; + volatile vtim_real *tmo; + vtim_real idle; }; /* cache_waiter.c */ diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index 4ae43643d..4b1703e6c 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -122,7 +122,7 @@ static unsigned vsm_status = 0; #define NOTIF_MAXLEN 256 static char notification_message[NOTIF_MAXLEN] = ""; -static double notification_eol = 0.0; +static vtim_mono notification_eol = 0.0; static void init_hitrate(void) diff --git a/include/vnum.h b/include/vnum.h index b24545015..266f156f5 100644 --- a/include/vnum.h +++ b/include/vnum.h @@ -31,5 +31,5 @@ /* from libvarnish/vnum.c */ double VNUM(const char *p); double VNUMpfx(const char *p, const char **e); -double VNUM_duration(const char *p); +vtim_dur VNUM_duration(const char *p); const char *VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel); diff --git a/include/vtim.h b/include/vtim.h index 1968bc568..b1898a39d 100644 --- a/include/vtim.h +++ b/include/vtim.h @@ -31,10 +31,10 @@ /* from libvarnish/vtim.c */ extern unsigned VTIM_postel; #define VTIM_FORMAT_SIZE 30 -void VTIM_format(double t, char *p); -double VTIM_parse(const char *p); +void VTIM_format(vtim_real t, char *p); +vtim_real VTIM_parse(const char *p); vtim_mono VTIM_mono(void); vtim_real VTIM_real(void); void VTIM_sleep(vtim_dur t); -struct timespec VTIM_timespec(vtim_dur t); -struct timeval VTIM_timeval(vtim_dur t); +struct timespec VTIM_timespec(vtim_real t); +struct timeval VTIM_timeval(vtim_real t); diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index 01f044fff..a936e08d6 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -165,7 +165,7 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy) int c, chosen = -1; uint32_t ringsz; VCL_BACKEND be; - double changed; + vtim_real changed; struct shard_be_info *sbe; AN(state); @@ -312,7 +312,7 @@ sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo, unsigned retval = 0; VCL_BACKEND be; unsigned u; - double c; + vtim_real c; CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index 84895c365..78f8d9c40 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -180,7 +180,7 @@ vdir_any_healthy(struct vdir *vd, const struct busyobj *bo, double *changed) unsigned retval = 0; VCL_BACKEND be; unsigned u; - double c; + vtim_real c; CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:13 +0000 (UTC) Subject: [6.0] 9e77fc812 Add some testing bits for VSL -R Message-ID: <20190208123113.2A84461986@lists.varnish-cache.org> commit 9e77fc8121bf58c4d3cffae4b14c92352c2e495e Author: Dag Haavi Finstad Date: Mon Dec 3 11:39:42 2018 +0100 Add some testing bits for VSL -R diff --git a/bin/varnishtest/tests/u00003.vtc b/bin/varnishtest/tests/u00003.vtc index e18d9b912..cd4a4880a 100644 --- a/bin/varnishtest/tests/u00003.vtc +++ b/bin/varnishtest/tests/u00003.vtc @@ -26,7 +26,7 @@ varnish v1 -vcl+backend { shell { varnishncsa -n ${v1_name} -D -P ${tmpdir}/ncsa.pid \ - -w ${tmpdir}/ncsa.log + -w ${tmpdir}/ncsa.log -R 100/s } delay 1 diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index 2f648ab8f..9ffb331e7 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -12,7 +12,7 @@ process p1 { } -start shell { exec varnishlog -n ${v1_name} -D -P ${tmpdir}/vlog.pid \ - -w ${tmpdir}/vlog.bin + -w ${tmpdir}/vlog.bin -R 10/s } shell -match "Usage: .*varnishlog " \ @@ -79,6 +79,25 @@ shell -err -expect "Expected integer got ']'" \ shell -err -expect "Expected positive integer" \ {varnishlog -q "ReqHeader:foo[a]"} +shell -err -expect "-R: Syntax error" \ + "varnishlog -R 1foo" +shell -err -expect "-R: Range error" \ + "varnishlog -R 0" +shell -err -expect "-R: Range error" \ + "varnishlog -R -1" +shell -err -expect "-R: Range error" \ + "varnishlog -R 3000000000" +shell -err -expect "-R: Syntax error: Invalid duration" \ + "varnishlog -R 10/0s" +shell -err -expect "-R: Syntax error: Invalid duration" \ + "varnishlog -R 10/-10s" +shell -err -expect "-R: Syntax error: Invalid duration" \ + "varnishlog -R 10/1q" +shell -err -expect "-R: Syntax error: Invalid duration" \ + "varnishlog -R 10/11f09s" +shell -err -expect "-R: Syntax error" \ + "varnishlog -R 1000000000/1000000000000000000000000000000s" + process p1 -wait shell {grep -q "0 CLI" ${tmpdir}/vlog} shell -match "0 CLI[ ]+- Wr 200 [0-9]+ PONG" \ From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:13 +0000 (UTC) Subject: [6.0] ceed5fe67 Tolerate null IP addresses for ACL matches Message-ID: <20190208123113.4E5F261996@lists.varnish-cache.org> commit ceed5fe671111abeb05676ba770ba5be7e89fd4a Author: Dridi Boukelmoune Date: Wed Nov 28 11:30:18 2018 +0100 Tolerate null IP addresses for ACL matches A vmod may return a null IP. This relaxes the check in VRT_acl_match to fail the transaction instead of crashing. Refs #2842 diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 9f838132f..4ff427f35 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -83,6 +83,10 @@ VRT_acl_match(VRT_CTX, VCL_ACL acl, VCL_IP ip) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(acl, VRT_ACL_MAGIC); + if (ip == NULL) { + VRT_fail(ctx, "Cannot match a null IP address"); + return (0); + } assert(VSA_Sane(ip)); return (acl->match(ctx, ip)); } diff --git a/bin/varnishtest/tests/r01504.vtc b/bin/varnishtest/tests/r01504.vtc index 02182a519..9767ff0b4 100644 --- a/bin/varnishtest/tests/r01504.vtc +++ b/bin/varnishtest/tests/r01504.vtc @@ -1,10 +1,26 @@ -varnishtest "unreferenced acls" +varnishtest "unreferenced or null acls" varnish v1 -arg "-p vcc_err_unref=off" -vcl { + import vtc; backend s1 { - .host = "127.0.0.1"; + .host = "${bad_backend}"; } acl foo { "127.0.0.1"; } -} + acl bar { + "127.0.0.1"; + } + sub vcl_recv { + if (vtc.no_ip() ~ bar) { + return (synth(200)); + } + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 503 + expect resp.body ~ "VCL failed" +} -run diff --git a/lib/libvmod_vtc/vmod.vcc b/lib/libvmod_vtc/vmod.vcc index 94f8f75b6..248f4b2d6 100644 --- a/lib/libvmod_vtc/vmod.vcc +++ b/lib/libvmod_vtc/vmod.vcc @@ -66,6 +66,10 @@ $Function STEVEDORE no_stevedore() Fails at storage selection. +$Function IP no_ip() + +Returns a null IP address, not even a bogo_ip. + $Function VOID panic(STRING_LIST) It can be useful to crash the child process in order to test the robustness diff --git a/lib/libvmod_vtc/vmod_vtc.c b/lib/libvmod_vtc/vmod_vtc.c index 6b6501adf..197fe81f0 100644 --- a/lib/libvmod_vtc/vmod_vtc.c +++ b/lib/libvmod_vtc/vmod_vtc.c @@ -89,6 +89,14 @@ vmod_no_stevedore(VRT_CTX) return (NULL); } +VCL_IP v_matchproto_(td_vtc_no_ip) +vmod_no_ip(VRT_CTX) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + return (NULL); +} + /*--------------------------------------------------------------------*/ VCL_VOID v_matchproto_(td_vtc_panic) From dridi.boukelmoune at gmail.com Fri Feb 8 12:31:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 12:31:13 +0000 (UTC) Subject: [6.0] 1845b8937 vsl -R: Add strtol underflow/overflow handling Message-ID: <20190208123113.71229619AD@lists.varnish-cache.org> commit 1845b8937c45df862bc36138a86b977f058392f8 Author: Dag Haavi Finstad Date: Mon Dec 3 14:31:35 2018 +0100 vsl -R: Add strtol underflow/overflow handling Conflicts: lib/libvarnishapi/vsl_arg.c For some reason it was missing errno.h... diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c index 2929b4ccf..25402e583 100644 --- a/lib/libvarnishapi/vsl_arg.c +++ b/lib/libvarnishapi/vsl_arg.c @@ -31,6 +31,7 @@ #include "config.h" #include +#include #include #include #include @@ -310,7 +311,10 @@ vsl_R_arg(struct VSL_data *vsl, const char *arg) AN(arg); CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC); + errno = 0; l = strtol(arg, &p, 0); + if ((l == LONG_MIN || l == LONG_MAX) && errno == ERANGE) + return (vsl_diag(vsl, "-R: Range error")); if (l <= 0 || l > INT_MAX) return (vsl_diag(vsl, "-R: Range error")); vsl->R_opt_l = l; From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:09 +0000 (UTC) Subject: [6.0] 58d10b093 rename OC_F_PASS -> OC_F_HFM - zero code change Message-ID: <20190208131309.574186473B@lists.varnish-cache.org> commit 58d10b0936fa659a6d7954707b9845d82f51f13a Author: Nils Goroll Date: Mon Dec 3 21:11:05 2018 +0100 rename OC_F_PASS -> OC_F_HFM - zero code change Now that we got hit-for-pass (hfp) and hit-for-miss (hfm), we should name the latter consistently to avoid unnecessary confusion diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index be6dc162c..4193c4e2c 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -127,7 +127,7 @@ vbf_beresp2obj(struct busyobj *bo) l += l2; if (bo->uncacheable) - bo->fetch_objcore->flags |= OC_F_PASS; + bo->fetch_objcore->flags |= OC_F_HFM; if (!vbf_allocobj(bo, l)) { if (vary != NULL) @@ -196,7 +196,7 @@ vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo) if (bo->stale_oc != NULL && ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND) && (bo->stale_oc->boc != NULL || ObjGetLen(wrk, bo->stale_oc) != 0)) { - AZ(bo->stale_oc->flags & (OC_F_PASS|OC_F_PRIVATE)); + AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE)); q = HTTP_GetHdrPack(bo->wrk, bo->stale_oc, H_Last_Modified); if (q != NULL) http_PrintfHeader(bo->bereq0, @@ -355,7 +355,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) if (http_IsStatus(bo->beresp, 304)) { if (bo->stale_oc != NULL && ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) { - AZ(bo->stale_oc->flags & (OC_F_PASS|OC_F_PRIVATE)); + AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE)); if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) { /* * If we changed the gzip status of the object @@ -418,7 +418,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) wrk->handling = VCL_RET_DELIVER; } if (bo->do_pass || bo->uncacheable) - bo->fetch_objcore->flags |= OC_F_PASS; + bo->fetch_objcore->flags |= OC_F_HFM; assert(wrk->handling == VCL_RET_DELIVER); @@ -454,7 +454,7 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) * We don't fail the fetch, in order for HitMiss * objects to be created. */ - AN(vfc->oc->flags & OC_F_PASS); + AN(vfc->oc->flags & OC_F_HFM); VSLb(wrk->vsl, SLT_Debug, "Fetch: Pass delivery abandoned"); bo->htc->doclose = SC_RX_BODY; @@ -608,7 +608,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_##U, 1); #include "tbl/obj_attr.h" - if (!(bo->fetch_objcore->flags & OC_F_PASS) && + if (!(bo->fetch_objcore->flags & OC_F_HFM) && http_IsStatus(bo->beresp, 200) && ( http_GetHdr(bo->beresp, H_Last_Modified, &p) || http_GetHdr(bo->beresp, H_ETag, &p))) diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index bb32d4fc2..f8029cb98 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -445,7 +445,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, ObjGetXID(wrk, oc), EXP_Dttl(req, oc)); oc = NULL; retval = HSH_MISS; - } else if (oc->flags & OC_F_PASS) { + } else if (oc->flags & OC_F_HFM) { wrk->stats->cache_hitmiss++; VSLb(req->vsl, SLT_HitMiss, "%u %.6f", ObjGetXID(wrk, oc), EXP_Dttl(req, oc)); @@ -472,7 +472,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, } } - if (exp_oc != NULL && exp_oc->flags & OC_F_PASS) { + if (exp_oc != NULL && exp_oc->flags & OC_F_HFM) { wrk->stats->cache_hitmiss++; VSLb(req->vsl, SLT_HitMiss, "%u %.6f", ObjGetXID(wrk, exp_oc), EXP_Dttl(req, exp_oc)); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index cd01c2976..cafdd010f 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -401,7 +401,7 @@ cnt_transmit(struct worker *wrk, struct req *req) http_Unset(req->resp, H_Content_Length); } else if (clval >= 0 && clval == req->resp_len) { /* Reuse C-L header */ - } else if (head && req->objcore->flags & OC_F_PASS) { + } else if (head && req->objcore->flags & OC_F_HFM) { /* * Don't touch C-L header (debatable) * @@ -423,9 +423,9 @@ cnt_transmit(struct worker *wrk, struct req *req) VSLb_ts_req(req, "Resp", W_TIM_real(wrk)); - if (req->objcore->flags & (OC_F_PRIVATE | OC_F_PASS | OC_F_HFP)) { + if (req->objcore->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)) { if (req->objcore->flags & OC_F_HFP) - AN(req->objcore->flags & OC_F_PASS); + AN(req->objcore->flags & OC_F_HFM); if (boc != NULL) { HSH_Abandon(req->objcore); ObjWaitState(req->objcore, BOS_FINISHED); @@ -534,7 +534,7 @@ cnt_lookup(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AZ(oc->flags & OC_F_BUSY); req->objcore = oc; - AZ(oc->flags & OC_F_PASS); + AZ(oc->flags & OC_F_HFM); VSLb(req->vsl, SLT_Hit, "%u %.6f %.6f %.6f", ObjGetXID(wrk, req->objcore), @@ -547,7 +547,7 @@ cnt_lookup(struct worker *wrk, struct req *req) switch (wrk->handling) { case VCL_RET_DELIVER: if (busy != NULL) { - AZ(oc->flags & OC_F_PASS); + AZ(oc->flags & OC_F_HFM); CHECK_OBJ_NOTNULL(busy->boc, BOC_MAGIC); // XXX: shouldn't we go to miss? VBF_Fetch(wrk, req, busy, oc, VBF_BACKGROUND); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 405bb4310..d3787b193 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -823,7 +823,7 @@ VRT_r_obj_uncacheable(VRT_CTX) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC); - return (ctx->req->objcore->flags & OC_F_PASS ? 1 : 0); + return (ctx->req->objcore->flags & OC_F_HFM ? 1 : 0); } /*--------------------------------------------------------------------*/ diff --git a/include/tbl/oc_flags.h b/include/tbl/oc_flags.h index fc8788b64..84fd86f80 100644 --- a/include/tbl/oc_flags.h +++ b/include/tbl/oc_flags.h @@ -30,7 +30,7 @@ OC_FLAG(PURGED, purged, (1<<0)) //lint !e835 OC_FLAG(BUSY, busy, (1<<1)) -OC_FLAG(PASS, pass, (1<<2)) +OC_FLAG(HFM, hfm, (1<<2)) OC_FLAG(HFP, hfp, (1<<3)) OC_FLAG(ABANDON, abandon, (1<<4)) OC_FLAG(PRIVATE, private, (1<<5)) From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:09 +0000 (UTC) Subject: [6.0] 3bb4686b0 merge HSH_EXP and HSH_EXPBUSY into HSH_GRACE Message-ID: <20190208131309.6DFB86473E@lists.varnish-cache.org> commit 3bb4686b07d5a52c24ca9c887caf6ed536c9f01d Author: Nils Goroll Date: Mon Dec 3 21:17:05 2018 +0100 merge HSH_EXP and HSH_EXPBUSY into HSH_GRACE they are the same case diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index f8029cb98..4ee3dbce6 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -496,7 +496,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, } else { if (exp_oc->hits < LONG_MAX) exp_oc->hits++; - retval = HSH_EXPBUSY; + retval = HSH_GRACE; } } else { Lck_Unlock(&oh->mtx); @@ -518,7 +518,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, assert(HSH_DerefObjHead(wrk, &oh)); if (exp_oc->hits < LONG_MAX) exp_oc->hits++; - return (HSH_EXP); + return (HSH_GRACE); } /* There are one or more busy objects, wait for them */ diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index cafdd010f..65c90e6b7 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -556,7 +556,7 @@ cnt_lookup(struct worker *wrk, struct req *req) } wrk->stats->cache_hit++; req->is_hit = 1; - if (lr == HSH_EXP || lr == HSH_EXPBUSY) + if (lr == HSH_GRACE) wrk->stats->cache_hit_grace++; req->req_step = R_STP_DELIVER; return (REQ_FSM_MORE); diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index d6cad78ae..aa4f284f4 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -53,8 +53,7 @@ enum lookup_e { HSH_MISS, HSH_BUSY, HSH_HIT, - HSH_EXP, - HSH_EXPBUSY + HSH_GRACE }; /* mgt_hash.c */ From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:09 +0000 (UTC) Subject: [6.0] 91c3dfc03 Introduce HSH_HITPASS and HSH_HITMISS and clarify grace/keep HSH_HITMISS Message-ID: <20190208131309.893B964741@lists.varnish-cache.org> commit 91c3dfc03b2369fe2d8efe27376a638f88375cbe Author: Nils Goroll Date: Mon Dec 3 21:34:12 2018 +0100 Introduce HSH_HITPASS and HSH_HITMISS and clarify grace/keep HSH_HITMISS diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 4ee3dbce6..66fd223f5 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -444,14 +444,14 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, VSLb(req->vsl, SLT_HitPass, "%u %.6f", ObjGetXID(wrk, oc), EXP_Dttl(req, oc)); oc = NULL; - retval = HSH_MISS; + retval = HSH_HITPASS; } else if (oc->flags & OC_F_HFM) { wrk->stats->cache_hitmiss++; VSLb(req->vsl, SLT_HitMiss, "%u %.6f", ObjGetXID(wrk, oc), EXP_Dttl(req, oc)); *bocp = hsh_insert_busyobj(wrk, oh); oc->refcnt++; - retval = HSH_MISS; + retval = HSH_HITMISS; } else { oc->refcnt++; if (oc->hits < LONG_MAX) @@ -473,11 +473,17 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, } if (exp_oc != NULL && exp_oc->flags & OC_F_HFM) { + /* + * expired HFM ("grace/keep HFM") + * + * XXX should HFM objects actually have grace/keep ? + */ wrk->stats->cache_hitmiss++; VSLb(req->vsl, SLT_HitMiss, "%u %.6f", ObjGetXID(wrk, exp_oc), EXP_Dttl(req, exp_oc)); - exp_oc = NULL; - busy_found = 0; + *bocp = hsh_insert_busyobj(wrk, oh); + Lck_Unlock(&oh->mtx); + return (HSH_HITMISS); } if (!busy_found) { diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 65c90e6b7..85af28381 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -516,21 +516,23 @@ cnt_lookup(struct worker *wrk, struct req *req) } AZ(req->objcore); - if (lr == HSH_MISS) { - if (busy != NULL) { - /* hitmiss, out-of-grace or ordinary miss */ - AN(busy->flags & OC_F_BUSY); - req->objcore = busy; - req->stale_oc = oc; - req->req_step = R_STP_MISS; - } else { - /* hitpass */ - AZ(oc); - req->req_step = R_STP_PASS; - } + if (lr == HSH_MISS || lr == HSH_HITMISS) { + AN(busy); + AN(busy->flags & OC_F_BUSY); + req->objcore = busy; + req->stale_oc = oc; + req->req_step = R_STP_MISS; + return (REQ_FSM_MORE); + } + if (lr == HSH_HITPASS) { + AZ(busy); + AZ(oc); + req->req_step = R_STP_PASS; return (REQ_FSM_MORE); } + assert(lr == HSH_HIT || lr == HSH_GRACE); + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AZ(oc->flags & OC_F_BUSY); req->objcore = oc; diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index aa4f284f4..9434ce874 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -53,6 +53,8 @@ enum lookup_e { HSH_MISS, HSH_BUSY, HSH_HIT, + HSH_HITMISS, + HSH_HITPASS, HSH_GRACE }; From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:09 +0000 (UTC) Subject: [6.0] fab3b3e19 reduce amount of work done under the objhead mutex Message-ID: <20190208131309.A4D9C64746@lists.varnish-cache.org> commit fab3b3e197b73f91f966761ca751390b02034ce1 Author: Nils Goroll Date: Mon Dec 3 22:11:53 2018 +0100 reduce amount of work done under the objhead mutex in particular, move logging diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 66fd223f5..64c12686d 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -348,6 +348,8 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, int busy_found; enum lookup_e retval; const uint8_t *vary; + unsigned xid = 0; + float dttl = 0.0; AN(ocp); *ocp = NULL; @@ -440,15 +442,13 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, assert(oh->refcnt > 1); assert(oc->objhead == oh); if (oc->flags & OC_F_HFP) { - wrk->stats->cache_hitpass++; - VSLb(req->vsl, SLT_HitPass, "%u %.6f", - ObjGetXID(wrk, oc), EXP_Dttl(req, oc)); + xid = ObjGetXID(wrk, oc); + dttl = EXP_Dttl(req, oc); oc = NULL; retval = HSH_HITPASS; } else if (oc->flags & OC_F_HFM) { - wrk->stats->cache_hitmiss++; - VSLb(req->vsl, SLT_HitMiss, "%u %.6f", - ObjGetXID(wrk, oc), EXP_Dttl(req, oc)); + xid = ObjGetXID(wrk, oc); + dttl = EXP_Dttl(req, oc); *bocp = hsh_insert_busyobj(wrk, oh); oc->refcnt++; retval = HSH_HITMISS; @@ -462,6 +462,24 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, *ocp = oc; if (*bocp == NULL) assert(HSH_DerefObjHead(wrk, &oh)); + + switch (retval) { + case HSH_HITPASS: + wrk->stats->cache_hitpass++; + VSLb(req->vsl, SLT_HitPass, "%u %.6f", + xid, dttl); + break; + case HSH_HITMISS: + wrk->stats->cache_hitmiss++; + VSLb(req->vsl, SLT_HitMiss, "%u %.6f", + xid, dttl); + break; + case HSH_HIT: + break; + default: + INCOMPL(); + } + return (retval); } if (EXP_Ttl(NULL, oc) < req->t_req && /* ignore req.ttl */ @@ -478,11 +496,13 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, * * XXX should HFM objects actually have grace/keep ? */ - wrk->stats->cache_hitmiss++; - VSLb(req->vsl, SLT_HitMiss, "%u %.6f", ObjGetXID(wrk, exp_oc), - EXP_Dttl(req, exp_oc)); + xid = ObjGetXID(wrk, exp_oc); + dttl = EXP_Dttl(req, exp_oc); *bocp = hsh_insert_busyobj(wrk, oh); Lck_Unlock(&oh->mtx); + + wrk->stats->cache_hitmiss++; + VSLb(req->vsl, SLT_HitMiss, "%u %.6f", xid, dttl); return (HSH_HITMISS); } @@ -528,13 +548,11 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, } /* There are one or more busy objects, wait for them */ + VTAILQ_INSERT_TAIL(&oh->waitinglist, req, w_list); + Lck_Unlock(&oh->mtx); AZ(req->hash_ignore_busy); - VTAILQ_INSERT_TAIL(&oh->waitinglist, req, w_list); - if (DO_DEBUG(DBG_WAITINGLIST)) - VSLb(req->vsl, SLT_Debug, "on waiting list <%p>", oh); - wrk->stats->busy_sleep++; /* * The objhead reference transfers to the sess, we get it @@ -544,7 +562,10 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, req->hash_objhead = oh; req->wrk = NULL; req->waitinglist = 1; - Lck_Unlock(&oh->mtx); + + if (DO_DEBUG(DBG_WAITINGLIST)) + VSLb(req->vsl, SLT_Debug, "on waiting list <%p>", oh); + return (HSH_BUSY); } From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:09 +0000 (UTC) Subject: [6.0] 7519d5330 Add req.is_hitmiss and req.is_hitpass Message-ID: <20190208131309.C45676474C@lists.varnish-cache.org> commit 7519d53304e2b2b491060bfa89aab48141689cc2 Author: Nils Goroll Date: Mon Dec 3 22:43:50 2018 +0100 Add req.is_hitmiss and req.is_hitpass Closes #2743 Conflicts: doc/changes.rst doc/changes.rst was considered binary in master, it won't be in 6.0 so there will be no need to pick a1d2db691eef3ee09ec30c591006c9aa2a5026f5 too. Refs a1d2db691eef3ee09ec30c591006c9aa2a5026f5 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 85af28381..b4bcc71ce 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -522,12 +522,15 @@ cnt_lookup(struct worker *wrk, struct req *req) req->objcore = busy; req->stale_oc = oc; req->req_step = R_STP_MISS; + if (lr == HSH_HITMISS) + req->is_hitmiss = 1; return (REQ_FSM_MORE); } if (lr == HSH_HITPASS) { AZ(busy); AZ(oc); req->req_step = R_STP_PASS; + req->is_hitpass = 1; return (REQ_FSM_MORE); } @@ -828,6 +831,8 @@ cnt_recv_prep(struct req *req, const char *ci) req->vdc->retval = 0; req->is_hit = 0; + req->is_hitmiss = 0; + req->is_hitpass = 0; } /*-------------------------------------------------------------------- * We have a complete request, set everything up and start it. diff --git a/bin/varnishtest/tests/c00014.vtc b/bin/varnishtest/tests/c00014.vtc index a0dbf5493..6fc238813 100644 --- a/bin/varnishtest/tests/c00014.vtc +++ b/bin/varnishtest/tests/c00014.vtc @@ -16,6 +16,12 @@ server s1 { } -start varnish v1 -vcl+backend { + sub vcl_miss { + set req.http.hitmiss = req.is_hitmiss; + } + sub vcl_deliver { + set resp.http.hitmiss = req.http.hitmiss; + } sub vcl_backend_response { set beresp.do_stream = false; set beresp.uncacheable = true; @@ -28,6 +34,7 @@ client c1 { expect resp.status == 200 expect resp.bodylen == 12 expect resp.http.x-varnish == "1001" + expect resp.http.hitmiss == "false" } -start barrier b1 sync @@ -39,6 +46,7 @@ client c2 { expect resp.status == 200 expect resp.bodylen == 6 expect resp.http.x-varnish == "1004" + expect resp.http.hitmiss == "true" } -run client c1 -wait diff --git a/bin/varnishtest/tests/c00081.vtc b/bin/varnishtest/tests/c00081.vtc index b08fa3250..ffd344080 100644 --- a/bin/varnishtest/tests/c00081.vtc +++ b/bin/varnishtest/tests/c00081.vtc @@ -12,10 +12,10 @@ server s1 { varnish v1 -vcl+backend { sub vcl_miss { - set req.http.miss = "True"; + set req.http.miss = true; } sub vcl_pass { - set req.http.pass = "True"; + set req.http.hitpass = req.is_hitpass; } sub vcl_backend_response { @@ -24,7 +24,7 @@ varnish v1 -vcl+backend { sub vcl_deliver { set resp.http.miss = req.http.miss; - set resp.http.pass = req.http.pass; + set resp.http.hitpass = req.http.hitpass; } } -start @@ -36,17 +36,17 @@ logexpect l1 -v v1 -g vxid { client c1 { txreq rxresp - expect resp.http.miss == True + expect resp.http.miss == true txreq rxresp - expect resp.http.pass == True + expect resp.http.hitpass == true delay 3 txreq rxresp - expect resp.http.miss == True + expect resp.http.miss == true } -run logexpect l1 -wait diff --git a/doc/changes.rst b/doc/changes.rst index 5ac9da18f..94583e701 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -34,6 +34,10 @@ Varnish Cache 6.0.3 (unreleased) renamed the red/black tree macros from ``VRB_*`` to ``VRBT_*`` to disambiguate from the acronym for Varnish Request Body. +* added ``req.is_hitmiss`` and ``req.is_hitpass`` (2743_) + +.. _2743: https://github.com/varnishcache/varnish-cache/issues/2743 + ================================ Varnish Cache 6.0.2 (2018-11-07) ================================ diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 4efbdff39..e8c04d928 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -391,6 +391,21 @@ req.hash_always_miss This is useful to force-update the cache without invalidating existing entries in case the fetch fails. +req.is_hitmiss + + Type: BOOL + + Readable from: client + + If this request resulted in a hitmiss + +req.is_hitpass + + Type: BOOL + + Readable from: client + + If this request resulted in a hitpass req_top.method diff --git a/include/tbl/req_flags.h b/include/tbl/req_flags.h index e23b9c1a2..2c0dbe803 100644 --- a/include/tbl/req_flags.h +++ b/include/tbl/req_flags.h @@ -34,6 +34,8 @@ REQ_FLAG(disable_esi, 0, 0, "") REQ_FLAG(hash_ignore_busy, 1, 1, "") REQ_FLAG(hash_always_miss, 1, 1, "") REQ_FLAG(is_hit, 0, 0, "") +REQ_FLAG(is_hitmiss, 1, 0, "") +REQ_FLAG(is_hitpass, 1, 0, "") REQ_FLAG(waitinglist, 0, 0, "") REQ_FLAG(want100cont, 0, 0, "") REQ_FLAG(late100cont, 0, 0, "") From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:09 +0000 (UTC) Subject: [6.0] 14be7fce0 add a lookup enum never to escape HSH_Lookup Message-ID: <20190208131309.DE53964758@lists.varnish-cache.org> commit 14be7fce0a4959a1b1ad24383cd944f7834ce348 Author: Nils Goroll Date: Mon Dec 3 22:49:51 2018 +0100 add a lookup enum never to escape HSH_Lookup This is caught by assert(lr == HSH_HIT || lr == HSH_GRACE); in cnt_lookup() diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index 9434ce874..6c17b8374 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -50,6 +50,7 @@ struct hash_slinger { }; enum lookup_e { + HSH_CONTINUE, HSH_MISS, HSH_BUSY, HSH_HIT, From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:10 +0000 (UTC) Subject: [6.0] e2d1a6504 Move HSH_DeleteObjHead into the hash deref methods Message-ID: <20190208131310.06EC86476C@lists.varnish-cache.org> commit e2d1a6504539fb111b69732ee1feb57719db65a3 Author: Martin Blix Grydeland Date: Tue Nov 20 12:48:49 2018 +0100 Move HSH_DeleteObjHead into the hash deref methods Move the clean up call to HSH_DeleteObjHead inside of the hash deref methods that require it, instead of having the outside (single) caller do it based off the return value. This just cleans up and makes the logic more transparent. diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 64c12686d..fd0deeb19 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -983,7 +983,6 @@ HSH_DerefObjHead(struct worker *wrk, struct objhead **poh) { struct objhead *oh; struct rush rush; - int r; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); TAKE_OBJ_NOTNULL(oh, poh, OBJHEAD_MAGIC); @@ -1016,10 +1015,7 @@ HSH_DerefObjHead(struct worker *wrk, struct objhead **poh) Lck_Unlock(&oh->mtx); assert(oh->refcnt > 0); - r = hash->deref(oh); - if (!r) - HSH_DeleteObjHead(wrk, oh); - return (r); + return (hash->deref(wrk, oh)); } void diff --git a/bin/varnishd/hash/hash_classic.c b/bin/varnishd/hash/hash_classic.c index 3440b9bd2..d3303c875 100644 --- a/bin/varnishd/hash/hash_classic.c +++ b/bin/varnishd/hash/hash_classic.c @@ -172,7 +172,7 @@ hcl_lookup(struct worker *wrk, const void *digest, struct objhead **noh) */ static int v_matchproto_(hash_deref_f) -hcl_deref(struct objhead *oh) +hcl_deref(struct worker *wrk, struct objhead *oh) { struct hcl_hd *hp; int ret; @@ -187,6 +187,8 @@ hcl_deref(struct objhead *oh) } else ret = 1; Lck_Unlock(&hp->mtx); + if (!ret) + HSH_DeleteObjHead(wrk, oh); return (ret); } diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c index c673595b2..1c453148f 100644 --- a/bin/varnishd/hash/hash_critbit.c +++ b/bin/varnishd/hash/hash_critbit.c @@ -347,13 +347,15 @@ hcb_start(void) } static int v_matchproto_(hash_deref_f) -hcb_deref(struct objhead *oh) +hcb_deref(struct worker *wrk, struct objhead *oh) { + int r; + (void)wrk; CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); Lck_Lock(&oh->mtx); assert(oh->refcnt > 0); - oh->refcnt--; + r = --oh->refcnt; if (oh->refcnt == 0) { Lck_Lock(&hcb_mtx); hcb_delete(&hcb_root, oh); @@ -364,7 +366,7 @@ hcb_deref(struct objhead *oh) #ifdef PHK fprintf(stderr, "hcb_defef %d %d <%s>\n", __LINE__, r, oh->hash); #endif - return (1); + return (r); } static struct objhead * v_matchproto_(hash_lookup_f) diff --git a/bin/varnishd/hash/hash_simple_list.c b/bin/varnishd/hash/hash_simple_list.c index 4c1e9025a..643148ed0 100644 --- a/bin/varnishd/hash/hash_simple_list.c +++ b/bin/varnishd/hash/hash_simple_list.c @@ -108,7 +108,7 @@ hsl_lookup(struct worker *wrk, const void *digest, struct objhead **noh) */ static int v_matchproto_(hash_deref_f) -hsl_deref(struct objhead *oh) +hsl_deref(struct worker *wrk, struct objhead *oh) { int ret; @@ -119,6 +119,8 @@ hsl_deref(struct objhead *oh) } else ret = 1; Lck_Unlock(&hsl_mtx); + if (!ret) + HSH_DeleteObjHead(wrk, oh); return (ret); } diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index 6c17b8374..0ff95cca2 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -36,7 +36,7 @@ typedef void hash_start_f(void); typedef void hash_prep_f(struct worker *); typedef struct objhead *hash_lookup_f(struct worker *, const void *digest, struct objhead **); -typedef int hash_deref_f(struct objhead *); +typedef int hash_deref_f(struct worker *, struct objhead *); struct hash_slinger { unsigned magic; From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:10 +0000 (UTC) Subject: [6.0] abb63ed7f Change hash slingers deref func to take a locked objhead Message-ID: <20190208131310.213C864771@lists.varnish-cache.org> commit abb63ed7f32db2aaa7437e0bb59f65b648871a60 Author: Martin Blix Grydeland Date: Tue Nov 20 12:59:36 2018 +0100 Change hash slingers deref func to take a locked objhead This changes the hash slingers deref function to take a locked objhead on input, and unlocking it before returning. This saves a lock/unlock sequence for critbit (which is the critical one) on HSH_DerefObjHead. diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index fd0deeb19..9468504d5 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -1012,7 +1012,6 @@ HSH_DerefObjHead(struct worker *wrk, struct objhead **poh) hsh_rush2(wrk, &rush); Lck_Lock(&oh->mtx); } - Lck_Unlock(&oh->mtx); assert(oh->refcnt > 0); return (hash->deref(wrk, oh)); diff --git a/bin/varnishd/hash/hash_classic.c b/bin/varnishd/hash/hash_classic.c index d3303c875..cb3b0b3a2 100644 --- a/bin/varnishd/hash/hash_classic.c +++ b/bin/varnishd/hash/hash_classic.c @@ -178,6 +178,9 @@ hcl_deref(struct worker *wrk, struct objhead *oh) int ret; CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); + Lck_AssertHeld(&oh->mtx); + Lck_Unlock(&oh->mtx); + CAST_OBJ_NOTNULL(hp, oh->hoh_head, HCL_HEAD_MAGIC); assert(oh->refcnt > 0); Lck_Lock(&hp->mtx); diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c index 1c453148f..fcedbef5f 100644 --- a/bin/varnishd/hash/hash_critbit.c +++ b/bin/varnishd/hash/hash_critbit.c @@ -353,7 +353,7 @@ hcb_deref(struct worker *wrk, struct objhead *oh) (void)wrk; CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - Lck_Lock(&oh->mtx); + Lck_AssertHeld(&oh->mtx); assert(oh->refcnt > 0); r = --oh->refcnt; if (oh->refcnt == 0) { diff --git a/bin/varnishd/hash/hash_simple_list.c b/bin/varnishd/hash/hash_simple_list.c index 643148ed0..8c0858e18 100644 --- a/bin/varnishd/hash/hash_simple_list.c +++ b/bin/varnishd/hash/hash_simple_list.c @@ -112,6 +112,10 @@ hsl_deref(struct worker *wrk, struct objhead *oh) { int ret; + CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); + Lck_AssertHeld(&oh->mtx); + Lck_Unlock(&oh->mtx); + Lck_Lock(&hsl_mtx); if (--oh->refcnt == 0) { VTAILQ_REMOVE(&hsl_head, oh, hoh_list); From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:10 +0000 (UTC) Subject: [6.0] fa716ba28 Rename HSH_DerefObjhead() to hsh_deref_objhead() Message-ID: <20190208131310.3CA526477D@lists.varnish-cache.org> commit fa716ba280f59d6b9b2d0543b59eefcbfab637d4 Author: Martin Blix Grydeland Date: Tue Dec 4 10:55:19 2018 +0100 Rename HSH_DerefObjhead() to hsh_deref_objhead() Make this function lower case for consistency. diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 9468504d5..80a8a6f92 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -76,7 +76,7 @@ static struct objhead *private_oh; static void hsh_rush1(const struct worker *, struct objhead *, struct rush *, int); static void hsh_rush2(struct worker *, struct rush *); -static int HSH_DerefObjHead(struct worker *wrk, struct objhead **poh); +static int hsh_deref_objhead(struct worker *wrk, struct objhead **poh); /*---------------------------------------------------------------------*/ @@ -461,7 +461,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, Lck_Unlock(&oh->mtx); *ocp = oc; if (*bocp == NULL) - assert(HSH_DerefObjHead(wrk, &oh)); + assert(hsh_deref_objhead(wrk, &oh)); switch (retval) { case HSH_HITPASS: @@ -541,7 +541,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, Lck_Unlock(&oh->mtx); *ocp = exp_oc; - assert(HSH_DerefObjHead(wrk, &oh)); + assert(hsh_deref_objhead(wrk, &oh)); if (exp_oc->hits < LONG_MAX) exp_oc->hits++; return (HSH_GRACE); @@ -974,12 +974,12 @@ HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp, int rushmax) /* Drop our ref on the objhead */ assert(oh->refcnt > 0); - (void)HSH_DerefObjHead(wrk, &oh); + (void)hsh_deref_objhead(wrk, &oh); return (0); } static int -HSH_DerefObjHead(struct worker *wrk, struct objhead **poh) +hsh_deref_objhead(struct worker *wrk, struct objhead **poh) { struct objhead *oh; struct rush rush; From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:10 +0000 (UTC) Subject: [6.0] e4c302679 Add hsh_deref_objhead_unlock(), taking a locked objhead Message-ID: <20190208131310.5E5DF647A2@lists.varnish-cache.org> commit e4c3026796dede208efd7dcb547ab9c2068cfc6d Author: Martin Blix Grydeland Date: Tue Dec 4 10:57:11 2018 +0100 Add hsh_deref_objhead_unlock(), taking a locked objhead Split hsh_deref_objhead() into two parts, with the new hsh_deref_objhead_unlock() function expecting a locked objhead reference on input. diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 80a8a6f92..5a13bffd0 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -77,6 +77,7 @@ static void hsh_rush1(const struct worker *, struct objhead *, struct rush *, int); static void hsh_rush2(struct worker *, struct rush *); static int hsh_deref_objhead(struct worker *wrk, struct objhead **poh); +static int hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh); /*---------------------------------------------------------------------*/ @@ -979,7 +980,7 @@ HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp, int rushmax) } static int -hsh_deref_objhead(struct worker *wrk, struct objhead **poh) +hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh) { struct objhead *oh; struct rush rush; @@ -988,9 +989,10 @@ hsh_deref_objhead(struct worker *wrk, struct objhead **poh) TAKE_OBJ_NOTNULL(oh, poh, OBJHEAD_MAGIC); INIT_OBJ(&rush, RUSH_MAGIC); + Lck_AssertHeld(&oh->mtx); + if (oh == private_oh) { assert(VTAILQ_EMPTY(&oh->waitinglist)); - Lck_Lock(&oh->mtx); assert(oh->refcnt > 1); oh->refcnt--; Lck_Unlock(&oh->mtx); @@ -1005,7 +1007,6 @@ hsh_deref_objhead(struct worker *wrk, struct objhead **poh) * just make the hold the same ref's as objcore, that would * confuse hashers. */ - Lck_Lock(&oh->mtx); while (oh->refcnt == 1 && !VTAILQ_EMPTY(&oh->waitinglist)) { hsh_rush1(wrk, oh, &rush, HSH_RUSH_ALL); Lck_Unlock(&oh->mtx); @@ -1017,6 +1018,18 @@ hsh_deref_objhead(struct worker *wrk, struct objhead **poh) return (hash->deref(wrk, oh)); } +static int +hsh_deref_objhead(struct worker *wrk, struct objhead **poh) +{ + struct objhead *oh; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + TAKE_OBJ_NOTNULL(oh, poh, OBJHEAD_MAGIC); + + Lck_Lock(&oh->mtx); + return (hsh_deref_objhead_unlock(wrk, &oh)); +} + void HSH_Init(const struct hash_slinger *slinger) { From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:10 +0000 (UTC) Subject: [6.0] 02dd83ba1 Reduce locking in HSH_Lookup() Message-ID: <20190208131310.75F95647AE@lists.varnish-cache.org> commit 02dd83ba173406d2f0c8e4808f2c2dbc6061dd65 Author: Martin Blix Grydeland Date: Tue Dec 4 10:57:11 2018 +0100 Reduce locking in HSH_Lookup() Make use of the new hsh_deref_objhead_unlock() in HSH_Lookup() to reduce the number of times the objhead mutex is taken during cache hits. diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 5a13bffd0..cfda2bb9c 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -459,10 +459,11 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, oc->hits++; retval = HSH_HIT; } - Lck_Unlock(&oh->mtx); *ocp = oc; if (*bocp == NULL) - assert(hsh_deref_objhead(wrk, &oh)); + AN(hsh_deref_objhead_unlock(wrk, &oh)); + else + Lck_Unlock(&oh->mtx); switch (retval) { case HSH_HITPASS: @@ -539,12 +540,10 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, assert(oh->refcnt > 1); assert(exp_oc->objhead == oh); exp_oc->refcnt++; - Lck_Unlock(&oh->mtx); *ocp = exp_oc; - - assert(hsh_deref_objhead(wrk, &oh)); if (exp_oc->hits < LONG_MAX) exp_oc->hits++; + AN(hsh_deref_objhead_unlock(wrk, &oh)); return (HSH_GRACE); } From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:10 +0000 (UTC) Subject: [6.0] b2f9f2fba Display the VCL syntax during a panic Message-ID: <20190208131310.921A4647B8@lists.varnish-cache.org> commit b2f9f2fba172b280404790a114f253061e341691 Author: Dridi Boukelmoune Date: Tue Dec 4 16:01:13 2018 +0100 Display the VCL syntax during a panic diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index d7a891802..410724e5d 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -212,6 +212,7 @@ VCL_Panic(struct vsb *vsb, const struct vcl *vcl) VSB_printf(vsb, "conf = NULL\n"); } else { PAN_CheckMagic(vsb, vcl->conf, VCL_CONF_MAGIC); + VSB_printf(vsb, "syntax = \"%d\",\n", vcl->conf->syntax); VSB_printf(vsb, "srcname = {\n"); VSB_indent(vsb, 2); for (i = 0; i < vcl->conf->nsrc; ++i) From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:10 +0000 (UTC) Subject: [6.0] daa0f945a Printf format fix Message-ID: <20190208131310.AEB63647CC@lists.varnish-cache.org> commit daa0f945a38d774d289f558cf006ae4d71dd03ef Author: Poul-Henning Kamp Date: Tue Dec 4 21:53:36 2018 +0000 Printf format fix diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 410724e5d..4cf196ea0 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -212,7 +212,7 @@ VCL_Panic(struct vsb *vsb, const struct vcl *vcl) VSB_printf(vsb, "conf = NULL\n"); } else { PAN_CheckMagic(vsb, vcl->conf, VCL_CONF_MAGIC); - VSB_printf(vsb, "syntax = \"%d\",\n", vcl->conf->syntax); + VSB_printf(vsb, "syntax = \"%u\",\n", vcl->conf->syntax); VSB_printf(vsb, "srcname = {\n"); VSB_indent(vsb, 2); for (i = 0; i < vcl->conf->nsrc; ++i) From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:10 +0000 (UTC) Subject: [6.0] a79e90ffe Simplify HSH_Lookup 1/N Message-ID: <20190208131310.CBF79647D8@lists.varnish-cache.org> commit a79e90ffe12e4656f125344d0dcfaf190a7ede26 Author: Poul-Henning Kamp Date: Tue Dec 4 21:54:16 2018 +0000 Simplify HSH_Lookup 1/N Move successful loop termination out of the loop. Inspired by: #2856 diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index cfda2bb9c..15c8fe8bb 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -439,51 +439,11 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, } if (EXP_Ttl(req, oc) > req->t_req) { - /* If still valid, use it */ assert(oh->refcnt > 1); assert(oc->objhead == oh); - if (oc->flags & OC_F_HFP) { - xid = ObjGetXID(wrk, oc); - dttl = EXP_Dttl(req, oc); - oc = NULL; - retval = HSH_HITPASS; - } else if (oc->flags & OC_F_HFM) { - xid = ObjGetXID(wrk, oc); - dttl = EXP_Dttl(req, oc); - *bocp = hsh_insert_busyobj(wrk, oh); - oc->refcnt++; - retval = HSH_HITMISS; - } else { - oc->refcnt++; - if (oc->hits < LONG_MAX) - oc->hits++; - retval = HSH_HIT; - } - *ocp = oc; - if (*bocp == NULL) - AN(hsh_deref_objhead_unlock(wrk, &oh)); - else - Lck_Unlock(&oh->mtx); - - switch (retval) { - case HSH_HITPASS: - wrk->stats->cache_hitpass++; - VSLb(req->vsl, SLT_HitPass, "%u %.6f", - xid, dttl); - break; - case HSH_HITMISS: - wrk->stats->cache_hitmiss++; - VSLb(req->vsl, SLT_HitMiss, "%u %.6f", - xid, dttl); - break; - case HSH_HIT: - break; - default: - INCOMPL(); - } - - return (retval); + break; } + if (EXP_Ttl(NULL, oc) < req->t_req && /* ignore req.ttl */ oc->t_origin > exp_t_origin) { /* record the newest object */ @@ -492,6 +452,48 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, } } + if (oc != NULL) { + if (oc->flags & OC_F_HFP) { + xid = ObjGetXID(wrk, oc); + dttl = EXP_Dttl(req, oc); + oc = NULL; + retval = HSH_HITPASS; + } else if (oc->flags & OC_F_HFM) { + xid = ObjGetXID(wrk, oc); + dttl = EXP_Dttl(req, oc); + *bocp = hsh_insert_busyobj(wrk, oh); + oc->refcnt++; + retval = HSH_HITMISS; + } else { + oc->refcnt++; + if (oc->hits < LONG_MAX) + oc->hits++; + retval = HSH_HIT; + } + *ocp = oc; + if (*bocp == NULL) + AN(hsh_deref_objhead_unlock(wrk, &oh)); + else + Lck_Unlock(&oh->mtx); + + switch (retval) { + case HSH_HITPASS: + wrk->stats->cache_hitpass++; + VSLb(req->vsl, SLT_HitPass, "%u %.6f", xid, dttl); + break; + case HSH_HITMISS: + wrk->stats->cache_hitmiss++; + VSLb(req->vsl, SLT_HitMiss, "%u %.6f", xid, dttl); + break; + case HSH_HIT: + break; + default: + INCOMPL(); + } + + return (retval); + } + if (exp_oc != NULL && exp_oc->flags & OC_F_HFM) { /* * expired HFM ("grace/keep HFM") From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:10 +0000 (UTC) Subject: [6.0] 5ee0e2573 Simplify HSH_Lookup 2/N Message-ID: <20190208131310.ED5C6647EA@lists.varnish-cache.org> commit 5ee0e2573a77c1508faf0dccf2d6432c7d3aa561 Author: Poul-Henning Kamp Date: Tue Dec 4 22:02:05 2018 +0000 Simplify HSH_Lookup 2/N Refactor the hit-cases Inspired by: #2856 diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 15c8fe8bb..c5f5d3c7d 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -452,46 +452,31 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, } } + if (oc != NULL && oc->flags & OC_F_HFP) { + xid = ObjGetXID(wrk, oc); + dttl = EXP_Dttl(req, oc); + AN(hsh_deref_objhead_unlock(wrk, &oh)); + wrk->stats->cache_hitpass++; + VSLb(req->vsl, SLT_HitPass, "%u %.6f", xid, dttl); + return (HSH_HITPASS); + } + if (oc != NULL) { - if (oc->flags & OC_F_HFP) { - xid = ObjGetXID(wrk, oc); - dttl = EXP_Dttl(req, oc); - oc = NULL; - retval = HSH_HITPASS; - } else if (oc->flags & OC_F_HFM) { + *ocp = oc; + oc->refcnt++; + if (oc->flags & OC_F_HFM) { xid = ObjGetXID(wrk, oc); dttl = EXP_Dttl(req, oc); *bocp = hsh_insert_busyobj(wrk, oh); - oc->refcnt++; - retval = HSH_HITMISS; - } else { - oc->refcnt++; - if (oc->hits < LONG_MAX) - oc->hits++; - retval = HSH_HIT; - } - *ocp = oc; - if (*bocp == NULL) - AN(hsh_deref_objhead_unlock(wrk, &oh)); - else Lck_Unlock(&oh->mtx); - - switch (retval) { - case HSH_HITPASS: - wrk->stats->cache_hitpass++; - VSLb(req->vsl, SLT_HitPass, "%u %.6f", xid, dttl); - break; - case HSH_HITMISS: wrk->stats->cache_hitmiss++; VSLb(req->vsl, SLT_HitMiss, "%u %.6f", xid, dttl); - break; - case HSH_HIT: - break; - default: - INCOMPL(); - } - - return (retval); + return (HSH_HITMISS); + } + if (oc->hits < LONG_MAX) + oc->hits++; + AN(hsh_deref_objhead_unlock(wrk, &oh)); + return (HSH_HIT); } if (exp_oc != NULL && exp_oc->flags & OC_F_HFM) { @@ -499,6 +484,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, * expired HFM ("grace/keep HFM") * * XXX should HFM objects actually have grace/keep ? + * XXX also: why isn't *ocp = exp_oc ? */ xid = ObjGetXID(wrk, exp_oc); dttl = EXP_Dttl(req, exp_oc); From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:11 +0000 (UTC) Subject: [6.0] 718918663 Simplify HSH_Lookup 3/N Message-ID: <20190208131311.188BA647F6@lists.varnish-cache.org> commit 718918663cacd1159efa550da1fce8aed669feb3 Author: Poul-Henning Kamp Date: Tue Dec 4 23:08:02 2018 +0000 Simplify HSH_Lookup 3/N Use req->hash_always_miss directly Inspired by: #2856 diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index c5f5d3c7d..94b7d2dd9 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -338,8 +338,7 @@ hsh_insert_busyobj(struct worker *wrk, struct objhead *oh) */ enum lookup_e -HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, - int always_insert) +HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) { struct worker *wrk; struct objhead *oh; @@ -384,7 +383,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); Lck_AssertHeld(&oh->mtx); - if (always_insert) { + if (req->hash_always_miss) { /* XXX: should we do predictive Vary in this case ? */ /* Insert new objcore in objecthead and release mutex */ *bocp = hsh_insert_busyobj(wrk, oh); @@ -472,7 +471,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, wrk->stats->cache_hitmiss++; VSLb(req->vsl, SLT_HitMiss, "%u %.6f", xid, dttl); return (HSH_HITMISS); - } + } if (oc->hits < LONG_MAX) oc->hits++; AN(hsh_deref_objhead_unlock(wrk, &oh)); diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h index 9e90d5b32..e12463981 100644 --- a/bin/varnishd/cache/cache_objhead.h +++ b/bin/varnishd/cache/cache_objhead.h @@ -68,8 +68,7 @@ int HSH_DerefObjCore(struct worker *, struct objcore **, int rushmax); #define HSH_RUSH_POLICY -1 #define HSH_RUSH_ALL INT_MAX -enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **, - int always_insert); +enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **); void HSH_Ref(struct objcore *o); void HSH_AddString(struct req *, void *ctx, const char *str); unsigned HSH_Purge(struct worker *, struct objhead *, vtim_real ttl_now, diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index b4bcc71ce..78395d7e5 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -495,7 +495,7 @@ cnt_lookup(struct worker *wrk, struct req *req) AZ(req->objcore); if (req->hash_objhead) had_objhead = 1; - lr = HSH_Lookup(req, &oc, &busy, req->hash_always_miss ? 1 : 0); + lr = HSH_Lookup(req, &oc, &busy); if (lr == HSH_BUSY) { /* * We lost the session to a busy object, disembark the @@ -986,7 +986,8 @@ cnt_purge(struct worker *wrk, struct req *req) VRY_Prep(req); AZ(req->objcore); - lr = HSH_Lookup(req, &oc, &boc, 1); + req->hash_always_miss = 1; + lr = HSH_Lookup(req, &oc, &boc); assert (lr == HSH_MISS); AZ(oc); CHECK_OBJ_NOTNULL(boc, OBJCORE_MAGIC); From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:11 +0000 (UTC) Subject: [6.0] 0f0464411 Simplify HSH_Lookup 4/4 Message-ID: <20190208131311.36B06647FF@lists.varnish-cache.org> commit 0f0464411a7db6395fefec185e2a24df42ddb2b3 Author: Poul-Henning Kamp Date: Tue Dec 4 23:45:27 2018 +0000 Simplify HSH_Lookup 4/4 Simplify the !busy cases Inspired by: #2856 diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 94b7d2dd9..197ebc4c7 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -346,7 +346,6 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) struct objcore *exp_oc; double exp_t_origin; int busy_found; - enum lookup_e retval; const uint8_t *vary; unsigned xid = 0; float dttl = 0.0; @@ -448,6 +447,8 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) /* record the newest object */ exp_oc = oc; exp_t_origin = oc->t_origin; + assert(oh->refcnt > 1); + assert(exp_oc->objhead == oh); } } @@ -489,43 +490,31 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) dttl = EXP_Dttl(req, exp_oc); *bocp = hsh_insert_busyobj(wrk, oh); Lck_Unlock(&oh->mtx); - wrk->stats->cache_hitmiss++; VSLb(req->vsl, SLT_HitMiss, "%u %.6f", xid, dttl); return (HSH_HITMISS); } if (!busy_found) { - /* Insert objcore in objecthead */ *bocp = hsh_insert_busyobj(wrk, oh); if (exp_oc != NULL) { - assert(oh->refcnt > 1); - assert(exp_oc->objhead == oh); exp_oc->refcnt++; - Lck_Unlock(&oh->mtx); *ocp = exp_oc; - - if (EXP_Ttl_grace(req, exp_oc) < req->t_req) { - retval = HSH_MISS; - } else { + if (EXP_Ttl_grace(req, exp_oc) > req->t_req) { if (exp_oc->hits < LONG_MAX) exp_oc->hits++; - retval = HSH_GRACE; + Lck_Unlock(&oh->mtx); + return (HSH_GRACE); } - } else { - Lck_Unlock(&oh->mtx); - retval = HSH_MISS; } - - return (retval); + Lck_Unlock(&oh->mtx); + return (HSH_MISS); } AN(busy_found); if (exp_oc != NULL && EXP_Ttl_grace(req, exp_oc) >= req->t_req) { /* we do not wait on the busy object if in grace */ - assert(oh->refcnt > 1); - assert(exp_oc->objhead == oh); exp_oc->refcnt++; *ocp = exp_oc; if (exp_oc->hits < LONG_MAX) From dridi.boukelmoune at gmail.com Fri Feb 8 13:13:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Feb 2019 13:13:11 +0000 (UTC) Subject: [6.0] 150d4c4ee Clean up dead code in hsh_deref_objhead_unlock Message-ID: <20190208131311.4E66F64809@lists.varnish-cache.org> commit 150d4c4ee2841d46652d256c5a5a42efa944ce6e Author: Dag Haavi Finstad Date: Wed Dec 5 10:25:57 2018 +0100 Clean up dead code in hsh_deref_objhead_unlock req's on waiting list do indeed hold an objhead ref (req->hash_objhead), so the condition for the while loop in hsh_deref_objhead_unlock will always evaluate to false. This gets rid of the needless rushing code and replaces it with an assert. diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 197ebc4c7..6d030c9ad 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -776,8 +776,10 @@ HSH_Unbusy(struct worker *wrk, struct objcore *oc) VTAILQ_REMOVE(&oh->objcs, oc, hsh_list); VTAILQ_INSERT_HEAD(&oh->objcs, oc, hsh_list); oc->flags &= ~OC_F_BUSY; - if (!VTAILQ_EMPTY(&oh->waitinglist)) + if (!VTAILQ_EMPTY(&oh->waitinglist)) { + assert(oh->refcnt > 1); hsh_rush1(wrk, oh, &rush, HSH_RUSH_POLICY); + } Lck_Unlock(&oh->mtx); if (!(oc->flags & OC_F_PRIVATE)) EXP_Insert(wrk, oc); @@ -932,8 +934,10 @@ HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp, int rushmax) r = --oc->refcnt; if (!r) VTAILQ_REMOVE(&oh->objcs, oc, hsh_list); - if (!VTAILQ_EMPTY(&oh->waitinglist)) + if (!VTAILQ_EMPTY(&oh->waitinglist)) { + assert(oh->refcnt > 1); hsh_rush1(wrk, oh, &rush, rushmax); + } Lck_Unlock(&oh->mtx); hsh_rush2(wrk, &rush); if (r != 0) @@ -958,11 +962,9 @@ static int hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh) { struct objhead *oh; - struct rush rush; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); TAKE_OBJ_NOTNULL(oh, poh, OBJHEAD_MAGIC); - INIT_OBJ(&rush, RUSH_MAGIC); Lck_AssertHeld(&oh->mtx); @@ -974,20 +976,8 @@ hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh) return(1); } - /* - * Make absolutely certain that we do not let the final ref - * disappear until the waitinglist is empty. - * This is necessary because the req's on the waiting list do - * not hold any ref on the objhead of their own, and we cannot - * just make the hold the same ref's as objcore, that would - * confuse hashers. - */ - while (oh->refcnt == 1 && !VTAILQ_EMPTY(&oh->waitinglist)) { - hsh_rush1(wrk, oh, &rush, HSH_RUSH_ALL); - Lck_Unlock(&oh->mtx); - hsh_rush2(wrk, &rush); - Lck_Lock(&oh->mtx); - } + if (oh->refcnt == 1) + assert(VTAILQ_EMPTY(&oh->waitinglist)); assert(oh->refcnt > 0); return (hash->deref(wrk, oh)); From nils.goroll at uplex.de Mon Feb 11 10:18:03 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Feb 2019 10:18:03 +0000 (UTC) Subject: [master] 0942fd55b add debug bit to count contended locks Message-ID: <20190211101803.AAB6B92E91@lists.varnish-cache.org> commit 0942fd55bd51faeab8dd9765aac14db129ddf502 Author: Nils Goroll Date: Mon Feb 11 11:16:40 2019 +0100 add debug bit to count contended locks Thank you to phk for the improvement over the initial suggestion diff --git a/bin/varnishd/VSC_lck.vsc b/bin/varnishd/VSC_lck.vsc index 1a6b6ee8b..0470964a5 100644 --- a/bin/varnishd/VSC_lck.vsc +++ b/bin/varnishd/VSC_lck.vsc @@ -30,5 +30,27 @@ :oneliner: Lock Operations +.. varnish_vsc:: dbg_busy + :type: counter + :level: debug + :oneliner: Contended lock operations + + If the ``lck`` debug bit is set: Lock operations which + returned EBUSY on the first locking attempt. + + If the ``lck`` debug bit is unset, this counter will never be + incremented even if lock operations are contended. + +.. varnish_vsc:: dbg_try_fail + :type: counter + :level: debug + :oneliner: Contended trylock operations + + If the ``lck`` debug bit is set: Trylock operations which + returned EBUSY. + + If the ``lck`` debug bit is unset, this counter will never be + incremented even if lock operations are contended. + .. varnish_vsc_end:: lck diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c index 30cbbab4a..af21c7ba4 100644 --- a/bin/varnishd/cache/cache_lck.c +++ b/bin/varnishd/cache/cache_lck.c @@ -107,11 +107,20 @@ void v_matchproto_() Lck__Lock(struct lock *lck, const char *p, int l) { struct ilck *ilck; + int r = EINVAL; CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC); if (DO_DEBUG(DBG_WITNESS)) Lck_Witness_Lock(ilck, p, l, ""); - AZ(pthread_mutex_lock(&ilck->mtx)); + else if (DO_DEBUG(DBG_LCK)) { + r = pthread_mutex_trylock(&ilck->mtx); + if (r == EBUSY) + ilck->stat->dbg_busy++; + else + AZ(r); + } + if (r) + AZ(pthread_mutex_lock(&ilck->mtx)); AZ(ilck->held); ilck->stat->locks++; ilck->owner = pthread_self(); @@ -162,7 +171,8 @@ Lck__Trylock(struct lock *lck, const char *p, int l) ilck->held = 1; ilck->stat->locks++; ilck->owner = pthread_self(); - } + } else if (DO_DEBUG(DBG_LCK)) + ilck->stat->dbg_try_fail++; return (r); } diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h index 8bf099d5d..83206e80d 100644 --- a/include/tbl/debug_bits.h +++ b/include/tbl/debug_bits.h @@ -51,6 +51,7 @@ DEBUG_BIT(VMOD_SO_KEEP, vmod_so_keep, "Keep copied VMOD libraries") DEBUG_BIT(PROCESSORS, processors, "Fetch/Deliver processors") DEBUG_BIT(PROTOCOL, protocol, "Protocol debugging") DEBUG_BIT(VCL_KEEP, vcl_keep, "Keep VCL C and so files") +DEBUG_BIT(LCK, lck, "Additional lock statistics") #undef DEBUG_BIT /*lint -restore */ From nils.goroll at uplex.de Mon Feb 11 10:22:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Feb 2019 10:22:08 +0000 (UTC) Subject: [master] f7d1c8bf3 fix oversight from previous commit Message-ID: <20190211102208.7250993148@lists.varnish-cache.org> commit f7d1c8bf39f348c6b214760b9bcf46877a3b5cb1 Author: Nils Goroll Date: Mon Feb 11 11:21:19 2019 +0100 fix oversight from previous commit diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c index af21c7ba4..fba7e03a2 100644 --- a/bin/varnishd/cache/cache_lck.c +++ b/bin/varnishd/cache/cache_lck.c @@ -112,7 +112,7 @@ Lck__Lock(struct lock *lck, const char *p, int l) CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC); if (DO_DEBUG(DBG_WITNESS)) Lck_Witness_Lock(ilck, p, l, ""); - else if (DO_DEBUG(DBG_LCK)) { + if (DO_DEBUG(DBG_LCK)) { r = pthread_mutex_trylock(&ilck->mtx); if (r == EBUSY) ilck->stat->dbg_busy++; From phk at FreeBSD.org Mon Feb 11 11:34:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 11 Feb 2019 11:34:09 +0000 (UTC) Subject: [master] 2cba061d8 Don't increment the contested counter until we have the lock. Message-ID: <20190211113409.3784E94A08@lists.varnish-cache.org> commit 2cba061d8d47934af1226b2c87701324d6b84cee Author: Poul-Henning Kamp Date: Mon Feb 11 11:32:51 2019 +0000 Don't increment the contested counter until we have the lock. diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c index fba7e03a2..4021166e4 100644 --- a/bin/varnishd/cache/cache_lck.c +++ b/bin/varnishd/cache/cache_lck.c @@ -114,14 +114,13 @@ Lck__Lock(struct lock *lck, const char *p, int l) Lck_Witness_Lock(ilck, p, l, ""); if (DO_DEBUG(DBG_LCK)) { r = pthread_mutex_trylock(&ilck->mtx); - if (r == EBUSY) - ilck->stat->dbg_busy++; - else - AZ(r); + assert(r == 0 || r == EBUSY); } if (r) AZ(pthread_mutex_lock(&ilck->mtx)); AZ(ilck->held); + if (r == EBUSY) + ilck->stat->dbg_busy++; ilck->stat->locks++; ilck->owner = pthread_self(); ilck->held = 1; From dridi.boukelmoune at gmail.com Mon Feb 11 13:14:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Feb 2019 13:14:09 +0000 (UTC) Subject: [master] 7b7b9c70c Typo Message-ID: <20190211131409.3592196CB8@lists.varnish-cache.org> commit 7b7b9c70cc4fac2e97e7ccee77a518ab2575660e Author: Dridi Boukelmoune Date: Mon Feb 11 14:13:14 2019 +0100 Typo diff --git a/doc/sphinx/phk/lucky.rst b/doc/sphinx/phk/lucky.rst index b7737f442..7a1df5ba5 100644 --- a/doc/sphinx/phk/lucky.rst +++ b/doc/sphinx/phk/lucky.rst @@ -4,7 +4,7 @@ Do you feel lucky ? =================== -Whenever a corporate cotton-mouth says anything, their laywers +Whenever a corporate cotton-mouth says anything, their lawers always make them add a footnote to the effect that *"Past performance is not predictive of future results."* so they do not get sued for being insufficiently clairvoyant. @@ -87,8 +87,8 @@ Around the time of the 2019-03-15 release we will gather for a VDD and then we will know and be ready to say something more detailed. I don't think it is realistic to roll out any kind of H3 support -in the september release, that release will probably only contain -a some of the necessary prepatory reorganization, so expect to +in the September release, that release will probably only contain +a some of the necessary preparatory reorganization, so expect to run production on 6.0 LTS for a while. --------------------- @@ -138,7 +138,7 @@ to receive the reply from their heartbroken spouse, that you are many months too late [#f2]_. Jacob was not a major persona on the Internet, but between doing a -lot of interesting stuff as a multi-disicpline phd. in physics, +lot of interesting stuff as a multi-discipline phd. in physics, being a really good Ada programmer, a huge Lego enthusiast, an incredibly helpful person *and* really *good* at helping, he had a lot of friends in many corners of the Internet. From dridi.boukelmoune at gmail.com Mon Feb 11 13:17:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Feb 2019 13:17:09 +0000 (UTC) Subject: [master] fb22c6081 Typo in my typo fix Message-ID: <20190211131709.A107096FED@lists.varnish-cache.org> commit fb22c6081a992357727892a7f63802a0f7e05918 Author: Dridi Boukelmoune Date: Mon Feb 11 14:16:04 2019 +0100 Typo in my typo fix diff --git a/doc/sphinx/phk/lucky.rst b/doc/sphinx/phk/lucky.rst index 7a1df5ba5..bf84ce746 100644 --- a/doc/sphinx/phk/lucky.rst +++ b/doc/sphinx/phk/lucky.rst @@ -4,7 +4,7 @@ Do you feel lucky ? =================== -Whenever a corporate cotton-mouth says anything, their lawers +Whenever a corporate cotton-mouth says anything, their lawyers always make them add a footnote to the effect that *"Past performance is not predictive of future results."* so they do not get sued for being insufficiently clairvoyant. From nils.goroll at uplex.de Mon Feb 11 13:38:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Feb 2019 13:38:08 +0000 (UTC) Subject: [master] 31bd91829 Improse FetchError on first byte timeout Message-ID: <20190211133808.67717978A1@lists.varnish-cache.org> commit 31bd91829dc07c25525abcf2e4a4b91f484d7a37 Author: Nils Goroll Date: Mon Feb 11 14:30:23 2019 +0100 Improse FetchError on first byte timeout Fixes #2185 Test case by @fgsch diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index f1b0ac3cb..fa456a408 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -199,6 +199,10 @@ V1F_FetchRespHdr(struct busyobj *bo) VSLb(bo->vsl, SLT_FetchError, "overflow"); htc->doclose = SC_RX_OVERFLOW; break; + case HTC_S_IDLE: + VSLb(bo->vsl, SLT_FetchError, "first byte timeout"); + htc->doclose = SC_RX_TIMEOUT; + break; default: VSLb(bo->vsl, SLT_FetchError, "HTC %s (%d)", HTC_Status(hs), hs); diff --git a/bin/varnishtest/tests/b00023.vtc b/bin/varnishtest/tests/b00023.vtc index 9ee5ed340..fd64cb2bd 100644 --- a/bin/varnishtest/tests/b00023.vtc +++ b/bin/varnishtest/tests/b00023.vtc @@ -19,12 +19,18 @@ varnish v1 -vcl+backend { } } -start +logexpect l1 -v v1 { + expect * 1002 FetchError "first byte timeout" +} -start + client c1 { txreq rxresp expect resp.status == 503 } -run +logexpect l1 -wait + server s1 { rxreq delay 0.2 From hermunn at varnish-software.com Mon Feb 11 14:11:09 2019 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Mon, 11 Feb 2019 14:11:09 +0000 (UTC) Subject: [4.1] 61367ed17 Prepare for 4.1.11 release Message-ID: <20190211141109.295189B56F@lists.varnish-cache.org> commit 61367ed17d08a9ef80a2d42dc84caef79cdeee7a Author: P?l Hermunn Johansen Date: Thu Jan 3 15:25:15 2019 +0100 Prepare for 4.1.11 release diff --git a/configure.ac b/configure.ac index e4c246277..905b5b75a 100644 --- a/configure.ac +++ b/configure.ac @@ -1,8 +1,8 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS -Copyright (c) 2006-2018 Varnish Software]) +Copyright (c) 2006-2019 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [4.1.10], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [4.1.11], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index 629f27725..82a615b2d 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,5 +1,5 @@ ================================= -Varnish Cache 4.1.11 (unreleased) +Varnish Cache 4.1.11 (2019-02-11) ================================= Changes since 4.1.10: diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c index 61711654e..b2258ccde 100644 --- a/lib/libvarnish/version.c +++ b/lib/libvarnish/version.c @@ -44,5 +44,5 @@ VCS_Message(const char *progname) { fprintf(stderr, "%s (%s)\n", progname, VCS_version); fprintf(stderr, "Copyright (c) 2006 Verdens Gang AS\n"); - fprintf(stderr, "Copyright (c) 2006-2015 Varnish Software AS\n"); + fprintf(stderr, "Copyright (c) 2006-2019 Varnish Software AS\n"); } From phk at FreeBSD.org Tue Feb 12 10:02:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 12 Feb 2019 10:02:10 +0000 (UTC) Subject: [master] 5735a543c Insist that VCL_RET_FAIL goes through VRT_fail() and not VRT_handling() Message-ID: <20190212100210.DD5BE7A06@lists.varnish-cache.org> commit 5735a543c720655c484ef11c70b6cde7f201d5ca Author: Poul-Henning Kamp Date: Tue Feb 12 10:01:00 2019 +0000 Insist that VCL_RET_FAIL goes through VRT_fail() and not VRT_handling() diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 45b4e1399..8940a6797 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -491,11 +491,11 @@ VRT_handling(VRT_CTX, unsigned hand) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (ctx->handling == NULL) - return; + assert(hand != VCL_RET_FAIL); + AN(ctx->handling); + AZ(*ctx->handling); assert(hand > 0); assert(hand < VCL_RET_MAX); - // XXX:NOTYET assert(*ctx->handling == 0); *ctx->handling = hand; } @@ -507,16 +507,21 @@ VRT_fail(VRT_CTX, const char *fmt, ...) va_list ap; assert(ctx->vsl != NULL || ctx->msg != NULL); + AN(ctx->handling); + if (*ctx->handling == VCL_RET_FAIL) + return; + AZ(*ctx->handling); + AN(fmt); AZ(strchr(fmt, '\n')); va_start(ap, fmt); - if (ctx->vsl != NULL) + if (ctx->vsl != NULL) { VSLbv(ctx->vsl, SLT_VCL_Error, fmt, ap); - else { + } else { VSB_vprintf(ctx->msg, fmt, ap); VSB_putc(ctx->msg, '\n'); } va_end(ap); - VRT_handling(ctx, VCL_RET_FAIL); + *ctx->handling = VCL_RET_FAIL; } /*-------------------------------------------------------------------- @@ -772,9 +777,8 @@ VRT_purge(VRT_CTX, VCL_DURATION ttl, VCL_DURATION grace, VCL_DURATION keep) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if ((ctx->method & (VCL_MET_HIT|VCL_MET_MISS)) == 0) { - VSLb(ctx->vsl, SLT_VCL_Error, + VRT_fail(ctx, "purge can only happen in vcl_hit{} or vcl_miss{}"); - VRT_handling(ctx, VCL_RET_FAIL); return (0); } diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index ded906870..02d323a14 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -350,7 +350,10 @@ vcc_act_return(struct vcc *tl, struct token *t, struct symbol *sym) } } ERRCHK(tl); - Fb(tl, 1, "VRT_handling(ctx, VCL_RET_%s);\n", h); + if (hand == VCL_RET_FAIL) + Fb(tl, 1, "VRT_fail(ctx, \"Failed from VCL\");\n"); + else + Fb(tl, 1, "VRT_handling(ctx, VCL_RET_%s);\n", h); SkipToken(tl, ')'); SkipToken(tl, ';'); } diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 13994eb6b..203f34465 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -318,8 +318,15 @@ EmitInitFini(const struct vcc *tl) if (VSB_len(p->event)) has_event = 1; } + + /* Handle failures from vcl_init */ + Fc(tl, 0, "\n"); + Fc(tl, 0, "\tif (*ctx->handling != VCL_RET_OK)\n"); + Fc(tl, 0, "\t\treturn(1);\n"); + VTAILQ_FOREACH(sy, &tl->sym_objects, sideways) { Fc(tl, 0, "\tif (!%s) {\n", sy->rname); + Fc(tl, 0, "\t\t*ctx->handling = 0;\n"); Fc(tl, 0, "\t\tVRT_fail(ctx, " "\"Object %s not initialized\");\n" , sy->name); Fc(tl, 0, "\t\treturn(1);\n"); @@ -655,7 +662,12 @@ vcc_CompileSource(struct vcc *tl, struct source *sp) /* Tie vcl_init/fini in */ ifp = New_IniFin(tl); - VSB_printf(ifp->ini, "\tVGC_function_vcl_init(ctx);"); + VSB_printf(ifp->ini, "\tVGC_function_vcl_init(ctx);\n"); + /* + * We do not return(1) if this fails, because the failure + * could be half way into vcl_init{} so vcl_fini{} must + * always be called, also on failure. + */ VSB_printf(ifp->fin, "\t\tVGC_function_vcl_fini(ctx);"); /* Emit method functions */ From dridi.boukelmoune at gmail.com Tue Feb 12 10:03:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 12 Feb 2019 10:03:09 +0000 (UTC) Subject: [6.0] 2dc0afd17 make two identical comparisons the same Message-ID: <20190212100309.B8E0E7C6C@lists.varnish-cache.org> commit 2dc0afd17f1e123513c83abbe36d9c9f63df083a Author: Nils Goroll Date: Wed Dec 5 11:40:13 2018 +0100 make two identical comparisons the same see 11 lines down Conflicts: bin/varnishd/cache/cache_hash.c diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 6d030c9ad..78b61ce11 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -501,9 +501,8 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) if (exp_oc != NULL) { exp_oc->refcnt++; *ocp = exp_oc; - if (EXP_Ttl_grace(req, exp_oc) > req->t_req) { - if (exp_oc->hits < LONG_MAX) - exp_oc->hits++; + if (EXP_Ttl_grace(req, exp_oc) >= req->t_req) { + exp_oc->hits++; Lck_Unlock(&oh->mtx); return (HSH_GRACE); } From dridi.boukelmoune at gmail.com Tue Feb 12 10:03:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 12 Feb 2019 10:03:09 +0000 (UTC) Subject: [6.0] a95864291 fsm graphviz: polish the waitinglist arrow Message-ID: <20190212100309.D78AD7C6F@lists.varnish-cache.org> commit a9586429145fb3defc62b1d65f244d95b0fc4812 Author: Nils Goroll Date: Wed Dec 5 11:56:44 2018 +0100 fsm graphviz: polish the waitinglist arrow diff --git a/doc/graphviz/cache_req_fsm.dot b/doc/graphviz/cache_req_fsm.dot index 3501b7bb1..b9f6a80c2 100644 --- a/doc/graphviz/cache_req_fsm.dot +++ b/doc/graphviz/cache_req_fsm.dot @@ -124,7 +124,7 @@ digraph cache_req_fsm { label="{cnt_lookup:|{vcl_hit\{\}|{req.*|obj.*}}|{fail|deliver|miss|restart|synth|pass}}" ] } - lookup:busy:e -> lookup:top:e [label="(waitinglist)", + lookup:busy:s -> lookup:top:ne [label=" waitinglist", color=grey, fontcolor=grey] lookup:miss:s -> miss [style=bold,color=blue] From dridi.boukelmoune at gmail.com Tue Feb 12 10:03:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 12 Feb 2019 10:03:09 +0000 (UTC) Subject: [6.0] 461d1a042 fsm graphviz: add hfm Message-ID: <20190212100309.F20CF7C73@lists.varnish-cache.org> commit 461d1a0427920066293ab8d751cd068be74367ec Author: Nils Goroll Date: Wed Dec 5 11:57:18 2018 +0100 fsm graphviz: add hfm diff --git a/doc/graphviz/cache_req_fsm.dot b/doc/graphviz/cache_req_fsm.dot index b9f6a80c2..9dda7bf9e 100644 --- a/doc/graphviz/cache_req_fsm.dot +++ b/doc/graphviz/cache_req_fsm.dot @@ -117,7 +117,7 @@ digraph cache_req_fsm { shape=record color=grey fontcolor=grey - label="{cnt_lookup:|hash lookup|{hit?|miss?|hit-for-pass?|busy?}}" + label="{cnt_lookup:|hash lookup|{hit?|miss?|hit-for-miss?|hit-for-pass?|busy?}}" ] lookup2 [ shape=record @@ -128,6 +128,7 @@ digraph cache_req_fsm { color=grey, fontcolor=grey] lookup:miss:s -> miss [style=bold,color=blue] + lookup:hfm:s -> miss [style=bold,color=blue] lookup:hfp:s -> pass [style=bold,color=red] lookup:h:s -> lookup2 [style=bold,color=green] From dridi.boukelmoune at gmail.com Tue Feb 12 10:03:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 12 Feb 2019 10:03:10 +0000 (UTC) Subject: [6.0] 5db08bed7 fsm graphviz: label hfm/hfp Message-ID: <20190212100310.174DB7C77@lists.varnish-cache.org> commit 5db08bed7580a761f81328a5c94faaa607201dcd Author: Nils Goroll Date: Wed Dec 5 12:09:24 2018 +0100 fsm graphviz: label hfm/hfp Ref #2743 diff --git a/doc/graphviz/cache_req_fsm.dot b/doc/graphviz/cache_req_fsm.dot index 9dda7bf9e..a33de6d5b 100644 --- a/doc/graphviz/cache_req_fsm.dot +++ b/doc/graphviz/cache_req_fsm.dot @@ -128,8 +128,8 @@ digraph cache_req_fsm { color=grey, fontcolor=grey] lookup:miss:s -> miss [style=bold,color=blue] - lookup:hfm:s -> miss [style=bold,color=blue] - lookup:hfp:s -> pass [style=bold,color=red] + lookup:hfm:s -> miss [style=bold,color=blue,label=" req.\n is_hitmiss"] + lookup:hfp:s -> pass [style=bold,color=red,label=" req.\n is_hitpass"] lookup:h:s -> lookup2 [style=bold,color=green] lookup2:deliver:s -> deliver:n [style=bold,color=green] From dridi.boukelmoune at gmail.com Tue Feb 12 10:03:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 12 Feb 2019 10:03:10 +0000 (UTC) Subject: [6.0] 55d4f7eac sm graphviz: document that miss from hit is going to die Message-ID: <20190212100310.2F9287C7B@lists.varnish-cache.org> commit 55d4f7eac5ebf8c8356aa6bd9ad2cbaafe20d69e Author: Nils Goroll Date: Wed Dec 5 12:21:02 2018 +0100 sm graphviz: document that miss from hit is going to die and polish the output formatting by reordering Ref #1799 diff --git a/doc/graphviz/cache_req_fsm.dot b/doc/graphviz/cache_req_fsm.dot index a33de6d5b..629072dc5 100644 --- a/doc/graphviz/cache_req_fsm.dot +++ b/doc/graphviz/cache_req_fsm.dot @@ -121,7 +121,7 @@ digraph cache_req_fsm { ] lookup2 [ shape=record - label="{cnt_lookup:|{vcl_hit\{\}|{req.*|obj.*}}|{fail|deliver|miss|restart|synth|pass}}" + label="{cnt_lookup:|{vcl_hit\{\}|{req.*|obj.*}}|{fail|deliver|pass|restart|synth|miss}}" ] } lookup:busy:s -> lookup:top:ne [label=" waitinglist", @@ -133,9 +133,7 @@ digraph cache_req_fsm { lookup:h:s -> lookup2 [style=bold,color=green] lookup2:deliver:s -> deliver:n [style=bold,color=green] - lookup2:miss:s -> miss [style=bold,color=blue] - // XXX should not happen - // lookup2:miss:s -> pass [style=bold,color=red,label="(no busy obj)"] + lookup2:miss:s -> miss [color=blue,label=" #1799 \n EOL"] lookup2:pass:s -> pass [style=bold,color=red] /* cnt_miss */ From dridi.boukelmoune at gmail.com Tue Feb 12 10:03:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 12 Feb 2019 10:03:10 +0000 (UTC) Subject: [6.0] fb936e871 update the generated graphviz svg Message-ID: <20190212100310.4BA407C7F@lists.varnish-cache.org> commit fb936e871111195f7b84e325f42171de0d8ee8b9 Author: Nils Goroll Date: Wed Dec 5 12:25:26 2018 +0100 update the generated graphviz svg diff --git a/doc/graphviz/cache_req_fsm.svg b/doc/graphviz/cache_req_fsm.svg index 60c71acd1..eeb0a75c8 100644 --- a/doc/graphviz/cache_req_fsm.svg +++ b/doc/graphviz/cache_req_fsm.svg @@ -4,496 +4,509 @@ -