From dridi.boukelmoune at gmail.com Wed Sep 1 05:30:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 1 Sep 2021 05:30:10 +0000 (UTC) Subject: [master] d2e526ce9 man: Add missing sections to the vtc(7) manual Message-ID: <20210901053010.A816B10EAD1@lists.varnish-cache.org> commit d2e526ce95d6d8426ec170b4cfc78d41abc07179 Author: Dridi Boukelmoune Date: Wed Sep 1 05:52:35 2021 +0200 man: Add missing sections to the vtc(7) manual diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 2cdb34a68..406193dbb 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -937,6 +937,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) n = 0; /* SECTION: stream.spec.zexpect.ping PING specific + * * ping.data * The 8-bytes string of the PING frame payload. * ping.ack (PING) @@ -952,6 +953,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) return (buf); } /* SECTION: stream.spec.zexpect.winup WINDOW_UPDATE specific + * * winup.size * The size of the upgrade given by the WINDOW_UPDATE frame. */ @@ -960,6 +962,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) RETURN_BUFFED(f->md.winup_size); } /* SECTION: stream.spec.zexpect.prio PRIORITY specific + * * prio.stream * The stream ID announced. * @@ -983,6 +986,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) RETURN_BUFFED(f->md.prio.weight); } /* SECTION: stream.spec.zexpect.rst RESET_STREAM specific + * * rst.err * The error code (as integer) of the RESET_STREAM frame. */ @@ -1037,6 +1041,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) if (!strcmp(spec, "hdrsize")) { RETURN_SETTINGS(6); } } /* SECTION: stream.spec.zexpect.push PUSH_PROMISE specific + * * push.id * The id of the promised stream. */ @@ -1045,6 +1050,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) RETURN_BUFFED(f->md.promised); } /* SECTION: stream.spec.zexpect.goaway GOAWAY specific + * * goaway.err * The error code (as integer) of the GOAWAY frame. * @@ -1066,6 +1072,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) return (f->md.goaway.debug); } /* SECTION: stream.spec.zexpect.zframe Generic frame + * * frame.data * Payload of the last frame * @@ -1101,6 +1108,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) } } /* SECTION: stream.spec.zexpect.zstream Stream + * * stream.window * The current window size of the stream, or, if on stream 0, * of the connection. @@ -1131,6 +1139,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) return (NULL); } /* SECTION: stream.spec.zexpect.ztable Index tables + * * tbl.dec.size / tbl.enc.size * Size (bytes) of the decoding/encoding table. * diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py index 03451ce51..c168e05a7 100644 --- a/doc/sphinx/vtc-syntax.py +++ b/doc/sphinx/vtc-syntax.py @@ -38,7 +38,7 @@ import re def parse_file(fn, cl, tl, sl): p = False section = "" - resec = re.compile("[ /]\* SECTION: ") + resec = re.compile("\s*/\* SECTION: ") try: # Python3 @@ -57,13 +57,13 @@ def parse_file(fn, cl, tl, sl): cl[section] = [] if len(a) > 3: tl[section] = re.sub( - r"^[\t ]*\/?\* SECTION: [^ ]+ +", + r"^\s*/?\* SECTION: [^ ]+ +", "", l) else: tl[section] = "" p = 1 elif p: - cl[section].append(re.sub(r"^ \* ?", "", l)) + cl[section].append(re.sub(r"^\s*\* ?", "", l)) f.close() if __name__ == "__main__": From dridi.boukelmoune at gmail.com Wed Sep 1 07:29:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 1 Sep 2021 07:29:05 +0000 (UTC) Subject: [master] 86e89aed0 changes: Use reStructuredText ``code`` delimiters Message-ID: <20210901072905.6A21D111C21@lists.varnish-cache.org> commit 86e89aed05ce7645a92cbbc3e623efdfbc3ba606 Author: Dridi Boukelmoune Date: Wed Sep 1 07:45:53 2021 +0200 changes: Use reStructuredText ``code`` delimiters diff --git a/doc/changes.rst b/doc/changes.rst index 0588c5359..f18804a3c 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -35,10 +35,10 @@ release process. Varnish Cache 7.x.x (2021-09-15) ================================ -* The minimum `session_workspace` is now 384 bytes +* The minimum ``session_workspace`` is now 384 bytes -* The default workdir (the default `-n` argument) is now `/var/run` - instead of `${prefix}/var`. +* The default workdir (the default ``-n`` argument) is now ``/var/run`` + instead of ``${prefix}/var``. (Packages usually configure this to match local customs.) * 'Scientific Notation' numbers like 6.62607004e-34 are no longer @@ -47,17 +47,17 @@ Varnish Cache 7.x.x (2021-09-15) in context of HTTP headers. * Accept-Ranges headers are no longer generated for passed objects, - but must either come from the backend or be created in `vcl_deliver{}` + but must either come from the backend or be created in ``vcl_deliver{}`` -* ACLs no longer produce VSL `VCL_acl` records by default, this must be - explicitly enabled with `vcl +log { ... }`. +* ACLs no longer produce VSL ``VCL_acl`` records by default, this must be + explicitly enabled with ``vcl +log { ... }``. * ACLs can be compiled into a table format, which runs a little bit slower, but compiles much faster for large ACLs. -* ACLs default to `pedantic` which is now a per-ACL feature flag. +* ACLs default to ``pedantic`` which is now a per-ACL feature flag. -* `varnishncsa -j` will now accept to print fields with control characters. +* ``varnishncsa -j`` will now accept to print fields with control characters. ================================ Varnish Cache 6.6.0 (2021-03-15) From dridi.boukelmoune at gmail.com Wed Sep 1 07:29:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 1 Sep 2021 07:29:05 +0000 (UTC) Subject: [master] 021890f16 man: Hide comment from the vtc(7) manual Message-ID: <20210901072905.80B4B111C24@lists.varnish-cache.org> commit 021890f160bda052ee9aeb98fa53221fc84a64e6 Author: Dridi Boukelmoune Date: Wed Sep 1 09:28:06 2021 +0200 man: Hide comment from the vtc(7) manual diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 34c139db8..5a018cf89 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -138,7 +138,7 @@ * expect * fail clear * - * XXX can we come up with a better solution which is still safe? + * .. XXX can we come up with a better solution which is still safe? */ #include "config.h" From dridi.boukelmoune at gmail.com Wed Sep 1 10:50:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 1 Sep 2021 10:50:07 +0000 (UTC) Subject: [master] dbff3e425 changes: Collect noteworthy changes since 6.6.0 Message-ID: <20210901105007.2F7541170E0@lists.varnish-cache.org> commit dbff3e425003c2d4813bb6ec5a65eafee1db9919 Author: Dridi Boukelmoune Date: Wed Sep 1 12:46:27 2021 +0200 changes: Collect noteworthy changes since 6.6.0 diff --git a/doc/changes.rst b/doc/changes.rst index f18804a3c..c290a644c 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -32,32 +32,140 @@ individual releases. These documents are updated as part of the release process. ================================ -Varnish Cache 7.x.x (2021-09-15) +Varnish Cache 7.0.0 (2021-09-15) ================================ -* The minimum ``session_workspace`` is now 384 bytes +* Added convenience ``vrt_null_strands`` and ``vrt_null_blob`` constants. -* The default workdir (the default ``-n`` argument) is now ``/var/run`` - instead of ``${prefix}/var``. - (Packages usually configure this to match local customs.) +* New VCL flag syntax ``foo +bar -baz { ... }``, starting with ACL + flags ``log``, ``pedantic`` and ``table``. + +* ACLs no longer produce VSL ``VCL_acl`` records by default, this must be + explicitly enabled with ``acl +log { ... }``. + +* ACLs can be compiled into a table format, which runs a little bit + slower, but compiles much faster for large ACLs. + +* ACLs default to ``pedantic`` which is now a per-ACL feature flag. + +* New ``glob`` flag for VCL ``include`` (3193_). + +* The maximum number of headers for a request or a response in ``varnishtest`` + was increased to 64. + +* The backend lock class from struct backend was moved to struct director and + renamed accordingly. + +* New ``%{sec,msec,usec,msec_frac,usec_frac}t`` formats in ``varnishncsa``. + +* ``vstrerror()`` was renamed to ``VAS_errtxt()``. + +* New ``varnishncsa -j`` option to format for JSON (3595_). + +* To skip a test in the *presence* of a feature instead of it absence, a new + ``feature !`` syntax was added to ``varnishtest``. + +* Accept-Ranges headers are no longer generated for passed objects, + but must either come from the backend or be created in ``vcl_deliver{}`` + (3251_). + +* The busyobj ``do_pass`` flag is gone in favor of ``uncacheable``. + +* The objcore flag ABANDON was renamed to CANCEL. * 'Scientific Notation' numbers like 6.62607004e-34 are no longer supported in VCL. (The preparation of RFC8941 made it clear that there are neither reason nor any need to support scientific notation in context of HTTP headers. -* Accept-Ranges headers are no longer generated for passed objects, - but must either come from the backend or be created in ``vcl_deliver{}`` +* New ``tunnel`` command in ``varnishtest`` to gain the ability to + shape traffic between two peers without having to change their + implementation. -* ACLs no longer produce VSL ``VCL_acl`` records by default, this must be - explicitly enabled with ``vcl +log { ... }``. +* Global VCL symbols can be defined after use (3555_). -* ACLs can be compiled into a table format, which runs a little bit - slower, but compiles much faster for large ACLs. +* New ``req.hash_ignore_vary`` flag in VCL. -* ACLs default to ``pedantic`` which is now a per-ACL feature flag. +* ``varnishtest`` can register macros backed by functions, which is the case + for ``${date}`` and the brand new ``${string,[,...]}`` macro + (3627_). + +* Migration to pcre2 with extensive changes to the VRE API, parameters renamed + to ``pcre2_match_limit`` and ``pcre2_depth_limit``, and the addition of a + new ``pcre2_jit_compilation`` parameter. The ``varnishtest`` undocumented + feature check ``pcre_jit`` is gone (3635_). This change is transparent at + the VRT layer and only affects direct VRE consumers. + +* New inverted mode in ``vtc-bisect.sh`` to find the opposite of regressions. + +* The default values for ``workspace_client``, ``workspace_backend`` and + ``vsl_buffer`` on 64bit systems were increased to respectively 96kB, 96kB + and 16kB (3648_). + +* The deprecated ``WS_Inside()`` was replaced with ``WS_Allocated()`` and + ``WS_Front()`` was removed. + +* VCL header names can be quoted, for example ``req.http."valid.name"``. + +* Added ``VRT_UnsetHdr()`` and removed ``vrt_magic_string_unset``. + +* Removed depcreated ``STRING_LIST`` in favor of ``STRANDS``. All functions + that previously took a ``STRING_LIST`` had ``const char *, ...`` arguments, + they now take ``const char *, VCL_STRANDS`` arguments. The magic cookie + ``vrt_magic_string_end`` is gone and ``VRT_CollectStrands()`` was renamed to + ``VRT_STRANDS_string()``. + +* The default value for ``thread_pool_stack`` was increased to 80kB for 64bit + systems and 64kB for 32bit to accomodate the PCRE2 jit compiler. + +* Removed deprecated ``VSB_new()`` and ``VSB_delete()``, which resulted in a + major soname bump of libvarnishapi to 3.0.0, instead of the 2.7.0 version + initially planned. + +* The default workdir (the default ``-n`` argument) is now ``/var/run`` + instead of ``${prefix}/var`` (3672_). Packages usually configure this to + match local customs. + +* The minimum ``session_workspace`` is now 384 bytes + +* Emit minimal 500 response if ``vcl_synth`` fails (3441_). + +* New ``--enable-coverage`` configure flag, and renovated sanitizer setup. + +* New feature checks in ``varnishtest``: ``sanitizer``, ``asan``, ``lsan``, + ``msan``, ``ubsan`` and ``coverage``. + +* New ``--enable-workspace-emulator`` configure flag to swap the worksapce + implementation with a sparse one ideal for fuzzing (3644_). + +* Strict comparison of items from the HTTP grammar (3650_). + +* New request body h2 window handling using a buffer to avoid stalling an + entire h2 session until the relevant stream starts consuming DATA frames. + As a result the minimum value for ``h2_initial_window_size`` is now 65535B + to avoid running out of buffer with a negative window that was simpler to + not tolerate, and a new ``h2_rxbuf_storage`` parameter was added (3661_). + +* ``SLT_Hit`` now includes streaming progress when relevant. + +* The ``http_range_support`` adds consistency checks for pass transactions + (3673_). + +* New ``VNUM_uint()`` and ``VNUM_hex()`` functions geared at token parsing. -* ``varnishncsa -j`` will now accept to print fields with control characters. +.. _3193: https://github.com/varnishcache/varnish-cache/issues/3193 +.. _3251: https://github.com/varnishcache/varnish-cache/issues/3251 +.. _3441: https://github.com/varnishcache/varnish-cache/issues/3441 +.. _3555: https://github.com/varnishcache/varnish-cache/issues/3555 +.. _3595: https://github.com/varnishcache/varnish-cache/issues/3595 +.. _3627: https://github.com/varnishcache/varnish-cache/issues/3627 +.. _3635: https://github.com/varnishcache/varnish-cache/issues/3635 +.. _3644: https://github.com/varnishcache/varnish-cache/issues/3644 +.. _3648: https://github.com/varnishcache/varnish-cache/issues/3648 +.. _3650: https://github.com/varnishcache/varnish-cache/issues/3650 +.. _3661: https://github.com/varnishcache/varnish-cache/issues/3661 +.. _3672: https://github.com/varnishcache/varnish-cache/issues/3672 +.. _3673: https://github.com/varnishcache/varnish-cache/issues/3673 ================================ Varnish Cache 6.6.0 (2021-03-15) From dridi.boukelmoune at gmail.com Wed Sep 1 12:48:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 1 Sep 2021 12:48:06 +0000 (UTC) Subject: [master] f6ee60cb6 vtc: Attempt at stabilizing t02020 Message-ID: <20210901124806.B4A7F11A2F8@lists.varnish-cache.org> commit f6ee60cb6c3791568e08da013f0ebc16004d9640 Author: Dridi Boukelmoune Date: Wed Sep 1 14:44:35 2021 +0200 vtc: Attempt at stabilizing t02020 Suggestions from Martin. diff --git a/bin/varnishtest/tests/t02020.vtc b/bin/varnishtest/tests/t02020.vtc index 77738e17f..d1bd95c32 100644 --- a/bin/varnishtest/tests/t02020.vtc +++ b/bin/varnishtest/tests/t02020.vtc @@ -15,10 +15,12 @@ server s1 { } -start varnish v1 -vcl+backend { + import std; import vtc; sub vcl_recv { if (req.url == "/3") { vtc.barrier_sync("${b1_sock}"); + std.cache_req_body(128k); } } } -start @@ -26,6 +28,7 @@ varnish v1 -vcl+backend { varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.reset h2_initial_window_size" varnish v1 -cliok "param.reset h2_rx_window_low_water" +varnish v1 -cliok "param.set fetch_chunksize 129k" varnish v1 -cliok "param.set debug +syncvsl" client c1 { From dridi.boukelmoune at gmail.com Wed Sep 1 14:08:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 1 Sep 2021 14:08:06 +0000 (UTC) Subject: [master] bb098e3f0 range: Check content-range unsatisfied-range Message-ID: <20210901140806.C0749466D@lists.varnish-cache.org> commit bb098e3f09f3c93454a305e0e32f63829af6148d Author: Dridi Boukelmoune Date: Wed Sep 1 15:55:24 2021 +0200 range: Check content-range unsatisfied-range And while at it add coverage for `content-range: bytes */*` for good measure. Fixes #3683 diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 701ff3a81..35f0f4a67 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -898,8 +898,6 @@ http_GetContentRange(const struct http *hp, ssize_t *lo, ssize_t *hi) if (*lo > *hi) return (-2); assert(cl >= -1); - if (cl == -1) - return (-1); if (*lo >= cl || *hi >= cl) return (-2); AN(cl); diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index b9944aa6e..71793ab4e 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -287,7 +287,7 @@ VRG_CheckBo(struct busyobj *bo) return (-1); } - if (crlo < 0 || crhi < 0) { + if (crlo < 0 && crhi < 0 && crlen < 0) { AZ(http_GetHdr(bo->beresp, H_Content_Range, NULL)); return (0); } @@ -297,6 +297,13 @@ VRG_CheckBo(struct busyobj *bo) return (-1); } + if (crlo < 0) { // Content-Range: bytes */123 + assert(crhi < 0); + assert(crlen > 0); + crlo = 0; + crhi = crlen - 1; + } + #define RANGE_CHECK(val, op, crval, what) \ do { \ if (val >= 0 && !(val op crval)) { \ diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc index d34b91cbe..df6bd6056 100644 --- a/bin/varnishtest/tests/c00034.vtc +++ b/bin/varnishtest/tests/c00034.vtc @@ -246,9 +246,18 @@ server s1 { txresp -status 206 -hdr "content-range: bytes 10-19/100" -bodylen 40 rxreq - expect req.url == "/4" + expect req.url == "/?unexpected=content-range" expect req.http.range == txresp -hdr "content-range: bytes 0-49/100" -bodylen 40 + + rxreq + expect req.url == "/?unexpected=unsatisfied-range" + expect req.http.range == + txresp -hdr "content-range: bytes */100" -bodylen 100 + + rxreq + expect req.http.range == "bytes=0-0" + txresp -hdr "content-range: bytes */*" -bodylen 100 } -start varnish v1 -vcl+backend { @@ -270,7 +279,15 @@ client c8 { rxresp expect resp.status == 503 - txreq -url /4 + txreq -url "/?unexpected=content-range" + rxresp + expect resp.status == 503 + + txreq -url "/?unexpected=unsatisfied-range" + rxresp + expect resp.status == 503 + + txreq -hdr "range: bytes=0-0" -hdr "return: pass" rxresp expect resp.status == 503 } -run @@ -288,7 +305,7 @@ varnish v1 -vcl+backend { } } -client c1 { +client c9 { txreq -url /5 rxresp expect resp.status == 200 From dridi.boukelmoune at gmail.com Wed Sep 1 14:30:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 1 Sep 2021 14:30:10 +0000 (UTC) Subject: [master] c718adc4e varnishncsa: Fall time_firstbyte back to "missing int" Message-ID: <20210901143010.56631524C@lists.varnish-cache.org> commit c718adc4ea935a8e7d8a393e18d5b6878164ed96 Author: Dridi Boukelmoune Date: Wed Sep 1 16:28:11 2021 +0200 varnishncsa: Fall time_firstbyte back to "missing int" Fixes #3682 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 9c9faf8f6..9ae918b0e 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -607,7 +607,7 @@ parse_x_format(char *buf) int slt; if (!strcmp(buf, "Varnish:time_firstbyte")) { - addf_fragment(&CTX.frag[F_ttfb], ""); + addf_fragment(&CTX.frag[F_ttfb], CTX.missing_int); return; } if (!strcmp(buf, "Varnish:hitmiss")) { From dridi.boukelmoune at gmail.com Wed Sep 1 15:46:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 1 Sep 2021 15:46:05 +0000 (UTC) Subject: [master] a04abff76 Revert "vtc: Attempt at stabilizing t02020" Message-ID: <20210901154605.773EC78B2@lists.varnish-cache.org> commit a04abff768331e3e44b0e932ab2090438129e985 Author: Dridi Boukelmoune Date: Wed Sep 1 17:45:22 2021 +0200 Revert "vtc: Attempt at stabilizing t02020" This reverts commit f6ee60cb6c3791568e08da013f0ebc16004d9640. A better attempt is on its way. diff --git a/bin/varnishtest/tests/t02020.vtc b/bin/varnishtest/tests/t02020.vtc index d1bd95c32..77738e17f 100644 --- a/bin/varnishtest/tests/t02020.vtc +++ b/bin/varnishtest/tests/t02020.vtc @@ -15,12 +15,10 @@ server s1 { } -start varnish v1 -vcl+backend { - import std; import vtc; sub vcl_recv { if (req.url == "/3") { vtc.barrier_sync("${b1_sock}"); - std.cache_req_body(128k); } } } -start @@ -28,7 +26,6 @@ varnish v1 -vcl+backend { varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.reset h2_initial_window_size" varnish v1 -cliok "param.reset h2_rx_window_low_water" -varnish v1 -cliok "param.set fetch_chunksize 129k" varnish v1 -cliok "param.set debug +syncvsl" client c1 { From martin at varnish-software.com Wed Sep 1 15:55:06 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Wed, 1 Sep 2021 15:55:06 +0000 (UTC) Subject: [master] c8d0781df Stabilise r02020.vtc Message-ID: <20210901155506.6A1817EBE@lists.varnish-cache.org> commit c8d0781df30107481b3b199995973eee547dd034 Author: Martin Blix Grydeland Date: Wed Sep 1 17:52:29 2021 +0200 Stabilise r02020.vtc This makes use of the fact that Varnish will always send a session window update immediately when receiving a data frame. The four rxwinup before barrier sync on stream 0 matches up with the four txdata frames sent early on the stream, and makes sure that the session thread has exhausted its send window completely before opening up and starting to consume rxbuf data by unblocking the client thread stuck in vcl_recv. From that point on window updates will also be sent on the stream. diff --git a/bin/varnishtest/tests/t02020.vtc b/bin/varnishtest/tests/t02020.vtc index 77738e17f..6a2ace287 100644 --- a/bin/varnishtest/tests/t02020.vtc +++ b/bin/varnishtest/tests/t02020.vtc @@ -1,6 +1,6 @@ varnishtest "H/2 received data frames with padding" -barrier b1 sock 2 +barrier b1 sock 3 server s1 { rxreq @@ -34,7 +34,26 @@ client c1 { txdata -data abcde -padlen 1 rxresp expect resp.status == 200 - } -run + } -start +} -run + +client c2 { + # This makes use of the fact that Varnish will always send a + # session window update immediately when receiving a data + # frame. The four rxwinup before barrier sync on stream 0 matches + # up with the four txdata frames sent early on the stream, and + # makes sure that the session thread has exhausted its send window + # completely before opening up and starting to consume rxbuf data + # by unblocking the client thread stuck in vcl_recv. From that + # point on window updates will also be sent on the stream. + + stream 0 { + rxwinup + rxwinup + rxwinup + rxwinup + barrier b1 sync + } -start stream 3 { txreq -req POST -url /3 -hdr "content-length" "131072" -nostrend @@ -55,12 +74,14 @@ client c1 { txdata -datalen 672 rxresp expect resp.status == 200 - } -run + } -start +} -run +client c3 { stream 5 { txreq -req POST -url /5 -nostrend txdata -data a -padlen 255 rxresp expect resp.status == 200 - } -run + } -start } -run From dridi.boukelmoune at gmail.com Wed Sep 1 17:13:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 1 Sep 2021 17:13:05 +0000 (UTC) Subject: [master] b96d77c89 range: Let 416s pass through Message-ID: <20210901171305.2AC0D62605@lists.varnish-cache.org> commit b96d77c89cd0983e52e3ddca87b49fb7adc555df Author: Dridi Boukelmoune Date: Wed Sep 1 18:50:37 2021 +0200 range: Let 416s pass through Reported by Simon as a followup to #3683. diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index 71793ab4e..402884e5f 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -300,6 +300,8 @@ VRG_CheckBo(struct busyobj *bo) if (crlo < 0) { // Content-Range: bytes */123 assert(crhi < 0); assert(crlen > 0); + if (http_GetStatus(bo->beresp) == 416) + return (0); crlo = 0; crhi = crlen - 1; } diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc index df6bd6056..9f68605f1 100644 --- a/bin/varnishtest/tests/c00034.vtc +++ b/bin/varnishtest/tests/c00034.vtc @@ -245,6 +245,10 @@ server s1 { expect req.http.range == "bytes=10-19" txresp -status 206 -hdr "content-range: bytes 10-19/100" -bodylen 40 + rxreq + expect req.http.range == "bytes=90-119" + txresp -status 416 -hdr "content-range: bytes */100" -bodylen 100 + rxreq expect req.url == "/?unexpected=content-range" expect req.http.range == @@ -279,6 +283,10 @@ client c8 { rxresp expect resp.status == 503 + txreq -hdr "range: bytes=90-119" -hdr "return: pass" + rxresp + expect resp.status == 416 + txreq -url "/?unexpected=content-range" rxresp expect resp.status == 503 From dridi.boukelmoune at gmail.com Thu Sep 2 08:11:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 2 Sep 2021 08:11:08 +0000 (UTC) Subject: [master] 29b1b0666 range: Distinguish missing separator from unknown unit Message-ID: <20210902081108.41163114957@lists.varnish-cache.org> commit 29b1b066626d0cf3578feeacf757004f6757151c Author: Dridi Boukelmoune Date: Thu Sep 2 08:49:02 2021 +0200 range: Distinguish missing separator from unknown unit diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 245b7aa98..6955c1c8b 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -669,7 +669,7 @@ extern const char H__Reason[]; #define http_expect_eq(str, tok) http_tok_eq(str, #tok) // rfc7233,l,1207,1208 -#define http_range_at(str, tok) http_ctok_at(str, #tok) +#define http_range_at(str, tok, l) http_tok_at(str, #tok, l) /* cache_main.c */ #define VXID(u) ((u) & VSL_IDENTMASK) diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 35f0f4a67..c30d21ee6 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -846,7 +846,7 @@ ssize_t http_GetContentRange(const struct http *hp, ssize_t *lo, ssize_t *hi) { ssize_t tmp, cl; - const char *b; + const char *b, *t; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -860,14 +860,14 @@ http_GetContentRange(const struct http *hp, ssize_t *lo, ssize_t *hi) if (!http_GetHdr(hp, H_Content_Range, &b)) return (-1); - if (strncasecmp("bytes", b, strlen("bytes"))) - return (-1); // Unknown range unit, ignore - b += strlen("bytes"); + t = strchr(b, ' '); + if (t == NULL) + return (-2); // Missing space after range unit - if (!vct_islws(*b)) + if (!http_range_at(b, bytes, t - b)) return (-1); // Unknown range unit, ignore - while (vct_islws(*b)) - b++; + b = t + 1; + if (*b == '*') { // Content-Range: bytes */123 *lo = *hi = -1; b++; @@ -908,7 +908,7 @@ const char * http_GetRange(const struct http *hp, ssize_t *lo, ssize_t *hi) { ssize_t tmp; - const char *b; + const char *b, *t; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -922,9 +922,13 @@ http_GetRange(const struct http *hp, ssize_t *lo, ssize_t *hi) if (!http_GetHdr(hp, H_Range, &b)) return (NULL); - if (!http_range_at(b, bytes=)) + t = strchr(b, '='); + if (t == NULL) + return ("Missing '='"); + + if (!http_range_at(b, bytes, t - b)) return ("Not Bytes"); - b += strlen("bytes="); + b = t + 1; *lo = VNUM_uint(b, NULL, &b); if (*lo == -2) diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc index 9f68605f1..bc2af742e 100644 --- a/bin/varnishtest/tests/c00034.vtc +++ b/bin/varnishtest/tests/c00034.vtc @@ -259,6 +259,11 @@ server s1 { expect req.http.range == txresp -hdr "content-range: bytes */100" -bodylen 100 + rxreq + expect req.url == "/?invalid=content-range" + expect req.http.range == + txresp -hdr "content-range: bytes=0-99/100" -bodylen 100 + rxreq expect req.http.range == "bytes=0-0" txresp -hdr "content-range: bytes */*" -bodylen 100 @@ -295,6 +300,10 @@ client c8 { rxresp expect resp.status == 503 + txreq -url "/?invalid=content-range" + rxresp + expect resp.status == 503 + txreq -hdr "range: bytes=0-0" -hdr "return: pass" rxresp expect resp.status == 503 From dridi.boukelmoune at gmail.com Thu Sep 2 08:25:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 2 Sep 2021 08:25:07 +0000 (UTC) Subject: [master] 167e3bdcd Start skeleton release notes for the next version. Message-ID: <20210902082507.0C70F115066@lists.varnish-cache.org> commit 167e3bdcd3225fb5a4329da71dbcecaadcb295c2 Author: Geoff Simmons Date: Tue Sep 25 16:31:17 2018 +0200 Start skeleton release notes for the next version. Restructured so that: * 'Upgrading' is limited to work that has to be done to upgrade from a current deployment to the new version. * 'Changes' is a comprehensive, user-level description of changes and new features. Conflicts: doc/sphinx/whats-new/index.rst diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst new file mode 100644 index 000000000..0a9e8457f --- /dev/null +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -0,0 +1,78 @@ +.. + Copyright 2021 Varnish Software + SPDX-License-Identifier: BSD-2-Clause + See LICENSE file for full text of license + +**Note: This is a working document for a future release, with running +updates for changes in the development branch. For changes in the +released versions of Varnish, see:** :ref:`whats-new-index` + +.. _whatsnew_changes_CURRENT: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +Changes in Varnish **$NEXT_RELEASE** +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +For information about updating your current Varnish deployment to the +new version, see :ref:`whatsnew_upgrading_CURRENT`. + +A more detailed and technical account of changes in Varnish, with +links to issues that have been fixed and pull requests that have been +merged, may be found in the `change log`_. + +.. _change log: https://github.com/varnishcache/varnish-cache/blob/master/doc/changes.rst + +varnishd +======== + +Parameters +~~~~~~~~~~ + +**XXX changes in -p parameters** + +Other changes in varnishd +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Changes to VCL +============== + +VCL variables +~~~~~~~~~~~~~ + +**XXX new, deprecated or removed variables, or changed semantics** + +Other changes to VCL +~~~~~~~~~~~~~~~~~~~~ + +VMODs +===== + +**XXX changes in the bundled VMODs** + +varnishlog +========== + +**XXX changes concerning varnishlog(1) and/or vsl(7)** + +varnishadm +========== + +**XXX changes concerning varnishadm(1) and/or varnish-cli(7)** + +varnishstat +=========== + +**XXX changes concerning varnishstat(1) and/or varnish-counters(7)** + +varnishtest +=========== + +**XXX changes concerning varnishtest(1) and/or vtc(7)** + +Changes for developers and VMOD authors +======================================= + +**XXX changes concerning VRT, the public APIs, source code organization, +builds etc.** + +*eof* diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst index 37ad90d28..6eaf369a6 100644 --- a/doc/sphinx/whats-new/index.rst +++ b/doc/sphinx/whats-new/index.rst @@ -13,6 +13,19 @@ This section describes the changes and improvements between different versions of Varnish, and what upgrading between the different versions entail. +Varnish **$NEXT_RELEASE** +------------------------- + +**Note: These are working documents for a future release, with running +updates for changes in the development branch. For changes in the +released versions of Varnish, see the chapters listed below.** + +.. toctree:: + :maxdepth: 2 + + changes-trunk + upgrading-trunk + Varnish 6.6 ----------- diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst new file mode 100644 index 000000000..9f2dab437 --- /dev/null +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -0,0 +1,38 @@ +.. + Copyright 2021 Varnish Software + SPDX-License-Identifier: BSD-2-Clause + See LICENSE file for full text of license + +**Note: This is a working document for a future release, with running +updates for changes in the development branch. For changes in the +released versions of Varnish, see:** :ref:`whats-new-index` + +.. _whatsnew_upgrading_CURRENT: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +Upgrading to Varnish **$NEXT_RELEASE** +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +**XXX: how to upgrade from previous deployments to this +version. Limited to work that has to be done for an upgrade, new +features are listed in "Changes". Explicitly mention what does *not* +have to be changed, especially in VCL. May include, but is not limited +to:** + +* Elements of VCL that have been removed or are deprecated, or whose + semantics have changed. + +* -p parameters that have been removed or are deprecated, or whose + semantics have changed. + +* Changes in the CLI. + +* Changes in the output or interpretation of stats or the log, including + changes affecting varnishncsa/-hist/-top. + +* Changes that may be necessary in VTCs or in the use of varnishtest. + +* Changes in public APIs that may require changes in VMODs or VAPI/VUT + clients. + +*eof* From dridi.boukelmoune at gmail.com Thu Sep 2 11:57:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 2 Sep 2021 11:57:05 +0000 (UTC) Subject: [master] aacf14f71 whats-new: First pass on the changes since 6.6.0 Message-ID: <20210902115705.51D3A11A863@lists.varnish-cache.org> commit aacf14f71056e1aaf27ea0fefd9f4c73c137a072 Author: Dridi Boukelmoune Date: Thu Sep 2 13:53:56 2021 +0200 whats-new: First pass on the changes since 6.6.0 I didn't bother adding references to other parts of the documentation. diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst index 0a9e8457f..dda4957ab 100644 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -22,56 +22,245 @@ merged, may be found in the `change log`_. .. _change log: https://github.com/varnishcache/varnish-cache/blob/master/doc/changes.rst +PCRE2 +===== + +One major change for this release is the migration of the regular expression +engine from PCRE to PCRE2. This change should be mostly transparent anywhere +regular expressions are used, like VCL, ban expressions, VSL queries etc. + +There were some parameters changes, see the upgrade notes for more details. + +Structured Fields +================= + +TODO (probably deserves its own section too) + varnishd ======== Parameters ~~~~~~~~~~ -**XXX changes in -p parameters** +There were changes to the parameters: + +- new ``pcre2_jit_compilation`` boolean defaulting to on +- the default value increased to 16kB for ``vsl_buffer`` +- the default value increased to 96kB for ``workspace_client`` +- the default value increased to 96kB for ``workspace_backend`` +- the minimum value increased to 384B for ``workspace_session`` +- the minimum value increased to 65535B for ``h2_initial_window_size`` +- the minimum value increased to 10ms for ``thread_pool_fail_delay`` +- the default value increased to 80kB for ``thread_pool_stack`` +- the default value increased to 64kB for ``thread_pool_stack`` on 32bit + systems +- ``vcc_acl_pedantic`` was removed, see upgrade notes for more details. +- ``pcre_match_limit`` was renamed to ``pcre2_match_limit`` +- ``pcre_match_limit_recursion`` was renamed to ``pcre2_depth_limit`` +- new ``h2_rxbuf_storage`` defaulting to ``Transient``, see upgrade notes for + more details. Other changes in varnishd ~~~~~~~~~~~~~~~~~~~~~~~~~ +For pass transactions, ``varnishd`` no longer strips ``Range`` headers from +client requests or ``Accept-Range`` and ``Content-Range`` headers from backend +responses to allow partial delivery directly from the backend. + +When ``http_range_support`` is on (the default) a consistency check is +performed on the backend response and malformed or inconsistent responses +are treated as fetch errors. + +There is a new buffer for HTTP/2 request bodies allocated from storage. + +See upgrade notes for more details on both topics. + Changes to VCL ============== VCL variables ~~~~~~~~~~~~~ -**XXX new, deprecated or removed variables, or changed semantics** +A new ``req.hash_ignore_vary`` flag allows to skip vary header checks during a +lookup. This can be useful when only the freshness of a resource is relevant, +and not a slight difference in representation. + +For interoperability purposes, it is now possible to quote header names that +aren't valid VCL symbols, but valid HTTP header names, for example:: + + req.http."dotted.name" + +This is rarely observed and should only be needed to better integrate with the +specific needs of certain clients or servers. + +Some global VCL symbols can be referenced before their declaration, this was +extended to all global VCL symbols for the following keywords: + +- ``acl`` +- ``backend`` +- ``probe`` +- ``sub`` + +Consider the following example:: + + sub vcl_recv { + set req.backend_hint = b; + } + + backend b { + .host = "example.org"; + } + +It used to fail the VCL compilation with "Symbol not found: 'b'" in +``vcl_recv``, and is now supported. + +Bit flags +~~~~~~~~~ + +There is a new bit flag syntax for certain VCL keywords:: + + keyword +flag -other ... + +Similarly to bit flag ``varnishd`` parameters ``debug``, ``feature`` and +``vsl_mask``, a ``+`` prefix means that a flag is raised and a ``-`` prefix +that a flag is cleared. + +The ``acl`` keyword supports the following flags: + +- ``log`` +- ``pedantic`` (enabled by default) +- ``table`` + +For example:: + + acl +log -pedantic { ... } + +The ``include`` keyword supports a ``glob`` flag. + +For example:: + + include +glob "example.org/*.vcl"; -Other changes to VCL -~~~~~~~~~~~~~~~~~~~~ +See upgrade notes for more details. VMODs ===== -**XXX changes in the bundled VMODs** +New ``BASE64CF`` encoding scheme in ``vmod_blob``. It is similar to +``BASE64URL``, with the following changes to ``BASE64``: + +- ``+`` replaced with ``-`` +- ``/`` replaced with ``~`` +- ``_`` as the padding character + +It is used by a certain CDN provider who also inspired the name. varnishlog ========== -**XXX changes concerning varnishlog(1) and/or vsl(7)** +If a cache hit occurs on a streaming object, an object that is still being +fetched, ``Hit`` records contain progress of the fetch task. This should help +troubleshooting when cache hits appear to be slow, whether or not the backend +is still serving the response. + +By default ``VCL_acl`` records are no longer emitted. They can be brought back +by adding a ``+log`` flag to the ACL declaration. + +varnishncsa +=========== + +New ``%{...}t`` time formats: + +- ``sec`` +- ``msec`` +- ``usec`` +- ``msec_frac`` +- ``usec_frac`` + +See the varnishncsa manual for more information. varnishadm ========== -**XXX changes concerning varnishadm(1) and/or varnish-cli(7)** +The ``-t`` option sets up the timeout for both attaching to a running +``varnishd`` instance and individual commands sent to that instance. -varnishstat -=========== - -**XXX changes concerning varnishstat(1) and/or varnish-counters(7)** +Command completion should be more accurate in interactive mode. varnishtest =========== -**XXX changes concerning varnishtest(1) and/or vtc(7)** +Test cases should be generally more reactive, whether it is detecting +a ``varnishd`` startup failure, waiting for ``varnishd`` to stop, or +when fail tests and there are barriers waiting for a synchronization. + +Clients and servers can have up to 64 headers in requests and responses. + +The ``feature`` command allows to skip gracefully test cases that are +missing specific requirements. It is now possible to skip a test based on +the presence of a feature. + +For example, for test cases targeting 32bit environment with a working DNS +setup:: + + feature dns !64bit + +There are new feature checks: + +- ``coverage`` +- ``asan`` +- ``msan`` +- ``tsan`` +- ``ubsan`` +- ``sanitizer`` +- ``workspace_emulator`` + +The undocumented ``pcre_jit`` feature check is gone. + +There is a new ``tunnel`` command that acts as a proxy between two peers. A +tunnel can pause and control how much data goes in each direction, and can +be used to trigger socket timeouts, possibly in the middle of protocol frames, +without having to change how the peers are implemented. + +There is a new dynamic macro ``${string,repeat,,}`` to avoid +very long lines or potential mistakes when maintained by hand. For example, +the two following strings are equivalent:: + + "AAA" + "${string,repeat,3,A}" + +There were also various improvements to HTTP/2 testing, and more should be +expected. Changes for developers and VMOD authors ======================================= +Varnish now comes with a second workspace implementation called the workspace +emulator. It needs to be enabled during the build with the configure flag +``--enable-workspace-emulator``. + +The workspace emulator performs sparse allocations and can help VMOD authors +interested in fuzzing, especially when the Address Sanitizer is enabled as +well. + +In order to make the emulator possible, some adjustments were needed for the +workspace API. Deprecated functions ``WS_Front()`` and ``WS_Inside()`` were +removed independently of the emulator. + +The ``STRING_LIST`` type is gone in favor of ``STRANDS``. All the VRT symbols +related to ``STRING_LIST`` are either gone or changed. + +Convenience constants ``vrt_null_strands`` and ``vrt_null_blob`` were added. + +The migration to PCRE2 also brought many changes to the VRE API. The VRT +functions handling ``REGEX`` arguments didn't change. + +The VNUM API also changed substantially for structured field number parsing. + +The deprecated functions ``VSB_new()`` and ``VSB_delete()`` were removed. + +See upgrade notes for more information. + **XXX changes concerning VRT, the public APIs, source code organization, builds etc.** From dridi.boukelmoune at gmail.com Thu Sep 2 15:05:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 2 Sep 2021 15:05:07 +0000 (UTC) Subject: [master] dab48c1b3 doc: Connected timestamp disambiguation attempt Message-ID: <20210902150507.3FCF361448@lists.varnish-cache.org> commit dab48c1b3e720f25d5edb75a0795697d4648e8cc Author: Dridi Boukelmoune Date: Thu Sep 2 17:03:06 2021 +0200 doc: Connected timestamp disambiguation attempt Spotted by Martin. diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst index cf63089db..3e174a979 100644 --- a/doc/sphinx/reference/vsl.rst +++ b/doc/sphinx/reference/vsl.rst @@ -95,7 +95,8 @@ Fetch Came off vcl_backend_fetch ready to send the backend request. Connected - Successfully established or reused a backend connection. + Successfully established a backend connection, or selected a recycled + connection from the pool. Bereq Backend request sent. From dridi.boukelmoune at gmail.com Thu Sep 2 17:12:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 2 Sep 2021 17:12:06 +0000 (UTC) Subject: [master] 473897cda vre: Don't count on the capture of the 0th group Message-ID: <20210902171206.7123A654E3@lists.varnish-cache.org> commit 473897cda00a35188a7bb11cfe2af913c3406e6f Author: Dridi Boukelmoune Date: Thu Sep 2 18:59:01 2021 +0200 vre: Don't count on the capture of the 0th group Using groups[0].e turns out to be unreliable to print the suffix of the subject string for a regsub operation. On Debian buster, with the help of ASAN we can observe uninitialized memory through the remains of ASAN's 0xbe pattern that leads later to a complaint about an invalid pointer: runtime error: pointer index expression with base 0x6310000a0816 overflowed to 0xbebf21cebec8c6d4 With a simple subtraction we can confirm the offset added to the base address: 0xbebf21cebec8c6d4 - 0x6310000a0816 = 0xbebebebebebebebe To work around the possibility of an uninitialized ovector depending on the pcre2 version, we initialize all offsets to PCRE2_UNSET and when we encounter that value we capture a safe empty token. This means that at the end of VRE_sub() we can no longer count on the capture of the 0th group and revert back to using the offset. diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index 0cf5c9bfd..f5dcc848e 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -188,7 +188,7 @@ vre_capture(const vre_t *code, const char *subject, size_t length, { pcre2_match_data *data; pcre2_code *re; - PCRE2_SIZE *ovector; + PCRE2_SIZE *ovector, b, e; size_t nov, g; int matches; @@ -202,6 +202,11 @@ vre_capture(const vre_t *code, const char *subject, size_t length, AN(data); } + ovector = pcre2_get_ovector_pointer(data); + nov = 2 * pcre2_get_ovector_count(data); + for (g = 0; g < nov; g++) + ovector[g] = PCRE2_UNSET; + matches = pcre2_match(re, (PCRE2_SPTR)subject, length, offset, options, data, code->re_ctx); @@ -213,8 +218,14 @@ vre_capture(const vre_t *code, const char *subject, size_t length, if (nov > *count) nov = *count; for (g = 0; g < nov; g++) { - groups->b = subject + ovector[2 * g]; - groups->e = subject + ovector[2 * g + 1]; + b = ovector[2 * g]; + e = ovector[2 * g + 1]; + if (b == PCRE2_UNSET) { + groups->b = groups->e = ""; + } else { + groups->b = subject + b; + groups->e = subject + e; + } groups++; } *count = nov; @@ -334,7 +345,7 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement, } /* Copy suffix to match */ - VSB_cat(vsb, groups[0].e); + VSB_cat(vsb, subject + offset); return (1); } From dridi.boukelmoune at gmail.com Fri Sep 3 09:19:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 Sep 2021 09:19:08 +0000 (UTC) Subject: [master] dd3157c6a whats-new: Clarify what the TODO is about Message-ID: <20210903091908.200DA118CA3@lists.varnish-cache.org> commit dd3157c6ae9b952e4f253c983ad182125f27cb5b Author: Dridi Boukelmoune Date: Fri Sep 3 11:17:58 2021 +0200 whats-new: Clarify what the TODO is about diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst index dda4957ab..591001b86 100644 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -31,8 +31,8 @@ regular expressions are used, like VCL, ban expressions, VSL queries etc. There were some parameters changes, see the upgrade notes for more details. -Structured Fields -================= +Structured Fields numbers +========================= TODO (probably deserves its own section too) From dridi.boukelmoune at gmail.com Fri Sep 3 09:49:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 Sep 2021 09:49:05 +0000 (UTC) Subject: [master] 6b94d9d73 whats-new: Remove incorrect change Message-ID: <20210903094905.AB4681199F7@lists.varnish-cache.org> commit 6b94d9d73839a33ec7d9cdc01597d9e0435fb5d1 Author: Dridi Boukelmoune Date: Fri Sep 3 11:48:09 2021 +0200 whats-new: Remove incorrect change diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst index 591001b86..f2fe4f9de 100644 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -50,7 +50,6 @@ There were changes to the parameters: - the default value increased to 96kB for ``workspace_backend`` - the minimum value increased to 384B for ``workspace_session`` - the minimum value increased to 65535B for ``h2_initial_window_size`` -- the minimum value increased to 10ms for ``thread_pool_fail_delay`` - the default value increased to 80kB for ``thread_pool_stack`` - the default value increased to 64kB for ``thread_pool_stack`` on 32bit systems From dridi.boukelmoune at gmail.com Fri Sep 3 10:18:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 Sep 2021 10:18:05 +0000 (UTC) Subject: [master] 0600f9b92 whats-new: First pass on upgrade notes Message-ID: <20210903101805.7206611A711@lists.varnish-cache.org> commit 0600f9b9240c6f8905732f0ea95378c538bcb978 Author: Dridi Boukelmoune Date: Fri Sep 3 12:13:45 2021 +0200 whats-new: First pass on upgrade notes I believe I have laid out all upgrade-worthy sections, and started filling them out. diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst index f2fe4f9de..fce689251 100644 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -132,7 +132,7 @@ The ``acl`` keyword supports the following flags: For example:: - acl +log -pedantic { ... } + acl +log -pedantic { ... } The ``include`` keyword supports a ``glob`` flag. diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst index 9f2dab437..fbeabfed0 100644 --- a/doc/sphinx/whats-new/upgrading-trunk.rst +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -13,26 +13,150 @@ released versions of Varnish, see:** :ref:`whats-new-index` Upgrading to Varnish **$NEXT_RELEASE** %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -**XXX: how to upgrade from previous deployments to this -version. Limited to work that has to be done for an upgrade, new -features are listed in "Changes". Explicitly mention what does *not* -have to be changed, especially in VCL. May include, but is not limited -to:** +PCRE2 +===== -* Elements of VCL that have been removed or are deprecated, or whose - semantics have changed. +The migration from PCRE to PCRE2 led to many changes, starting with a +change of build dependencies. See the current installation notes for +package dependencies on various platforms. -* -p parameters that have been removed or are deprecated, or whose - semantics have changed. +Previously the Just In Time (jit) compilation of regular expressions was +always enabled at run time if it was present during the build. From now +on jit compilation is enabled by default, but can be disabled with the +``--disable-pcre2-jit`` configure option. Once enabled, jit compilation +is merely attempted and failures are ignored since they are not essential. -* Changes in the CLI. +The new ``varnishd`` parameter ``pcre2_jit_compilation`` controls whether +jit compilation should be attempted and has no effect if jit support was +disabled at configure time. -* Changes in the output or interpretation of stats or the log, including - changes affecting varnishncsa/-hist/-top. +The former parameters ``pcre_match_limit`` and ``pcre_match_limit_recursion`` +were renamed to ``pcre2_match_limit`` and ``pcre2_depth_limit``. With older +PCRE2 libraries, it is possible to see the depth limit being referred to as +recursion limit in error messages. -* Changes that may be necessary in VTCs or in the use of varnishtest. +The syntax of regular expression should be the same, but it is possible to +run into subtle differences. We are aware one such difference, PCRE2 fails +the compilation of unknown escape sequences. For example PCRE interprets +``"\T"`` as ``"T"`` and ignores the escape character, but PCRE2 sees it as +a syntax error. The solution is to simply use ``"T"`` and in general remove +all spurious escape characters. -* Changes in public APIs that may require changes in VMODs or VAPI/VUT - clients. +While PCRE2 can capture named groups and has its own substitution syntax +where captured groups can be referred to by position with ``$`` or even +by name. The substitution syntax for VCL's ``regsub()`` remains the same and +captured groups still require the ``\`` syntax where ``\1`` refers to +the first group. + +For this reason, there shouldn't be changes required to existing VCL, ban +expressions, VSL queries, or anything working with regular expression in +Varnish, except maybe where PCRE2 seems to be stricter and refuses invalid +escape sequences. + +VMOD authors manipulating ``VCL_REGEX`` arguments should not be affected by +this migration if they only use the VRT API. However, the underlying VRE API +was substantially changed and the new revision of VRE allowed to cover all +the Varnish use cases so that ``libvarnish`` is now the only binary linking +*directly* to ``libpcre2-8``. + +The migration implies that bans persisted in the deprecated persistent storage +are no longer compatible and a new deprecated persistent storage should be +rebuilt from scratch. + +Structured Fields numbers +========================= + +TODO + +Memory footprint +================ + +In order to lower the likelihood of flushing the logs of a single task more +than once, the default value for ``vsl_buffer`` was increased to 16kB. This +should generally result in better performance with tools like ``varnishlog`` +or ``varnishncsa`` except for ``raw`` grouping. + +To accommodate this extra workspace consumption and add even more headroom +on top of it, ``workspace_client`` and ``workspace_backend`` both increased +to 96kB by default. + +The PCRE2 jit compiler produces code that consumes more stack, so the default +value of ``thread_pool_stack`` was increased to 80kB, and to 64kB on 32bit +systems. + +If you are relying on default values, this will result in an increase of +virtual memory consumption proportional to the number of concurrent client +requests and backend fetches being processed. This memory is not accounted +for in the storage limits that can be applied. + +To address a potential head of line blocking scenario with HTTP/2, request +bodies are now buffered between the HTTP/2 session (stream 0) and the client +request. This is allocated on storage, controlled by the ``h2_rxbuf_storage`` +parameter and comes in addition to the existing buffering between a client +request and a backend fetch also allocated on storage. The new buffer size +depends on ``h2_initial_window_size`` that has a new default value of 65535B +to avoid having streams with negative windows. + +Range requests +============== + +Varnish only supports bytes units for range requests and always stripped +``Accept-Range`` headers coming from the backend. This is no longer the case +for pass transactions. + +To find out whether an ``Accept-Range`` header came from the backend, the +``obj.uncacheable`` in ``vcl_deliver`` indicates whether this was a pass +transaction. + +When ``http_range_support`` is on, a consistency check is added to ensure +the backend doesn't act as a bad gateway. If an unexpected ``Content-Range`` +header is received, or if it doesn't match the client's ``Range`` header, +it is considered an error and a 503 response is generated instead. + +If your backend adds spurious ``Content-Range`` headers that you can assess +are safe to ignore, you can amend the response in VCL:: + + sub vcl_backend_response { + if (!bereq.http.range) { + unset beresp.http.content-range; + } + } + +When a consistency check fails, an error is logged with the specific range +problem encountered. + +ACL +=== + +The ``acl`` keyword in VCL now supports bit flags: + +- ``log`` +- ``pedantic`` (enabled by default) +- ``table`` + +The global parameter ``vcc_acl_pedantic`` (off by default) was removed, and +as a result ACLs are now pedantic by default. TODO: reference to manual. + +They are also quiet by default, the following ACL declarations are +equivalent:: + + acl { ... } + acl -log +pedantic -table { ... } + +This means that the entry an ACL matched is no longer logged as ``VCL_acl`` by +default. + +To restore the previous default behavior, declare your ACL like this:: + + acl +log -pedantic { ... } + +ACLs are optimized for runtime performance by default, which can increase +significantly the VCL compilation time with very large ACLs. The ``table`` +flag improves compilation time at the expense of runtime performance. + +Changes for VMOD authors +======================== + +TODO *eof* From dridi.boukelmoune at gmail.com Fri Sep 3 10:38:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 Sep 2021 10:38:06 +0000 (UTC) Subject: [master] ff44f062f param: Turn h2_rxbuf_storage into a string parameter Message-ID: <20210903103806.4FC9211B0BC@lists.varnish-cache.org> commit ff44f062ff70dfc1dfc7525537c927b8b92f37ce Author: Dridi Boukelmoune Date: Fri Sep 3 12:27:31 2021 +0200 param: Turn h2_rxbuf_storage into a string parameter And simply require string parameters to define their tweaks. Refs #3661 diff --git a/include/tbl/params.h b/include/tbl/params.h index ade2b944a..4e23d7845 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1551,11 +1551,12 @@ PARAM_THREAD( * String parameters */ -# define PARAM_STRING(nm, pv, def, ...) \ - PARAM(, , nm, tweak_string, pv, NULL, NULL, def, NULL, __VA_ARGS__) +# define PARAM_STRING(nm, tw, pv, def, ...) \ + PARAM(, , nm, tw, pv, NULL, NULL, def, NULL, __VA_ARGS__) PARAM_STRING( /* name */ cc_command, + /* tweak */ tweak_string, /* priv */ &mgt_cc_cmd, /* def */ VCC_CC, /* descr */ @@ -1566,8 +1567,20 @@ PARAM_STRING( /* flags */ MUST_RELOAD ) +PARAM_STRING( + /* name */ h2_rxbuf_storage, + /* tweak */ tweak_h2_rxbuf_storage, + /* priv */ &mgt_stv_h2_rxbuf, + /* def */ "Transient", + /* descr */ + "The name of the storage backend that HTTP/2 receive buffers " + "should be allocated from.", + /* flags */ MUST_RESTART +) + PARAM_STRING( /* name */ vcl_path, + /* tweak */ tweak_string, /* priv */ &mgt_vcl_path, /* def */ VARNISH_VCL_DIR, /* descr */ @@ -1582,6 +1595,7 @@ PARAM_STRING( PARAM_STRING( /* name */ vmod_path, + /* tweak */ tweak_string, /* priv */ &mgt_vmod_path, /* def */ VARNISH_VMOD_DIR, /* descr */ @@ -1657,28 +1671,10 @@ PARAM_PCRE2( " messages." ) -/*-------------------------------------------------------------------- - * Custom parameters with separate tweak function - */ - -# define PARAM_CUSTOM(nm, pv, def, ...) \ - PARAM(, , nm, tweak_ ## nm, pv, NULL, NULL, def, NULL, __VA_ARGS__) - -PARAM_CUSTOM( - /* name */ h2_rxbuf_storage, - /* priv */ &mgt_stv_h2_rxbuf, - /* def */ "Transient", - /* descr */ - "The name of the storage backend that HTTP/2 receive buffers" - " should be allocated from.", - /* flags */ MUST_RESTART -) - # undef PARAM_ALL # undef PARAM_PCRE2 # undef PARAM_STRING # undef PARAM_VCC -# undef PARAM_CUSTOM #endif /* defined(PARAM_ALL) */ #undef PARAM_MEMPOOL From dridi.boukelmoune at gmail.com Fri Sep 3 10:38:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 Sep 2021 10:38:06 +0000 (UTC) Subject: [master] e37c74b7f tweak: Rename the storage tweak to be neutral Message-ID: <20210903103806.6A2AE11B0BF@lists.varnish-cache.org> commit e37c74b7f97a42174f0b73ffc107db6242a9339f Author: Dridi Boukelmoune Date: Fri Sep 3 12:32:57 2021 +0200 tweak: Rename the storage tweak to be neutral We could have shortlived_storage and req_body_storage parameters to stop requiring Transient for those special cases. Refs #3661 diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index 29f472299..f31fc1078 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -71,6 +71,7 @@ tweak_t tweak_bytes; tweak_t tweak_bytes_u; tweak_t tweak_double; tweak_t tweak_poolparam; +tweak_t tweak_storage; tweak_t tweak_string; tweak_t tweak_thread_pool_min; tweak_t tweak_thread_pool_max; @@ -78,7 +79,6 @@ tweak_t tweak_timeout; tweak_t tweak_uint; tweak_t tweak_vsl_buffer; tweak_t tweak_vsl_reclen; -tweak_t tweak_h2_rxbuf_storage; extern struct parspec mgt_parspec[]; /* mgt_param_tbl.c */ extern struct parspec VSL_parspec[]; /* mgt_param_vsl.c */ diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c index 192cb5259..2971eaa1b 100644 --- a/bin/varnishd/mgt/mgt_param_tweak.c +++ b/bin/varnishd/mgt/mgt_param_tweak.c @@ -491,13 +491,12 @@ tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par, } /*-------------------------------------------------------------------- - * Tweak 'h2_rxbuf_storage' + * Tweak storage * */ int v_matchproto_(tweak_t) -tweak_h2_rxbuf_storage(struct vsb *vsb, const struct parspec *par, - const char *arg) +tweak_storage(struct vsb *vsb, const struct parspec *par, const char *arg) { struct stevedore *stv; diff --git a/include/tbl/params.h b/include/tbl/params.h index 4e23d7845..8f3eaf8d4 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1569,7 +1569,7 @@ PARAM_STRING( PARAM_STRING( /* name */ h2_rxbuf_storage, - /* tweak */ tweak_h2_rxbuf_storage, + /* tweak */ tweak_storage, /* priv */ &mgt_stv_h2_rxbuf, /* def */ "Transient", /* descr */ From dridi.boukelmoune at gmail.com Fri Sep 3 13:59:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 Sep 2021 13:59:05 +0000 (UTC) Subject: [master] 4ad052f3d whats-new: First pass on the changes for developers Message-ID: <20210903135905.E6508611DF@lists.varnish-cache.org> commit 4ad052f3d57f4376ba34c1cb2492f47570d04b9d Author: Dridi Boukelmoune Date: Fri Sep 3 14:57:48 2021 +0200 whats-new: First pass on the changes for developers diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst index fbeabfed0..6b1a01990 100644 --- a/doc/sphinx/whats-new/upgrading-trunk.rst +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -154,8 +154,105 @@ ACLs are optimized for runtime performance by default, which can increase significantly the VCL compilation time with very large ACLs. The ``table`` flag improves compilation time at the expense of runtime performance. -Changes for VMOD authors -======================== +Changes for developers +======================= + +Build +----- + +There is a new ``--enable-workspace-emulator`` configure flag to replace the +regular "packed allocation" workspace with a "sparse allocation" alternative. +Combined with the Address Sanitizer it can help VMOD authors find memory +handling issues like buffer overflows that could otherwise be missed on a +regular workspace. + +Workspace API +------------- + +The deprecated functions ``WS_Front()`` and ``WS_Inside()`` are gone, they +were replaced by ``WS_Reservation()`` and ``WS_Allocated()``. For this reason +``WS_Assert_Allocated()`` was removed despite not being deprecated, since it +became redundant with ``assert(WS_Allocated(...))``. Accessing the workspace +front pointer only makes sense during a reservation, that's why ``WS_Front()`` +was deprecated in a previous release. + +It should no longer be needed to access ``struct ws`` fields directly, and +everything should be possible with the ``WS_*()`` functions. It even becomes +mandatory when the workspace emulator is enabled, the ``struct ws`` fields +have different semantics. + +``STRING_LIST`` +--------------- + +VMOD authors can no longer take ``STRING_LIST`` arguments in functions or +object methods. To work with string fragments, use ``VCL_STRANDS`` instead. + +As a result the following symbols are gone: + +- ``VRT_String()`` +- ``VRT_StringList()`` +- ``VRT_CollectString()`` +- ``vrt_magic_string_end`` + +Functions that used to take a ``STRING_LIST`` in the form of a prototype +ending with ``const char *, ...`` now take ``const char *, VCL_STRANDS``: + +- ``VRT_l_client_identity()`` +- ``VRT_l_req_method()`` +- ``VRT_l_req_url()`` +- ``VRT_l_req_proto()`` +- ``VRT_l_bereq_method()`` +- ``VRT_l_bereq_url()`` +- ``VRT_l_bereq_proto()`` +- ``VRT_l_beresp_body()`` +- ``VRT_l_beresp_proto()`` +- ``VRT_l_beresp_reason()`` +- ``VRT_l_beresp_storage_hint()`` +- ``VRT_l_beresp_filters()`` +- ``VRT_l_resp_body()`` +- ``VRT_l_resp_proto()`` +- ``VRT_l_resp_reason()`` +- ``VRT_l_resp_filters()`` + +The ``VRT_SetHdr()`` function also used to take a ``STRING_LIST`` and now +takes a ``const char *, VCL_STRANDS`` too. But, in addition to this change, +it also no longer accepts the special ``vrt_magic_string_unset`` argument. + +Instead, a new ``VRT_UnsetHdr()`` function was added. + +The ``VRT_CollectStrands()`` function was renamed to ``VRT_STRANDS_string()``, +which was its original intended name. + +Null sentinels +-------------- + +Two convenience sentinels ``vrt_null_strands`` and ``vrt_null_blob`` were +added to avoid ``NULL`` usage. ``VRT_blob()`` returns ``vrt_null_blob`` when +the source is null or the length is zero. The null blob has the type +``VRT_NULL_BLOB_TYPE``. + +libvarnishapi +------------- + +Deprecated functions ``VSB_new()`` and ``VSB_delete()`` were removed. Use +``VSB_init()`` and ``VSB_fini()`` for static buffers and ``VSB_new_auto()`` +and ``VSB_destroy()`` for dynamic buffers. + +Their removal resulted in bumping the soname to 3.0.0 for libvarnishapi. + +libvarnish +---------- + +Other changes were made to libvarnish, those are only available to VMOD +authors since they are not exposed by libvarnishapi. + +VNUM +'''' + +TODO + +VRE +''' TODO diff --git a/include/vrt.h b/include/vrt.h index 24d11fd10..5e2e8f7d4 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -59,8 +59,24 @@ * VCL_STRINGLIST, vrt_magic_string_end removed * VRT_String(), VRT_StringList(), VRT_CollectString() removed * VRT_CollectStrands() renamed to VRT_STRANDS_string() - * VRT_l_{beresp|resp}_body() changed to takes STRANDS argument - * VRT_SetHdr() changed to take `const char*` & `STRANDS` arguments. + * The following functions changed to take `const char *, STRANDS`: + * - VRT_l_client_identity() + * - VRT_l_req_method() + * - VRT_l_req_url() + * - VRT_l_req_proto() + * - VRT_l_bereq_method() + * - VRT_l_bereq_url() + * - VRT_l_bereq_proto() + * - VRT_l_beresp_body() + * - VRT_l_beresp_proto() + * - VRT_l_beresp_reason() + * - VRT_l_beresp_storage_hint() + * - VRT_l_beresp_filters() + * - VRT_l_resp_body() + * - VRT_l_resp_proto() + * - VRT_l_resp_reason() + * - VRT_l_resp_filters() + * - VRT_SetHdr() * VRT_UnsetHdr() added * vrt_magic_string_unset removed (use VRT_UnsetHdr() instead) * VNUMpfx() removed, SF_Parse_{Integer|Decimal|Number} added From dridi.boukelmoune at gmail.com Fri Sep 3 16:08:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 Sep 2021 16:08:06 +0000 (UTC) Subject: [master] 7a3e0b3bb whats-new: Upgrade notes for VRE Message-ID: <20210903160806.BD37D65482@lists.varnish-cache.org> commit 7a3e0b3bb066160060d65c789d0631565ea5e631 Author: Dridi Boukelmoune Date: Fri Sep 3 17:58:29 2021 +0200 whats-new: Upgrade notes for VRE diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst index 6b1a01990..23d9d85f3 100644 --- a/doc/sphinx/whats-new/upgrading-trunk.rst +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -254,6 +254,40 @@ TODO VRE ''' -TODO +The VRE API completely changed in preparation for the PCRE2 migration, in +order to funnel all PCRE usage in the Varnish source code through VRE. + +Similarly to how parameters were renamed, the ``match_recursion`` field from +``struct vre_limits`` was renamed to ``depth``. It has otherwise the same +meaning and purpose. + +Notable breaking changes: + +- ``VRE_compile()`` signature changed +- ``VRE_exec()`` was replaced: + - ``VRE_match()`` does simple matching + - ``VRE_capture()`` captures matched groups in a ``txt`` array + - ``VRE_sub()`` substitute matches with a replacement in a VSB +- ``VRE_error()`` prints an error message for all the functions above in a VSB +- ``VRE_export()`` packs a usable ``vre_t`` that can be persisted as a byte + stream + +The ``VRE_ERROR_NOMATCH`` symbol is now hard-linked like ``VRE_CASELESS``, and +``VRE_NOTEMPTY`` is no longer supported. There are no match options left in +the VRE facade but the ``VRE_match()``, ``VRE_capture()`` and ``VRE_sub()`` +functions still take an ``options`` argument to keep the ability of allowing +match options in the future. + +The ``VRE_ERROR_LEN`` gives a size that should be safe to avoid truncated +error messages in a static buffer. + +To gain full access to PCRE2 features from a regular expression provided via +``vre_t`` a backend-specific ``vre_pcre2.h`` contains a ``VRE_unpack()`` +function. This opens for example the door to ``pcre2_substitute()`` with the +PCRE2 substitution syntax and named capture groups as an alternative to VCL's +``regsub()`` syntax backed by ``VRE_sub()``. + +Ideally, ``vre_pcre2.h`` will be the only breaking change next time we move +to a different regular expression engine. Hopefully not too soon. *eof* diff --git a/include/vre.h b/include/vre.h index bb8702b22..854e96058 100644 --- a/include/vre.h +++ b/include/vre.h @@ -52,7 +52,7 @@ typedef struct vre vre_t; /* This maps to PCRE2 error codes */ extern const int VRE_ERROR_NOMATCH; -/* And those to PCRE2 options */ +/* And those to PCRE2 compilation options */ extern const unsigned VRE_CASELESS; vre_t *VRE_compile(const char *, unsigned, int *, int *, unsigned); From dridi.boukelmoune at gmail.com Fri Sep 3 16:08:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 Sep 2021 16:08:06 +0000 (UTC) Subject: [master] 34dbf94ca whats-new: A quick word on vdef.h changes Message-ID: <20210903160806.D102A65488@lists.varnish-cache.org> commit 34dbf94ca45be7f134758b101b8a7865ad94097e Author: Dridi Boukelmoune Date: Fri Sep 3 18:05:25 2021 +0200 whats-new: A quick word on vdef.h changes There's technically nothing breaking there that require upgrade notes, but it doesn't hurt, since it contains items related to both VNUM and VRE breaking changes. diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst index 23d9d85f3..d748425b6 100644 --- a/doc/sphinx/whats-new/upgrading-trunk.rst +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -166,6 +166,21 @@ Combined with the Address Sanitizer it can help VMOD authors find memory handling issues like buffer overflows that could otherwise be missed on a regular workspace. +``vdef.h`` +---------- + +The ``vdef.h`` header is no longer self-contained, it includes ``stddef.h``. + +Since it is the first header that should be included when working with Varnish +bindings, some definitions were promoted to ``vdef.h``: + +- a fallback for the ``__has_feature()`` macro in its absence +- VRT macros for Structured Fields number limits +- ``struct txt`` and its companion macros (the macros require ``vas.h`` too) + +This header is implicitly included by ``vrt.h`` and ``cache.h`` and should not +concern VMOD authors. + Workspace API ------------- From dridi.boukelmoune at gmail.com Fri Sep 3 16:58:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 Sep 2021 16:58:06 +0000 (UTC) Subject: [master] 7ff12fdc6 vre: Homework for later Message-ID: <20210903165806.CAD8C1000A9@lists.varnish-cache.org> commit 7ff12fdc69b0d0ac9ee3d287020157e000c437c6 Author: Dridi Boukelmoune Date: Fri Sep 3 18:48:01 2021 +0200 vre: Homework for later diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst index d748425b6..470271060 100644 --- a/doc/sphinx/whats-new/upgrading-trunk.rst +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -287,6 +287,10 @@ Notable breaking changes: - ``VRE_export()`` packs a usable ``vre_t`` that can be persisted as a byte stream +An exported regular expression takes the form of a byte stream of a given size +that can be used as-is by the various matching functions. Care should be taken +to always maintain pointer alignment of an exported ``vre_t``. + The ``VRE_ERROR_NOMATCH`` symbol is now hard-linked like ``VRE_CASELESS``, and ``VRE_NOTEMPTY`` is no longer supported. There are no match options left in the VRE facade but the ``VRE_match()``, ``VRE_capture()`` and ``VRE_sub()`` diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index f5dcc848e..a1ad3da51 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -135,6 +135,10 @@ pcre2_code * VRE_unpack(const vre_t *code) { + /* XXX: The ban code ensures that regex "lumps" are pointer-aligned, + * but coming for example from a VMOD there is no guarantee. Should + * we formally require that code is properly aligned? + */ CHECK_OBJ_NOTNULL(code, VRE_MAGIC); if (code->re == VRE_PACKED_RE) { AZ(code->re_ctx); From dridi.boukelmoune at gmail.com Mon Sep 6 04:12:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 6 Sep 2021 04:12:07 +0000 (UTC) Subject: [master] 9a0a613c6 tools: Help magic_check.sh work with other awks Message-ID: <20210906041207.5106F11647C@lists.varnish-cache.org> commit 9a0a613c649f06298348ce9e77cfafd1fba648af Author: Dridi Boukelmoune Date: Mon Sep 6 05:59:03 2021 +0200 tools: Help magic_check.sh work with other awks Like the one used by default on Debian Buster. Also, we don't want to `set -e` by default when we use `false` on purpose... diff --git a/tools/magic_check.sh b/tools/magic_check.sh index b5ec57ee0..18db22c27 100755 --- a/tools/magic_check.sh +++ b/tools/magic_check.sh @@ -33,7 +33,6 @@ # - 0x00112233 (most magic numbers) # - 0x00 (where size matters) -set -e set -u ROOT=$(git rev-parse --show-cdup 2>/dev/null) || exit 77 @@ -46,7 +45,7 @@ tr '[:upper:]' '[:lower:]' | sort | uniq -c | sort | -awk '$1 != 1 || $2 !~ /^0x[[:xdigit:]]*$/ || length($2) !~ /^(4|10)$/' | +awk '$1 != 1 || $2 !~ /^0x[0-9a-f]*$/ || length($2) !~ /^(4|10)$/' | while read -r COUNT MAGIC do if [ $COUNT -eq 1 ] From dridi.boukelmoune at gmail.com Mon Sep 6 05:14:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 6 Sep 2021 05:14:06 +0000 (UTC) Subject: [master] 9703c6cd4 ws: Move common code to a new cache_ws_common.c Message-ID: <20210906051406.13668117E9F@lists.varnish-cache.org> commit 9703c6cd40112a80990309411c3f8e5decb02db3 Author: Dridi Boukelmoune Date: Mon Sep 6 06:50:40 2021 +0200 ws: Move common code to a new cache_ws_common.c diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index e129b9a8a..583f9c5dc 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -53,6 +53,7 @@ varnishd_SOURCES = \ cache/cache_vrt_vcl.c \ cache/cache_vrt_vmod.c \ cache/cache_wrk.c \ + cache/cache_ws_common.c \ common/common_vsc.c \ common/common_vsmw.c \ hash/hash_classic.c \ diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 6955c1c8b..90b9d781e 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -780,13 +780,10 @@ void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv); /* cache_ws.c */ - void WS_Init(struct ws *ws, const char *id, void *space, unsigned len); unsigned WS_ReserveSize(struct ws *, unsigned); unsigned WS_ReserveAll(struct ws *); -unsigned WS_ReserveLumps(struct ws *ws, size_t sz); -void WS_MarkOverflow(struct ws *ws); void WS_Release(struct ws *ws, unsigned bytes); void WS_ReleaseP(struct ws *ws, const char *ptr); void WS_Assert(const struct ws *ws); @@ -794,14 +791,9 @@ void WS_Reset(struct ws *ws, uintptr_t); void *WS_Alloc(struct ws *ws, unsigned bytes); void *WS_Copy(struct ws *ws, const void *str, int len); uintptr_t WS_Snapshot(struct ws *ws); -int WS_Overflowed(const struct ws *ws); -const char *WS_Printf(struct ws *ws, const char *fmt, ...) v_printflike_(2, 3); int WS_Allocated(const struct ws *ws, const void *ptr, ssize_t len); unsigned WS_Dump(const struct ws *ws, char, size_t off, void *buf, size_t len); -void WS_VSB_new(struct vsb *, struct ws *); -char *WS_VSB_finish(struct vsb *, struct ws *, size_t *); - static inline void * WS_Reservation(const struct ws *ws) { @@ -820,6 +812,23 @@ WS_ReservationSize(const struct ws *ws) return (ws->r - ws->f); } +static inline unsigned +WS_ReserveLumps(struct ws *ws, size_t sz) +{ + + AN(sz); + return (WS_ReserveAll(ws) / sz); +} + +/* cache_ws_common.c */ +void WS_MarkOverflow(struct ws *ws); +int WS_Overflowed(const struct ws *ws); + +const char *WS_Printf(struct ws *ws, const char *fmt, ...) v_printflike_(2, 3); + +void WS_VSB_new(struct vsb *, struct ws *); +char *WS_VSB_finish(struct vsb *, struct ws *, size_t *); + /* cache_rfc2616.c */ void RFC2616_Ttl(struct busyobj *, vtim_real now, vtim_real *t_origin, float *ttl, float *grace, float *keep); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 47213fff6..fda1e4dba 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -527,7 +527,6 @@ void WRK_Init(void); void WRK_AddStat(const struct worker *); /* cache_ws.c */ -void WS_Id(const struct ws *ws, char *id); void WS_Panic(struct vsb *, const struct ws *); static inline int WS_IsReserved(const struct ws *ws) @@ -536,11 +535,14 @@ WS_IsReserved(const struct ws *ws) return (ws->r != NULL); } -void WS_Rollback(struct ws *, uintptr_t); void *WS_AtOffset(const struct ws *ws, unsigned off, unsigned len); unsigned WS_ReservationOffset(const struct ws *ws); unsigned WS_ReqPipeline(struct ws *, const void *b, const void *e); +/* cache_ws_common.c */ +void WS_Id(const struct ws *ws, char *id); +void WS_Rollback(struct ws *, uintptr_t); + /* http1/cache_http1_pipe.c */ void V1P_Init(void); diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 1db047a1a..02a3a867c 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -1,9 +1,10 @@ /*- * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2011 Varnish Software AS + * Copyright (c) 2006-2021 Varnish Software AS * All rights reserved. * * Author: Poul-Henning Kamp + * Author: Dridi Boukelmoune * * SPDX-License-Identifier: BSD-2-Clause * @@ -34,8 +35,6 @@ #include "cache_varnishd.h" -#include - #define WS_REDZONE_END '\x15' static const uintptr_t snap_overflowed = (uintptr_t)&snap_overflowed; @@ -100,32 +99,6 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len) WS_Assert(ws); } -void -WS_Id(const struct ws *ws, char *id) -{ - - WS_Assert(ws); - AN(id); - memcpy(id, ws->id, WS_ID_SIZE); - id[0] |= 0x20; // cheesy tolower() -} - -void -WS_MarkOverflow(struct ws *ws) -{ - CHECK_OBJ_NOTNULL(ws, WS_MAGIC); - - ws->id[0] &= ~0x20; // cheesy toupper() -} - -static void -ws_ClearOverflow(struct ws *ws) -{ - CHECK_OBJ_NOTNULL(ws, WS_MAGIC); - - ws->id[0] |= 0x20; // cheesy tolower() -} - /* * Reset a WS to a cookie from WS_Snapshot * @@ -156,24 +129,6 @@ WS_Reset(struct ws *ws, uintptr_t pp) WS_Assert(ws); } -/* - * Reset the WS to a cookie or its start and clears any overflow - * - * for varnishd internal use only - */ - -void -WS_Rollback(struct ws *ws, uintptr_t pp) -{ - WS_Assert(ws); - - if (pp == 0) - pp = (uintptr_t)ws->s; - - ws_ClearOverflow(ws); - WS_Reset(ws, pp); -} - /* * Make a reservation and optionally pipeline a memory region that may or * may not originate from the same workspace. @@ -252,28 +207,6 @@ WS_Copy(struct ws *ws, const void *str, int len) return (r); } -const char * -WS_Printf(struct ws *ws, const char *fmt, ...) -{ - unsigned u, v; - va_list ap; - char *p; - - u = WS_ReserveAll(ws); - p = ws->f; - va_start(ap, fmt); - v = vsnprintf(p, u, fmt, ap); - va_end(ap); - if (v >= u) { - WS_Release(ws, 0); - WS_MarkOverflow(ws); - p = NULL; - } else { - WS_Release(ws, v + 1); - } - return (p); -} - uintptr_t WS_Snapshot(struct ws *ws) { @@ -332,12 +265,6 @@ WS_ReserveSize(struct ws *ws, unsigned bytes) return (bytes); } -unsigned -WS_ReserveLumps(struct ws *ws, size_t sz) -{ - return (WS_ReserveAll(ws) / sz); -} - void WS_Release(struct ws *ws, unsigned bytes) { @@ -364,17 +291,6 @@ WS_ReleaseP(struct ws *ws, const char *ptr) WS_Assert(ws); } -int -WS_Overflowed(const struct ws *ws) -{ - CHECK_OBJ_NOTNULL(ws, WS_MAGIC); - AN(ws->id[0]); - - if (ws->id[0] & 0x20) // cheesy islower() - return (0); - return (1); -} - void * WS_AtOffset(const struct ws *ws, unsigned off, unsigned len) { @@ -394,60 +310,6 @@ WS_ReservationOffset(const struct ws *ws) return (ws->f - ws->s); } -/*--------------------------------------------------------------------- - * Build a VSB on a workspace. - * Usage pattern: - * - * struct vsb vsb[1]; - * char *p; - * - * WS_VSB_new(vsb, ctx->ws); - * VSB_printf(vsb, "blablabla"); - * p = WS_VSB_finish(vsb, ctx->ws, NULL); - * if (p == NULL) - * return (FAILURE); - */ - -void -WS_VSB_new(struct vsb *vsb, struct ws *ws) -{ - unsigned u; - static char bogus[2]; // Smallest possible vsb - - AN(vsb); - WS_Assert(ws); - u = WS_ReserveAll(ws); - if (WS_Overflowed(ws) || u < 2) - AN(VSB_init(vsb, bogus, sizeof bogus)); - else - AN(VSB_init(vsb, WS_Reservation(ws), u)); -} - -char * -WS_VSB_finish(struct vsb *vsb, struct ws *ws, size_t *szp) -{ - char *p; - - AN(vsb); - WS_Assert(ws); - if (!VSB_finish(vsb)) { - p = VSB_data(vsb); - if (p == ws->f) { - WS_Release(ws, VSB_len(vsb) + 1); - if (szp != NULL) - *szp = VSB_len(vsb); - VSB_fini(vsb); - return (p); - } - } - WS_MarkOverflow(ws); - VSB_fini(vsb); - WS_Release(ws, 0); - if (szp) - *szp = 0; - return (NULL); -} - /*--------------------------------------------------------------------*/ unsigned diff --git a/bin/varnishd/cache/cache_ws_common.c b/bin/varnishd/cache/cache_ws_common.c new file mode 100644 index 000000000..a23cd06af --- /dev/null +++ b/bin/varnishd/cache/cache_ws_common.c @@ -0,0 +1,171 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2021 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * Author: Dridi Boukelmoune + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include "config.h" + +#include + +#include "cache_varnishd.h" + +void +WS_Id(const struct ws *ws, char *id) +{ + + WS_Assert(ws); + AN(id); + memcpy(id, ws->id, WS_ID_SIZE); + id[0] |= 0x20; // cheesy tolower() +} + +void +WS_MarkOverflow(struct ws *ws) +{ + CHECK_OBJ_NOTNULL(ws, WS_MAGIC); + + ws->id[0] &= ~0x20; // cheesy toupper() +} + +static void +ws_ClearOverflow(struct ws *ws) +{ + CHECK_OBJ_NOTNULL(ws, WS_MAGIC); + + ws->id[0] |= 0x20; // cheesy tolower() +} + +int +WS_Overflowed(const struct ws *ws) +{ + CHECK_OBJ_NOTNULL(ws, WS_MAGIC); + AN(ws->id[0]); + + if (ws->id[0] & 0x20) // cheesy islower() + return (0); + return (1); +} + +/* + * Reset the WS to a cookie or its start and clears any overflow + * + * for varnishd internal use only + */ + +void +WS_Rollback(struct ws *ws, uintptr_t pp) +{ + + WS_Assert(ws); + + if (pp == 0) + pp = (uintptr_t)ws->s; + ws_ClearOverflow(ws); + WS_Reset(ws, pp); +} + +/*--------------------------------------------------------------------*/ + +const char * +WS_Printf(struct ws *ws, const char *fmt, ...) +{ + unsigned u, v; + va_list ap; + char *p; + + u = WS_ReserveAll(ws); + p = ws->f; + va_start(ap, fmt); + v = vsnprintf(p, u, fmt, ap); + va_end(ap); + if (v >= u) { + WS_Release(ws, 0); + WS_MarkOverflow(ws); + p = NULL; + } else { + WS_Release(ws, v + 1); + } + return (p); +} + +/*--------------------------------------------------------------------- + * Build a VSB on a workspace. + * Usage pattern: + * + * struct vsb vsb[1]; + * char *p; + * + * WS_VSB_new(vsb, ctx->ws); + * VSB_printf(vsb, "blablabla"); + * p = WS_VSB_finish(vsb, ctx->ws, NULL); + * if (p == NULL) + * return (FAILURE); + */ + +void +WS_VSB_new(struct vsb *vsb, struct ws *ws) +{ + unsigned u; + static char bogus[2]; // Smallest possible vsb + + AN(vsb); + WS_Assert(ws); + u = WS_ReserveAll(ws); + if (WS_Overflowed(ws) || u < 2) + AN(VSB_init(vsb, bogus, sizeof bogus)); + else + AN(VSB_init(vsb, WS_Reservation(ws), u)); +} + +char * +WS_VSB_finish(struct vsb *vsb, struct ws *ws, size_t *szp) +{ + char *p; + + AN(vsb); + WS_Assert(ws); + if (!VSB_finish(vsb)) { + p = VSB_data(vsb); + if (p == ws->f) { + WS_Release(ws, VSB_len(vsb) + 1); + if (szp != NULL) + *szp = VSB_len(vsb); + VSB_fini(vsb); + return (p); + } + } + WS_MarkOverflow(ws); + VSB_fini(vsb); + WS_Release(ws, 0); + if (szp) + *szp = 0; + return (NULL); +} diff --git a/bin/varnishd/cache/cache_ws_emu.c b/bin/varnishd/cache/cache_ws_emu.c index 631045bdd..95cb80800 100644 --- a/bin/varnishd/cache/cache_ws_emu.c +++ b/bin/varnishd/cache/cache_ws_emu.c @@ -39,7 +39,6 @@ #include "cache_varnishd.h" -#include #include struct ws_alloc { @@ -182,32 +181,6 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len) WS_Assert(ws); } -void -WS_Id(const struct ws *ws, char *id) -{ - - WS_Assert(ws); - AN(id); - memcpy(id, ws->id, WS_ID_SIZE); - id[0] |= 0x20; // cheesy tolower() -} - -void -WS_MarkOverflow(struct ws *ws) -{ - CHECK_OBJ_NOTNULL(ws, WS_MAGIC); - - ws->id[0] &= ~0x20; // cheesy toupper() -} - -static void -ws_ClearOverflow(struct ws *ws) -{ - CHECK_OBJ_NOTNULL(ws, WS_MAGIC); - - ws->id[0] |= 0x20; // cheesy tolower() -} - static void ws_alloc_free(struct ws_emu *we, struct ws_alloc **wap) { @@ -248,18 +221,6 @@ WS_Reset(struct ws *ws, uintptr_t pp) WS_Assert(ws); } -void -WS_Rollback(struct ws *ws, uintptr_t pp) -{ - - WS_Assert(ws); - - if (pp == 0) - pp = (uintptr_t)ws->s; - ws_ClearOverflow(ws); - WS_Reset(ws, pp); -} - unsigned WS_ReqPipeline(struct ws *ws, const void *b, const void *e) { @@ -375,28 +336,6 @@ WS_Copy(struct ws *ws, const void *str, int len) return (NULL); } -const char * -WS_Printf(struct ws *ws, const char *fmt, ...) -{ - unsigned u, v; - va_list ap; - char *p; - - u = WS_ReserveAll(ws); - p = ws->f; - va_start(ap, fmt); - v = vsnprintf(p, u, fmt, ap); - va_end(ap); - if (v >= u) { - WS_Release(ws, 0); - WS_MarkOverflow(ws); - p = NULL; - } else { - WS_Release(ws, v + 1); - } - return (p); -} - uintptr_t WS_Snapshot(struct ws *ws) { @@ -464,12 +403,6 @@ WS_ReserveSize(struct ws *ws, unsigned bytes) return (bytes); } -unsigned -WS_ReserveLumps(struct ws *ws, size_t sz) -{ - return (WS_ReserveAll(ws) / sz); -} - static void ws_release(struct ws *ws, unsigned bytes) { @@ -520,17 +453,6 @@ WS_ReleaseP(struct ws *ws, const char *ptr) DSL(DBG_WORKSPACE, 0, "WS_ReleaseP(%p, %p (%u))", ws, ptr, l); } -int -WS_Overflowed(const struct ws *ws) -{ - CHECK_OBJ_NOTNULL(ws, WS_MAGIC); - AN(ws->id[0]); - - if (ws->id[0] & 0x20) // cheesy islower() - return (0); - return (1); -} - void * WS_AtOffset(const struct ws *ws, unsigned off, unsigned len) { @@ -566,46 +488,6 @@ WS_ReservationOffset(const struct ws *ws) return (wa->off); } -void -WS_VSB_new(struct vsb *vsb, struct ws *ws) -{ - unsigned u; - static char bogus[2]; // Smallest possible vsb - - AN(vsb); - WS_Assert(ws); - u = WS_ReserveAll(ws); - if (WS_Overflowed(ws) || u < 2) - AN(VSB_init(vsb, bogus, sizeof bogus)); - else - AN(VSB_init(vsb, WS_Reservation(ws), u)); -} - -char * -WS_VSB_finish(struct vsb *vsb, struct ws *ws, size_t *szp) -{ - char *p; - - AN(vsb); - WS_Assert(ws); - if (!VSB_finish(vsb)) { - p = VSB_data(vsb); - if (p == ws->f) { - WS_Release(ws, VSB_len(vsb) + 1); - if (szp != NULL) - *szp = VSB_len(vsb); - VSB_fini(vsb); - return (p); - } - } - WS_MarkOverflow(ws); - VSB_fini(vsb); - WS_Release(ws, 0); - if (szp) - *szp = 0; - return (NULL); -} - unsigned WS_Dump(const struct ws *ws, char where, size_t off, void *buf, size_t len) { From phk at FreeBSD.org Mon Sep 6 06:19:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 6 Sep 2021 06:19:05 +0000 (UTC) Subject: [master] 46e3f3fdf Attempt to fix SunOs/gcc12 Message-ID: <20210906061905.9DD47119A1A@lists.varnish-cache.org> commit 46e3f3fdfdea1e4bff7c323f4129593c66419245 Author: Poul-Henning Kamp Date: Mon Sep 6 06:17:43 2021 +0000 Attempt to fix SunOs/gcc12 diff --git a/bin/varnishd/fuzzers/esi_parse_fuzzer.c b/bin/varnishd/fuzzers/esi_parse_fuzzer.c index 2cc659778..246e9180a 100644 --- a/bin/varnishd/fuzzers/esi_parse_fuzzer.c +++ b/bin/varnishd/fuzzers/esi_parse_fuzzer.c @@ -85,6 +85,13 @@ WS_Alloc(struct ws *ws, unsigned bytes) return (calloc(1, bytes)); } +unsigned +WS_ReserveAll(struct ws *ws) +{ + (void)ws; + WRONG("Should not be called"); +} + int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { From dridi.boukelmoune at gmail.com Mon Sep 6 07:08:39 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 6 Sep 2021 07:08:39 +0000 (UTC) Subject: [master] aadc45de1 waiter: Whitespace OCD Message-ID: <20210906070839.7228B11AEF8@lists.varnish-cache.org> commit aadc45de150aa03178110606306a252e4c356bb6 Author: Dridi Boukelmoune Date: Mon Sep 6 09:07:09 2021 +0200 waiter: Whitespace OCD diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c index ea77af96c..b3ab4b618 100644 --- a/bin/varnishd/waiter/cache_waiter_poll.c +++ b/bin/varnishd/waiter/cache_waiter_poll.c @@ -97,7 +97,7 @@ static void vwp_add(struct vwp *vwp, struct waited *wp) { -VSL(SLT_Debug, wp->fd, "ADD"); + VSL(SLT_Debug, wp->fd, "ADD"); CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC); CHECK_OBJ_NOTNULL(vwp, VWP_MAGIC); if (vwp->hpoll == vwp->npoll) From dridi.boukelmoune at gmail.com Mon Sep 6 07:13:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 6 Sep 2021 07:13:05 +0000 (UTC) Subject: [master] 7c738c290 waiter: Dereference wp after checking it Message-ID: <20210906071305.2163C11B2B3@lists.varnish-cache.org> commit 7c738c2907836e6504514f298ecc6fe129b9e8b7 Author: Dridi Boukelmoune Date: Mon Sep 6 09:11:19 2021 +0200 waiter: Dereference wp after checking it diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c index b3ab4b618..3ff15321e 100644 --- a/bin/varnishd/waiter/cache_waiter_poll.c +++ b/bin/varnishd/waiter/cache_waiter_poll.c @@ -97,8 +97,8 @@ static void vwp_add(struct vwp *vwp, struct waited *wp) { - VSL(SLT_Debug, wp->fd, "ADD"); CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC); + VSL(SLT_Debug, wp->fd, "ADD"); CHECK_OBJ_NOTNULL(vwp, VWP_MAGIC); if (vwp->hpoll == vwp->npoll) vwp_extend_pollspace(vwp); From phk at FreeBSD.org Tue Sep 7 05:58:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 7 Sep 2021 05:58:07 +0000 (UTC) Subject: [master] 25aae185d Silence FlexeLint Message-ID: <20210907055807.8570E4B62@lists.varnish-cache.org> commit 25aae185d818e19b5b2850862316d7b71a6ec582 Author: Poul-Henning Kamp Date: Tue Sep 7 05:57:24 2021 +0000 Silence FlexeLint diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index a1ad3da51..d994f13fe 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -207,7 +207,7 @@ vre_capture(const vre_t *code, const char *subject, size_t length, } ovector = pcre2_get_ovector_pointer(data); - nov = 2 * pcre2_get_ovector_count(data); + nov = 2L * pcre2_get_ovector_count(data); for (g = 0; g < nov; g++) ovector[g] = PCRE2_UNSET; From dridi.boukelmoune at gmail.com Tue Sep 7 12:03:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 7 Sep 2021 12:03:05 +0000 (UTC) Subject: [master] e0624cda8 doc: Promote design patterns to a top level section Message-ID: <20210907120305.7D97F100C31@lists.varnish-cache.org> commit e0624cda8c8db86366a89a8c4eb12fe5a671f37d Author: Dridi Boukelmoune Date: Tue Sep 7 12:53:22 2021 +0200 doc: Promote design patterns to a top level section diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index a017cf10a..8a1522a31 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -74,6 +74,7 @@ Longer listings like example command output and VCL look like this:: tutorial/index.rst users-guide/index.rst reference/index.rst + vcl-design-patterns/index.rst whats-new/index.rst dev-guide/index.rst phk/index.rst diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst index c5c751019..63bc4db88 100644 --- a/doc/sphinx/reference/index.rst +++ b/doc/sphinx/reference/index.rst @@ -9,6 +9,8 @@ The Varnish Reference Manual %%%%%%%%%%%%%%%%%%%%%%%%%%%% +.. _reference-vcl: + The VCL language ---------------- @@ -21,13 +23,8 @@ The VCL language VCL backend health probe states.rst -VCL Design Patterns -------------------- - -.. toctree:: - :maxdepth: 1 - - dp_vcl_resp_status.rst +A collection of :ref:`vcl-design-patterns-index` is available in addition to +these reference manuals. Bundled VMODs ------------- diff --git a/doc/sphinx/vcl-design-patterns/index.rst b/doc/sphinx/vcl-design-patterns/index.rst new file mode 100644 index 000000000..ed903ffa9 --- /dev/null +++ b/doc/sphinx/vcl-design-patterns/index.rst @@ -0,0 +1,19 @@ +.. + Copyright (c) 2021 Varnish Software AS + SPDX-License-Identifier: BSD-2-Clause + See LICENSE file for full text of license + +.. _vcl-design-patterns-index: + +%%%%%%%%%%%%%%%%%%% +VCL Design Patterns +%%%%%%%%%%%%%%%%%%% + +This section showcases design patterns for :ref:`reference-vcl`. To keep code +examples short, some aspects not directly related to a given design pattern +may be simplified. + +.. toctree:: + :maxdepth: 1 + + resp-status.rst diff --git a/doc/sphinx/reference/dp_vcl_resp_status.rst b/doc/sphinx/vcl-design-patterns/resp-status.rst similarity index 100% rename from doc/sphinx/reference/dp_vcl_resp_status.rst rename to doc/sphinx/vcl-design-patterns/resp-status.rst From dridi.boukelmoune at gmail.com Tue Sep 7 12:03:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 7 Sep 2021 12:03:05 +0000 (UTC) Subject: [master] f52d66456 doc: Remove trailing blanks Message-ID: <20210907120305.960C9100C34@lists.varnish-cache.org> commit f52d664560232903bbf61f64839ea818fdfc223d Author: Dridi Boukelmoune Date: Tue Sep 7 14:01:37 2021 +0200 doc: Remove trailing blanks diff --git a/doc/sphinx/phk/VSV00003.rst b/doc/sphinx/phk/VSV00003.rst index f899cd98e..06fbae2b6 100644 --- a/doc/sphinx/phk/VSV00003.rst +++ b/doc/sphinx/phk/VSV00003.rst @@ -83,7 +83,7 @@ that to a German Engineer... Nils Goroll from UPLEX didn't quite say *"Halten Sie Mein Bier?"*, but he did produce a VCL workaround right away, once again using -the inline-C capability, to frob things which are normally +the inline-C capability, to frob things which are normally "No User Serviceable Parts Behind This Door". Bravo Nils! diff --git a/doc/sphinx/phk/ip_address.rst b/doc/sphinx/phk/ip_address.rst index 840177e09..04b43d437 100644 --- a/doc/sphinx/phk/ip_address.rst +++ b/doc/sphinx/phk/ip_address.rst @@ -14,7 +14,7 @@ I'm old enough to remember `HOSTS.TXT` and the introduction of the DNS system. Those were the days when you got a class B network by sending a polite letter to California, getting a polite letter back, and then, some months later, when -`RFC1166 INTERNET NUMBERS `_ +`RFC1166 INTERNET NUMBERS `_ arrived with in semi-annual packet of printed RFCs, find out that letter had at typo and you had configured all of the European Parliaments 1200 computers on 136.172/16 instead of @@ -43,7 +43,7 @@ why `ping 0x7f000001` becomes `127.0.0.1` and `0127.0.0.1` becomes `87.0.0.1`. If you read the manual page for `getaddrinfo(3)` you will find -that it does not tell you that, it merely says it +that it does not tell you that, it merely says it `conforms to IEEE Std 1001`. But everybody knew what that was back in 1990, and nobody had firewalls @@ -74,7 +74,7 @@ That has just become CVE-2021-29418 and CVE-2021-28918 and will probably become a dozen more, once the CVE-trophy-hunters go to town. All IP number strings enter Varnish from trusted points, either -as command line arguments (`-a`, `-b`, `-M` etc.), +as command line arguments (`-a`, `-b`, `-M` etc.), in the VCL source (`backend`, `acl` etc.) or as PROXYv1 header strings from the TLS-stripper in front of Varnish. diff --git a/doc/sphinx/phk/legacy.rst b/doc/sphinx/phk/legacy.rst index d605a4fda..f0754ac23 100644 --- a/doc/sphinx/phk/legacy.rst +++ b/doc/sphinx/phk/legacy.rst @@ -32,7 +32,7 @@ also using their own HDLC frame layout. you have used it: That was the hot way to do error-correction on modems, when dialing into a BBS or ISP in the 1990ies. -I guess I should say "legacy-modems" ? +I guess I should say "legacy-modems" ? Big-endianess, storing the bytes the sensible way for hex-dumps, is rapidly becoming legacy, as the final old HP and SUN irons are diff --git a/doc/sphinx/phk/patent.rst b/doc/sphinx/phk/patent.rst index b90847ff2..446956802 100644 --- a/doc/sphinx/phk/patent.rst +++ b/doc/sphinx/phk/patent.rst @@ -216,7 +216,7 @@ I took myself out of the game: QcQAn18fGLT4Lx2ACBivtk5wEFy6fUcu =3V52 -----END PGP SIGNATURE----- - -- + -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst index 63bc4db88..3ce03678a 100644 --- a/doc/sphinx/reference/index.rst +++ b/doc/sphinx/reference/index.rst @@ -61,7 +61,7 @@ Logging and monitoring VarnishNCSA - Logging in NCSA format VarnishHist - Realtime reponse histogram display VarnishTop - Realtime activity display - + Counters and statistics ----------------------- diff --git a/doc/sphinx/reference/shell_tricks.rst b/doc/sphinx/reference/shell_tricks.rst index 065d8ac3c..d09fd4df4 100644 --- a/doc/sphinx/reference/shell_tricks.rst +++ b/doc/sphinx/reference/shell_tricks.rst @@ -6,7 +6,7 @@ .. _ref-shell_tricks: %%%%%%%%%%%% -Shell Tricks +Shell Tricks %%%%%%%%%%%% All the varnish programs can be invoked with the single diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 0a5704848..844487467 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -20,7 +20,7 @@ HTTP accelerator daemon SYNOPSIS ======== -varnishd +varnishd [-a [name=][listen_address[,PROTO]] [-b [host[:port]|path]] [-C] diff --git a/doc/sphinx/reference/vcl-backend.rst b/doc/sphinx/reference/vcl-backend.rst index 4dea4dfa4..fa8c0cfe9 100644 --- a/doc/sphinx/reference/vcl-backend.rst +++ b/doc/sphinx/reference/vcl-backend.rst @@ -52,7 +52,7 @@ IPv4/IPv6 address or a domain name which resolves to *at most* one IPv4 and one IPv6 address:: .host = "127.0.0.1"; - + .host = "[::1]:8080"; .host = "example.com:8081"; diff --git a/doc/sphinx/reference/vcl-probe.rst b/doc/sphinx/reference/vcl-probe.rst index 6d05e48ae..25eeb5aa1 100644 --- a/doc/sphinx/reference/vcl-probe.rst +++ b/doc/sphinx/reference/vcl-probe.rst @@ -140,7 +140,7 @@ is considered fully healthy. An additional ``.initial`` fills that many "happy" entries in the shift register when the VCL is loaded, so that backends can quickly -become healthy, even if their health is normally considered over +become healthy, even if their health is normally considered over many samples:: .interval = 1s; diff --git a/doc/sphinx/reference/vcl-var.rst b/doc/sphinx/reference/vcl-var.rst index 150f6b103..286d9bf9d 100644 --- a/doc/sphinx/reference/vcl-var.rst +++ b/doc/sphinx/reference/vcl-var.rst @@ -76,21 +76,21 @@ status message. SEE ALSO ======== - + * :ref:`varnishd(1)` * :ref:`vcl(7)` HISTORY ======= - + VCL was developed by Poul-Henning Kamp in cooperation with Verdens Gang AS, Redpill Linpro and Varnish Software. This manual page is written by Per Buer, Poul-Henning Kamp, Martin Blix Grydeland, Kristian Lyngst?l, Lasse Karstensen and others. - + COPYRIGHT ========= - + This document is licensed under the same license as Varnish itself. See LICENSE for details. diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst index f300b4488..11b13f68b 100644 --- a/doc/sphinx/tutorial/introduction.rst +++ b/doc/sphinx/tutorial/introduction.rst @@ -19,7 +19,7 @@ Varnish, and it is a pretty apt metafor:: ??????? ??????????? ??????? ??????????? ????????????? ? app ? --- ? Network ? -- ? TLS ? -- ? Varnish ? -- ? Backend ?? ??????? ??????????? ??????? ??????????? ??????????? - / + / ?????????????? ? API-client ? ?????????????? @@ -65,7 +65,7 @@ be examined and monitored in real time or recorded for later use in native or NCSA format, and when we say 'everything' we mean *everything*:: - * << Request >> 318737 + * << Request >> 318737 - Begin req 318736 rxreq - Timestamp Start: 1612787907.221931 0.000000 0.000000 - Timestamp Req: 1612787907.221931 0.000000 0.000000 diff --git a/doc/sphinx/users-guide/run_cli.rst b/doc/sphinx/users-guide/run_cli.rst index 9ac130311..fd5cb1c2d 100644 --- a/doc/sphinx/users-guide/run_cli.rst +++ b/doc/sphinx/users-guide/run_cli.rst @@ -26,19 +26,19 @@ with command-completion, command-history and other comforts: .. code-block:: text - critter phk> ./varnishadm - 200 + critter phk> ./varnishadm + 200 ----------------------------- Varnish Cache CLI 1.0 ----------------------------- FreeBSD,13.0-CURRENT,amd64,-jnone,-sdefault,-sdefault,-hcritbit varnish-trunk revision 2bd5d2adfc407216ebaa653fae882d3c8d47f5e1 - + Type 'help' for command list. Type 'quit' to close CLI session. Type 'start' to launch worker process. - - varnish> + + varnish> The CLI always returns a three digit status code to tell how things went. diff --git a/doc/sphinx/vcl-design-patterns/resp-status.rst b/doc/sphinx/vcl-design-patterns/resp-status.rst index 00a98d7df..33dbc3573 100644 --- a/doc/sphinx/vcl-design-patterns/resp-status.rst +++ b/doc/sphinx/vcl-design-patterns/resp-status.rst @@ -5,7 +5,7 @@ .. _dp_vcl_resp_status: -Using extra digits in resp.status +Using extra digits in resp.status ================================= In Varnish the ``.status`` variables can hold more than three From dridi.boukelmoune at gmail.com Tue Sep 7 13:28:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 7 Sep 2021 13:28:06 +0000 (UTC) Subject: [master] 0209ce360 build: Distribute doc/sphinx/vcl-design-patterns/ Message-ID: <20210907132806.B4F0C10311F@lists.varnish-cache.org> commit 0209ce360820503f0fa0e73764347eb862d9b2c1 Author: Dridi Boukelmoune Date: Tue Sep 7 15:26:59 2021 +0200 build: Distribute doc/sphinx/vcl-design-patterns/ diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 8327b478f..cb4e42881 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -61,6 +61,7 @@ EXTRA_DIST = \ tutorial \ reference \ users-guide \ + vcl-design-patterns \ vtc-syntax.py \ whats-new From dridi.boukelmoune at gmail.com Tue Sep 7 13:28:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 7 Sep 2021 13:28:06 +0000 (UTC) Subject: [master] 33188acfa doc: Add a design pattern entry for hash_ignore_vary Message-ID: <20210907132806.C9B73103122@lists.varnish-cache.org> commit 33188acfa13f0abe5f025c63a6c181433888dd5e Author: Dridi Boukelmoune Date: Tue Sep 7 14:51:04 2021 +0200 doc: Add a design pattern entry for hash_ignore_vary diff --git a/doc/sphinx/vcl-design-patterns/index.rst b/doc/sphinx/vcl-design-patterns/index.rst index ed903ffa9..142ded002 100644 --- a/doc/sphinx/vcl-design-patterns/index.rst +++ b/doc/sphinx/vcl-design-patterns/index.rst @@ -17,3 +17,4 @@ may be simplified. :maxdepth: 1 resp-status.rst + req-hash_ignore_vary.rst diff --git a/doc/sphinx/vcl-design-patterns/req-hash_ignore_vary.rst b/doc/sphinx/vcl-design-patterns/req-hash_ignore_vary.rst new file mode 100644 index 000000000..b5b620c96 --- /dev/null +++ b/doc/sphinx/vcl-design-patterns/req-hash_ignore_vary.rst @@ -0,0 +1,56 @@ +.. + Copyright (c) 2021 Varnish Software AS + SPDX-License-Identifier: BSD-2-Clause + See LICENSE file for full text of license + +Ignoring the Vary header for bots +================================= + +Varnish supports HTTP variants out of the box, but the *Vary* header is +somewhat limited since it operates on complete header values. If you want for +example to conduct an A/B testing campaign or perform blue/green deployment +you can make clients "remember" their path with a first-party cookie. + +When a search engine bot asks for contents however, there's a high chance that +they don't process cookies and in all likelihood you would prefer to serve a +response quickly. In that case you would probably prefer not to even try to +attribute a category to the client, but in that case you create a new variant +in your cache that is none of A, B, blue, green, or whatever your backend +serves. + +If the way content is served makes no difference to the bot, because you +changed the color of a button or something else orthogonal to the content +itself, then you risk a cache miss with the detrimental effects of adding a +needless variant to the cache and serving it with extra latency. + +If latency is paramount, you can use ``req.hash_ignore_vary`` to opt out of +the Vary match during the lookup and get the freshest variant. + +Ignoring how the cookie is set, and assuming the backend always provides an +accurate *Cache-Control* even when cookies are present, below is an example of +an A/B testing setup where bots are served the freshest variant:: + + + import cookie; + + include "devicedetect.vcl"; + + sub vcl_recv { + call devicedetect; + if (req.http.X-UA-Device ~ "bot") { + set req.hash_ignore_vary = true; + } + } + + sub vcl_req_cookie { + cookie.parse(req.http.Cookie); + set req.http.X-AB-Test = cookie.get("ab-test"); + return; + } + + sub vcl_deliver { + unset resp.http.Vary; + } + +It is also assumed that the backend replies with a ``Vary: X-AB-Test`` header +and varies on no other header. diff --git a/doc/sphinx/vcl-design-patterns/resp-status.rst b/doc/sphinx/vcl-design-patterns/resp-status.rst index 33dbc3573..29feeb59b 100644 --- a/doc/sphinx/vcl-design-patterns/resp-status.rst +++ b/doc/sphinx/vcl-design-patterns/resp-status.rst @@ -3,8 +3,6 @@ SPDX-License-Identifier: BSD-2-Clause See LICENSE file for full text of license -.. _dp_vcl_resp_status: - Using extra digits in resp.status ================================= From dridi.boukelmoune at gmail.com Tue Sep 7 15:45:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 7 Sep 2021 15:45:06 +0000 (UTC) Subject: [master] 6f955e441 build: Relocate line where it belongs Message-ID: <20210907154506.A60DF108A5D@lists.varnish-cache.org> commit 6f955e4413f0ccf03b521b75563615494af0195b Author: Dridi Boukelmoune Date: Tue Sep 7 16:47:41 2021 +0200 build: Relocate line where it belongs diff --git a/configure.ac b/configure.ac index ccc5afd8f..097b8a763 100644 --- a/configure.ac +++ b/configure.ac @@ -800,6 +800,8 @@ case $CFLAGS in ;; esac +AC_SUBST(AM_LT_LDFLAGS) + # --enable-coverage AC_ARG_ENABLE(coverage, AS_HELP_STRING([--enable-coverage], @@ -841,7 +843,6 @@ if test "$enable_debugging_symbols" != no; then fi fi -AC_SUBST(AM_LT_LDFLAGS) # Command line for compiling VCL code. I wish there were a simple way # to figure this out dynamically without introducing a run-time From dridi.boukelmoune at gmail.com Tue Sep 7 15:45:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 7 Sep 2021 15:45:06 +0000 (UTC) Subject: [master] 7cbbe58c2 build: New --enable-oss-fuzz configure option Message-ID: <20210907154506.C6E1C108A60@lists.varnish-cache.org> commit 7cbbe58c2016f25e6d85a726bce70d907bdc3e30 Author: Dridi Boukelmoune Date: Tue Sep 7 16:55:51 2021 +0200 build: New --enable-oss-fuzz configure option To avoid the maintenance of a downstream patch. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 583f9c5dc..e70b43de9 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -205,11 +205,17 @@ esi_parse_fuzzer_SOURCES = \ cache/cache_esi_parse.c \ fuzzers/esi_parse_fuzzer.c esi_parse_fuzzer_CFLAGS = \ - -DNOT_IN_A_VMOD -DTEST_DRIVER + -DNOT_IN_A_VMOD esi_parse_fuzzer_LDADD = \ $(top_builddir)/lib/libvarnish/libvarnish.la \ $(top_builddir)/lib/libvgz/libvgz.a +if ENABLE_OSS_FUZZ +esi_parse_fuzzer_LDFLAGS = $(LIB_FUZZING_ENGINE) +else +esi_parse_fuzzer_CFLAGS += -DTEST_DRIVER +endif + TESTS = vhp_table_test vhp_decode_test # diff --git a/configure.ac b/configure.ac index 097b8a763..513bf9fbe 100644 --- a/configure.ac +++ b/configure.ac @@ -806,15 +806,15 @@ AC_SUBST(AM_LT_LDFLAGS) AC_ARG_ENABLE(coverage, AS_HELP_STRING([--enable-coverage], [enable coverage (implies debugging symbols, default is NO)]), - [], - [enable_coverage=no]) + [], + [enable_coverage=no]) # --enable-debugging-symbols AC_ARG_ENABLE(debugging-symbols, AS_HELP_STRING([--enable-debugging-symbols], [enable debugging symbols (default is NO)]), - [], - [enable_debugging_symbols=no]) + [], + [enable_debugging_symbols=no]) if test "$enable_coverage" != no; then AC_DEFINE([ENABLE_COVERAGE], [1], [Define to 1 if code coverage is enabled.]) @@ -843,6 +843,14 @@ if test "$enable_debugging_symbols" != no; then fi fi +# --enable-oss-fuzz +AC_ARG_ENABLE(oss-fuzz, + AS_HELP_STRING([--enable-oss-fuzz], + [enable build tweaks for OSS-Fuzz (default is NO)]), + [], + [enable_oss_fuzz=no]) + +AM_CONDITIONAL(ENABLE_OSS_FUZZ, [test "$enable_oss_fuzz" != no]) # Command line for compiling VCL code. I wish there were a simple way # to figure this out dynamically without introducing a run-time From nils.goroll at uplex.de Wed Sep 8 07:53:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 8 Sep 2021 07:53:08 +0000 (UTC) Subject: [master] 3714f9fa4 Fix beresp.do_esi docs Message-ID: <20210908075308.53ED765C8B@lists.varnish-cache.org> commit 3714f9fa4c4074a1a3794d4c15b668ad4480a756 Author: Nils Goroll Date: Wed Sep 8 09:44:58 2021 +0200 Fix beresp.do_esi docs The variable is independent of req.esi / resp.do_esi: Whether or not the backend side prepares an object for ESI has nothing to do with whether or not the client side processes it. diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index cce173636..0aa911b82 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -759,8 +759,11 @@ beresp.do_esi Default: ``false``. - Set it to true to parse the object for ESI directives. - Will only be honored if req.esi is true. + Set it to true to parse the object for ESI directives. This is + necessary for later ESI processing on the client side. If + beresp.do_esi is false when an object enters the cache, client + side ESI processing will not be possible (obj.can_esi will be + false). It is a VCL error to use beresp.do_esi after setting beresp.filters. @@ -1254,8 +1257,8 @@ resp.do_esi ``VCL >= 4.1`` Default: obj.can_esi - This can be used to selectively disable ESI processing, - even though ESI parsing happened during fetch. + This can be used to selectively disable ESI processing, even + though ESI parsing happened during fetch (see beresp.do_esi). This is useful when Varnish caches peer with each other. It is a VCL error to use resp.do_esi after setting resp.filters. From nils.goroll at uplex.de Wed Sep 8 07:53:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 8 Sep 2021 07:53:08 +0000 (UTC) Subject: [master] 047d0a276 Doc polish Message-ID: <20210908075308.68F9E65C8E@lists.varnish-cache.org> commit 047d0a2763847f8e211e71745c99ab482e30ebbf Author: Nils Goroll Date: Wed Sep 8 09:46:34 2021 +0200 Doc polish diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 0aa911b82..4271824ac 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -784,8 +784,8 @@ beresp.do_stream have been sent to the client may get freed early, depending on the storage engine used. - This variable has no effect if do_esi is true or when the - response body is empty. + This variable has no effect if beresp.do_esi is true or when + the response body is empty. beresp.do_gzip From dridi.boukelmoune at gmail.com Thu Sep 9 09:50:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 9 Sep 2021 09:50:10 +0000 (UTC) Subject: [master] 81544c7b7 circleci: Trade Ubuntu xenial for focal Message-ID: <20210909095010.2459910388C@lists.varnish-cache.org> commit 81544c7b762a3f0ed07bff4d683bbb370b7e86a7 Author: Dridi Boukelmoune Date: Thu Sep 9 08:43:43 2021 +0200 circleci: Trade Ubuntu xenial for focal And bionic inherits the "oldest Ubuntu" 32bit setting. diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ed135706..8b834c526 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -162,7 +162,7 @@ jobs: description: the Linux distribution (debian|ubuntu) type: string release: - description: the release name (stretch|buster|xenial|bionic) + description: the release name (stretch|buster|bionic|focal) type: string extra_conf: description: platform-specific configure arguments @@ -328,9 +328,9 @@ workflows: release: buster extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator - distcheck: - name: distcheck_ubuntu_xenial + name: distcheck_ubuntu_bionic dist: ubuntu - release: xenial + release: bionic extra_conf: CFLAGS='-g -O2 -m32' - distcheck: name: distcheck_alpine @@ -361,7 +361,7 @@ workflows: parameters: platform: - ubuntu:bionic - - ubuntu:xenial + - ubuntu:focal - debian:bullseye - debian:buster - debian:stretch From dridi.boukelmoune at gmail.com Thu Sep 9 17:00:13 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 9 Sep 2021 17:00:13 +0000 (UTC) Subject: [master] b016f662c build: Bump autotools dependencies to ancient versions Message-ID: <20210909170013.610B2110BCF@lists.varnish-cache.org> commit b016f662c86cbee67846ecb553403bea11f64d6e Author: Dridi Boukelmoune Date: Thu Sep 9 18:56:12 2021 +0200 build: Bump autotools dependencies to ancient versions The previous ones were antiquated. diff --git a/autogen.des b/autogen.des index 7fffce8c3..294053b87 100755 --- a/autogen.des +++ b/autogen.des @@ -25,10 +25,6 @@ fi rm -f $SRCDIR/configure autoreconf -i -v $SRCDIR -# autoconf prior to 2.62 has issues with zsh 4.2 and newer -CONFIG_SHELL=/bin/sh -export CONFIG_SHELL - # NB: Workaround for make distcheck not working with # NB: FreeBSD's make on -current # env MAKE=gmake \ diff --git a/configure.ac b/configure.ac index 513bf9fbe..eb055da0a 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_PREREQ(2.59) +AC_PREREQ(2.69) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2021 Varnish Software]) AC_REVISION([$Id$]) @@ -19,7 +19,7 @@ AC_CANONICAL_SYSTEM AC_LANG(C) AM_MAINTAINER_MODE([disable]) -AM_INIT_AUTOMAKE([1.11 foreign color-tests parallel-tests subdir-objects]) +AM_INIT_AUTOMAKE([1.13 foreign color-tests parallel-tests subdir-objects]) AM_EXTRA_RECURSIVE_TARGETS([recheck]) AM_SILENT_RULES([yes]) AC_DISABLE_STATIC diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst index 470271060..c2c3350cf 100644 --- a/doc/sphinx/whats-new/upgrading-trunk.rst +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -160,6 +160,10 @@ Changes for developers Build ----- +Building from source requires autoconf 2.69 or newer and automake 1.13 or +newer. Neither are needed when building from a release archive since they +are already bootstrapped. + There is a new ``--enable-workspace-emulator`` configure flag to replace the regular "packed allocation" workspace with a "sparse allocation" alternative. Combined with the Address Sanitizer it can help VMOD authors find memory From dridi.boukelmoune at gmail.com Thu Sep 9 17:03:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 9 Sep 2021 17:03:07 +0000 (UTC) Subject: [master] e4a04d593 build: Typo Message-ID: <20210909170307.7FDBA110E60@lists.varnish-cache.org> commit e4a04d593422f8213a8a0f77c7cda6fe807ecf30 Author: Dridi Boukelmoune Date: Thu Sep 9 19:02:03 2021 +0200 build: Typo diff --git a/configure.ac b/configure.ac index eb055da0a..da9bb0603 100644 --- a/configure.ac +++ b/configure.ac @@ -543,7 +543,7 @@ AC_CHECK_DECL([SO_ACCEPTFILTER], # implement them. # # Varnish will build and run without these, but connections will not -# time out, which may leave Varnish vulnerable to denail-of-service +# time out, which may leave Varnish vulnerable to denial-of-service # attacks which would not be possible on other platforms. # # Newer Solaris releases with the Volo framework (Solaris 11, From dridi.boukelmoune at gmail.com Fri Sep 10 09:03:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 10 Sep 2021 09:03:05 +0000 (UTC) Subject: [master] ce9e5d22b vtc: Move more VMOD tests to the vmod/ tree Message-ID: <20210910090305.8121B104FDB@lists.varnish-cache.org> commit ce9e5d22b319cc3c92229ea5be5ff290c48f45f7 Author: Dridi Boukelmoune Date: Fri Sep 10 10:51:26 2021 +0200 vtc: Move more VMOD tests to the vmod/ tree I'm purposefully leaving the shard direcor and vmod_std alone for now. diff --git a/bin/varnishtest/tests/o00005.vtc b/bin/varnishtest/tests/o00005.vtc index 53250dc84..d635dc09d 100644 --- a/bin/varnishtest/tests/o00005.vtc +++ b/bin/varnishtest/tests/o00005.vtc @@ -1,82 +1,15 @@ -varnishtest "PROXY v2 TLV test" +varnishtest "PROXY v2 TLV overflow" # this does does not work with IPv6, the workspace overflow test is too brittle feature ipv4 -server s1 { - rxreq - txresp -} -start - -varnish v1 -arg "-p pool_sess=0,0,0" -proto "PROXY" -vcl+backend { - import vtc; - import proxy; - - sub vcl_deliver { - set resp.http.is_ssl = proxy.is_ssl(); - set resp.http.client_has_cert_sess = proxy.client_has_cert_sess(); - set resp.http.client_has_cert_conn = proxy.client_has_cert_conn(); - set resp.http.ssl_verify_result = proxy.ssl_verify_result() == 0; - set resp.http.alpn = proxy.alpn(); - set resp.http.authority = proxy.authority(); - set resp.http.ssl-version = proxy.ssl_version(); - set resp.http.cipher = proxy.ssl_cipher(); - set resp.http.key = proxy.cert_key(); - set resp.http.sign = proxy.cert_sign(); - set resp.http.cn = proxy.client_cert_cn(); - set resp.http.ws_free = vtc.workspace_free(session); - } +varnish v1 -arg "-p pool_sess=0,0,0" -proto "PROXY" -vcl { + backend be none; } -start logexpect l1 -v v1 -g raw { expect * 1000 Begin "sess 0 PROXY" - expect * 1000 Proxy "2 217.70.181.33 60822 95.142.168.34 443" - expect * 1000 Link "req 1001 rxreq" -} -start - -client c1 { - # PROXY2 with CRC32C TLV - sendhex { - 0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a - 21 11 00 65 - d9 46 b5 21 - 5f 8e a8 22 - ed 96 - 01 bb - 03 00 04 95 03 ee 75 - 01 00 02 68 32 - 02 00 0a 68 6f 63 64 65 74 2e 6e 65 74 - 20 00 3d - 01 00 00 00 00 - 21 00 07 54 4c 53 76 31 2e 33 - 25 00 05 45 43 32 35 36 - 24 00 0a 52 53 41 2d 53 48 41 32 35 36 - 23 00 16 41 45 41 44 2d 41 45 53 31 32 38 - 2d 47 43 4d 2d 53 48 41 32 35 36 - } - txreq - rxresp - expect resp.status == 200 - expect resp.http.is_ssl == true - expect resp.http.client_has_cert_sess == false - expect resp.http.client_has_cert_conn == false - expect resp.http.ssl_verify_result == true - expect resp.http.alpn == h2 - expect resp.http.authority == hocdet.net - expect resp.http.ssl-version == TLSv1.3 - expect resp.http.cipher == AEAD-AES128-GCM-SHA256 - expect resp.http.key == EC256 - expect resp.http.sign == RSA-SHA256 - expect resp.http.cn == "" -} -run - -varnish v1 -vsl_catchup - -logexpect l1 -wait - -logexpect l1 -v v1 -g raw { - expect * 1003 Begin "sess 0 PROXY" - expect * 1003 ProxyGarbage "PROXY2: CRC error" + expect * 1000 ProxyGarbage "PROXY2: CRC error" } -start client c1 { @@ -108,8 +41,8 @@ varnish v1 -vsl_catchup logexpect l1 -wait logexpect l1 -v v1 -g raw { - expect * 1004 Begin "sess 0 PROXY" - expect * 1004 ProxyGarbage "PROXY2: TLV Dribble bytes" + expect * 1001 Begin "sess 0 PROXY" + expect * 1001 ProxyGarbage "PROXY2: TLV Dribble bytes" } -start client c1 { @@ -142,8 +75,8 @@ varnish v1 -vsl_catchup logexpect l1 -wait logexpect l1 -v v1 -g raw { - expect * 1005 Begin "sess 0 PROXY" - expect * 1005 ProxyGarbage "PROXY2: TLV Length Error" + expect * 1002 Begin "sess 0 PROXY" + expect * 1002 ProxyGarbage "PROXY2: TLV Length Error" } -start client c1 { @@ -175,8 +108,8 @@ varnish v1 -vsl_catchup logexpect l1 -wait logexpect l1 -v v1 -g raw { - expect * 1006 Begin "sess 0 PROXY" - expect * 1006 ProxyGarbage "PROXY2: TLV Length Error" + expect * 1003 Begin "sess 0 PROXY" + expect * 1003 ProxyGarbage "PROXY2: TLV Length Error" } -start client c1 { diff --git a/bin/varnishtest/tests/d00000.vtc b/vmod/tests/directors_b00000.vtc similarity index 100% rename from bin/varnishtest/tests/d00000.vtc rename to vmod/tests/directors_b00000.vtc diff --git a/bin/varnishtest/tests/d00001.vtc b/vmod/tests/directors_b00001.vtc similarity index 100% rename from bin/varnishtest/tests/d00001.vtc rename to vmod/tests/directors_b00001.vtc diff --git a/bin/varnishtest/tests/d00002.vtc b/vmod/tests/directors_b00002.vtc similarity index 100% rename from bin/varnishtest/tests/d00002.vtc rename to vmod/tests/directors_b00002.vtc diff --git a/bin/varnishtest/tests/d00003.vtc b/vmod/tests/directors_b00003.vtc similarity index 100% rename from bin/varnishtest/tests/d00003.vtc rename to vmod/tests/directors_b00003.vtc diff --git a/bin/varnishtest/tests/d00027.vtc b/vmod/tests/directors_b00004.vtc similarity index 100% rename from bin/varnishtest/tests/d00027.vtc rename to vmod/tests/directors_b00004.vtc diff --git a/bin/varnishtest/tests/d00028.vtc b/vmod/tests/directors_b00005.vtc similarity index 100% rename from bin/varnishtest/tests/d00028.vtc rename to vmod/tests/directors_b00005.vtc diff --git a/bin/varnishtest/tests/d00031.vtc b/vmod/tests/directors_b00006.vtc similarity index 100% rename from bin/varnishtest/tests/d00031.vtc rename to vmod/tests/directors_b00006.vtc diff --git a/bin/varnishtest/tests/d00004.vtc b/vmod/tests/directors_c00000.vtc similarity index 100% rename from bin/varnishtest/tests/d00004.vtc rename to vmod/tests/directors_c00000.vtc diff --git a/bin/varnishtest/tests/d00005.vtc b/vmod/tests/directors_c00001.vtc similarity index 100% rename from bin/varnishtest/tests/d00005.vtc rename to vmod/tests/directors_c00001.vtc diff --git a/bin/varnishtest/tests/d00006.vtc b/vmod/tests/directors_c00002.vtc similarity index 100% rename from bin/varnishtest/tests/d00006.vtc rename to vmod/tests/directors_c00002.vtc diff --git a/vmod/tests/proxy_b00000.vtc b/vmod/tests/proxy_b00000.vtc new file mode 100644 index 000000000..7ebb2fa13 --- /dev/null +++ b/vmod/tests/proxy_b00000.vtc @@ -0,0 +1,72 @@ +varnishtest "PROXY v2 TLV test" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -proto "PROXY" -vcl+backend { + import vtc; + import proxy; + + sub vcl_deliver { + set resp.http.is_ssl = proxy.is_ssl(); + set resp.http.client_has_cert_sess = proxy.client_has_cert_sess(); + set resp.http.client_has_cert_conn = proxy.client_has_cert_conn(); + set resp.http.ssl_verify_result = proxy.ssl_verify_result() == 0; + set resp.http.alpn = proxy.alpn(); + set resp.http.authority = proxy.authority(); + set resp.http.ssl-version = proxy.ssl_version(); + set resp.http.cipher = proxy.ssl_cipher(); + set resp.http.key = proxy.cert_key(); + set resp.http.sign = proxy.cert_sign(); + set resp.http.cn = proxy.client_cert_cn(); + set resp.http.ws_free = vtc.workspace_free(session); + } +} -start + +logexpect l1 -v v1 -g raw { + expect * 1000 Begin "sess 0 PROXY" + expect * 1000 Proxy "2 217.70.181.33 60822 95.142.168.34 443" + expect * 1000 Link "req 1001 rxreq" +} -start + +client c1 { + # PROXY2 with CRC32C TLV + sendhex { + 0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a + 21 11 00 65 + d9 46 b5 21 + 5f 8e a8 22 + ed 96 + 01 bb + 03 00 04 95 03 ee 75 + 01 00 02 68 32 + 02 00 0a 68 6f 63 64 65 74 2e 6e 65 74 + 20 00 3d + 01 00 00 00 00 + 21 00 07 54 4c 53 76 31 2e 33 + 25 00 05 45 43 32 35 36 + 24 00 0a 52 53 41 2d 53 48 41 32 35 36 + 23 00 16 41 45 41 44 2d 41 45 53 31 32 38 + 2d 47 43 4d 2d 53 48 41 32 35 36 + } + txreq + rxresp + expect resp.status == 200 + expect resp.http.is_ssl == true + expect resp.http.client_has_cert_sess == false + expect resp.http.client_has_cert_conn == false + expect resp.http.ssl_verify_result == true + expect resp.http.alpn == h2 + expect resp.http.authority == hocdet.net + expect resp.http.ssl-version == TLSv1.3 + expect resp.http.cipher == AEAD-AES128-GCM-SHA256 + expect resp.http.key == EC256 + expect resp.http.sign == RSA-SHA256 + expect resp.http.cn == "" +} -run + +varnish v1 -vsl_catchup + +logexpect l1 -wait diff --git a/bin/varnishtest/tests/m00031.vtc b/vmod/tests/purge_c00000.vtc similarity index 100% rename from bin/varnishtest/tests/m00031.vtc rename to vmod/tests/purge_c00000.vtc diff --git a/bin/varnishtest/tests/m00047.vtc b/vmod/tests/unix_c00000.vtc similarity index 100% rename from bin/varnishtest/tests/m00047.vtc rename to vmod/tests/unix_c00000.vtc From dridi.boukelmoune at gmail.com Fri Sep 10 09:03:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 10 Sep 2021 09:03:05 +0000 (UTC) Subject: [master] 7cfe14da7 vtc: Retire irrelevant test case Message-ID: <20210910090305.ABC6C104FDE@lists.varnish-cache.org> commit 7cfe14da72407830114ff80e7075e5de38334035 Author: Dridi Boukelmoune Date: Fri Sep 10 10:53:46 2021 +0200 vtc: Retire irrelevant test case diff --git a/bin/varnishtest/tests/r00263.vtc b/bin/varnishtest/tests/r00263.vtc deleted file mode 100644 index 5b75b6f55..000000000 --- a/bin/varnishtest/tests/r00263.vtc +++ /dev/null @@ -1,24 +0,0 @@ -varnishtest "Test refcounting backends from director" - -# XXX: This test is probably not relevant with VMOD.directors - -varnish v1 -vcl { - import directors; - - backend node1 { .host = "10.0.0.1"; .port = "80"; } - backend node2 { .host = "10.0.0.2"; .port = "80"; } - backend node3 { .host = "10.0.0.3"; .port = "80"; } - - sub vcl_init { - new cluster = directors.random(); - cluster.add_backend(node1, 1); - cluster.add_backend(node2, 1); - cluster.add_backend(node3, 1); - } - - sub vcl_backend_fetch { - set bereq.backend = cluster.backend(); - } -} -start - -varnish v1 -wait From dridi.boukelmoune at gmail.com Fri Sep 10 10:17:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 10 Sep 2021 10:17:06 +0000 (UTC) Subject: [master] 10e3b1770 vtc: Move the remaining vmod_directors test cases Message-ID: <20210910101706.10268107102@lists.varnish-cache.org> commit 10e3b1770f733eac8b900db92edceacebba690c6 Author: Dridi Boukelmoune Date: Fri Sep 10 12:03:01 2021 +0200 vtc: Move the remaining vmod_directors test cases With a tiny bit of polish here and there. diff --git a/bin/varnishtest/tests/README b/bin/varnishtest/tests/README index 1d01105de..4070daf05 100644 --- a/bin/varnishtest/tests/README +++ b/bin/varnishtest/tests/README @@ -17,7 +17,7 @@ Naming scheme id ~ ^a02 --> HTTP2 id ~ ^b --> Basic functionality tests id ~ ^c --> Complex functionality tests - id ~ ^d --> Director VMOD tests + id ~ ^d --> Director facility tests id ~ ^e --> ESI tests id ~ ^f --> Security related tests id ~ ^g --> GZIP tests @@ -34,3 +34,5 @@ Naming scheme id ~ ^t02 --> HTTP2 id ~ ^u --> Utilities and background processes id ~ ^v --> VCL tests: execute VRT functions + + Coverage for individual VMODs is in "${top_srcdir}vmod/tests". diff --git a/bin/varnishtest/tests/d00025.vtc b/vmod/tests/directors_b00007.vtc similarity index 100% rename from bin/varnishtest/tests/d00025.vtc rename to vmod/tests/directors_b00007.vtc diff --git a/bin/varnishtest/tests/d00018.vtc b/vmod/tests/directors_c00003.vtc similarity index 55% rename from bin/varnishtest/tests/d00018.vtc rename to vmod/tests/directors_c00003.vtc index 429bc9014..5a81f479c 100644 --- a/bin/varnishtest/tests/d00018.vtc +++ b/vmod/tests/directors_c00003.vtc @@ -2,17 +2,17 @@ varnishtest "shard director/int key" server s1 { rxreq - txresp -body "ech3Ooj" + txresp } -start server s2 { rxreq - txresp -body "ieQu2qua" + txresp } -start server s3 { rxreq - txresp -body "xiuFi3Pe" + txresp } -start varnish v1 -vcl+backend { @@ -36,31 +36,26 @@ varnish v1 -vcl+backend { } sub vcl_recv { - if (req.url == "/1") { - set req.backend_hint = vd.backend(by=KEY, key=1); - } - if (req.url == "/2") { - set req.backend_hint = vd.backend(by=KEY, key=2147483647); - } - if (req.url == "/3") { - set req.backend_hint = vd.backend(by=KEY, key=4294967295); - } + set req.backend_hint = vd.backend(by=KEY, key=std.integer(req.http.key)); return(pass); } + sub vcl_backend_response { + set beresp.http.backend = beresp.backend; + } } -start client c1 { - txreq -url /1 + txreq -hdr "key: 1" rxresp - expect resp.body == "ech3Ooj" + expect resp.http.backend == s1 - txreq -url /2 + txreq -hdr "key: 2147483647" rxresp - expect resp.body == "ieQu2qua" + expect resp.http.backend == s2 - txreq -url /3 + txreq -hdr "key: 4294967295" rxresp - expect resp.body == "xiuFi3Pe" + expect resp.http.backend == s3 } -run diff --git a/bin/varnishtest/tests/d00029.vtc b/vmod/tests/directors_c00004.vtc similarity index 97% rename from bin/varnishtest/tests/d00029.vtc rename to vmod/tests/directors_c00004.vtc index 26fc5b0b5..668d6c5a8 100644 --- a/bin/varnishtest/tests/d00029.vtc +++ b/vmod/tests/directors_c00004.vtc @@ -1,4 +1,4 @@ -varnishtest "shard director LAZY - d18.vtc" +varnishtest "shard director LAZY - directors_c00003.vtc" server s1 -repeat 2 { rxreq diff --git a/bin/varnishtest/tests/d00019.vtc b/vmod/tests/directors_c00005.vtc similarity index 73% rename from bin/varnishtest/tests/d00019.vtc rename to vmod/tests/directors_c00005.vtc index 9695457db..0fea75d1a 100644 --- a/bin/varnishtest/tests/d00019.vtc +++ b/vmod/tests/directors_c00005.vtc @@ -2,17 +2,17 @@ varnishtest "shard director by req.url (default)" server s1 { rxreq - txresp -body "ech3Ooj" + txresp } -start server s2 { rxreq - txresp -body "ieQu2qua" + txresp } -start server s3 { rxreq - txresp -body "xiuFi3Pe" + txresp } -start varnish v1 -vcl+backend { @@ -31,19 +31,22 @@ varnish v1 -vcl+backend { return(pass); } + sub vcl_backend_response { + set beresp.http.backend = beresp.backend; + } } -start client c1 { txreq -url /Boo0aixe rxresp - expect resp.body == "ech3Ooj" + expect resp.http.backend == s1 txreq -url /eishoSu2 rxresp - expect resp.body == "ieQu2qua" + expect resp.http.backend == s2 txreq -url /Aunah3uo rxresp - expect resp.body == "xiuFi3Pe" + expect resp.http.backend == s3 } -run diff --git a/bin/varnishtest/tests/d00041.vtc b/vmod/tests/directors_c00006.vtc similarity index 99% rename from bin/varnishtest/tests/d00041.vtc rename to vmod/tests/directors_c00006.vtc index 3e108b81b..794cc904e 100644 --- a/bin/varnishtest/tests/d00041.vtc +++ b/vmod/tests/directors_c00006.vtc @@ -1,4 +1,4 @@ -varnishtest "d00017.vtc but with weights" +varnishtest "directors_c00005.vtc but with weights" server s1 { rxreq diff --git a/bin/varnishtest/tests/d00017.vtc b/vmod/tests/directors_c00007.vtc similarity index 100% rename from bin/varnishtest/tests/d00017.vtc rename to vmod/tests/directors_c00007.vtc diff --git a/bin/varnishtest/tests/d00026.vtc b/vmod/tests/directors_c00008.vtc similarity index 90% rename from bin/varnishtest/tests/d00026.vtc rename to vmod/tests/directors_c00008.vtc index 0cfd81649..c571f5d2f 100644 --- a/bin/varnishtest/tests/d00026.vtc +++ b/vmod/tests/directors_c00008.vtc @@ -1,4 +1,4 @@ -varnishtest "shard director - same as d00017.vtc but setting backend in fetch" +varnishtest "directors_c00007.vtc but setting backend in fetch" server s1 { rxreq diff --git a/bin/varnishtest/tests/d00020.vtc b/vmod/tests/directors_c00009.vtc similarity index 100% rename from bin/varnishtest/tests/d00020.vtc rename to vmod/tests/directors_c00009.vtc diff --git a/bin/varnishtest/tests/d00015.vtc b/vmod/tests/directors_c00010.vtc similarity index 100% rename from bin/varnishtest/tests/d00015.vtc rename to vmod/tests/directors_c00010.vtc diff --git a/bin/varnishtest/tests/d00016.vtc b/vmod/tests/directors_c00011.vtc similarity index 100% rename from bin/varnishtest/tests/d00016.vtc rename to vmod/tests/directors_c00011.vtc diff --git a/bin/varnishtest/tests/d00021.vtc b/vmod/tests/directors_c00012.vtc similarity index 100% rename from bin/varnishtest/tests/d00021.vtc rename to vmod/tests/directors_c00012.vtc diff --git a/bin/varnishtest/tests/d00022.vtc b/vmod/tests/directors_c00013.vtc similarity index 100% rename from bin/varnishtest/tests/d00022.vtc rename to vmod/tests/directors_c00013.vtc diff --git a/bin/varnishtest/tests/d00023.vtc b/vmod/tests/directors_c00014.vtc similarity index 100% rename from bin/varnishtest/tests/d00023.vtc rename to vmod/tests/directors_c00014.vtc diff --git a/bin/varnishtest/tests/d00030.vtc b/vmod/tests/directors_c00015.vtc similarity index 100% rename from bin/varnishtest/tests/d00030.vtc rename to vmod/tests/directors_c00015.vtc diff --git a/bin/varnishtest/tests/d00024.vtc b/vmod/tests/directors_s00000.vtc similarity index 100% rename from bin/varnishtest/tests/d00024.vtc rename to vmod/tests/directors_s00000.vtc diff --git a/bin/varnishtest/tests/d00039.vtc b/vmod/tests/directors_v00000.vtc similarity index 100% rename from bin/varnishtest/tests/d00039.vtc rename to vmod/tests/directors_v00000.vtc From dridi.boukelmoune at gmail.com Fri Sep 10 12:05:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 10 Sep 2021 12:05:08 +0000 (UTC) Subject: [master] b42114edd vtc: Move vmod coverage around Message-ID: <20210910120508.3982B109E6D@lists.varnish-cache.org> commit b42114edd870ce7f9fafe551f7381ee6d71bdd73 Author: Dridi Boukelmoune Date: Fri Sep 10 12:23:59 2021 +0200 vtc: Move vmod coverage around To avoid mixing vmod_std coverage with the VMOD feature. diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index d8a9960a1..3cc101a62 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -1,4 +1,4 @@ -varnishtest "Test std, vtc & debug vmod" +varnishtest "Test vmod usage and test vmods (debug & vtc)" server s1 { rxreq @@ -14,31 +14,11 @@ varnish v1 -vcl+backend { import debug as dbg; import debug as dbg; // again - sub log { - std.log("VCL" + " initiated " + "log"); - std.log("01030507090b0d0f" + - "11131517191b1d1f" + - "21232527292b2d2f" + - "31333537393b3d3f" + - "41434547494b4d4f" + - "51535557595b5d5f" + - "61636567696b6d6f" + - "71737577797b7d7f" + - "81838587898b8d8f" + - "91939597999b9d9f" + - "a1a3a5a7a9abadaf" + - "b1b3b5b7b9bbbdbf" + - "c1c3c5c7c9cbcdcf" + - "d1d3d5d7d9dbdddf" + - "e1e3e5e7e9ebedef" + - "f1f3f5f7f9fbfdff"); - } - sub vcl_init { new objx = dbg.obj(); objx.obj(); dbg.vsc_new(); - call log; + std.log("inside vcl_init"); } sub vcl_synth { @@ -66,8 +46,6 @@ varnish v1 -vcl+backend { } sub vcl_deliver { - set resp.http.foo = std.toupper(resp.http.foo); - set resp.http.bar = std.tolower(resp.http.bar); set resp.http.who = debug.author(mithrandir); set resp.http.really = debug.author(); set resp.http.when = objx.date(); @@ -77,7 +55,6 @@ varnish v1 -vcl+backend { debug.test_priv_vcl(); objx.test_priv_call(); objx.test_priv_vcl(); - call log; std.syslog(8 + 7, "Somebody runs varnishtest"); debug.rot52(resp); } @@ -90,8 +67,6 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == "4" - expect resp.http.foo == "BAR" - expect resp.http.bar == "foo" expect resp.http.who == "Tollef" expect resp.http.really == "Poul-Henning" expect resp.http.when == "Thu, 01 Jan 1970 00:00:21 GMT" @@ -113,23 +88,15 @@ varnish v1 -expect DEBUG.count == 1 logexpect l1 -v v1 -g raw -d 1 { expect * 0 CLI {^Rd vcl.load} - expect 0 0 VCL_Log {^VCL initiated log} - # vsl_reclen = 255b minus NUL byte (last "ff" missing) - expect 0 0 VCL_Log {^01030507090b0d0f11131517191b1d1f21232527292b2d2f31333537393b3d3f41434547494b4d4f51535557595b5d5f61636567696b6d6f71737577797b7d7f81838587898b8d8f91939597999b9d9fa1a3a5a7a9abadafb1b3b5b7b9bbbdbfc1c3c5c7c9cbcdcfd1d3d5d7d9dbdddfe1e3e5e7e9ebedeff1f3f5f7f9fbfd$} + expect 0 0 VCL_Log {^inside vcl_init} expect 0 0 Debug {VCL_EVENT_WARM} expect * 1001 VCL_call {^DELIVER} - expect 0 = RespUnset {^foo: bAr} - expect 0 = RespHeader {^foo: BAR} - expect 0 = RespUnset {^bar: fOo} - expect 0 = RespHeader {^bar: foo} expect 0 = RespHeader {^who: Tollef} expect 0 = RespHeader {^really: Poul-Henning} expect 0 = RespHeader {^when: Thu, 01 Jan 1970 00:00:21 GMT} expect 0 = RespHeader {^what: [1-9][0-9]} expect 0 = RespHeader {^not: -1} - expect 0 = VCL_Log {^VCL initiated log} - expect 0 = VCL_Log {^01030507090b0d0f11131517191b1d1f21232527292b2d2f31333537393b3d3f41434547494b4d4f51535557595b5d5f61636567696b6d6f71737577797b7d7f81838587898b8d8f91939597999b9d9fa1a3a5a7a9abadafb1b3b5b7b9bbbdbfc1c3c5c7c9cbcdcfd1d3d5d7d9dbdddfe1e3e5e7e9ebedeff1f3f5f7f9fbfd$} expect 0 = RespHeader {^Encrypted: ROT52} expect 0 = VCL_return {^deliver} } -start @@ -190,7 +157,6 @@ varnish v1 -errvcl {Failed initialization} { varnish v1 -cliok "param.set vcc_allow_inline_c on" varnish v1 -vcl { - import std; import vtc; backend be none; @@ -303,3 +269,29 @@ client c1 { expect resp.http.csc-match == true expect resp.http.uzp-match == true } -run + +varnish v1 -vcl+backend { } + +varnish v1 -cliok "debug.vmod" +varnish v1 -cliok "vcl.list" + +varnish v1 -expect vmods == 3 + +varnish v1 -cliok "vcl.discard vcl[1-8]" +varnish v1 -cliok "vcl.list" +varnish v1 -cliok "debug.vmod" + +varnish v1 -expect vmods == 0 + +varnish v1 -errvcl {Symbol 'std' type (vmod) can not be used in expression.} { + import std; + + sub vcl_recv { + if (std == 2) { + } + } +} + +varnish v1 -cliok "debug.vmod" +varnish v1 -cliok "vcl.list" +varnish v1 -expect vmods == 0 diff --git a/bin/varnishtest/tests/m00001.vtc b/bin/varnishtest/tests/m00001.vtc index 1a676c673..ac5be793a 100644 --- a/bin/varnishtest/tests/m00001.vtc +++ b/bin/varnishtest/tests/m00001.vtc @@ -54,27 +54,65 @@ client c1 { expect resp.http.vcl42 == "false" } -run -varnish v1 -vcl+backend { } - -varnish v1 -cliok "debug.vmod" -varnish v1 -cliok "vcl.list" +server s1 { + rxreq + txresp -hdr "foo: bAr" -hdr "bar: fOo" +} -start -varnish v1 -expect vmods == 1 +varnish v1 -vcl+backend { + import std; -varnish v1 -cliok "vcl.discard vcl1" -varnish v1 -cliok "vcl.discard vcl2" -varnish v1 -cliok "vcl.list" -varnish v1 -cliok "debug.vmod" + sub log { + std.log("VCL" + " initiated " + "log"); + std.log("01030507090b0d0f" + + "11131517191b1d1f" + + "21232527292b2d2f" + + "31333537393b3d3f" + + "41434547494b4d4f" + + "51535557595b5d5f" + + "61636567696b6d6f" + + "71737577797b7d7f" + + "81838587898b8d8f" + + "91939597999b9d9f" + + "a1a3a5a7a9abadaf" + + "b1b3b5b7b9bbbdbf" + + "c1c3c5c7c9cbcdcf" + + "d1d3d5d7d9dbdddf" + + "e1e3e5e7e9ebedef" + + "f1f3f5f7f9fbfdff"); + } -varnish v1 -errvcl {Symbol 'std' type (vmod) can not be used in expression.} { - import std; + sub vcl_init { + call log; + } - sub vcl_recv { - if (std == 2) { - } + sub vcl_deliver { + set resp.http.foo = std.toupper(resp.http.foo); + set resp.http.bar = std.tolower(resp.http.bar); + std.syslog(8 + 7, "Somebody runs varnishtest"); + call log; } } -varnish v1 -cliok "debug.vmod" -varnish v1 -cliok "vcl.list" -varnish v1 -expect vmods == 0 +client c1 { + txreq + rxresp + expect resp.http.foo == "BAR" + expect resp.http.bar == "foo" +} -run + +logexpect l1 -v v1 -g raw -d 1 { + # vsl_reclen = 255b minus NUL byte (last "ff" missing) + expect * 0 VCL_Log {^VCL initiated log} + expect 0 = VCL_Log {^01030507090b0d0f11131517191b1d1f21232527292b2d2f31333537393b3d3f41434547494b4d4f51535557595b5d5f61636567696b6d6f71737577797b7d7f81838587898b8d8f91939597999b9d9fa1a3a5a7a9abadafb1b3b5b7b9bbbdbfc1c3c5c7c9cbcdcfd1d3d5d7d9dbdddfe1e3e5e7e9ebedeff1f3f5f7f9fbfd$} + expect * 1006 VCL_call {^DELIVER} + expect 0 = RespUnset {^foo: bAr} + expect 0 = RespHeader {^foo: BAR} + expect 0 = RespUnset {^bar: fOo} + expect 0 = RespHeader {^bar: foo} + expect 0 = VCL_Log {^VCL initiated log} + expect 0 = VCL_Log {^01030507090b0d0f11131517191b1d1f21232527292b2d2f31333537393b3d3f41434547494b4d4f51535557595b5d5f61636567696b6d6f71737577797b7d7f81838587898b8d8f91939597999b9d9fa1a3a5a7a9abadafb1b3b5b7b9bbbdbfc1c3c5c7c9cbcdcfd1d3d5d7d9dbdddfe1e3e5e7e9ebedeff1f3f5f7f9fbfd$} + expect 0 = VCL_return {^deliver} +} -run + +#logexpect l1 -wait From dridi.boukelmoune at gmail.com Fri Sep 10 12:45:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 10 Sep 2021 12:45:07 +0000 (UTC) Subject: [master] 480c5d2dc vtc: Move vmod_std coverage to the vmod/ tree Message-ID: <20210910124507.41A9210D335@lists.varnish-cache.org> commit 480c5d2dc32291208491df2f74d92a409ee3156f Author: Dridi Boukelmoune Date: Fri Sep 10 14:27:08 2021 +0200 vtc: Move vmod_std coverage to the vmod/ tree diff --git a/bin/varnishtest/tests/README b/bin/varnishtest/tests/README index 4070daf05..0f78207b7 100644 --- a/bin/varnishtest/tests/README +++ b/bin/varnishtest/tests/README @@ -25,7 +25,7 @@ Naming scheme id ~ ^i --> Interoperability and standards compliance id ~ ^j --> JAIL tests id ~ ^l --> VSL tests - id ~ ^m --> VMOD tests excluding director + id ~ ^m --> VMOD facility, vmod_debug and vmod_vtc id ~ ^o --> prOxy protocol id ~ ^p --> Persistent tests id ~ ^r --> Regression tests, same number as ticket diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 3cc101a62..fb3a90e3d 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -99,9 +99,7 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 = RespHeader {^not: -1} expect 0 = RespHeader {^Encrypted: ROT52} expect 0 = VCL_return {^deliver} -} -start - -logexpect l1 -wait +} -run varnish v1 -errvcl {Wrong enum value. Expected one of:} { import debug; diff --git a/bin/varnishtest/tests/m00004.vtc b/vmod/tests/std_b00000.vtc similarity index 100% rename from bin/varnishtest/tests/m00004.vtc rename to vmod/tests/std_b00000.vtc diff --git a/bin/varnishtest/tests/m00005.vtc b/vmod/tests/std_b00002.vtc similarity index 100% rename from bin/varnishtest/tests/m00005.vtc rename to vmod/tests/std_b00002.vtc diff --git a/bin/varnishtest/tests/m00006.vtc b/vmod/tests/std_b00003.vtc similarity index 100% rename from bin/varnishtest/tests/m00006.vtc rename to vmod/tests/std_b00003.vtc diff --git a/bin/varnishtest/tests/m00007.vtc b/vmod/tests/std_b00004.vtc similarity index 100% rename from bin/varnishtest/tests/m00007.vtc rename to vmod/tests/std_b00004.vtc diff --git a/bin/varnishtest/tests/m00011.vtc b/vmod/tests/std_b00005.vtc similarity index 100% rename from bin/varnishtest/tests/m00011.vtc rename to vmod/tests/std_b00005.vtc diff --git a/bin/varnishtest/tests/m00013.vtc b/vmod/tests/std_b00006.vtc similarity index 100% rename from bin/varnishtest/tests/m00013.vtc rename to vmod/tests/std_b00006.vtc diff --git a/bin/varnishtest/tests/m00014.vtc b/vmod/tests/std_b00007.vtc similarity index 100% rename from bin/varnishtest/tests/m00014.vtc rename to vmod/tests/std_b00007.vtc diff --git a/bin/varnishtest/tests/m00015.vtc b/vmod/tests/std_b00008.vtc similarity index 100% rename from bin/varnishtest/tests/m00015.vtc rename to vmod/tests/std_b00008.vtc diff --git a/bin/varnishtest/tests/m00016.vtc b/vmod/tests/std_b00009.vtc similarity index 100% rename from bin/varnishtest/tests/m00016.vtc rename to vmod/tests/std_b00009.vtc diff --git a/bin/varnishtest/tests/m00018.vtc b/vmod/tests/std_b00010.vtc similarity index 100% rename from bin/varnishtest/tests/m00018.vtc rename to vmod/tests/std_b00010.vtc diff --git a/bin/varnishtest/tests/m00020.vtc b/vmod/tests/std_b00011.vtc similarity index 100% rename from bin/varnishtest/tests/m00020.vtc rename to vmod/tests/std_b00011.vtc diff --git a/bin/varnishtest/tests/m00026.vtc b/vmod/tests/std_b00012.vtc similarity index 78% rename from bin/varnishtest/tests/m00026.vtc rename to vmod/tests/std_b00012.vtc index 3659eb367..cc341d8cc 100644 --- a/bin/varnishtest/tests/m00026.vtc +++ b/vmod/tests/std_b00012.vtc @@ -1,5 +1,7 @@ varnishtest "Test std.getenv" +setenv FOO "BAR BAZ" + varnish v1 -vcl { import std; @@ -10,6 +12,7 @@ varnish v1 -vcl { } sub vcl_synth { + set resp.http.X-FOO = std.getenv("FOO"); set resp.http.X-PATH = std.getenv("PATH"); set resp.http.X-unset = std.getenv(req.http.unset); } @@ -18,6 +21,7 @@ varnish v1 -vcl { client c1 { txreq rxresp + expect resp.http.X-FOO == "BAR BAZ" expect resp.http.X-PATH ~ "^/" expect resp.http.X-unset == "" } -run diff --git a/bin/varnishtest/tests/m00029.vtc b/vmod/tests/std_b00013.vtc similarity index 100% rename from bin/varnishtest/tests/m00029.vtc rename to vmod/tests/std_b00013.vtc diff --git a/bin/varnishtest/tests/m00050.vtc b/vmod/tests/std_b00014.vtc similarity index 100% rename from bin/varnishtest/tests/m00050.vtc rename to vmod/tests/std_b00014.vtc diff --git a/bin/varnishtest/tests/m00001.vtc b/vmod/tests/std_c00000.vtc similarity index 99% rename from bin/varnishtest/tests/m00001.vtc rename to vmod/tests/std_c00000.vtc index ac5be793a..a2f779558 100644 --- a/bin/varnishtest/tests/m00001.vtc +++ b/vmod/tests/std_c00000.vtc @@ -114,5 +114,3 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 = VCL_Log {^01030507090b0d0f11131517191b1d1f21232527292b2d2f31333537393b3d3f41434547494b4d4f51535557595b5d5f61636567696b6d6f71737577797b7d7f81838587898b8d8f91939597999b9d9fa1a3a5a7a9abadafb1b3b5b7b9bbbdbfc1c3c5c7c9cbcdcfd1d3d5d7d9dbdddfe1e3e5e7e9ebedeff1f3f5f7f9fbfd$} expect 0 = VCL_return {^deliver} } -run - -#logexpect l1 -wait diff --git a/bin/varnishtest/tests/m00017.vtc b/vmod/tests/std_c00001.vtc similarity index 100% rename from bin/varnishtest/tests/m00017.vtc rename to vmod/tests/std_c00001.vtc diff --git a/bin/varnishtest/tests/m00046.vtc b/vmod/tests/std_c00002.vtc similarity index 100% rename from bin/varnishtest/tests/m00046.vtc rename to vmod/tests/std_c00002.vtc From dridi.boukelmoune at gmail.com Fri Sep 10 12:45:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 10 Sep 2021 12:45:07 +0000 (UTC) Subject: [master] 7275194b5 vtc: Remove test case redundant with m0 Message-ID: <20210910124507.5CDC410D338@lists.varnish-cache.org> commit 7275194b5ec3ab8fd513c80021595b1e409d9a3c Author: Dridi Boukelmoune Date: Fri Sep 10 14:27:54 2021 +0200 vtc: Remove test case redundant with m0 diff --git a/bin/varnishtest/tests/m00032.vtc b/bin/varnishtest/tests/m00032.vtc deleted file mode 100644 index 711df628f..000000000 --- a/bin/varnishtest/tests/m00032.vtc +++ /dev/null @@ -1,26 +0,0 @@ -varnishtest "VMOD blob vcl.use and .discard" - -server s1 { -} -start - -varnish v1 -vcl+backend { - import blob; -} -start - -varnish v1 -vcl+backend { -} - -varnish v1 -cli "vcl.list" -varnish v1 -cli "vcl.use vcl1" - -varnish v1 -cli "vcl.use vcl2" -varnish v1 -cli "vcl.use vcl1" - -varnish v1 -expect vmods == 1 - -varnish v1 -cli "vcl.show vcl1" -varnish v1 -cli "vcl.use vcl2" -varnish v1 -cli "vcl.discard vcl1" -varnish v1 -cli "vcl.list" - -varnish v1 -expect vmods == 0 From dridi.boukelmoune at gmail.com Fri Sep 10 12:45:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 10 Sep 2021 12:45:07 +0000 (UTC) Subject: [master] 9091a9ac0 vtc: Move setenv coverage to the right category Message-ID: <20210910124507.A182410D33C@lists.varnish-cache.org> commit 9091a9ac0c3237f2e2de9858b077171bcab2cc2a Author: Dridi Boukelmoune Date: Fri Sep 10 14:31:37 2021 +0200 vtc: Move setenv coverage to the right category diff --git a/bin/varnishtest/tests/a00022.vtc b/bin/varnishtest/tests/a00022.vtc new file mode 100644 index 000000000..86a26e86f --- /dev/null +++ b/bin/varnishtest/tests/a00022.vtc @@ -0,0 +1,10 @@ +varnishtest "Test setenv" + +setenv FOO "BAR BAZ" +shell -expect "BAR BAZ" {echo $FOO} + +setenv -ifunset FOO QUUX +shell -expect "BAR BAZ" {echo $FOO} + +setenv FOO QUUX +shell -expect QUUX {echo $FOO} diff --git a/bin/varnishtest/tests/m00028.vtc b/bin/varnishtest/tests/m00028.vtc deleted file mode 100644 index 3f527a646..000000000 --- a/bin/varnishtest/tests/m00028.vtc +++ /dev/null @@ -1,73 +0,0 @@ -varnishtest "Test setenv" - -setenv FOO "BAR BAZ" - -varnish v1 -vcl { - import std; - - backend dummy { .host = "${bad_ip}"; .port = "9080"; } - - sub vcl_recv { - return(synth(200)); - } - - sub vcl_synth { - set resp.http.X-FOO = std.getenv("FOO"); - } -} -start - -client c1 { - txreq - rxresp - expect resp.http.X-FOO == "BAR BAZ" -} -run - -varnish v1 -stop - -setenv -ifunset FOO QUUX - -varnish v2 -vcl { - import std; - - backend dummy { .host = "${bad_ip}"; .port = "9080"; } - - sub vcl_recv { - return(synth(200)); - } - - sub vcl_synth { - set resp.http.X-FOO = std.getenv("FOO"); - } -} -start - -client c2 -connect ${v2_sock} { - txreq - rxresp - expect resp.http.X-FOO == "BAR BAZ" -} -run - -varnish v2 -stop - -setenv FOO QUUX - -varnish v3 -vcl { - import std; - - backend dummy { .host = "${bad_ip}"; .port = "9080"; } - - sub vcl_recv { - return(synth(200)); - } - - sub vcl_synth { - set resp.http.X-FOO = std.getenv("FOO"); - } -} -start - -client c3 -connect ${v3_sock} { - txreq - rxresp - expect resp.http.X-FOO == "QUUX" -} -run - -varnish v2 -stop From dridi.boukelmoune at gmail.com Fri Sep 10 12:45:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 10 Sep 2021 12:45:07 +0000 (UTC) Subject: [master] 2dfb0deeb vtc: Fail unattended logs with logexpect Message-ID: <20210910124507.CC71B10D340@lists.varnish-cache.org> commit 2dfb0deebb4dbb1c8dc4eb8c54e2fac250b17830 Author: Dridi Boukelmoune Date: Fri Sep 10 14:37:30 2021 +0200 vtc: Fail unattended logs with logexpect diff --git a/bin/varnishtest/tests/m00022.vtc b/bin/varnishtest/tests/m00022.vtc index ce60baa00..97bfd3409 100644 --- a/bin/varnishtest/tests/m00022.vtc +++ b/bin/varnishtest/tests/m00022.vtc @@ -8,9 +8,10 @@ server s1 { varnish v1 -vcl+backend { } -start logexpect l1 -v v1 -g raw { - expect * 0 VCL_Log "Should happen first" - expect 0 0 VCL_Log "Should happen second" - + fail add * VCL_Log "Should not happen" + expect * 0 VCL_Log "Should happen first" + expect 0 0 VCL_Log "Should happen second" + fail clear } -start varnish v1 -errvcl "Forced failure" { @@ -27,6 +28,7 @@ varnish v1 -errvcl "Forced failure" { debug.fail(); std.log("Should not happen"); } + sub vcl_fini { std.log("Should happen second"); } @@ -34,8 +36,6 @@ varnish v1 -errvcl "Forced failure" { logexpect l1 -wait -shell -match "^$" {varnishlog -n ${v1_name} -d -g raw -I "Should not happen"} - varnish v1 -cliok "param.set nuke_limit 42" varnish v1 -errvcl "nuke_limit is not the answer." { diff --git a/bin/varnishtest/tests/m00027.vtc b/bin/varnishtest/tests/m00027.vtc index d4b8ff767..730cbffd2 100644 --- a/bin/varnishtest/tests/m00027.vtc +++ b/bin/varnishtest/tests/m00027.vtc @@ -4,6 +4,13 @@ server s1 { } -start varnish v1 -vcl+backend { } -start +logexpect l1 -v v1 -g raw { + fail add * VCL_Log "Should not happen" + expect * 0 VCL_Log "Should happen first" + expect 0 0 VCL_Log "Should happen second" + fail clear +} -start + varnish v1 -errvcl "Missing dynamic backend address or port" { import debug; import std; @@ -13,9 +20,14 @@ varnish v1 -errvcl "Missing dynamic backend address or port" { } sub vcl_init { + std.log("Should happen first"); new objx = debug.dyn("", ""); std.log("Should not happen"); } + + sub vcl_fini { + std.log("Should happen second"); + } } -shell -match "^$" {varnishlog -n ${v1_name} -d -g raw -I "Should not happen"} +logexpect l1 -wait From dridi.boukelmoune at gmail.com Mon Sep 13 09:12:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 13 Sep 2021 09:12:08 +0000 (UTC) Subject: [master] 24b901cc8 CONTRIBUTING: Sync with reality Message-ID: <20210913091208.A257510906D@lists.varnish-cache.org> commit 24b901cc80c84dce7e795134a6776b7006f7d77c Author: Dridi Boukelmoune Date: Mon Sep 13 11:10:58 2021 +0200 CONTRIBUTING: Sync with reality Spotted by @Algunenano. diff --git a/CONTRIBUTING b/CONTRIBUTING index 429d58a4e..804b83385 100644 --- a/CONTRIBUTING +++ b/CONTRIBUTING @@ -15,7 +15,7 @@ Our main project communication is through or developer IRC channel:: (That channel is not for user questions, use the #varnish channel for that.) -Mondays at 13:00 EU time we hold our weekly "bugwash" where we go +Mondays at 15:00 EU time we hold our weekly "bugwash" where we go through new (and old) tickets. It speeds things up a lot if you can join the channel and answer questions directly when we go over the ticket. From phk at FreeBSD.org Mon Sep 13 11:47:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 13 Sep 2021 11:47:05 +0000 (UTC) Subject: [master] cf3fa6ee9 Add text about RFC8941 Structured Fields Message-ID: <20210913114705.CFDAF10F4B2@lists.varnish-cache.org> commit cf3fa6ee91bc9c940b485c9b2101a5bb84899c8b Author: Poul-Henning Kamp Date: Mon Sep 13 11:46:18 2021 +0000 Add text about RFC8941 Structured Fields diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst index fce689251..21a2ffae3 100644 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -31,10 +31,39 @@ regular expressions are used, like VCL, ban expressions, VSL queries etc. There were some parameters changes, see the upgrade notes for more details. -Structured Fields numbers -========================= +RFC8941 - Structured Fields +=========================== -TODO (probably deserves its own section too) +It will come as no surprise to VCL writers that HTTP headers use what can +charitably be described as "heterogenous syntax". + +In 2016, on the train back from the HTTP Workshop in Stockholm, and +in response to proposals to allow JSON in HTTP headers, I started an effort +which culminated with the publication of `RFC 8941 Structured Fields`_. + +The syntax in Structured Fields is distilled from current standardized headers, +which means that the vast majority of existing HTTP headers are already +covered. +There are unfortunate exceptions, most notably the Cookie headers. + +Starting with this release, we are gently migrating VCL towards the +Structured Field semantics. + +The first change is that it is now possible to include BLOBs in VCL, +by using the RFC 8941 syntax of:: + + ':' + + ':' + +The second and likely more significant change is numbers in VCL +now conform to RFC8941 as well: Up to 15 digits and at most 3 +decimal places. + +(These restrictions were chosen after much careful deliberation, to +ensure that no overflows would occur, even when HTTP headers are +processed in languages where numbers are represented by IEEE-754 +64 binary floating point variables,) + +.. _RFC 8941 Structured Fields: https://www.rfc-editor.org/rfc/rfc8941.html varnishd ======== From phk at FreeBSD.org Mon Sep 13 11:56:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 13 Sep 2021 11:56:05 +0000 (UTC) Subject: [master] 2bb623d6f Note that scientific notation is no longer allowed Message-ID: <20210913115605.1601910F9E0@lists.varnish-cache.org> commit 2bb623d6f80d7d6122d31314bc303cc5a30c3e15 Author: Poul-Henning Kamp Date: Mon Sep 13 11:55:13 2021 +0000 Note that scientific notation is no longer allowed diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst index 21a2ffae3..0ccb84325 100644 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -56,7 +56,7 @@ by using the RFC 8941 syntax of:: The second and likely more significant change is numbers in VCL now conform to RFC8941 as well: Up to 15 digits and at most 3 -decimal places. +decimal places, and "scientific notation" is no longer allowed. (These restrictions were chosen after much careful deliberation, to ensure that no overflows would occur, even when HTTP headers are From nils.goroll at uplex.de Mon Sep 13 12:49:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 13 Sep 2021 12:49:07 +0000 (UTC) Subject: [master] 33ffbbc5b Add -p option to varnishadm usage Message-ID: <20210913124907.7CE071110FF@lists.varnish-cache.org> commit 33ffbbc5b04849428621fc29b69d826dae3a1847 Author: Nils Goroll Date: Mon Sep 13 14:47:33 2021 +0200 Add -p option to varnishadm usage and sort options alphabetically while being at it. Relates to ##3687 diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index d95cc8845..913584bb3 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -369,8 +369,8 @@ static void v_noreturn_ usage(int status) { fprintf(stderr, - "Usage: varnishadm [-h] [-n ident] [-t timeout] [-S secretfile] " - "[-T [address]:port] [command [...]]\n"); + "Usage: varnishadm [-h] [-p] [-n ident] [-S secretfile] " + "[-t timeout] [-T [address]:port] [command [...]]\n"); fprintf(stderr, "\t-n is mutually exclusive with -S and -T\n"); exit(status); } From dridi at varni.sh Mon Sep 13 13:22:45 2021 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 13 Sep 2021 13:22:45 +0000 Subject: [master] 33ffbbc5b Add -p option to varnishadm usage In-Reply-To: <20210913124907.7CE071110FF@lists.varnish-cache.org> References: <20210913124907.7CE071110FF@lists.varnish-cache.org> Message-ID: On Mon, Sep 13, 2021 at 12:49 PM Nils Goroll wrote: > > > commit 33ffbbc5b04849428621fc29b69d826dae3a1847 > Author: Nils Goroll > Date: Mon Sep 13 14:47:33 2021 +0200 > > Add -p option to varnishadm usage > > and sort options alphabetically while being at it. > > Relates to ##3687 > > diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c > index d95cc8845..913584bb3 100644 > --- a/bin/varnishadm/varnishadm.c > +++ b/bin/varnishadm/varnishadm.c > @@ -369,8 +369,8 @@ static void v_noreturn_ > usage(int status) > { > fprintf(stderr, > - "Usage: varnishadm [-h] [-n ident] [-t timeout] [-S secretfile] " > - "[-T [address]:port] [command [...]]\n"); > + "Usage: varnishadm [-h] [-p] [-n ident] [-S secretfile] " > + "[-t timeout] [-T [address]:port] [command [...]]\n"); How dare you break lexicographic order? (n < p && T < t) > fprintf(stderr, "\t-n is mutually exclusive with -S and -T\n"); > exit(status); > } > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi.boukelmoune at gmail.com Mon Sep 13 17:07:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 13 Sep 2021 17:07:06 +0000 (UTC) Subject: [master] d654495cd whats-new: Upgrade notes on numbers Message-ID: <20210913170706.D126A117E3E@lists.varnish-cache.org> commit d654495cdd54ec5ab7f96743d6fa158d337404c9 Author: Dridi Boukelmoune Date: Mon Sep 13 19:02:56 2021 +0200 whats-new: Upgrade notes on numbers diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst index c2c3350cf..a96a10b20 100644 --- a/doc/sphinx/whats-new/upgrading-trunk.rst +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -66,7 +66,13 @@ rebuilt from scratch. Structured Fields numbers ========================= -TODO +VCL types INTEGER and REAL now map respectively to Structured Fields integer +and decimal numbers. Numbers outside of the Structured Fields bounds are no +longer accepted by the VCL compiler and the various conversion functions from +vmod_std will fail the transaction for numbers out of bounds. + +The scientific notation is no longer supported, for example ``12.34e+3`` must +be spelled out as ``12340`` instead. Memory footprint ================ @@ -155,7 +161,7 @@ significantly the VCL compilation time with very large ACLs. The ``table`` flag improves compilation time at the expense of runtime performance. Changes for developers -======================= +====================== Build ----- @@ -268,7 +274,23 @@ authors since they are not exposed by libvarnishapi. VNUM '''' -TODO +The ``VNUMpfx()`` function was replaced by ``SF_Parse_Number()`` that parses +both decimal and integer numbers from RFC8941. In addition there are new +``SF_Parse_Decimal()`` and ``SF_Parse_Integer()`` more specialized functions. + +``VNUM_bytes_unit()`` returns an integer and no longer parses factional bytes. + +New token parsers ``VNUM_uint()`` and ``VNUM_hex()`` were added. + +The other VNUM functions rely on the new SF functions for parsing, with the +same limitations. + +The following macros define the Structured Fields number bounds: + +- ``VRT_INTEGER_MIN`` +- ``VRT_INTEGER_MAX`` +- ``VRT_DECIMAL_MIN`` +- ``VRT_DECIMAL_MAX`` VRE ''' From dridi.boukelmoune at gmail.com Tue Sep 14 08:12:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 14 Sep 2021 08:12:07 +0000 (UTC) Subject: [master] 0dc55228f Fix bullet list formatting Message-ID: <20210914081207.352AE109CBE@lists.varnish-cache.org> commit 0dc55228fedb372182ad8c49337fe970896d403a Author: Arnaud Rocher Date: Tue Sep 14 09:46:28 2021 +0200 Fix bullet list formatting diff --git a/doc/sphinx/users-guide/command-line.rst b/doc/sphinx/users-guide/command-line.rst index 97f79b88b..3cecbae61 100644 --- a/doc/sphinx/users-guide/command-line.rst +++ b/doc/sphinx/users-guide/command-line.rst @@ -9,6 +9,7 @@ Required command line arguments ------------------------------- There a two command line arguments you have to set when starting Varnish, these are: + * what TCP port to serve HTTP from, and * where the backend server can be contacted. From dridi.boukelmoune at gmail.com Tue Sep 14 09:56:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 14 Sep 2021 09:56:07 +0000 (UTC) Subject: [master] 8b875005b doc: Generate references for vtc-syntax sections Message-ID: <20210914095607.C906E10EC78@lists.varnish-cache.org> commit 8b875005b5c39314fd8ebd6f7929934f9a7e2ec7 Author: Dridi Boukelmoune Date: Tue Sep 14 11:52:25 2021 +0200 doc: Generate references for vtc-syntax sections diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py index c168e05a7..f5533e84a 100644 --- a/doc/sphinx/vtc-syntax.py +++ b/doc/sphinx/vtc-syntax.py @@ -74,6 +74,7 @@ if __name__ == "__main__": parse_file(fn, cl, tl, sl) sl.sort() for section in sl: + print(".. _vtc-" + section + ":\n") print(tl[section], end="") a = section c = section.count(".") From dridi.boukelmoune at gmail.com Tue Sep 14 09:56:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 14 Sep 2021 09:56:07 +0000 (UTC) Subject: [master] 04455d6c3 doc: Release notes cross references and improvements Message-ID: <20210914095607.E3C5810EC7B@lists.varnish-cache.org> commit 04455d6c3d8b2d810007239cb1cb2b740d7ec8ab Author: Dridi Boukelmoune Date: Tue Sep 14 11:53:30 2021 +0200 doc: Release notes cross references and improvements diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 05b3ef9e9..9b1f6a026 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -430,9 +430,9 @@ barrier_sync(struct barrier *b, struct vtclog *vl) * * If you wish to synchronize the VCL, you need to declare a "sock" barrier. * This will emit a macro definition named "bNAME_sock" that you can use in - * VCL (after importing the debug vmod):: + * VCL (after importing the vtc vmod):: * - * debug.barrier_sync("${bNAME_sock}"); + * vtc.barrier_sync("${bNAME_sock}"); * * This function returns 0 if everything went well and is the equivalent of * ``barrier bNAME sync`` at the VTC top-level. diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 9efe6fd5a..ac5f6df96 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -321,7 +321,6 @@ cmd_setenv(CMD_ARGS) * * Sleep for the number of seconds specified in the argument. The number * can include a fractional part, e.g. 1.5. - * */ void cmd_delay(CMD_ARGS) @@ -454,7 +453,7 @@ addr_no_randomize_works(void) * Varnish was built with its workspace emulator. * * A feature name can be prefixed with an exclamation mark (!) to skip a - * test if a feature is present. + * test if the feature is present. * * Be careful with ignore_unknown_macro, because it may cause a test with a * misspelled macro to fail silently. You should only need it if you must diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 51b9316ab..863856505 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -68,6 +68,8 @@ considers varnish to be the client. It is possible to keep two varnishncsa instances running, one in backend mode, and one in client mode, logging to different files. +.. _ncsa-format: + FORMAT ====== diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 3513f0eb6..b309a737c 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -205,14 +205,19 @@ the flag within parens following a question mark, like this:: ... } +.. _vcl-include: Include statement ----------------- -To include a VCL file in another file use the include keyword:: +To include a VCL file in another file use the ``include`` keyword:: include "foo.vcl"; +Optionally, the ``include`` keyword can take a ``+glob`` flag to include all +files matching a glob pattern:: + + include +glob "example.org/*.vcl"; Import statement ---------------- @@ -248,6 +253,8 @@ Backends and health probes Please see :ref:`vcl-backend(7)` and :ref:`vcl-probe(7)` +.. _vcl-acl: + Access Control List (ACL) ------------------------- diff --git a/doc/sphinx/reference/vtc.rst b/doc/sphinx/reference/vtc.rst index cd74e6c70..13fd80295 100644 --- a/doc/sphinx/reference/vtc.rst +++ b/doc/sphinx/reference/vtc.rst @@ -57,6 +57,8 @@ being the command and the following ones being its arguments. To continue over to a new line without breaking the argument string, you can escape the newline character (\\n) with a backslash (\\). +.. _vtc-macros: + MACROS ====== diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst index 0ccb84325..66aec6c01 100644 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -44,6 +44,7 @@ which culminated with the publication of `RFC 8941 Structured Fields`_. The syntax in Structured Fields is distilled from current standardized headers, which means that the vast majority of existing HTTP headers are already covered. + There are unfortunate exceptions, most notably the Cookie headers. Starting with this release, we are gently migrating VCL towards the @@ -54,6 +55,11 @@ by using the RFC 8941 syntax of:: ':' + + ':' +For example, the VCL string ``"helloworld"`` can be represented as the BLOB +literal ``:aGVsbG93b3JsZA==:`` without involving a VMOD. + +See :ref:`vmod_blob(3)`. + The second and likely more significant change is numbers in VCL now conform to RFC8941 as well: Up to 15 digits and at most 3 decimal places, and "scientific notation" is no longer allowed. @@ -61,7 +67,7 @@ decimal places, and "scientific notation" is no longer allowed. (These restrictions were chosen after much careful deliberation, to ensure that no overflows would occur, even when HTTP headers are processed in languages where numbers are represented by IEEE-754 -64 binary floating point variables,) +64 binary floating point variables.) .. _RFC 8941 Structured Fields: https://www.rfc-editor.org/rfc/rfc8941.html @@ -163,12 +169,16 @@ For example:: acl +log -pedantic { ... } +See :ref:`vcl-acl`. + The ``include`` keyword supports a ``glob`` flag. For example:: include +glob "example.org/*.vcl"; +See :ref:`vcl-include`. + See upgrade notes for more details. VMODs @@ -183,6 +193,8 @@ New ``BASE64CF`` encoding scheme in ``vmod_blob``. It is similar to It is used by a certain CDN provider who also inspired the name. +See the ``vmod_blob`` manual (:ref:`vmod_blob-base64`). + varnishlog ========== @@ -191,9 +203,13 @@ fetched, ``Hit`` records contain progress of the fetch task. This should help troubleshooting when cache hits appear to be slow, whether or not the backend is still serving the response. +See :ref:`vcl(7)`. + By default ``VCL_acl`` records are no longer emitted. They can be brought back by adding a ``+log`` flag to the ACL declaration. +See :ref:`vcl-acl`. + varnishncsa =========== @@ -205,7 +221,7 @@ New ``%{...}t`` time formats: - ``msec_frac`` - ``usec_frac`` -See the varnishncsa manual for more information. +See the ``varnishncsa`` manual (:ref:`ncsa-format`) for more information. varnishadm ========== @@ -220,7 +236,7 @@ varnishtest Test cases should be generally more reactive, whether it is detecting a ``varnishd`` startup failure, waiting for ``varnishd`` to stop, or -when fail tests and there are barriers waiting for a synchronization. +when tests fail and there are barriers waiting for a synchronization. Clients and servers can have up to 64 headers in requests and responses. @@ -245,11 +261,15 @@ There are new feature checks: The undocumented ``pcre_jit`` feature check is gone. +See the VTC manual (:ref:`vtc-feature`) for more details. + There is a new ``tunnel`` command that acts as a proxy between two peers. A tunnel can pause and control how much data goes in each direction, and can be used to trigger socket timeouts, possibly in the middle of protocol frames, without having to change how the peers are implemented. +See the VTC manual (:ref:`vtc-tunnel`) for more details. + There is a new dynamic macro ``${string,repeat,,}`` to avoid very long lines or potential mistakes when maintained by hand. For example, the two following strings are equivalent:: @@ -257,6 +277,8 @@ the two following strings are equivalent:: "AAA" "${string,repeat,3,A}" +See the VTC manual (:ref:`vtc-macros`) for more details. + There were also various improvements to HTTP/2 testing, and more should be expected. @@ -289,7 +311,4 @@ The deprecated functions ``VSB_new()`` and ``VSB_delete()`` were removed. See upgrade notes for more information. -**XXX changes concerning VRT, the public APIs, source code organization, -builds etc.** - *eof* diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst index a96a10b20..a6b127ee8 100644 --- a/doc/sphinx/whats-new/upgrading-trunk.rst +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -30,11 +30,16 @@ The new ``varnishd`` parameter ``pcre2_jit_compilation`` controls whether jit compilation should be attempted and has no effect if jit support was disabled at configure time. +See :ref:`ref_param_pcre2_jit_compilation`. + The former parameters ``pcre_match_limit`` and ``pcre_match_limit_recursion`` were renamed to ``pcre2_match_limit`` and ``pcre2_depth_limit``. With older PCRE2 libraries, it is possible to see the depth limit being referred to as recursion limit in error messages. +See :ref:`ref_param_pcre2_depth_limit` and :ref:`ref_param_pcre2_depth_limit` +for more information. + The syntax of regular expression should be the same, but it is possible to run into subtle differences. We are aware one such difference, PCRE2 fails the compilation of unknown escape sequences. For example PCRE interprets @@ -160,6 +165,8 @@ ACLs are optimized for runtime performance by default, which can increase significantly the VCL compilation time with very large ACLs. The ``table`` flag improves compilation time at the expense of runtime performance. +See :ref:`vcl-acl`. + Changes for developers ====================== diff --git a/vmod/vmod_blob.vcc b/vmod/vmod_blob.vcc index b3644c66c..b6256eeac 100644 --- a/vmod/vmod_blob.vcc +++ b/vmod/vmod_blob.vcc @@ -122,6 +122,8 @@ also be written as:: The *case* ENUM MUST be set to ``DEFAULT`` for ``IDENTITY`` encodings. +.. _vmod_blob-base64: + BASE64* ~~~~~~~ From dridi.boukelmoune at gmail.com Tue Sep 14 11:30:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 14 Sep 2021 11:30:08 +0000 (UTC) Subject: [master] bf1c04bc7 varnishadm: Only print the status in interactive mode Message-ID: <20210914113008.DCC101115A0@lists.varnish-cache.org> commit bf1c04bc7d0928dcdde900ef3795b3ab5d8b3950 Author: Dridi Boukelmoune Date: Thu Sep 9 18:35:39 2021 +0200 varnishadm: Only print the status in interactive mode Or when the -p option is used. Fixes #3687 diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index 913584bb3..888bd2a44 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -76,6 +76,10 @@ exit(status); \ } while (0) +enum pass_mode_e { + pass_script, + pass_interactive, +}; static double timeout = 5; static int p_arg = 0; @@ -161,7 +165,7 @@ cli_sock(const char *T_arg, const char *S_arg) } static unsigned -pass_answer(int fd) +pass_answer(int fd, enum pass_mode_e mode) { unsigned u, status; char *answer = NULL; @@ -180,7 +184,8 @@ pass_answer(int fd) } else if (p_arg) { printf("%-3u %-8u\n", status, 0U); } else { - printf("%u\n", status); + if (mode == pass_interactive) + printf("%u\n", status); if (answer != NULL) printf("%s\n", answer); if (status == CLIS_TRUNCATED) @@ -204,7 +209,7 @@ do_args(int sock, int argc, char * const *argv) cli_write(sock, argv[i]); } cli_write(sock, "\n"); - status = pass_answer(sock); + status = pass_answer(sock, pass_script); closefd(&sock); if (status == CLIS_OK || status == CLIS_TRUNCATED) exit(0); @@ -317,7 +322,7 @@ interactive(int sock) if (fds[0].revents & POLLIN) { /* Get rid of the prompt, kinda hackish */ printf("\r \r"); - (void)pass_answer(fds[0].fd); + (void)pass_answer(fds[0].fd, pass_interactive); rl_forced_update_display(); } if (fds[1].revents & POLLIN) { @@ -348,7 +353,7 @@ pass(int sock) } assert(i > 0); if (fds[0].revents & POLLIN) - (void)pass_answer(fds[0].fd); + (void)pass_answer(fds[0].fd, pass_script); if (fds[1].revents & POLLIN || fds[1].revents & POLLHUP) { n = read(fds[1].fd, buf, sizeof buf - 1); if (n == 0) { From dridi.boukelmoune at gmail.com Tue Sep 14 11:44:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 14 Sep 2021 11:44:06 +0000 (UTC) Subject: [master] 2be7bb5d3 varnishadm: Add -p to the synopsis Message-ID: <20210914114406.ED746111D0A@lists.varnish-cache.org> commit 2be7bb5d3e6092039e00a42e6736ef92862fdafe Author: Dridi Boukelmoune Date: Tue Sep 14 13:41:19 2021 +0200 varnishadm: Add -p to the synopsis Refs 33ffbbc5b048 diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index 888bd2a44..1a7f7c00c 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -374,8 +374,8 @@ static void v_noreturn_ usage(int status) { fprintf(stderr, - "Usage: varnishadm [-h] [-p] [-n ident] [-S secretfile] " - "[-t timeout] [-T [address]:port] [command [...]]\n"); + "Usage: varnishadm [-h] [-n ident] [-p] [-S secretfile] " + "[-T [address]:port] [-t timeout] [command [...]]\n"); fprintf(stderr, "\t-n is mutually exclusive with -S and -T\n"); exit(status); } diff --git a/doc/sphinx/reference/varnishadm.rst b/doc/sphinx/reference/varnishadm.rst index a0bb09c64..4d89b52f3 100644 --- a/doc/sphinx/reference/varnishadm.rst +++ b/doc/sphinx/reference/varnishadm.rst @@ -19,7 +19,7 @@ Control a running Varnish instance SYNOPSIS ======== -varnishadm [-h] [-n ident] [-t timeout] [-S secretfile] [-T [address]:port] [command [...]] +varnishadm [-h] [-n ident] [-p] [-S secretfile] [-T [address]:port] [-t timeout] [command [...]] DESCRIPTION From dridi.boukelmoune at gmail.com Tue Sep 14 11:47:04 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 14 Sep 2021 11:47:04 +0000 (UTC) Subject: [master] 4e2c05dee doc: Reorder varnishadm options Message-ID: <20210914114705.03830111FA2@lists.varnish-cache.org> commit 4e2c05dee1a953c31ccba26ed81fc70696540e9b Author: Dridi Boukelmoune Date: Tue Sep 14 13:44:50 2021 +0200 doc: Reorder varnishadm options Refs 33ffbbc5b048 diff --git a/doc/sphinx/reference/varnishadm.rst b/doc/sphinx/reference/varnishadm.rst index 4d89b52f3..8cfdc6983 100644 --- a/doc/sphinx/reference/varnishadm.rst +++ b/doc/sphinx/reference/varnishadm.rst @@ -43,25 +43,25 @@ OPTIONS -h Print program usage and exit. +-n ident + Connect to the instance of `varnishd` with this name. + -p Force `pass` mode and make the output follow the VCLI protocol. This disables command-history/command-completion and makes it easier for programs to parse the response(s). --n ident - Connect to the instance of `varnishd` with this name. - -S secretfile Specify the authentication secret file. This should be the same -S argument as was given to `varnishd`. Only processes which can read the contents of this file, will be able to authenticate the CLI connection. --t timeout - Wait no longer than this many seconds for an operation to finish. - -T Connect to the management interface at the specified address and port. +-t timeout + Wait no longer than this many seconds for an operation to finish. + The syntax and operation of the actual CLI interface is described in the :ref:`varnish-cli(7)` manual page. Parameters are described in From dridi.boukelmoune at gmail.com Wed Sep 15 08:27:04 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 15 Sep 2021 08:27:04 +0000 (UTC) Subject: [7.0] 454733b82 Prepare for 7.0.0 Message-ID: <20210915082704.DEDF810E69D@lists.varnish-cache.org> commit 454733b82a3279a1603516b4f0a07f8bad4bcd55 Author: Dridi Boukelmoune Date: Wed Sep 15 07:25:07 2021 +0200 Prepare for 7.0.0 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 6c6484236..2e4ae1650 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -166,7 +166,7 @@ Resp_Setup_Deliver(struct req *req) http_PrintfHeader(h, "Age: %.0f", floor(fmax(0., req->t_prev - oc->t_origin))); - http_SetHeader(h, "Via: 1.1 varnish (Varnish/6.6)"); + http_SetHeader(h, "Via: 1.1 varnish (Varnish/7.0)"); if (cache_param->http_gzip_support && ObjCheckFlag(req->wrk, oc, OF_GZIPED) && diff --git a/configure.ac b/configure.ac index da9bb0603..162a26d48 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.69) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2021 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [trunk], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [7.0.0], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index 8a1522a31..16af4e23e 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -40,9 +40,9 @@ Conventions used in this manual include: Longer listings like example command output and VCL look like this:: $ /opt/varnish/sbin/varnishd -V - varnishd (varnish-trunk revision 199de9b) + varnishd (varnish-trunk revision 1234567) Copyright (c) 2006 Verdens Gang AS - Copyright (c) 2006-2020 Varnish Software + Copyright (c) 2006-2021 Varnish Software .. For maintainers: diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-7.0.rst similarity index 96% rename from doc/sphinx/whats-new/changes-trunk.rst rename to doc/sphinx/whats-new/changes-7.0.rst index 66aec6c01..625d7b08c 100644 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ b/doc/sphinx/whats-new/changes-7.0.rst @@ -3,18 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause See LICENSE file for full text of license -**Note: This is a working document for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see:** :ref:`whats-new-index` +.. _whatsnew_changes_7.0: -.. _whatsnew_changes_CURRENT: - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -Changes in Varnish **$NEXT_RELEASE** -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%% +Changes in Varnish 7.0 +%%%%%%%%%%%%%%%%%%%%%% For information about updating your current Varnish deployment to the -new version, see :ref:`whatsnew_upgrading_CURRENT`. +new version, see :ref:`whatsnew_upgrading_7.0`. A more detailed and technical account of changes in Varnish, with links to issues that have been fixed and pull requests that have been diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst index 6eaf369a6..6257163eb 100644 --- a/doc/sphinx/whats-new/index.rst +++ b/doc/sphinx/whats-new/index.rst @@ -13,18 +13,14 @@ This section describes the changes and improvements between different versions of Varnish, and what upgrading between the different versions entail. -Varnish **$NEXT_RELEASE** -------------------------- - -**Note: These are working documents for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see the chapters listed below.** +Varnish 7.0 +----------- .. toctree:: :maxdepth: 2 - changes-trunk - upgrading-trunk + changes-7.0 + upgrading-7.0 Varnish 6.6 ----------- diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-7.0.rst similarity index 97% rename from doc/sphinx/whats-new/upgrading-trunk.rst rename to doc/sphinx/whats-new/upgrading-7.0.rst index a6b127ee8..6fd865b34 100644 --- a/doc/sphinx/whats-new/upgrading-trunk.rst +++ b/doc/sphinx/whats-new/upgrading-7.0.rst @@ -3,15 +3,11 @@ SPDX-License-Identifier: BSD-2-Clause See LICENSE file for full text of license -**Note: This is a working document for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see:** :ref:`whats-new-index` +.. _whatsnew_upgrading_7.0: -.. _whatsnew_upgrading_CURRENT: - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -Upgrading to Varnish **$NEXT_RELEASE** -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%% +Upgrading to Varnish 7.0 +%%%%%%%%%%%%%%%%%%%%%%%% PCRE2 ===== diff --git a/include/vrt.h b/include/vrt.h index 5e2e8f7d4..1406e7b59 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -53,7 +53,7 @@ * Whenever something is deleted or changed in a way which is not * binary/load-time compatible, increment MAJOR version * - * NEXT (2021-09-15) + * 14.0 (2021-09-15) * VIN_n_Arg() no directly returns the directory name. * VSB_new() and VSB_delete() removed * VCL_STRINGLIST, vrt_magic_string_end removed @@ -225,7 +225,7 @@ * vrt_acl type added */ -#define VRT_MAJOR_VERSION 13U +#define VRT_MAJOR_VERSION 14U #define VRT_MINOR_VERSION 0U diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c index 9bc77a378..af4fd8835 100644 --- a/lib/libvarnish/version.c +++ b/lib/libvarnish/version.c @@ -74,7 +74,7 @@ VCS_String(const char *which) ")" "\n" "Copyright (c) 2006 Verdens Gang AS\n" - "Copyright (c) 2006-2020 Varnish Software\n" + "Copyright (c) 2006-2021 Varnish Software\n" ); default: WRONG("Wrong argument to VCS_String"); From dridi.boukelmoune at gmail.com Wed Sep 15 09:15:09 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 15 Sep 2021 09:15:09 +0000 (UTC) Subject: [master] 454733b82 Prepare for 7.0.0 Message-ID: <20210915091509.CDB1510FCCE@lists.varnish-cache.org> commit 454733b82a3279a1603516b4f0a07f8bad4bcd55 Author: Dridi Boukelmoune Date: Wed Sep 15 07:25:07 2021 +0200 Prepare for 7.0.0 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 6c6484236..2e4ae1650 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -166,7 +166,7 @@ Resp_Setup_Deliver(struct req *req) http_PrintfHeader(h, "Age: %.0f", floor(fmax(0., req->t_prev - oc->t_origin))); - http_SetHeader(h, "Via: 1.1 varnish (Varnish/6.6)"); + http_SetHeader(h, "Via: 1.1 varnish (Varnish/7.0)"); if (cache_param->http_gzip_support && ObjCheckFlag(req->wrk, oc, OF_GZIPED) && diff --git a/configure.ac b/configure.ac index da9bb0603..162a26d48 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.69) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2021 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [trunk], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [7.0.0], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index 8a1522a31..16af4e23e 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -40,9 +40,9 @@ Conventions used in this manual include: Longer listings like example command output and VCL look like this:: $ /opt/varnish/sbin/varnishd -V - varnishd (varnish-trunk revision 199de9b) + varnishd (varnish-trunk revision 1234567) Copyright (c) 2006 Verdens Gang AS - Copyright (c) 2006-2020 Varnish Software + Copyright (c) 2006-2021 Varnish Software .. For maintainers: diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-7.0.rst similarity index 96% rename from doc/sphinx/whats-new/changes-trunk.rst rename to doc/sphinx/whats-new/changes-7.0.rst index 66aec6c01..625d7b08c 100644 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ b/doc/sphinx/whats-new/changes-7.0.rst @@ -3,18 +3,14 @@ SPDX-License-Identifier: BSD-2-Clause See LICENSE file for full text of license -**Note: This is a working document for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see:** :ref:`whats-new-index` +.. _whatsnew_changes_7.0: -.. _whatsnew_changes_CURRENT: - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -Changes in Varnish **$NEXT_RELEASE** -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%% +Changes in Varnish 7.0 +%%%%%%%%%%%%%%%%%%%%%% For information about updating your current Varnish deployment to the -new version, see :ref:`whatsnew_upgrading_CURRENT`. +new version, see :ref:`whatsnew_upgrading_7.0`. A more detailed and technical account of changes in Varnish, with links to issues that have been fixed and pull requests that have been diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst index 6eaf369a6..6257163eb 100644 --- a/doc/sphinx/whats-new/index.rst +++ b/doc/sphinx/whats-new/index.rst @@ -13,18 +13,14 @@ This section describes the changes and improvements between different versions of Varnish, and what upgrading between the different versions entail. -Varnish **$NEXT_RELEASE** -------------------------- - -**Note: These are working documents for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see the chapters listed below.** +Varnish 7.0 +----------- .. toctree:: :maxdepth: 2 - changes-trunk - upgrading-trunk + changes-7.0 + upgrading-7.0 Varnish 6.6 ----------- diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-7.0.rst similarity index 97% rename from doc/sphinx/whats-new/upgrading-trunk.rst rename to doc/sphinx/whats-new/upgrading-7.0.rst index a6b127ee8..6fd865b34 100644 --- a/doc/sphinx/whats-new/upgrading-trunk.rst +++ b/doc/sphinx/whats-new/upgrading-7.0.rst @@ -3,15 +3,11 @@ SPDX-License-Identifier: BSD-2-Clause See LICENSE file for full text of license -**Note: This is a working document for a future release, with running -updates for changes in the development branch. For changes in the -released versions of Varnish, see:** :ref:`whats-new-index` +.. _whatsnew_upgrading_7.0: -.. _whatsnew_upgrading_CURRENT: - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -Upgrading to Varnish **$NEXT_RELEASE** -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%% +Upgrading to Varnish 7.0 +%%%%%%%%%%%%%%%%%%%%%%%% PCRE2 ===== diff --git a/include/vrt.h b/include/vrt.h index 5e2e8f7d4..1406e7b59 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -53,7 +53,7 @@ * Whenever something is deleted or changed in a way which is not * binary/load-time compatible, increment MAJOR version * - * NEXT (2021-09-15) + * 14.0 (2021-09-15) * VIN_n_Arg() no directly returns the directory name. * VSB_new() and VSB_delete() removed * VCL_STRINGLIST, vrt_magic_string_end removed @@ -225,7 +225,7 @@ * vrt_acl type added */ -#define VRT_MAJOR_VERSION 13U +#define VRT_MAJOR_VERSION 14U #define VRT_MINOR_VERSION 0U diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c index 9bc77a378..af4fd8835 100644 --- a/lib/libvarnish/version.c +++ b/lib/libvarnish/version.c @@ -74,7 +74,7 @@ VCS_String(const char *which) ")" "\n" "Copyright (c) 2006 Verdens Gang AS\n" - "Copyright (c) 2006-2020 Varnish Software\n" + "Copyright (c) 2006-2021 Varnish Software\n" ); default: WRONG("Wrong argument to VCS_String"); From dridi.boukelmoune at gmail.com Wed Sep 15 09:15:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 15 Sep 2021 09:15:10 +0000 (UTC) Subject: [master] 45cd6f626 Merge tag 'varnish-7.0.0' Message-ID: <20210915091510.092BB10FCD1@lists.varnish-cache.org> commit 45cd6f626a16cb7bbdf8effaa00a109932de65a1 Merge: 4e2c05dee 454733b82 Author: Dridi Boukelmoune Date: Wed Sep 15 11:07:28 2021 +0200 Merge tag 'varnish-7.0.0' Releasing 7.0.0 From martin at varnish-software.com Wed Sep 15 11:38:06 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Wed, 15 Sep 2021 11:38:06 +0000 (UTC) Subject: [7.0] fd41231a1 Prepare the release branch for package building Message-ID: <20210915113807.0A3CD116490@lists.varnish-cache.org> commit fd41231a1edfdf52bc110901517e74350404e5dd Author: Martin Blix Grydeland Date: Wed Sep 15 13:35:41 2021 +0200 Prepare the release branch for package building The CircleCI default commit behaviour is changed to package building now that this is a release branch. diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b834c526..705a0162c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ parameters: default: "HEAD" pkg-commit: type: string - default: "master" + default: "7.0" configure_args: type: string default: | @@ -312,43 +312,6 @@ jobs: workflows: version: 2 commit: - jobs: - - dist - - distcheck: - name: distcheck_centos_7 - dist: centos - release: "7" - - distcheck: - name: distcheck_centos_8 - dist: centos - release: "8" - - distcheck: - name: distcheck_debian_buster - dist: debian - release: buster - extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator - - distcheck: - name: distcheck_ubuntu_bionic - dist: ubuntu - release: bionic - extra_conf: CFLAGS='-g -O2 -m32' - - distcheck: - name: distcheck_alpine - dist: alpine - release: "latest" - #extra_conf: --without-jemalloc - - distcheck: - name: distcheck_archlinux - dist: archlinux - release: "latest" - nightly: - triggers: - - schedule: - cron: "0 4 * * *" - filters: - branches: - only: - - master jobs: - dist - tar_pkg_tools From martin at varnish-software.com Wed Sep 15 12:05:07 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Wed, 15 Sep 2021 12:05:07 +0000 (UTC) Subject: [7.0] d24694967 Adjust CircleCI config to take dist-url as an arg Message-ID: <20210915120507.8FE37117194@lists.varnish-cache.org> commit d24694967ff2b6a6d88f91257443512141a62569 Author: Martin Blix Grydeland Date: Wed Sep 15 14:03:38 2021 +0200 Adjust CircleCI config to take dist-url as an arg diff --git a/.circleci/config.yml b/.circleci/config.yml index 705a0162c..d9ba4fa58 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,12 @@ parameters: pkg-commit: type: string default: "7.0" + dist-url: + type: string + default: "" + dist-url-sha256: + type: string + default: "" configure_args: type: string default: | @@ -17,7 +23,7 @@ parameters: --with-persistent-storage \ jobs: dist: - description: Builds varnish-x.y.z.tar.gz that is used later for the packaging jobs + description: Build or download varnish-x.y.z.tar.gz that is used later for the packaging jobs docker: - image: centos:7 steps: @@ -37,17 +43,40 @@ jobs: python3 \ python-sphinx - checkout - - run: - name: Create the dist tarball - command: | - git checkout << pipeline.parameters.vc-commit >> - # if version is "trunk", it's a weekly tarball, override the version - if grep 'AC_INIT.*trunk.*' ./configure.ac; then - sed -i -e "s/^AC_INIT.*trunk.*/AC_INIT([Varnish], [$(date +%Y%m%d)], [varnish-dev at varnish-cache.org])/" ./configure.ac - touch .is_weekly - fi - ./autogen.des --quiet - make dist -j 16 + - when: + condition: << pipeline.parameters.dist-url >> + steps: + - run: + name: Download the dist tarball + command: | + curl -s << pipeline.parameters.dist-url >> -o varnish-dist.tar.gz + - when: + condition: << pipeline.parameters.dist-url-sha256 >> + steps: + - run: + name: Verify downloaded tarball + command: | + echo "<< pipeline.parameters.dist-url-sha256 >> varnish-dist.tar.gz" | sha256sum -c + -run: + name: Rename the dist tarball by parsed version + command: | + mkdir parse-version-tmp + cd parse-version-tmp + tar xzf ../varnish-dist.tar.gz + VERSION=$(varnish-*/configure --version | awk 'NR == 1 {print $NF}') + cd .. + mv -v varnish-dist.tar.gz varnish-${VERSION}.tar.gz + - unless: + condition: << pipeline.parameters.dist-url >> + steps: + - run: + name: Create the dist tarball + command: | + git checkout << pipeline.parameters.vc-commit >> + # Locally built tarballs are always weekly - built with date in package name + touch .is_weekly + ./autogen.des --quiet + make dist -j 16 - persist_to_workspace: root: . paths: From martin at varnish-software.com Wed Sep 15 12:07:04 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Wed, 15 Sep 2021 12:07:04 +0000 (UTC) Subject: [7.0] 0e73c0f85 Fix up yaml syntax Message-ID: <20210915120704.AD6111173C0@lists.varnish-cache.org> commit 0e73c0f85338b589bbeb4db1d8395ce6681f3d0c Author: Martin Blix Grydeland Date: Wed Sep 15 14:06:16 2021 +0200 Fix up yaml syntax diff --git a/.circleci/config.yml b/.circleci/config.yml index d9ba4fa58..f6d0cbc91 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,15 +57,15 @@ jobs: name: Verify downloaded tarball command: | echo "<< pipeline.parameters.dist-url-sha256 >> varnish-dist.tar.gz" | sha256sum -c - -run: - name: Rename the dist tarball by parsed version - command: | - mkdir parse-version-tmp - cd parse-version-tmp - tar xzf ../varnish-dist.tar.gz - VERSION=$(varnish-*/configure --version | awk 'NR == 1 {print $NF}') - cd .. - mv -v varnish-dist.tar.gz varnish-${VERSION}.tar.gz + - run: + name: Rename the dist tarball by parsed version + command: | + mkdir parse-version-tmp + cd parse-version-tmp + tar xzf ../varnish-dist.tar.gz + VERSION=$(varnish-*/configure --version | awk 'NR == 1 {print $NF}') + cd .. + mv -v varnish-dist.tar.gz varnish-${VERSION}.tar.gz - unless: condition: << pipeline.parameters.dist-url >> steps: From martin at varnish-software.com Wed Sep 15 14:17:05 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Wed, 15 Sep 2021 14:17:05 +0000 (UTC) Subject: [master] d3b60b06a Adjust CircleCI config to be compatible with release branches Message-ID: <20210915141705.4CB7211ABED@lists.varnish-cache.org> commit d3b60b06a5c34d5d6b3ea086cc0161841f8828b4 Author: Martin Blix Grydeland Date: Wed Sep 15 14:03:38 2021 +0200 Adjust CircleCI config to be compatible with release branches Add the necessary bits to the master CCI configuration to make the package building parts also work in the release branches. This way there is no need to tailor the release branches' CCI setup on release day. This should not (famous last words) change how the weekly packages from the master branch are generated. diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b834c526..b459dc3d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,12 @@ parameters: pkg-commit: type: string default: "master" + dist-url: + type: string + default: "" + dist-url-sha256: + type: string + default: "" configure_args: type: string default: | @@ -17,7 +23,7 @@ parameters: --with-persistent-storage \ jobs: dist: - description: Builds varnish-x.y.z.tar.gz that is used later for the packaging jobs + description: Build or download varnish-x.y.z.tar.gz that is used later for the packaging jobs docker: - image: centos:7 steps: @@ -37,17 +43,46 @@ jobs: python3 \ python-sphinx - checkout - - run: - name: Create the dist tarball - command: | - git checkout << pipeline.parameters.vc-commit >> - # if version is "trunk", it's a weekly tarball, override the version - if grep 'AC_INIT.*trunk.*' ./configure.ac; then - sed -i -e "s/^AC_INIT.*trunk.*/AC_INIT([Varnish], [$(date +%Y%m%d)], [varnish-dev at varnish-cache.org])/" ./configure.ac - touch .is_weekly - fi - ./autogen.des --quiet - make dist -j 16 + - when: + condition: << pipeline.parameters.dist-url >> + steps: + - run: + name: Download the dist tarball + command: | + curl -s << pipeline.parameters.dist-url >> -o varnish-dist.tar.gz + - when: + condition: << pipeline.parameters.dist-url-sha256 >> + steps: + - run: + name: Verify downloaded tarball + command: | + echo "<< pipeline.parameters.dist-url-sha256 >> varnish-dist.tar.gz" | sha256sum -c + - run: + name: Rename the dist tarball by parsed version + command: | + mkdir parse-version-tmp + cd parse-version-tmp + tar xzf ../varnish-dist.tar.gz + VERSION=$(varnish-*/configure --version | awk 'NR == 1 {print $NF}') + cd .. + mv -v varnish-dist.tar.gz varnish-${VERSION}.tar.gz + - unless: + condition: << pipeline.parameters.dist-url >> + steps: + - run: + name: Create the dist tarball + command: | + git checkout << pipeline.parameters.vc-commit >> + # Locally built tarballs are always built with weekly in package name + touch .is_weekly + # If version is "trunk", override version to add date + if grep 'AC_INIT.*trunk.*' ./configure.ac; then + sed -e "s/AC_INIT(\[\(.*\)\], \[\(.*\)\], \[\(.*\)\])/AC_INIT([\1], [$(date +%Y%m%d)], [\3])/" ./configure.ac + else + sed -e "s/AC_INIT(\[\(.*\)\], \[\(.*\)\], \[\(.*\)\])/AC_INIT([\1], [\2-$(date +%Y%m%d)], [\3])/" ./configure.ac + fi + ./autogen.des --quiet + make dist -j 16 - persist_to_workspace: root: . paths: From martin at varnish-software.com Thu Sep 16 06:39:08 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 16 Sep 2021 06:39:08 +0000 (UTC) Subject: [master] edd7b69f0 Fix up CCI config Message-ID: <20210916063908.7E124110C6F@lists.varnish-cache.org> commit edd7b69f0e8dc4736baa520faa929d315b05d06d Author: Martin Blix Grydeland Date: Thu Sep 16 08:38:17 2021 +0200 Fix up CCI config Forgot to pass -i for inplace sed run diff --git a/.circleci/config.yml b/.circleci/config.yml index b459dc3d1..1a32fef41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,9 +77,9 @@ jobs: touch .is_weekly # If version is "trunk", override version to add date if grep 'AC_INIT.*trunk.*' ./configure.ac; then - sed -e "s/AC_INIT(\[\(.*\)\], \[\(.*\)\], \[\(.*\)\])/AC_INIT([\1], [$(date +%Y%m%d)], [\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\], \[\(.*\)\], \[\(.*\)\])/AC_INIT([\1], [$(date +%Y%m%d)], [\3])/" ./configure.ac else - sed -e "s/AC_INIT(\[\(.*\)\], \[\(.*\)\], \[\(.*\)\])/AC_INIT([\1], [\2-$(date +%Y%m%d)], [\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\], \[\(.*\)\], \[\(.*\)\])/AC_INIT([\1], [\2-$(date +%Y%m%d)], [\3])/" ./configure.ac fi ./autogen.des --quiet make dist -j 16 From dridi.boukelmoune at gmail.com Mon Sep 20 05:50:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 Sep 2021 05:50:10 +0000 (UTC) Subject: [master] 2ac661197 vtc: Add varnishadm pass/script/interactive coverage Message-ID: <20210920055010.B736C114F9E@lists.varnish-cache.org> commit 2ac661197a84c4ae0242e3a277a659f50dc9cec1 Author: Dridi Boukelmoune Date: Mon Sep 20 07:36:41 2021 +0200 vtc: Add varnishadm pass/script/interactive coverage To avoid trailing blanks in diffs I decided to use underscores. In curses test cases, pack process writes with the following expectations. Refs #3691 diff --git a/bin/varnishtest/tests/u00001.vtc b/bin/varnishtest/tests/u00001.vtc new file mode 100644 index 000000000..d2e7e34ab --- /dev/null +++ b/bin/varnishtest/tests/u00001.vtc @@ -0,0 +1,62 @@ +varnishtest "trivial run of varnishadm in script mode" + +feature cmd "python3 -c 'import json'" + +varnish v1 -vcl {backend be none;} -start + +shell { + cat >expected.txt <<-EOF + Child_in_state_running + EOF + varnishadm -n ${v1_name} status | tr ' ' _ >actual.txt + diff -u expected.txt actual.txt +} + +# same command via stdin +shell { + echo status | varnishadm -n ${v1_name} | tr ' ' _ >actual.txt + diff -u expected.txt actual.txt +} + +shell { + cat >expected.txt <<-EOF + 200_22______ + Child_in_state_running + EOF + { + varnishadm -n ${v1_name} -p status + printf '\n' # easier to add here than remove in expected.txt + } | tr ' ' _ >actual.txt + diff -u expected.txt actual.txt +} + +# same command via stdin +shell { + echo status | varnishadm -n ${v1_name} -p | tr ' ' _ >actual.txt + printf '\n' >>actual.txt + diff -u expected.txt actual.txt +} + +shell { + varnishadm -n ${v1_name} status -j | + python3 -c 'import sys, json; print(json.load(sys.stdin))' +} + +# same command via stdin +shell { + echo 'status -j' | + varnishadm -n ${v1_name} | + python3 -c 'import sys, json; print(json.load(sys.stdin))' +} + +shell -err { + varnishadm -n ${v1_name} -p status -j | + python3 -c 'import sys, json; print(json.load(sys.stdin))' +} + +# same command via stdin +shell -err { + echo 'status -j' | + varnishadm -n ${v1_name} -p | + python3 -c 'import sys, json; print(json.load(sys.stdin))' +} diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc index 439fbfafc..1561ad1e4 100644 --- a/bin/varnishtest/tests/u00011.vtc +++ b/bin/varnishtest/tests/u00011.vtc @@ -18,16 +18,15 @@ process p1 -log {varnishadm -n ${v1_name}} -start process p1 -expect-text 0 1 "Type 'quit' to close CLI session." +# expect a status in interactive mode (after a banner) process p1 -write "pin\t\r" - -process p1 -expect-text 0 1 "PONG" +process p1 -expect-text 12 1 "200 " +process p1 -expect-text 13 1 PONG process p1 -write "vcl.li\t\r" - process p1 -expect-text 0 1 "active auto warm" process p1 -write "vcl.s\t\th\t vcl1\r" - process p1 -expect-text 0 1 "backend s1" process p1 -screen_dump -write "quit\r" -wait diff --git a/bin/varnishtest/tests/u00012.vtc b/bin/varnishtest/tests/u00012.vtc index 5dc3fcb9e..6c5af113f 100644 --- a/bin/varnishtest/tests/u00012.vtc +++ b/bin/varnishtest/tests/u00012.vtc @@ -16,18 +16,15 @@ varnish v1 -vsl_catchup process p1 -log {varnishadm -p -n ${v1_name}} -start +# expect a full status line in pass mode (no banner) process p1 -write "ping\r" - -process p1 -expect-text 0 1 "200 19" - -process p1 -expect-text 0 1 "PONG" +process p1 -expect-text 2 1 "200 19 " +process p1 -expect-text 3 1 PONG process p1 -write "vcl.list\r" - process p1 -expect-text 0 0 "auto warm" process p1 -write "vcl.show vcl1\r" - process p1 -expect-text 0 0 "backend s1" process p1 -screen_dump -write "\x04" -wait From dridi.boukelmoune at gmail.com Mon Sep 20 06:53:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 Sep 2021 06:53:05 +0000 (UTC) Subject: [master] b60d6ebab varnishtest: Remove the err_shell command Message-ID: <20210920065305.D4E36116B6A@lists.varnish-cache.org> commit b60d6ebabe760114a26f5f6bf9db97907d798412 Author: Dridi Boukelmoune Date: Mon Sep 20 08:19:39 2021 +0200 varnishtest: Remove the err_shell command It has been deprecated long enough and could have gone away with the 7.0 release. Refs 3fb17e94de3e diff --git a/bin/varnishtest/cmds.h b/bin/varnishtest/cmds.h index dccdcf5bb..8afee5140 100644 --- a/bin/varnishtest/cmds.h +++ b/bin/varnishtest/cmds.h @@ -43,7 +43,6 @@ CMD_GLOBAL(shell) #define CMD_TOP(x) #endif CMD_TOP(client) -CMD_TOP(err_shell) CMD_TOP(feature) CMD_TOP(haproxy) #ifdef VTEST_WITH_VTC_LOGEXPECT diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index ac5f6df96..a70d6aa55 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -246,34 +246,6 @@ cmd_shell(CMD_ARGS) cmd_shell_engine(vl, ok, av[n], expect, re); } -/* SECTION: err_shell err_shell - * - * NOTICE: err_shell is deprecated, use `shell -err -expect` instead. - * - * This is very similar to the the ``shell`` command, except it takes a first - * string as argument before the command:: - * - * err_shell "foo" "echo foo" - * - * err_shell expect the shell command to fail AND stdout to match the string, - * failing the test case otherwise. - */ - -void v_matchproto_(cmd_f) -cmd_err_shell(CMD_ARGS) -{ - (void)priv; - - if (av == NULL) - return; - AN(av[1]); - AN(av[2]); - AZ(av[3]); - vtc_log(vl, 1, - "NOTICE: err_shell is deprecated, use 'shell -err -expect'"); - cmd_shell_engine(vl, -1, av[2], av[1], NULL); -} - /* SECTION: setenv setenv * * Set or change an environment variable:: From dridi.boukelmoune at gmail.com Mon Sep 20 06:53:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 Sep 2021 06:53:05 +0000 (UTC) Subject: [master] fa1f5632e param: Remove vsm_space Message-ID: <20210920065305.EB250116B6D@lists.varnish-cache.org> commit fa1f5632e41e3d4cfa69a3fa532a413a3da1f685 Author: Dridi Boukelmoune Date: Mon Sep 20 08:32:17 2021 +0200 param: Remove vsm_space It has been deprecated long enough and could have gone away with the 7.0 release. Refs 7a85739b1bd5 diff --git a/include/tbl/params.h b/include/tbl/params.h index 8f3eaf8d4..2cea9074a 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1065,18 +1065,6 @@ PARAM_SIMPLE( /* flags */ MUST_RESTART ) -PARAM_SIMPLE( - /* name */ vsm_space, - /* type */ bytes, - /* min */ "1M", - /* max */ "1G", - /* def */ "1M", - /* units */ "bytes", - /* descr */ - "DEPRECATED: This parameter is ignored.\n" - "There is no global limit on amount of shared memory now." -) - PARAM_SIMPLE( /* name */ workspace_backend, /* type */ bytes_u, From dridi.boukelmoune at gmail.com Mon Sep 20 07:21:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 Sep 2021 07:21:07 +0000 (UTC) Subject: [master] 8c8f686e8 doc: Remove stale vsm_space reference Message-ID: <20210920072107.2CA8D117A72@lists.varnish-cache.org> commit 8c8f686e887e2349d566cabaff1a101793d89ef1 Author: Dridi Boukelmoune Date: Mon Sep 20 09:19:28 2021 +0200 doc: Remove stale vsm_space reference diff --git a/doc/sphinx/whats-new/upgrading-5.2.rst b/doc/sphinx/whats-new/upgrading-5.2.rst index 570a92c52..a3c40d80f 100644 --- a/doc/sphinx/whats-new/upgrading-5.2.rst +++ b/doc/sphinx/whats-new/upgrading-5.2.rst @@ -18,13 +18,12 @@ counters, but these should all be transparent at the user-level. varnishd parameters =================== -The :ref:`ref_param_vsm_space` and ``cli_buffer`` -parameters are now deprecated and ignored. They will be removed -in a future major release. +The ``vsm_space`` and ``cli_buffer`` parameters are now deprecated and +ignored. They will be removed in a future major release. The updated shared memory implementation manages space automatically, so -it no longer needs :ref:`ref_param_vsm_space`. Memory for the CLI -command buffer is now dynamically allocated. +it no longer needs ``vsm_space``. Memory for the CLI command buffer is now +dynamically allocated. We have updated the documentation for :ref:`ref_param_send_timeout`, :ref:`ref_param_idle_send_timeout`, :ref:`ref_param_timeout_idle` and From phk at FreeBSD.org Mon Sep 20 08:08:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 20 Sep 2021 08:08:07 +0000 (UTC) Subject: [master] f33f798ed Eliminate annoying stderr diagnostic Message-ID: <20210920080807.5096D118F9E@lists.varnish-cache.org> commit f33f798ede8a6fa0524f2a69d81c64848174174a Author: Poul-Henning Kamp Date: Mon Sep 20 08:07:33 2021 +0000 Eliminate annoying stderr diagnostic diff --git a/tools/vtest.sh b/tools/vtest.sh index ef99fadde..bf46d894e 100755 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -220,10 +220,14 @@ while [ $MAXRUNS -eq 0 ] || [ $i -lt $MAXRUNS ] do i=$((i + 1)) - (cd "${SRCDIR}" && git reset --hard > /dev/null 2>&1 || true) - (cd "${SRCDIR}" && git clean -df > /dev/null 2>&1 || true) - (cd "${SRCDIR}" && git pull > /dev/null 2>&1 || true) - (cd "${SRCDIR}" && chmod -R +w varnish-trunk && rm -rf varnish-trunk > /dev/null 2>&1 || true) + ( + cd "${SRCDIR}" + chmod -R +w varnish-trunk > /dev/null 2>&1 || true + rm -rf varnish-trunk > /dev/null 2>&1 || true + git reset --hard > /dev/null 2>&1 || true + git clean -df > /dev/null 2>&1 || true + git pull > /dev/null 2>&1 + ) rev=`cd "${SRCDIR}" && git show -s --pretty=format:%H` if [ "x${rev}" != "x${orev}" ] ; then waitcur=${WAITMIN} From dridi.boukelmoune at gmail.com Mon Sep 20 08:43:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 Sep 2021 08:43:06 +0000 (UTC) Subject: [master] eae509ef7 circleci: Help troubleshoot configure failures Message-ID: <20210920084306.D4521119F6C@lists.varnish-cache.org> commit eae509ef78a56472ad04056abf080b673ecc348e Author: Dridi Boukelmoune Date: Mon Sep 20 10:41:37 2021 +0200 circleci: Help troubleshoot configure failures diff --git a/.circleci/config.yml b/.circleci/config.yml index 1a32fef41..1b4a24342 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,7 +81,7 @@ jobs: else sed -i -e "s/AC_INIT(\[\(.*\)\], \[\(.*\)\], \[\(.*\)\])/AC_INIT([\1], [\2-$(date +%Y%m%d)], [\3])/" ./configure.ac fi - ./autogen.des --quiet + ./autogen.des make dist -j 16 - persist_to_workspace: root: . @@ -318,7 +318,6 @@ jobs: autoreconf -i -v sudo -u varnish \ ./configure \ - --quiet \ << pipeline.parameters.configure_args >> \ << parameters.extra_conf >> sudo -u varnish \ From dridi.boukelmoune at gmail.com Mon Sep 20 09:15:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 Sep 2021 09:15:07 +0000 (UTC) Subject: [master] 3aecdc8a8 build: Manage libvcc with libtool Message-ID: <20210920091507.3987A11AE76@lists.varnish-cache.org> commit 3aecdc8a8076c409ab47a906f9ded5ed9c279128 Author: Dridi Boukelmoune Date: Mon Sep 20 10:50:39 2021 +0200 build: Manage libvcc with libtool It's still a static library, but the build is slightly more uniform. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index e70b43de9..4524dd229 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -175,7 +175,7 @@ varnishd_CFLAGS = \ varnishd_LDFLAGS = -export-dynamic varnishd_LDADD = \ - $(top_builddir)/lib/libvcc/libvcc.a \ + $(top_builddir)/lib/libvcc/libvcc.la \ $(top_builddir)/lib/libvarnish/libvarnish.la \ $(top_builddir)/lib/libvgz/libvgz.a \ @JEMALLOC_LDADD@ \ diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am index b6b8bca3e..dc6037f2b 100644 --- a/lib/libvcc/Makefile.am +++ b/lib/libvcc/Makefile.am @@ -6,9 +6,9 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/include \ -I$(top_builddir)/include -noinst_LIBRARIES = libvcc.a +noinst_LTLIBRARIES = libvcc.la -libvcc_a_SOURCES = \ +libvcc_la_SOURCES = \ vcc_compile.h \ vcc_namespace.h \ vcc_token_defs.h \ From dridi.boukelmoune at gmail.com Mon Sep 20 09:15:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 Sep 2021 09:15:07 +0000 (UTC) Subject: [master] 7e265ab1d build: Manage libvgz with libtool Message-ID: <20210920091507.61E4711AE79@lists.varnish-cache.org> commit 7e265ab1d5aca44f2e7a03c662c59129d94977e8 Author: Dridi Boukelmoune Date: Mon Sep 20 10:56:07 2021 +0200 build: Manage libvgz with libtool It's still a static library, but the build is slightly more uniform. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 4524dd229..7ca48c873 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -177,7 +177,7 @@ varnishd_LDFLAGS = -export-dynamic varnishd_LDADD = \ $(top_builddir)/lib/libvcc/libvcc.la \ $(top_builddir)/lib/libvarnish/libvarnish.la \ - $(top_builddir)/lib/libvgz/libvgz.a \ + $(top_builddir)/lib/libvgz/libvgz.la \ @JEMALLOC_LDADD@ \ ${DL_LIBS} ${PTHREAD_LIBS} ${NET_LIBS} ${RT_LIBS} ${LIBM} @@ -208,7 +208,7 @@ esi_parse_fuzzer_CFLAGS = \ -DNOT_IN_A_VMOD esi_parse_fuzzer_LDADD = \ $(top_builddir)/lib/libvarnish/libvarnish.la \ - $(top_builddir)/lib/libvgz/libvgz.a + $(top_builddir)/lib/libvgz/libvgz.la if ENABLE_OSS_FUZZ esi_parse_fuzzer_LDFLAGS = $(LIB_FUZZING_ENGINE) diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index 3a79b6fff..4937381d0 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -54,7 +54,7 @@ varnishtest_SOURCES = \ varnishtest_LDADD = \ $(top_builddir)/lib/libvarnishapi/libvarnishapi.la \ $(top_builddir)/lib/libvarnish/libvarnish.la \ - $(top_builddir)/lib/libvgz/libvgz.a \ + $(top_builddir)/lib/libvgz/libvgz.la \ ${PTHREAD_LIBS} ${NET_LIBS} ${LIBM} varnishtest_CFLAGS = \ diff --git a/lib/libvgz/Makefile.am b/lib/libvgz/Makefile.am index 5cee560e7..52fd2f0fb 100644 --- a/lib/libvgz/Makefile.am +++ b/lib/libvgz/Makefile.am @@ -2,12 +2,12 @@ AM_LDFLAGS = $(AM_LT_LDFLAGS) -noinst_LIBRARIES = libvgz.a +noinst_LTLIBRARIES = libvgz.la -libvgz_a_CFLAGS = -D_LARGEFILE64_SOURCE=1 -DZLIB_CONST \ +libvgz_la_CFLAGS = -D_LARGEFILE64_SOURCE=1 -DZLIB_CONST \ $(libvgz_extra_cflags) -libvgz_a_SOURCES = \ +libvgz_la_SOURCES = \ adler32.c \ crc32.c \ crc32.h \ From dridi.boukelmoune at gmail.com Mon Sep 20 09:15:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 Sep 2021 09:15:07 +0000 (UTC) Subject: [master] c5ed00b63 circleci: Double quote CFLAGS in the bionic job Message-ID: <20210920091507.987AA11AE7C@lists.varnish-cache.org> commit c5ed00b63c9ceb0af1d244f2e90878bfd73afd12 Author: Dridi Boukelmoune Date: Mon Sep 20 11:13:14 2021 +0200 circleci: Double quote CFLAGS in the bionic job Otherwise they interfere with the surrounding sh -c '...' command. diff --git a/.circleci/config.yml b/.circleci/config.yml index 1b4a24342..2cb0c96de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -365,7 +365,7 @@ workflows: name: distcheck_ubuntu_bionic dist: ubuntu release: bionic - extra_conf: CFLAGS='-g -O2 -m32' + extra_conf: CFLAGS="-g -O2 -m32" - distcheck: name: distcheck_alpine dist: alpine From dridi.boukelmoune at gmail.com Mon Sep 20 11:28:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 Sep 2021 11:28:05 +0000 (UTC) Subject: [master] f8974400e circleci: Escape spaces instead of quoting them Message-ID: <20210920112805.46A5B7169@lists.varnish-cache.org> commit f8974400e43e973811f54643d35cc667a5b94efe Author: Dridi Boukelmoune Date: Mon Sep 20 13:23:33 2021 +0200 circleci: Escape spaces instead of quoting them It turns out, in addition to beeing enclosed inside sh -c '...' the CFLAGS may also be nested inside "double" quotes. diff --git a/.circleci/config.yml b/.circleci/config.yml index 2cb0c96de..01534fc90 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -365,7 +365,7 @@ workflows: name: distcheck_ubuntu_bionic dist: ubuntu release: bionic - extra_conf: CFLAGS="-g -O2 -m32" + extra_conf: CFLAGS=-g\ -O2\ -m32 - distcheck: name: distcheck_alpine dist: alpine From dridi.boukelmoune at gmail.com Mon Sep 20 12:40:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 Sep 2021 12:40:07 +0000 (UTC) Subject: [master] 5dc880134 miniobj: Add a FINI_OBJ() macro Message-ID: <20210920124007.ED39661750@lists.varnish-cache.org> commit 5dc8801348904cbe7146fa92113b3c3fb7ac7097 Author: Dridi Boukelmoune Date: Mon Sep 6 07:42:17 2021 +0200 miniobj: Add a FINI_OBJ() macro It operates on a pointer like FREE_OBJ() and clears it. diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index f3a9619c0..f0013de1b 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -425,8 +425,7 @@ ses_handle(struct waited *wp, enum wait_event ev, vtim_real now) CAST_OBJ_NOTNULL(xp, (const void*)wp->priv2, TRANSPORT_MAGIC); AN(wp->priv2); assert(WS_Reservation(sp->ws) == wp); - wp->magic = 0; - wp = NULL; + FINI_OBJ(wp); /* The WS was reserved in SES_Wait() */ WS_Release(sp->ws, 0); diff --git a/bin/varnishd/hpack/vhp_table.c b/bin/varnishd/hpack/vhp_table.c index 4415f47b8..33798f39b 100644 --- a/bin/varnishd/hpack/vhp_table.c +++ b/bin/varnishd/hpack/vhp_table.c @@ -122,7 +122,7 @@ vht_trim(struct vht_table *tbl, ssize_t max) assert(e->offset == u); u += ENTRYLEN(e); v++; - e->magic = 0; + FINI_OBJ(e); } else { /* Fixup offset */ assert(e->offset >= u); @@ -257,7 +257,7 @@ VHT_NewEntry_Indexed(struct vht_table *tbl, unsigned idx) assert(e->offset == 0); lname = e->namelen; lentry = ENTRYLEN(e); - e->magic = 0; + FINI_OBJ(e); memmove(TBLENTRY(tbl, 1), TBLENTRY(tbl, 0), (tbl->n - 1) * sizeof *e); tbl->n--; @@ -462,7 +462,7 @@ VHT_Init(struct vht_table *tbl, size_t protomax) INIT_OBJ(tbl, VHT_TABLE_MAGIC); r = VHT_SetProtoMax(tbl, protomax); if (r) { - tbl->magic = 0; + FINI_OBJ(tbl); return (r); } tbl->maxsize = tbl->protomax; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 6bc062ee1..6e67b55c5 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -216,7 +216,7 @@ h2h_decode_fini(const struct h2_sess *h2) ret = H2CE_COMPRESSION_ERROR; } else ret = d->error; - d->magic = 0; + FINI_OBJ(d); return (ret); } diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index bcf00c75b..c5c335848 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -252,18 +252,21 @@ sml_slim(struct worker *wrk, struct objcore *oc) static void v_matchproto_(objfree_f) sml_objfree(struct worker *wrk, struct objcore *oc) { + struct storage *st; struct object *o; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); sml_slim(wrk, oc); CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC); - o->magic = 0; + st = o->objstore; + CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); + FINI_OBJ(o); if (oc->boc == NULL && oc->stobj->stevedore->lru != NULL) LRU_Remove(oc); - sml_stv_free(oc->stobj->stevedore, o->objstore); + sml_stv_free(oc->stobj->stevedore, st); memset(oc->stobj, 0, sizeof oc->stobj); diff --git a/include/miniobj.h b/include/miniobj.h index 8da6ef42e..8babaa251 100644 --- a/include/miniobj.h +++ b/include/miniobj.h @@ -17,6 +17,12 @@ (to)->magic = (type_magic); \ } while (0) +#define FINI_OBJ(to) \ + do { \ + ZERO_OBJ(&(to)->magic, sizeof (to)->magic); \ + to = NULL; \ + } while (0) + #define ALLOC_OBJ(to, type_magic) \ do { \ (to) = calloc(1, sizeof *(to)); \ diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c index ab3544899..aacdbbb23 100644 --- a/lib/libvarnish/vev.c +++ b/lib/libvarnish/vev.c @@ -337,8 +337,8 @@ VEV_Stop(struct vev_root *evb, struct vev *e) es->happened = 0; } - e->magic = 0; e->__vevb = NULL; + FINI_OBJ(e); } /*--------------------------------------------------------------------*/ From dridi.boukelmoune at gmail.com Mon Sep 20 14:03:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 Sep 2021 14:03:06 +0000 (UTC) Subject: [master] 45eef9620 fuzz: Set up a workspace emulator in esi_parse_fuzzer Message-ID: <20210920140306.9970C64529@lists.varnish-cache.org> commit 45eef962011f5a2f2fa944f537a7d628fd8e26cd Author: Dridi Boukelmoune Date: Mon Sep 6 09:23:42 2021 +0200 fuzz: Set up a workspace emulator in esi_parse_fuzzer While at it, try to better formalize all the setup steps. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 7ca48c873..27d931ab3 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -202,10 +202,12 @@ vhp_decode_test_LDADD = $(top_builddir)/lib/libvarnish/libvarnish.la noinst_PROGRAMS += esi_parse_fuzzer esi_parse_fuzzer_SOURCES = \ + cache/cache_ws_emu.c \ + cache/cache_ws_common.c \ cache/cache_esi_parse.c \ fuzzers/esi_parse_fuzzer.c esi_parse_fuzzer_CFLAGS = \ - -DNOT_IN_A_VMOD + -DNOT_IN_A_VMOD -DENABLE_WORKSPACE_EMULATOR esi_parse_fuzzer_LDADD = \ $(top_builddir)/lib/libvarnish/libvarnish.la \ $(top_builddir)/lib/libvgz/libvgz.la diff --git a/bin/varnishd/fuzzers/esi_parse_fuzzer.c b/bin/varnishd/fuzzers/esi_parse_fuzzer.c index 246e9180a..fb1e5ecb5 100644 --- a/bin/varnishd/fuzzers/esi_parse_fuzzer.c +++ b/bin/varnishd/fuzzers/esi_parse_fuzzer.c @@ -36,22 +36,39 @@ #include #include -#include "cache/cache.h" +#include "cache/cache_varnishd.h" #include "cache/cache_vgz.h" /* enum vgz_flag */ #include "cache/cache_esi.h" #include "cache/cache_filter.h" /* struct vfp_ctx */ -#include "common/common_param.h" /* struct params */ -#include "VSC_main.h" #include "vfil.h" -#include "vsb.h" int LLVMFuzzerTestOneInput(const uint8_t *, size_t); -extern struct VSC_main *VSC_C_main; struct VSC_main *VSC_C_main; -extern struct params *cache_param; -struct params *cache_param; +volatile struct params *cache_param; + +int +PAN__DumpStruct(struct vsb *vsb, int block, int track, const void *ptr, + const char *smagic, unsigned magic, const char *fmt, ...) +{ + (void)vsb; + (void)block; + (void)track; + (void)ptr; + (void)smagic; + (void)magic; + (void)fmt; + return (0); +} + +void +VSL(enum VSL_tag_e tag, uint32_t vxid, const char *fmt, ...) +{ + (void)tag; + (void)vxid; + (void)fmt; +} void VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) @@ -72,38 +89,20 @@ VSLb_ts(struct vsl_log *l, const char *event, vtim_real first, vtim_real *pprev, (void)now; } -void -WS_Assert(const struct ws *ws) -{ - (void)ws; -} - -void * -WS_Alloc(struct ws *ws, unsigned bytes) -{ - (void)ws; - return (calloc(1, bytes)); -} - -unsigned -WS_ReserveAll(struct ws *ws) -{ - (void)ws; - WRONG("Should not be called"); -} - int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { struct VSC_main __VSC_C_main; struct params __cache_param; - struct http req = { .magic = HTTP_MAGIC }; - struct http resp = { .magic = HTTP_MAGIC }; - struct vfp_ctx vc = { .magic = VFP_CTX_MAGIC }; + struct http req[1]; + struct http resp[1]; + struct vfp_ctx vc[1]; + struct worker wrk[1]; + struct ws ws[1]; struct vep_state *vep; struct vsb *vsb; - struct worker wrk; txt hd[HTTP_HDR_URL + 1]; + char ws_buf[1024]; if (size < 1) return (0); @@ -125,21 +124,34 @@ LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) BSET(__cache_param.feature_bits, FEATURE_ESI_REMOVE_BOM); #undef BSET + /* Setup ws */ + WS_Init(ws, "req", ws_buf, sizeof ws_buf); + /* Setup req */ - req.hd = hd; - req.hd[HTTP_HDR_URL].b = "/"; + INIT_OBJ(req, HTTP_MAGIC); + req->hd = hd; + req->hd[HTTP_HDR_URL].b = "/"; + req->ws = ws; + + /* Setup resp */ + INIT_OBJ(resp, HTTP_MAGIC); + resp->ws = ws; + + /* Setup wrk */ + INIT_OBJ(wrk, WORKER_MAGIC); /* Setup vc */ - vc.wrk = &wrk; - vc.resp = &resp; + INIT_OBJ(vc, VFP_CTX_MAGIC); + vc->wrk = wrk; + vc->resp = resp; - vep = VEP_Init(&vc, &req, NULL, NULL); + vep = VEP_Init(vc, req, NULL, NULL); AN(vep); VEP_Parse(vep, (const char *)data, size); vsb = VEP_Finish(vep); if (vsb != NULL) VSB_destroy(&vsb); - free(vep); + WS_Rollback(ws, 0); return (0); } @@ -155,6 +167,7 @@ main(int argc, char **argv) for (i = 1; i < argc; i++) { len = 0; buf = VFIL_readfile(NULL, argv[i], &len); + AN(buf); LLVMFuzzerTestOneInput((uint8_t *)buf, len); free(buf); } From phk at FreeBSD.org Tue Sep 21 08:14:06 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 21 Sep 2021 08:14:06 +0000 (UTC) Subject: [master] fe96d9828 Release the panicstr_mtx before calling abort(2), but still protect against reentrancy. Message-ID: <20210921081406.3AFF611BDB0@lists.varnish-cache.org> commit fe96d9828b263c903adfd7d4a90e2d7a9abc20cf Author: Poul-Henning Kamp Date: Tue Sep 21 08:12:19 2021 +0000 Release the panicstr_mtx before calling abort(2), but still protect against reentrancy. Fixes: #3688 diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 190ac90b7..3d01e6090 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -718,9 +718,11 @@ pan_ic(const char *func, const char *file, int line, const char *cond, struct sigaction sa; int err = errno; - AZ(pthread_mutex_lock(&panicstr_mtx)); /* Won't be released, - we're going to die - anyway */ + AZ(pthread_mutex_lock(&panicstr_mtx)); + + /* If we already panic'ed, do nothing */ + while (heritage.panic_str[0]) + sleep(1); /* * should we trigger a SIGSEGV while handling a panic, our sigsegv @@ -809,6 +811,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond, VSB_putc(pan_vsb, '\0'); /* NUL termination */ v_gcov_flush(); + AZ(pthread_mutex_unlock(&panicstr_mtx)); abort(); } From dridi.boukelmoune at gmail.com Wed Sep 22 07:00:12 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 Sep 2021 07:00:12 +0000 (UTC) Subject: [master] 50dbec196 circleci: Revert recent changes Message-ID: <20210922070012.3CA4197D9@lists.varnish-cache.org> commit 50dbec1965f53c7fae7e537919def094598da3a8 Author: Dridi Boukelmoune Date: Wed Sep 22 08:56:18 2021 +0200 circleci: Revert recent changes Going back to pretending 32bit bionic builds are fine until #3702 is sorted out. We have 32bit coverage in our vtest CI anyway. Reverts f8974400e43e Reverts c5ed00b63c9c diff --git a/.circleci/config.yml b/.circleci/config.yml index 01534fc90..1b4a24342 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -365,7 +365,7 @@ workflows: name: distcheck_ubuntu_bionic dist: ubuntu release: bionic - extra_conf: CFLAGS=-g\ -O2\ -m32 + extra_conf: CFLAGS='-g -O2 -m32' - distcheck: name: distcheck_alpine dist: alpine From dridi.boukelmoune at gmail.com Wed Sep 22 07:42:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 Sep 2021 07:42:05 +0000 (UTC) Subject: [master] 1e997e24c [vnum] more lenient test Message-ID: <20210922074205.DB42863E13@lists.varnish-cache.org> commit 1e997e24c1022862f9bdf1b2dba8d3d5b28921c2 Author: Guillaume Quintard Date: Tue Sep 21 22:11:19 2021 -0700 [vnum] more lenient test fixes #3697 diff --git a/include/vnum.h b/include/vnum.h index fe9e78bd1..5189b541a 100644 --- a/include/vnum.h +++ b/include/vnum.h @@ -31,6 +31,7 @@ */ /* from libvarnish/vnum.c */ +#define VNUM_EPSILON (1e-9d) double VNUM(const char *p); vtim_dur VNUM_duration_unit(vtim_dur r, const char *b, const char *e); vtim_dur VNUM_duration(const char *p); diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index fa3a8cc97..88c688f08 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -693,9 +693,10 @@ main(int argc, char *argv[]) ++ec; } d1 = VNUM_duration(" 365.24219d "); - if (d1 != 31556908.8) { + d2 = 31556908.8; + if (fabs(d1 - d2) > VNUM_EPSILON) { printf("%s: VNUM_Duration() wrong, %.3f delta = %e\n", - *argv, d1, d1 - 31556908.8); + *argv, d1, d1 - d2); ++ec; } /* TODO: test invalid strings */ From dridi.boukelmoune at gmail.com Wed Sep 22 08:04:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 Sep 2021 08:04:06 +0000 (UTC) Subject: [master] a32b7c000 vnum: 'd' is not a standard suffix Message-ID: <20210922080406.0CF6764AA2@lists.varnish-cache.org> commit a32b7c000a65e263d964647c561272eedf115c75 Author: Dridi Boukelmoune Date: Wed Sep 22 10:02:38 2021 +0200 vnum: 'd' is not a standard suffix My bad. diff --git a/include/vnum.h b/include/vnum.h index 5189b541a..176a235e9 100644 --- a/include/vnum.h +++ b/include/vnum.h @@ -31,7 +31,7 @@ */ /* from libvarnish/vnum.c */ -#define VNUM_EPSILON (1e-9d) +#define VNUM_EPSILON (1e-9) double VNUM(const char *p); vtim_dur VNUM_duration_unit(vtim_dur r, const char *b, const char *e); vtim_dur VNUM_duration(const char *p); From martin at varnish-software.com Wed Sep 22 08:40:06 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Wed, 22 Sep 2021 08:40:06 +0000 (UTC) Subject: [master] 20e007a5b Cast to (u)intmax_t for %ju and %jd Message-ID: <20210922084007.0154865F29@lists.varnish-cache.org> commit 20e007a5b17c1f68f70ab42080de384f9e192900 Author: Martin Blix Grydeland Date: Wed Sep 22 10:35:27 2021 +0200 Cast to (u)intmax_t for %ju and %jd This to silence errors on OSX where apparently int64_t isn't type equivalent to intmax_t, causing printf-errors when using %jd. Fixes: #3699 diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c index b9720070a..06ea40c15 100644 --- a/bin/varnishd/http2/cache_http2_panic.c +++ b/bin/varnishd/http2/cache_http2_panic.c @@ -31,6 +31,8 @@ #include "config.h" +#include + #include "cache/cache_varnishd.h" #include "cache/cache_transport.h" @@ -108,13 +110,14 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp) VSB_printf(vsb, "t_send = %f, t_winupd = %f,\n", r2->t_send, r2->t_winupd); VSB_printf(vsb, "t_window = %jd, r_window = %jd,\n", - r2->t_window, r2->r_window); + (intmax_t)r2->t_window, (intmax_t)r2->r_window); if (!PAN_dump_struct(vsb, r2->rxbuf, H2_RXBUF_MAGIC, "rxbuf")) { VSB_printf(vsb, "stvbuf = %p,\n", r2->rxbuf->stvbuf); VSB_printf(vsb, "{size, tail, head} = {%u, %ju, %ju},\n", - r2->rxbuf->size, r2->rxbuf->tail, r2->rxbuf->head); + r2->rxbuf->size, (uintmax_t)r2->rxbuf->tail, + (uintmax_t)r2->rxbuf->head); VSB_indent(vsb, -2); VSB_cat(vsb, "},\n"); } From dridi.boukelmoune at gmail.com Wed Sep 22 16:37:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 Sep 2021 16:37:05 +0000 (UTC) Subject: [master] 02ada45c5 ws_emu: Remove unused variable Message-ID: <20210922163705.871B910DD74@lists.varnish-cache.org> commit 02ada45c5d5119c0f86a6db24fffd696e0feb626 Author: Dridi Boukelmoune Date: Wed Sep 22 18:35:15 2021 +0200 ws_emu: Remove unused variable Spotted by oss-fuzz. diff --git a/bin/varnishd/cache/cache_ws_emu.c b/bin/varnishd/cache/cache_ws_emu.c index 95cb80800..43ee87f85 100644 --- a/bin/varnishd/cache/cache_ws_emu.c +++ b/bin/varnishd/cache/cache_ws_emu.c @@ -493,7 +493,7 @@ WS_Dump(const struct ws *ws, char where, size_t off, void *buf, size_t len) { struct ws_emu *we; struct ws_alloc *wa; - unsigned l, r; + unsigned l; char *b; WS_Assert(ws); @@ -526,13 +526,11 @@ WS_Dump(const struct ws *ws, char where, size_t off, void *buf, size_t len) } b = buf; - r = 0; if (where == 'f' && ws->r != NULL) { if (l > len) l = len; memcpy(b, wa->ptr, l); b += l; - r += l; len -= l; } @@ -547,7 +545,6 @@ WS_Dump(const struct ws *ws, char where, size_t off, void *buf, size_t len) l = len; memcpy(b, wa->ptr, l); b += l; - r += l; len -= l; } } From dridi.boukelmoune at gmail.com Wed Sep 22 16:46:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 Sep 2021 16:46:05 +0000 (UTC) Subject: [master] 0bbd1d06a build: Use -Wunused-but-set-variable if available Message-ID: <20210922164605.97D6010E27E@lists.varnish-cache.org> commit 0bbd1d06a05e83f311ab4946f8284a509e300051 Author: Dridi Boukelmoune Date: Wed Sep 22 18:43:57 2021 +0200 build: Use -Wunused-but-set-variable if available Refs 02ada45c5d51 diff --git a/wflags.py b/wflags.py index 9e9e4e902..1b8c48fc0 100644 --- a/wflags.py +++ b/wflags.py @@ -64,6 +64,7 @@ DESIRABLE_WFLAGS = [ "-Wunused-parameter", "-Wunused-parameters", "-Wunused-result", + "-Wunused-but-set-variable", "-Wwrite-strings", ] From dridi.boukelmoune at gmail.com Thu Sep 23 10:55:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 23 Sep 2021 10:55:07 +0000 (UTC) Subject: [master] ef91cff55 vtc: Another c69 and c93 stabilization attempt Message-ID: <20210923105507.B42B51063E2@lists.varnish-cache.org> commit ef91cff55128f66cdc0611a1a63a07968f586150 Author: Dridi Boukelmoune Date: Thu Sep 23 12:39:48 2021 +0200 vtc: Another c69 and c93 stabilization attempt Even thought the first rxresp implies that the backend fetch must have completed, it is still possible to observe a streaming hit. Since resp.is_streaming merely looks at the presence of a boc, and not whether its state is lower than FINISHED, waiting until we see the fetch end in the logs and forcing a new client session should reduce the race window in those test cases. We could also try to settle on the value of resp.is_streaming when we hit an object in the cache or resume from a backend fetch and change the semantic to whether resp was streaming when VCL entered the subroutine. By that I mean, with a proper check of the boc state. diff --git a/bin/varnishtest/tests/c00069.vtc b/bin/varnishtest/tests/c00069.vtc index 9aefaf4ea..6963d4cdc 100644 --- a/bin/varnishtest/tests/c00069.vtc +++ b/bin/varnishtest/tests/c00069.vtc @@ -33,18 +33,24 @@ varnish v1 -vcl+backend { } } -start +logexpect l1 -v v1 -q "Begin ~ bereq" -i End { + expect 0 1002 End +} -start + client c1 { txreq rxresp - expect resp.http.streaming == "true" + expect resp.http.streaming == true +} -run - delay 0.1 +logexpect l1 -wait +client c2 { txreq rxresp - expect resp.http.streaming == "false" + expect resp.http.streaming == false txreq -url /synth rxresp - expect resp.http.streaming == "false" + expect resp.http.streaming == false } -run diff --git a/bin/varnishtest/tests/c00093.vtc b/bin/varnishtest/tests/c00093.vtc index 0fa9aa2bd..23a36ad36 100644 --- a/bin/varnishtest/tests/c00093.vtc +++ b/bin/varnishtest/tests/c00093.vtc @@ -27,14 +27,20 @@ varnish v1 -vcl+backend { } } -start +logexpect l1 -v v1 -q "Begin ~ bereq" -i End { + expect 0 1002 End +} -start + client c1 { txreq rxresp - expect resp.http.streaming == "true" + expect resp.http.streaming == true +} -run - delay 0.1 +logexpect l1 -wait +client c2 { txreq rxresp - expect resp.http.streaming == "false" + expect resp.http.streaming == false } -run From dridi.boukelmoune at gmail.com Fri Sep 24 08:38:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 24 Sep 2021 08:38:06 +0000 (UTC) Subject: [master] ac9258739 vtc: Polish using the new asan feature check Message-ID: <20210924083806.CCA801043F2@lists.varnish-cache.org> commit ac9258739219da2bd494c66fe19c1ea30a573257 Author: Dridi Boukelmoune Date: Fri Sep 24 10:36:29 2021 +0200 vtc: Polish using the new asan feature check diff --git a/bin/varnishtest/tests/c00057.vtc b/bin/varnishtest/tests/c00057.vtc index a4d1ae1ce..db3919572 100644 --- a/bin/varnishtest/tests/c00057.vtc +++ b/bin/varnishtest/tests/c00057.vtc @@ -2,7 +2,7 @@ varnishtest "test sigsegv handler" # Under ASAN, the stack layout is different and STACK OVERFLOW is # never printed. -feature cmd "test -z $(env | grep ASAN_OPTIONS)" +feature !asan server s1 { rxreq From nils.goroll at uplex.de Sat Sep 25 10:39:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 25 Sep 2021 10:39:07 +0000 (UTC) Subject: [master] b6fef8a4d Move struct storage definition to storage_simple.h Message-ID: <20210925103907.A73D2108A8A@lists.varnish-cache.org> commit b6fef8a4d2323b753e089a7e4beba1d5e283bcff Author: Nils Goroll Date: Sat Sep 25 12:24:50 2021 +0200 Move struct storage definition to storage_simple.h Use of SML is supported by the storage API though the sml_* function pointers. Yet, like struct object, struct storage is part of the simple storage (sml) implementation and does not belong to the storage API. Storage not using SML (that is, code which does not include storage_simple.h) should not be concerned with struct storage. diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 193b273e3..060c41a24 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -47,21 +47,6 @@ enum baninfo { BI_DROP }; -/* Storage -----------------------------------------------------------*/ - -struct storage { - unsigned magic; -#define STORAGE_MAGIC 0x1a4e51c0 - - - VTAILQ_ENTRY(storage) list; - void *priv; - - unsigned char *ptr; - unsigned len; - unsigned space; -}; - /* Prototypes --------------------------------------------------------*/ typedef void storage_init_f(struct stevedore *, int ac, char * const *av); @@ -80,6 +65,7 @@ typedef void *storage_allocbuf_f(struct worker *, const struct stevedore *, typedef void storage_freebuf_f(struct worker *, const struct stevedore *, uintptr_t priv); +struct storage; typedef struct object *sml_getobj_f(struct worker *, struct objcore *); typedef struct storage *sml_alloc_f(const struct stevedore *, size_t size); typedef void sml_free_f(struct storage *); diff --git a/bin/varnishd/storage/storage_simple.h b/bin/varnishd/storage/storage_simple.h index 3f288aef7..881e0b857 100644 --- a/bin/varnishd/storage/storage_simple.h +++ b/bin/varnishd/storage/storage_simple.h @@ -33,6 +33,21 @@ * */ +/* Storage -----------------------------------------------------------*/ + +struct storage { + unsigned magic; +#define STORAGE_MAGIC 0x1a4e51c0 + + + VTAILQ_ENTRY(storage) list; + void *priv; + + unsigned char *ptr; + unsigned len; + unsigned space; +}; + /* Object ------------------------------------------------------------*/ VTAILQ_HEAD(storagehead, storage); From dridi.boukelmoune at gmail.com Mon Sep 27 14:00:14 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 27 Sep 2021 14:00:14 +0000 (UTC) Subject: [master] 0fb88f9fc build: Simply require that SO_???TIMEO are present Message-ID: <20210927140014.CE3D5113B83@lists.varnish-cache.org> commit 0fb88f9fc246cff65173ec4d1ecc07ac9b26b711 Author: Dridi Boukelmoune Date: Wed Sep 22 19:46:18 2021 +0200 build: Simply require that SO_???TIMEO are present Closes #3692 diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 33dc99130..d868e0cac 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -94,14 +94,8 @@ static struct tcp_opt { TCPO(SOL_SOCKET, SO_LINGER, struct linger, 0) TCPO(SOL_SOCKET, SO_KEEPALIVE, int, 0) TCPO(IPPROTO_TCP, TCP_NODELAY, int, 1) - -#ifdef SO_SNDTIMEO_WORKS TCPO(SOL_SOCKET, SO_SNDTIMEO, struct timeval, 0) -#endif - -#ifdef SO_RCVTIMEO_WORKS TCPO(SOL_SOCKET, SO_RCVTIMEO, struct timeval, 0) -#endif #ifdef HAVE_TCP_KEEP TCPO(IPPROTO_TCP, TCP_KEEPIDLE, int, 1) @@ -193,16 +187,12 @@ vca_tcp_opt_init(void) } \ } while (0) -#ifdef SO_SNDTIMEO_WORKS } else if (!strcmp(to->strname, "SO_SNDTIMEO")) { tv = VTIM_timeval(cache_param->idle_send_timeout); NEW_VAL(to, tv); -#endif -#ifdef SO_RCVTIMEO_WORKS } else if (!strcmp(to->strname, "SO_RCVTIMEO")) { tv = VTIM_timeval(cache_param->timeout_idle); NEW_VAL(to, tv); -#endif #ifdef HAVE_TCP_KEEP } else if (!strcmp(to->strname, "TCP_KEEPIDLE")) { x = (int)(cache_param->tcp_keepalive_time); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 2b05aac01..2294e742c 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -1000,14 +1000,9 @@ HTTP_VAR(beresp) static inline void set_idle_send_timeout(const struct sess *sp, VCL_DURATION d) { -#ifdef SO_SNDTIMEO_WORKS struct timeval tv = VTIM_timeval(d); VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof tv)); -#else - (void)sp; - (void)d; -#endif } #define SESS_VAR_DUR(x, setter) \ diff --git a/bin/varnishtest/tests/b00020.vtc b/bin/varnishtest/tests/b00020.vtc index 1f0427c11..fc7c7c2d8 100644 --- a/bin/varnishtest/tests/b00020.vtc +++ b/bin/varnishtest/tests/b00020.vtc @@ -1,7 +1,5 @@ varnishtest "Check that between_bytes_timeout behaves from parameters" -feature SO_RCVTIMEO_WORKS - server s1 { rxreq send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n" diff --git a/bin/varnishtest/tests/b00021.vtc b/bin/varnishtest/tests/b00021.vtc index 46be8cb4f..f0c4c0ce2 100644 --- a/bin/varnishtest/tests/b00021.vtc +++ b/bin/varnishtest/tests/b00021.vtc @@ -1,7 +1,5 @@ varnishtest "Check the between_bytes_timeout behaves from vcl" -feature SO_RCVTIMEO_WORKS - server s1 { rxreq send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n" diff --git a/bin/varnishtest/tests/b00022.vtc b/bin/varnishtest/tests/b00022.vtc index 7bf6d9575..fee5db3d9 100644 --- a/bin/varnishtest/tests/b00022.vtc +++ b/bin/varnishtest/tests/b00022.vtc @@ -1,7 +1,5 @@ varnishtest "Check the between_bytes_timeout behaves from backend definition" -feature SO_RCVTIMEO_WORKS - server s1 { rxreq send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n" diff --git a/bin/varnishtest/tests/b00023.vtc b/bin/varnishtest/tests/b00023.vtc index fd64cb2bd..5948d6bef 100644 --- a/bin/varnishtest/tests/b00023.vtc +++ b/bin/varnishtest/tests/b00023.vtc @@ -1,7 +1,5 @@ varnishtest "Check that the first_byte_timeout works" -feature SO_RCVTIMEO_WORKS - # From VCL server s1 { diff --git a/bin/varnishtest/tests/b00026.vtc b/bin/varnishtest/tests/b00026.vtc index 48fce004b..e12676985 100644 --- a/bin/varnishtest/tests/b00026.vtc +++ b/bin/varnishtest/tests/b00026.vtc @@ -1,7 +1,5 @@ varnishtest "Check the precedence for timeouts" -feature SO_RCVTIMEO_WORKS - server s1 { rxreq expect req.url == "from_backend" diff --git a/bin/varnishtest/tests/s00010.vtc b/bin/varnishtest/tests/s00010.vtc index fea3ea89d..f9e0bbb15 100644 --- a/bin/varnishtest/tests/s00010.vtc +++ b/bin/varnishtest/tests/s00010.vtc @@ -3,8 +3,6 @@ varnishtest "client h1 send timeouts - tcp" # XXX See https://github.com/varnishcache/varnish-cache/pull/2980#issuecomment-486214661 feature cmd {test $(uname) != "SunOS" && test $(uname) != "Darwin"} -feature SO_RCVTIMEO_WORKS - barrier b1 cond 2 -cyclic barrier b2 cond 2 -cyclic diff --git a/bin/varnishtest/tests/s00012.vtc b/bin/varnishtest/tests/s00012.vtc index 9726f050f..e635ad03d 100644 --- a/bin/varnishtest/tests/s00012.vtc +++ b/bin/varnishtest/tests/s00012.vtc @@ -2,8 +2,6 @@ varnishtest "client h1 send timeouts - uds" feature cmd {test $(uname) != "SunOS"} -feature SO_RCVTIMEO_WORKS - server s1 { rxreq txresp -bodylen 100000 diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index a70d6aa55..59bde405f 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -382,8 +382,6 @@ addr_no_randomize_works(void) * the test otherwise; or change the interpretation of the test, as * documented below. feature takes any number of arguments from this list: * - * SO_RCVTIMEO_WORKS - * The SO_RCVTIMEO socket option is working * 64bit * The environment is 64 bits * ipv4 @@ -480,12 +478,6 @@ static const unsigned with_persistent_storage = 1; static const unsigned with_persistent_storage = 0; #endif -#ifdef SO_RCVTIMEO_WORKS -static const unsigned so_rcvtimeo_works = 1; -#else -static const unsigned so_rcvtimeo_works = 0; -#endif - void v_matchproto_(cmd_f) cmd_feature(CMD_ARGS) { @@ -538,7 +530,6 @@ cmd_feature(CMD_ARGS) FEATURE("tsan", tsan); FEATURE("ubsan", ubsan); FEATURE("sanitizer", sanitizer); - FEATURE("SO_RCVTIMEO_WORKS", so_rcvtimeo_works); FEATURE("workspace_emulator", workspace_emulator); if (!strcmp(feat, "cmd")) { diff --git a/configure.ac b/configure.ac index da9bb0603..ca1a49c29 100644 --- a/configure.ac +++ b/configure.ac @@ -532,91 +532,24 @@ AC_CHECK_MEMBER([struct sockaddr.sa_len], AC_CHECK_DECL([SO_ACCEPTFILTER], AC_DEFINE(HAVE_ACCEPT_FILTERS,1,[Define to 1 if you have accept filters]), - , - [ -#include -#include - ] -) - -# Older Solaris versions define SO_{RCV,SND}TIMEO, but do not -# implement them. -# -# Varnish will build and run without these, but connections will not -# time out, which may leave Varnish vulnerable to denial-of-service -# attacks which would not be possible on other platforms. -# -# Newer Solaris releases with the Volo framework (Solaris 11, -# Opensolaris starting with onnv_106) do support SO_{RCV,SND}TIMEO -# (see PSARC 2007/587, initially committed into onnv-gate / -# OpenSolaris 8348:4137e18bfaf0 Thu Dec 11 20:04:13 2008) - -save_LIBS="${LIBS}" -LIBS="${LIBS} ${NET_LIBS}" -AC_CACHE_CHECK([whether SO_RCVTIMEO works], - [ac_cv_so_rcvtimeo_works], - [AC_RUN_IFELSE( - [AC_LANG_PROGRAM([[ -#include -#include -#include -#include - ]],[[ -int s = socket(AF_INET, SOCK_STREAM, 0); -struct timeval tv = { 1, 0 }; -if (s < 0 && errno == EPROTONOSUPPORT) - s = socket(AF_INET6, SOCK_STREAM, 0); -if (setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof tv) == 0) { - socklen_t l = sizeof tv; - if (getsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, &l) == 0) { - return (l != sizeof tv); - } -} -return 1; - ]])], - [ac_cv_so_rcvtimeo_works=yes], - [ac_cv_so_rcvtimeo_works=no]) - ]) -if test "$ac_cv_so_rcvtimeo_works" = yes; then - AC_DEFINE([SO_RCVTIMEO_WORKS], [1], [Define if SO_RCVTIMEO works]) -fi -LIBS="${save_LIBS}" + [], [ + #include + #include + ]) -save_LIBS="${LIBS}" -LIBS="${LIBS} ${NET_LIBS}" -AC_CACHE_CHECK([whether SO_SNDTIMEO works], - [ac_cv_so_sndtimeo_works], - [AC_RUN_IFELSE( - [AC_LANG_PROGRAM([[ -#include -#include -#include -#include - ]],[[ -int s = socket(AF_INET, SOCK_STREAM, 0); -struct timeval tv = { 1, 0 }; -if (s < 0 && errno == EPROTONOSUPPORT) - s = socket(AF_INET6, SOCK_STREAM, 0); -if (setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof tv) == 0) { - socklen_t l = sizeof tv; - if (getsockopt(s, SOL_SOCKET, SO_SNDTIMEO, &tv, &l) == 0) { - return (l != sizeof tv); - } -} -return 1; - ]])], - [ac_cv_so_sndtimeo_works=yes], - [ac_cv_so_sndtimeo_works=no]) - ]) -if test "$ac_cv_so_sndtimeo_works" = yes; then - AC_DEFINE([SO_SNDTIMEO_WORKS], [1], [Define if SO_SNDTIMEO works]) -fi +AC_CHECK_DECL([SO_RCVTIMEO], + [], + AC_MSG_ERROR([SO_RCVTIMEO is needed to build Varnish.]), [ + #include + #include + ]) -if test "$ac_cv_so_rcvtimeo_works" = no || - test "$ac_cv_so_sndtimeo_works" = no; then - AC_MSG_WARN([connection timeouts will not work]) -fi -LIBS="${save_LIBS}" +AC_CHECK_DECL([SO_SNDTIMEO], + [], + AC_MSG_ERROR([SO_SNDTIMEO is needed to build Varnish.]), [ + #include + #include + ]) # Check if the OS supports TCP_KEEP(CNT|IDLE|INTVL) socket options save_LIBS="${LIBS}" diff --git a/include/tbl/params.h b/include/tbl/params.h index 2cea9074a..270cec0de 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -610,11 +610,6 @@ PARAM_SIMPLE( "of that the response is allowed to take up." ) -#if defined(SO_SNDTIMEO_WORKS) -# define PLATFORM_FLAGS DELAYED_EFFECT -#else -# define PLATFORM_FLAGS NOT_IMPLEMENTED -#endif PARAM_SIMPLE( /* name */ idle_send_timeout, /* type */ timeout, @@ -628,9 +623,8 @@ PARAM_SIMPLE( "When this timeout is hit, the session is closed.\n\n" "See the man page for `setsockopt(2)` or `socket(7)` under" " ``SO_SNDTIMEO`` for more information.", - /* flags */ PLATFORM_DEPENDENT | PLATFORM_FLAGS + /* flags */ DELAYED_EFFECT ) -#undef PLATFORM_FLAGS PARAM_SIMPLE( /* name */ listen_depth, @@ -781,11 +775,6 @@ PARAM_SIMPLE( /* flags */ EXPERIMENTAL ) -#if defined(SO_SNDTIMEO_WORKS) -# define PLATFORM_FLAGS DELAYED_EFFECT -#else -# define PLATFORM_FLAGS NOT_IMPLEMENTED -#endif PARAM_SIMPLE( /* name */ send_timeout, /* type */ timeout, @@ -800,9 +789,8 @@ PARAM_SIMPLE( " timeout is extended unless the total time already taken for sending" " the response in its entirety exceeds this many seconds.\n\n" "When this timeout is hit, the session is closed", - /* flags */ PLATFORM_DEPENDENT | PLATFORM_FLAGS + /* flags */ DELAYED_EFFECT ) -#undef PLATFORM_FLAGS PARAM_SIMPLE( /* name */ shortlived, @@ -914,11 +902,6 @@ PARAM_SIMPLE( ) #undef PLATFORM_FLAGS -#if defined(SO_RCVTIMEO_WORKS) -# define PLATFORM_FLAGS 0 -#else -# define PLATFORM_FLAGS NOT_IMPLEMENTED -#endif PARAM_SIMPLE( /* name */ timeout_idle, /* type */ timeout, @@ -932,10 +915,8 @@ PARAM_SIMPLE( " request headers.\n\n" "This parameter is particularly relevant for HTTP1 keepalive " " connections which are closed unless the next request is received" - " before this timeout is reached.", - /* flags */ PLATFORM_DEPENDENT | PLATFORM_FLAGS + " before this timeout is reached." ) -#undef PLATFORM_FLAGS PARAM_SIMPLE( /* name */ timeout_linger, diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 9e43f5da0..6c1d154a2 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -349,8 +349,8 @@ VTCP_close(int *s) void VTCP_set_read_timeout(int s, vtim_dur seconds) { -#ifdef SO_RCVTIMEO_WORKS struct timeval timeout = VTIM_timeval(seconds); + /* * Solaris bug (present at least in snv_151 and older): If this fails * with EINVAL, the socket is half-closed (SS_CANTSENDMORE) and the @@ -359,10 +359,6 @@ VTCP_set_read_timeout(int s, vtim_dur seconds) */ VTCP_Assert(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout)); -#else - (void)s; - (void)seconds; -#endif } /*-------------------------------------------------------------------- From nils.goroll at uplex.de Tue Sep 28 14:48:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 28 Sep 2021 14:48:07 +0000 (UTC) Subject: [master] f6b4f8065 ban_evaluate(): Do not call VRE_match on a NULL subject Message-ID: <20210928144807.2BC2E117DF6@lists.varnish-cache.org> commit f6b4f806564e65b531b34cba8ae09e3ac65b0654 Author: Nils Goroll Date: Tue Sep 28 16:44:24 2021 +0200 ban_evaluate(): Do not call VRE_match on a NULL subject Fixes #3706 diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index bd02f3001..ab523cd6e 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -568,15 +568,19 @@ ban_evaluate(struct worker *wrk, const uint8_t *bsarg, struct objcore *oc, } break; case BANS_OPER_MATCH: + if (arg1 == NULL) + return (0); rv = VRE_match(bt.arg2_spec, arg1, 0, 0, NULL); xxxassert(rv >= -1); - if (arg1 == NULL || rv < 0) + if (rv < 0) return (0); break; case BANS_OPER_NMATCH: + if (arg1 == NULL) + return (0); rv = VRE_match(bt.arg2_spec, arg1, 0, 0, NULL); xxxassert(rv >= -1); - if (arg1 == NULL || rv >= 0) + if (rv >= 0) return (0); break; case BANS_OPER_GT: From dridi.boukelmoune at gmail.com Tue Sep 28 15:47:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 28 Sep 2021 15:47:05 +0000 (UTC) Subject: [master] 2b7389394 vtc: Test case for #3706 Message-ID: <20210928154705.7B2C011981C@lists.varnish-cache.org> commit 2b7389394b84914a475272665930ad73540972e0 Author: Dridi Boukelmoune Date: Tue Sep 28 17:45:22 2021 +0200 vtc: Test case for #3706 diff --git a/bin/varnishtest/tests/r03706.vtc b/bin/varnishtest/tests/r03706.vtc new file mode 100644 index 000000000..582c6f922 --- /dev/null +++ b/bin/varnishtest/tests/r03706.vtc @@ -0,0 +1,18 @@ +varnishtest "Null regex subject in ban expression" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { } -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +varnish v1 -cliok "ban req.http.nonexistent ~ foo" + +client c1 -run From nils.goroll at uplex.de Tue Sep 28 16:35:10 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 28 Sep 2021 16:35:10 +0000 (UTC) Subject: [master] fc1ce6daf Fix use of ObjGetSpace() for synthetic() Message-ID: <20210928163510.DE84811AD4B@lists.varnish-cache.org> commit fc1ce6daf6a966df3aa8b7519228f60b040ff072 Author: Nils Goroll Date: Tue Sep 28 18:24:28 2021 +0200 Fix use of ObjGetSpace() for synthetic() The size argument of ObjGetSpace() is only a hint and storage engines are free to return whatever size. If the transient storage returned less space than requested, the session was closed (no response body sent) with SC_OVERLOAD and "Error: Could not get storage" logged. diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 2e4ae1650..24c87bd1b 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -304,6 +304,7 @@ cnt_synth(struct worker *wrk, struct req *req) struct vsb *synth_body; ssize_t sz, szl; uint8_t *ptr; + const char *body; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -368,20 +369,26 @@ cnt_synth(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); szl = -1; if (STV_NewObject(wrk, req->objcore, stv_transient, 1024)) { + body = VSB_data(synth_body); szl = VSB_len(synth_body); assert(szl >= 0); - sz = szl; - if (sz > 0 && - ObjGetSpace(wrk, req->objcore, &sz, &ptr) && sz >= szl) { - memcpy(ptr, VSB_data(synth_body), szl); - ObjExtend(wrk, req->objcore, szl, 1); - } else if (sz > 0) { - szl = -1; + while (szl > 0) { + sz = szl; + if (! ObjGetSpace(wrk, req->objcore, &sz, &ptr)) { + szl = -1; + break; + } + if (sz > szl) + sz = szl; + szl -= sz; + memcpy(ptr, body, sz); + ObjExtend(wrk, req->objcore, sz, szl == 0 ? 1 : 0); + body += sz; } } if (szl >= 0) - AZ(ObjSetU64(wrk, req->objcore, OA_LEN, szl)); + AZ(ObjSetU64(wrk, req->objcore, OA_LEN, VSB_len(synth_body))); HSH_DerefBoc(wrk, req->objcore); VSB_destroy(&synth_body); From nils.goroll at uplex.de Tue Sep 28 16:45:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 28 Sep 2021 16:45:08 +0000 (UTC) Subject: [7.0] 46bbfe258 ban_evaluate(): Do not call VRE_match on a NULL subject Message-ID: <20210928164508.47C7611B310@lists.varnish-cache.org> commit 46bbfe2584380dec4c9036cfdc4a28d6c3d1ab9c Author: Nils Goroll Date: Tue Sep 28 16:44:24 2021 +0200 ban_evaluate(): Do not call VRE_match on a NULL subject Fixes #3706 diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index bd02f3001..ab523cd6e 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -568,15 +568,19 @@ ban_evaluate(struct worker *wrk, const uint8_t *bsarg, struct objcore *oc, } break; case BANS_OPER_MATCH: + if (arg1 == NULL) + return (0); rv = VRE_match(bt.arg2_spec, arg1, 0, 0, NULL); xxxassert(rv >= -1); - if (arg1 == NULL || rv < 0) + if (rv < 0) return (0); break; case BANS_OPER_NMATCH: + if (arg1 == NULL) + return (0); rv = VRE_match(bt.arg2_spec, arg1, 0, 0, NULL); xxxassert(rv >= -1); - if (arg1 == NULL || rv >= 0) + if (rv >= 0) return (0); break; case BANS_OPER_GT: From nils.goroll at uplex.de Tue Sep 28 16:45:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 28 Sep 2021 16:45:08 +0000 (UTC) Subject: [7.0] 1996366df vtc: Test case for #3706 Message-ID: <20210928164508.6814B11B313@lists.varnish-cache.org> commit 1996366df42bf5c89e7f2e74a8871f247e4693a3 Author: Dridi Boukelmoune Date: Tue Sep 28 17:45:22 2021 +0200 vtc: Test case for #3706 diff --git a/bin/varnishtest/tests/r03706.vtc b/bin/varnishtest/tests/r03706.vtc new file mode 100644 index 000000000..582c6f922 --- /dev/null +++ b/bin/varnishtest/tests/r03706.vtc @@ -0,0 +1,18 @@ +varnishtest "Null regex subject in ban expression" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { } -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +varnish v1 -cliok "ban req.http.nonexistent ~ foo" + +client c1 -run From nils.goroll at uplex.de Tue Sep 28 19:26:03 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 28 Sep 2021 19:26:03 +0000 (UTC) Subject: [master] 2235d9ed3 Introduce a debug storage which is just malloc with a different name Message-ID: <20210928192603.0E45D5B78@lists.varnish-cache.org> commit 2235d9ed3a7c0f0d65d2d96baa7fa737dcd31379 Author: Nils Goroll Date: Tue Sep 28 19:15:59 2021 +0200 Introduce a debug storage which is just malloc with a different name To avoid code duplication, we only provide an init callback, which copies all other callbacks from the malloc storage. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 27d931ab3..af04e2995 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -101,6 +101,7 @@ varnishd_SOURCES = \ storage/storage_file.c \ storage/storage_lru.c \ storage/storage_malloc.c \ + storage/storage_debug.c \ storage/storage_simple.c \ storage/storage_umem.c \ waiter/cache_waiter.c \ diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c index f96f841ce..c593c4d9d 100644 --- a/bin/varnishd/storage/mgt_stevedore.c +++ b/bin/varnishd/storage/mgt_stevedore.c @@ -152,6 +152,7 @@ static const struct stevedore smp_fake_stevedore = { static const struct choice STV_choice[] = { { "file", &smf_stevedore }, { "malloc", &sma_stevedore }, + { "debug", &smd_stevedore }, #ifdef WITH_PERSISTENT_STORAGE { "deprecated_persistent", &smp_stevedore }, { "persistent", &smp_fake_stevedore }, diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 060c41a24..e6c0a0cfd 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -144,5 +144,6 @@ void LRU_Touch(struct worker *, struct objcore *, vtim_real now); /*--------------------------------------------------------------------*/ extern const struct stevedore smu_stevedore; extern const struct stevedore sma_stevedore; +extern const struct stevedore smd_stevedore; extern const struct stevedore smf_stevedore; extern const struct stevedore smp_stevedore; diff --git a/bin/varnishd/storage/storage_debug.c b/bin/varnishd/storage/storage_debug.c new file mode 100644 index 000000000..f91c74f1e --- /dev/null +++ b/bin/varnishd/storage/storage_debug.c @@ -0,0 +1,58 @@ +/*- + * Copyright 2021 UPLEX - Nils Goroll Systemoptimierung + * All rights reserved. + * + * Author: Nils Goroll + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * debug helper storage based on malloc + */ + +#include "config.h" + +#include "cache/cache_varnishd.h" + +#include "storage/storage.h" +#include "storage/storage_simple.h" + +static void +smd_init(struct stevedore *parent, int ac, char * const *av) +{ + const char *ident; + + ident = parent->ident; + memcpy(parent, &sma_stevedore, sizeof *parent); + parent->ident = ident; + parent->name = smd_stevedore.name; + + sma_stevedore.init(parent, ac, av); +} + +const struct stevedore smd_stevedore = { + .magic = STEVEDORE_MAGIC, + .name = "debug", + .init = smd_init, + // other callbacks initialized in smd_init() +}; diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c index 6bb789e8b..cf0bf2544 100644 --- a/bin/varnishd/storage/storage_malloc.c +++ b/bin/varnishd/storage/storage_malloc.c @@ -187,19 +187,20 @@ sma_init(struct stevedore *parent, int ac, char * const *av) AZ(av[ac]); if (ac > 1) - ARGV_ERR("(-smalloc) too many arguments\n"); + ARGV_ERR("(-s%s) too many arguments\n", parent->name); if (ac == 0 || *av[0] == '\0') return; e = VNUM_2bytes(av[0], &u, 0); if (e != NULL) - ARGV_ERR("(-smalloc) size \"%s\": %s\n", av[0], e); + ARGV_ERR("(-s%s) size \"%s\": %s\n", parent->name, av[0], e); if ((u != (uintmax_t)(size_t)u)) - ARGV_ERR("(-smalloc) size \"%s\": too big\n", av[0]); + ARGV_ERR("(-s%s) size \"%s\": too big\n", parent->name, av[0]); if (u < 1024*1024) - ARGV_ERR("(-smalloc) size \"%s\": too small, " - "did you forget to specify M or G?\n", av[0]); + ARGV_ERR("(-s%s) size \"%s\": too small, " + "did you forget to specify M or G?\n", parent->name, + av[0]); sc->sma_max = u; } diff --git a/bin/varnishtest/tests/b00017.vtc b/bin/varnishtest/tests/b00017.vtc index 29fbe9e69..c886e17e1 100644 --- a/bin/varnishtest/tests/b00017.vtc +++ b/bin/varnishtest/tests/b00017.vtc @@ -1,6 +1,6 @@ -varnishtest "Check that we do not close one error" +varnishtest "Check set resp.body in vcl_synth" -varnish v1 -vcl { +varnish v1 -arg "-sTransient=debug" -vcl { backend foo { .host = "${bad_backend}"; } From nils.goroll at uplex.de Tue Sep 28 19:26:03 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 28 Sep 2021 19:26:03 +0000 (UTC) Subject: [master] 8b3e91687 Teach the debug storage engine to return less space then requested Message-ID: <20210928192603.23A705B7C@lists.varnish-cache.org> commit 8b3e9168742378469eadf1c67f174ad6fc844931 Author: Nils Goroll Date: Tue Sep 28 20:05:41 2021 +0200 Teach the debug storage engine to return less space then requested for the objgetspace callback This makes b17.vtc fail with fc1ce6daf6a966df3aa8b7519228f60b040ff072 reverted. diff --git a/bin/varnishd/storage/storage_debug.c b/bin/varnishd/storage/storage_debug.c index f91c74f1e..23cd5040c 100644 --- a/bin/varnishd/storage/storage_debug.c +++ b/bin/varnishd/storage/storage_debug.c @@ -33,20 +33,57 @@ #include "config.h" #include "cache/cache_varnishd.h" +#include "cache/cache_obj.h" + +#include #include "storage/storage.h" #include "storage/storage_simple.h" +/* returns one byte less than requested */ +static int v_matchproto_(objgetspace_f) +smd_lsp_getspace(struct worker *wrk, struct objcore *oc, ssize_t *sz, + uint8_t **ptr) +{ + AN(sz); + if (*sz > 1) + (*sz)--; + return (SML_methods.objgetspace(wrk, oc, sz, ptr)); +} + static void -smd_init(struct stevedore *parent, int ac, char * const *av) +smd_init(struct stevedore *parent, int aac, char * const *aav) { + struct obj_methods *methods; const char *ident; + int i, ac = 0; + char **av; ident = parent->ident; memcpy(parent, &sma_stevedore, sizeof *parent); parent->ident = ident; parent->name = smd_stevedore.name; + methods = malloc(sizeof *methods); + AN(methods); + memcpy(methods, &SML_methods, sizeof *methods); + parent->methods = methods; + + av = malloc(sizeof *av * (aac + 1)); + AN(av); + for (i = 0; i < aac; i++) { + if (aav[i] == NULL) { + av[ac++] = NULL; + continue; + } + if (! strcmp(aav[i], "lessspace")) { + methods->objgetspace = smd_lsp_getspace; + continue; + } + av[ac++] = strdup(aav[i]); + } + av[ac] = NULL; + sma_stevedore.init(parent, ac, av); } diff --git a/bin/varnishtest/tests/b00017.vtc b/bin/varnishtest/tests/b00017.vtc index c886e17e1..b205a8064 100644 --- a/bin/varnishtest/tests/b00017.vtc +++ b/bin/varnishtest/tests/b00017.vtc @@ -1,6 +1,6 @@ varnishtest "Check set resp.body in vcl_synth" -varnish v1 -arg "-sTransient=debug" -vcl { +varnish v1 -arg "-sTransient=debug,lessspace" -vcl { backend foo { .host = "${bad_backend}"; } From nils.goroll at uplex.de Wed Sep 29 08:59:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 29 Sep 2021 08:59:07 +0000 (UTC) Subject: [master] 6bdb447b2 Polish smd_init() as suggested by Dridi Message-ID: <20210929085907.E5D20A6363@lists.varnish-cache.org> commit 6bdb447b2cd6ee81563800ee056369278977ad6b Author: Nils Goroll Date: Wed Sep 29 10:54:51 2021 +0200 Polish smd_init() as suggested by Dridi Ref #3707 diff --git a/bin/varnishd/storage/storage_debug.c b/bin/varnishd/storage/storage_debug.c index 23cd5040c..379d97647 100644 --- a/bin/varnishd/storage/storage_debug.c +++ b/bin/varnishd/storage/storage_debug.c @@ -69,11 +69,12 @@ smd_init(struct stevedore *parent, int aac, char * const *aav) memcpy(methods, &SML_methods, sizeof *methods); parent->methods = methods; - av = malloc(sizeof *av * (aac + 1)); + av = calloc(aac + 1, sizeof *av); AN(av); for (i = 0; i < aac; i++) { if (aav[i] == NULL) { - av[ac++] = NULL; + AZ(av[ac]); + ac++; continue; } if (! strcmp(aav[i], "lessspace")) { @@ -82,7 +83,7 @@ smd_init(struct stevedore *parent, int aac, char * const *aav) } av[ac++] = strdup(aav[i]); } - av[ac] = NULL; + AZ(av[ac]); sma_stevedore.init(parent, ac, av); } From nils.goroll at uplex.de Wed Sep 29 10:35:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 29 Sep 2021 10:35:07 +0000 (UTC) Subject: [master] c138d1779 Dridilinting Message-ID: <20210929103507.627C7A91EF@lists.varnish-cache.org> commit c138d17796f5d7c26478e6a30046b102fc07a25e Author: Nils Goroll Date: Wed Sep 29 12:34:07 2021 +0200 Dridilinting Ref #3707 diff --git a/bin/varnishd/storage/storage_debug.c b/bin/varnishd/storage/storage_debug.c index 379d97647..aa8672fe6 100644 --- a/bin/varnishd/storage/storage_debug.c +++ b/bin/varnishd/storage/storage_debug.c @@ -81,7 +81,8 @@ smd_init(struct stevedore *parent, int aac, char * const *aav) methods->objgetspace = smd_lsp_getspace; continue; } - av[ac++] = strdup(aav[i]); + REPLACE(av[ac], aav[i]); + ac++; } AZ(av[ac]); From nils.goroll at uplex.de Wed Sep 29 10:39:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 29 Sep 2021 10:39:05 +0000 (UTC) Subject: [master] 2f02345a4 Code compaction after Dridilinting Message-ID: <20210929103905.661BBA9556@lists.varnish-cache.org> commit 2f02345a4868e309f36d9c4a38c44ba50a1374ae Author: Nils Goroll Date: Wed Sep 29 12:38:15 2021 +0200 Code compaction after Dridilinting Ref #3707 diff --git a/bin/varnishd/storage/storage_debug.c b/bin/varnishd/storage/storage_debug.c index aa8672fe6..ad069ef5f 100644 --- a/bin/varnishd/storage/storage_debug.c +++ b/bin/varnishd/storage/storage_debug.c @@ -72,12 +72,7 @@ smd_init(struct stevedore *parent, int aac, char * const *aav) av = calloc(aac + 1, sizeof *av); AN(av); for (i = 0; i < aac; i++) { - if (aav[i] == NULL) { - AZ(av[ac]); - ac++; - continue; - } - if (! strcmp(aav[i], "lessspace")) { + if (aav[i] != NULL && ! strcmp(aav[i], "lessspace")) { methods->objgetspace = smd_lsp_getspace; continue; } From nils.goroll at uplex.de Wed Sep 29 15:22:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 29 Sep 2021 15:22:06 +0000 (UTC) Subject: [master] 57155c045 Close potential race: Do not access a request after HSH_BUSY Message-ID: <20210929152206.D90EEB148D@lists.varnish-cache.org> commit 57155c045e9a0027c230880657815a3d56394664 Author: Nils Goroll Date: Wed Sep 29 17:04:47 2021 +0200 Close potential race: Do not access a request after HSH_BUSY Conceptually, when HSH_Lookup() returns with HSH_BUSY, ownership of the request has been ceded: Right after Lck_Unlock(&oh->mtx) in HSH_Lookup(), it could have got picked up from the waitinglist by another thread and, in the extreme, could already be done with when HSH_Lookup() returns. diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 24c87bd1b..cb4d3fa1a 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -555,10 +555,6 @@ cnt_lookup(struct worker *wrk, struct req *req) had_objhead = 1; wrk->strangelove = 0; lr = HSH_Lookup(req, &oc, &busy); - assert(wrk->strangelove >= 0); - if ((unsigned)wrk->strangelove >= cache_param->vary_notice) - VSLb(req->vsl, SLT_Notice, "vsl: High number of variants (%d)", - wrk->strangelove); if (lr == HSH_BUSY) { /* * We lost the session to a busy object, disembark the @@ -568,6 +564,10 @@ cnt_lookup(struct worker *wrk, struct req *req) */ return (REQ_FSM_DISEMBARK); } + assert(wrk->strangelove >= 0); + if ((unsigned)wrk->strangelove >= cache_param->vary_notice) + VSLb(req->vsl, SLT_Notice, "vsl: High number of variants (%d)", + wrk->strangelove); if (had_objhead) VSLb_ts_req(req, "Waitinglist", W_TIM_real(wrk)); From nils.goroll at uplex.de Wed Sep 29 15:30:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 29 Sep 2021 15:30:08 +0000 (UTC) Subject: [master] 39a5eb2ff Buy fetch steps some white space Message-ID: <20210929153008.2AC8EB19FA@lists.varnish-cache.org> commit 39a5eb2ff4155b8c34691feb02c37116c6a8f5e8 Author: Nils Goroll Date: Wed Sep 29 17:23:57 2021 +0200 Buy fetch steps some white space diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index eb09990ff..ea5553b3f 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -60,7 +60,7 @@ struct fetch_step { #define FETCH_STEP(l, U) \ static vbf_state_f vbf_stp_##l; \ - static const struct fetch_step F_STP_##U[1] = {{ .name = "Fetch Step" #l, .func = vbf_stp_##l, }}; + static const struct fetch_step F_STP_##U[1] = {{ .name = "Fetch Step " #l, .func = vbf_stp_##l, }}; FETCH_STEPS #undef FETCH_STEP From nils.goroll at uplex.de Wed Sep 29 15:30:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 29 Sep 2021 15:30:08 +0000 (UTC) Subject: [master] c1bef1592 Name request steps Req Step Message-ID: <20210929153008.4BDBCB19FD@lists.varnish-cache.org> commit c1bef1592e9418452387964179c852f8a10f17f2 Author: Nils Goroll Date: Wed Sep 29 17:25:26 2021 +0200 Name request steps Req Step Closes #3708 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index cb4d3fa1a..5116c8c4c 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -72,7 +72,7 @@ #define REQ_STEP(l, U, priv) \ static req_state_f cnt_##l; \ priv const struct req_step R_STP_##U[1] = {{ \ - .name = "Fetch Step" #l, \ + .name = "Req Step " #l, \ .func = cnt_##l, \ }}; REQ_STEPS From guillaume.quintard at gmail.com Wed Sep 15 22:43:10 2021 From: guillaume.quintard at gmail.com (Guillaume Quintard) Date: Wed, 15 Sep 2021 22:43:10 -0000 Subject: [master] 6120dcc72 [vtc] wait for logexpect to finish Message-ID: <20210915224309.D71B71028D5@lists.varnish-cache.org> commit 6120dcc72bf3c50afdab47df585ef79816ed6d9e Author: Guillaume Quintard Date: Wed Sep 15 15:41:35 2021 -0700 [vtc] wait for logexpect to finish fixes #3698 diff --git a/bin/varnishtest/tests/r02451.vtc b/bin/varnishtest/tests/r02451.vtc index 50bf1c71b..88a8e6d94 100644 --- a/bin/varnishtest/tests/r02451.vtc +++ b/bin/varnishtest/tests/r02451.vtc @@ -72,3 +72,5 @@ client c1 { rxresp expect resp.body == "xiuFi3Pe" } -run + +logexpect l1 -wait