From phk at FreeBSD.org Mon Dec 4 08:56:10 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 4 Dec 2023 08:56:10 +0000 (UTC) Subject: [master] 84f2a4703 Add missing reference to vsl-query Message-ID: <20231204085610.4960C11E510@lists.varnish-cache.org> commit 84f2a47038b5c4f4e9215a2b88004e4eb5888ee4 Author: Poul-Henning Kamp Date: Wed Nov 22 09:11:06 2023 +0000 Add missing reference to vsl-query diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 1585b2ae7..173ea8a63 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -261,6 +261,7 @@ SEE ALSO :ref:`varnishd(1)` :ref:`varnishlog(1)` :ref:`varnishstat(1)` +:ref:`vsl-query(7)` :ref:`vsl(7)` HISTORY From phk at FreeBSD.org Mon Dec 4 08:56:10 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 4 Dec 2023 08:56:10 +0000 (UTC) Subject: [master] f8936e78a Change the panic reentrancy check so that Coverity can (hopefully) grok it. Message-ID: <20231204085610.6BCC411E512@lists.varnish-cache.org> commit f8936e78a50153f3203bb077be8a067eac0c9663 Author: Poul-Henning Kamp Date: Mon Dec 4 08:53:22 2023 +0000 Change the panic reentrancy check so that Coverity can (hopefully) grok it. diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 3e093c50b..7b433c596 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -81,6 +81,7 @@ static pthread_key_t req_key; static pthread_key_t bo_key; static pthread_key_t wrk_key; pthread_key_t witness_key; +pthread_key_t panic_key; void THR_SetBusyobj(const struct busyobj *bo) @@ -404,6 +405,7 @@ child_main(int sigmagic, size_t altstksz) PTOK(pthread_key_create(&wrk_key, NULL)); PTOK(pthread_key_create(&witness_key, free)); PTOK(pthread_key_create(&name_key, NULL)); + PTOK(pthread_key_create(&panic_key, NULL)); THR_SetName("cache-main"); diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 221d9835c..eca20c32b 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -67,7 +67,6 @@ static struct vsb pan_vsb_storage, *pan_vsb; static pthread_mutex_t panicstr_mtx; -static pthread_t panicy; static void pan_sess(struct vsb *, const struct sess *); static void pan_req(struct vsb *, const struct req *); @@ -744,18 +743,23 @@ pan_ic(const char *func, const char *file, int line, const char *cond, struct busyobj *bo; struct worker *wrk; struct sigaction sa; - int err = errno; + int i, err = errno; - /* If we already panicing in another thread, do nothing */ - while (heritage.panic_str[0] && panicy != pthread_self()) - sleep(1); - - if (pthread_mutex_lock(&panicstr_mtx)) { - /* Reentrant panic */ + if (pthread_getspecific(panic_key) != NULL) { VSB_cat(pan_vsb, "\n\nPANIC REENTRANCY\n\n"); abort(); } - panicy = pthread_self(); + + /* If we already panicing in another thread, do nothing */ + do { + i = pthread_mutex_trylock(&panicstr_mtx); + if (i != 0) + sleep (1); + } while (i != 0); + + assert (VSB_len(pan_vsb) == 0); + + AZ(pthread_setspecific(panic_key, pan_vsb)); /* * should we trigger a SIGSEGV while handling a panic, our sigsegv @@ -844,6 +848,14 @@ pan_ic(const char *func, const char *file, int line, const char *cond, VSB_putc(pan_vsb, '\0'); /* NUL termination */ v_gcov_flush(); + + /* + * Do a little song and dance for static checkers which + * are not smart enough to figure out that calling abort() + * with a mutex held is OK and probably very intentional. + */ + if (pthread_getspecific(panic_key)) /* ie: always */ + abort(); PTOK(pthread_mutex_unlock(&panicstr_mtx)); abort(); } diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index b534eb174..e9ffad17c 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -303,6 +303,7 @@ unsigned HTTP1_Write(const struct worker *w, const struct http *hp, const int*); /* cache_main.c */ vxid_t VXID_Get(const struct worker *, uint64_t marker); +extern pthread_key_t panic_key; extern pthread_key_t witness_key; void THR_SetName(const char *name); From phk at FreeBSD.org Mon Dec 4 09:12:05 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 4 Dec 2023 09:12:05 +0000 (UTC) Subject: [master] ddeb24214 Add comment about ObjSubscribeEvents() not being atomic. Message-ID: <20231204091205.67E4111F002@lists.varnish-cache.org> commit ddeb24214599de85f8e77be72ce9fc29d386a8a7 Author: Poul-Henning Kamp Date: Mon Dec 4 09:09:12 2023 +0000 Add comment about ObjSubscribeEvents() not being atomic. diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index d92fc1f0d..2ff54cb8e 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -672,6 +672,10 @@ static VTAILQ_HEAD(,oev_entry) oev_list; static pthread_rwlock_t oev_rwl; static unsigned oev_mask; +/* + * NB: ObjSubscribeEvents() is not atomic: + * oev_mask is checked optimistically in ObjSendEvent() + */ uintptr_t ObjSubscribeEvents(obj_event_f *func, void *priv, unsigned mask) { From dridi.boukelmoune at gmail.com Mon Dec 4 10:05:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 10:05:06 +0000 (UTC) Subject: [master] 2aac04085 vtim: Make test driver compile Message-ID: <20231204100506.C50501209C6@lists.varnish-cache.org> commit 2aac04085cb1b83080dc9e5c04c1006bc4e962e0 Author: Dridi Boukelmoune Date: Mon Dec 4 11:02:01 2023 +0100 vtim: Make test driver compile diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 1ca0d836d..92e5535b2 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -504,7 +504,7 @@ tst_delta_check(const char *name, double begin, double end, vtim_dur ref) } static void -tst_delta() +tst_delta(void) { vtim_mono m_begin, m_end; vtim_real r_begin, r_end; @@ -527,7 +527,7 @@ tst_delta() } static void -bench() +bench(void) { vtim_mono s, e; vtim_mono t_m; @@ -536,6 +536,7 @@ bench() int i; char buf[64]; + t_m = 0; t_r = 0; s = VTIM_mono(); for (i=0; i<100000; i++) @@ -575,7 +576,7 @@ bench() e - s, i, 1e9 * (e - s) / i, t_i, buf); } -void +static void parse_check(time_t t, const char *s) { vtim_real tt; @@ -598,6 +599,9 @@ main(int argc, char **argv) char buf[BUFSIZ]; char buf1[BUFSIZ]; + (void)argc; + (void)argv; + AZ(setenv("TZ", "UTC", 1)); assert(sizeof t >= 8); From dridi.boukelmoune at gmail.com Mon Dec 4 10:05:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 10:05:06 +0000 (UTC) Subject: [master] 11ed4f243 build: Add vtim_test to the libvarnish test suite Message-ID: <20231204100506.E279E1209C9@lists.varnish-cache.org> commit 11ed4f243299c046cb4e2d823bb094a570531c4a Author: Dridi Boukelmoune Date: Mon Dec 4 11:03:28 2023 +0100 build: Add vtim_test to the libvarnish test suite diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index d19e61409..946966cf9 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -46,7 +46,14 @@ libvarnish_la_SOURCES = \ libvarnish_la_LIBADD = @PCRE2_LIBS@ $(LIBM) -TESTS = vav_test vbh_test vct_test vjsn_test vnum_c_test vsb_test +TESTS = \ + vav_test \ + vbh_test \ + vct_test \ + vjsn_test \ + vnum_c_test \ + vsb_test \ + vtim_test noinst_PROGRAMS = ${TESTS} @@ -73,3 +80,7 @@ vjsn_test_LDADD = $(AM_LDFLAGS) libvarnish.la vsb_test_SOURCES = vsb_test.c vsb_test_CFLAGS = $(AM_CFLAGS) -DVSB_TEST vsb_test_LDADD = $(AM_LDFLAGS) libvarnish.la + +vtim_test_SOURCES = vtim.c +vtim_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER +vtim_test_LDADD = $(AM_LDFLAGS) libvarnish.la diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 92e5535b2..31a216ca6 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -454,16 +454,8 @@ VTIM_timespec(vtim_dur t) return (tv); } - #ifdef TEST_DRIVER -/********************************************************************** - * Compile with: - * cc -o foo -DTEST_DRIVER -I../.. -I../../include vtim.c vas.c -lm - * Test with: - * ./foo - */ - #include static void @@ -642,4 +634,5 @@ main(int argc, char **argv) return (0); } + #endif From dridi.boukelmoune at gmail.com Mon Dec 4 10:07:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 10:07:05 +0000 (UTC) Subject: [master] 60bb4e4ed doc: Use a raw string for regular expressions Message-ID: <20231204100705.5B125120D89@lists.varnish-cache.org> commit 60bb4e4ed661d21de0fcb0132ade509520ebb932 Author: Dridi Boukelmoune Date: Mon Dec 4 10:45:20 2023 +0100 doc: Use a raw string for regular expressions Since Python 3.12 unknown escape sequences trigger a warning. diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py index cfc5a05ab..acf6f356c 100644 --- a/doc/sphinx/vtc-syntax.py +++ b/doc/sphinx/vtc-syntax.py @@ -38,7 +38,7 @@ import re def parse_file(fn, cl, tl, sl): p = False section = "" - resec = re.compile("\s*/?\* SECTION: ") + resec = re.compile(r"\s*/?\* SECTION: ") try: # Python3 From nils.goroll at uplex.de Mon Dec 4 10:36:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Dec 2023 10:36:06 +0000 (UTC) Subject: [master] 3fc690dfc Add test case for 304 response on zero-sized object Message-ID: <20231204103606.5F1CB121CA0@lists.varnish-cache.org> commit 3fc690dfc6c81ac2d9e96768bbe0516e8c67b75f Author: Nils Goroll Date: Mon Dec 4 11:30:24 2023 +0100 Add test case for 304 response on zero-sized object Test case by Martin Blix Grydeland, taken from #4013 diff --git a/bin/varnishtest/tests/b000082.vtc b/bin/varnishtest/tests/b000082.vtc new file mode 100644 index 000000000..8b659a913 --- /dev/null +++ b/bin/varnishtest/tests/b000082.vtc @@ -0,0 +1,39 @@ +varnishtest "Backend IMS 304 reponse with Content-Length 0" + +# this case tests invalid behaviour, which we should handle gracefully anyway + +server s1 { + rxreq + txresp -nolen -hdr "Content-Length: 0" -hdr {Etag: "foo"} + + rxreq + txresp -status 304 -nolen -hdr "Content-Length: 0" -hdr {Etag: "foo"} +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.ttl = 1s; + } +} -start + +client c1 { + txreq + rxresp + + txreq + rxresp +} -run + +delay 1 + +client c2 { + txreq + rxresp +} -run + +delay 0.1 + +client c3 { + txreq + rxresp +} -run From nils.goroll at uplex.de Mon Dec 4 10:56:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Dec 2023 10:56:06 +0000 (UTC) Subject: [master] 9a4653bbc Fix file name format of newly added test Message-ID: <20231204105606.5A2641226ED@lists.varnish-cache.org> commit 9a4653bbca9284d91dc96b9f454b78743c2ca72b Author: Nils Goroll Date: Mon Dec 4 11:54:58 2023 +0100 Fix file name format of newly added test sorry for the glitch diff --git a/bin/varnishtest/tests/b000082.vtc b/bin/varnishtest/tests/b00082.vtc similarity index 100% rename from bin/varnishtest/tests/b000082.vtc rename to bin/varnishtest/tests/b00082.vtc From dridi.boukelmoune at gmail.com Mon Dec 4 11:20:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 11:20:06 +0000 (UTC) Subject: [master] 93974b3f9 build: Ignore y2k warning for vtim_test Message-ID: <20231204112006.DA6B3123322@lists.varnish-cache.org> commit 93974b3f9f51c5591345b0fc37e51437393659f3 Author: Dridi Boukelmoune Date: Mon Dec 4 12:18:58 2023 +0100 build: Ignore y2k warning for vtim_test diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index 946966cf9..aa88f486a 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -82,5 +82,5 @@ vsb_test_CFLAGS = $(AM_CFLAGS) -DVSB_TEST vsb_test_LDADD = $(AM_LDFLAGS) libvarnish.la vtim_test_SOURCES = vtim.c -vtim_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER +vtim_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER -Wno-error=format-y2k vtim_test_LDADD = $(AM_LDFLAGS) libvarnish.la From nils.goroll at uplex.de Mon Dec 4 11:33:03 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Dec 2023 11:33:03 +0000 (UTC) Subject: [master] 245c768fa Split out vcc_acl_insert_entry() again Message-ID: <20231204113303.8AD94123A60@lists.varnish-cache.org> commit 245c768fa4e509d19bdf5bd308f0fbfc71d280fb Author: Nils Goroll Date: Mon May 10 17:05:07 2021 +0200 Split out vcc_acl_insert_entry() again Reverts d89674d129ef0ed1c495e1169f83ec3203db1b42 Reinstantiates bbb5db47e02f431598e5a7ff7106a8ac316db8be diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index dffd9013c..c8219dcc8 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -208,11 +208,30 @@ vcc_acl_chk(struct vcc *tl, const struct acl_e *ae, const int l, return (r); } +static void +vcc_acl_insert_entry(struct vcc *tl, struct acl_e **aenp) +{ + struct acl_e *ae2; + + CHECK_OBJ_NOTNULL(*aenp, VCC_ACL_E_MAGIC); + ae2 = VRBT_INSERT(acl_tree, &tl->acl->acl_tree, *aenp); + if (ae2 != NULL) { + if (ae2->not != (*aenp)->not) { + VSB_cat(tl->sb, "Conflicting ACL entries:\n"); + vcc_ErrWhere(tl, ae2->t_addr); + VSB_cat(tl->sb, "vs:\n"); + vcc_ErrWhere(tl, (*aenp)->t_addr); + } + return; + } + *aenp = NULL; +} + static void vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l, unsigned char *u, int fam) { - struct acl_e *aen, *ae2; + struct acl_e *aen; if (fam == PF_INET && ae->mask > 32) { VSB_printf(tl->sb, @@ -247,14 +266,9 @@ vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l, assert(l + 1UL <= sizeof aen->data); memcpy(aen->data + 1L, u, l); - ae2 = VRBT_INSERT(acl_tree, &tl->acl->acl_tree, aen); - if (ae2 != NULL && ae2->not != aen->not) { - VSB_cat(tl->sb, "Conflicting ACL entries:\n"); - vcc_ErrWhere(tl, ae2->t_addr); - VSB_cat(tl->sb, "vs:\n"); - vcc_ErrWhere(tl, aen->t_addr); + vcc_acl_insert_entry(tl, &aen); + if (aen != NULL) vcl_acl_free(&aen); - } } static void From nils.goroll at uplex.de Mon Dec 4 11:33:03 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Dec 2023 11:33:03 +0000 (UTC) Subject: [master] 51523bc88 vcc_acl: change compare functions to differentiate cases Message-ID: <20231204113303.A18F7123A63@lists.varnish-cache.org> commit 51523bc883525005a0d87c83467e753a88abb0ed Author: Nils Goroll Date: Fri Dec 13 18:08:59 2019 +0100 vcc_acl: change compare functions to differentiate cases This is in preparation of a follow-up commit to merge acl entries and detect supersedes from supernets, but these changes are backwards compatible with the previous CMP() if being used as a comparison function for which only negative, zero and positive result are relevant. The A in CMPA() stands for "adjacent". CMPA() returns -3/3 for left of/right of. diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index c8219dcc8..d4b78ad8d 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -74,14 +74,29 @@ struct acl_e { struct token *t_mask; }; -/* Compare two acl rules for ordering */ +/* + * Compare two acl rules for ordering + * returns: + * 0 same + * -1/1 strictly less/greater + * -2/2 b contains a / a contains b + * -3/3 a left of b / b left of a + */ -#define CMP(a, b) \ +#define CMP(n, a, b) \ do { \ if ((a) < (b)) \ - return (-1); \ + return (-n); \ else if ((b) < (a)) \ - return (1); \ + return (n); \ + } while (0) + +#define CMPA(a, b) \ + do { \ + if (((a) | 1) == (b)) \ + return (-3); \ + else if (((b) | 1) == (a)) \ + return (3); \ } while (0) static void @@ -99,6 +114,7 @@ vcl_acl_cmp(const struct acl_e *ae1, const struct acl_e *ae2) { const unsigned char *p1, *p2; unsigned m; + unsigned char a1, a2; CHECK_OBJ_NOTNULL(ae1, VCC_ACL_E_MAGIC); CHECK_OBJ_NOTNULL(ae2, VCC_ACL_E_MAGIC); @@ -107,17 +123,22 @@ vcl_acl_cmp(const struct acl_e *ae1, const struct acl_e *ae2) p2 = ae2->data; m = vmin_t(unsigned, ae1->mask, ae2->mask); for (; m >= 8; m -= 8) { - CMP(*p1, *p2); + CMP(1, *p1, *p2); p1++; p2++; } if (m) { - m = 0xff00 >> m; - m &= 0xff; - CMP(*p1 & m, *p2 & m); + assert (m < 8); + a1 = *p1 >> (8 - m); + a2 = *p2 >> (8 - m); + if (ae1->mask == ae2->mask) + CMPA(a1, a2); + CMP(1, a1, a2); + } else if (ae1->mask == ae2->mask) { + CMPA(*p1, *p2); } /* Long mask is less than short mask */ - CMP(ae2->mask, ae1->mask); + CMP(2, ae2->mask, ae1->mask); return (0); } @@ -135,14 +156,14 @@ vcl_acl_disjoint(const struct acl_e *ae1, const struct acl_e *ae2) p2 = ae2->data; m = vmin_t(unsigned, ae1->mask, ae2->mask); for (; m >= 8; m -= 8) { - CMP(*p1, *p2); + CMP(1, *p1, *p2); p1++; p2++; } if (m) { m = 0xff00 >> m; m &= 0xff; - CMP(*p1 & m, *p2 & m); + CMP(1, *p1 & m, *p2 & m); } return (0); } From nils.goroll at uplex.de Mon Dec 4 11:33:03 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Dec 2023 11:33:03 +0000 (UTC) Subject: [master] 01bf0dd2b +fold acl flag: remove subnets and fold adjacent entries Message-ID: <20231204113303.D51AB123A68@lists.varnish-cache.org> commit 01bf0dd2b0702588c724db1c0a79373e6f9fe97a Author: Nils Goroll Date: Sat Mar 27 13:56:25 2021 +0100 +fold acl flag: remove subnets and fold adjacent entries Function: With the fold acl flag enabled (default: disabled, keeping the existing behavior), ACLs are optimized in that subnets contained in other entries are skipped (e.g. if 1.2.3.0/24 is part of the ACL, an entry for 1.2.3.128/25 will not be added) and adjacent entries get folded (e.g. if both 1.2.3.0/25 and 1.2.3.128/25 are added, they will be folded to 1.2.3.0/24). Skip and fold operations on VCL entries are output as warnings during VCL compilation as entries from the VCL are processed in order. Logging under the VCL_acl tag can change with this parameter enabled: Matches on skipped subnet entries are now logged as matches on the respective supernet entry. Matches on folded entries are logged with a shorter netmask which might not be contained in the original ACL as defined in VCL. Such log entries are marked by "fixed: folded". Negated ACL entries are excluded from folds. Implementation: The sort functions are changed such that the previous semantics are preserved: negative return values signify "a < b", positive return values signify "a > b". But additionally the values -2/2 and -3/3 are introduced (and given enums) to signify "contained in supernet" and "directly adjacent to". This allows for mostly unchanged code with vcc_acl_fold disabled. For the "subnet contained in supernet" case, all contained subnets are removed. By sort order, caontained subnets are always to be found left of supernets. For the "fold adjacent" case, the netmask of the entry with the smaller network number is decreased by one and the other entry removed. Because changing the netmask might affect sort order, we reinsert the changed entry. diff --git a/bin/varnishtest/tests/c00005.vtc b/bin/varnishtest/tests/c00005.vtc index 4520088c7..8854329a4 100644 --- a/bin/varnishtest/tests/c00005.vtc +++ b/bin/varnishtest/tests/c00005.vtc @@ -86,6 +86,14 @@ varnish v1 -vcl { "1.3.0.0"/21; "1.4.7"; "1.4.6.0"/24; + + # bad notation (confusing) + "affe::affe:0304"/120; + "affe::affe:0342"/122; + + # more specific wins + "bad:cafe::"/32; + "bad:cafe::"/31; } sub vcl_recv { @@ -110,6 +118,12 @@ varnish v1 -vcl { set req.http.ip = "1.4.5.255"; call t; set req.http.ip = "1.4.6.64"; call t; set req.http.ip = "1.4.7.64"; call t; + + set req.http.ip = "affe::affe:0300"; call t; + set req.http.ip = "affe::affe:033f"; call t; + set req.http.ip = "affe::affe:0340"; call t; + + set req.http.ip = "0bad:cafe:1234::1"; call t; } } @@ -140,6 +154,15 @@ logexpect l1 -v v1 -g raw { expect 0 = VCL_acl {^\QMATCH acl1 "1.4.6.0"/24\E$} expect 1 = ReqHeader {^\Qip: 1.4.7.64\E$} expect 0 = VCL_acl {^\QMATCH acl1 "1.4.7"\E$} + + expect 1 = ReqHeader {^\Qip: affe::affe:0300\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "affe::affe:0304"/120 fixed: affe::affe:300/120\E$} + expect 1 = ReqHeader {^\Qip: affe::affe:033f\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "affe::affe:0304"/120 fixed: affe::affe:300/120\E$} + expect 1 = ReqHeader {^\Qip: affe::affe:0340\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "affe::affe:0342"/122 fixed: affe::affe:340/122\E$} + expect 1 = ReqHeader {^\Qip: 0bad:cafe:1234::1\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "bad:cafe::"/32\E$} } -start client c1 { @@ -162,3 +185,192 @@ varnish v1 -errvcl {Non-zero bits in masked part} { if (client.ip ~ acl1) {} } } + +varnish v1 -errvcl {Non-zero bits in masked part} { + import std; + + backend dummy None; + + acl acl1 +pedantic { + "affe::affe:0304"/120; + } + + sub vcl_recv { + if (client.ip ~ acl1) {} + } +} + +# this is both an OK test for pedantic and fold +varnish v1 -vcl { + import std; + + backend dummy None; + + acl acl1 +log +pedantic +fold { + # bad notation (confusing) + "1.2.3.0"/24; + "1.2.3.64"/26; + + # all contained in 1.3.0.0/21 and 1.4.4.0/22 + "1.4.4.0"/22; + "1.3.4.0"/23; + "1.3.5.0"/26; + "1.3.6.0"/25; + "1.3.6.128"/25; + "1.3.0.0"/21; + "1.4.7"; + "1.4.6.0"/24; + + # right,left adjacent + "2.3.2.0"/23; + "2.3.0.0"/23; + # left,right adjacent + "2.3.4.0"/23; + "2.3.6.0"/23; + + # 12/14 folded, not 10 + "2.10.0.0"/15; + "2.12.0.0"/15; + "2.14.0.0"/15; + + # 226/227 folded, not 225 + "2.225.0.0"/16; + "2.226.0.0"/16; + "2.227.0.0"/16; + + # phks test case + "10.0.0.0"/23; + "10.0.2.0"/23; + + "10.1.0.0"/24; + "10.1.1.0"/24; + + "10.2.0.0"/25; + "10.2.0.128"/25; + + # contained + "bad:cafe::"/32; + "bad:cafe::"/31; + } + + sub vcl_recv { + return (synth(200)); + } + sub t { + if (std.ip(req.http.ip) ~ acl1) { } + } + sub vcl_synth { + # variables would be nice, but not in core (yet?) + set req.http.ip = "1.2.3.0"; call t; + set req.http.ip = "1.2.3.63"; call t; + set req.http.ip = "1.2.3.64"; call t; + + set req.http.ip = "1.3.4.255"; call t; + set req.http.ip = "1.3.5.0"; call t; + set req.http.ip = "1.3.5.255"; call t; + set req.http.ip = "1.3.6.0"; call t; + set req.http.ip = "1.3.6.140"; call t; + set req.http.ip = "1.3.7.255"; call t; + + set req.http.ip = "1.4.5.255"; call t; + set req.http.ip = "1.4.6.64"; call t; + set req.http.ip = "1.4.7.64"; call t; + + set req.http.ip = "2.3.0.0"; call t; + set req.http.ip = "2.3.5.255"; call t; + + set req.http.ip = "2.2.255.255";call t; + set req.http.ip = "2.3.8.0"; call t; + + set req.http.ip = "2.9.1.1"; call t; + set req.http.ip = "2.10.1.1"; call t; + set req.http.ip = "2.12.0.0"; call t; + set req.http.ip = "2.15.255.255";call t; + set req.http.ip = "2.16.1.1"; call t; + + set req.http.ip = "2.224.1.1"; call t; + set req.http.ip = "2.225.1.1"; call t; + set req.http.ip = "2.226.1.1"; call t; + set req.http.ip = "2.227.1.1"; call t; + + set req.http.ip = "10.0.3.255"; call t; + set req.http.ip = "10.1.1.255"; call t; + set req.http.ip = "10.2.0.255"; call t; + + set req.http.ip = "0bad:cafe:1234::1"; call t; + } +} + +logexpect l1 -v v1 -g raw { + expect * 1009 ReqHeader {^\Qip: 1.2.3.0\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "1.2.3.0"/24\E$} + expect 1 = ReqHeader {^\Qip: 1.2.3.63\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "1.2.3.0"/24\E$} + expect 1 = ReqHeader {^\Qip: 1.2.3.64\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "1.2.3.0"/24\E$} + + expect 1 = ReqHeader {^\Qip: 1.3.4.255\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "1.3.0.0"/21\E$} + expect 1 = ReqHeader {^\Qip: 1.3.5.0\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "1.3.0.0"/21\E$} + expect 1 = ReqHeader {^\Qip: 1.3.5.255\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "1.3.0.0"/21\E$} + expect 1 = ReqHeader {^\Qip: 1.3.6.0\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "1.3.0.0"/21\E$} + expect 1 = ReqHeader {^\Qip: 1.3.6.140\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "1.3.0.0"/21\E$} + expect 1 = ReqHeader {^\Qip: 1.3.7.255\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "1.3.0.0"/21\E$} + + expect 1 = ReqHeader {^\Qip: 1.4.5.255\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "1.4.4.0"/22\E$} + expect 1 = ReqHeader {^\Qip: 1.4.6.64\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "1.4.4.0"/22\E$} + expect 1 = ReqHeader {^\Qip: 1.4.7.64\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "1.4.4.0"/22\E$} + + expect 1 = ReqHeader {^\Qip: 2.3.0.0\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "2.3.0.0"/21 fixed: folded\E} + expect 1 = ReqHeader {^\Qip: 2.3.5.255\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "2.3.0.0"/21 fixed: folded\E} + expect 1 = ReqHeader {^\Qip: 2.2.255.255\E$$} + expect 0 = VCL_acl {^\QNO_MATCH acl1\E$} + expect 1 = ReqHeader {^\Qip: 2.3.8.0\E$} + expect 0 = VCL_acl {^\QNO_MATCH acl1\E$} + + expect 1 = ReqHeader {^\Qip: 2.9.1.1\E$} + expect 0 = VCL_acl {^\QNO_MATCH acl1\E$} + expect 1 = ReqHeader {^\Qip: 2.10.1.1\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "2.10.0.0"/15\E$} + expect 1 = ReqHeader {^\Qip: 2.12.0.0\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "2.12.0.0"/14 fixed: folded\E} + expect 1 = ReqHeader {^\Qip: 2.15.255.255\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "2.12.0.0"/14 fixed: folded\E} + expect 1 = ReqHeader {^\Qip: 2.16.1.1\E$} + expect 0 = VCL_acl {^\QNO_MATCH acl1\E} + + expect 1 = ReqHeader {^\Qip: 2.224.1.1\E$} + expect 0 = VCL_acl {^\QNO_MATCH acl1\E$} + expect 1 = ReqHeader {^\Qip: 2.225.1.1\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "2.225.0.0"/16\E$} + expect 1 = ReqHeader {^\Qip: 2.226.1.1\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "2.226.0.0"/15 fixed: folded\E} + expect 1 = ReqHeader {^\Qip: 2.227.1.1\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "2.226.0.0"/15 fixed: folded\E} + + expect 1 = ReqHeader {^\Qip: 10.0.3.255\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "10.0.0.0"/22 fixed: folded\E} + expect 1 = ReqHeader {^\Qip: 10.1.1.255\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "10.1.0.0"/23 fixed: folded\E} + expect 1 = ReqHeader {^\Qip: 10.2.0.255\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "10.2.0.0"/24 fixed: folded\E} + expect 1 = ReqHeader {^\Qip: 0bad:cafe:1234::1\E$} + expect 0 = VCL_acl {^\QMATCH acl1 "bad:cafe::"/31} +} -start + +client c1 { + txreq + rxresp +} -run + +logexpect l1 -wait diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index b218ae416..1d4d3b897 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -299,6 +299,27 @@ individually: However, if the name resolves to both IPv4 and IPv6 you will still get an error. +* `+fold` - Fold ACL supernets and adjacent networks. + + With this parameter set to on, ACLs are optimized in that subnets + contained in other entries are skipped (e.g. if 1.2.3.0/24 is part + of the ACL, an entry for 1.2.3.128/25 will not be added) and + adjacent entries get folded (e.g. if both 1.2.3.0/25 and + 1.2.3.128/25 are added, they will be folded to 1.2.3.0/24). + + Skip and fold operations on VCL entries are output as warnings + during VCL compilation as entries from the VCL are processed in + order. + + Logging under the ``VCL_acl`` tag can change with this parameter + enabled: Matches on skipped subnet entries are now logged as matches + on the respective supernet entry. Matches on folded entries are + logged with a shorter netmask which might not be contained in the + original ACL as defined in VCL. Such log entries are marked by + ``fixed: folded``. + + Negated ACL entries are never folded. + VCL objects ----------- diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 20a563dae..ebcb3ac36 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -264,15 +264,19 @@ SLTM(Fetch_Body, 0, "Body fetched from backend", SLTM(VCL_acl, 0, "VCL ACL check results", "ACLs with the `+log` flag emits this record with the result.\n\n" "The format is::\n\n" - "\t%s %s [%s [fixed: %s]]\n" - "\t| | | |\n" - "\t| | | +- Fixed entry (see acl +pedantic flag)\n" - "\t| | +------------ Matching entry (only for MATCH)\n" - "\t| +---------------- Name of the ACL\n" - "\t+-------------------- MATCH or NO_MATCH\n" - "\n" - "MATCH denotes an ACL match\n" - "NO_MATCH denotes that a checked ACL has not matched\n" + "\t%s [%s [%s [fixed: %s]]]\n" + "\t| | | |\n" + "\t| | | +- Fix info (see below)\n" + "\t| | +------------ Matching entry (only for MATCH)\n" + "\t| +---------------- Name of the ACL for MATCH or NO_MATCH\n" + "\t+-------------------- MATCH, NO_MATCH or NO_FAM\n" + "\n" + "* Fix info: either contains network/mask for non-canonical entries " + "(see acl +pedantic flag) or ``folded`` for entries " + "which were the result of a fold operation (see acl +fold flag).\n" + "* ``MATCH`` denotes an ACL match\n" + "* ``NO_MATCH`` denotes that a checked ACL has not matched\n" + "* ``NO_FAM`` denotes a missing address family and should not occur.\n" "\n" ) diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index d4b78ad8d..3585c3c81 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -53,6 +53,7 @@ struct acl { #define VCC_ACL_MAGIC 0xb9fb3cd0 int flag_log; + int flag_fold; int flag_pedantic; int flag_table; @@ -74,19 +75,26 @@ struct acl_e { struct token *t_mask; }; +enum acl_cmp_e { + ACL_EQ = 0, + ACL_LT = -1, // a < b + ACL_GT = 1, // b > a + ACL_CONTAINED = -2, // b contains a + ACL_CONTAINS = 2, // a contains b + ACL_LEFT = -3, // a + 1 == b + ACL_RIGHT = 3 // a == b + 1 +}; + +static void vcc_acl_insert_entry(struct vcc *, struct acl_e **); + /* - * Compare two acl rules for ordering - * returns: - * 0 same - * -1/1 strictly less/greater - * -2/2 b contains a / a contains b - * -3/3 a left of b / b left of a + * Compare two acl rules for relation */ #define CMP(n, a, b) \ do { \ if ((a) < (b)) \ - return (-n); \ + return (enum acl_cmp_e)(-n); \ else if ((b) < (a)) \ return (n); \ } while (0) @@ -94,9 +102,9 @@ struct acl_e { #define CMPA(a, b) \ do { \ if (((a) | 1) == (b)) \ - return (-3); \ + return (ACL_LEFT); \ else if (((b) | 1) == (a)) \ - return (3); \ + return (ACL_RIGHT); \ } while (0) static void @@ -109,7 +117,7 @@ vcl_acl_free(struct acl_e **aep) FREE_OBJ(a); } -static int +static enum acl_cmp_e vcl_acl_cmp(const struct acl_e *ae1, const struct acl_e *ae2) { const unsigned char *p1, *p2; @@ -123,7 +131,9 @@ vcl_acl_cmp(const struct acl_e *ae1, const struct acl_e *ae2) p2 = ae2->data; m = vmin_t(unsigned, ae1->mask, ae2->mask); for (; m >= 8; m -= 8) { - CMP(1, *p1, *p2); + if (m == 8 && ae1->mask == ae2->mask) + CMPA(*p1, *p2); + CMP(ACL_GT, *p1, *p2); p1++; p2++; } @@ -133,14 +143,14 @@ vcl_acl_cmp(const struct acl_e *ae1, const struct acl_e *ae2) a2 = *p2 >> (8 - m); if (ae1->mask == ae2->mask) CMPA(a1, a2); - CMP(1, a1, a2); + CMP(ACL_GT, a1, a2); } else if (ae1->mask == ae2->mask) { CMPA(*p1, *p2); } /* Long mask is less than short mask */ - CMP(2, ae2->mask, ae1->mask); + CMP(ACL_CONTAINS, ae2->mask, ae1->mask); - return (0); + return (ACL_EQ); } static int @@ -156,14 +166,14 @@ vcl_acl_disjoint(const struct acl_e *ae1, const struct acl_e *ae2) p2 = ae2->data; m = vmin_t(unsigned, ae1->mask, ae2->mask); for (; m >= 8; m -= 8) { - CMP(1, *p1, *p2); + CMP(ACL_GT, *p1, *p2); p1++; p2++; } if (m) { m = 0xff00 >> m; m &= 0xff; - CMP(1, *p1 & m, *p2 & m); + CMP(ACL_GT, *p1 & m, *p2 & m); } return (0); } @@ -171,6 +181,8 @@ vcl_acl_disjoint(const struct acl_e *ae1, const struct acl_e *ae2) VRBT_GENERATE_INSERT_COLOR(acl_tree, acl_e, branch, static) VRBT_GENERATE_INSERT_FINISH(acl_tree, acl_e, branch, static) VRBT_GENERATE_INSERT(acl_tree, acl_e, branch, vcl_acl_cmp, static) +VRBT_GENERATE_REMOVE_COLOR(acl_tree, acl_e, branch, static) +VRBT_GENERATE_REMOVE(acl_tree, acl_e, branch, static) VRBT_GENERATE_MINMAX(acl_tree, acl_e, branch, static) VRBT_GENERATE_NEXT(acl_tree, acl_e, branch, static) VRBT_GENERATE_PREV(acl_tree, acl_e, branch, static) @@ -229,10 +241,66 @@ vcc_acl_chk(struct vcc *tl, const struct acl_e *ae, const int l, return (r); } +static void +vcl_acl_fold(struct vcc *tl, struct acl_e **l, struct acl_e **r) +{ + enum acl_cmp_e cmp; + + AN(l); + AN(r); + CHECK_OBJ_NOTNULL(*l, VCC_ACL_E_MAGIC); + CHECK_OBJ_NOTNULL(*r, VCC_ACL_E_MAGIC); + + if ((*l)->not || (*r)->not) + return; + + cmp = vcl_acl_cmp(*l, *r); + + assert(cmp < 0); + if (cmp == ACL_LT) + return; + + do { + switch (cmp) { + case ACL_CONTAINED: + VSB_cat(tl->sb, "ACL entry:\n"); + vcc_ErrWhere(tl, (*r)->t_addr); + VSB_cat(tl->sb, "supersedes / removes:\n"); + vcc_ErrWhere(tl, (*l)->t_addr); + vcc_Warn(tl); + VRBT_REMOVE(acl_tree, &tl->acl->acl_tree, *l); + FREE_OBJ(*l); + *l = VRBT_PREV(acl_tree, &tl->acl->acl_tree, *r); + break; + case ACL_LEFT: + (*l)->mask--; + (*l)->fixed = "folded"; + VSB_cat(tl->sb, "ACL entry:\n"); + vcc_ErrWhere(tl, (*l)->t_addr); + VSB_cat(tl->sb, "left of:\n"); + vcc_ErrWhere(tl, (*r)->t_addr); + VSB_printf(tl->sb, "removing the latter and expanding " + "mask of the former by one to /%d\n", + (*l)->mask - 8); + vcc_Warn(tl); + VRBT_REMOVE(acl_tree, &tl->acl->acl_tree, *r); + FREE_OBJ(*r); + VRBT_REMOVE(acl_tree, &tl->acl->acl_tree, *l); + vcc_acl_insert_entry(tl, l); + return; + default: + INCOMPL(); + } + if (*l == NULL || *r == NULL) + break; + cmp = vcl_acl_cmp(*l, *r); + } while (cmp != ACL_LT); +} + static void vcc_acl_insert_entry(struct vcc *tl, struct acl_e **aenp) { - struct acl_e *ae2; + struct acl_e *ae2, *l, *r; CHECK_OBJ_NOTNULL(*aenp, VCC_ACL_E_MAGIC); ae2 = VRBT_INSERT(acl_tree, &tl->acl->acl_tree, *aenp); @@ -245,7 +313,24 @@ vcc_acl_insert_entry(struct vcc *tl, struct acl_e **aenp) } return; } + + r = *aenp; *aenp = NULL; + + if (tl->acl->flag_fold == 0) + return; + + l = VRBT_PREV(acl_tree, &tl->acl->acl_tree, r); + if (l != NULL) { + vcl_acl_fold(tl, &l, &r); + } + if (r == NULL) + return; + l = r; + r = VRBT_NEXT(acl_tree, &tl->acl->acl_tree, l); + if (r == NULL) + return; + vcl_acl_fold(tl, &l, &r); } static void @@ -749,6 +834,9 @@ vcc_ParseAcl(struct vcc *tl) if (vcc_IdIs(tl->t, "log")) { acl->flag_log = sign; vcc_NextToken(tl); + } else if (vcc_IdIs(tl->t, "fold")) { + acl->flag_fold = sign; + vcc_NextToken(tl); } else if (vcc_IdIs(tl->t, "pedantic")) { acl->flag_pedantic = sign; vcc_NextToken(tl); From nils.goroll at uplex.de Mon Dec 4 11:38:05 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Dec 2023 11:38:05 +0000 (UTC) Subject: [master] 736a5692f Flexelint polish Message-ID: <20231204113805.2224B40EF@lists.varnish-cache.org> commit 736a5692f61a61ca1bf05cfff5a2c5d017f11689 Author: Nils Goroll Date: Mon Dec 4 12:37:04 2023 +0100 Flexelint polish Ref 63e4f71efb9392407863f87ca4a89127d252ecb0 diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c index 07ad6ce30..e80054244 100644 --- a/bin/varnishd/mgt/mgt_param_tweak.c +++ b/bin/varnishd/mgt/mgt_param_tweak.c @@ -611,7 +611,7 @@ static inline void bit_clear(uint8_t *p, unsigned l) { - memset(p, 0, (l + 7) >> 3); + memset(p, 0, ((size_t)l + 7) >> 3); } /*-------------------------------------------------------------------- From nils.goroll at uplex.de Mon Dec 4 11:41:05 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Dec 2023 11:41:05 +0000 (UTC) Subject: [master] 9edc3449c Flexelint polish Message-ID: <20231204114105.1A5814411@lists.varnish-cache.org> commit 9edc3449c7f4224444952067324e27cd0444d7e2 Author: Nils Goroll Date: Mon Dec 4 12:39:41 2023 +0100 Flexelint polish Ref 01bf0dd2b0702588c724db1c0a79373e6f9fe97a diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 3585c3c81..b360fa682 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -280,7 +280,7 @@ vcl_acl_fold(struct vcc *tl, struct acl_e **l, struct acl_e **r) VSB_cat(tl->sb, "left of:\n"); vcc_ErrWhere(tl, (*r)->t_addr); VSB_printf(tl->sb, "removing the latter and expanding " - "mask of the former by one to /%d\n", + "mask of the former by one to /%u\n", (*l)->mask - 8); vcc_Warn(tl); VRBT_REMOVE(acl_tree, &tl->acl->acl_tree, *r); From phk at FreeBSD.org Mon Dec 4 12:01:05 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 4 Dec 2023 12:01:05 +0000 (UTC) Subject: [master] 510bcca23 Disable persistent storage on s390 Message-ID: <20231204120105.C85756295@lists.varnish-cache.org> commit 510bcca2308c0772bd53afbbeed97ab5b142f11b Author: Poul-Henning Kamp Date: Mon Dec 4 11:59:57 2023 +0000 Disable persistent storage on s390 The ASLR is now so aggresive that it never works. diff --git a/autogen.des b/autogen.des index d256a9bcb..606c3b62a 100755 --- a/autogen.des +++ b/autogen.des @@ -22,6 +22,13 @@ else DST="--prefix=/opt/varnish --mandir=/opt/varnish/man" fi +PERSISTENT=--with-persistent-storage + +if [ `uname -m` = "s390x" ] ; then + # ASLR makes this impossible + PERSISTENT= +fi + rm -f $SRCDIR/configure autoreconf -i -v $SRCDIR @@ -34,6 +41,6 @@ $SRCDIR/configure \ --enable-developer-warnings \ --enable-debugging-symbols \ --enable-dependency-tracking \ - --with-persistent-storage \ + ${PERSISTENT} \ --with-contrib \ "$@" From nils.goroll at uplex.de Mon Dec 4 12:30:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Dec 2023 12:30:06 +0000 (UTC) Subject: [master] aac2bc452 Changelog TLC Message-ID: <20231204123006.6DE3F7562@lists.varnish-cache.org> commit aac2bc45262e575cb5bcad7a39f62a49a77a6d21 Author: Nils Goroll Date: Mon Dec 4 13:07:38 2023 +0100 Changelog TLC diff --git a/doc/changes.rst b/doc/changes.rst index 786803a7c..33d9ab3f4 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -41,6 +41,35 @@ Varnish Cache NEXT (2024-03-15) .. PLEASE keep this roughly in commit order as shown by git-log / tig (new to old) +* The ``fold`` flag has been added to Access Control Lists (ACLs) + in VCL. When it is activated with ``acl ... +fold {}``, ACL entries + get optimized in that subnets contained in other entries are skipped + (e.g. if 1.2.3.0/24 is part of the ACL, an entry for 1.2.3.128/25 + will not be added) and adjacent entries get folded (e.g. if both + 1.2.3.0/25 and 1.2.3.128/25 are added, they will be folded to + 1.2.3.0/24) (3563_). + + Logging under the ``VCL_acl`` tag can change with this flag. + + Negated ACL entries are never folded. + +* Fixed handling of failing sub-requests: A VCL failure on the client + side or the ``vcl_req_reset`` feature could trigger a panic, because + it is not allowed to generate a minimal response. For sub-requests, + we now masquerade the fail transition as a deliver and trade the + illegal minimal response for the synthetic response (4022_). + +* The ``param.reset [-j]`` CLI command has been added to reset flags + to their default. Consequently, the ``param.set ... default`` + special value is now deprecated. + +* The ``param.set`` CLI command now supports the ``none`` and ``all`` + values to achieve setting "absolute" values atomically as in + ``param.set foo none,+bar,+baz`` or ``param.set foo all,-bar,-baz``. + +* A glitch in CLI command parsing has been fixed where individually + quoted arguments like ``"help"`` were rejected. + * The ``vcl_req_reset`` feature (controllable through the ``feature`` parameter, see `varnishd(1)`) has been added and enabled by default to terminate client side VCL processing early when the client is @@ -118,6 +147,8 @@ Varnish Cache NEXT (2024-03-15) .. _3984: https://github.com/varnishcache/varnish-cache/issues/3984 .. _3995: https://github.com/varnishcache/varnish-cache/issues/3995 .. _3996: https://github.com/varnishcache/varnish-cache/issues/3996 +.. _4022: https://github.com/varnishcache/varnish-cache/issues/4022 +.. _3563: https://github.com/varnishcache/varnish-cache/pull/3563 .. _3997: https://github.com/varnishcache/varnish-cache/pull/3997 .. _3998: https://github.com/varnishcache/varnish-cache/pull/3998 .. _3999: https://github.com/varnishcache/varnish-cache/pull/3999 From nils.goroll at uplex.de Mon Dec 4 12:30:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Dec 2023 12:30:06 +0000 (UTC) Subject: [master] 90fe6063c Build: Use GCC pragma to suppress y2k warning Message-ID: <20231204123006.84F967566@lists.varnish-cache.org> commit 90fe6063c61b79b6585077c91a598ebdeaa23e98 Author: Nils Goroll Date: Mon Dec 4 13:26:38 2023 +0100 Build: Use GCC pragma to suppress y2k warning The problem with 93974b3f9f51c5591345b0fc37e51437393659f3 is that the negating option from Makefile.am comes before CFLAGS, which does not disable it for gcc. Tested locally with gcc and clang. diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index aa88f486a..946966cf9 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -82,5 +82,5 @@ vsb_test_CFLAGS = $(AM_CFLAGS) -DVSB_TEST vsb_test_LDADD = $(AM_LDFLAGS) libvarnish.la vtim_test_SOURCES = vtim.c -vtim_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER -Wno-error=format-y2k +vtim_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER vtim_test_LDADD = $(AM_LDFLAGS) libvarnish.la diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 31a216ca6..2d3569e86 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -456,6 +456,8 @@ VTIM_timespec(vtim_dur t) #ifdef TEST_DRIVER +#pragma GCC diagnostic ignored "-Wformat-y2k" + #include static void From dridi at varni.sh Mon Dec 4 12:38:03 2023 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 12:38:03 +0000 Subject: [master] 90fe6063c Build: Use GCC pragma to suppress y2k warning In-Reply-To: <20231204123006.84F967566@lists.varnish-cache.org> References: <20231204123006.84F967566@lists.varnish-cache.org> Message-ID: On Mon, Dec 4, 2023 at 12:30?PM Nils Goroll wrote: > > > commit 90fe6063c61b79b6585077c91a598ebdeaa23e98 > Author: Nils Goroll > Date: Mon Dec 4 13:26:38 2023 +0100 > > Build: Use GCC pragma to suppress y2k warning Thanks for taking care of it. > The problem with 93974b3f9f51c5591345b0fc37e51437393659f3 is that the > negating option from Makefile.am comes before CFLAGS, which does not > disable it for gcc. It did on my machine, it turned the error into a simple warning. > Tested locally with gcc and clang. > > diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am > index aa88f486a..946966cf9 100644 > --- a/lib/libvarnish/Makefile.am > +++ b/lib/libvarnish/Makefile.am > @@ -82,5 +82,5 @@ vsb_test_CFLAGS = $(AM_CFLAGS) -DVSB_TEST > vsb_test_LDADD = $(AM_LDFLAGS) libvarnish.la > > vtim_test_SOURCES = vtim.c > -vtim_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER -Wno-error=format-y2k > +vtim_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER > vtim_test_LDADD = $(AM_LDFLAGS) libvarnish.la > diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c > index 31a216ca6..2d3569e86 100644 > --- a/lib/libvarnish/vtim.c > +++ b/lib/libvarnish/vtim.c > @@ -456,6 +456,8 @@ VTIM_timespec(vtim_dur t) > > #ifdef TEST_DRIVER > > +#pragma GCC diagnostic ignored "-Wformat-y2k" > + > #include > > static void > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at FreeBSD.org Mon Dec 4 12:50:06 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 4 Dec 2023 12:50:06 +0000 (UTC) Subject: [master] 90f4f1494 Fix cast for %jd Message-ID: <20231204125006.4567E9534@lists.varnish-cache.org> commit 90f4f1494cd31976cf38f6a48f01c3a44445f4dc Author: Poul-Henning Kamp Date: Mon Dec 4 12:48:51 2023 +0000 Fix cast for %jd diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 2d3569e86..1894fa0ee 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -561,8 +561,8 @@ bench(void) s = VTIM_mono(); for (i=0; i<100000; i++) { snprintf(buf, sizeof(buf), "%ju.%06ju", - (uint64_t)floor(s), - (uint64_t)floor((s * 1e6)) % 1000000UL); + (uintmax_t)floor(s), + (uintmax_t)floor((s * 1e6)) % 1000000UL); t_i += buf[4]; } e = VTIM_mono(); From nils.goroll at uplex.de Mon Dec 4 13:13:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Dec 2023 13:13:06 +0000 (UTC) Subject: [master] 8151a2daa Generalize ignored test dropping patterns Message-ID: <20231204131306.488A8653B9@lists.varnish-cache.org> commit 8151a2daae9cb75f2fc5490ae8876ddad74d9b7d Author: Nils Goroll Date: Mon Dec 4 14:11:23 2023 +0100 Generalize ignored test dropping patterns and, in particular, ignore lib/libvarnish/vtim_test too diff --git a/.gitignore b/.gitignore index 2e8612b52..a6eb65683 100644 --- a/.gitignore +++ b/.gitignore @@ -113,19 +113,12 @@ cscope.*out /nbproject/private/ # Test droppings -/bin/varnishd/vhp_decode_test -/bin/varnishd/vhp_table_test +/bin/varnishd/*_test /bin/varnishtest/tests/*.log-t /include/vrt_test* /include/vbm_test -/lib/libvarnish/vav_test -/lib/libvarnish/vbh_test -/lib/libvarnish/vct_test -/lib/libvarnish/vjsn_test -/lib/libvarnish/vnum_c_test -/lib/libvarnish/vsb_test -/lib/libvarnishapi/vsl_glob_test -/lib/libvarnishapi/vxp_test +/lib/libvarnish/*_test +/lib/libvarnishapi/*_test # GCOV droppings *.gcda From nils.goroll at uplex.de Mon Dec 4 15:33:05 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Dec 2023 15:33:05 +0000 (UTC) Subject: [master] 4dd951312 Adjust vtim.c test code to 32bit time_t Message-ID: <20231204153305.972F4103B36@lists.varnish-cache.org> commit 4dd951312f1f0f2da546a51deeb95508a8b493db Author: Nils Goroll Date: Mon Dec 4 16:30:41 2023 +0100 Adjust vtim.c test code to 32bit time_t Fixes vtest error on SunOS gcc4.7.3 -m32: vtim.c: In function 'main': vtim.c:605:2: error: overflow in implicit constant conversion [-Werror=overflow] vtim.c:605:2: error: comparison is always true due to limited range of data type [-Werror=type-limits] diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 1894fa0ee..8f89fae58 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -585,6 +585,9 @@ parse_check(time_t t, const char *s) } } +#define TTEST_MIN (sizeof(time_t) >= 8 ? -2209852800 : INT32_MIN) +#define TTEST_MAX (sizeof(time_t) >= 8 ? 20000000000 : INT32_MAX) + int main(int argc, char **argv) { @@ -602,7 +605,7 @@ main(int argc, char **argv) bench(); /* Brute force test against libc version */ - for (t = -2209852800; t < 20000000000; t += 3599) { + for (t = TTEST_MIN; t < TTEST_MAX; t += 3599) { gmtime_r(&t, &tm); strftime(buf1, sizeof buf1, "%a, %d %b %Y %T GMT", &tm); VTIM_format(t, buf); From dridi.boukelmoune at gmail.com Mon Dec 4 16:36:07 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 16:36:07 +0000 (UTC) Subject: [master] 2d2fc8990 vte: Don't count '\t' and '\n' delimiters Message-ID: <20231204163607.51FE0105B6E@lists.varnish-cache.org> commit 2d2fc8990c99641cf460dded11179d2a2edb4c5d Author: Dridi Boukelmoune Date: Mon Dec 4 17:19:47 2023 +0100 vte: Don't count '\t' and '\n' delimiters Since the maximum space between fields is 3, it should have been obvious that something was off while inspecting test cases. Before this fix, the fields were aligned, but sometimes the space between fields was too wide and in particular before right-aligned fields. diff --git a/bin/varnishtest/tests/r02471.vtc b/bin/varnishtest/tests/r02471.vtc index 994480af5..a9e30c92f 100644 --- a/bin/varnishtest/tests/r02471.vtc +++ b/bin/varnishtest/tests/r02471.vtc @@ -23,7 +23,7 @@ varnish v1 -vcl+backend {} varnish v1 -cliok "vcl.state vcl1 cold" # Nothing holds vcl1, so it should go gold. -varnish v1 -cliexpect "cold cold 0 vcl1" "vcl.list" +varnish v1 -cliexpect "cold cold 0 vcl1" "vcl.list" # Grab hold of vcl1 @@ -43,7 +43,7 @@ client c1 { # There should still be a single busy hold on vcl1 varnish v1 -cliok "vcl.state vcl1 cold" -varnish v1 -cliexpect "cold busy [12] vcl1" "vcl.list" +varnish v1 -cliexpect "cold busy [12] vcl1" "vcl.list" # Release hold on vcl1 varnish v1 -cliok "vcl.state vcl1 auto" @@ -60,6 +60,6 @@ client c1 { rxresp } -run -# Nothing holds vcl1, so it should go gold. +# Nothing holds vcl1, so it should go cold. varnish v1 -cliok "vcl.state vcl1 cold" -varnish v1 -cliexpect "cold .... [01] vcl1" "vcl.list" +varnish v1 -cliexpect "cold .... [01] vcl1" "vcl.list" diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc index d3baa09d6..63df292be 100644 --- a/bin/varnishtest/tests/u00000.vtc +++ b/bin/varnishtest/tests/u00000.vtc @@ -108,7 +108,7 @@ shell -expect {VCL 'vcl1' now active} { varnishadm -n ${tmpdir}/v1 vcl.use vcl1 } -shell -expect {active auto warm - vcl1} { +shell -expect {active auto warm - vcl1} { varnishadm -n ${tmpdir}/v1 vcl.list } @@ -152,8 +152,8 @@ shell { } varnish v2 -arg "-f ${tmpdir}/ok1" -arg "-f ${tmpdir}/ok2" -start -varnish v2 -cliexpect {available auto warm 0 boot0} "vcl.list" -varnish v2 -cliexpect {active auto warm 0 boot1} "vcl.list" +varnish v2 -cliexpect {available auto warm 0 boot0} "vcl.list" +varnish v2 -cliexpect {active auto warm 0 boot1} "vcl.list" varnish v2 -stop -wait # Test multiple -f options with a bad VCL diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc index 42152f824..57273fb39 100644 --- a/bin/varnishtest/tests/u00011.vtc +++ b/bin/varnishtest/tests/u00011.vtc @@ -24,7 +24,7 @@ process p1 -expect-text 12 1 "200 " process p1 -expect-text 13 1 PONG process p1 -write "vcl.li\t\r" -process p1 -expect-text 0 1 "active auto warm" +process p1 -expect-text 0 1 "active auto warm" process p1 -write "vcl.s\t\th\t vcl1\r" process p1 -expect-text 0 1 "backend s1" diff --git a/bin/varnishtest/tests/u00012.vtc b/bin/varnishtest/tests/u00012.vtc index 3f4a19ec6..cecf70e9f 100644 --- a/bin/varnishtest/tests/u00012.vtc +++ b/bin/varnishtest/tests/u00012.vtc @@ -22,7 +22,7 @@ process p1 -expect-text 2 1 "200 19 " process p1 -expect-text 3 1 PONG process p1 -write "vcl.list\r" -process p1 -expect-text 0 0 "auto warm" +process p1 -expect-text 0 0 "auto warm" process p1 -write "vcl.show vcl1\r" process p1 -expect-text 0 0 "backend s1" diff --git a/bin/varnishtest/tests/v00045.vtc b/bin/varnishtest/tests/v00045.vtc index 812c0392a..795d25436 100644 --- a/bin/varnishtest/tests/v00045.vtc +++ b/bin/varnishtest/tests/v00045.vtc @@ -17,7 +17,7 @@ varnish v1 -cliok "vcl.state vcl1 cold" # We should now see it as cooling delay 1 -varnish v1 -cliexpect "available cold cooling 0 vcl1" vcl.list +varnish v1 -cliexpect "available cold cooling 0 vcl1" vcl.list varnish v1 -clijson "vcl.list -j" # It can't be warmed up yet @@ -26,7 +26,7 @@ varnish v1 -cliexpect "vmod-debug ref on vcl1" "vcl.state vcl1 warm" # It will eventually cool down delay 2 -varnish v1 -cliexpect "available cold cold 0 vcl1" vcl.list +varnish v1 -cliexpect "available cold cold 0 vcl1" vcl.list varnish v1 -clijson "vcl.list -j" # At this point it becomes possible to warm up again diff --git a/lib/libvarnish/vte.c b/lib/libvarnish/vte.c index c97a95441..cd2620251 100644 --- a/lib/libvarnish/vte.c +++ b/lib/libvarnish/vte.c @@ -138,7 +138,7 @@ vte_update(struct vte *vte) vte->f_off = 0; vte->f_sz = 0; vte->l_sz = 0; - } else { + } else if (*p != '\t') { vte->f_sz++; vte->l_sz++; } @@ -261,9 +261,9 @@ VTE_format(const struct vte *vte, VTE_format_f *func, void *priv) just_left = 0; while (*p != 0) { if (*p == '\v') { - if (p - 1 > q) { /* exclude previous separator */ + if (p > q) { VTE_FORMAT(func, priv, "%.*s%s", - (int)((p - 1) - q), q, sep); + (int)(p - q), q, sep); } q = ++p; just_left = 1; From dridi.boukelmoune at gmail.com Mon Dec 4 16:36:07 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 16:36:07 +0000 (UTC) Subject: [master] 4e386aeff mgt_vcl: Retire artificial formatting from vcl.list Message-ID: <20231204163607.904FC105B72@lists.varnish-cache.org> commit 4e386aefff035bb1fe533d53071262a663665cb8 Author: Dridi Boukelmoune Date: Mon Dec 4 17:25:57 2023 +0100 mgt_vcl: Retire artificial formatting from vcl.list It is the VTE's job to deal with alignment now. diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 882c828aa..9c4e01e9c 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -849,7 +849,7 @@ mcf_vcl_list(struct cli *cli, const char * const *av, void *priv) vp == mgt_vcl_active ? "active" : "available"); vs = vp->warm ? VCL_STATE_WARM : VCL_STATE_COLD; VTE_printf(vte, "\t%s\t%s", vp->state->name, vs->name); - VTE_printf(vte, "\t%6s\t%s", "-", vp->name); + VTE_printf(vte, "\t-\t%s", vp->name); if (mcf_is_label(vp)) { vd = VTAILQ_FIRST(&vp->dfrom); AN(vd); diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc index 63df292be..8b1e9de32 100644 --- a/bin/varnishtest/tests/u00000.vtc +++ b/bin/varnishtest/tests/u00000.vtc @@ -108,7 +108,7 @@ shell -expect {VCL 'vcl1' now active} { varnishadm -n ${tmpdir}/v1 vcl.use vcl1 } -shell -expect {active auto warm - vcl1} { +shell -expect {active auto warm - vcl1} { varnishadm -n ${tmpdir}/v1 vcl.list } From dridi.boukelmoune at gmail.com Mon Dec 4 16:36:07 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 16:36:07 +0000 (UTC) Subject: [master] 282c49525 vte: Add a basic test case covering several facets Message-ID: <20231204163607.B6290105B75@lists.varnish-cache.org> commit 282c4952570682aad1556d06c6df8d523f14ab55 Author: Dridi Boukelmoune Date: Mon Dec 4 17:27:15 2023 +0100 vte: Add a basic test case covering several facets diff --git a/lib/libvarnish/vte.c b/lib/libvarnish/vte.c index cd2620251..f27598feb 100644 --- a/lib/libvarnish/vte.c +++ b/lib/libvarnish/vte.c @@ -300,3 +300,85 @@ VTE_format(const struct vte *vte, VTE_format_f *func, void *priv) VTE_FORMAT(func, priv, "%s", q); return (0); } + +#ifdef TEST_DRIVER + +#include + +static const char *test_vte = + "name\tref\tcomment\n" + "foo\t\v1\tthe foo\n" + "bar\t\v10\tthe bars\n" + "baz\t\v0\t\n" + "qux\t\v-1\tno eol"; + +static const char *test_fmt = + "name ref comment\n" + "foo 1 the foo\n" + "bar 10 the bars\n" + "baz 0 \n" + "qux -1 no eol"; + +static int +test_vsb_format(void *priv, const char *fmt, ...) +{ + struct vsb *vsb; + va_list ap; + int res; + + CAST_OBJ_NOTNULL(vsb, priv, VSB_MAGIC); + AN(fmt); + + va_start(ap, fmt); + res = VSB_vprintf(vsb, fmt, ap); + va_end(ap); + + return (res); +} + +int +main(int argc, char **argv) +{ + struct vte *vte; + struct vsb *vsb; + int err = 0; + + (void)argc; + (void)argv; + + vte = VTE_new(3, 20); + AN(vte); + AZ(VTE_cat(vte, test_vte)); + AZ(VTE_finish(vte)); + + vsb = VSB_new_auto(); + AN(vsb); + AZ(VTE_format(vte, test_vsb_format, vsb)); + AZ(VSB_finish(vsb)); + + assert(vte->o_sep == 2); + assert(vte->f_maxsz[0] == 4); + assert(vte->f_maxsz[1] == 3); + assert(vte->f_maxsz[2] == 8); + + if (strcmp(VSB_data(vsb), test_fmt)) { + fprintf(stderr, + "Error: VTE output mismatch\n" + "<<<<<<<\n" + "%s\n" + "=======\n" + "%s\n" + ">>>>>>>\n" + "FAIL\n", + VSB_data(vsb), test_fmt); + err = 1; + } + + VSB_destroy(&vsb); + VTE_destroy(&vte); + if (!err) + printf("PASS\n"); + return (err); +} + +#endif From dridi.boukelmoune at gmail.com Mon Dec 4 16:36:07 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 16:36:07 +0000 (UTC) Subject: [master] 2bf97ffac build: Add vte_test to the test suite Message-ID: <20231204163607.E6D83105B7B@lists.varnish-cache.org> commit 2bf97ffacacf0d597fd138b512de9ca04adb8911 Author: Dridi Boukelmoune Date: Mon Dec 4 17:28:30 2023 +0100 build: Add vte_test to the test suite diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index 946966cf9..d06f1b5b2 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -53,6 +53,7 @@ TESTS = \ vjsn_test \ vnum_c_test \ vsb_test \ + vte_test \ vtim_test noinst_PROGRAMS = ${TESTS} @@ -81,6 +82,10 @@ vsb_test_SOURCES = vsb_test.c vsb_test_CFLAGS = $(AM_CFLAGS) -DVSB_TEST vsb_test_LDADD = $(AM_LDFLAGS) libvarnish.la +vte_test_SOURCES = vte.c +vte_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER +vte_test_LDADD = $(AM_LDFLAGS) libvarnish.la + vtim_test_SOURCES = vtim.c vtim_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER vtim_test_LDADD = $(AM_LDFLAGS) libvarnish.la From dridi.boukelmoune at gmail.com Mon Dec 4 17:00:09 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 17:00:09 +0000 (UTC) Subject: [master] 1ab71cf18 h2: the :scheme pseudo header is not optional Message-ID: <20231204170009.A8ACA106CA9@lists.varnish-cache.org> commit 1ab71cf18b448478ebe16bd7a7fa78fe6d5e0dae Author: Asad Sajjad Ahmed Date: Wed Sep 28 14:58:38 2022 +0200 h2: the :scheme pseudo header is not optional The :scheme pseudo header is not optional in H/2 except when doing CONNECT. There is also a strict requirement for it appear only once. Signed-off-by: Asad Sajjad Ahmed Conflicts: bin/varnishtest/tests/t02025.vtc diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index ca2e65993..61d0a1d78 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -218,11 +218,14 @@ vtr_deliver_f h2_deliver; vtr_minimal_response_f h2_minimal_response; #endif /* TRANSPORT_MAGIC */ +#define H2H_DECODE_FLAG_SCHEME_SEEN 0x1 + /* http2/cache_http2_hpack.c */ struct h2h_decode { unsigned magic; #define H2H_DECODE_MAGIC 0xd092bde4 + int flags; h2_error error; enum vhd_ret_e vhd_ret; char *out; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index b5451eb22..472b86294 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -127,7 +127,7 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } static h2_error -h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) +h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) { /* XXX: This might belong in cache/cache_http.c */ const char *b0; @@ -188,9 +188,18 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal header, stripping the first ':'. */ + if (*flags & H2H_DECODE_FLAG_SCHEME_SEEN) { + VSLb(hp->vsl, SLT_BogoHeader, + "Duplicate pseudo-header %.*s%.*s", + (int)namelen, b0, + (int)(len > 20 ? 20 : len), b); + return (H2SE_PROTOCOL_ERROR); + } + b++; len-=1; n = hp->nhd; + *flags |= H2H_DECODE_FLAG_SCHEME_SEEN; for (p = b + namelen, u = 0; u < len-namelen; p++, u++) { @@ -380,7 +389,8 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->out_u); if (d->error) break; - d->error = h2h_addhdr(hp, d->out, d->namelen, d->out_u); + d->error = h2h_addhdr(hp, d->out, d->namelen, d->out_u, + &d->flags); if (d->error) break; d->out[d->out_u++] = '\0'; /* Zero guard */ diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index edcc709d9..a474020b2 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -601,11 +601,13 @@ static h2_error h2_end_headers(struct worker *wrk, struct h2_sess *h2, struct req *req, struct h2_req *r2) { + int scheme_seen; h2_error h2e; ssize_t cl; ASSERT_RXTHR(h2); assert(r2->state == H2_S_OPEN); + scheme_seen = h2->decode->flags & H2H_DECODE_FLAG_SCHEME_SEEN; h2e = h2h_decode_fini(h2); h2->new_req = NULL; if (h2e != NULL) { @@ -656,10 +658,17 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, VSLb(h2->vsl, SLT_Debug, "Missing :method"); return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 } + if (req->http->hd[HTTP_HDR_URL].b == NULL) { VSLb(h2->vsl, SLT_Debug, "Missing :path"); return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 } + + if (!(scheme_seen)) { + VSLb(h2->vsl, SLT_Debug, "Missing :scheme"); + return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 + } + AN(req->http->hd[HTTP_HDR_PROTO].b); if (*req->http->hd[HTTP_HDR_URL].b == '*' && diff --git a/bin/varnishtest/tests/t02026.vtc b/bin/varnishtest/tests/t02026.vtc new file mode 100644 index 000000000..b464f8cdc --- /dev/null +++ b/bin/varnishtest/tests/t02026.vtc @@ -0,0 +1,48 @@ +varnishtest "Dublicate pseudo-headers" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -arg "-p feature=+http2" -vcl+backend { +} -start + +#client c1 { +# txreq -url "/some/path" -url "/some/other/path" +# rxresp +# expect resp.status == 400 +#} -run + +#client c1 { +# txreq -req "GET" -req "POST" +# rxresp +# expect resp.status == 400 +#} -run + +#client c1 { +# txreq -proto "HTTP/1.1" -proto "HTTP/2.0" +# rxresp +# expect resp.status == 400 +#} -run + +client c1 { + stream 1 { + txreq -url "/some/path" -url "/some/other/path" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -scheme "http" -scheme "https" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -req "GET" -req "POST" + rxrst + } -run +} -run From dridi.boukelmoune at gmail.com Mon Dec 4 17:55:07 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 17:55:07 +0000 (UTC) Subject: [master] ca22c744e h2: Manage missing :scheme as a custom error Message-ID: <20231204175507.45BDC108995@lists.varnish-cache.org> commit ca22c744e9badc9df7ba360b8ab4caebe90448d9 Author: Dridi Boukelmoune Date: Mon Dec 4 18:30:06 2023 +0100 h2: Manage missing :scheme as a custom error There is room for further improvement in the dynamic between HPACK and the HTTP/2 session, but this will serve as the first step. diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 61d0a1d78..f73b5657e 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -218,14 +218,12 @@ vtr_deliver_f h2_deliver; vtr_minimal_response_f h2_minimal_response; #endif /* TRANSPORT_MAGIC */ -#define H2H_DECODE_FLAG_SCHEME_SEEN 0x1 - /* http2/cache_http2_hpack.c */ struct h2h_decode { unsigned magic; #define H2H_DECODE_MAGIC 0xd092bde4 - int flags; + unsigned has_scheme:1; h2_error error; enum vhd_ret_e vhd_ret; char *out; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 472b86294..daf4b93c5 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -127,7 +127,8 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } static h2_error -h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) +h2h_addhdr(struct h2h_decode *d, struct http *hp, char *b, size_t namelen, + size_t len) { /* XXX: This might belong in cache/cache_http.c */ const char *b0; @@ -188,7 +189,7 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal header, stripping the first ':'. */ - if (*flags & H2H_DECODE_FLAG_SCHEME_SEEN) { + if (d->has_scheme) { VSLb(hp->vsl, SLT_BogoHeader, "Duplicate pseudo-header %.*s%.*s", (int)namelen, b0, @@ -199,7 +200,7 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) b++; len-=1; n = hp->nhd; - *flags |= H2H_DECODE_FLAG_SCHEME_SEEN; + d->has_scheme = 1; for (p = b + namelen, u = 0; u < len-namelen; p++, u++) { @@ -302,6 +303,9 @@ h2h_decode_fini(const struct h2_sess *h2) VSLb(h2->new_req->http->vsl, SLT_BogoHeader, "HPACK compression error/fini (%s)", VHD_Error(d->vhd_ret)); ret = H2CE_COMPRESSION_ERROR; + } else if (d->error == NULL && !d->has_scheme) { + VSLb(h2->vsl, SLT_Debug, "Missing :scheme"); + ret = H2SE_MISSING_SCHEME; //rfc7540,l,3087,3090 } else ret = d->error; FINI_OBJ(d); @@ -389,8 +393,8 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->out_u); if (d->error) break; - d->error = h2h_addhdr(hp, d->out, d->namelen, d->out_u, - &d->flags); + d->error = h2h_addhdr(d, hp, d->out, + d->namelen, d->out_u); if (d->error) break; d->out[d->out_u++] = '\0'; /* Zero guard */ diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index a474020b2..afdbdeebe 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -601,13 +601,11 @@ static h2_error h2_end_headers(struct worker *wrk, struct h2_sess *h2, struct req *req, struct h2_req *r2) { - int scheme_seen; h2_error h2e; ssize_t cl; ASSERT_RXTHR(h2); assert(r2->state == H2_S_OPEN); - scheme_seen = h2->decode->flags & H2H_DECODE_FLAG_SCHEME_SEEN; h2e = h2h_decode_fini(h2); h2->new_req = NULL; if (h2e != NULL) { @@ -664,11 +662,6 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 } - if (!(scheme_seen)) { - VSLb(h2->vsl, SLT_Debug, "Missing :scheme"); - return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 - } - AN(req->http->hd[HTTP_HDR_PROTO].b); if (*req->http->hd[HTTP_HDR_URL].b == '*' && diff --git a/include/tbl/h2_error.h b/include/tbl/h2_error.h index c6c6bed89..345e1549b 100644 --- a/include/tbl/h2_error.h +++ b/include/tbl/h2_error.h @@ -156,6 +156,13 @@ H2_ERROR( /* descr */ "http/2 rapid reset detected" ) +H2_ERROR( + /* name */ MISSING_SCHEME, + /* val */ 1, /* PROTOCOL_ERROR */ + /* types */ 2, + /* reason */ SC_NULL, + /* descr */ "Missing :scheme pseudo-header" +) # undef H2_CUSTOM_ERRORS #endif From dridi.boukelmoune at gmail.com Mon Dec 4 17:55:07 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 17:55:07 +0000 (UTC) Subject: [master] 2ee000602 hash: Retire unused HSH_CONTINUE enum Message-ID: <20231204175507.61A43108997@lists.varnish-cache.org> commit 2ee0006020bb668addfb8ccbd153560b5034dc26 Author: Dridi Boukelmoune Date: Mon Oct 2 13:25:29 2023 +0200 hash: Retire unused HSH_CONTINUE enum diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index a1a9c0e82..c73147544 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -52,7 +52,6 @@ struct hash_slinger { }; enum lookup_e { - HSH_CONTINUE, HSH_MISS, HSH_BUSY, HSH_HIT, From dridi.boukelmoune at gmail.com Mon Dec 4 17:55:07 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 4 Dec 2023 17:55:07 +0000 (UTC) Subject: [master] 6b04b7da6 hash: Move enum lookup_e to cache_objhead.h Message-ID: <20231204175507.8C00D10899B@lists.varnish-cache.org> commit 6b04b7da656b7baceaa41a9e5aab1a4741979a35 Author: Dridi Boukelmoune Date: Mon Oct 2 13:35:49 2023 +0200 hash: Move enum lookup_e to cache_objhead.h It has nothing to do with the hash lookup implementation, this is the outcome of a cache lookup. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 027b617f5..55c14f1d4 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -34,7 +34,6 @@ #include "cache_varnishd.h" #include "cache_filter.h" #include "cache_objhead.h" -#include "hash/hash_slinger.h" #include "storage/storage.h" #include "vcl.h" #include "vtim.h" diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h index bc1782379..58b536502 100644 --- a/bin/varnishd/cache/cache_objhead.h +++ b/bin/varnishd/cache/cache_objhead.h @@ -56,6 +56,15 @@ struct objhead { #define hoh_head _u.n.u_n_hoh_head }; +enum lookup_e { + HSH_MISS, + HSH_HITMISS, + HSH_HITPASS, + HSH_HIT, + HSH_GRACE, + HSH_BUSY, +}; + void HSH_Fail(struct objcore *); void HSH_Kill(struct objcore *); void HSH_Insert(struct worker *, const void *hash, struct objcore *, diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index bca871765..3cbd62734 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -41,7 +41,6 @@ #include "vtim.h" #include "storage/storage.h" -#include "hash/hash_slinger.h" /*---------------------------------------------------------------------- * Pull the req.body in via/into a objcore diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index c0508cd4c..aa0a0db98 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -46,7 +46,6 @@ #include "cache_transport.h" #include "vcc_interface.h" -#include "hash/hash_slinger.h" #include "http1/cache_http1.h" #include "storage/storage.h" #include "vcl.h" diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h index c73147544..1f0e980a7 100644 --- a/bin/varnishd/hash/hash_slinger.h +++ b/bin/varnishd/hash/hash_slinger.h @@ -51,15 +51,6 @@ struct hash_slinger { hash_deref_f *deref; }; -enum lookup_e { - HSH_MISS, - HSH_BUSY, - HSH_HIT, - HSH_HITMISS, - HSH_HITPASS, - HSH_GRACE -}; - /* mgt_hash.c */ void HSH_config(const char *); From nils.goroll at uplex.de Tue Dec 5 07:14:05 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Dec 2023 07:14:05 +0000 (UTC) Subject: [master] 7bc5cf683 Fix vtest reporting when build dir is just _build/, not _build/sub/ Message-ID: <20231205071405.99A1D6C10@lists.varnish-cache.org> commit 7bc5cf683c54bb3457c927e0b8cb3695a0946aa6 Author: Nils Goroll Date: Tue Dec 5 08:10:14 2023 +0100 Fix vtest reporting when build dir is just _build/, not _build/sub/ See "cope with older automake" a couple of lines up Should fix the issue seen with Solaris vtesters. bash -x output: + vtc=./varnish-trunk/_build/bin/varnishtest/tests/r02686.vtc ++ echo ./varnish-trunk/_build/bin/varnishtest/tests/r02686.trs ++ sed -e 's/trs$/log/' + logfile=./varnish-trunk/_build/bin/varnishtest/tests/r02686.log + log=r02686.log ++ git log -n 1 --pretty=format:%H ./varnish-trunk/_build/bin/varnishtest/tests/r02686.vtc fatal: ambiguous argument './varnish-trunk/_build/bin/varnishtest/tests/r02686.vtc': unknown revision or path not in the working tree. Use '--' to separate paths from revisions, like this: 'git [...] -- [...]' + rev= diff --git a/tools/vtest.sh b/tools/vtest.sh index 00620502f..bf6c37a8a 100755 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -194,7 +194,7 @@ failedtests () ( while read trs do name=`basename "${trs}" .trs` - vtc=`echo $trs | sed -e 's/trs$/vtc/' -e 's/.*sub\///' ` + vtc=`echo $trs | sed -e 's/trs$/vtc/' -e 's/.*_build\/$sub\/$\?//'` logfile=`echo $trs | sed -e 's/trs$/log/'` log="${name}.log" rev=`git log -n 1 --pretty=format:%H "${vtc}"` From phk at FreeBSD.org Tue Dec 5 07:59:06 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Dec 2023 07:59:06 +0000 (UTC) Subject: [master] d7a9386e8 Yet an attempt to fix vtim_iter on 32 bit Message-ID: <20231205075906.1D0CC1000BF@lists.varnish-cache.org> commit d7a9386e8d7263ec3a405a34dcef01d46475b14a Author: Poul-Henning Kamp Date: Tue Dec 5 07:57:52 2023 +0000 Yet an attempt to fix vtim_iter on 32 bit diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 8f89fae58..8a4aade1a 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -592,6 +592,7 @@ int main(int argc, char **argv) { time_t t; + intmax_t iter; struct tm tm; char buf[BUFSIZ]; char buf1[BUFSIZ]; @@ -605,7 +606,8 @@ main(int argc, char **argv) bench(); /* Brute force test against libc version */ - for (t = TTEST_MIN; t < TTEST_MAX; t += 3599) { + for (iter = TTEST_MIN; iter < TTEST_MAX; iter += 3599) { + t = (time_t)iter; gmtime_r(&t, &tm); strftime(buf1, sizeof buf1, "%a, %d %b %Y %T GMT", &tm); VTIM_format(t, buf); From phk at FreeBSD.org Tue Dec 5 08:41:05 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Dec 2023 08:41:05 +0000 (UTC) Subject: [master] 8df29b6b6 Make vtest work with non-vtc test-fails Message-ID: <20231205084105.47C56101FD7@lists.varnish-cache.org> commit 8df29b6b6145d9484c92bf961ee769093c8fe90f Author: Poul-Henning Kamp Date: Tue Dec 5 08:40:21 2023 +0000 Make vtest work with non-vtc test-fails diff --git a/tools/vtest.sh b/tools/vtest.sh index bf6c37a8a..1d82f65a3 100755 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -194,10 +194,14 @@ failedtests () ( while read trs do name=`basename "${trs}" .trs` - vtc=`echo $trs | sed -e 's/trs$/vtc/' -e 's/.*_build\/$sub\/$\?//'` + vtc=`echo $trs | sed -E -e 's/trs$/vtc/' -e 's/.*_build\/(sub\/)?//'` logfile=`echo $trs | sed -e 's/trs$/log/'` log="${name}.log" - rev=`git log -n 1 --pretty=format:%H "${vtc}"` + if [ -f ${vtc} ] ; then + rev=`git log -n 1 --pretty=format:%H "${vtc}"` + else + rev="?" + fi cp "${logfile}" "${REPORTDIR}/_${log}" echo "VTCGITREV ${name} ${rev}" echo "MANIFEST _${log}" From phk at FreeBSD.org Tue Dec 5 08:52:06 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Dec 2023 08:52:06 +0000 (UTC) Subject: [master] 9ef2b77c6 Make vtim_test not assert on 32 bit systems. Message-ID: <20231205085206.194AF1026E1@lists.varnish-cache.org> commit 9ef2b77c66a74e9f05bccff7164fd9add6847163 Author: Poul-Henning Kamp Date: Tue Dec 5 08:51:35 2023 +0000 Make vtim_test not assert on 32 bit systems. diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 8a4aade1a..3918e5eb2 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -601,7 +601,6 @@ main(int argc, char **argv) (void)argv; AZ(setenv("TZ", "UTC", 1)); - assert(sizeof t >= 8); bench(); From dridi.boukelmoune at gmail.com Tue Dec 5 10:14:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Dec 2023 10:14:05 +0000 (UTC) Subject: [master] 89c1688f8 Call trimstore only once when copying the body after a 304 Message-ID: <20231205101405.7D9A010609C@lists.varnish-cache.org> commit 89c1688f8845f7630dcc0955ed748861156d74c2 Author: Nils Goroll Date: Tue Oct 31 13:46:37 2023 +0100 Call trimstore only once when copying the body after a 304 It is my understanding that the objtrimstore stevedore API function is only to be called once when the object is complete, which I believe is also in line with the comment on ObjExtend() that "The final flag must be set on the last call". If this understanding of the API is correct, we did not adhere to it in the fetch code when we made a copy of an existing stale object after a 304 response: There, we iterated over the stale object and did not set the final flag just once when the object was complete, but rather after each storage segment was copied. This commit fixes this, adds some pedentry to the simple storage and extends b00062.vtc to test this behavior specifically. On top, g6.vtc also triggered without the fix but the duplicate trim detection in place. This issue has originally surfaced in the SLASH/fellow storage where the trimstore function implicitly asserted to only be called once. Ref 115742b07c8bad6d465f1c981ee264f934a4492b Ref https://gitlab.com/uplex/varnish/slash/-/issues/33 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 55c14f1d4..e365f6c88 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -769,7 +769,8 @@ vbf_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) uint8_t *pd; CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC); - (void)flush; + + flush &= OBJ_ITER_END; while (len > 0) { l = len; @@ -777,7 +778,7 @@ vbf_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) return (1); l = vmin(l, len); memcpy(pd, ps, l); - VFP_Extend(bo->vfc, l, l == len ? VFP_END : VFP_OK); + VFP_Extend(bo->vfc, l, flush && l == len ? VFP_END : VFP_OK); ps += l; len -= l; } diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index c94dcccb4..b91fbcb32 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -44,6 +44,9 @@ /* Flags for allocating memory in sml_stv_alloc */ #define LESS_MEM_ALLOCED_IS_OK 1 +// marker pointer for sml_trimstore +static void *trim_once = &trim_once; + /*-------------------------------------------------------------------*/ static struct storage * @@ -257,7 +260,8 @@ sml_bocfini(const struct stevedore *stv, struct boc *boc) CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); CHECK_OBJ_NOTNULL(boc, BOC_MAGIC); - if (boc->stevedore_priv == NULL) + if (boc->stevedore_priv == NULL || + boc->stevedore_priv == trim_once) return; /* Free any leftovers from Trim */ @@ -533,6 +537,10 @@ sml_trimstore(struct worker *wrk, struct objcore *oc) stv = oc->stobj->stevedore; CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); + if (oc->boc->stevedore_priv != NULL) + WRONG("sml_trimstore already called"); + oc->boc->stevedore_priv = trim_once; + if (stv->sml_free == NULL) return; @@ -566,7 +574,6 @@ sml_trimstore(struct worker *wrk, struct objcore *oc) VTAILQ_INSERT_HEAD(&o->list, st1, list); Lck_Unlock(&oc->boc->mtx); /* sml_bocdone frees this */ - AZ(oc->boc->stevedore_priv); oc->boc->stevedore_priv = st; } diff --git a/bin/varnishtest/tests/b00062.vtc b/bin/varnishtest/tests/b00062.vtc index 511a4b416..4a828a53b 100644 --- a/bin/varnishtest/tests/b00062.vtc +++ b/bin/varnishtest/tests/b00062.vtc @@ -2,7 +2,11 @@ varnishtest "Test that we properly wait for certain 304 cases" server s1 { rxreq - txresp -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" -body "Geoff Still Rules" + txresp -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr "Geoff: Still Rules" \ + -bodylen 130560 + + # 2*64k-512 ^^^ see sml_trimstore() st->space - st->len < 512 # The IMS request we will spend some time to process for the sake of # this test. @@ -17,7 +21,7 @@ server s1 { txresp -body "x" } -start -varnish v1 -vcl+backend { +varnish v1 -arg "-p fetch_maxchunksize=64k" -vcl+backend { sub vcl_backend_response { set beresp.ttl = 1s; set beresp.grace = 1s; @@ -30,7 +34,8 @@ client c1 { txreq rxresp expect resp.status == 200 - expect resp.body == "Geoff Still Rules" + expect resp.http.Geoff == "Still Rules" + expect resp.bodylen == 130560 } -run # let the object's ttl and grace expire @@ -42,7 +47,8 @@ client c2 { rxresp # we did not disable grace in the request, so we should get the graced object here expect resp.status == 200 - expect resp.body == "Geoff Still Rules" + expect resp.http.Geoff == "Still Rules" + expect resp.bodylen == 130560 } -start delay .1 @@ -52,7 +58,8 @@ client c3 { txreq rxresp expect resp.status == 200 - expect resp.body == "Geoff Still Rules" + expect resp.http.Geoff == "Still Rules" + expect resp.bodylen == 130560 } -start client c2 -wait From dridi.boukelmoune at gmail.com Tue Dec 5 10:14:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Dec 2023 10:14:05 +0000 (UTC) Subject: [master] 98b6b9548 Optimize 304 template object copying Message-ID: <20231205101405.96FFC10609F@lists.varnish-cache.org> commit 98b6b954848b3a0fe1d81c852142d79d95edf395 Author: Nils Goroll Date: Wed Nov 1 10:48:25 2023 +0100 Optimize 304 template object copying When copying a stale object after a 304 revalidation, we iterated over it and allocated new storage for each storage segment. So, at best, we kept the fragmentation of the existing object, or made it even worse. This is particularly relevant when the existing object was created from a chunked response, in which case the original segments might have been particularly small and, consequently, many in number. Because we wait for the stale object to complete, we know the total length (OA_LEN) upfront, and can ask the storage serving the copy for exactly the right length. This is much more efficient, as fragmentation is lowered and the storage engine might make a much better allocation decision when it knows the full length rather than getting piecemeal requests. We implement the improved allocation scheme through additional state kept for the duration of the template object copy: struct vbf_objiter_priv holds the pointer to the newly created busy object, the total yet unallocated length (initialized to OA_LEN of the existing object) and pointer/length to the unused portion of the currently open segment, if any. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index e365f6c88..9a6ed4ec6 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -760,28 +760,56 @@ vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo) /*-------------------------------------------------------------------- */ +struct vbf_objiter_priv { + unsigned magic; +#define VBF_OBITER_PRIV_MAGIC 0x3c272a17 + struct busyobj *bo; + // not yet allocated + ssize_t l; + // current allocation + uint8_t *p; + ssize_t pl; +}; + static int v_matchproto_(objiterate_f) vbf_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) { - struct busyobj *bo; + struct vbf_objiter_priv *vop; ssize_t l; const uint8_t *ps = ptr; - uint8_t *pd; - CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC); + CAST_OBJ_NOTNULL(vop, priv, VBF_OBITER_PRIV_MAGIC); + CHECK_OBJ_NOTNULL(vop->bo, BUSYOBJ_MAGIC); flush &= OBJ_ITER_END; while (len > 0) { - l = len; - if (VFP_GetStorage(bo->vfc, &l, &pd) != VFP_OK) - return (1); - l = vmin(l, len); - memcpy(pd, ps, l); - VFP_Extend(bo->vfc, l, flush && l == len ? VFP_END : VFP_OK); + if (vop->pl == 0) { + vop->p = NULL; + AN(vop->l); + vop->pl = vop->l; + if (VFP_GetStorage(vop->bo->vfc, &vop->pl, &vop->p) + != VFP_OK) + return (1); + if (vop->pl < vop->l) + vop->l -= vop->pl; + else + vop->l = 0; + } + AN(vop->pl); + AN(vop->p); + + l = vmin(vop->pl, len); + memcpy(vop->p, ps, l); + VFP_Extend(vop->bo->vfc, l, + flush && l == len ? VFP_END : VFP_OK); ps += l; + vop->p += l; len -= l; + vop->pl -= l; } + if (flush) + AZ(vop->l); return (0); } @@ -791,6 +819,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) struct boc *stale_boc; enum boc_state_e stale_state; struct objcore *oc, *stale_oc; + struct vbf_objiter_priv vop[1]; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); @@ -850,7 +879,10 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) ObjSetState(wrk, oc, BOS_STREAM); } - if (ObjIterate(wrk, stale_oc, bo, vbf_objiterate, 0)) + INIT_OBJ(vop, VBF_OBITER_PRIV_MAGIC); + vop->bo = bo; + vop->l = ObjGetLen(bo->wrk, stale_oc); + if (ObjIterate(wrk, stale_oc, vop, vbf_objiterate, 0)) (void)VFP_Error(bo->vfc, "Template object failed"); if (bo->vfc->failed) { From dridi.boukelmoune at gmail.com Tue Dec 5 10:28:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Dec 2023 10:28:05 +0000 (UTC) Subject: [master] 4250083b8 req_fsm: Use status 408 for reset streams Message-ID: <20231205102805.C140D106AD3@lists.varnish-cache.org> commit 4250083b875d3445ab6b3a5dca6e9228cef22d96 Author: Dridi Boukelmoune Date: Fri Dec 1 16:25:00 2023 +0100 req_fsm: Use status 408 for reset streams The 503 synth and 500 minimal response status codes are too misleading in this context, where the failure is attributed to the client. Among existing 4XX status codes, this is the closest if we stretch the timeout definition to "didn't complete rapidly enough before the client went away". diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index aa0a0db98..b49f4dadd 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -288,8 +288,13 @@ cnt_vclfail(struct worker *wrk, struct req *req) Req_Rollback(ctx); - req->err_code = 503; - req->err_reason = "VCL failed"; + if (req->req_reset) { + req->err_code = 408; + req->err_reason = "Client disconnected"; + } else { + req->err_code = 503; + req->err_reason = "VCL failed"; + } req->req_step = R_STP_SYNTH; req->doclose = SC_VCL_FAILURE; req->filter_list = NULL; @@ -305,6 +310,7 @@ cnt_synth(struct worker *wrk, struct req *req) { struct vsb *synth_body; ssize_t sz, szl; + uint16_t status; uint8_t *ptr; const char *body; @@ -339,7 +345,8 @@ cnt_synth(struct worker *wrk, struct req *req) } VSB_destroy(&synth_body); (void)VRB_Ignore(req); - (void)req->transport->minimal_response(req, 500); + status = req->req_reset ? 408 : 500; + (void)req->transport->minimal_response(req, status); req->doclose = SC_VCL_FAILURE; // XXX: Not necessary any more ? VSLb_ts_req(req, "Resp", W_TIM_real(wrk)); http_Teardown(req->resp); diff --git a/bin/varnishtest/tests/t02025.vtc b/bin/varnishtest/tests/t02025.vtc index 578dbf5c8..4b83fe95b 100644 --- a/bin/varnishtest/tests/t02025.vtc +++ b/bin/varnishtest/tests/t02025.vtc @@ -46,7 +46,7 @@ varnish v1 -expect req_reset == 1 # is interpreted as before a second elapsed. Session VXIDs showing up # numerous times become increasingly more suspicious. The format can of # course be extended to add anything else useful for data mining. -shell -expect "1000 ${localhost}" { +shell -expect "1000 ${localhost} 408" { varnishncsa -n ${v1_name} -d \ - -q 'Timestamp:Reset[2] < 1.0' -F '%{VSL:Begin[2]}x %h' + -q 'Timestamp:Reset[2] < 1.0' -F '%{VSL:Begin[2]}x %h %s' } diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst index 98c126d2d..e3c6d0db8 100644 --- a/doc/sphinx/reference/vsl.rst +++ b/doc/sphinx/reference/vsl.rst @@ -79,7 +79,8 @@ Restart Reset The client closed its connection, reset its stream or caused a stream error that forced Varnish to reset the stream. Request - processing is interrupted and considered failed. + processing is interrupted and considered failed, with a 408 + "Request Timeout" status code. Pipe handling timestamps ~~~~~~~~~~~~~~~~~~~~~~~~ From dridi.boukelmoune at gmail.com Tue Dec 5 11:16:07 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Dec 2023 11:16:07 +0000 (UTC) Subject: [7.3] d9dd7826a req_fsm: Use status 408 for reset streams Message-ID: <20231205111607.8AD601089B1@lists.varnish-cache.org> commit d9dd7826ac75c4a334d576e8f5169293eee8bff8 Author: Dridi Boukelmoune Date: Fri Dec 1 16:25:00 2023 +0100 req_fsm: Use status 408 for reset streams The 503 synth and 500 minimal response status codes are too misleading in this context, where the failure is attributed to the client. Among existing 4XX status codes, this is the closest if we stretch the timeout definition to "didn't complete rapidly enough before the client went away". diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 9373ec4ef..d587fa1f9 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -289,8 +289,13 @@ cnt_vclfail(struct worker *wrk, struct req *req) Req_Rollback(ctx); - req->err_code = 503; - req->err_reason = "VCL failed"; + if (req->req_reset) { + req->err_code = 408; + req->err_reason = "Client disconnected"; + } else { + req->err_code = 503; + req->err_reason = "VCL failed"; + } req->req_step = R_STP_SYNTH; req->doclose = SC_VCL_FAILURE; req->filter_list = NULL; @@ -306,6 +311,7 @@ cnt_synth(struct worker *wrk, struct req *req) { struct vsb *synth_body; ssize_t sz, szl; + uint16_t status; uint8_t *ptr; const char *body; @@ -340,7 +346,8 @@ cnt_synth(struct worker *wrk, struct req *req) } VSB_destroy(&synth_body); (void)VRB_Ignore(req); - (void)req->transport->minimal_response(req, 500); + status = req->req_reset ? 408 : 500; + (void)req->transport->minimal_response(req, status); req->doclose = SC_VCL_FAILURE; // XXX: Not necessary any more ? VSLb_ts_req(req, "Resp", W_TIM_real(wrk)); http_Teardown(req->resp); diff --git a/bin/varnishtest/tests/t02025.vtc b/bin/varnishtest/tests/t02025.vtc index 578dbf5c8..4b83fe95b 100644 --- a/bin/varnishtest/tests/t02025.vtc +++ b/bin/varnishtest/tests/t02025.vtc @@ -46,7 +46,7 @@ varnish v1 -expect req_reset == 1 # is interpreted as before a second elapsed. Session VXIDs showing up # numerous times become increasingly more suspicious. The format can of # course be extended to add anything else useful for data mining. -shell -expect "1000 ${localhost}" { +shell -expect "1000 ${localhost} 408" { varnishncsa -n ${v1_name} -d \ - -q 'Timestamp:Reset[2] < 1.0' -F '%{VSL:Begin[2]}x %h' + -q 'Timestamp:Reset[2] < 1.0' -F '%{VSL:Begin[2]}x %h %s' } diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst index 98c126d2d..e3c6d0db8 100644 --- a/doc/sphinx/reference/vsl.rst +++ b/doc/sphinx/reference/vsl.rst @@ -79,7 +79,8 @@ Restart Reset The client closed its connection, reset its stream or caused a stream error that forced Varnish to reset the stream. Request - processing is interrupted and considered failed. + processing is interrupted and considered failed, with a 408 + "Request Timeout" status code. Pipe handling timestamps ~~~~~~~~~~~~~~~~~~~~~~~~ From dridi.boukelmoune at gmail.com Tue Dec 5 11:22:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Dec 2023 11:22:05 +0000 (UTC) Subject: [7.3] 48c299c13 doc: Use a raw string for regular expressions Message-ID: <20231205112205.6E1D9108EE4@lists.varnish-cache.org> commit 48c299c13a3b59ebad4bda1ffddad6fc75fba551 Author: Dridi Boukelmoune Date: Mon Dec 4 10:45:20 2023 +0100 doc: Use a raw string for regular expressions Since Python 3.12 unknown escape sequences trigger a warning. diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py index cfc5a05ab..acf6f356c 100644 --- a/doc/sphinx/vtc-syntax.py +++ b/doc/sphinx/vtc-syntax.py @@ -38,7 +38,7 @@ import re def parse_file(fn, cl, tl, sl): p = False section = "" - resec = re.compile("\s*/?\* SECTION: ") + resec = re.compile(r"\s*/?\* SECTION: ") try: # Python3 From dridi.boukelmoune at gmail.com Tue Dec 5 11:24:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Dec 2023 11:24:05 +0000 (UTC) Subject: [6.0] 20f65265c man: Add missing sections to the vtc(7) manual Message-ID: <20231205112405.D261B109196@lists.varnish-cache.org> commit 20f65265c7bd1842dcfb2c759df43fe86e90348b Author: Dridi Boukelmoune Date: Wed Sep 1 05:52:35 2021 +0200 man: Add missing sections to the vtc(7) manual diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 5d16f3e15..d4d0443a2 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -913,6 +913,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) n = 0; /* SECTION: stream.spec.zexpect.ping PING specific + * * ping.data * The 8-bytes string of the PING frame payload. * ping.ack (PING) @@ -928,6 +929,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) return (buf); } /* SECTION: stream.spec.zexpect.winup WINDOW_UPDATE specific + * * winup.size * The size of the upgrade given by the WINDOW_UPDATE frame. */ @@ -936,6 +938,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) RETURN_BUFFED(f->md.winup_size); } /* SECTION: stream.spec.zexpect.prio PRIORITY specific + * * prio.stream * The stream ID announced. * @@ -959,6 +962,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) RETURN_BUFFED(f->md.prio.weight); } /* SECTION: stream.spec.zexpect.rst RESET_STREAM specific + * * rst.err * The error code (as integer) of the RESET_STREAM frame. */ @@ -1013,6 +1017,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) if (!strcmp(spec, "hdrsize")) { RETURN_SETTINGS(6); } } /* SECTION: stream.spec.zexpect.push PUSH_PROMISE specific + * * push.id * The id of the promised stream. */ @@ -1021,6 +1026,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) RETURN_BUFFED(f->md.promised); } /* SECTION: stream.spec.zexpect.goaway GOAWAY specific + * * goaway.err * The error code (as integer) of the GOAWAY frame. * @@ -1042,6 +1048,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) return (f->md.goaway.debug); } /* SECTION: stream.spec.zexpect.zframe Generic frame + * * frame.data * Payload of the last frame * @@ -1077,6 +1084,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) } } /* SECTION: stream.spec.zexpect.zstream Stream + * * stream.window * The current window size of the stream, or, if on stream 0, * of the connection. @@ -1107,6 +1115,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) return (NULL); } /* SECTION: stream.spec.zexpect.ztable Index tables + * * tbl.dec.size / tbl.enc.size * Size (bytes) of the decoding/encoding table. * diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py index 8b15ca7e9..33aad14b6 100644 --- a/doc/sphinx/vtc-syntax.py +++ b/doc/sphinx/vtc-syntax.py @@ -36,7 +36,7 @@ import re def parse_file(fn, cl, tl, sl): p = False section = "" - resec = re.compile("[ /]\* SECTION: ") + resec = re.compile("\s*/\* SECTION: ") f = open(fn, "r", encoding="UTF-8") @@ -50,13 +50,13 @@ def parse_file(fn, cl, tl, sl): cl[section] = [] if len(a) > 3: tl[section] = re.sub( - r"^[\t ]*\/?\* SECTION: [^ ]+ +", + r"^\s*/?\* SECTION: [^ ]+ +", "", l) else: tl[section] = "" p = 1 elif p: - cl[section].append(re.sub(r"^ \* ?", "", l)) + cl[section].append(re.sub(r"^\s*\* ?", "", l)) f.close() if __name__ == "__main__": From dridi.boukelmoune at gmail.com Tue Dec 5 11:24:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Dec 2023 11:24:05 +0000 (UTC) Subject: [6.0] ca5770ada man: Fix rendering of certain sections in vtc(7) Message-ID: <20231205112405.E5A23109199@lists.varnish-cache.org> commit ca5770ada1b18e44e6961b92b62d197f5b80863d Author: Dridi Boukelmoune Date: Mon Nov 28 17:17:15 2022 +0100 man: Fix rendering of certain sections in vtc(7) The matching regular expression did not align with the substitution regular expression. Refs d2e526ce95d6d8426ec170b4cfc78d41abc07179 diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py index 33aad14b6..1c40f30d5 100644 --- a/doc/sphinx/vtc-syntax.py +++ b/doc/sphinx/vtc-syntax.py @@ -36,7 +36,7 @@ import re def parse_file(fn, cl, tl, sl): p = False section = "" - resec = re.compile("\s*/\* SECTION: ") + resec = re.compile("\s*/?\* SECTION: ") f = open(fn, "r", encoding="UTF-8") From dridi.boukelmoune at gmail.com Tue Dec 5 11:24:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 5 Dec 2023 11:24:06 +0000 (UTC) Subject: [6.0] e9af6a8cd doc: Use a raw string for regular expressions Message-ID: <20231205112406.0EAE910919D@lists.varnish-cache.org> commit e9af6a8cd480c739629d787eeb99abe51dfbadf1 Author: Dridi Boukelmoune Date: Mon Dec 4 10:45:20 2023 +0100 doc: Use a raw string for regular expressions Since Python 3.12 unknown escape sequences trigger a warning. diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py index 1c40f30d5..497c7b4b3 100644 --- a/doc/sphinx/vtc-syntax.py +++ b/doc/sphinx/vtc-syntax.py @@ -36,7 +36,7 @@ import re def parse_file(fn, cl, tl, sl): p = False section = "" - resec = re.compile("\s*/?\* SECTION: ") + resec = re.compile(r"\s*/?\* SECTION: ") f = open(fn, "r", encoding="UTF-8") From phk at FreeBSD.org Tue Dec 5 12:38:06 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Dec 2023 12:38:06 +0000 (UTC) Subject: [master] d7cb26c50 Add some debugging to figure out how old fashioned Solaris is. Message-ID: <20231205123806.0B7A810FA4F@lists.varnish-cache.org> commit d7cb26c509f34c90e886757695006dedf9e29fd1 Author: Poul-Henning Kamp Date: Tue Dec 5 12:37:01 2023 +0000 Add some debugging to figure out how old fashioned Solaris is. diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc index 8b1e9de32..95fac23dd 100644 --- a/bin/varnishtest/tests/u00000.vtc +++ b/bin/varnishtest/tests/u00000.vtc @@ -174,8 +174,9 @@ shell -err -expect {Cannot read -f file} { process p1 { /bin/echo -n foobar > ${tmpdir}/_foobar + ls -l ${tmpdir}/_foobar exec varnishd -n ${tmpdir}/v0 -d -a :0 -I ${tmpdir}/_foobar 2>&1 -} -expect-exit 2 -start +} -expect-exit 2 -dump -start process p1 -expect-text 0 0 "-I file had incomplete CLI command at the end" process p1 -screen-dump From phk at FreeBSD.org Tue Dec 5 13:04:06 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Dec 2023 13:04:06 +0000 (UTC) Subject: [master] c64ef384b Try to work around "echo -n" being broken on Solaris Message-ID: <20231205130406.20B1F1109BB@lists.varnish-cache.org> commit c64ef384bf42d177ac5581ab2321e061cdc94e16 Author: Poul-Henning Kamp Date: Tue Dec 5 13:03:12 2023 +0000 Try to work around "echo -n" being broken on Solaris diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc index 95fac23dd..8ce91b74b 100644 --- a/bin/varnishtest/tests/u00000.vtc +++ b/bin/varnishtest/tests/u00000.vtc @@ -172,9 +172,9 @@ shell -err -expect {Cannot read -f file} { # Test that incomplete CLI commands in -I causes failure +filewrite ${tmpdir}/_foobar foobar + process p1 { - /bin/echo -n foobar > ${tmpdir}/_foobar - ls -l ${tmpdir}/_foobar exec varnishd -n ${tmpdir}/v0 -d -a :0 -I ${tmpdir}/_foobar 2>&1 } -expect-exit 2 -dump -start From dridi at varni.sh Tue Dec 5 15:20:31 2023 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 5 Dec 2023 15:20:31 +0000 Subject: [master] c64ef384b Try to work around "echo -n" being broken on Solaris In-Reply-To: <20231205130406.20B1F1109BB@lists.varnish-cache.org> References: <20231205130406.20B1F1109BB@lists.varnish-cache.org> Message-ID: On Tue, Dec 5, 2023 at 1:04?PM Poul-Henning Kamp wrote: > > > commit c64ef384bf42d177ac5581ab2321e061cdc94e16 > Author: Poul-Henning Kamp > Date: Tue Dec 5 13:03:12 2023 +0000 > > Try to work around "echo -n" being broken on Solaris echo(1) has no portable option, so why not printf(1) instead? > diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc > index 95fac23dd..8ce91b74b 100644 > --- a/bin/varnishtest/tests/u00000.vtc > +++ b/bin/varnishtest/tests/u00000.vtc > @@ -172,9 +172,9 @@ shell -err -expect {Cannot read -f file} { > > # Test that incomplete CLI commands in -I causes failure > > +filewrite ${tmpdir}/_foobar foobar > + > process p1 { > - /bin/echo -n foobar > ${tmpdir}/_foobar > - ls -l ${tmpdir}/_foobar > exec varnishd -n ${tmpdir}/v0 -d -a :0 -I ${tmpdir}/_foobar 2>&1 > } -expect-exit 2 -dump -start > > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at phk.freebsd.dk Tue Dec 5 15:49:11 2023 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 05 Dec 2023 15:49:11 +0000 Subject: [master] c64ef384b Try to work around "echo -n" being broken on Solaris In-Reply-To: References: <20231205130406.20B1F1109BB@lists.varnish-cache.org> Message-ID: <202312051549.3B5FnBk6043249@critter.freebsd.dk> -------- Dridi Boukelmoune writes: > On Tue, Dec 5, 2023 at 1:04=E2=80=AFPM Poul-Henning Kamp = > wrote: > > > > > > commit c64ef384bf42d177ac5581ab2321e061cdc94e16 > > Author: Poul-Henning Kamp > > Date: Tue Dec 5 13:03:12 2023 +0000 > > > > Try to work around "echo -n" being broken on Solaris > > echo(1) has no portable option, so why not printf(1) instead? because I'm sure there are systems without printf(1) too :-) -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From nils.goroll at uplex.de Tue Dec 5 20:47:18 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Dec 2023 20:47:18 +0000 (UTC) Subject: [master] 5583667fe VRT minor version bump for #4013 Message-ID: <20231205204718.2EC4811F918@lists.varnish-cache.org> commit 5583667fedcb969d112b17fdb4d84f2e910e75fe Author: Nils Goroll Date: Tue Dec 5 21:44:49 2023 +0100 VRT minor version bump for #4013 we need a signal if the fix is in diff --git a/include/vrt.h b/include/vrt.h index 26509f344..b378920a2 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -48,7 +48,7 @@ #define VRT_MAJOR_VERSION 18U -#define VRT_MINOR_VERSION 0U +#define VRT_MINOR_VERSION 1U /*********************************************************************** * Major and minor VRT API versions. From dridi at varni.sh Tue Dec 5 22:32:10 2023 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 5 Dec 2023 22:32:10 +0000 Subject: [master] 5583667fe VRT minor version bump for #4013 In-Reply-To: <20231205204718.2EC4811F918@lists.varnish-cache.org> References: <20231205204718.2EC4811F918@lists.varnish-cache.org> Message-ID: On Tue, Dec 5, 2023 at 8:47?PM Nils Goroll wrote: > > > commit 5583667fedcb969d112b17fdb4d84f2e910e75fe > Author: Nils Goroll > Date: Tue Dec 5 21:44:49 2023 +0100 > > VRT minor version bump for #4013 > > we need a signal if the fix is in Have you considered running a VTC at configure time instead? > diff --git a/include/vrt.h b/include/vrt.h > index 26509f344..b378920a2 100644 > --- a/include/vrt.h > +++ b/include/vrt.h > @@ -48,7 +48,7 @@ > > #define VRT_MAJOR_VERSION 18U > > -#define VRT_MINOR_VERSION 0U > +#define VRT_MINOR_VERSION 1U > > /*********************************************************************** > * Major and minor VRT API versions. > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Wed Dec 6 09:51:30 2023 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 6 Dec 2023 09:51:30 +0000 Subject: [master] f8936e78a Change the panic reentrancy check so that Coverity can (hopefully) grok it. In-Reply-To: <20231204085610.6BCC411E512@lists.varnish-cache.org> References: <20231204085610.6BCC411E512@lists.varnish-cache.org> Message-ID: On Mon, Dec 4, 2023 at 8:56?AM Poul-Henning Kamp wrote: > > > commit f8936e78a50153f3203bb077be8a067eac0c9663 > Author: Poul-Henning Kamp > Date: Mon Dec 4 08:53:22 2023 +0000 > > Change the panic reentrancy check so that Coverity can (hopefully) grok it. Defects eliminated: 2 That's a good start, but I think there were plenty more. > diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c > index 3e093c50b..7b433c596 100644 > --- a/bin/varnishd/cache/cache_main.c > +++ b/bin/varnishd/cache/cache_main.c > @@ -81,6 +81,7 @@ static pthread_key_t req_key; > static pthread_key_t bo_key; > static pthread_key_t wrk_key; > pthread_key_t witness_key; > +pthread_key_t panic_key; > > void > THR_SetBusyobj(const struct busyobj *bo) > @@ -404,6 +405,7 @@ child_main(int sigmagic, size_t altstksz) > PTOK(pthread_key_create(&wrk_key, NULL)); > PTOK(pthread_key_create(&witness_key, free)); > PTOK(pthread_key_create(&name_key, NULL)); > + PTOK(pthread_key_create(&panic_key, NULL)); > > THR_SetName("cache-main"); > > diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c > index 221d9835c..eca20c32b 100644 > --- a/bin/varnishd/cache/cache_panic.c > +++ b/bin/varnishd/cache/cache_panic.c > @@ -67,7 +67,6 @@ > > static struct vsb pan_vsb_storage, *pan_vsb; > static pthread_mutex_t panicstr_mtx; > -static pthread_t panicy; > > static void pan_sess(struct vsb *, const struct sess *); > static void pan_req(struct vsb *, const struct req *); > @@ -744,18 +743,23 @@ pan_ic(const char *func, const char *file, int line, const char *cond, > struct busyobj *bo; > struct worker *wrk; > struct sigaction sa; > - int err = errno; > + int i, err = errno; > > - /* If we already panicing in another thread, do nothing */ > - while (heritage.panic_str[0] && panicy != pthread_self()) > - sleep(1); > - > - if (pthread_mutex_lock(&panicstr_mtx)) { > - /* Reentrant panic */ > + if (pthread_getspecific(panic_key) != NULL) { > VSB_cat(pan_vsb, "\n\nPANIC REENTRANCY\n\n"); > abort(); > } > - panicy = pthread_self(); > + > + /* If we already panicing in another thread, do nothing */ > + do { > + i = pthread_mutex_trylock(&panicstr_mtx); > + if (i != 0) > + sleep (1); > + } while (i != 0); > + > + assert (VSB_len(pan_vsb) == 0); > + > + AZ(pthread_setspecific(panic_key, pan_vsb)); > > /* > * should we trigger a SIGSEGV while handling a panic, our sigsegv > @@ -844,6 +848,14 @@ pan_ic(const char *func, const char *file, int line, const char *cond, > VSB_putc(pan_vsb, '\0'); /* NUL termination */ > > v_gcov_flush(); > + > + /* > + * Do a little song and dance for static checkers which > + * are not smart enough to figure out that calling abort() > + * with a mutex held is OK and probably very intentional. > + */ > + if (pthread_getspecific(panic_key)) /* ie: always */ > + abort(); > PTOK(pthread_mutex_unlock(&panicstr_mtx)); > abort(); > } > diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h > index b534eb174..e9ffad17c 100644 > --- a/bin/varnishd/cache/cache_varnishd.h > +++ b/bin/varnishd/cache/cache_varnishd.h > @@ -303,6 +303,7 @@ unsigned HTTP1_Write(const struct worker *w, const struct http *hp, const int*); > > /* cache_main.c */ > vxid_t VXID_Get(const struct worker *, uint64_t marker); > +extern pthread_key_t panic_key; > extern pthread_key_t witness_key; > > void THR_SetName(const char *name); > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at phk.freebsd.dk Wed Dec 6 09:55:29 2023 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Wed, 06 Dec 2023 09:55:29 +0000 Subject: [master] f8936e78a Change the panic reentrancy check so that Coverity can (hopefully) grok it. In-Reply-To: References: <20231204085610.6BCC411E512@lists.varnish-cache.org> Message-ID: <202312060955.3B69tTV8075403@critter.freebsd.dk> -------- Dridi Boukelmoune writes: > Defects eliminated: 2 > > That's a good start, but I think there were plenty more. Yes, I'm working my way through them. Some of them are surprisingly thorny from a QA and philosophical point of view. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at FreeBSD.org Wed Dec 6 10:12:08 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Dec 2023 10:12:08 +0000 (UTC) Subject: [master] af24f081c Fix an insignificant race found by Coverity Message-ID: <20231206101208.9112A117404@lists.varnish-cache.org> commit af24f081cecb98e5afe6a34f4ff5d65ce55d4542 Author: Poul-Henning Kamp Date: Wed Dec 6 09:31:37 2023 +0000 Fix an insignificant race found by Coverity diff --git a/vmod/vmod_directors_fall_back.c b/vmod/vmod_directors_fall_back.c index b24e25e8e..a5cc37057 100644 --- a/vmod/vmod_directors_fall_back.c +++ b/vmod/vmod_directors_fall_back.c @@ -169,9 +169,9 @@ vmod_fallback_resolve(VRT_CTX, VCL_BACKEND dir) if (++fb->cur == fb->vd->n_backend) fb->cur = 0; } - vdir_unlock(fb->vd); if (u == fb->vd->n_backend) be = NULL; + vdir_unlock(fb->vd); return (be); } From phk at FreeBSD.org Wed Dec 6 10:12:08 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Dec 2023 10:12:08 +0000 (UTC) Subject: [master] f5d952c6a Consistently lock 'vsc' variable even if not necessary (for Coverity) Message-ID: <20231206101208.AEEBC117407@lists.varnish-cache.org> commit f5d952c6a3e1a692d913f1ebf0332e18f49ccdf3 Author: Poul-Henning Kamp Date: Wed Dec 6 10:10:27 2023 +0000 Consistently lock 'vsc' variable even if not necessary (for Coverity) diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 2f5390f09..9a98fe700 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -714,8 +714,12 @@ event_discard(VRT_CTX, void *priv) * of this VMOD, so we should only carry out cleanup on the * last discard event. */ - if (vsc) + PTOK(pthread_mutex_lock(&vsc_mtx)); + if (vsc != NULL) { VSC_debug_Destroy(&vsc_seg); + vsc = NULL; + } + PTOK(pthread_mutex_unlock(&vsc_mtx)); return (0); } @@ -772,8 +776,10 @@ VCL_VOID xyzzy_vsc_count(VRT_CTX, VCL_INT cnt) { (void)ctx; + PTOK(pthread_mutex_lock(&vsc_mtx)); AN(vsc); vsc->count += cnt; + PTOK(pthread_mutex_unlock(&vsc_mtx)); } VCL_VOID From nils.goroll at uplex.de Wed Dec 6 17:34:01 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 6 Dec 2023 18:34:01 +0100 Subject: [master] 5583667fe VRT minor version bump for #4013 In-Reply-To: References: <20231205204718.2EC4811F918@lists.varnish-cache.org> Message-ID: <95ca722c-f160-da53-e224-38e01f75cd7e@uplex.de> On 05.12.23 23:32, Dridi Boukelmoune wrote: > Have you considered running a VTC at configure time instead? No, in my mind, this is what the VRT version is for ("bump minor number for new functionality") and it is much simpler to use on the consumer side. Do you have a different opinion? Nils -- Nils Goroll (he/him) ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ From dridi at varni.sh Thu Dec 7 09:00:13 2023 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 7 Dec 2023 09:00:13 +0000 Subject: [master] 5583667fe VRT minor version bump for #4013 In-Reply-To: <95ca722c-f160-da53-e224-38e01f75cd7e@uplex.de> References: <20231205204718.2EC4811F918@lists.varnish-cache.org> <95ca722c-f160-da53-e224-38e01f75cd7e@uplex.de> Message-ID: On Wed, Dec 6, 2023 at 5:34?PM Nils Goroll wrote: > > On 05.12.23 23:32, Dridi Boukelmoune wrote: > > Have you considered running a VTC at configure time instead? > > No, in my mind, this is what the VRT version is for ("bump minor number for new > functionality") and it is much simpler to use on the consumer side. > > Do you have a different opinion? As a matter of fact, I do. VRT is minor-bumped for additive changes to the not-clearly-defined VRT ABI. --->8--- $ git log varnish-7.4.0..master -- include/vrt*.h bin/varnishd/cache/cache.h commit 5583667fedcb969d112b17fdb4d84f2e910e75fe Author: Nils Goroll Date: Tue Dec 5 21:44:49 2023 +0100 VRT minor version bump for #4013 we need a signal if the fix is in commit 4d584ddb082c1b343f70eb467578814f29f75033 Author: Nils Goroll Date: Wed Oct 18 12:08:14 2023 +0200 Start NEXT sections in vrt.h and changes.rst to anchor patches ---8<--- There hasn't been an ABI change registered since the last bump. Cheers, Dridi From nils.goroll at uplex.de Thu Dec 7 10:02:40 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Dec 2023 11:02:40 +0100 Subject: [master] 5583667fe VRT minor version bump for #4013 In-Reply-To: References: <20231205204718.2EC4811F918@lists.varnish-cache.org> <95ca722c-f160-da53-e224-38e01f75cd7e@uplex.de> Message-ID: Yes, I probably should add a comment to vrt.h regarding the vbf_objiterate() change. On 07.12.23 10:00, Dridi Boukelmoune wrote: > VRT is minor-bumped for additive changes to the not-clearly-defined VRT ABI. I agree to not-clearly-defined. Is the stevedore API part of the VRT ABI? Directly or indirectly? Regarding "additive changes", I read * Whenever something is deleted or changed in a way which is not * binary/load-time compatible, increment MAJOR version as applicable for an incompatible change. Overall, which harm is caused by a minor bump given that we practive unnecessary major bumps for releases "to be on the safe side"? Nils -- Nils Goroll (he/him) ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ From nils.goroll at uplex.de Thu Dec 7 13:20:10 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Dec 2023 13:20:10 +0000 (UTC) Subject: [master] 8c0c414f0 Document VRT 18.1 change Message-ID: <20231207132010.431779323@lists.varnish-cache.org> commit 8c0c414f0631ff4cc1bc7324b691a2114b119130 Author: Nils Goroll Date: Thu Dec 7 14:18:44 2023 +0100 Document VRT 18.1 change Ref 5583667fedcb969d112b17fdb4d84f2e910e75fe Ref #4013 diff --git a/include/vrt.h b/include/vrt.h index b378920a2..fe292e3ce 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -58,6 +58,8 @@ * binary/load-time compatible, increment MAJOR version * * NEXT (2024-03-15) + * 18.1 (2023-12-05) + * vbf_objiterate() implementation changed #4013 * 18.0 (2023-09-15) * [cache_filter.h] struct vdp gained priv1 member * VRT_trace() added From nils.goroll at uplex.de Mon Dec 11 10:20:10 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Dec 2023 10:20:10 +0000 (UTC) Subject: [master] 69615e855 Reflect vfp_filter_list field name in panic string Message-ID: <20231211102010.97D6511D65C@lists.varnish-cache.org> commit 69615e85541eb3c024bbf4b74153779b3bda7c95 Author: Nils Goroll Date: Mon Dec 11 08:04:57 2023 +0100 Reflect vfp_filter_list field name in panic string diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index eca20c32b..428962b20 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -378,8 +378,10 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) if (bo->vfc != NULL) pan_vfp(vsb, bo->vfc); - if (bo->vfp_filter_list != NULL) - VSB_printf(vsb, "filter_list = \"%s\",\n", bo->vfp_filter_list); + if (bo->vfp_filter_list != NULL) { + VSB_printf(vsb, "vfp_filter_list = \"%s\",\n", + bo->vfp_filter_list); + } WS_Panic(vsb, bo->ws); VSB_printf(vsb, "ws_bo = %p,\n", (void *)bo->ws_bo); From nils.goroll at uplex.de Mon Dec 11 11:28:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Dec 2023 11:28:06 +0000 (UTC) Subject: [master] 81d406e1d Consistent naming of struct vdp_ctx * arguments Message-ID: <20231211112806.3328311F892@lists.varnish-cache.org> commit 81d406e1d8f03479408228accd5fa77748e7c1c1 Author: Nils Goroll Date: Mon Dec 11 12:15:12 2023 +0100 Consistent naming of struct vdp_ctx * arguments for improved greppability, cosistently name struct vdp_ctx * variables vdc. diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 7fb7b87be..936191ea3 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -309,7 +309,7 @@ ved_vdp_esi_fini(struct vdp_ctx *vdc, void **priv) } static int v_matchproto_(vdp_bytes_f) -ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, +ved_vdp_esi_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { const uint8_t *q, *r; @@ -324,14 +324,14 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, act = VDP_FLUSH; AN(priv); - CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); pp = ptr; while (1) { switch (ecx->state) { case 0: - ecx->p = ObjGetAttr(vdx->wrk, ecx->preq->objcore, + ecx->p = ObjGetAttr(vdc->wrk, ecx->preq->objcore, OA_ESIDATA, &l); AN(ecx->p); assert(l > 0); @@ -339,7 +339,7 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, if (*ecx->p == VEC_GZ) { if (ecx->pecx == NULL) - retval = VDP_bytes(vdx, VDP_NULL, + retval = VDP_bytes(vdc, VDP_NULL, gzip_hdr, 10); ecx->l_crc = 0; ecx->crc = crc32(0L, Z_NULL, 0); @@ -357,14 +357,14 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, case VEC_V1: case VEC_V2: case VEC_V8: - ecx->l = ved_decode_len(vdx->vsl, &ecx->p); + ecx->l = ved_decode_len(vdc->vsl, &ecx->p); if (ecx->l < 0) return (-1); if (ecx->isgzip) { assert(*ecx->p == VEC_C1 || *ecx->p == VEC_C2 || *ecx->p == VEC_C8); - l = ved_decode_len(vdx->vsl, &ecx->p); + l = ved_decode_len(vdc->vsl, &ecx->p); if (l < 0) return (-1); icrc = vbe32dec(ecx->p); @@ -378,7 +378,7 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, case VEC_S1: case VEC_S2: case VEC_S8: - ecx->l = ved_decode_len(vdx->vsl, &ecx->p); + ecx->l = ved_decode_len(vdc->vsl, &ecx->p); if (ecx->l < 0) return (-1); Debug("SKIP1(%d)\n", (int)ecx->l); @@ -395,7 +395,7 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, q++; r = (void*)strchr((const char*)q, '\0'); AN(r); - if (VDP_bytes(vdx, VDP_FLUSH, NULL, 0)) { + if (VDP_bytes(vdc, VDP_FLUSH, NULL, 0)) { ecx->p = ecx->e; break; } @@ -406,7 +406,7 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, ecx->p = r + 1; break; default: - VSLb(vdx->vsl, SLT_Error, + VSLb(vdc->vsl, SLT_Error, "ESI corruption line %d 0x%02x [%s]\n", __LINE__, *ecx->p, ecx->p); WRONG("ESI-codes: Illegal code"); @@ -439,7 +439,7 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, ecx->crc, ecx->l_crc); ecx->pecx->l_crc += ecx->l_crc; } - retval = VDP_bytes(vdx, VDP_END, ptr, len); + retval = VDP_bytes(vdc, VDP_END, ptr, len); ecx->state = 99; return (retval); case 3: @@ -451,7 +451,7 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, */ if (ecx->l <= len) { if (ecx->state == 3) - retval = VDP_bytes(vdx, act, + retval = VDP_bytes(vdc, act, pp, ecx->l); len -= ecx->l; pp += ecx->l; @@ -459,7 +459,7 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, break; } if (ecx->state == 3 && len > 0) - retval = VDP_bytes(vdx, act, pp, len); + retval = VDP_bytes(vdc, act, pp, len); ecx->l -= len; return (retval); case 99: @@ -524,7 +524,7 @@ ved_pretend_gzip_fini(struct vdp_ctx *vdc, void **priv) } static int v_matchproto_(vdp_bytes_f) -ved_pretend_gzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, +ved_pretend_gzip_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, const void *pv, ssize_t l) { uint8_t buf1[5], buf2[5]; @@ -532,7 +532,7 @@ ved_pretend_gzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, uint16_t lx; struct ecx *ecx; - CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); (void)priv; @@ -652,7 +652,7 @@ ved_gzgz_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) */ static int v_matchproto_(vdp_bytes_f) -ved_gzgz_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, +ved_gzgz_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { struct ved_foo *foo; @@ -660,7 +660,7 @@ ved_gzgz_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, ssize_t dl; ssize_t l; - (void)vdx; + (void)vdc; CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); pp = ptr; if (len > 0) { @@ -834,12 +834,12 @@ ved_vdp_fini(struct vdp_ctx *vdc, void **priv) } static int v_matchproto_(vdp_bytes_f) -ved_vdp_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, +ved_vdp_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { struct ecx *ecx; - (void)vdx; + (void)vdc; CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); return (ved_bytes(ecx, act, ptr, len)); } diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index c1434dc62..d49b03c41 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -288,7 +288,7 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags) */ static int v_matchproto_(vdp_init_f) -vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdp, void **priv, struct objcore *oc) +vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) { struct vgz *vg; struct boc *boc; @@ -299,9 +299,9 @@ vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdp, void **priv, struct objcore *oc) uint64_t u; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vdp, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC); - req = vdp->req; + req = vdc->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); vg = VGZ_NewGunzip(req->vsl, "U D -"); @@ -353,7 +353,7 @@ vdp_gunzip_fini(struct vdp_ctx *vdc, void **priv) } static int v_matchproto_(vdp_bytes_f) -vdp_gunzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, +vdp_gunzip_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { enum vgzret_e vr; @@ -362,8 +362,8 @@ vdp_gunzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, struct worker *wrk; struct vgz *vg; - CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); - wrk = vdx->wrk; + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + wrk = vdc->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); (void)act; @@ -385,9 +385,9 @@ vdp_gunzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, if (vr < VGZ_OK) return (-1); if (vg->m_len == vg->m_sz || vr != VGZ_OK) { - if (VDP_bytes(vdx, vr == VGZ_END ? VDP_END : VDP_FLUSH, + if (VDP_bytes(vdc, vr == VGZ_END ? VDP_END : VDP_FLUSH, vg->m_buf, vg->m_len)) - return (vdx->retval); + return (vdc->retval); vg->m_len = 0; VGZ_Obuf(vg, vg->m_buf, vg->m_sz); } diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index bde81ccc7..d7398f33f 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -65,7 +65,7 @@ vrg_range_fini(struct vdp_ctx *vdc, void **priv) } static int v_matchproto_(vdp_bytes_f) -vrg_range_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, +vrg_range_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { int retval = 0; @@ -73,7 +73,7 @@ vrg_range_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const char *p = ptr; struct vrg_priv *vrg_priv; - CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); AN(priv); CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC); @@ -93,9 +93,9 @@ vrg_range_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, } if (l > 0) - retval = VDP_bytes(vdx, act, p, l); + retval = VDP_bytes(vdc, act, p, l); else if (l == 0 && act > VDP_NULL) - retval = VDP_bytes(vdx, act, p, 0); + retval = VDP_bytes(vdc, act, p, 0); return (retval || act == VDP_END ? 1 : 0); } diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index e9ffad17c..404ebc894 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -189,7 +189,7 @@ void VDI_Event(const struct director *d, enum vcl_event_e ev); void VDI_Init(void); /* cache_deliver_proc.c */ -void VDP_Init(struct vdp_ctx *vdx, struct worker *wrk, struct vsl_log *vsl, +void VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl, struct req *req); uint64_t VDP_Close(struct vdp_ctx *, struct objcore *, struct boc *); void VDP_Panic(struct vsb *vsb, const struct vdp_ctx *vdc); diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index 86a2dfd0e..6bed2b08d 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -40,19 +40,19 @@ /*--------------------------------------------------------------------*/ static int v_matchproto_(vdp_bytes_f) -v1d_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, +v1d_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { ssize_t wl = 0; - CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); (void)priv; - AZ(vdx->nxt); /* always at the bottom of the pile */ + AZ(vdc->nxt); /* always at the bottom of the pile */ if (len > 0) - wl = V1L_Write(vdx->wrk, ptr, len); - if (act > VDP_NULL && V1L_Flush(vdx->wrk) != SC_NULL) + wl = V1L_Write(vdc->wrk, ptr, len); + if (act > VDP_NULL && V1L_Flush(vdc->wrk) != SC_NULL) return (-1); if (len != wl) return (-1); diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 632d999ef..267ea91ea 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -111,12 +111,12 @@ h2_fini(struct vdp_ctx *vdc, void **priv) } static int v_matchproto_(vdp_bytes_f) -h2_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, +h2_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { struct h2_req *r2; - CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); CAST_OBJ_NOTNULL(r2, *priv, H2_REQ_MAGIC); (void)act; @@ -124,9 +124,9 @@ h2_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, return (-1); if (len == 0) return (0); - H2_Send_Get(vdx->wrk, r2->h2sess, r2); - vdx->bytes_done = 0; - H2_Send(vdx->wrk, r2, H2_F_DATA, H2FF_NONE, len, ptr, &vdx->bytes_done); + H2_Send_Get(vdc->wrk, r2->h2sess, r2); + vdc->bytes_done = 0; + H2_Send(vdc->wrk, r2, H2_F_DATA, H2FF_NONE, len, ptr, &vdc->bytes_done); H2_Send_Rel(r2->h2sess, r2); return (0); } diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 9a98fe700..8f3f5636e 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -116,18 +116,18 @@ xyzzy_vfp_rot13_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore * } static int v_matchproto_(vdp_bytes_f) -xyzzy_vfp_rot13_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, +xyzzy_vfp_rot13_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { char *q; const char *pp; int i, j, retval = 0; - CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); AN(priv); AN(*priv); if (len <= 0) - return (VDP_bytes(vdx, act, ptr, len)); + return (VDP_bytes(vdc, act, ptr, len)); AN(ptr); if (act != VDP_END) act = VDP_FLUSH; @@ -142,14 +142,14 @@ xyzzy_vfp_rot13_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, else q[i] = pp[j]; if (i == ROT13_BUFSZ - 1 && j < len - 1) { - retval = VDP_bytes(vdx, VDP_FLUSH, q, ROT13_BUFSZ); + retval = VDP_bytes(vdc, VDP_FLUSH, q, ROT13_BUFSZ); if (retval != 0) return (retval); i = -1; } } if (i >= 0) - retval = VDP_bytes(vdx, act, q, i); + retval = VDP_bytes(vdc, act, q, i); return (retval); } @@ -212,7 +212,7 @@ static const struct vmod_priv_methods priv_pedantic_methods[1] = {{ }}; static int v_matchproto_(vdp_init_f) -xyzzy_pedantic_init(VRT_CTX, struct vdp_ctx *vdx, void **priv, +xyzzy_pedantic_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) { struct vdp_state_s *vdps; @@ -226,7 +226,7 @@ xyzzy_pedantic_init(VRT_CTX, struct vdp_ctx *vdx, void **priv, return (-1); assert(vdps->state == VDPS_NULL); - p = VRT_priv_task(ctx, (void *)vdx); + p = VRT_priv_task(ctx, (void *)vdc); if (p == NULL) return (-1); p->priv = vdps; @@ -241,7 +241,7 @@ xyzzy_pedantic_init(VRT_CTX, struct vdp_ctx *vdx, void **priv, } static int v_matchproto_(vdp_bytes_f) -xyzzy_pedantic_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, +xyzzy_pedantic_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { struct vdp_state_s *vdps; @@ -255,15 +255,15 @@ xyzzy_pedantic_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, else vdps->state = VDPS_BYTES; - return (VDP_bytes(vdx, act, ptr, len)); + return (VDP_bytes(vdc, act, ptr, len)); } static int v_matchproto_(vdp_fini_f) -xyzzy_pedantic_fini(struct vdp_ctx *vdx, void **priv) +xyzzy_pedantic_fini(struct vdp_ctx *vdc, void **priv) { struct vdp_state_s *vdps; - (void) vdx; + (void) vdc; AN(priv); if (*priv == NULL) return (0); From nils.goroll at uplex.de Mon Dec 11 11:28:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Dec 2023 11:28:06 +0000 (UTC) Subject: [master] 3ca7b1d5a Prefer struct vdp_ctx members over req members Message-ID: <20231211112806.487F411F899@lists.varnish-cache.org> commit 3ca7b1d5a2c6c82cbae14c8926cc10b6941b14cd Author: Nils Goroll Date: Mon Dec 11 12:18:20 2023 +0100 Prefer struct vdp_ctx members over req members Struct vdp_ctx has wrk and vsl members, use these instead of req. diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index d49b03c41..476c261f5 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -304,7 +304,7 @@ vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) req = vdc->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - vg = VGZ_NewGunzip(req->vsl, "U D -"); + vg = VGZ_NewGunzip(vdc->vsl, "U D -"); AN(vg); if (vgz_getmbuf(vg)) { (void)VGZ_Destroy(&vg); @@ -325,12 +325,12 @@ vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) if (boc != NULL) { CHECK_OBJ(boc, BOC_MAGIC); bos = boc->state; - HSH_DerefBoc(req->wrk, oc); + HSH_DerefBoc(vdc->wrk, oc); if (bos < BOS_FINISHED) return (0); /* OA_GZIPBITS is not stable yet */ } - p = ObjGetAttr(req->wrk, oc, OA_GZIPBITS, &dl); + p = ObjGetAttr(vdc->wrk, oc, OA_GZIPBITS, &dl); if (p != NULL && dl == 32) { u = vbe64dec(p + 24); if (u != 0) From phk at FreeBSD.org Mon Dec 11 13:35:08 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 11 Dec 2023 13:35:08 +0000 (UTC) Subject: [master] d55093f67 Move two time consuming curses operations out of the mutex Message-ID: <20231211133508.5014C12365B@lists.varnish-cache.org> commit d55093f671b58b8a2f9dddd7a2aef15ac2895f27 Author: Poul-Henning Kamp Date: Mon Dec 11 12:37:57 2023 +0000 Move two time consuming curses operations out of the mutex diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index ddec07a11..fa26a5ca8 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -151,8 +151,6 @@ update(void) int i, j; unsigned k, l; - AC(erase()); - /* Draw horizontal axis */ for (k = 0; k < n; ++k) (void)mvaddch(LINES - 2, k, '-'); @@ -202,8 +200,6 @@ update(void) for (; l < (bm[k] + bh[k]) / scale; ++l) (void)mvaddch((LINES - 3) - l, k, '|'); } - - AC(refresh()); } inline static void @@ -382,9 +378,12 @@ do_curses(void *arg) AC(curs_set(0)); AC(erase()); while (!VSIG_int && !VSIG_term && !VSIG_hup) { + + AC(erase()); PTOK(pthread_mutex_lock(&mtx)); update(); PTOK(pthread_mutex_unlock(&mtx)); + AC(refresh()); assert(ms_delay > 0); timeout(ms_delay); From phk at FreeBSD.org Mon Dec 11 13:35:08 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 11 Dec 2023 13:35:08 +0000 (UTC) Subject: [master] 835284ee8 Stop reading ``-r file`` on signals Message-ID: <20231211133508.6880D12365E@lists.varnish-cache.org> commit 835284ee871d2076c95addcdd78de19d16e398b0 Author: Poul-Henning Kamp Date: Mon Dec 11 13:20:20 2023 +0000 Stop reading ``-r file`` on signals diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index fa26a5ca8..5bdc59956 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -232,6 +232,9 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], (void)priv; for (tr = pt[0]; tr != NULL; tr = *++pt) { + if (VSIG_int || VSIG_term || VSIG_hup) + return (-1); + if (tr->reason == VSL_r_esi) /* Skip ESI requests */ continue; From phk at FreeBSD.org Mon Dec 11 13:35:08 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 11 Dec 2023 13:35:08 +0000 (UTC) Subject: [master] e0d796b65 Uniformly put vsl_t* timestamps under mutex (Found by Coverity) Message-ID: <20231211133508.8799B123661@lists.varnish-cache.org> commit e0d796b65dcdada1bb0c61da37d0ac53ce12b1d2 Author: Poul-Henning Kamp Date: Mon Dec 11 13:24:25 2023 +0000 Uniformly put vsl_t* timestamps under mutex (Found by Coverity) diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 5bdc59956..c8c5d9568 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -217,6 +217,26 @@ upd_vsl_ts(const char *p) vsl_ts = vmax_t(double, vsl_ts, strtod(p + 1, NULL)); } +static void +deloran(void) +{ + int i; + double t = VTIM_mono(); + + if (vsl_t0 == 0) + vsl_to = vsl_t0 = vsl_ts; + + assert(t > t0); + vsl_to = vsl_t0 + (t - t0) * timebend; + + if (vsl_ts > vsl_to) { + double when = VTIM_real() + vsl_ts - vsl_to; + struct timespec ts = VTIM_timespec(when); + i = pthread_cond_timedwait(&timebend_cv, &mtx, &ts); + assert(i == 0 || i == ETIMEDOUT); + } +} + static int v_matchproto_ (VSLQ_dispatch_f) accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], void *priv) @@ -225,8 +245,8 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], unsigned u; double value = 0; struct VSL_transaction *tr; - double t; const char *tsp; + enum vsl_status stat; (void)vsl; (void)priv; @@ -235,25 +255,26 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (VSIG_int || VSIG_term || VSIG_hup) return (-1); - if (tr->reason == VSL_r_esi) + if (tr->reason == VSL_r_esi) { /* Skip ESI requests */ continue; + } hit = 0; skip = 0; match = 0; tsp = NULL; while (skip == 0) { - i = VSL_Next(tr->c); - if (i == -3) { - /* overrun - need to skip forward */ + stat = VSL_Next(tr->c); + if (stat == vsl_e_overrun) { + /* need to skip forward */ PTOK(pthread_mutex_lock(&mtx)); vsl_to = vsl_t0 = vsl_ts = 0; t0 = VTIM_mono(); PTOK(pthread_mutex_unlock(&mtx)); break; } - if (i != 1) + if (stat != vsl_more) break; /* get the value we want and register if it's a hit */ @@ -341,29 +362,10 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (++next_hist == HIST_N) { next_hist = 0; } + if (vsl_ts >= vsl_to) + deloran(); PTOK(pthread_mutex_unlock(&mtx)); } - - if (vsl_ts < vsl_to) - return (0); - - t = VTIM_mono(); - - PTOK(pthread_mutex_lock(&mtx)); - if (vsl_t0 == 0) - vsl_to = vsl_t0 = vsl_ts; - - assert(t > t0); - vsl_to = vsl_t0 + (t - t0) * timebend; - - if (vsl_ts > vsl_to) { - double when = VTIM_real() + vsl_ts - vsl_to; - struct timespec ts = VTIM_timespec(when); - i = pthread_cond_timedwait(&timebend_cv, &mtx, &ts); - assert(i == 0 || i == ETIMEDOUT); - } - PTOK(pthread_mutex_unlock(&mtx)); - return (0); } From phk at FreeBSD.org Mon Dec 11 13:57:04 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 11 Dec 2023 13:57:04 +0000 (UTC) Subject: [master] c1aa3c20b Proper locking around n_backend. Found by Coverity. Message-ID: <20231211135704.F39C9124424@lists.varnish-cache.org> commit c1aa3c20b4377e1eedb545fdf15edd60c4b72c9a Author: Poul-Henning Kamp Date: Mon Dec 11 13:45:34 2023 +0000 Proper locking around n_backend. Found by Coverity. diff --git a/vmod/vmod_directors.c b/vmod/vmod_directors.c index 2334e0c14..44b98217f 100644 --- a/vmod/vmod_directors.c +++ b/vmod/vmod_directors.c @@ -94,9 +94,11 @@ vdir_release(struct vdir *vd) CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); + vdir_wrlock(vd); for (u = 0; u < vd->n_backend; u++) VRT_Assign_Backend(&vd->backend[u], NULL); vd->n_backend = 0; + vdir_unlock(vd); } void diff --git a/vmod/vmod_directors_round_robin.c b/vmod/vmod_directors_round_robin.c index 015e060da..f2ee59f04 100644 --- a/vmod/vmod_directors_round_robin.c +++ b/vmod/vmod_directors_round_robin.c @@ -90,9 +90,9 @@ vmod_rr_resolve(VRT_CTX, VCL_BACKEND dir) break; } rr->nxt = nxt; - vdir_unlock(rr->vd); if (u == rr->vd->n_backend) be = NULL; + vdir_unlock(rr->vd); return (be); } From nils.goroll at uplex.de Mon Dec 11 14:07:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Dec 2023 14:07:06 +0000 (UTC) Subject: [master] 13cf51e70 Rename (struct req).filter_list to vdp_filter_list Message-ID: <20231211140707.20A29124A63@lists.varnish-cache.org> commit 13cf51e70c00b912ce39110d7eff50ccc01b7bb9 Author: Nils Goroll Date: Mon Dec 11 08:10:54 2023 +0100 Rename (struct req).filter_list to vdp_filter_list for consistency with (struct busyobj).vfp_filter_list diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index b6af25537..a7fcac6bf 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -533,7 +533,7 @@ struct req { /* Deliver pipeline */ struct vdp_ctx *vdc; - const char *filter_list; + const char *vdp_filter_list; /* Transaction VSL buffer */ struct vsl_log vsl[1]; diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index eb1700df1..cfea8fce2 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -251,7 +251,7 @@ Req_Rollback(VRT_CTX) if (IS_TOPREQ(req)) VCL_TaskEnter(req->top->privs); HTTP_Clone(req->http, req->http0); - req->filter_list = NULL; + req->vdp_filter_list = NULL; req->vcf = NULL; if (WS_Overflowed(req->ws)) req->wrk->stats->ws_client_overflow++; diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index b49f4dadd..bb7a43d4e 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -297,7 +297,7 @@ cnt_vclfail(struct worker *wrk, struct req *req) } req->req_step = R_STP_SYNTH; req->doclose = SC_VCL_FAILURE; - req->filter_list = NULL; + req->vdp_filter_list = NULL; return (REQ_FSM_MORE); } @@ -327,7 +327,7 @@ cnt_synth(struct worker *wrk, struct req *req) Resp_Setup_Synth(req); - req->filter_list = NULL; + req->vdp_filter_list = NULL; synth_body = VSB_new_auto(); AN(synth_body); @@ -461,10 +461,10 @@ cnt_transmit(struct worker *wrk, struct req *req) } VDP_Init(req->vdc, req->wrk, req->vsl, req); - if (req->filter_list == NULL) - req->filter_list = resp_Get_Filter_List(req); - if (req->filter_list == NULL || - VCL_StackVDP(req, req->vcl, req->filter_list)) { + if (req->vdp_filter_list == NULL) + req->vdp_filter_list = resp_Get_Filter_List(req); + if (req->vdp_filter_list == NULL || + VCL_StackVDP(req, req->vcl, req->vdp_filter_list)) { VSLb(req->vsl, SLT_Error, "Failure to push processors"); req->doclose = SC_OVERLOAD; } else { @@ -513,7 +513,7 @@ cnt_transmit(struct worker *wrk, struct req *req) (void)HSH_DerefObjCore(wrk, &req->objcore, HSH_RUSH_POLICY); http_Teardown(req->resp); - req->filter_list = NULL; + req->vdp_filter_list = NULL; req->res_mode = 0; return (REQ_FSM_DONE); } diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c index bcd7ed1b3..e0ca18321 100644 --- a/bin/varnishd/cache/cache_vrt_filter.c +++ b/bin/varnishd/cache/cache_vrt_filter.c @@ -448,4 +448,4 @@ resp_Get_Filter_List(struct req *req) } FILTER_VAR(beresp, bo, VBF_Get_Filter_List, vfp_filter_list) -FILTER_VAR(resp, req, resp_Get_Filter_List, filter_list) +FILTER_VAR(resp, req, resp_Get_Filter_List, vdp_filter_list) diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 8810fcc51..c1282c149 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -1002,7 +1002,7 @@ VRT_r_resp_is_streaming(VRT_CTX) static inline int resp_filter_fixed(VRT_CTX, const char *s) { - if (ctx->req->filter_list == NULL) + if (ctx->req->vdp_filter_list == NULL) return (0); VRT_fail(ctx, "resp.filters are already fixed, %s is undefined", s); return (1); diff --git a/include/vrt.h b/include/vrt.h index fe292e3ce..2a478b372 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -58,6 +58,7 @@ * binary/load-time compatible, increment MAJOR version * * NEXT (2024-03-15) + * [cache.h] (struct req).filter_list renamed to vdp_filter_list * 18.1 (2023-12-05) * vbf_objiterate() implementation changed #4013 * 18.0 (2023-09-15) From nils.goroll at uplex.de Mon Dec 11 16:36:05 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Dec 2023 16:36:05 +0000 (UTC) Subject: [master] 24b434383 Test that we do not explode hitting max_esi_depth Message-ID: <20231211163606.023FB101B42@lists.varnish-cache.org> commit 24b434383c616639d5aa9be9b5ba3647a418d64c Author: Nils Goroll Date: Mon Dec 11 17:33:44 2023 +0100 Test that we do not explode hitting max_esi_depth diff --git a/bin/varnishtest/tests/e00029.vtc b/bin/varnishtest/tests/e00029.vtc new file mode 100644 index 000000000..556c05cca --- /dev/null +++ b/bin/varnishtest/tests/e00029.vtc @@ -0,0 +1,41 @@ +varnishtest "ESI max_esi_depth" + +# test that the default stack size is sufficent for hitting +# max_esi_depth + +server s1 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunked {} + chunkedlen 1024 + chunked {} + chunkedlen 1024 + chunked {} + chunkedlen 1024 + chunked {} + chunkedlen 0 +} -start + +varnish v1 -arg "-p feature=+esi_include_onerror" -vcl+backend { + sub vcl_backend_response { + set beresp.do_esi = true; + set beresp.do_gzip = true; + } +} -start + +logexpect l1 -v v1 -g raw { + expect * * VCL_Error {^ESI depth limit reached} + expect 0 = VCL_Error {^ESI depth limit reached} + expect * * VCL_Error {^ESI depth limit reached} + expect 0 = VCL_Error {^ESI depth limit reached} + expect * * VCL_Error {^ESI depth limit reached} + expect 0 = VCL_Error {^ESI depth limit reached} + #... more +} -start + +client c1 { + txreq -hdr "Host: foo" -hdr "Accept-Encoding: gzip" + rxresp +} -run + +logexpect l1 -wait \ No newline at end of file From dridi.boukelmoune at gmail.com Mon Dec 18 08:51:09 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Dec 2023 08:51:09 +0000 (UTC) Subject: [6.0] 83881e9f0 req_fsm: Use status 408 for reset streams Message-ID: <20231218085110.1DECD1170E6@lists.varnish-cache.org> commit 83881e9f0e897603ab7eb3aea1fd0f5a4a7f7a5c Author: Dridi Boukelmoune Date: Fri Dec 1 16:25:00 2023 +0100 req_fsm: Use status 408 for reset streams The 503 synth and 500 minimal response status codes are too misleading in this context, where the failure is attributed to the client. Among existing 4XX status codes, this is the closest if we stretch the timeout definition to "didn't complete rapidly enough before the client went away". Conflicts: bin/varnishd/cache/cache_req_fsm.c There is no minimal 500 response on this branch. diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 43d3e6d68..c71c5e67f 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -275,8 +275,13 @@ cnt_vclfail(const struct worker *wrk, struct req *req) Req_Rollback(req); - req->err_code = 503; - req->err_reason = "VCL failed"; + if (req->req_reset) { + req->err_code = 408; + req->err_reason = "Client disconnected"; + } else { + req->err_code = 503; + req->err_reason = "VCL failed"; + } req->req_step = R_STP_SYNTH; req->doclose = SC_VCL_FAILURE; return (REQ_FSM_MORE); diff --git a/bin/varnishtest/tests/t02025.vtc b/bin/varnishtest/tests/t02025.vtc index 578dbf5c8..4b83fe95b 100644 --- a/bin/varnishtest/tests/t02025.vtc +++ b/bin/varnishtest/tests/t02025.vtc @@ -46,7 +46,7 @@ varnish v1 -expect req_reset == 1 # is interpreted as before a second elapsed. Session VXIDs showing up # numerous times become increasingly more suspicious. The format can of # course be extended to add anything else useful for data mining. -shell -expect "1000 ${localhost}" { +shell -expect "1000 ${localhost} 408" { varnishncsa -n ${v1_name} -d \ - -q 'Timestamp:Reset[2] < 1.0' -F '%{VSL:Begin[2]}x %h' + -q 'Timestamp:Reset[2] < 1.0' -F '%{VSL:Begin[2]}x %h %s' } diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst index b52956226..83ffc36e2 100644 --- a/doc/sphinx/reference/vsl.rst +++ b/doc/sphinx/reference/vsl.rst @@ -74,7 +74,8 @@ Restart Reset The client closed its connection, reset its stream or caused a stream error that forced Varnish to reset the stream. Request - processing is interrupted and considered failed. + processing is interrupted and considered failed, with a 408 + "Request Timeout" status code. Pipe handling timestamps ~~~~~~~~~~~~~~~~~~~~~~~~ From dridi at varni.sh Tue Dec 19 06:30:59 2023 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 19 Dec 2023 06:30:59 +0000 Subject: [master] 5583667fe VRT minor version bump for #4013 In-Reply-To: References: <20231205204718.2EC4811F918@lists.varnish-cache.org> <95ca722c-f160-da53-e224-38e01f75cd7e@uplex.de>

Message-ID: On Thu, Dec 7, 2023 at 10:02?AM Nils Goroll wrote: > > Yes, I probably should add a comment to vrt.h regarding the vbf_objiterate() change. > > On 07.12.23 10:00, Dridi Boukelmoune wrote: > > VRT is minor-bumped for additive changes to the not-clearly-defined VRT ABI. > > I agree to not-clearly-defined. Is the stevedore API part of the VRT ABI? > Directly or indirectly? The stevedore API is exposed in cache_varnishd.h and is clearly not $VRT abi. > Regarding "additive changes", I read > > * Whenever something is deleted or changed in a way which is not > * binary/load-time compatible, increment MAJOR version > > as applicable for an incompatible change. > > > Overall, which harm is caused by a minor bump given that we practive unnecessary > major bumps for releases "to be on the safe side"? Another reason for major bumps "safety" is also that we branch out. If trunk major-bumps to 18.0 when we release 7.4, and we minor-bump trunk, then if in the future we need to minor-bump 7.4 after a security release for example, then we have two competing 18.1 VRT versions. I have suggested in the past to add namespaces to VRT so we could have something like this: - varnish-cache-trunk 18.0 - varnish-cache-7.4 18.0 - varnish-cache-6.0 7.1 - varnish-enterprise-6.0 7.5 On top of the current X.Y rule we could add an exact match requirement for the namespace, to make sure that a VMOD built for a given ABI could not load on a different one that happens to share the same VRT version. This way, we can drop the "just to be safe" attitude we have today and start bumping trunk only when relevant. In this specific case, that's irrelevant since this is not a VRT change. Dridi From dridi.boukelmoune at gmail.com Tue Dec 19 08:22:10 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 19 Dec 2023 08:22:10 +0000 (UTC) Subject: [master] a0e31b706 vtc: Polish c{97..99} tests Message-ID: <20231219082210.3097711D98D@lists.varnish-cache.org> commit a0e31b7060fff4ffcbe6e317cb942603e951fa20 Author: Dridi Boukelmoune Date: Thu Sep 28 08:53:26 2023 +0200 vtc: Polish c{97..99} tests diff --git a/bin/varnishtest/tests/c00097.vtc b/bin/varnishtest/tests/c00097.vtc index afc77fbd9..796c6481c 100644 --- a/bin/varnishtest/tests/c00097.vtc +++ b/bin/varnishtest/tests/c00097.vtc @@ -53,7 +53,7 @@ client c4 { } -start # Wait until c2-c4 are on the waitinglist -delay 1 +varnish v1 -vsl_catchup varnish v1 -expect busy_sleep == 3 # Open up the response headers from s1, and as a result HSH_Unbusy diff --git a/bin/varnishtest/tests/c00098.vtc b/bin/varnishtest/tests/c00098.vtc index 5b34d3a75..148245275 100644 --- a/bin/varnishtest/tests/c00098.vtc +++ b/bin/varnishtest/tests/c00098.vtc @@ -70,17 +70,17 @@ server s6 { varnish v1 -arg "-p thread_pools=1" -arg "-p thread_pool_min=30" -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -arg "-p debug=+waitinglist" -vcl+backend { sub vcl_backend_fetch { - if (bereq.http.client == "1") { + if (bereq.http.user-agent == "c1") { set bereq.backend = s1; - } else if (bereq.http.client == "2") { + } else if (bereq.http.user-agent == "c2") { set bereq.backend = s2; - } else if (bereq.http.client == "3") { + } else if (bereq.http.user-agent == "c3") { set bereq.backend = s3; - } else if (bereq.http.client == "4") { + } else if (bereq.http.user-agent == "c4") { set bereq.backend = s4; - } else if (bereq.http.client == "5") { + } else if (bereq.http.user-agent == "c5") { set bereq.backend = s5; - } else if (bereq.http.client == "6") { + } else if (bereq.http.user-agent == "c6") { set bereq.backend = s6; } } @@ -90,7 +90,7 @@ varnish v1 -arg "-p thread_pools=1" -arg "-p thread_pool_min=30" -arg "-p rush_e } -start client c1 { - txreq -url /hfp -hdr "Client: 1" + txreq rxresp } -start @@ -98,32 +98,32 @@ client c1 { barrier b1 sync client c2 { - txreq -url /hfp -hdr "Client: 2" + txreq rxresp } -start client c3 { - txreq -url /hfp -hdr "Client: 3" + txreq rxresp } -start client c4 { - txreq -url /hfp -hdr "Client: 4" + txreq rxresp } -start client c5 { - txreq -url /hfp -hdr "Client: 5" + txreq rxresp } -start client c6 { - txreq -url /hfp -hdr "Client: 6" + txreq rxresp } -start # Wait until c2-c6 are on the waitinglist -delay 1 +varnish v1 -vsl_catchup varnish v1 -expect busy_sleep == 5 # Open up the response headers from s1, and as a result HSH_Unbusy diff --git a/bin/varnishtest/tests/c00099.vtc b/bin/varnishtest/tests/c00099.vtc index 4bbd904a0..9ee177597 100644 --- a/bin/varnishtest/tests/c00099.vtc +++ b/bin/varnishtest/tests/c00099.vtc @@ -70,17 +70,17 @@ server s6 { varnish v1 -arg "-p thread_pools=1" -arg "-p thread_pool_min=30" -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -arg "-p debug=+waitinglist" -vcl+backend { sub vcl_backend_fetch { - if (bereq.http.client == "1") { + if (bereq.http.user-agent == "c1") { set bereq.backend = s1; - } else if (bereq.http.client == "2") { + } else if (bereq.http.user-agent == "c2") { set bereq.backend = s2; - } else if (bereq.http.client == "3") { + } else if (bereq.http.user-agent == "c3") { set bereq.backend = s3; - } else if (bereq.http.client == "4") { + } else if (bereq.http.user-agent == "c4") { set bereq.backend = s4; - } else if (bereq.http.client == "5") { + } else if (bereq.http.user-agent == "c5") { set bereq.backend = s5; - } else if (bereq.http.client == "6") { + } else if (bereq.http.user-agent == "c6") { set bereq.backend = s6; } } @@ -90,7 +90,7 @@ varnish v1 -arg "-p thread_pools=1" -arg "-p thread_pool_min=30" -arg "-p rush_e } -start client c1 { - txreq -url /hfm -hdr "Client: 1" + txreq rxresp } -start @@ -98,32 +98,32 @@ client c1 { barrier b1 sync client c2 { - txreq -url /hfm -hdr "Client: 2" + txreq rxresp } -start client c3 { - txreq -url /hfm -hdr "Client: 3" + txreq rxresp } -start client c4 { - txreq -url /hfm -hdr "Client: 4" + txreq rxresp } -start client c5 { - txreq -url /hfm -hdr "Client: 5" + txreq rxresp } -start client c6 { - txreq -url /hfm -hdr "Client: 6" + txreq rxresp } -start # Wait until c2-c6 are on the waitinglist -delay 1 +varnish v1 -vsl_catchup varnish v1 -expect busy_sleep == 5 # Open up the response headers from s1, and as a result HSH_Unbusy From nils.goroll at uplex.de Thu Dec 21 09:33:11 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 21 Dec 2023 09:33:11 +0000 (UTC) Subject: [master] 2cae35978 For e29.vtc, also execute a reasonably complex regex match Message-ID: <20231221093311.2314F106372@lists.varnish-cache.org> commit 2cae35978727b706a02b77fce0c82e2863649687 Author: Nils Goroll Date: Mon Dec 11 19:54:32 2023 +0100 For e29.vtc, also execute a reasonably complex regex match diff --git a/bin/varnishtest/tests/e00029.vtc b/bin/varnishtest/tests/e00029.vtc index 556c05cca..40b266dfe 100644 --- a/bin/varnishtest/tests/e00029.vtc +++ b/bin/varnishtest/tests/e00029.vtc @@ -17,6 +17,11 @@ server s1 { } -start varnish v1 -arg "-p feature=+esi_include_onerror" -vcl+backend { + sub vcl_deliver { + # ref r01576.vtc + set resp.http.torture = + req.url + "${string,repeat,128,AB}" ~ "^/(A{1,2}B)+$"; + } sub vcl_backend_response { set beresp.do_esi = true; set beresp.do_gzip = true; @@ -24,6 +29,7 @@ varnish v1 -arg "-p feature=+esi_include_onerror" -vcl+backend { } -start logexpect l1 -v v1 -g raw { + fail add * VCL_Error {Regexp matching failed} expect * * VCL_Error {^ESI depth limit reached} expect 0 = VCL_Error {^ESI depth limit reached} expect * * VCL_Error {^ESI depth limit reached} @@ -31,11 +37,13 @@ logexpect l1 -v v1 -g raw { expect * * VCL_Error {^ESI depth limit reached} expect 0 = VCL_Error {^ESI depth limit reached} #... more + fail clear } -start client c1 { txreq -hdr "Host: foo" -hdr "Accept-Encoding: gzip" rxresp + expect resp.status == 200 } -run logexpect l1 -wait \ No newline at end of file From nils.goroll at uplex.de Thu Dec 21 16:18:10 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 21 Dec 2023 16:18:10 +0000 (UTC) Subject: [master] 6c8340126 Momentarily revert "For e29.vtc, also execute a reasonably complex regex match" Message-ID: <20231221161810.40207116289@lists.varnish-cache.org> commit 6c83401261df3365a2b55195e6c9ecf94c72dc35 Author: Nils Goroll Date: Thu Dec 21 17:14:20 2023 +0100 Momentarily revert "For e29.vtc, also execute a reasonably complex regex match" This reverts commit 2cae35978727b706a02b77fce0c82e2863649687. See #4033 diff --git a/bin/varnishtest/tests/e00029.vtc b/bin/varnishtest/tests/e00029.vtc index 40b266dfe..556c05cca 100644 --- a/bin/varnishtest/tests/e00029.vtc +++ b/bin/varnishtest/tests/e00029.vtc @@ -17,11 +17,6 @@ server s1 { } -start varnish v1 -arg "-p feature=+esi_include_onerror" -vcl+backend { - sub vcl_deliver { - # ref r01576.vtc - set resp.http.torture = - req.url + "${string,repeat,128,AB}" ~ "^/(A{1,2}B)+$"; - } sub vcl_backend_response { set beresp.do_esi = true; set beresp.do_gzip = true; @@ -29,7 +24,6 @@ varnish v1 -arg "-p feature=+esi_include_onerror" -vcl+backend { } -start logexpect l1 -v v1 -g raw { - fail add * VCL_Error {Regexp matching failed} expect * * VCL_Error {^ESI depth limit reached} expect 0 = VCL_Error {^ESI depth limit reached} expect * * VCL_Error {^ESI depth limit reached} @@ -37,13 +31,11 @@ logexpect l1 -v v1 -g raw { expect * * VCL_Error {^ESI depth limit reached} expect 0 = VCL_Error {^ESI depth limit reached} #... more - fail clear } -start client c1 { txreq -hdr "Host: foo" -hdr "Accept-Encoding: gzip" rxresp - expect resp.status == 200 } -run logexpect l1 -wait \ No newline at end of file From nils.goroll at uplex.de Wed Dec 27 09:28:09 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 27 Dec 2023 09:28:09 +0000 (UTC) Subject: [master] b5d8e5713 Move the V1L-based VDP to cache_http1_line.c Message-ID: <20231227092809.A61721094AF@lists.varnish-cache.org> commit b5d8e57138c488efcc47df6f41a150063f1cf178 Author: Nils Goroll Date: Wed Dec 27 10:16:06 2023 +0100 Move the V1L-based VDP to cache_http1_line.c This is to prepare for reuse, but arguably the new home might actually be the better place? diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h index a9f905376..d6639f273 100644 --- a/bin/varnishd/http1/cache_http1.h +++ b/bin/varnishd/http1/cache_http1.h @@ -65,3 +65,4 @@ void V1L_Open(struct worker *, struct ws *, int *fd, struct vsl_log *, stream_close_t V1L_Flush(const struct worker *w); stream_close_t V1L_Close(struct worker *w, uint64_t *cnt); size_t V1L_Write(const struct worker *w, const void *ptr, ssize_t len); +extern const struct vdp * const v1l_vdp; diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index 6bed2b08d..657ce9e24 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -39,31 +39,6 @@ /*--------------------------------------------------------------------*/ -static int v_matchproto_(vdp_bytes_f) -v1d_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, - const void *ptr, ssize_t len) -{ - ssize_t wl = 0; - - CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - (void)priv; - - AZ(vdc->nxt); /* always at the bottom of the pile */ - - if (len > 0) - wl = V1L_Write(vdc->wrk, ptr, len); - if (act > VDP_NULL && V1L_Flush(vdc->wrk) != SC_NULL) - return (-1); - if (len != wl) - return (-1); - return (0); -} - -static const struct vdp v1d_vdp = { - .name = "V1B", - .bytes = v1d_bytes, -}; - static void v1d_error(struct req *req, const char *msg) { @@ -122,7 +97,7 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody) } INIT_OBJ(ctx, VRT_CTX_MAGIC); VCL_Req2Ctx(ctx, req); - if (VDP_Push(ctx, req->vdc, req->ws, &v1d_vdp, NULL)) { + if (VDP_Push(ctx, req->vdc, req->ws, v1l_vdp, NULL)) { v1d_error(req, "Failure to push v1d processor"); return; } diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index 8c6f53b16..9c13fb90d 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -43,6 +43,7 @@ #include #include "cache/cache_varnishd.h" +#include "cache/cache_filter.h" #include @@ -339,3 +340,32 @@ V1L_EndChunk(const struct worker *wrk) v1l->cliov = 0; (void)V1L_Write(wrk, "0\r\n\r\n", -1); } + +/*-------------------------------------------------------------------- + * VDP using V1L + */ + +static int v_matchproto_(vdp_bytes_f) +v1l_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, + const void *ptr, ssize_t len) +{ + ssize_t wl = 0; + + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + (void)priv; + + AZ(vdc->nxt); /* always at the bottom of the pile */ + + if (len > 0) + wl = V1L_Write(vdc->wrk, ptr, len); + if (act > VDP_NULL && V1L_Flush(vdc->wrk) != SC_NULL) + return (-1); + if (len != wl) + return (-1); + return (0); +} + +const struct vdp * const v1l_vdp = &(struct vdp){ + .name = "V1B", + .bytes = v1l_bytes, +}; From nils.goroll at uplex.de Wed Dec 27 09:28:09 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 27 Dec 2023 09:28:09 +0000 (UTC) Subject: [master] 8d0b37d65 Rename the V1L VDP to get in line with oder VDPs Message-ID: <20231227092809.BC69B1094B5@lists.varnish-cache.org> commit 8d0b37d65da9348cab1183e9169e8d377abc52cf Author: Nils Goroll Date: Wed Dec 27 10:19:29 2023 +0100 Rename the V1L VDP to get in line with oder VDPs diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h index d6639f273..dc454bcab 100644 --- a/bin/varnishd/http1/cache_http1.h +++ b/bin/varnishd/http1/cache_http1.h @@ -65,4 +65,4 @@ void V1L_Open(struct worker *, struct ws *, int *fd, struct vsl_log *, stream_close_t V1L_Flush(const struct worker *w); stream_close_t V1L_Close(struct worker *w, uint64_t *cnt); size_t V1L_Write(const struct worker *w, const void *ptr, ssize_t len); -extern const struct vdp * const v1l_vdp; +extern const struct vdp * const VDP_v1l; diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index 657ce9e24..d82443367 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -97,7 +97,7 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody) } INIT_OBJ(ctx, VRT_CTX_MAGIC); VCL_Req2Ctx(ctx, req); - if (VDP_Push(ctx, req->vdc, req->ws, v1l_vdp, NULL)) { + if (VDP_Push(ctx, req->vdc, req->ws, VDP_v1l, NULL)) { v1d_error(req, "Failure to push v1d processor"); return; } diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index 9c13fb90d..82ae2fe26 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -365,7 +365,7 @@ v1l_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, return (0); } -const struct vdp * const v1l_vdp = &(struct vdp){ +const struct vdp * const VDP_v1l = &(struct vdp){ .name = "V1B", .bytes = v1l_bytes, }; From nils.goroll at uplex.de Wed Dec 27 14:07:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 27 Dec 2023 14:07:06 +0000 (UTC) Subject: [master] 610d17740 Simplify initialization of the H2 VDP Message-ID: <20231227140706.2A7801154A1@lists.varnish-cache.org> commit 610d17740861a521fcf86d656bdc35f4199484c0 Author: Nils Goroll Date: Wed Dec 27 10:49:01 2023 +0100 Simplify initialization of the H2 VDP Initialize the private pointer directly, instead of indirectly via the request struct. diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 267ea91ea..cdd356d7d 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -75,12 +75,14 @@ V2D_Init(void) static int v_matchproto_(vdp_init_f) h2_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) { + struct h2_req *r2; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + AN(priv); + CAST_OBJ_NOTNULL(r2, *priv, H2_REQ_MAGIC); + (void)r2; (void)oc; - CHECK_OBJ_NOTNULL(vdc->req, REQ_MAGIC); - *priv = vdc->req->transport_priv; return (0); } @@ -346,7 +348,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) if (sendbody) { INIT_OBJ(ctx, VRT_CTX_MAGIC); VCL_Req2Ctx(ctx, req); - if (!VDP_Push(ctx, req->vdc, req->ws, &h2_vdp, NULL)) + if (!VDP_Push(ctx, req->vdc, req->ws, &h2_vdp, r2)) (void)VDP_DeliverObj(req->vdc, req->objcore); } From nils.goroll at uplex.de Sun Dec 31 16:59:09 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 31 Dec 2023 16:59:09 +0000 (UTC) Subject: [master] 5752421e0 Polish: Consistent use of sizeof in SML_allocobj() Message-ID: <20231231165909.E7F6F10D102@lists.varnish-cache.org> commit 5752421e0219f1563ef432b748a572342f4819de Author: Nils Goroll Date: Sun Dec 31 17:31:22 2023 +0100 Polish: Consistent use of sizeof in SML_allocobj() There are two sizeof calls here with the same semantics, make them the same. diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index b91fbcb32..c34f6e394 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -148,7 +148,7 @@ SML_allocobj(struct worker *wrk, const struct stevedore *stv, AN(stv->sml_alloc); - ltot = sizeof(struct object) + PRNDUP(wsl); + ltot = sizeof(*o) + PRNDUP(wsl); do { st = stv->sml_alloc(stv, ltot); From nils.goroll at uplex.de Sun Dec 31 16:59:10 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 31 Dec 2023 16:59:10 +0000 (UTC) Subject: [master] 532e9c207 Clarify and fix STV_NewObject() len/wsl argument use Message-ID: <20231231165910.1BC1E10D105@lists.varnish-cache.org> commit 532e9c207c93c954807422e3c748ff668b140333 Author: Nils Goroll Date: Sun Dec 31 17:09:17 2023 +0100 Clarify and fix STV_NewObject() len/wsl argument use It was not documented explicitly anywhere, but the len/wsl argument to STV_NewObject() is the amount of space to allocate for all OBJ_VARATTRs combined which are going to be written to this object. For now, these are OA_HEADERS and OA_VARY only (see include/tbl/obj_attr.h). So for one, there is no reason to force this argument be greater than zero when we know that no OBJ_VARATTRs are going to be set. Secondly, this might actually represent a minor shortcoming of the stevedore API, because the amount of space which a stevedore implementation actually needs might be larger than what the simple stevedores use. So for now, the semantics of this argument are, more specifically, "the equivalent of space for OBJ_VARATTRs as for simple storage". In practice, this probably does not matter much... But even before this clarification, the API was not used consistently: For the call from vrb_pull(), the maximum request body size (to cache) was used as the wsl argument, yet it has nothing to do with the object body size, it specifies the amount of space to allocate for variable sized object attributes (see above). For the call from cnt_synth(), a somehow arbitrary value of 1KB was used. In both cases, the amount of space actually required is zero, because the only attribute used on the objects created is OA_LEN, which is fixed and thus always present. diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index 3cbd62734..70996a2f1 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -59,7 +59,6 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) enum vfp_status vfps = VFP_ERROR; const struct stevedore *stv; ssize_t req_bodybytes = 0; - unsigned hint; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -77,8 +76,7 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) req->storage = NULL; - hint = maxsize > 0 ? maxsize : 1; - if (STV_NewObject(req->wrk, req->body_oc, stv, hint) == 0) { + if (STV_NewObject(req->wrk, req->body_oc, stv, 0) == 0) { req->req_body_status = BS_ERROR; HSH_DerefBoc(req->wrk, req->body_oc); AZ(HSH_DerefObjCore(req->wrk, &req->body_oc, 0)); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index bb7a43d4e..c7469f5eb 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -383,7 +383,7 @@ cnt_synth(struct worker *wrk, struct req *req) req->objcore = HSH_Private(wrk); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); szl = -1; - if (STV_NewObject(wrk, req->objcore, stv_transient, 1024)) { + if (STV_NewObject(wrk, req->objcore, stv_transient, 0)) { body = VSB_data(synth_body); szl = VSB_len(synth_body); assert(szl >= 0); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 404ebc894..062041f49 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -564,6 +564,7 @@ const struct stevedore *STV_next(void); int STV_BanInfoDrop(const uint8_t *ban, unsigned len); int STV_BanInfoNew(const uint8_t *ban, unsigned len); void STV_BanExport(const uint8_t *banlist, unsigned len); +// STV_NewObject() len is space for OBJ_VARATTR int STV_NewObject(struct worker *, struct objcore *, const struct stevedore *, unsigned len); diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 9c2847997..7813c424f 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -84,7 +84,6 @@ STV_NewObject(struct worker *wrk, struct objcore *oc, CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); - assert(wsl > 0); wrk->strangelove = cache_param->nuke_limit; AN(stv->allocobj);