From dridi.boukelmoune at gmail.com Tue Jun 6 14:16:10 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 6 Jun 2023 14:16:10 +0000 (UTC) Subject: [master] d915e2201 mgt_vcc: Missing new line in error message Message-ID: <20230606141610.BD3B810430E@lists.varnish-cache.org> commit d915e22017a7778dd19fa944fa165bcacb5cc2c5 Author: Dridi Boukelmoune Date: Tue Jun 6 16:14:52 2023 +0200 mgt_vcc: Missing new line in error message diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 6561e7e77..0fdc53ab0 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -256,7 +256,8 @@ mgt_vcc_touchfile(const char *fn, struct vsb *sb) i = open(fn, O_WRONLY|O_CREAT|O_TRUNC, 0640); if (i < 0) { - VSB_printf(sb, "Failed to create %s: %s", fn, VAS_errtxt(errno)); + VSB_printf(sb, "Failed to create %s: %s\n", + fn, VAS_errtxt(errno)); return (2); } closefd(&i); From dridi.boukelmoune at gmail.com Wed Jun 7 14:32:10 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 7 Jun 2023 14:32:10 +0000 (UTC) Subject: [master] 0fea7e117 build: Generate VSCs .c and .h files separately Message-ID: <20230607143210.5A1CB10D65F@lists.varnish-cache.org> commit 0fea7e117ec12e2069fc16af0996b039a4065c58 Author: Dridi Boukelmoune Date: Tue May 16 14:12:14 2023 +0200 build: Generate VSCs .c and .h files separately diff --git a/vsc.am b/vsc.am index 0a8da842a..e38d43e19 100644 --- a/vsc.am +++ b/vsc.am @@ -7,7 +7,10 @@ VSC_GEN = $(VSC_SRC:.vsc=.c) $(VSC_SRC:.vsc=.h) $(VSC_GEN): $(VSCTOOL) .vsc.c: - $(AM_V_GEN) $(PYTHON) $(VSCTOOL) -ch $< + $(AM_V_GEN) $(PYTHON) $(VSCTOOL) -c $< + +.vsc.h: + $(AM_V_GEN) $(PYTHON) $(VSCTOOL) -h $< clean-local: vsc-clean From dridi.boukelmoune at gmail.com Wed Jun 7 14:32:10 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 7 Jun 2023 14:32:10 +0000 (UTC) Subject: [master] b701fb54d build: Add $(VSC_GEN) to $(BUILT_SOURCES) Message-ID: <20230607143210.771C410D663@lists.varnish-cache.org> commit b701fb54d69cd2bd30b5fad8dd3db524e4e819fd Author: Dridi Boukelmoune Date: Tue May 16 14:36:22 2023 +0200 build: Add $(VSC_GEN) to $(BUILT_SOURCES) This guarantees that VSC code is generated before compiling anything. Fixes #3928 diff --git a/lib/libvsc/Makefile.am b/lib/libvsc/Makefile.am index 243f82587..9f9ae0373 100644 --- a/lib/libvsc/Makefile.am +++ b/lib/libvsc/Makefile.am @@ -19,4 +19,6 @@ VSC_SRC = \ noinst_LTLIBRARIES = libvsc.la libvsc_la_SOURCES = $(VSC_SRC) +BUILT_SOURCES = $(VSC_GEN) + dist_pkgdata_SCRIPTS = vsctool.py diff --git a/vmod/Makefile.am b/vmod/Makefile.am index a485c0372..694e6c284 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -41,6 +41,8 @@ VSC_SRC = VSC_debug.vsc libvmod_debug_la_SOURCES += $(VSC_SRC) +BUILT_SOURCES = $(VSC_GEN) + # Allow Vmod_wrong*_Data to be exported vmod_debug_symbols_regex = 'Vmod_.*_Data' diff --git a/vsc.am b/vsc.am index e38d43e19..49fe25e43 100644 --- a/vsc.am +++ b/vsc.am @@ -1,6 +1,7 @@ # Generic rule to generate C code from VSC files. VSC files must be listed # in the $(VSC_SRC) variable. The $(VSCTOOL) variable must point to the # location of vsctool.py, normally set up by varnish.m4 at configure time. +# The resulting $(VSC_GEN) variable must be added to $(BUILT_SOURCES). VSC_GEN = $(VSC_SRC:.vsc=.c) $(VSC_SRC:.vsc=.h) From dridi.boukelmoune at gmail.com Wed Jun 7 14:32:10 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 7 Jun 2023 14:32:10 +0000 (UTC) Subject: [master] 0796b6c6d build: Add a .vsc.rst suffix rule to vsc.am Message-ID: <20230607143210.9058310D667@lists.varnish-cache.org> commit 0796b6c6d4a264101708329140af1b9a23d7e0fe Author: Dridi Boukelmoune Date: Mon May 22 15:29:05 2023 +0200 build: Add a .vsc.rst suffix rule to vsc.am Suggested by @nigoroll. diff --git a/vsc.am b/vsc.am index 49fe25e43..b3d7c4bdb 100644 --- a/vsc.am +++ b/vsc.am @@ -1,11 +1,18 @@ # Generic rule to generate C code from VSC files. VSC files must be listed # in the $(VSC_SRC) variable. The $(VSCTOOL) variable must point to the # location of vsctool.py, normally set up by varnish.m4 at configure time. -# The resulting $(VSC_GEN) variable must be added to $(BUILT_SOURCES). +# The resulting $(VSC_GEN) variable must be added to $(BUILT_SOURCES). The +# $(VSC_RST) variable references RST file names for manual pages includes. VSC_GEN = $(VSC_SRC:.vsc=.c) $(VSC_SRC:.vsc=.h) -$(VSC_GEN): $(VSCTOOL) +VSC_RST = $(VSC_SRC:.vsc=.rst) + +_VSC_RST = $(VSC_SRC:.vsc=.rst_) + +$(VSC_GEN) $(VSC_RST): $(VSCTOOL) + +SUFFIXES = .vsc .c .h .rst .vsc.c: $(AM_V_GEN) $(PYTHON) $(VSCTOOL) -c $< @@ -13,7 +20,11 @@ $(VSC_GEN): $(VSCTOOL) .vsc.h: $(AM_V_GEN) $(PYTHON) $(VSCTOOL) -h $< +.vsc.rst: + $(AM_V_GEN) $(PYTHON) $(VSCTOOL) -r $< >${@}_ + @mv ${@}_ $@ + clean-local: vsc-clean vsc-clean: - rm -f $(VSC_GEN) + rm -f $(VSC_GEN) $(VSC_RST) $(_VSC_RST) From dridi.boukelmoune at gmail.com Wed Jun 7 14:32:10 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 7 Jun 2023 14:32:10 +0000 (UTC) Subject: [master] c661cb725 build: Use vsc.am for varnish-counters(3) includes Message-ID: <20230607143210.B456E10D66C@lists.varnish-cache.org> commit c661cb7253b319f4dfd4d14114a6dfc2d4af6cfd Author: Dridi Boukelmoune Date: Mon May 22 17:40:32 2023 +0200 build: Use vsc.am for varnish-counters(3) includes With that we lose the ability to rebuild the manual from the doc/sphinx directory if the sources were modified, just like any other dependency on $(top_builddir) sources. This one is linked once and for all, but doesn't fit the link_srcdir target. diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 96fc5fc1a..2dcdcd55f 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -85,22 +85,8 @@ include/params.rst: $(top_builddir)/bin/varnishd/varnishd mv -f ${@}_ ${@} BUILT_SOURCES += include/params.rst -COUNTERS = \ - $(top_srcdir)/lib/libvsc/VSC_main.vsc \ - $(top_srcdir)/lib/libvsc/VSC_mgt.vsc \ - $(top_srcdir)/lib/libvsc/VSC_mempool.vsc \ - $(top_srcdir)/lib/libvsc/VSC_sma.vsc \ - $(top_srcdir)/lib/libvsc/VSC_smu.vsc \ - $(top_srcdir)/lib/libvsc/VSC_smf.vsc \ - $(top_srcdir)/lib/libvsc/VSC_vbe.vsc \ - $(top_srcdir)/lib/libvsc/VSC_lck.vsc - -include/counters.rst: $(top_srcdir)/lib/libvsc/vsctool.py $(COUNTERS) - echo -n '' > ${@}_ - for i in $(COUNTERS); do \ - $(PYTHON) $(top_srcdir)/lib/libvsc/vsctool.py -r $$i >> ${@}_ ; \ - done - mv -f ${@}_ ${@} +include/counters.rst: + ln -s $(abs_top_builddir)/lib/libvsc/counters.rst $@ BUILT_SOURCES += include/counters.rst include/varnishncsa_options.rst: $(top_builddir)/bin/varnishncsa/varnishncsa diff --git a/lib/libvsc/Makefile.am b/lib/libvsc/Makefile.am index 9f9ae0373..992b4d94c 100644 --- a/lib/libvsc/Makefile.am +++ b/lib/libvsc/Makefile.am @@ -22,3 +22,11 @@ libvsc_la_SOURCES = $(VSC_SRC) BUILT_SOURCES = $(VSC_GEN) dist_pkgdata_SCRIPTS = vsctool.py + +nodist_noinst_DATA = counters.rst + +counters.rst: $(VSC_RST) + $(AM_V_GEN)cat $(VSC_RST) >${@}_ + @mv ${@}_ $@ + +CLEANFILES = $(nodist_noinst_DATA) From dridi.boukelmoune at gmail.com Wed Jun 7 15:03:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 7 Jun 2023 15:03:06 +0000 (UTC) Subject: [master] 8b9d7df02 git: Ignore /lib/libvsc/VSC_*.rst Message-ID: <20230607150306.4451410EB79@lists.varnish-cache.org> commit 8b9d7df02b1ba7cad61d319cc200eaf7e398a899 Author: Dridi Boukelmoune Date: Wed Jun 7 17:01:30 2023 +0200 git: Ignore /lib/libvsc/VSC_*.rst Refs #3929 diff --git a/.gitignore b/.gitignore index 5762b8a9e..f77dc5baf 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,7 @@ cscope.*out # Stats /lib/libvsc/VSC_*.c /lib/libvsc/VSC_*.h +/lib/libvsc/VSC_*.rst # Misc. generated files for included vmods. /vmod/VSC_*.c From dridi.boukelmoune at gmail.com Wed Jun 7 15:37:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 7 Jun 2023 15:37:06 +0000 (UTC) Subject: [master] b2005c0fa vrt: Ignore HSH_DerefObjCore(bereq) return value Message-ID: <20230607153706.101D0110718@lists.varnish-cache.org> commit b2005c0fa70f5d7564b7bd6c2eb816be72a4c31e Author: Dridi Boukelmoune Date: Wed Jun 7 17:34:01 2023 +0200 vrt: Ignore HSH_DerefObjCore(bereq) return value Spotted by Coverity Scan. Refs #3914 diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 1ec09a74c..8810fcc51 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -614,7 +614,7 @@ VRT_u_bereq_body(VRT_CTX) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); if (ctx->bo->bereq_body != NULL) { - HSH_DerefObjCore(ctx->bo->wrk, &ctx->bo->bereq_body, 0); + (void)HSH_DerefObjCore(ctx->bo->wrk, &ctx->bo->bereq_body, 0); http_Unset(ctx->bo->bereq, H_Content_Length); } From nils.goroll at uplex.de Wed Jun 7 18:43:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 7 Jun 2023 18:43:06 +0000 (UTC) Subject: [master] f53ec1029 Humble beginnings with editorconfig Message-ID: <20230607184306.7295611686C@lists.varnish-cache.org> commit f53ec1029b658f008b865468bb43581d0ca15144 Author: Nils Goroll Date: Thu Jun 1 15:43:48 2023 +0200 Humble beginnings with editorconfig https://editorconfig.org/ diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..a1c4c2a6d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +root = true + +[*] +charset = utf-8 +max_line_length = 80 +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.{c,h}] +indent_style = tab +indent_size = 8 +tab_width = 8 From nils.goroll at uplex.de Wed Jun 7 18:43:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 7 Jun 2023 18:43:06 +0000 (UTC) Subject: [master] d0ea1b676 Excercise a test of the BACKEND .resolve() method Message-ID: <20230607184306.85C4A11686E@lists.varnish-cache.org> commit d0ea1b6762e8620bd2de686aa90572b0a6db48ee Author: Nils Goroll Date: Wed Jun 7 20:18:45 2023 +0200 Excercise a test of the BACKEND .resolve() method motivated by #3935 diff --git a/vmod/tests/directors_b00001.vtc b/vmod/tests/directors_b00001.vtc index 1ebae04d2..ad7aad5c8 100644 --- a/vmod/tests/directors_b00001.vtc +++ b/vmod/tests/directors_b00001.vtc @@ -2,16 +2,19 @@ varnishtest "Test fallback director" server s1 { rxreq + expect req.http.resolved == "s1" txresp -hdr "Foo: 1" } -start server s2 { rxreq + expect req.http.resolved == "s2" txresp -hdr "Foo: 2" } -start server s3 { rxreq + expect req.http.resolved == "s3" txresp -hdr "Foo: 3" } -start @@ -40,6 +43,7 @@ varnish v1 -vcl+backend { sub vcl_backend_fetch { set bereq.backend = fb1.backend(); + set bereq.http.resolved = bereq.backend.resolve(); } } -start From nils.goroll at uplex.de Wed Jun 7 18:43:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 7 Jun 2023 18:43:06 +0000 (UTC) Subject: [master] c169cc9c3 Document director layering and .resolve() Message-ID: <20230607184306.A0967116872@lists.varnish-cache.org> commit c169cc9c3d10ebc871813b09398b58836e291a1b Author: Nils Goroll Date: Wed Jun 7 20:39:09 2023 +0200 Document director layering and .resolve() Also motivated by #3935 diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst index a4708b8b1..72927da4a 100644 --- a/doc/sphinx/users-guide/vcl-backends.rst +++ b/doc/sphinx/users-guide/vcl-backends.rst @@ -309,6 +309,53 @@ not to change the probe config if you do a lot of VCL loading. Unloading the VCL will discard the probes. For more information on how to do this please see ref:`reference-vcl-director`. +Layering +-------- + +By default, most directors' ``.backend()`` methods return a reference +to the director itself. This allows for layering, like in this +example:: + + import directors; + + sub vcl_init { + new dc1 = directors.round_robin(); + dc1.add_backend(server1A); + dc1.add_backend(server1B); + + new dc2 = directors.round_robin(); + dc2.add_backend(server2A); + dc2.add_backend(server2B); + + new dcprio = directors.fallback(); + dcprio.add_backend(dc1); + dcprio.add_backend(dc2); + } + +With this initialization, ``dcprio.backend()`` will resolve to either +``server1A`` or ``server1B`` if both are healthy or one of them if +only one is healthy. Only if both are sick will a healthy server from +``dc2`` be returned, if any. + +Director Resolution +------------------- + +The actual resolution happens when the backend connection is prepared +after a return from ``vcl_backend_fetch {}``. + +In some cases like server sharding the resolution outcome is required +already in VCL. For such cases, the ``.resolve()`` method can be used, +like in this example:: + + set req.backend_hint = dcprio.backend().resolve(); + +When using this statement with the previous example code, +``req.backend_hint`` will be set to one of the ``server*`` backends or +the ``none`` backend if they were all sick. + +``.resolve()`` works on any object of the ``BACKEND`` type. + + .. _users-guide-advanced_backend_connection-pooling: Connection Pooling From dridi.boukelmoune at gmail.com Thu Jun 8 05:50:09 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 8 Jun 2023 05:50:09 +0000 (UTC) Subject: [master] 17ffd4690 vrb: Reduce storage pressure from req.body Message-ID: <20230608055009.535FC10261C@lists.varnish-cache.org> commit 17ffd469089744da5bc501621aab0a73d22be8d6 Author: Dridi Boukelmoune Date: Mon Jun 5 08:09:21 2023 +0200 vrb: Reduce storage pressure from req.body Instead of trying to allocate storage for the entire request body when the content length is known, let it go in fetch_chunksize steps. This may prevent spurious cache evictions for large request bodies for pass transactions when we are merely streaming the body. On the other hand, when the content length is known and the goal is to cache the request body, we may still attempt a single allocation and all that entails. diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index 5ffd08b77..bca871765 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -60,6 +60,7 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) enum vfp_status vfps = VFP_ERROR; const struct stevedore *stv; ssize_t req_bodybytes = 0; + unsigned hint; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -77,7 +78,8 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) req->storage = NULL; - if (STV_NewObject(req->wrk, req->body_oc, stv, 8) == 0) { + hint = maxsize > 0 ? maxsize : 1; + if (STV_NewObject(req->wrk, req->body_oc, stv, hint) == 0) { req->req_body_status = BS_ERROR; HSH_DerefBoc(req->wrk, req->body_oc); AZ(HSH_DerefObjCore(req->wrk, &req->body_oc, 0)); @@ -111,7 +113,8 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) (void)VFP_Error(vfc, "Request body too big to cache"); break; } - l = yet; + /* NB: only attempt a full allocation when caching. */ + l = maxsize > 0 ? yet : 0; if (VFP_GetStorage(vfc, &l, &ptr) != VFP_OK) break; AZ(vfc->failed); @@ -122,6 +125,8 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) req_bodybytes += l; if (yet >= l) yet -= l; + else if (yet > 0) + yet = 0; if (func != NULL) { r = func(priv, 1, ptr, l); if (r) diff --git a/bin/varnishtest/tests/c00123.vtc b/bin/varnishtest/tests/c00123.vtc new file mode 100644 index 000000000..c72c55267 --- /dev/null +++ b/bin/varnishtest/tests/c00123.vtc @@ -0,0 +1,75 @@ +varnishtest "Low req.body streaming pressure on storage" + +server s0 { + rxreq + txresp -status 200 + expect req.bodylen == 100000 +} -dispatch + +varnish v1 -vcl+backend { + import std; + + sub vcl_recv { + set req.storage = storage.s0; + if (req.http.cache) { + std.cache_req_body(100000b); + } + } +} -start + +# explicit setting to be robust against changes to the default value +varnish v1 -cliok "param.set fetch_chunksize 16k" + +# chunked req.body streaming uses approximately one fetch_chunksize'd chunk +client c1 { + txreq -req PUT -hdr "Transfer-encoding: chunked" + chunkedlen 100000 + chunkedlen 0 + rxresp + expect resp.status == 200 +} -run + +# in practice a little over fetch_chunksize is allocated +varnish v1 -expect SM?.s0.c_bytes < 20000 + +# reset s0 counters +varnish v1 -cliok stop +varnish v1 -cliok start +varnish v1 -expect SM?.s0.c_bytes == 0 + +# content-length req.body streaming also needs one chunk +client c2 { + txreq -req PUT -bodylen 100000 + rxresp + expect resp.status == 200 +} -run + +varnish v1 -expect SM?.s0.c_bytes < 20000 + +# reset s0 counters +varnish v1 -cliok stop +varnish v1 -cliok start + +# chunked req.body caching allocates storage for the entire body +client c3 { + txreq -req PUT -hdr "cache: body" -hdr "Transfer-encoding: chunked" + chunkedlen 100000 + chunkedlen 0 + rxresp + expect resp.status == 200 +} -run + +varnish v1 -expect SM?.s0.c_bytes > 100000 + +# reset s0 counters +varnish v1 -cliok stop +varnish v1 -cliok start + +# content-length req.body caching allocates storage for the entire body +client c4 { + txreq -req PUT -hdr "cache: body" -bodylen 100000 + rxresp + expect resp.status == 200 +} -run + +varnish v1 -expect SM?.s0.c_bytes > 100000 From nils.goroll at uplex.de Thu Jun 8 12:37:08 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 8 Jun 2023 12:37:08 +0000 (UTC) Subject: [master] c60c2886c I always forget vcl_pipe {} from the varnish twilight zone Message-ID: <20230608123708.348AE111991@lists.varnish-cache.org> commit c60c2886cebc0f7789525fcdcdff60f565098c0d Author: Nils Goroll Date: Thu Jun 8 14:35:43 2023 +0200 I always forget vcl_pipe {} from the varnish twilight zone diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst index 72927da4a..99142cbca 100644 --- a/doc/sphinx/users-guide/vcl-backends.rst +++ b/doc/sphinx/users-guide/vcl-backends.rst @@ -341,7 +341,7 @@ Director Resolution ------------------- The actual resolution happens when the backend connection is prepared -after a return from ``vcl_backend_fetch {}``. +after a return from ``vcl_backend_fetch {}`` or ``vcl_pipe {}``. In some cases like server sharding the resolution outcome is required already in VCL. For such cases, the ``.resolve()`` method can be used, From dridi.boukelmoune at gmail.com Fri Jun 9 10:08:11 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 9 Jun 2023 10:08:11 +0000 (UTC) Subject: [master] 97b61252d v1l: Preserve a copy of errno after writev(2) Message-ID: <20230609100811.423BF112CDF@lists.varnish-cache.org> commit 97b61252dba824545c5acb717ab54692ecd5b333 Author: Dridi Boukelmoune Date: Fri Jun 9 11:45:01 2023 +0200 v1l: Preserve a copy of errno after writev(2) This is to ensure we don't lose write errors by the time we make the final checks. diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index 544f80d8f..db5ca93c5 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -174,6 +174,7 @@ stream_close_t V1L_Flush(const struct worker *wrk) { ssize_t i; + int err; struct v1l *v1l; char cbuf[32]; @@ -228,7 +229,9 @@ V1L_Flush(const struct worker *wrk) * prevent slowloris attacks */ - if (errno == EWOULDBLOCK) { + err = errno; + + if (err == EWOULDBLOCK) { VSLb(v1l->vsl, SLT_Debug, "Hit idle send timeout, " "wrote = %zd/%zd; retrying", @@ -237,14 +240,14 @@ V1L_Flush(const struct worker *wrk) if (i > 0) v1l_prune(v1l, i); - } while (i > 0 || errno == EWOULDBLOCK); + } while (i > 0 || err == EWOULDBLOCK); if (i <= 0) { VSLb(v1l->vsl, SLT_Debug, "Write error, retval = %zd, len = %zd, errno = %s", - i, v1l->liov, VAS_errtxt(errno)); + i, v1l->liov, VAS_errtxt(err)); assert(v1l->werr == SC_NULL); - if (errno == EPIPE) + if (err == EPIPE) v1l->werr = SC_REM_CLOSE; else v1l->werr = SC_TX_ERROR; From dridi.boukelmoune at gmail.com Fri Jun 9 10:08:11 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 9 Jun 2023 10:08:11 +0000 (UTC) Subject: [master] a84b3c941 build: Polish vtc-syntax.py dependency Message-ID: <20230609100811.69873112D53@lists.varnish-cache.org> commit a84b3c9416890a3f32e25ce4f6d6c87e40b10c79 Author: Dridi Boukelmoune Date: Fri Jun 9 12:00:17 2023 +0200 build: Polish vtc-syntax.py dependency Better diff with the --word-diff --word-diff-regex=. options. diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 2dcdcd55f..af4bc85c4 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -155,8 +155,8 @@ VTCSYN_SRC = \ $(top_srcdir)/bin/varnishtest/vtc_syslog.c \ $(top_srcdir)/bin/varnishtest/vtc_tunnel.c \ $(top_srcdir)/bin/varnishtest/vtc_varnish.c -include/vtc-syntax.rst: vtc-syntax.py $(VTCSYN_SRC) - $(AM_V_GEN) $(PYTHON) $(top_srcdir)/doc/sphinx/vtc-syntax.py $(VTCSYN_SRC) > ${@}_ +include/vtc-syntax.rst: $(srcdir)/vtc-syntax.py $(VTCSYN_SRC) + $(AM_V_GEN) $(PYTHON) $(srcdir)/vtc-syntax.py $(VTCSYN_SRC) > ${@}_ @mv -f ${@}_ ${@} BUILT_SOURCES += include/vtc-syntax.rst From dridi.boukelmoune at gmail.com Mon Jun 12 05:50:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Jun 2023 05:50:06 +0000 (UTC) Subject: [master] dc4662bc5 v1l: Always initialize err Message-ID: <20230612055006.9A3B3C07C1@lists.varnish-cache.org> commit dc4662bc5beafe05ba6e916441c1698e71844a70 Author: Dridi Boukelmoune Date: Mon Jun 12 07:44:33 2023 +0200 v1l: Always initialize err Should we give it an explicit ETIMEDOUT in the dripping write case? diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index db5ca93c5..8c6f53b16 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -205,6 +205,7 @@ V1L_Flush(const struct worker *wrk) } i = 0; + err = 0; do { if (VTIM_real() > v1l->deadline) { VSLb(v1l->vsl, SLT_Debug, From dridi.boukelmoune at gmail.com Mon Jun 12 13:43:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Jun 2023 13:43:05 +0000 (UTC) Subject: [master] 8f2cfe92b circleci: Stop at the witness.dot step Message-ID: <20230612134305.855C5105E72@lists.varnish-cache.org> commit 8f2cfe92b960de7dfefe8369b296ce437ad4aab9 Author: Dridi Boukelmoune Date: Mon Mar 6 05:59:09 2023 +0100 circleci: Stop at the witness.dot step We don't need to render witness.svg in CI, even if dot(1) is available. diff --git a/.circleci/config.yml b/.circleci/config.yml index accac691a..8ba92cc9a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -405,7 +405,7 @@ workflows: name: build_fedora_latest dist: fedora release: latest - make_target: witness + make_target: witness.dot - build: name: build_fedora_rawhide dist: fedora From dridi.boukelmoune at gmail.com Mon Jun 12 13:43:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Jun 2023 13:43:05 +0000 (UTC) Subject: [master] 942d61288 build: Turn dot into a "proper" maintainer program Message-ID: <20230612134305.BEA53105E75@lists.varnish-cache.org> commit 942d61288ca1398883f437cf118bb4a28580bbb5 Author: Dridi Boukelmoune Date: Mon Mar 6 06:21:28 2023 +0100 build: Turn dot into a "proper" maintainer program The --enable-maintainer-mode option has been used in autogen.des for a while now and it has allowed automatic reconfiguration and hot reload of makefiles. For the next step, we can turn certain programs into maintainer programs and only require them when they are absolutely needed. This is what we currently do with SVG files generated from dot files, checked in git and optionally required, with an error message when rebuild is not possible. It turns out we only need 3 lines in configure.ac to implement all that, with the following differences: - no more `./configure --with-dot[=...]` option - use `./configure DOT=...` to override the default `dot` command - or alternatively run configure with `DOT=...` in the environment - generic missing error message (but referencing dot) from make - no need to reconfigure once `dot` is in the PATH The DOT override is actually documented in `./configure --help` like the `--with-dot[=...]` option was. While dot is our most emblematic maintainer tool, with its build targets checked in all the way in the git repository, this could be applied to other maintainer programs like sphinx-build and rst2man that should not be required when building from a dist archive. diff --git a/Makefile.am b/Makefile.am index 5e2db4419..f9cad589d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -85,19 +85,8 @@ witness.dot: all lib/libvmod_*/ .dot.svg: -if ! HAVE_DOT - @echo ================================================== - @echo You need graphviz installed to generate svg output - @echo ================================================== - @false -else $(AM_V_GEN) $(DOT) -Tsvg $< >$@ -endif -if HAVE_DOT witness: witness.svg -else -witness: witness.dot -endif .PHONY: cscope witness.dot diff --git a/configure.ac b/configure.ac index fb0b8d202..c1877cee0 100644 --- a/configure.ac +++ b/configure.ac @@ -79,16 +79,9 @@ if test "x$RST2HTML" = "xno"; then [rst2html not found - (Weird, we found rst2man?!)]) fi -AC_ARG_WITH([dot], - AS_HELP_STRING([--with-dot=PATH], - [Location of the dot tool from graphviz (auto)]), - [DOT="$withval"], - [AC_CHECK_PROGS(DOT, [dot], [no]) - if test "x$DOT" = "xno"; then - AC_MSG_WARN( - [dot not found - build will fail if svg files are out of date.]) - fi]) -AM_CONDITIONAL(HAVE_DOT,[test "x$DOT" != "xno"]) +AC_ARG_VAR([DOT], [The dot program from graphviz to build SVG graphics]) +AM_MISSING_PROG([DOT], [dot]) +AC_CHECK_PROGS([DOT], [dot]) # Define VMOD flags _VARNISH_VMOD_LDFLAGS diff --git a/doc/graphviz/Makefile.am b/doc/graphviz/Makefile.am index f135000c9..6fed3ae87 100644 --- a/doc/graphviz/Makefile.am +++ b/doc/graphviz/Makefile.am @@ -35,10 +35,8 @@ SVGS = \ cache_req_fsm.svg \ cache_fetch.svg -if HAVE_DOT CLEANFILES = \ $(PDFS) -endif pdf: $(PDFS) @@ -46,31 +44,10 @@ html: $(SVGS) link_srcdir # XXX does not fit onto a4 unless in landscape cache_fetch.pdf: cache_fetch.dot -if ! HAVE_DOT - @echo ================================================== - @echo You need graphviz installed to generate pdf output - @echo ================================================== - @false -else - @DOT@ -Tpdf -Gsize=$(SIZE) -Grotate=90 $< >$@ -endif + $(AM_V_GEN) $(DOT) -Tpdf -Gsize=$(SIZE) -Grotate=90 $< >$@ .dot.pdf: -if ! HAVE_DOT - @echo ================================================== - @echo You need graphviz installed to generate pdf output - @echo ================================================== - @false -else - @DOT@ -Tpdf -Gsize=$(SIZE) $< >$@ -endif + $(AM_V_GEN) $(DOT) -Tpdf -Gsize=$(SIZE) $< >$@ .dot.svg: -if ! HAVE_DOT - @echo ================================================== - @echo You need graphviz installed to generate svg output - @echo ================================================== - @false -else - @DOT@ -Tsvg $< >$@ -endif + $(AM_V_GEN) $(DOT) -Tsvg $< >$@ From dridi.boukelmoune at gmail.com Mon Jun 12 13:43:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Jun 2023 13:43:05 +0000 (UTC) Subject: [master] 8f4e7d452 build: Migrate witness.sh execution to vmod/ directory Message-ID: <20230612134306.05CF5105E79@lists.varnish-cache.org> commit 8f4e7d4521ba84d501d7863469260eed330f4f4c Author: Dridi Boukelmoune Date: Mon Mar 6 07:19:44 2023 +0100 build: Migrate witness.sh execution to vmod/ directory It's about time... diff --git a/Makefile.am b/Makefile.am index f9cad589d..e050e2ba8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -82,7 +82,7 @@ gcov_digest: witness.dot: all $(MAKE) check AM_VTC_LOG_FLAGS=-pdebug=+witness $(AM_V_GEN) $(srcdir)/tools/witness.sh witness.dot bin/varnishtest/ \ - lib/libvmod_*/ + vmod/ .dot.svg: $(AM_V_GEN) $(DOT) -Tsvg $< >$@ From nils.goroll at uplex.de Mon Jun 12 18:38:10 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Jun 2023 18:38:10 +0000 (UTC) Subject: [master] d0e12c7f4 Fix max_esi_depth check with onerror=abort Message-ID: <20230612183810.C34B11115AA@lists.varnish-cache.org> commit d0e12c7f41e58ae32c168af953f2bb2620823dcb Author: Nils Goroll Date: Mon Jun 12 20:30:50 2023 +0200 Fix max_esi_depth check with onerror=abort This was meant to be tested by 054215b51ede8caefa856e27e8328fac0ddba737, but the test did not attempt to receive a second chunk, so it succeeded when it should not have. Fixes #3938 diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 6ebb063bb..82e6bc7a1 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -127,6 +127,8 @@ ved_include(struct req *preq, const char *src, const char *host, VSLb(preq->vsl, SLT_VCL_Error, "ESI depth limit reach (param max_esi_depth = %u)", cache_param->max_esi_depth); + if (!ecx->incl_cont) + preq->top->topreq->vdc->retval = -1; return; } diff --git a/bin/varnishtest/tests/r03865.vtc b/bin/varnishtest/tests/r03865.vtc index 3bb1994e5..aba495f27 100644 --- a/bin/varnishtest/tests/r03865.vtc +++ b/bin/varnishtest/tests/r03865.vtc @@ -34,6 +34,7 @@ client c1 { rxresphdrs expect resp.status == 200 rxchunk + rxchunk expect_close expect resp.body == "before " } -run @@ -46,6 +47,7 @@ client c1 { rxresphdrs expect resp.status == 200 rxchunk + rxchunk expect_close expect resp.body == "before " } -run From nils.goroll at uplex.de Mon Jun 12 18:38:10 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Jun 2023 18:38:10 +0000 (UTC) Subject: [master] 903611966 Polish error message Message-ID: <20230612183810.DD5041115AD@lists.varnish-cache.org> commit 903611966b88aa464d22a9867411f31790b505f9 Author: Nils Goroll Date: Mon Jun 12 20:32:24 2023 +0200 Polish error message diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 82e6bc7a1..b543e2a62 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -125,7 +125,7 @@ ved_include(struct req *preq, const char *src, const char *host, if (preq->esi_level >= cache_param->max_esi_depth) { VSLb(preq->vsl, SLT_VCL_Error, - "ESI depth limit reach (param max_esi_depth = %u)", + "ESI depth limit reached (param max_esi_depth = %u)", cache_param->max_esi_depth); if (!ecx->incl_cont) preq->top->topreq->vdc->retval = -1; From phk at FreeBSD.org Tue Jun 13 05:57:09 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Jun 2023 05:57:09 +0000 (UTC) Subject: [master] 1a1901950 Rename VSM "classes" to "categories" Message-ID: <20230613055709.7633865A20@lists.varnish-cache.org> commit 1a190195088027d670667615b0c2ae1990fc873d Author: Poul-Henning Kamp Date: Mon Jun 12 18:25:36 2023 +0000 Rename VSM "classes" to "categories" diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c index 0e9128eaa..7df98b836 100644 --- a/bin/varnishd/common/common_vsc.c +++ b/bin/varnishd/common/common_vsc.c @@ -83,14 +83,14 @@ vsm_lock_f *vsc_unlock = vsc_dummy_lock; static const size_t vsc_overhead = PRNDUP(sizeof(struct vsc_head)); static struct vsc_seg * -vrt_vsc_mksegv(struct vsmw_cluster *vc, const char *class, +vrt_vsc_mksegv(struct vsmw_cluster *vc, const char *category, size_t payload, const char *fmt, va_list va) { struct vsc_seg *vsg; ALLOC_OBJ(vsg, VSC_SEG_MAGIC); AN(vsg); - vsg->seg = VSMW_Allocv(heritage.proc_vsmw, vc, class, + vsg->seg = VSMW_Allocv(heritage.proc_vsmw, vc, category, VRT_VSC_Overhead(payload), fmt, va); AN(vsg->seg); vsg->vsm = heritage.proc_vsmw; @@ -101,13 +101,13 @@ vrt_vsc_mksegv(struct vsmw_cluster *vc, const char *class, } static struct vsc_seg * -vrt_vsc_mksegf(const char *class, size_t payload, const char *fmt, ...) +vrt_vsc_mksegf(const char *category, size_t payload, const char *fmt, ...) { va_list ap; struct vsc_seg *vsg; va_start(ap, fmt); - vsg = vrt_vsc_mksegv(NULL, class, payload, fmt, ap); + vsg = vrt_vsc_mksegv(NULL, category, payload, fmt, ap); va_end(ap); return (vsg); } diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 46b558ec7..85afe07c7 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -114,7 +114,7 @@ struct vsmwseg { VTAILQ_ENTRY(vsmwseg) list; struct vsmw_cluster *cluster; - char *class; + char *category; size_t off; size_t len; char *id; @@ -163,7 +163,7 @@ vsmw_fmt_index(const struct vsmw *vsmw, const struct vsmwseg *seg, char act) seg->cluster->fn, seg->off, seg->len, - seg->class, + seg->category, seg->id); } @@ -257,7 +257,7 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg) REPLACE(t, NULL); vsmw->nsubs = 0; } - REPLACE(seg->class, NULL); + REPLACE(seg->category, NULL); REPLACE(seg->id, NULL); FREE_OBJ(seg); } @@ -317,7 +317,7 @@ VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx) vc->cseg = seg; seg->len = vc->len; seg->cluster = vc; - REPLACE(seg->class, ""); + REPLACE(seg->category, ""); REPLACE(seg->id, ""); vc->refs++; vc->named = 1; @@ -365,7 +365,7 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vsmcp) void * VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc, - const char *class, size_t payload, + const char *category, size_t payload, const char *fmt, va_list va) { struct vsmwseg *seg; @@ -375,7 +375,7 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc, ALLOC_OBJ(seg, VSMWSEG_MAGIC); AN(seg); - REPLACE(seg->class, class); + REPLACE(seg->category, category); seg->len = PRNDUP(payload); VSB_clear(vsmw->vsb); @@ -384,7 +384,7 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc, REPLACE(seg->id, VSB_data(vsmw->vsb)); if (vc == NULL) - vc = vsmw_newcluster(vsmw, seg->len, class); + vc = vsmw_newcluster(vsmw, seg->len, category); AN(vc); vc->refs++; @@ -402,13 +402,13 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc, void * VSMW_Allocf(struct vsmw *vsmw, struct vsmw_cluster *vc, - const char *class, size_t len, const char *fmt, ...) + const char *category, size_t len, const char *fmt, ...) { va_list ap; void *p; va_start(ap, fmt); - p = VSMW_Allocv(vsmw, vc, class, len, fmt, ap); + p = VSMW_Allocv(vsmw, vc, category, len, fmt, ap); va_end(ap); return (p); } diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 9677e44b4..3d133db34 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -196,7 +196,7 @@ extern struct params mgt_param; /* mgt_shmem.c */ void mgt_SHM_Init(void); void mgt_SHM_static_alloc(const void *, ssize_t size, - const char *class, const char *ident); + const char *category, const char *ident); void mgt_SHM_Create(void); void mgt_SHM_Destroy(int keep); diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c index ca08e2e10..858dfafdd 100644 --- a/bin/varnishd/mgt/mgt_shmem.c +++ b/bin/varnishd/mgt/mgt_shmem.c @@ -55,11 +55,11 @@ static struct vsmw *mgt_vsmw; void mgt_SHM_static_alloc(const void *ptr, ssize_t size, - const char *class, const char *ident) + const char *category, const char *ident) { void *p; - p = VSMW_Allocf(mgt_vsmw, NULL, class, size, "%s", ident); + p = VSMW_Allocf(mgt_vsmw, NULL, category, size, "%s", ident); AN(p); memcpy(p, ptr, size); } diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h index a2f6f186e..289dfc9fb 100644 --- a/include/vapi/vsc.h +++ b/include/vapi/vsc.h @@ -30,7 +30,7 @@ * * This is the public API for the VSC access. * - * VSC is a "subclass" of VSM. + * VSC is a sub-category of VSM. * */ diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h index 81a613d3d..065161787 100644 --- a/include/vapi/vsm.h +++ b/include/vapi/vsm.h @@ -31,7 +31,7 @@ * * This is the public API for the VSM access. * - * The VSM "class" acts as parent class for the VSL and VSC subclasses. + * The VSM category acts as parent category for the VSL and VSC sub-categories. * */ @@ -49,7 +49,7 @@ struct vsm_fantom { uintptr_t priv2; /* VSM private */ void *b; /* first byte of payload */ void *e; /* first byte past payload */ - char *class; + char *category; char *ident; }; @@ -73,7 +73,7 @@ struct vsm *VSM_New(void); void VSM_Destroy(struct vsm **vd); /* * Close and deallocate all storage and mappings. - * (including any VSC and VSL "sub-classes" XXX?) + * (including any VSC and VSL "sub-category" XXX?) */ const char *VSM_Error(const struct vsm *vd); @@ -173,14 +173,14 @@ const struct vsm_valid *VSM_StillValid(const struct vsm *, const struct vsm_fant */ int VSM_Get(struct vsm *, struct vsm_fantom *vf, - const char *class, const char *ident); + const char *category, const char *ident); /* * Find a chunk, produce fantom for it. * Returns zero on failure. - * class is mandatory, ident optional. + * category is mandatory, ident optional. */ -char *VSM_Dup(struct vsm*, const char *class, const char *ident); +char *VSM_Dup(struct vsm*, const char *category, const char *ident); /* * Returns a malloc'ed copy of the fanton. * diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 4d2f65c03..fc1f06467 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -362,7 +362,7 @@ vsc_add_seg(const struct vsc *vsc, struct vsm *vsm, const struct vsm_fantom *fp) assert(sp->head->ready > 0); sp->body = (char*)sp->fantom->b + sp->head->body_offset; - if (!strcmp(fp->class, VSC_CLASS)) { + if (!strcmp(fp->category, VSC_CLASS)) { VTAILQ_FOREACH(spd, &vsc->segs, list) if (spd->head->doc_id == sp->head->doc_id) break; @@ -385,7 +385,7 @@ vsc_add_seg(const struct vsc *vsc, struct vsm *vsm, const struct vsm_fantom *fp) VSB_destroy(&vsb); return (sp); } - assert(!strcmp(fp->class, VSC_DOC_CLASS)); + assert(!strcmp(fp->category, VSC_DOC_CLASS)); sp->vj = vjsn_parse(sp->body, &e); XXXAZ(e); AN(sp->vj); @@ -456,9 +456,9 @@ VSC_Iter(struct vsc *vsc, struct vsm *vsm, VSC_iter_f *fiter, void *priv) AN(vsm); sp = VTAILQ_FIRST(&vsc->segs); VSM_FOREACH(&ifantom, vsm) { - AN(ifantom.class); - if (strcmp(ifantom.class, VSC_CLASS) && - strcmp(ifantom.class, VSC_DOC_CLASS)) + AN(ifantom.category); + if (strcmp(ifantom.category, VSC_CLASS) && + strcmp(ifantom.category, VSC_DOC_CLASS)) continue; while (sp != NULL && (strcmp(ifantom.ident, sp->fantom->ident) || diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index 5242b0726..6ada5dfc0 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -878,9 +878,9 @@ VSM__itern(struct vsm *vd, struct vsm_fantom *vf) memset(vf, 0, sizeof *vf); vf->priv = VSM_PRIV_MERGE(vg->serial, vd->serial); vf->priv2 = (uintptr_t)vg; - vf->class = vg->av[4]; + vf->category = vg->av[4]; vf->ident = vg->av[5]; - AN(vf->class); + AN(vf->category); return (1); } @@ -901,7 +901,7 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf) return (vsm_diag(vd, "VSM_Map: bad fantom")); assert(vg->serial == VSM_PRIV_LOW(vf->priv)); - assert(vg->av[4] == vf->class); + assert(vg->av[4] == vf->category); assert(vg->av[5] == vf->ident); if (vg->b != NULL) { @@ -1018,13 +1018,13 @@ VSM_StillValid(const struct vsm *vd, const struct vsm_fantom *vf) int VSM_Get(struct vsm *vd, struct vsm_fantom *vf, - const char *class, const char *ident) + const char *category, const char *ident) { CHECK_OBJ_NOTNULL(vd, VSM_MAGIC); AN(vd->attached); VSM_FOREACH(vf, vd) { - if (strcmp(vf->class, class)) + if (strcmp(vf->category, category)) continue; if (ident != NULL && strcmp(vf->ident, ident)) continue; @@ -1037,7 +1037,7 @@ VSM_Get(struct vsm *vd, struct vsm_fantom *vf, /*--------------------------------------------------------------------*/ char * -VSM_Dup(struct vsm *vd, const char *class, const char *ident) +VSM_Dup(struct vsm *vd, const char *category, const char *ident) { struct vsm_fantom vf; char *p = NULL; @@ -1045,7 +1045,7 @@ VSM_Dup(struct vsm *vd, const char *class, const char *ident) CHECK_OBJ_NOTNULL(vd, VSM_MAGIC); AN(vd->attached); VSM_FOREACH(&vf, vd) { - if (strcmp(vf.class, class)) + if (strcmp(vf.category, category)) continue; if (ident != NULL && strcmp(vf.ident, ident)) continue; From phk at FreeBSD.org Tue Jun 13 05:57:09 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Jun 2023 05:57:09 +0000 (UTC) Subject: [master] a24ad62d5 Rename argument from "try" to "attempt" Message-ID: <20230613055709.89A4465A23@lists.varnish-cache.org> commit a24ad62d510f5d570e4ef677a61b304232099f85 Author: Poul-Henning Kamp Date: Mon Jun 12 18:27:22 2023 +0000 Rename argument from "try" to "attempt" diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c index 3de001cf0..cccf48cc1 100644 --- a/bin/varnishd/cache/cache_lck.c +++ b/bin/varnishd/cache/cache_lck.c @@ -59,7 +59,8 @@ struct ilck { /*--------------------------------------------------------------------*/ static void -Lck_Witness_Lock(const struct ilck *il, const char *p, int l, const char *try) +Lck_Witness_Lock(const struct ilck *il, const char *p, int l, + const char *attempt) { char *q, t[10]; //lint -e429 int emit; @@ -74,7 +75,7 @@ Lck_Witness_Lock(const struct ilck *il, const char *p, int l, const char *try) emit = *q != '\0'; strcat(q, " "); strcat(q, il->w); - strcat(q, try); + strcat(q, attempt); strcat(q, ","); strcat(q, p); strcat(q, ","); From phk at FreeBSD.org Tue Jun 13 06:09:06 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Jun 2023 06:09:06 +0000 (UTC) Subject: [master] f6ce3c9c6 Constify Message-ID: <20230613060906.1AD201003F7@lists.varnish-cache.org> commit f6ce3c9c676ad94609508ff2801463635b2a8316 Author: Poul-Henning Kamp Date: Tue Jun 13 06:08:17 2023 +0000 Constify diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index b543e2a62..59e304ee2 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -312,7 +312,7 @@ static int v_matchproto_(vdp_bytes_f) ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { - uint8_t *q, *r; + const uint8_t *q, *r; ssize_t l = 0; uint32_t icrc = 0; uint8_t tailbuf[8 + 5]; From nils.goroll at uplex.de Tue Jun 13 13:27:09 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Jun 2023 13:27:09 +0000 (UTC) Subject: [master] d95b853f2 Reduce duplication in vmodtool.py Message-ID: <20230613132709.E4BF510FED0@lists.varnish-cache.org> commit d95b853f20b6eb4e2cbc4287fc67027432ad52f6 Author: Nils Goroll Date: Tue Jun 13 15:10:50 2023 +0200 Reduce duplication in vmodtool.py diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 58fbc3041..4112e05e8 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -710,18 +710,9 @@ class EventStanza(Stanza): jl.append(["$EVENT", "%s._event" % self.vcc.csn]) -class FunctionStanza(Stanza): +class FunctionMethodStanzaBase(Stanza): - ''' $Function TYPE name ( ARGUMENTS ) ''' - - def parse(self): - self.proto = ProtoType(self) - self.rstlbl = '%s.%s()' % (self.vcc.modname, self.proto.name) - self.vcc.contents.append(self) - self.restrict = None - - def cstuff(self, fo, where): - fo.write(self.proto.cproto(['VRT_CTX'], where)) + ''' Base class for $Function and Method ''' def cstruct(self, fo, define): self.fmt_cstruct_proto(fo, self.proto, define) @@ -733,12 +724,29 @@ class FunctionStanza(Stanza): self.restrict.rstdoc(fo, unused_man) def json(self, jl): - jl.append(["$FUNC", "%s" % self.proto.name]) self.proto.jsonproto(jl[-1], self.proto.cname()) if (self.restrict is not None): self.restrict.json(jl) +class FunctionStanza(FunctionMethodStanzaBase): + + ''' $Function TYPE name ( ARGUMENTS ) ''' + + def parse(self): + self.proto = ProtoType(self) + self.rstlbl = '%s.%s()' % (self.vcc.modname, self.proto.name) + self.vcc.contents.append(self) + self.restrict = None + + def cstuff(self, fo, where): + fo.write(self.proto.cproto(['VRT_CTX'], where)) + + def json(self, jl): + jl.append(["$FUNC", "%s" % self.proto.name]) + super().json(jl) + + class ObjectStanza(Stanza): ''' $Object TYPE class ( ARGUMENTS ) ''' @@ -828,7 +836,7 @@ class ObjectStanza(Stanza): ####################################################################### -class MethodStanza(Stanza): +class MethodStanza(FunctionMethodStanzaBase): ''' $Method TYPE . method ( ARGUMENTS ) ''' @@ -845,20 +853,9 @@ class MethodStanza(Stanza): p.methods.append(self) self.restrict = None - def cstruct(self, fo, define): - self.fmt_cstruct_proto(fo, self.proto, define) - - def rstdoc(self, fo, unused_man): - super().rstdoc(fo,unused_man) - if (self.restrict is not None): - fo.write("\nRestricted to: ``%s``\n\n" % ', '.join(self.restrict.restrict_toks)) - self.restrict.rstdoc(fo, unused_man) - def json(self, jl): jl.append(["$METHOD", self.proto.name[len(self.pfx)+1:]]) - self.proto.jsonproto(jl[-1], self.proto.cname()) - if (self.restrict is not None): - self.restrict.json(jl) + super().json(jl) class RestrictStanza(Stanza): From nils.goroll at uplex.de Tue Jun 13 13:27:09 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Jun 2023 13:27:09 +0000 (UTC) Subject: [master] b6c7bf9d6 Polish $Restrict doc formatting Message-ID: <20230613132709.F147010FED2@lists.varnish-cache.org> commit b6c7bf9d6b87c9f6f21d17f3fee54d6c3bdb9917 Author: Nils Goroll Date: Tue Jun 13 15:19:11 2023 +0200 Polish $Restrict doc formatting Format individual subroutines as code, add a trailing period. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 4112e05e8..dbb00fba6 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -720,7 +720,8 @@ class FunctionMethodStanzaBase(Stanza): def rstdoc(self, fo, unused_man): super().rstdoc(fo,unused_man) if (self.restrict is not None): - fo.write("\nRestricted to: ``%s``\n\n" % ', '.join(self.restrict.restrict_toks)) + fo.write("\nRestricted to: ``%s``.\n\n" % + '``, ``'.join(self.restrict.restrict_toks)) self.restrict.rstdoc(fo, unused_man) def json(self, jl): From nils.goroll at uplex.de Tue Jun 13 16:47:07 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Jun 2023 16:47:07 +0000 (UTC) Subject: [master] f7767df01 Ensure vxids are fully written before the memory barrier Message-ID: <20230613164707.A119A1178AB@lists.varnish-cache.org> commit f7767df01b0ab17d242c4eeb4d92389a42bfae06 Author: Nils Goroll Date: Tue Jun 13 17:49:25 2023 +0200 Ensure vxids are fully written before the memory barrier Seen while staring at #3937 diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 67150f69a..9c1fefe33 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -273,9 +273,11 @@ vslr(enum VSL_tag_e tag, vxid_t vxid, const char *b, unsigned len) memcpy(p + VSL_OVERHEAD, b, len); /* - * vsl_hdr() writes p[1] again, but we want to make sure it - * has hit memory because we work on the live buffer here. + * the vxid needs to be written before the barrier to + * ensure it is valid when vsl_hdr() marks the record + * ready by writing p[0] */ + p[2] = vxid.vxid >> 32; p[1] = vxid.vxid; VWMB(); (void)vsl_hdr(tag, p, len, vxid); From nils.goroll at uplex.de Tue Jun 13 16:47:07 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Jun 2023 16:47:07 +0000 (UTC) Subject: [master] fa98d41e3 Replace magic number with VSL_IDSHIFT Message-ID: <20230613164707.A91CC1178AD@lists.varnish-cache.org> commit fa98d41e3a0f491512d7add166877880dcc8187f Author: Nils Goroll Date: Tue Jun 13 18:39:55 2023 +0200 Replace magic number with VSL_IDSHIFT diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 9c1fefe33..8a040ee70 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -355,7 +355,7 @@ VSL_Flush(struct vsl_log *vsl, int overflow) memcpy(p + VSL_OVERHEAD, vsl->wlb, l); p[1] = l; VWMB(); - p[0] = ((((unsigned)SLT__Batch & 0xff) << 24)); + p[0] = ((((unsigned)SLT__Batch & 0xff) << VSL_IDSHIFT)); vsl->wlp = vsl->wlb; vsl->wlr = 0; } From nils.goroll at uplex.de Tue Jun 13 16:47:07 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 13 Jun 2023 16:47:07 +0000 (UTC) Subject: [master] 692aeb54f git ignore /lib/libvsc/counters.rst Message-ID: <20230613164707.C08AB1178B0@lists.varnish-cache.org> commit 692aeb54ff8f3624d3f0c99cd4434fcdc369920d Author: Nils Goroll Date: Tue Jun 13 18:44:56 2023 +0200 git ignore /lib/libvsc/counters.rst diff --git a/.gitignore b/.gitignore index f77dc5baf..2e8612b52 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,7 @@ cscope.*out /lib/libvsc/VSC_*.c /lib/libvsc/VSC_*.h /lib/libvsc/VSC_*.rst +/lib/libvsc/counters.rst # Misc. generated files for included vmods. /vmod/VSC_*.c From phk at FreeBSD.org Wed Jun 14 07:01:06 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 14 Jun 2023 07:01:06 +0000 (UTC) Subject: [master] d4255aa17 Prefix struct member names with "f_" to avoid collisions with reserved words. Message-ID: <20230614070106.8C2C6108BD9@lists.varnish-cache.org> commit d4255aa17ddacf8373ed81a67b8e82cc18d9aef7 Author: Poul-Henning Kamp Date: Wed Jun 14 06:24:16 2023 +0000 Prefix struct member names with "f_" to avoid collisions with reserved words. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index dbb00fba6..e8c413793 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -499,7 +499,7 @@ class ProtoType(): ''' Produce VCL prototype as JSON ''' ll = [] self.retval.jsonproto(ll) - ll.append('%s.%s' % (self.st.vcc.csn, cfunc)) + ll.append('%s.f_%s' % (self.st.vcc.csn, cfunc)) if self.argstruct: # We cannot use VARGS() here, we are after the #undef ll.append('struct arg_%s%s_%s' % @@ -571,12 +571,12 @@ class Stanza(): fmt_cstruct( fo, proto.typedef_name(), - '*' + proto.cname() + ';' + '*f_' + proto.cname() + ';' ) else: fmt_cstruct( fo, - '.' + proto.cname() + ' =', + '.f_' + proto.cname() + ' =', self.vcc.sympfx + proto.cname() + ',' ) @@ -1190,13 +1190,13 @@ class vcc(): fo.write("\nconst struct vmod_data %s = {\n" % vmd) fo.write("\t.vrt_major =\t%s,\n" % self.vrt_major) fo.write("\t.vrt_minor =\t%s,\n" % self.vrt_minor) + fo.write("\t.file_id =\t\"%s\",\n" % self.file_id) fo.write('\t.name =\t\t"%s",\n' % self.modname) + fo.write('\t.func_name =\t"%s",\n' % self.csn) fo.write('\t.func =\t\t&%s,\n' % self.csn) fo.write('\t.func_len =\tsizeof(%s),\n' % self.csn) - fo.write('\t.func_name =\t"%s",\n' % self.csn) fo.write('\t.json =\t\tVmod_Json,\n') fo.write('\t.abi =\t\tVMOD_ABI_Version,\n') - fo.write("\t.file_id =\t\"%s\",\n" % self.file_id) fo.write("};\n") def mkcfile(self): From dridi.boukelmoune at gmail.com Fri Jun 16 10:06:15 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 16 Jun 2023 10:06:15 +0000 (UTC) Subject: [master] 620740dbf vtc: Stabilize h2 req.body tests Message-ID: <20230616100615.85AE964C92@lists.varnish-cache.org> commit 620740dbfb2ba38cb67a88aad9723c1b30c87958 Author: Dridi Boukelmoune Date: Fri Jun 16 11:58:03 2023 +0200 vtc: Stabilize h2 req.body tests With the recent change to stream the request body to the backend like chunked encoding does, even when we know the content length, two test cases became sensitive to the timing of window updates. The data frames contain for the most part a 16kB payload, which happens to be the same as fetch_chunksize's default value. With a value of 64kB, corresponding to one more byte than the default h2 initial window, we are garanteed to never risk a race between the VFP consuming the h2_rxbuf and the h2_sess task monitoring it to send window updates. This is also a good occasion to exercise stream.peer_window in t02019. Refs #3878 Refs #3934 diff --git a/bin/varnishtest/tests/t02019.vtc b/bin/varnishtest/tests/t02019.vtc index 3adcf2603..6955fcf85 100644 --- a/bin/varnishtest/tests/t02019.vtc +++ b/bin/varnishtest/tests/t02019.vtc @@ -15,6 +15,7 @@ varnish v1 -vcl+backend { } } -start +varnish v1 -cliok "param.set fetch_chunksize 64k" varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.reset h2_initial_window_size" varnish v1 -cliok "param.reset h2_rx_window_low_water" @@ -22,21 +23,35 @@ varnish v1 -cliok "param.reset h2_rx_window_low_water" client c1 { stream 1 { txreq -req POST -url /1 -hdr "content-length" "131072" -nostrend + + expect stream.peer_window == 65535 txdata -datalen 16384 -nostrend txdata -datalen 16384 -nostrend txdata -datalen 16384 -nostrend txdata -datalen 16383 -nostrend + expect stream.peer_window == 0 barrier b1 sync + rxwinup + expect stream.peer_window == 65535 txdata -datalen 16384 -nostrend + rxwinup + expect stream.peer_window == 65535 txdata -datalen 16384 -nostrend + rxwinup + expect stream.peer_window == 65535 txdata -datalen 16384 -nostrend + rxwinup + expect stream.peer_window == 65535 txdata -datalen 16384 -nostrend + rxwinup + expect stream.peer_window == 65535 txdata -datalen 1 + rxresp expect resp.status == 200 } -run diff --git a/bin/varnishtest/tests/t02020.vtc b/bin/varnishtest/tests/t02020.vtc index 6a2ace287..e2bcb76f4 100644 --- a/bin/varnishtest/tests/t02020.vtc +++ b/bin/varnishtest/tests/t02020.vtc @@ -4,7 +4,7 @@ barrier b1 sock 3 server s1 { rxreq - expect req.url == /1 + expect req.url == "/1" expect req.body == abcde txresp rxreq @@ -14,6 +14,7 @@ server s1 { expect req.body == a } -start +varnish v1 -cliok "param.set fetch_chunksize 64k" varnish v1 -vcl+backend { import vtc; sub vcl_recv { From dridi.boukelmoune at gmail.com Fri Jun 16 10:13:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 16 Jun 2023 10:13:05 +0000 (UTC) Subject: [master] 5a7b42e59 vtc: Polish t02019 Message-ID: <20230616101305.F3FA891183@lists.varnish-cache.org> commit 5a7b42e599909f1631a19a77e4b7327df4e6cf99 Author: Dridi Boukelmoune Date: Fri Jun 16 12:09:40 2023 +0200 vtc: Polish t02019 diff --git a/bin/varnishtest/tests/t02019.vtc b/bin/varnishtest/tests/t02019.vtc index 6955fcf85..5795ceed4 100644 --- a/bin/varnishtest/tests/t02019.vtc +++ b/bin/varnishtest/tests/t02019.vtc @@ -32,21 +32,11 @@ client c1 { expect stream.peer_window == 0 barrier b1 sync - rxwinup - expect stream.peer_window == 65535 - txdata -datalen 16384 -nostrend - - rxwinup - expect stream.peer_window == 65535 - txdata -datalen 16384 -nostrend - - rxwinup - expect stream.peer_window == 65535 - txdata -datalen 16384 -nostrend - - rxwinup - expect stream.peer_window == 65535 - txdata -datalen 16384 -nostrend + loop 4 { + rxwinup + expect stream.peer_window == 65535 + txdata -datalen 16384 -nostrend + } rxwinup expect stream.peer_window == 65535 From dridi.boukelmoune at gmail.com Fri Jun 16 10:17:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 16 Jun 2023 10:17:05 +0000 (UTC) Subject: [master] dff754805 vtc_varnish: Log h2 frames Message-ID: <20230616101706.3F8D0915C5@lists.varnish-cache.org> commit dff7548052bbcb3cd5a02dcb2f44837d68f185e4 Author: Dridi Boukelmoune Date: Thu Jun 15 19:45:53 2023 +0200 vtc_varnish: Log h2 frames diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index c8b6bba15..9c0662d25 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -417,7 +417,7 @@ varnish_launch(struct varnish *v) VSB_cat(vsb, " -p syslog_cli_traffic=off"); VSB_cat(vsb, " -p thread_pool_min=10"); VSB_cat(vsb, " -p debug=+vtc_mode"); - VSB_cat(vsb, " -p vsl_mask=+Debug"); + VSB_cat(vsb, " -p vsl_mask=+Debug,+H2RxHdr,+H2RxBody"); VSB_cat(vsb, " -p h2_initial_window_size=1m"); VSB_cat(vsb, " -p h2_rx_window_low_water=64k"); if (!v->has_a_arg) { From nils.goroll at uplex.de Fri Jun 16 13:10:12 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 16 Jun 2023 13:10:12 +0000 (UTC) Subject: [master] 10c606d37 Trivial refactor of the debug stevedore Message-ID: <20230616131013.038059C344@lists.varnish-cache.org> commit 10c606d3759e106b5786f106454350b57b9daa1a Author: Nils Goroll Date: Fri Jun 16 12:54:24 2023 +0200 Trivial refactor of the debug stevedore diff --git a/bin/varnishd/storage/storage_debug.c b/bin/varnishd/storage/storage_debug.c index 9a3360a42..dd3eee140 100644 --- a/bin/varnishd/storage/storage_debug.c +++ b/bin/varnishd/storage/storage_debug.c @@ -58,6 +58,7 @@ smd_init(struct stevedore *parent, int aac, char * const *aav) const char *ident; int i, ac = 0; size_t nac; + char *a; char **av; //lint -e429 ident = parent->ident; @@ -76,11 +77,14 @@ smd_init(struct stevedore *parent, int aac, char * const *aav) av = calloc(nac, sizeof *av); AN(av); for (i = 0; i < aac; i++) { - if (aav[i] != NULL && ! strcmp(aav[i], "lessspace")) { - methods->objgetspace = smd_lsp_getspace; - continue; + a = aav[i]; + if (a != NULL) { + if (! strcmp(a, "lessspace")) { + methods->objgetspace = smd_lsp_getspace; + continue; + } } - av[ac] = aav[i]; + av[ac] = a; ac++; } assert(ac >= 0); From nils.goroll at uplex.de Fri Jun 16 13:10:13 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 16 Jun 2023 13:10:13 +0000 (UTC) Subject: [master] 6efc2d556 Add init and open delays to the debug stevedore Message-ID: <20230616131013.126EE9C360@lists.varnish-cache.org> commit 6efc2d556015410acd5b3de09e2fbf99fb68c6bb Author: Nils Goroll Date: Fri Jun 16 13:26:31 2023 +0200 Add init and open delays to the debug stevedore To keep things simple, the open delay is global - the delay of the last debug stevedore applies to all of them. diff --git a/bin/varnishd/storage/storage_debug.c b/bin/varnishd/storage/storage_debug.c index dd3eee140..290606ba6 100644 --- a/bin/varnishd/storage/storage_debug.c +++ b/bin/varnishd/storage/storage_debug.c @@ -1,5 +1,5 @@ /*- - * Copyright 2021 UPLEX - Nils Goroll Systemoptimierung + * Copyright 2021,2023 UPLEX - Nils Goroll Systemoptimierung * All rights reserved. * * Author: Nils Goroll @@ -35,11 +35,22 @@ #include "cache/cache_varnishd.h" #include "cache/cache_obj.h" +#include #include #include "storage/storage.h" #include "storage/storage_simple.h" +#include "vtim.h" +#include "vnum.h" + +/* we cheat and make the open delay a static to avoid + * having to wrap all callbacks to unpack the priv + * pointer. Consequence: last dopen applies to all + * debug stevedores + */ +static vtim_dur dopen = 0.0; + /* returns one byte less than requested */ static int v_matchproto_(objgetspace_f) smd_lsp_getspace(struct worker *wrk, struct objcore *oc, ssize_t *sz, @@ -51,6 +62,18 @@ smd_lsp_getspace(struct worker *wrk, struct objcore *oc, ssize_t *sz, return (SML_methods.objgetspace(wrk, oc, sz, ptr)); } +#define dur_arg(a, s, d) \ + (! strncmp((a), (s), strlen(s)) \ + && (d = VNUM_duration(a + strlen(s))) != nan("")) + +static void smd_open(struct stevedore *stv) +{ + sma_stevedore.open(stv); + fprintf(stderr, "-sdebug open delay %fs\n", dopen); + if (dopen > 0.0) + VTIM_sleep(dopen); +} + static void v_matchproto_(storage_init_f) smd_init(struct stevedore *parent, int aac, char * const *aav) { @@ -58,8 +81,9 @@ smd_init(struct stevedore *parent, int aac, char * const *aav) const char *ident; int i, ac = 0; size_t nac; - char *a; + vtim_dur d, dinit = 0.0; char **av; //lint -e429 + char *a; ident = parent->ident; memcpy(parent, &sma_stevedore, sizeof *parent); @@ -83,6 +107,14 @@ smd_init(struct stevedore *parent, int aac, char * const *aav) methods->objgetspace = smd_lsp_getspace; continue; } + if (dur_arg(a, "dinit=", d)) { + dinit = d; + continue; + } + if (dur_arg(a, "dopen=", d)) { + dopen = d; + continue; + } } av[ac] = a; ac++; @@ -93,6 +125,12 @@ smd_init(struct stevedore *parent, int aac, char * const *aav) sma_stevedore.init(parent, ac, av); free(av); + fprintf(stderr, "-sdebug init delay %fs\n", dinit); + fprintf(stderr, "-sdebug open delay in init %fs\n", dopen); + if (dinit > 0.0) { + VTIM_sleep(dinit); + } + parent->open = smd_open; } const struct stevedore smd_stevedore = { From nils.goroll at uplex.de Fri Jun 16 20:14:14 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 16 Jun 2023 20:14:14 +0000 (UTC) Subject: [master] 185e73a10 doc polish: cli_timeout Message-ID: <20230616201414.41C2EAD9C2@lists.varnish-cache.org> commit 185e73a103f229f0598486c56bad3553e674ebe8 Author: Nils Goroll Date: Fri Jun 16 22:07:22 2023 +0200 doc polish: cli_timeout Varnish historians claim to have evidence that the odd wording "from mgt_param" could have been an unintended side effect of a struct rename in ancient times at around 2011. (8241149492e5649c8a3126ee3) diff --git a/include/tbl/params.h b/include/tbl/params.h index fcf47e31c..8a066361f 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -332,8 +332,7 @@ PARAM_SIMPLE( /* def */ "60.000", /* units */ "seconds", /* descr */ - "Timeout for the child's replies to CLI requests from the " - "mgt_param." + "Timeout for the child's replies to CLI requests." ) PARAM_SIMPLE( From phk at FreeBSD.org Tue Jun 20 08:53:09 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 20 Jun 2023 08:53:09 +0000 (UTC) Subject: [master] 3df9cdc1e Bite the bullet, and eliminate K&R functions in libvgz, in the most diff-friendly way. Message-ID: <20230620085310.00B5611AB4D@lists.varnish-cache.org> commit 3df9cdc1eda72ec736c603b818387fb8ab825569 Author: Poul-Henning Kamp Date: Tue Jun 20 08:36:55 2023 +0000 Bite the bullet, and eliminate K&R functions in libvgz, in the most diff-friendly way. If Mark Adler ever decides that the 21st century is real thing, we will adopt his diff. This should allow us to revert 79c7d17543d86 diff --git a/lib/libvgz/adler32.c b/lib/libvgz/adler32.c index ef1ae7e79..7a3d10ccc 100644 --- a/lib/libvgz/adler32.c +++ b/lib/libvgz/adler32.c @@ -186,10 +186,11 @@ uLong ZEXPORT adler32_combine64(adler1, adler2, len2) return adler32_combine_(adler1, adler2, len2); } #else /* NOVGZ */ -uLong ZEXPORT adler32(adler, buf, len) - uLong adler; - const Bytef *buf; - uInt len; +uLong ZEXPORT adler32( + uLong adler, + const Bytef *buf, + uInt len +) { (void)adler; (void)buf; diff --git a/lib/libvgz/crc32.c b/lib/libvgz/crc32.c index 6eab5e951..32c6d5b77 100644 --- a/lib/libvgz/crc32.c +++ b/lib/libvgz/crc32.c @@ -125,8 +125,9 @@ local z_crc_t x2nmodp OF((z_off64_t n, unsigned k)); */ local z_word_t byte_swap OF((z_word_t word)); -local z_word_t byte_swap(word) - z_word_t word; +local z_word_t byte_swap( + z_word_t word +) { # if W == 8 return @@ -550,9 +551,10 @@ local void braid(ltl, big, n, w) Return a(x) multiplied by b(x) modulo p(x), where p(x) is the CRC polynomial, reflected. For speed, this requires that a not be zero. */ -local z_crc_t multmodp(a, b) - z_crc_t a; - z_crc_t b; +local z_crc_t multmodp( + z_crc_t a, + z_crc_t b +) { z_crc_t m, p; @@ -574,9 +576,10 @@ local z_crc_t multmodp(a, b) Return x^(n * 2^k) modulo p(x). Requires that x2n_table[] has been initialized. */ -local z_crc_t x2nmodp(n, k) - z_off64_t n; - unsigned k; +local z_crc_t x2nmodp( + z_off64_t n, + unsigned k +) { z_crc_t p; @@ -594,7 +597,7 @@ local z_crc_t x2nmodp(n, k) * This function can be used by asm versions of crc32(), and to force the * generation of the CRC tables in a threaded application. */ -const z_crc_t FAR * ZEXPORT get_crc_table() +const z_crc_t FAR * ZEXPORT get_crc_table(void) { #ifdef DYNAMIC_CRC_TABLE once(&made, make_crc_table); @@ -727,8 +730,9 @@ unsigned long ZEXPORT crc32_z(crc, buf, len) */ local z_crc_t crc_word OF((z_word_t data)); -local z_crc_t crc_word(data) - z_word_t data; +local z_crc_t crc_word( + z_word_t data +) { int k; for (k = 0; k < W; k++) @@ -738,8 +742,9 @@ local z_crc_t crc_word(data) local z_word_t crc_word_big OF((z_word_t data)); -local z_word_t crc_word_big(data) - z_word_t data; +local z_word_t crc_word_big( + z_word_t data +) { int k; for (k = 0; k < W; k++) @@ -751,10 +756,11 @@ local z_word_t crc_word_big(data) #endif /* ========================================================================= */ -unsigned long ZEXPORT crc32_z(crc, buf, len) - unsigned long crc; - const unsigned char FAR *buf; - z_size_t len; +unsigned long ZEXPORT crc32_z( + unsigned long crc, + const unsigned char FAR *buf, + z_size_t len +) { /* Return initial CRC, if requested. */ if (buf == Z_NULL) return 0; @@ -1075,19 +1081,21 @@ unsigned long ZEXPORT crc32_z(crc, buf, len) #endif /* ========================================================================= */ -unsigned long ZEXPORT crc32(crc, buf, len) - unsigned long crc; - const unsigned char FAR *buf; - uInt len; +unsigned long ZEXPORT crc32( + unsigned long crc, + const unsigned char FAR *buf, + uInt len +) { return crc32_z(crc, buf, len); } /* ========================================================================= */ -uLong ZEXPORT crc32_combine64(crc1, crc2, len2) - uLong crc1; - uLong crc2; - z_off64_t len2; +uLong ZEXPORT crc32_combine64( + uLong crc1, + uLong crc2, + z_off64_t len2 +) { #ifdef DYNAMIC_CRC_TABLE once(&made, make_crc_table); @@ -1096,17 +1104,19 @@ uLong ZEXPORT crc32_combine64(crc1, crc2, len2) } /* ========================================================================= */ -uLong ZEXPORT crc32_combine(crc1, crc2, len2) - uLong crc1; - uLong crc2; - z_off_t len2; +uLong ZEXPORT crc32_combine( + uLong crc1, + uLong crc2, + z_off_t len2 +) { return crc32_combine64(crc1, crc2, (z_off64_t)len2); } /* ========================================================================= */ -uLong ZEXPORT crc32_combine_gen64(len2) - z_off64_t len2; +uLong ZEXPORT crc32_combine_gen64( + z_off64_t len2 +) { #ifdef DYNAMIC_CRC_TABLE once(&made, make_crc_table); @@ -1115,17 +1125,19 @@ uLong ZEXPORT crc32_combine_gen64(len2) } /* ========================================================================= */ -uLong ZEXPORT crc32_combine_gen(len2) - z_off_t len2; +uLong ZEXPORT crc32_combine_gen( + z_off_t len2 +) { return crc32_combine_gen64((z_off64_t)len2); } /* ========================================================================= */ -uLong ZEXPORT crc32_combine_op(crc1, crc2, op) - uLong crc1; - uLong crc2; - uLong op; +uLong ZEXPORT crc32_combine_op( + uLong crc1, + uLong crc2, + uLong op +) { return multmodp(op, crc1) ^ (crc2 & 0xffffffff); } diff --git a/lib/libvgz/deflate.c b/lib/libvgz/deflate.c index 75277afd4..32b395342 100644 --- a/lib/libvgz/deflate.c +++ b/lib/libvgz/deflate.c @@ -198,8 +198,9 @@ local const config configuration_table[10] = { * bit values at the expense of memory usage). We slide even when level == 0 to * keep the hash table consistent if we switch back to level > 0 later. */ -local void slide_hash(s) - deflate_state *s; +local void slide_hash( + deflate_state *s +) { unsigned n, m; Posf *p; @@ -241,16 +242,16 @@ int ZEXPORT deflateInit_(strm, level, version, stream_size) #endif /* NOVGZ */ /* ========================================================================= */ -int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, - version, stream_size) - z_streamp strm; - int level; - int method; - int windowBits; - int memLevel; - int strategy; - const char *version; - int stream_size; +int ZEXPORT deflateInit2_( + z_streamp strm, + int level, + int method, + int windowBits, + int memLevel, + int strategy, + const char *version, + int stream_size +) { deflate_state *s; int wrap = 1; @@ -393,8 +394,9 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, /* ========================================================================= * Check for a valid deflate stream state. Return 0 if ok, 1 if not. */ -local int deflateStateCheck (strm) - z_streamp strm; +local int deflateStateCheck ( + z_streamp strm +) { deflate_state *s; if (strm == Z_NULL || @@ -513,8 +515,9 @@ int ZEXPORT deflateGetDictionary (strm, dictionary, dictLength) #endif /* NOVGZ */ /* ========================================================================= */ -int ZEXPORT deflateResetKeep (strm) - z_streamp strm; +int ZEXPORT deflateResetKeep ( + z_streamp strm +) { deflate_state *s; @@ -552,8 +555,9 @@ int ZEXPORT deflateResetKeep (strm) } /* ========================================================================= */ -int ZEXPORT deflateReset (strm) - z_streamp strm; +int ZEXPORT deflateReset ( + z_streamp strm +) { int ret; @@ -566,9 +570,10 @@ int ZEXPORT deflateReset (strm) #ifdef NOVGZ /* ========================================================================= */ -int ZEXPORT deflateSetHeader (strm, head) - z_streamp strm; - gz_headerp head; +int ZEXPORT deflateSetHeader ( + z_streamp strm, + gz_headerp head +) { if (deflateStateCheck(strm) || strm->state->wrap != 2) return Z_STREAM_ERROR; @@ -577,10 +582,11 @@ int ZEXPORT deflateSetHeader (strm, head) } /* ========================================================================= */ -int ZEXPORT deflatePending (strm, pending, bits) - unsigned *pending; - int *bits; - z_streamp strm; +int ZEXPORT deflatePending ( + z_streamp strm, + unsigned *pending, + int *bits +) { if (deflateStateCheck(strm)) return Z_STREAM_ERROR; if (pending != Z_NULL) @@ -783,9 +789,10 @@ uLong ZEXPORT deflateBound(strm, sourceLen) * IN assertion: the stream state is correct and there is enough room in * pending_buf. */ -local void putShortMSB (s, b) - deflate_state *s; - uInt b; +local void putShortMSB ( + deflate_state *s, + uInt b +) { put_byte(s, (Byte)(b >> 8)); put_byte(s, (Byte)(b & 0xff)); @@ -797,8 +804,9 @@ local void putShortMSB (s, b) * applications may wish to modify it to avoid allocating a large * strm->next_out buffer and copying into it. (See also read_buf()). */ -local void flush_pending(strm) - z_streamp strm; +local void flush_pending( + z_streamp strm +) { unsigned len; deflate_state *s = strm->state; @@ -830,9 +838,10 @@ local void flush_pending(strm) } while (0) /* ========================================================================= */ -int ZEXPORT deflate (strm, flush) - z_streamp strm; - int flush; +int ZEXPORT deflate ( + z_streamp strm, + int flush +) { int old_flush; /* value of flush param for previous deflate call */ deflate_state *s; @@ -1158,8 +1167,9 @@ int ZEXPORT deflate (strm, flush) } /* ========================================================================= */ -int ZEXPORT deflateEnd (strm) - z_streamp strm; +int ZEXPORT deflateEnd ( + z_streamp strm +) { int status; @@ -1247,10 +1257,11 @@ int ZEXPORT deflateCopy (dest, source) * allocating a large strm->next_in buffer and copying from it. * (See also flush_pending()). */ -local unsigned read_buf(strm, buf, size) - z_streamp strm; - Bytef *buf; - unsigned size; +local unsigned read_buf( + z_streamp strm, + Bytef *buf, + unsigned size +) { unsigned len = strm->avail_in; @@ -1277,8 +1288,9 @@ local unsigned read_buf(strm, buf, size) /* =========================================================================== * Initialize the "longest match" routines for a new zlib stream */ -local void lm_init (s) - deflate_state *s; +local void lm_init ( + deflate_state *s +) { s->window_size = (ulg)2L*s->w_size; @@ -1310,9 +1322,10 @@ local void lm_init (s) * string (strstart) and its distance is <= MAX_DIST, and prev_length >= 1 * OUT assertion: the match length is not greater than s->lookahead. */ -local uInt longest_match(s, cur_match) - deflate_state *s; - IPos cur_match; /* current match */ +local uInt longest_match( + deflate_state *s, + IPos cur_match /* current match */ +) { unsigned chain_length = s->max_chain_length;/* max hash chain length */ register Bytef *scan = s->window + s->strstart; /* current string */ @@ -1559,8 +1572,9 @@ local void check_match(s, start, match, length) * performed for at least two bytes (required for the zip translate_eol * option -- not supported here). */ -local void fill_window(s) - deflate_state *s; +local void fill_window( + deflate_state *s +) { unsigned n; unsigned more; /* Amount of free space at the end of the window. */ @@ -1724,9 +1738,10 @@ local void fill_window(s) * copied. It is most efficient with large input and output buffers, which * maximizes the opportunities to have a single copy from next_in to next_out. */ -local block_state deflate_stored(s, flush) - deflate_state *s; - int flush; +local block_state deflate_stored( + deflate_state *s, + int flush +) { /* Smallest worthy block size when not flushing or finishing. By default * this is 32K. This can be as small as 507 bytes for memLevel == 1. For @@ -1914,9 +1929,10 @@ local block_state deflate_stored(s, flush) * new strings in the dictionary only for unmatched strings or for short * matches. It is used only for the fast compression options. */ -local block_state deflate_fast(s, flush) - deflate_state *s; - int flush; +local block_state deflate_fast( + deflate_state *s, + int flush +) { IPos hash_head; /* head of the hash chain */ int bflush; /* set if current block must be flushed */ @@ -2016,9 +2032,10 @@ local block_state deflate_fast(s, flush) * evaluation for matches: a match is finally adopted only if there is * no better match at the next window position. */ -local block_state deflate_slow(s, flush) - deflate_state *s; - int flush; +local block_state deflate_slow( + deflate_state *s, + int flush +) { IPos hash_head; /* head of hash chain */ int bflush; /* set if current block must be flushed */ diff --git a/lib/libvgz/inffast.c b/lib/libvgz/inffast.c index 903ebfae1..01b00436b 100644 --- a/lib/libvgz/inffast.c +++ b/lib/libvgz/inffast.c @@ -47,9 +47,10 @@ requires strm->avail_out >= 258 for each loop to avoid checking for output space. */ -void ZLIB_INTERNAL inflate_fast(strm, start) -z_streamp strm; -unsigned start; /* inflate()'s starting value for strm->avail_out */ +void ZLIB_INTERNAL inflate_fast( + z_streamp strm, + unsigned start /* inflate()'s starting value for strm->avail_out */ +) { struct inflate_state FAR *state; z_const unsigned char FAR *in; /* local strm->next_in */ diff --git a/lib/libvgz/inflate.c b/lib/libvgz/inflate.c index 5cdbe8b95..cfd708391 100644 --- a/lib/libvgz/inflate.c +++ b/lib/libvgz/inflate.c @@ -104,8 +104,9 @@ local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, unsigned len)); #endif /* NOVGZ */ -local int inflateStateCheck(strm) -z_streamp strm; +local int inflateStateCheck( + z_streamp strm +) { struct inflate_state FAR *state; if (strm == Z_NULL || @@ -118,8 +119,9 @@ z_streamp strm; return 0; } -int ZEXPORT inflateResetKeep(strm) -z_streamp strm; +int ZEXPORT inflateResetKeep( + z_streamp strm +) { struct inflate_state FAR *state; @@ -145,8 +147,9 @@ z_streamp strm; return Z_OK; } -int ZEXPORT inflateReset(strm) -z_streamp strm; +int ZEXPORT inflateReset( + z_streamp strm +) { struct inflate_state FAR *state; @@ -158,9 +161,10 @@ z_streamp strm; return inflateResetKeep(strm); } -int ZEXPORT inflateReset2(strm, windowBits) -z_streamp strm; -int windowBits; +int ZEXPORT inflateReset2( + z_streamp strm, + int windowBits +) { int wrap; struct inflate_state FAR *state; @@ -198,11 +202,12 @@ int windowBits; return inflateReset(strm); } -int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) -z_streamp strm; -int windowBits; -const char *version; -int stream_size; +int ZEXPORT inflateInit2_( + z_streamp strm, + int windowBits, + const char *version, + int stream_size +) { int ret; struct inflate_state FAR *state; @@ -244,18 +249,20 @@ int stream_size; #ifdef NOVGZ -int ZEXPORT inflateInit_(strm, version, stream_size) -z_streamp strm; -const char *version; -int stream_size; +int ZEXPORT inflateInit_( + z_streamp strm, + const char *version, + int stream_size +) { return inflateInit2_(strm, DEF_WBITS, version, stream_size); } -int ZEXPORT inflatePrime(strm, bits, value) -z_streamp strm; -int bits; -int value; +int ZEXPORT inflatePrime( + z_streamp strm, + int bits, + int value +) { struct inflate_state FAR *state; @@ -285,8 +292,9 @@ int value; used for threaded applications, since the rewriting of the tables and virgin may not be thread-safe. */ -local void fixedtables(state) -struct inflate_state FAR *state; +local void fixedtables( + struct inflate_state FAR *state +) { #ifdef BUILDFIXED static int virgin = 1; @@ -403,10 +411,11 @@ void makefixed() output will fall in the output data, making match copies simpler and faster. The advantage may be dependent on the size of the processor's data caches. */ -local int updatewindow(strm, end, copy) -z_streamp strm; -const Bytef *end; -unsigned copy; +local int updatewindow( + z_streamp strm, + const Bytef *end, + unsigned copy +) { struct inflate_state FAR *state; unsigned dist; @@ -629,9 +638,10 @@ unsigned copy; will return Z_BUF_ERROR if it has not reached the end of the stream. */ -int ZEXPORT inflate(strm, flush) -z_streamp strm; -int flush; +int ZEXPORT inflate( + z_streamp strm, + int flush +) { struct inflate_state FAR *state; z_const unsigned char FAR *next; /* next input */ @@ -1313,8 +1323,9 @@ int flush; return ret; } -int ZEXPORT inflateEnd(strm) -z_streamp strm; +int ZEXPORT inflateEnd( + z_streamp strm +) { struct inflate_state FAR *state; if (inflateStateCheck(strm)) diff --git a/lib/libvgz/inftrees.c b/lib/libvgz/inftrees.c index 18faf783c..a794b0e27 100644 --- a/lib/libvgz/inftrees.c +++ b/lib/libvgz/inftrees.c @@ -30,13 +30,14 @@ const char inflate_copyright[] = table index bits. It will differ if the request is greater than the longest code or if it is less than the shortest code. */ -int ZLIB_INTERNAL inflate_table(type, lens, codes, table, bits, work) -codetype type; -unsigned short FAR *lens; -unsigned codes; -code FAR * FAR *table; -unsigned FAR *bits; -unsigned short FAR *work; +int ZLIB_INTERNAL inflate_table( + codetype type, + unsigned short FAR *lens, + unsigned codes, + code FAR * FAR *table, + unsigned FAR *bits, + unsigned short FAR *work +) { unsigned len; /* a code's length in bits */ unsigned sym; /* index of code symbols */ diff --git a/lib/libvgz/trees.c b/lib/libvgz/trees.c index 1ccce63e6..87b5c5f32 100644 --- a/lib/libvgz/trees.c +++ b/lib/libvgz/trees.c @@ -229,7 +229,7 @@ local void send_bits(s, value, length) /* =========================================================================== * Initialize the various 'constant' tables. */ -local void tr_static_init() +local void tr_static_init(void) { #if defined(GEN_TREES_H) || !defined(STDC) static int static_init_done = 0; @@ -376,8 +376,9 @@ void gen_trees_header() /* =========================================================================== * Initialize the tree data structures for a new zlib stream. */ -void ZLIB_INTERNAL _tr_init(s) - deflate_state *s; +void ZLIB_INTERNAL _tr_init( + deflate_state *s +) { tr_static_init(); @@ -404,8 +405,9 @@ void ZLIB_INTERNAL _tr_init(s) /* =========================================================================== * Initialize a new block. */ -local void init_block(s) - deflate_state *s; +local void init_block( + deflate_state *s +) { int n; /* iterates over tree elements */ @@ -448,10 +450,11 @@ local void init_block(s) * when the heap property is re-established (each father smaller than its * two sons). */ -local void pqdownheap(s, tree, k) - deflate_state *s; - ct_data *tree; /* the tree to restore */ - int k; /* node to move down */ +local void pqdownheap( + deflate_state *s, + ct_data *tree, /* the tree to restore */ + int k /* node to move down */ +) { int v = s->heap[k]; int j = k << 1; /* left son of k */ @@ -483,9 +486,10 @@ local void pqdownheap(s, tree, k) * The length opt_len is updated; static_len is also updated if stree is * not null. */ -local void gen_bitlen(s, desc) - deflate_state *s; - tree_desc *desc; /* the tree descriptor */ +local void gen_bitlen( + deflate_state *s, + tree_desc *desc /* the tree descriptor */ +) { ct_data *tree = desc->dyn_tree; int max_code = desc->max_code; @@ -569,10 +573,11 @@ local void gen_bitlen(s, desc) * OUT assertion: the field code is set for all tree elements of non * zero code length. */ -local void gen_codes (tree, max_code, bl_count) - ct_data *tree; /* the tree to decorate */ - int max_code; /* largest code with non zero frequency */ - ushf *bl_count; /* number of codes at each bit length */ +local void gen_codes ( + ct_data *tree, /* the tree to decorate */ + int max_code, /* largest code with non zero frequency */ + ushf *bl_count /* number of codes at each bit length */ +) { ush next_code[MAX_BITS+1]; /* next code value for each bit length */ unsigned code = 0; /* running code value */ @@ -612,9 +617,10 @@ local void gen_codes (tree, max_code, bl_count) * and corresponding code. The length opt_len is updated; static_len is * also updated if stree is not null. The field max_code is set. */ -local void build_tree(s, desc) - deflate_state *s; - tree_desc *desc; /* the tree descriptor */ +local void build_tree( + deflate_state *s, + tree_desc *desc /* the tree descriptor */ +) { ct_data *tree = desc->dyn_tree; const ct_data *stree = desc->stat_desc->static_tree; @@ -700,10 +706,11 @@ local void build_tree(s, desc) * Scan a literal or distance tree to determine the frequencies of the codes * in the bit length tree. */ -local void scan_tree (s, tree, max_code) - deflate_state *s; - ct_data *tree; /* the tree to be scanned */ - int max_code; /* and its largest code of non zero frequency */ +local void scan_tree ( + deflate_state *s, + ct_data *tree, /* the tree to be scanned */ + int max_code /* and its largest code of non zero frequency */ +) { int n; /* iterates over all tree elements */ int prevlen = -1; /* last emitted length */ @@ -745,10 +752,11 @@ local void scan_tree (s, tree, max_code) * Send a literal or distance tree in compressed form, using the codes in * bl_tree. */ -local void send_tree (s, tree, max_code) - deflate_state *s; - ct_data *tree; /* the tree to be scanned */ - int max_code; /* and its largest code of non zero frequency */ +local void send_tree ( + deflate_state *s, + ct_data *tree, /* the tree to be scanned */ + int max_code /* and its largest code of non zero frequency */ +) { int n; /* iterates over all tree elements */ int prevlen = -1; /* last emitted length */ @@ -796,8 +804,9 @@ local void send_tree (s, tree, max_code) * Construct the Huffman tree for the bit lengths and return the index in * bl_order of the last bit length code to send. */ -local int build_bl_tree(s) - deflate_state *s; +local int build_bl_tree( + deflate_state *s +) { int max_blindex; /* index of last bit length code of non zero freq */ @@ -831,9 +840,12 @@ local int build_bl_tree(s) * lengths of the bit length codes, the literal tree and the distance tree. * IN assertion: lcodes >= 257, dcodes >= 1, blcodes >= 4. */ -local void send_all_trees(s, lcodes, dcodes, blcodes) - deflate_state *s; - int lcodes, dcodes, blcodes; /* number of codes for each tree */ +local void send_all_trees( + deflate_state *s, + int lcodes, + int dcodes, + int blcodes /* number of codes for each tree */ +) { int rank; /* index in bl_order */ @@ -860,11 +872,12 @@ local void send_all_trees(s, lcodes, dcodes, blcodes) /* =========================================================================== * Send a stored block */ -void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) - deflate_state *s; - charf *buf; /* input block */ - ulg stored_len; /* length of input block */ - int last; /* one if this is the last block for a file */ +void ZLIB_INTERNAL _tr_stored_block( + deflate_state *s, + charf *buf, /* input block */ + ulg stored_len, /* length of input block */ + int last /* one if this is the last block for a file */ +) { if (last) s->strm->last_bit = @@ -891,8 +904,9 @@ void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) /* =========================================================================== * Flush the bits in the bit buffer to pending output (leaves at most 7 bits) */ -void ZLIB_INTERNAL _tr_flush_bits(s) - deflate_state *s; +void ZLIB_INTERNAL _tr_flush_bits( + deflate_state *s +) { bi_flush(s); } @@ -901,8 +915,9 @@ void ZLIB_INTERNAL _tr_flush_bits(s) * Send one empty static block to give enough lookahead for inflate. * This takes 10 bits, of which 7 may remain in the bit buffer. */ -void ZLIB_INTERNAL _tr_align(s) - deflate_state *s; +void ZLIB_INTERNAL _tr_align( + deflate_state *s +) { send_bits(s, STATIC_TREES<<1, 3); send_code(s, END_BLOCK, static_ltree); @@ -916,11 +931,12 @@ void ZLIB_INTERNAL _tr_align(s) * Determine the best encoding for the current block: dynamic trees, static * trees or store, and write out the encoded block. */ -void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) - deflate_state *s; - charf *buf; /* input block, or NULL if too old */ - ulg stored_len; /* length of input block */ - int last; /* one if this is the last block for a file */ +void ZLIB_INTERNAL _tr_flush_block( + deflate_state *s, + charf *buf, /* input block, or NULL if too old */ + ulg stored_len, /* length of input block */ + int last /* one if this is the last block for a file */ +) { ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */ int max_blindex = 0; /* index of last bit length code of non zero freq */ @@ -1024,10 +1040,11 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) * Save the match info and tally the frequency counts. Return true if * the current block must be flushed. */ -int ZLIB_INTERNAL _tr_tally (s, dist, lc) - deflate_state *s; - unsigned dist; /* distance of matched string */ - unsigned lc; /* match length - MIN_MATCH or unmatched char (dist==0) */ +int ZLIB_INTERNAL _tr_tally ( + deflate_state *s, + unsigned dist, /* distance of matched string */ + unsigned lc /* match length - MIN_MATCH or unmatched char (dist==0) */ +) { s->sym_buf[s->sym_next++] = (uch)dist; s->sym_buf[s->sym_next++] = (uch)(dist >> 8); @@ -1052,10 +1069,11 @@ int ZLIB_INTERNAL _tr_tally (s, dist, lc) /* =========================================================================== * Send the block data compressed using the given Huffman trees */ -local void compress_block(s, ltree, dtree) - deflate_state *s; - const ct_data *ltree; /* literal tree */ - const ct_data *dtree; /* distance tree */ +local void compress_block( + deflate_state *s, + const ct_data *ltree, /* literal tree */ + const ct_data *dtree /* distance tree */ +) { unsigned dist; /* distance of matched string */ int lc; /* match length or unmatched char (if dist == 0) */ @@ -1112,8 +1130,9 @@ local void compress_block(s, ltree, dtree) * (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}). * IN assertion: the fields Freq of dyn_ltree are set. */ -local int detect_data_type(s) - deflate_state *s; +local int detect_data_type( + deflate_state *s +) { /* block_mask is the bit mask of block-listed bytes * set bits 0..6, 14..25, and 28..31 @@ -1146,9 +1165,10 @@ local int detect_data_type(s) * method would use a table) * IN assertion: 1 <= len <= 15 */ -local unsigned bi_reverse(code, len) - unsigned code; /* the value to invert */ - int len; /* its bit length */ +local unsigned bi_reverse( + unsigned code, /* the value to invert */ + int len /* its bit length */ +) { register unsigned res = 0; do { @@ -1161,8 +1181,9 @@ local unsigned bi_reverse(code, len) /* =========================================================================== * Flush the bit buffer, keeping at most 7 bits in it. */ -local void bi_flush(s) - deflate_state *s; +local void bi_flush( + deflate_state *s +) { if (s->bi_valid == 16) { put_short(s, s->bi_buf); @@ -1178,8 +1199,9 @@ local void bi_flush(s) /* =========================================================================== * Flush the bit buffer and align the output on a byte boundary */ -local void bi_windup(s) - deflate_state *s; +local void bi_windup( + deflate_state *s +) { if (s->bi_valid > 8) { put_short(s, s->bi_buf); diff --git a/lib/libvgz/zutil.c b/lib/libvgz/zutil.c index fd0cda93b..6a5b32c3e 100644 --- a/lib/libvgz/zutil.c +++ b/lib/libvgz/zutil.c @@ -307,19 +307,21 @@ extern voidp calloc OF((uInt items, uInt size)); extern void free OF((voidpf ptr)); #endif -voidpf ZLIB_INTERNAL zcalloc (opaque, items, size) - voidpf opaque; - unsigned items; - unsigned size; +voidpf ZLIB_INTERNAL zcalloc ( + voidpf opaque, + unsigned items, + unsigned size +) { (void)opaque; return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : (voidpf)calloc(items, size); } -void ZLIB_INTERNAL zcfree (opaque, ptr) - voidpf opaque; - voidpf ptr; +void ZLIB_INTERNAL zcfree ( + voidpf opaque, + voidpf ptr +) { (void)opaque; free(ptr); From nils.goroll at uplex.de Thu Jun 22 13:18:11 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 22 Jun 2023 13:18:11 +0000 (UTC) Subject: [master] 466504669 mgt: Send launch error to CLI also Message-ID: <20230622131811.63BBF107DD2@lists.varnish-cache.org> commit 466504669f9cf25b2ddb9ab36e8eb2647596f6a0 Author: Nils Goroll Date: Thu Jun 22 09:33:40 2023 +0200 mgt: Send launch error to CLI also diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 55259fe90..28e7b4570 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -291,6 +291,14 @@ child_poker(const struct vev *e, int what) * Launch the child process */ +#define mgt_launch_err(cli, status, ...) do { \ + MGT_Complain(C_ERR, __VA_ARGS__); \ + if (cli == NULL) \ + break; \ + VCLI_Out(cli, __VA_ARGS__); \ + VCLI_SetResult(cli, status); \ + } while (0) + static void mgt_launch_child(struct cli *cli) { @@ -467,8 +475,7 @@ mgt_launch_child(struct cli *cli) child_pid = pid; if (mgt_push_vcls(cli, &u, &p)) { - VCLI_SetResult(cli, u); - MGT_Complain(C_ERR, "Child (%jd) Pushing vcls failed:\n%s", + mgt_launch_err(cli, u, "Child (%jd) Pushing vcls failed:\n%s", (intmax_t)child_pid, p); free(p); MCH_Stop_Child(); @@ -476,8 +483,7 @@ mgt_launch_child(struct cli *cli) } if (mgt_cli_askchild(&u, &p, "start\n")) { - VCLI_SetResult(cli, u); - MGT_Complain(C_ERR, "Child (%jd) Acceptor start failed:\n%s", + mgt_launch_err(cli, u, "Child (%jd) Acceptor start failed:\n%s", (intmax_t)child_pid, p); free(p); MCH_Stop_Child(); From nils.goroll at uplex.de Fri Jun 23 10:27:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 23 Jun 2023 10:27:06 +0000 (UTC) Subject: [master] 0c2599e7c Try to stabilize m0.vtc on arm vtester Message-ID: <20230623102706.9F67510A653@lists.varnish-cache.org> commit 0c2599e7c3bad90a8ca61749d575533478292dda Author: Nils Goroll Date: Fri Jun 23 12:25:02 2023 +0200 Try to stabilize m0.vtc on arm vtester from the log: ** top === shell -exit 1 -expect {failing as requested} { **** top shell_cmd|exec 2>&1 ; **** top shell_cmd|\tvarnishadm -n /root/VT/_vtest_tmp/vtc.37277.0dc3c11b/v1 vcl.load f1 /root/VT/_vtest_tmp/vtc.37277.0dc3c11b/f1 **** dT 27.789 **** v1 vsl| 0 CLI - Rd vcl.load f1 vcl_f1.1686893821.277415/vgc.so 1auto **** v1 vsl| 0 CLI - Wr 300 62 VCL "f1" Failed initialization Message: failing as requested **** dT 27.893 **** v1 vsl| 0 CLI - Rd ping **** v1 vsl| 0 CLI - Wr 200 19 PONG 1686893826 1.0 **** dT 32.987 **** top shell_out|CLI communication error (hdr) **** top shell_status = 0x0002 ---- top shell_exit not as expected: got 0x0002 wanted 0x0001 * top RESETTING after ../../../../bin/varnishtest/tests/m00000.vtc **** dT 32.988 diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index a5d94aca4..715f5fa7b 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -187,7 +187,7 @@ shell { varnish v1 -clierr 300 "vcl.load f1 ${tmpdir}/f1" shell -exit 1 -expect {failing as requested} { - varnishadm -n ${tmpdir}/v1 vcl.load f1 ${tmpdir}/f1 + varnishadm -t 10 -n ${tmpdir}/v1 vcl.load f1 ${tmpdir}/f1 } varnish v1 -cliok "param.set vcc_feature +allow_inline_c" From nils.goroll at uplex.de Fri Jun 23 10:49:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 23 Jun 2023 10:49:06 +0000 (UTC) Subject: [master] d208f376e vtest: make the basename of the test directory available as vtcid Message-ID: <20230623104906.CF19910D4A4@lists.varnish-cache.org> commit d208f376e6c706a962686a20413b814efccd24d0 Author: Nils Goroll Date: Fri Jun 23 12:41:01 2023 +0200 vtest: make the basename of the test directory available as vtcid to have available a unique string for concurrently running tests which is not a path. diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 12647c6ee..55080ac27 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -600,6 +600,11 @@ exec_file(const char *fn, const char *script, const char *tmpdir, /* Move into our tmpdir */ AZ(chdir(tmpdir)); macro_def(vltop, NULL, "tmpdir", "%s", tmpdir); + p = strrchr(tmpdir, '/'); + AN(p); + p++; + AN(*p); + macro_def(vltop, NULL, "vtcid", "%s", p); /* Drop file to tell what was going on here */ f = fopen("INFO", "w"); From nils.goroll at uplex.de Fri Jun 23 10:49:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 23 Jun 2023 10:49:06 +0000 (UTC) Subject: [master] baf60b4d2 Make c121.vtc robust against concurrent execution Message-ID: <20230623104906.F1BD310D4A7@lists.varnish-cache.org> commit baf60b4d2ef5f418e6ac470f3562e43c37e2ad44 Author: Nils Goroll Date: Fri Jun 23 12:44:42 2023 +0200 Make c121.vtc robust against concurrent execution this happened on one of the uplex linux vtesters which runs gcc and clang in parallel un-namespaced * top VTEST Abstract UDS backend: change path, drop poll ** top === feature abstract_uds **** dT 0.004 ** top === server s1 -listen "@vtc.s1.sock" { ** s1 Starting server ---- s1 Server listen address (@vtc.s1.sock) cannot be resolved: bind(2) diff --git a/bin/varnishtest/tests/c00121.vtc b/bin/varnishtest/tests/c00121.vtc index 002a9aca5..d810306aa 100644 --- a/bin/varnishtest/tests/c00121.vtc +++ b/bin/varnishtest/tests/c00121.vtc @@ -2,7 +2,7 @@ varnishtest "Abstract UDS backend: change path, drop poll" feature abstract_uds -server s1 -listen "@vtc.s1.sock" { +server s1 -listen "@${vtcid}.s1.sock" { non_fatal timeout 3 loop 40 { @@ -12,7 +12,7 @@ server s1 -listen "@vtc.s1.sock" { } } -start -server s2 -listen "@vtc.s2.sock" { +server s2 -listen "@${vtcid}.s2.sock" { non_fatal timeout 3 loop 40 { @@ -22,7 +22,7 @@ server s2 -listen "@vtc.s2.sock" { } } -start -varnish v1 -arg "-a @vtc.v1.sock" -vcl { +varnish v1 -arg "-a @${vtcid}.v1.sock" -vcl { probe default { .window = 8; .initial = 7; @@ -30,7 +30,7 @@ varnish v1 -arg "-a @vtc.v1.sock" -vcl { .interval = 0.1s; } backend s1 { - .path = "@vtc.s2.sock"; + .path = "@${vtcid}.s2.sock"; } } -start @@ -44,7 +44,7 @@ varnish v1 -vcl { .interval = 0.1s; } backend s1 { - .path = "@vtc.s1.sock"; + .path = "@${vtcid}.s1.sock"; } } -cliok "vcl.use vcl2" -cliok "vcl.discard vcl1" @@ -52,7 +52,7 @@ delay 1 varnish v1 -vcl { backend s1 { - .path = "@vtc.s1.sock"; + .path = "@${vtcid}.s1.sock"; } } -cliok "vcl.use vcl3" -cliok "vcl.discard vcl2" @@ -69,7 +69,7 @@ server s1 -break { delay 1 -client c1 -connect "@vtc.v1.sock" { +client c1 -connect "@${vtcid}.v1.sock" { txreq -url /foo rxresp txreq -url /foo From phk at FreeBSD.org Mon Jun 26 08:05:11 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Jun 2023 08:05:11 +0000 (UTC) Subject: [master] 2459de453 s/this/that/ to not stomp on C++ reserved words. Message-ID: <20230626080511.F2D1811ADAA@lists.varnish-cache.org> commit 2459de453975767936009cadecede791b83c4a03 Author: Poul-Henning Kamp Date: Mon Jun 26 08:03:55 2023 +0000 s/this/that/ to not stomp on C++ reserved words. diff --git a/vmod/vmod_debug_acl.c b/vmod/vmod_debug_acl.c index 8f277e7a0..fadc6cea6 100644 --- a/vmod/vmod_debug_acl.c +++ b/vmod/vmod_debug_acl.c @@ -85,14 +85,14 @@ struct acl_sweep { uint8_t *probe_p; VCL_INT step; uint64_t reset; - uint64_t this; + uint64_t that; uint64_t count; }; static void reset_sweep(struct acl_sweep *asw) { - asw->this = asw->reset; + asw->that = asw->reset; } static int @@ -129,7 +129,7 @@ setup_sweep(VRT_CTX, struct acl_sweep *asw, VCL_IP ip0, VCL_IP ip1, } asw->reset = vbe64dec(asw->ip0_p + 8); } - asw->this = asw->reset; + asw->that = asw->reset; /* Dont try this at home */ asw->probe = malloc(vsa_suckaddr_len); @@ -156,12 +156,12 @@ step_sweep(struct acl_sweep *asw) AN(asw); asw->count++; - asw->this += asw->step; + asw->that += asw->step; if (asw->family == PF_INET) { - vbe32enc(asw->probe_p, asw->this); + vbe32enc(asw->probe_p, asw->that); return (memcmp(asw->probe_p, asw->ip1_p, 4)); } else { - vbe64enc(asw->probe_p + 8, asw->this); + vbe64enc(asw->probe_p + 8, asw->that); return (memcmp(asw->probe_p, asw->ip1_p, 16)); } } From dridi at varni.sh Mon Jun 26 08:31:19 2023 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 26 Jun 2023 08:31:19 +0000 Subject: [master] 2459de453 s/this/that/ to not stomp on C++ reserved words. In-Reply-To: <20230626080511.F2D1811ADAA@lists.varnish-cache.org> References: <20230626080511.F2D1811ADAA@lists.varnish-cache.org> Message-ID: On Mon, Jun 26, 2023 at 8:05?AM Poul-Henning Kamp wrote: > > > commit 2459de453975767936009cadecede791b83c4a03 > Author: Poul-Henning Kamp > Date: Mon Jun 26 08:03:55 2023 +0000 > > s/this/that/ to not stomp on C++ reserved words. Why not s/this/self/ like in python? From phk at phk.freebsd.dk Mon Jun 26 08:37:27 2023 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Mon, 26 Jun 2023 08:37:27 +0000 Subject: [master] 2459de453 s/this/that/ to not stomp on C++ reserved words. In-Reply-To: References: <20230626080511.F2D1811ADAA@lists.varnish-cache.org> Message-ID: <202306260837.35Q8bRPt065946@critter.freebsd.dk> -------- Dridi Boukelmoune writes: > On Mon, Jun 26, 2023 at 8:05=E2=80=AFAM Poul-Henning Kamp = > > commit 2459de453975767936009cadecede791b83c4a03 > > Author: Poul-Henning Kamp > > Date: Mon Jun 26 08:03:55 2023 +0000 > > > > s/this/that/ to not stomp on C++ reserved words. > > Why not s/this/self/ like in python? Because then it would become in the way when we migrate to Python ? :-) -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Mon Jun 26 08:57:34 2023 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 26 Jun 2023 08:57:34 +0000 Subject: [master] 2459de453 s/this/that/ to not stomp on C++ reserved words. In-Reply-To: <202306260837.35Q8bRPt065946@critter.freebsd.dk> References: <20230626080511.F2D1811ADAA@lists.varnish-cache.org> <202306260837.35Q8bRPt065946@critter.freebsd.dk> Message-ID: On Mon, Jun 26, 2023 at 8:37?AM Poul-Henning Kamp wrote: > > -------- > Dridi Boukelmoune writes: > > On Mon, Jun 26, 2023 at 8:05=E2=80=AFAM Poul-Henning Kamp = > > > > commit 2459de453975767936009cadecede791b83c4a03 > > > Author: Poul-Henning Kamp > > > Date: Mon Jun 26 08:03:55 2023 +0000 > > > > > > s/this/that/ to not stomp on C++ reserved words. > > > > Why not s/this/self/ like in python? > > Because then it would become in the way when we migrate to Python ? :-) I see From nils.goroll at uplex.de Mon Jun 26 13:41:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Jun 2023 13:41:06 +0000 (UTC) Subject: [master] 0a7f23d97 More backtrace levels Message-ID: <20230626134106.A81267EA1@lists.varnish-cache.org> commit 0a7f23d973c3a9ddde1389bf1c51516da7d935bc Author: Nils Goroll Date: Fri Jun 23 18:02:34 2023 +0200 More backtrace levels https://gitlab.com/uplex/varnish/slash/-/issues/13 is an example where 9 stack frames are used by abort handling alone. diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 7f6d66873..221d9835c 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -646,11 +646,7 @@ pan_backtrace(struct vsb *vsb) #else /* WITH_UNWIND */ -#if defined(ENABLE_SANITIZER) -# define BACKTRACE_LEVELS 20 -#else -# define BACKTRACE_LEVELS 10 -#endif +#define BACKTRACE_LEVELS 20 static void pan_backtrace(struct vsb *vsb) From dridi at varni.sh Mon Jun 26 13:44:55 2023 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 26 Jun 2023 13:44:55 +0000 Subject: [master] 0a7f23d97 More backtrace levels In-Reply-To: <20230626134106.A81267EA1@lists.varnish-cache.org> References: <20230626134106.A81267EA1@lists.varnish-cache.org> Message-ID: On Mon, Jun 26, 2023 at 1:41?PM Nils Goroll wrote: > > > commit 0a7f23d973c3a9ddde1389bf1c51516da7d935bc > Author: Nils Goroll > Date: Fri Jun 23 18:02:34 2023 +0200 > > More backtrace levels > > https://gitlab.com/uplex/varnish/slash/-/issues/13 is an example where > 9 stack frames are used by abort handling alone. Consider building Varnish --with-unwind then, it goes all the way down. > diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c > index 7f6d66873..221d9835c 100644 > --- a/bin/varnishd/cache/cache_panic.c > +++ b/bin/varnishd/cache/cache_panic.c > @@ -646,11 +646,7 @@ pan_backtrace(struct vsb *vsb) > > #else /* WITH_UNWIND */ > > -#if defined(ENABLE_SANITIZER) > -# define BACKTRACE_LEVELS 20 > -#else > -# define BACKTRACE_LEVELS 10 > -#endif > +#define BACKTRACE_LEVELS 20 > > static void > pan_backtrace(struct vsb *vsb) > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Mon Jun 26 14:01:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Jun 2023 14:01:06 +0000 (UTC) Subject: [master] d91514408 Warn about pitfall: workdir on noexec mountpoint Message-ID: <20230626140106.3863061E21@lists.varnish-cache.org> commit d91514408f4e9122c0b9163bb116039e7dba6a7f Author: Nils Goroll Date: Thu Jun 22 18:00:44 2023 +0200 Warn about pitfall: workdir on noexec mountpoint Fixes #3943 diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index ad811c4eb..63b9b9eac 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -486,6 +486,7 @@ VCL_Open(const char *fn, struct vsb *msg) if (dlh == NULL) { VSB_cat(msg, "Could not load compiled VCL.\n"); VSB_printf(msg, "\tdlopen() = %s\n", dlerror()); + VSB_cat(msg, "\thint: check for \"noexec\" mount\n"); return (NULL); } cnf = dlsym(dlh, "VCL_conf"); diff --git a/bin/varnishd/mgt/mgt_jail.c b/bin/varnishd/mgt/mgt_jail.c index 81c2fcdb8..8a133707f 100644 --- a/bin/varnishd/mgt/mgt_jail.c +++ b/bin/varnishd/mgt/mgt_jail.c @@ -39,6 +39,7 @@ #include #include #include +#include #include "mgt/mgt.h" #include "common/heritage.h" @@ -138,6 +139,7 @@ VJ_subproc(enum jail_subproc_e jse) int VJ_make_workdir(const char *dname) { + struct statvfs vfs[1]; int i; AN(dname); @@ -163,6 +165,17 @@ VJ_make_workdir(const char *dname) ARGV_ERR("Cannot create test-file in %s (%s)\n" "Check permissions (or delete old directory)\n", dname, VAS_errtxt(errno)); + +#ifdef ST_NOEXEC + /* deliberately ignore fstatvfs errors */ + if (! fstatvfs(i, vfs) && vfs->f_flag & ST_NOEXEC) { + closefd(&i); + AZ(unlink("_.testfile")); + ARGV_ERR("Working directory %s (-n argument) " + "can not reside on a file system mounted noexec\n", dname); + } +#endif + closefd(&i); AZ(unlink("_.testfile")); VJ_master(JAIL_MASTER_LOW); From nils.goroll at uplex.de Mon Jun 26 14:05:07 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Jun 2023 14:05:07 +0000 (UTC) Subject: [master] 27f4b714d build: Run test cases after refreshing the test suite Message-ID: <20230626140507.2C1BD6222F@lists.varnish-cache.org> commit 27f4b714dc0831a55621d5012735993c219eca77 Author: Dridi Boukelmoune Date: Fri Jun 23 15:52:04 2023 +0200 build: Run test cases after refreshing the test suite This was fixed after several attempts in the past [1] and I convinced myself that I was doing it wrong when I implemented generic VSC rules similarly to how VTC rules were centralized [2] but some lessons will never be learned. The reason why I was so easily convinced is that the '-local' suffix is clearly documented, but the '-am' one is not, which leads me to believe that it is an implementation detail we shouldn't rely on. The documentation clearly states the lack of '-local' ordering, one more reason not to rely on 'check-local'. If we can use neither 'check-local' nor 'check-am' reliably for the test suite refresh vs execution ordering, maybe there's a simpler solution? Fixes #3942 [1] 85e3d442c7895e96b4a59a697921a8491f820461 [2] dcaf616c66d93de69735237967cc091fa490bb93 diff --git a/vtc.am b/vtc.am index cf29b5c9b..0994f833f 100644 --- a/vtc.am +++ b/vtc.am @@ -11,7 +11,7 @@ VTC_LOG_COMPILER = $(top_builddir)/bin/varnishtest/varnishtest -v -i TEST_EXTENSIONS = .vtc -check-local: vtc-check-tests +check: vtc-check-tests vtc-check-tests: @mkdir -p tests ; \ From nils.goroll at uplex.de Mon Jun 26 14:12:05 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Jun 2023 14:12:05 +0000 (UTC) Subject: [master] dee5a1b16 Avoid unused variable on bsd Message-ID: <20230626141205.BAD15627EF@lists.varnish-cache.org> commit dee5a1b16e67d340b131fd58b6ae1b55d436c818 Author: Nils Goroll Date: Mon Jun 26 16:11:03 2023 +0200 Avoid unused variable on bsd diff --git a/bin/varnishd/mgt/mgt_jail.c b/bin/varnishd/mgt/mgt_jail.c index 8a133707f..61ccc87ad 100644 --- a/bin/varnishd/mgt/mgt_jail.c +++ b/bin/varnishd/mgt/mgt_jail.c @@ -139,7 +139,6 @@ VJ_subproc(enum jail_subproc_e jse) int VJ_make_workdir(const char *dname) { - struct statvfs vfs[1]; int i; AN(dname); @@ -167,6 +166,8 @@ VJ_make_workdir(const char *dname) dname, VAS_errtxt(errno)); #ifdef ST_NOEXEC + struct statvfs vfs[1]; + /* deliberately ignore fstatvfs errors */ if (! fstatvfs(i, vfs) && vfs->f_flag & ST_NOEXEC) { closefd(&i); From nils.goroll at uplex.de Mon Jun 26 15:00:07 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Jun 2023 15:00:07 +0000 (UTC) Subject: [master] c743dc0f8 Silence Flexelint Message-ID: <20230626150007.E916964500@lists.varnish-cache.org> commit c743dc0f87480fd0c8917c65cc23e55977bbaa26 Author: Nils Goroll Date: Mon Jun 26 16:52:52 2023 +0200 Silence Flexelint I believe this is the most invasive method: -e{766} has no effect --e{766} applies to the whole file -efile(766, ...) in flint.lnt applies to the (sub)tree diff --git a/bin/varnishd/mgt/mgt_jail.c b/bin/varnishd/mgt/mgt_jail.c index 61ccc87ad..c5b828a9b 100644 --- a/bin/varnishd/mgt/mgt_jail.c +++ b/bin/varnishd/mgt/mgt_jail.c @@ -39,6 +39,7 @@ #include #include #include +//lint -efile(766, sys/statvfs.h) #include #include "mgt/mgt.h" From nils.goroll at uplex.de Mon Jun 26 15:00:08 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Jun 2023 15:00:08 +0000 (UTC) Subject: [master] 91df6a1bf Remove flexelint options with no effect Message-ID: <20230626150008.1A6FE64506@lists.varnish-cache.org> commit 91df6a1bf0351fafda9d210a3bd859f32514dc11 Author: Nils Goroll Date: Mon Jun 26 16:58:17 2023 +0200 Remove flexelint options with no effect We have two forms of -efile(766, config.h) globally diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c index 749306835..d35d93726 100644 --- a/bin/varnishd/mgt/mgt_jail_solaris.c +++ b/bin/varnishd/mgt/mgt_jail_solaris.c @@ -204,7 +204,6 @@ * */ -//lint -e{766} #include "config.h" #ifdef HAVE_SETPPRIV diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c index ffeb33f90..04a74827d 100644 --- a/bin/varnishd/waiter/cache_waiter_epoll.c +++ b/bin/varnishd/waiter/cache_waiter_epoll.c @@ -32,7 +32,6 @@ * - thank you, Marc Alexander Lehmann */ -//lint -e{766} #include "config.h" #if defined(HAVE_EPOLL_CTL) diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c index 7d016c442..330265b59 100644 --- a/bin/varnishd/waiter/cache_waiter_kqueue.c +++ b/bin/varnishd/waiter/cache_waiter_kqueue.c @@ -30,7 +30,6 @@ * */ -//lint -e{766} #include "config.h" #if defined(HAVE_KQUEUE) diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c index e717fb868..5cbe6f6d1 100644 --- a/bin/varnishd/waiter/cache_waiter_ports.c +++ b/bin/varnishd/waiter/cache_waiter_ports.c @@ -63,7 +63,6 @@ * */ -//lint -e{766} #include "config.h" #if defined(HAVE_PORT_CREATE) From dridi.boukelmoune at gmail.com Mon Jun 26 15:41:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 26 Jun 2023 15:41:06 +0000 (UTC) Subject: [master] ee0939bf0 build: Retire zlib-specific flags Message-ID: <20230626154106.AE59D100D77@lists.varnish-cache.org> commit ee0939bf032fb9d5977a1e142a2fcaa2faa8078d Author: Dridi Boukelmoune Date: Mon Jun 26 17:32:46 2023 +0200 build: Retire zlib-specific flags But retain the ability to specify them: make VGZ_CFLAGS=-Wfoo We may need them again if new warnings crop up when we sync with zlib in the future, or if we retire libvgz in favor of a vanilla zlib. Refs 3df9cdc1eda72ec736c603b818387fb8ab825569 diff --git a/configure.ac b/configure.ac index c1877cee0..7a6af4796 100644 --- a/configure.ac +++ b/configure.ac @@ -695,13 +695,6 @@ if test $? -ne 0 ; then fi # zlib-specific flags -AX_CHECK_COMPILE_FLAG([-Wno-error=strict-prototypes], - [VGZ_CFLAGS="$VGZ_CFLAGS -Wno-error=strict-prototypes"]) - -dnl https://github.com/madler/zlib/issues/633 -AX_CHECK_COMPILE_FLAG([-Wno-error=deprecated-non-prototype], - [VGZ_CFLAGS="$VGZ_CFLAGS -Wno-error=deprecated-non-prototype"]) - AC_SUBST(VGZ_CFLAGS) # Support for visibility attribute (zlib) From nils.goroll at uplex.de Mon Jun 26 16:42:06 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Jun 2023 16:42:06 +0000 (UTC) Subject: [master] 1af069bc0 Bring back close_range() support proper Message-ID: <20230626164206.C1520102C81@lists.varnish-cache.org> commit 1af069bc0c48d020e4088eab895bfdf967fd8af5 Author: Nils Goroll Date: Mon Jun 26 18:24:35 2023 +0200 Bring back close_range() support proper As noted in 31baed29c5e77bafe8850d935240acefdb33cd4c, my commit 0c1aef58e57a5935116bf55176cbf48aad3d3a08 was wrong, and it was even worse than we thought: Despite what the linux man page suggests, the close_range() declaration is in unistd.h on Linux like on freebsd. We do not actually need linux/close_range.h, because it has only macro definitions which we do not need. We now add a specific configure test if close_range() not only exists but also works. Closes #3905 diff --git a/configure.ac b/configure.ac index 7a6af4796..5627b7aa9 100644 --- a/configure.ac +++ b/configure.ac @@ -623,6 +623,27 @@ if test "$ac_cv_have_tcp_fastopen" = yes; then fi LIBS="${save_LIBS}" +AC_CHECK_FUNCS([close_range]) + +# Check for working close_range() +if test "$ac_cv_func_close_range" = yes; then +AC_CACHE_CHECK([if close_range is working], + [ac_cv_have_working_close_range], + [AC_RUN_IFELSE( + [AC_LANG_PROGRAM([[ +#include + ]],[[ +return (close_range(0, 2, 0)); + ]])], + [ac_cv_have_working_close_range=yes], + [ac_cv_have_working_close_range=no]) + ]) +fi +if test "x$ac_cv_have_working_close_range" = xyes; then + AC_DEFINE([HAVE_WORKING_CLOSE_RANGE], [1], + [Define if OS has working close_range()]) +fi + # Run-time directory if test "${localstatedir}" = '${prefix}/var' ; then VARNISH_STATE_DIR='/var/run' diff --git a/lib/libvarnish/vsub.c b/lib/libvarnish/vsub.c index b528c12c4..27caec46d 100644 --- a/lib/libvarnish/vsub.c +++ b/lib/libvarnish/vsub.c @@ -37,8 +37,10 @@ #include #include // Solaris closefrom(3c) #include -#include -#ifndef HAVE_CLOSEFROM +#include // BSD/Linux close_range(2) +#ifdef HAVE_WORKING_CLOSE_RANGE +#elif HAVE_CLOSEFROM +#else # include #endif @@ -65,7 +67,10 @@ VSUB_closefrom(int fd) assert(fd >= 0); -#ifdef HAVE_CLOSEFROM +#ifdef HAVE_WORKING_CLOSE_RANGE + AZ(close_range(fd, ~0U, 0)); + return; +#elif HAVE_CLOSEFROM closefrom(fd); return; #else From dridi.boukelmoune at gmail.com Wed Jun 28 15:10:10 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 28 Jun 2023 15:10:10 +0000 (UTC) Subject: [master] 9755dc718 vcc: Plug inconsequential leak Message-ID: <20230628151010.5B832117195@lists.varnish-cache.org> commit 9755dc718e7946f873be9b32492fcda3dffa3c82 Author: Dridi Boukelmoune Date: Wed Jun 28 15:22:43 2023 +0200 vcc: Plug inconsequential leak The VCC_GlobalSymbol() function might be called twice for the same symbol. For example a subroutine symbol may be created when the sub keyword is first encountered, but it was referenced by a call action before the subroutine definition. The main problem the leak is causing is lsan's output polluting test cases looking at the screen output of varnishd, making the lines we care about scroll out to oblivion. To remedy this, VCC_GlobalSymbol() idempotence becomes free of side effects. diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 1478509ed..3425c1c86 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -513,12 +513,22 @@ VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, vcc_ns_t ns, vcc_kind_t kind) void VCC_GlobalSymbol(struct symbol *sym, vcc_type_t type) { + vcc_kind_t kind; struct vsb *vsb; CHECK_OBJ_NOTNULL(sym, SYMBOL_MAGIC); AN(type); AN(type->global_pfx); + kind = VCC_HandleKind(type); + + if (sym->lname != NULL) { + AN(sym->rname); + assert(sym->type == type); + assert(sym->kind == kind); + return; + } + vsb = VSB_new_auto(); AN(vsb); VSB_printf(vsb, "%s_", type->global_pfx); @@ -536,7 +546,7 @@ VCC_GlobalSymbol(struct symbol *sym, vcc_type_t type) VSB_destroy(&vsb); sym->type = type; - sym->kind = VCC_HandleKind(sym->type); + sym->kind = kind; if (sym->kind != SYM_NONE) { AZ(VCT_invalid_name(sym->rname, NULL)); if (type == SUB) From nils.goroll at uplex.de Thu Jun 29 06:57:08 2023 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 29 Jun 2023 06:57:08 +0000 (UTC) Subject: [master] 05810d097 Spray vsmw_assert_lock() Message-ID: <20230629065708.800E110EF88@lists.varnish-cache.org> commit 05810d09714cdd3fb1e0cdc572189f0a95931a51 Author: Nils Goroll Date: Wed Jun 28 17:52:52 2023 +0200 Spray vsmw_assert_lock() Motivated by #3948 diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 85afe07c7..758ac82fa 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -155,6 +155,7 @@ static void vsmw_fmt_index(const struct vsmw *vsmw, const struct vsmwseg *seg, char act) { + vsmw_assert_lock(); CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC); AN(seg->cluster); @@ -198,6 +199,7 @@ vsmw_append_record(struct vsmw *vsmw, struct vsmwseg *seg, char act) { int fd; + vsmw_assert_lock(); CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC); fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND | O_WRONLY); @@ -271,6 +273,7 @@ vsmw_newcluster(struct vsmw *vsmw, size_t len, const char *pfx) int fd; size_t ps; + vsmw_assert_lock(); ALLOC_OBJ(vc, VSMW_CLUSTER_MAGIC); AN(vc); @@ -331,6 +334,7 @@ static void vsmw_DestroyCluster_locked(struct vsmw *vsmw, struct vsmw_cluster *vc) { + vsmw_assert_lock(); CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); CHECK_OBJ_NOTNULL(vc, VSMW_CLUSTER_MAGIC); From dridi.boukelmoune at gmail.com Thu Jun 29 08:58:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 29 Jun 2023 08:58:06 +0000 (UTC) Subject: [master] 1c4506bd9 vtc: Polish r3159 Message-ID: <20230629085807.0A38A1143B6@lists.varnish-cache.org> commit 1c4506bd970d2842314a4adae4e3e13f708dc308 Author: Dridi Boukelmoune Date: Thu Jun 29 10:47:00 2023 +0200 vtc: Polish r3159 For both processes, dump the screen before checking expectations. For p1, we make sure we have something to show before proceeding, and can abuse CLI serialization for that. Since p2 was already waited for, the screen dump was already guaranteed to succeed, on the other hand the superfluous p2 -wait can go away. diff --git a/bin/varnishtest/tests/r03159.vtc b/bin/varnishtest/tests/r03159.vtc index b95abf320..d165d4c2d 100644 --- a/bin/varnishtest/tests/r03159.vtc +++ b/bin/varnishtest/tests/r03159.vtc @@ -14,10 +14,14 @@ process p1 -log { -l 2m 2>&1 } -start -expect-exit 0x40 -process p1 -expect-text 0 1 "Unused sub foo, defined:" -process p1 -expect-text 0 1 "(That was just a warning)" +shell { + # wait for startup vcl.load to complete + varnishadm -n ${tmpdir}/t ping +} process p1 -screen_dump +process p1 -expect-text 0 1 "Unused sub foo, defined:" +process p1 -expect-text 0 1 "(That was just a warning)" process p2 -log { set -e @@ -26,12 +30,8 @@ process p2 -log { varnishadm -n ${tmpdir}/t "vcl.list" } -run -process p2 -wait -process p1 -kill TERM - process p2 -screen_dump - process p2 -expect-text 0 1 "Unused sub foo, defined:" process p2 -expect-text 0 1 "(That was just a warning)" -process p1 -wait +process p1 -kill TERM