From nils.goroll at uplex.de  Mon Jan  1 11:46:05 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  1 Jan 2024 11:46:05 +0000 (UTC)
Subject: [master] 083505193 Fix use of objiterate_f in vrb_pull()
Message-ID: <20240101114605.B019010824C@lists.varnish-cache.org>


commit 083505193a5065fab0359568816b5e733b69edc3
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Sun Dec 31 18:09:42 2023 +0100

    Fix use of objiterate_f in vrb_pull()
    
    The objiterate_f interface requires that OBJ_ITER_END be sent,
    preferrably with the last chunk of data, but at least after it.

diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c
index 70996a2f1..c3afeb853 100644
--- a/bin/varnishd/cache/cache_req_body.c
+++ b/bin/varnishd/cache/cache_req_body.c
@@ -59,6 +59,7 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv)
 	enum vfp_status vfps = VFP_ERROR;
 	const struct stevedore *stv;
 	ssize_t req_bodybytes = 0;
+	unsigned flush = OBJ_ITER_FLUSH;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
@@ -125,7 +126,9 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv)
 			else if (yet > 0)
 				yet = 0;
 			if (func != NULL) {
-				r = func(priv, 1, ptr, l);
+				if (vfps == VFP_END)
+					flush |= OBJ_ITER_END;
+				r = func(priv, flush, ptr, l);
 				if (r)
 					break;
 			} else {
@@ -140,6 +143,8 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv)
 	if (func != NULL) {
 		HSH_DerefBoc(req->wrk, req->body_oc);
 		AZ(HSH_DerefObjCore(req->wrk, &req->body_oc, 0));
+		if (vfps == VFP_END && (flush & OBJ_ITER_END) == 0)
+			func(priv, flush | OBJ_ITER_END, NULL, 0);
 		if (vfps != VFP_END) {
 			req->req_body_status = BS_ERROR;
 			if (r == 0)

From dridi.boukelmoune at gmail.com  Tue Jan  2 10:36:06 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  2 Jan 2024 10:36:06 +0000 (UTC)
Subject: [master] 48ee17dac circleci: Remove Fedora Rawhide job
Message-ID: <20240102103606.C8B1A10FDA1@lists.varnish-cache.org>


commit 48ee17dacd4caa6eced41837a404a76a26595bea
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jan 2 11:34:45 2024 +0100

    circleci: Remove Fedora Rawhide job
    
    Rawhide is not stable and we already have a job for the latest stable
    branch.

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 5551e8dd7..58cb84ca4 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -406,10 +406,6 @@ workflows:
           dist: fedora
           release: latest
           make_target: witness.dot
-      - build:
-          name: build_fedora_rawhide
-          dist: fedora
-          release: rawhide
       # latest debian uses sanitizers
       - build:
           name: build_debian_buster

From nils.goroll at uplex.de  Tue Jan  2 16:49:12 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jan 2024 16:49:12 +0000 (UTC)
Subject: [master] 7aeca4d63 Flexelint-polish
Message-ID: <20240102164912.62F4B11BC02@lists.varnish-cache.org>


commit 7aeca4d632827e2db91bf63e67e4c61c9319d038
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jan 2 17:42:39 2024 +0100

    Flexelint-polish
    
    of 083505193a5065fab0359568816b5e733b69edc3

diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c
index c3afeb853..51d45471e 100644
--- a/bin/varnishd/cache/cache_req_body.c
+++ b/bin/varnishd/cache/cache_req_body.c
@@ -143,8 +143,8 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv)
 	if (func != NULL) {
 		HSH_DerefBoc(req->wrk, req->body_oc);
 		AZ(HSH_DerefObjCore(req->wrk, &req->body_oc, 0));
-		if (vfps == VFP_END && (flush & OBJ_ITER_END) == 0)
-			func(priv, flush | OBJ_ITER_END, NULL, 0);
+		if (vfps == VFP_END && r == 0 && (flush & OBJ_ITER_END) == 0)
+			r = func(priv, flush | OBJ_ITER_END, NULL, 0);
 		if (vfps != VFP_END) {
 			req->req_body_status = BS_ERROR;
 			if (r == 0)

From dridi.boukelmoune at gmail.com  Mon Jan  8 13:57:10 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon,  8 Jan 2024 13:57:10 +0000 (UTC)
Subject: [master] e26533571 stevedore: Restore null check for properties
Message-ID: <20240108135710.3176F11A69A@lists.varnish-cache.org>


commit e265335716815c56757ecc606c152721e05973df
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Jan 8 14:51:59 2024 +0100

    stevedore: Restore null check for properties
    
    There is a default value when the property is not implemented and has no
    callback.
    
    Fixes #4036

diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 7813c424f..34f3919fe 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -306,6 +306,8 @@ VRT_stevedore_##nm(VCL_STEVEDORE stv)			\
 {							\
 	if (stv == NULL)				\
 		return (0);				\
+	if (stv->var_##nm == NULL)			\
+		return (dval);				\
 	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);	\
 	return (stv->var_##nm(stv));			\
 }
diff --git a/bin/varnishtest/tests/r04036.vtc b/bin/varnishtest/tests/r04036.vtc
new file mode 100644
index 000000000..a689680b9
--- /dev/null
+++ b/bin/varnishtest/tests/r04036.vtc
@@ -0,0 +1,15 @@
+varnishtest "Undefined storage properties"
+
+varnish v1 -arg "-s malloc=malloc -s file=file,${tmpdir}/file,1M" -vcl {
+	backend be none;
+
+	sub vcl_recv {
+		set req.http.happy = storage.malloc.happy;
+		set req.http.space = storage.file.free_space;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+} -run

From nils.goroll at uplex.de  Mon Jan  8 16:51:07 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  8 Jan 2024 16:51:07 +0000 (UTC)
Subject: [master] 7b7572128 stevedore type methods: Change .happy default to
 true
Message-ID: <20240108165107.396F011FD24@lists.varnish-cache.org>


commit 7b7572128fe9989d9570ceba3dd524e17297d9b5
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jan 8 17:46:35 2024 +0100

    stevedore type methods: Change .happy default to true
    
    and document the default of the other methods.
    
    Ref #4036

diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index 5d320537f..23c169019 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -1826,6 +1826,8 @@ sess.xid	``VCL >= 4.1``
 storage
 -------
 
+.. XXX all of these are actually defined in generate.py
+
 .. _storage.free_space:
 
 storage.<name>.free_space
@@ -1834,6 +1836,7 @@ storage.<name>.free_space
 
 	Readable from: client, backend
 
+	Default: 0
 
 	Free space available in the named stevedore. Only available for
 	the malloc stevedore.
@@ -1847,6 +1850,7 @@ storage.<name>.happy
 
 	Readable from: client, backend
 
+	Default: true
 
 	Health status for the named stevedore. Not available in any of the
 	current stevedores.
@@ -1860,6 +1864,7 @@ storage.<name>.used_space
 
 	Readable from: client, backend
 
+	Default: 0
 
 	Used space in the named stevedore. Only available for the malloc
 	stevedore.
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index fe5bcc9b4..0660f2283 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -295,7 +295,7 @@ stv_variables = (
     Used space in the named stevedore. Only available for the malloc
     stevedore.
     """),
-    ('happy', 'BOOL', "0", 'storage.<name>.happy', """
+    ('happy', 'BOOL', "1", 'storage.<name>.happy', """
     Health status for the named stevedore. Not available in any of the
     current stevedores.
     """),

From nils.goroll at uplex.de  Mon Jan  8 16:51:07 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  8 Jan 2024 16:51:07 +0000 (UTC)
Subject: [master] c595770f9 Polish: VCL_BYTES is an integer type
Message-ID: <20240108165107.4FA8D11FD26@lists.varnish-cache.org>


commit c595770f9de9551ad6e28bdbc7b381967c2eb412
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jan 8 17:47:40 2024 +0100

    Polish: VCL_BYTES is an integer type
    
    or rather, has been for some time now.

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 0660f2283..10fedac24 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -287,11 +287,11 @@ def parse_var_doc(fn):
         parse_var(l[n-2:m-1])
 
 stv_variables = (
-    ('free_space', 'BYTES', "0.", 'storage.<name>.free_space', """
+    ('free_space', 'BYTES', "0", 'storage.<name>.free_space', """
     Free space available in the named stevedore. Only available for
     the malloc stevedore.
     """),
-    ('used_space', 'BYTES', "0.", 'storage.<name>.used_space', """
+    ('used_space', 'BYTES', "0", 'storage.<name>.used_space', """
     Used space in the named stevedore. Only available for the malloc
     stevedore.
     """),

From phk at FreeBSD.org  Tue Jan  9 09:52:09 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jan 2024 09:52:09 +0000 (UTC)
Subject: [master] 66ab5b05f Attempt to fix r03996 on systems without massive
 resources It's amazing how long time it takes to find a delay of half a
 second...
Message-ID: <20240109095209.A9D3911B35F@lists.varnish-cache.org>


commit 66ab5b05fbd0f3917ba9996259c69f66bd9b2d41
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 9 09:50:23 2024 +0000

    Attempt to fix r03996 on systems without massive resources
    It's amazing how long time it takes to find a delay of half a second...

diff --git a/bin/varnishtest/tests/r03996.vtc b/bin/varnishtest/tests/r03996.vtc
index cb1936027..c38ead8eb 100644
--- a/bin/varnishtest/tests/r03996.vtc
+++ b/bin/varnishtest/tests/r03996.vtc
@@ -17,6 +17,7 @@ varnish v1 -vcl+backend {
 	import vtc;
 
 	sub vcl_recv {
+		vtc.sleep(0.5s);
 		if (req.http.barrier) {
 			vtc.barrier_sync(req.http.barrier);
 		}
@@ -25,6 +26,8 @@ varnish v1 -vcl+backend {
 
 } -start
 
+varnish v1 -vsl_catchup
+
 client c1 {
 	stream 0 {
 		rxgoaway
@@ -43,6 +46,8 @@ client c1 {
 	stream 0 -wait
 } -run
 
+varnish v1 -vsl_catchup
+
 varnish v1 -expect sc_rapid_reset == 1
 
 varnish v1 -cliok "param.set feature -vcl_req_reset"
@@ -64,4 +69,6 @@ client c2 {
 	stream 0 -wait
 } -run
 
+varnish v1 -vsl_catchup
+
 varnish v1 -expect sc_rapid_reset == 2

From phk at FreeBSD.org  Wed Jan 10 12:36:09 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Jan 2024 12:36:09 +0000 (UTC)
Subject: [master] 389d414a0 Emit a usable ARGV_ERR if a -sfile is too small
Message-ID: <20240110123609.8B7D868D3@lists.varnish-cache.org>


commit 389d414a041343a3e73c19b17ec3de28d665b535
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 10 12:34:58 2024 +0000

    Emit a usable ARGV_ERR if a -sfile is too small

diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
index b5148ac36..4a01ae6d2 100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@ -420,6 +420,13 @@ smf_open(struct stevedore *st)
 	Lck_Lock(&sc->mtx);
 	smf_open_chunk(sc, sc->filesize, 0, &fail, &sum);
 	Lck_Unlock(&sc->mtx);
+	if (sum < MINPAGES * (off_t)getpagesize()) {
+		ARGV_ERR(
+		    "-sfile too small for this architecture,"
+		    " minimum size is %jd MB\n",
+		    (MINPAGES * (intmax_t)getpagesize()) >> 20
+		);
+	}
 	printf("SMF.%s mmap'ed %ju bytes of %ju\n",
 	    st->ident, (uintmax_t)sum, sc->filesize);
 

From phk at FreeBSD.org  Wed Jan 10 12:36:09 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Jan 2024 12:36:09 +0000 (UTC)
Subject: [master] edc10d177 Make r4036 work on OS/X which as bigger pages
Message-ID: <20240110123609.A1D8A68D6@lists.varnish-cache.org>


commit edc10d177a5693f6999ef722b29e807906eb507b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 10 12:35:26 2024 +0000

    Make r4036 work on OS/X which as bigger pages

diff --git a/bin/varnishtest/tests/r04036.vtc b/bin/varnishtest/tests/r04036.vtc
index a689680b9..babaac855 100644
--- a/bin/varnishtest/tests/r04036.vtc
+++ b/bin/varnishtest/tests/r04036.vtc
@@ -1,6 +1,6 @@
 varnishtest "Undefined storage properties"
 
-varnish v1 -arg "-s malloc=malloc -s file=file,${tmpdir}/file,1M" -vcl {
+varnish v1 -arg "-s malloc=malloc -s file=file,${tmpdir}/file,2M" -vcl {
 	backend be none;
 
 	sub vcl_recv {

From phk at FreeBSD.org  Wed Jan 10 12:58:05 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Jan 2024 12:58:05 +0000 (UTC)
Subject: [master] 32d1fc7fa Flexelinting
Message-ID: <20240110125805.2B3657AB3@lists.varnish-cache.org>


commit 32d1fc7fa7f39dd26f77554e5f3f74fb644a5894
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 10 12:57:40 2024 +0000

    Flexelinting

diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
index 4a01ae6d2..299464e19 100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@ -424,7 +424,7 @@ smf_open(struct stevedore *st)
 		ARGV_ERR(
 		    "-sfile too small for this architecture,"
 		    " minimum size is %jd MB\n",
-		    (MINPAGES * (intmax_t)getpagesize()) >> 20
+		    (MINPAGES * (intmax_t)getpagesize()) / (1<<20)
 		);
 	}
 	printf("SMF.%s mmap'ed %ju bytes of %ju\n",

From phk at FreeBSD.org  Mon Jan 15 08:53:10 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jan 2024 08:53:10 +0000 (UTC)
Subject: [master] 93047d4fa Take mutexes for sake of consistency (Coverity)
Message-ID: <20240115085310.0DBBA11E693@lists.varnish-cache.org>


commit 93047d4fa0544edf181413d64dffcdffdb31125e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 15 08:52:09 2024 +0000

    Take mutexes for sake of consistency (Coverity)

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 5bbe4f858..d44dc4f27 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -1067,7 +1067,9 @@ cmd_process(CMD_ARGS)
 		p = process_new(av[0]);
 	av++;
 
+	PTOK(pthread_mutex_lock(&p->mtx));
 	bsnap = p->stdout_bytes;
+	PTOK(pthread_mutex_unlock(&p->mtx));
 
 	for (; *av != NULL; av++) {
 		if (vtc_error)
diff --git a/vmod/vmod_debug_dyn.c b/vmod/vmod_debug_dyn.c
index f557e075c..4714c436b 100644
--- a/vmod/vmod_debug_dyn.c
+++ b/vmod/vmod_debug_dyn.c
@@ -155,10 +155,14 @@ xyzzy_dyn__fini(struct xyzzy_debug_dyn **dynp)
 VCL_BACKEND v_matchproto_()
 xyzzy_dyn_backend(VRT_CTX, struct xyzzy_debug_dyn *dyn)
 {
+	VCL_BACKEND retval;
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(dyn, VMOD_DEBUG_DYN_MAGIC);
-	AN(dyn->dir);
-	return (dyn->dir);
+	PTOK(pthread_mutex_lock(&dyn->mtx));
+	retval = dyn->dir;
+	PTOK(pthread_mutex_unlock(&dyn->mtx));
+	AN(retval);
+	return (retval);
 }
 
 VCL_VOID

From phk at FreeBSD.org  Mon Jan 15 09:21:06 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jan 2024 09:21:06 +0000 (UTC)
Subject: [master] b26a62dd2 Replace the mgt/wrk cli pipe(2) with a socketpair(AF_UNIX, SOCK_STREAM,…)
Message-ID: <20240115092106.4716A11F5FD@lists.varnish-cache.org>


commit b26a62dd24ffcbde77d58757e3ef08aaf73f1bdd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 15 09:18:52 2024 +0000

    Replace the mgt/wrk cli pipe(2) with a socketpair(AF_UNIX, SOCK_STREAM,?)
    
    This should be a no-op on all contemporary UNIX implementations, but
    let's see what vtest tells us...

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index b4fd329bb..0ea50c16e 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -42,6 +42,7 @@
 #include <syslog.h>
 #include <unistd.h>
 #include <sys/types.h>
+#include <sys/socket.h>
 #include <sys/time.h>
 #include <sys/resource.h>
 
@@ -318,7 +319,7 @@ mgt_launch_child(struct cli *cli)
 	child_state = CH_STARTING;
 
 	/* Open pipe for mgt->child CLI */
-	AZ(pipe(cp));
+	AZ(socketpair(AF_UNIX, SOCK_STREAM, 0, cp));
 	heritage.cli_in = cp[0];
 	assert(cp[0] > STDERR_FILENO);	// See #2782
 	assert(cp[1] > STDERR_FILENO);

From phk at FreeBSD.org  Mon Jan 15 09:33:05 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jan 2024 09:33:05 +0000 (UTC)
Subject: [master] 9784b3984 Merge HAproxy behavioural change
Message-ID: <20240115093305.593F811FC76@lists.varnish-cache.org>


commit 9784b3984f117250417ee50405ac59d26637e043
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 15 09:32:02 2024 +0000

    Merge HAproxy behavioural change
    
    https://github.com/vtest/VTest/pull/32

diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c
index b5053d0e8..d2d9d5daa 100644
--- a/bin/varnishtest/vtc_haproxy.c
+++ b/bin/varnishtest/vtc_haproxy.c
@@ -55,7 +55,6 @@
 #define HAPROXY_OPT_DAEMON	"-D"
 #define HAPROXY_SIGNAL		SIGINT
 #define HAPROXY_EXPECT_EXIT	(128 + HAPROXY_SIGNAL)
-#define HAPROXY_GOOD_CONF	"Configuration file is valid"
 
 struct envar {
 	VTAILQ_ENTRY(envar) list;
@@ -1077,8 +1076,9 @@ cmd_haproxy(CMD_ARGS)
 		if (!strcmp(*av, "-conf-OK")) {
 			AN(av[1]);
 			haproxy_store_conf(h, av[1], 0);
+			h->expect_exit = 0;
+			haproxy_check_conf(h, "");
 			av++;
-			haproxy_check_conf(h, HAPROXY_GOOD_CONF);
 			continue;
 		}
 		if (!strcmp(*av, "-conf-BAD")) {

From phk at FreeBSD.org  Mon Jan 15 13:26:06 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jan 2024 13:26:06 +0000 (UTC)
Subject: [master] 3cd73087d Exploit that socketpair(2) is bidirectional
Message-ID: <20240115132606.1104B67C3@lists.varnish-cache.org>


commit 3cd73087d53d1e3912a82ee3e729a317606cb48a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 15 13:25:07 2024 +0000

    Exploit that socketpair(2) is bidirectional

diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c
index 2dd203597..0a99bc3cf 100644
--- a/bin/varnishd/cache/cache_cli.c
+++ b/bin/varnishd/cache/cache_cli.c
@@ -99,10 +99,10 @@ CLI_Run(void)
 	add_check = 1;
 
 	/* Tell waiting MGT that we are ready to speak CLI */
-	AZ(VCLI_WriteResult(heritage.cli_out, CLIS_OK, "Ready"));
+	AZ(VCLI_WriteResult(heritage.cli_fd, CLIS_OK, "Ready"));
 
 	cli = VCLS_AddFd(cache_cls,
-	    heritage.cli_in, heritage.cli_out, NULL, NULL);
+	    heritage.cli_fd, heritage.cli_fd, NULL, NULL);
 	AN(cli);
 	cli->auth = 255;	// Non-zero to disable paranoia in vcli_serve
 
diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h
index 5b68d96a3..3112c1214 100644
--- a/bin/varnishd/common/heritage.h
+++ b/bin/varnishd/common/heritage.h
@@ -60,8 +60,7 @@ VTAILQ_HEAD(listen_sock_head, listen_sock);
 struct heritage {
 
 	/* Two pipe(2)'s for CLI connection between cache and mgt.  */
-	int				cli_in;
-	int				cli_out;
+	int				cli_fd;
 
 	/* File descriptor for stdout/stderr */
 	int				std_fd;
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 0ea50c16e..697b05826 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -63,8 +63,7 @@ static pid_t		child_pid = -1;
 
 static struct vbitmap	*fd_map;
 
-static int		child_cli_in = -1;
-static int		child_cli_out = -1;
+static int		child_cli_fd = -1;
 static int		child_output = -1;
 
 static enum {
@@ -320,17 +319,11 @@ mgt_launch_child(struct cli *cli)
 
 	/* Open pipe for mgt->child CLI */
 	AZ(socketpair(AF_UNIX, SOCK_STREAM, 0, cp));
-	heritage.cli_in = cp[0];
+	heritage.cli_fd = cp[0];
 	assert(cp[0] > STDERR_FILENO);	// See #2782
 	assert(cp[1] > STDERR_FILENO);
-	MCH_Fd_Inherit(heritage.cli_in, "cli_in");
-	child_cli_out = cp[1];
-
-	/* Open pipe for child->mgt CLI */
-	AZ(pipe(cp));
-	heritage.cli_out = cp[1];
-	MCH_Fd_Inherit(heritage.cli_out, "cli_out");
-	child_cli_in = cp[0];
+	MCH_Fd_Inherit(heritage.cli_fd, "cli_fd");
+	child_cli_fd = cp[1];
 
 	/*
 	 * Open pipe for child stdout/err
@@ -427,11 +420,8 @@ mgt_launch_child(struct cli *cli)
 	/* Close stuff the child got */
 	closefd(&heritage.std_fd);
 
-	MCH_Fd_Inherit(heritage.cli_in, NULL);
-	closefd(&heritage.cli_in);
-
-	MCH_Fd_Inherit(heritage.cli_out, NULL);
-	closefd(&heritage.cli_out);
+	MCH_Fd_Inherit(heritage.cli_fd, NULL);
+	closefd(&heritage.cli_fd);
 
 	child_std_vlu = VLU_New(child_line, NULL, 0);
 	AN(child_std_vlu);
@@ -440,7 +430,7 @@ mgt_launch_child(struct cli *cli)
 	bstart = mgt_param.startup_timeout >= mgt_param.cli_timeout;
 	dstart = bstart ? mgt_param.startup_timeout : mgt_param.cli_timeout;
 	t0 = VTIM_mono();
-	if (VCLI_ReadResult(child_cli_in, &u, NULL, dstart)) {
+	if (VCLI_ReadResult(child_cli_fd, &u, NULL, dstart)) {
 		assert(u == CLIS_COMMS);
 		if (VTIM_mono() - t0 < dstart)
 			mgt_launch_err(cli, u, "Child failed on launch ");
@@ -480,7 +470,7 @@ mgt_launch_child(struct cli *cli)
 		ev_poker = e;
 	}
 
-	mgt_cli_start_child(child_cli_in, child_cli_out);
+	mgt_cli_start_child(child_cli_fd, child_cli_fd);
 	child_pid = pid;
 
 	if (mgt_push_vcls(cli, &u, &p)) {
@@ -535,10 +525,8 @@ mgt_reap_child(void)
 	 * This signals orderly shut down to child
 	 */
 	mgt_cli_stop_child();
-	if (child_cli_out >= 0)
-		closefd(&child_cli_out);
-	if (child_cli_in >= 0)
-		closefd(&child_cli_in);
+	if (child_cli_fd >= 0)
+		closefd(&child_cli_fd);
 
 	/* Stop the poker */
 	if (ev_poker != NULL) {

From nils.goroll at uplex.de  Mon Jan 15 14:30:07 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 15 Jan 2024 14:30:07 +0000 (UTC)
Subject: [master] c44bd67f2 Fix order of COLD events for directors vs. VCL
Message-ID: <20240115143007.11C34100DDE@lists.varnish-cache.org>


commit c44bd67f25216746311b3f65ba39552a8971934b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Jan 5 21:33:25 2024 +0100

    Fix order of COLD events for directors vs. VCL
    
    vcldir_retire() only sends a VCL_EVENT_COLD when the vcl is warm. In
    other words, it (rightly) asserts that a COLD event (if any) has
    alrady been posted when a director is deleted on a COLD vcl.
    
    Yet, when the director is deleted upon a COLD vmod event, this assertion
    was wrong, because the COLD events for directors were only posted after
    vmod events.
    
    Given that vmods do things like deleting directors, it appears (more)
    correct to post VDI COLD events before VMOD COLD events.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 29d4b459f..90e41ece8 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -594,9 +594,9 @@ vcl_set_state(struct vcl *vcl, const char *state, struct vsb **msg)
 		if (vcl->busy == 0 && vcl->temp->is_warm) {
 			vcl->temp = VTAILQ_EMPTY(&vcl->ref_list) ?
 			    VCL_TEMP_COLD : VCL_TEMP_COOLING;
+			vcl_BackendEvent(vcl, VCL_EVENT_COLD);
 			AZ(vcl_send_event(vcl, VCL_EVENT_COLD, msg));
 			AZ(*msg);
-			vcl_BackendEvent(vcl, VCL_EVENT_COLD);
 		}
 		else if (vcl->busy)
 			vcl->temp = VCL_TEMP_BUSY;
diff --git a/include/vrt.h b/include/vrt.h
index 2a478b372..f5ea03bc5 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -59,6 +59,7 @@
  *
  * NEXT (2024-03-15)
  *	[cache.h] (struct req).filter_list renamed to vdp_filter_list
+ *	order of vcl/vmod and director COLD events reversed to directors first
  * 18.1 (2023-12-05)
  *	vbf_objiterate() implementation changed #4013
  * 18.0 (2023-09-15)

From phk at FreeBSD.org  Mon Jan 15 19:38:06 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jan 2024 19:38:06 +0000 (UTC)
Subject: [master] 5f67d3aee Slight reshuffling of some lines of code.
Message-ID: <20240115193806.1BDE410A461@lists.varnish-cache.org>


commit 5f67d3aee3a756beb46bae610e1a75548e03c07f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 15 19:36:58 2024 +0000

    Slight reshuffling of some lines of code.

diff --git a/lib/libvarnish/vcli_serve.c b/lib/libvarnish/vcli_serve.c
index dbb384a0f..3f3c71dc9 100644
--- a/lib/libvarnish/vcli_serve.c
+++ b/lib/libvarnish/vcli_serve.c
@@ -212,14 +212,27 @@ VCLS_func_help_json(struct cli *cli, const char * const *av, void *priv)
  */
 
 static void
-cls_dispatch(struct cli *cli, const struct cli_proto *cp,
-    char * const * av, int ac)
+cls_dispatch(struct cli *cli, struct VCLS *cs, char * const * av, int ac)
 {
 	int json = 0;
+	struct cli_proto *cp;
 
 	AN(av);
 	assert(ac >= 0);
 
+	VTAILQ_FOREACH(cp, &cs->funcs, list) {
+		if (cp->auth > cli->auth)
+			continue;
+		if (!strcmp(cp->desc->request, av[1]))
+			break;
+	}
+
+	if (cp == NULL && cs->wildcard && cs->wildcard->auth <= cli->auth)
+		cp = cs->wildcard;
+
+	if (cp == NULL)
+		return;
+
 	VSB_clear(cli->sb);
 
 	if (ac > 1 && !strcmp(av[2], "-j"))
@@ -263,7 +276,6 @@ static int
 cls_exec(struct VCLS_fd *cfd, char * const *av)
 {
 	struct VCLS *cs;
-	struct cli_proto *clp;
 	struct cli *cli;
 	int na;
 	ssize_t len;
@@ -313,17 +325,7 @@ cls_exec(struct VCLS_fd *cfd, char * const *av)
 		for (na = 0; av[na + 1] != NULL; na++)
 			continue;
 
-		VTAILQ_FOREACH(clp, &cs->funcs, list) {
-			if (clp->auth > cli->auth)
-				continue;
-			if (!strcmp(clp->desc->request, av[1])) {
-				cls_dispatch(cli, clp, av, na);
-				break;
-			}
-		}
-		if (clp == NULL &&
-		    cs->wildcard && cs->wildcard->auth <= cli->auth)
-			cls_dispatch(cli, cs->wildcard, av, na);
+		cls_dispatch(cli, cs, av, na);
 
 	} while (0);
 

From dridi.boukelmoune at gmail.com  Thu Jan 18 21:08:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 18 Jan 2024 21:08:05 +0000 (UTC)
Subject: [master] 9b27d997f http1_fetch: Log missing closing parenthesis
Message-ID: <20240118210805.E3D6A917C1@lists.varnish-cache.org>


commit 9b27d997f1775feb334f15330d4e44f9d2bde611
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Jan 18 22:06:34 2024 +0100

    http1_fetch: Log missing closing parenthesis

diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 7d0e6c015..05fe46418 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -154,7 +154,8 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes,
 
 	CHECK_OBJ_NOTNULL(sc, STREAM_CLOSE_MAGIC);
 	if (sc != SC_NULL) {
-		VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s) (%s",
+		VSLb(bo->vsl, SLT_FetchError,
+		    "backend write error: %d (%s) (%s)",
 		    errno, VAS_errtxt(errno), sc->desc);
 		VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
 		htc->doclose = sc;

From dridi.boukelmoune at gmail.com  Mon Jan 22 10:14:11 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 22 Jan 2024 10:14:11 +0000 (UTC)
Subject: [master] 3092670ea cache_shmlog: Missing assertion
Message-ID: <20240122101411.641A1AFEBC@lists.varnish-cache.org>


commit 3092670ea20e4baf07ea65210eca91c642dfecd2
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Jan 22 11:10:34 2024 +0100

    cache_shmlog: Missing assertion

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 971c65ae6..ee671afd9 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -110,6 +110,7 @@ vsl_sanity(const struct vsl_log *vsl)
 	AN(vsl->wlp);
 	AN(vsl->wlb);
 	AN(vsl->wle);
+	assert(vsl->wlb <= vsl->wlp);
 	assert(vsl->wlp <= vsl->wle);
 }
 

From nils.goroll at uplex.de  Mon Jan 22 18:23:07 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 22 Jan 2024 18:23:07 +0000 (UTC)
Subject: [master] 4d17b66b5 varnishstat: Fix missing description of the last
 VSC
Message-ID: <20240122182307.4E85893003@lists.varnish-cache.org>


commit 4d17b66b53877d95a6fb2153dc22f38c3187baf8
Author: Walid Boudebouda <walid.boudebouda at gmail.com>
Date:   Tue Jan 16 11:32:04 2024 +0100

    varnishstat: Fix missing description of the last VSC

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index c9d7117b3..6938fbdb3 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -847,7 +847,7 @@ draw_bar_b(void)
 	if (page_start + l_points < n_ptarray)
 		mvwprintw(w_bar_b, 0, x, "vvv");
 	x += 4;
-	if (current < n_ptarray - 1)
+	if (current < n_ptarray)
 		mvwprintw(w_bar_b, 0, x, "%s", ptarray[current]->vpt->name);
 
 	bprintf(buf, "%d-%d/%d", page_start + 1,
@@ -880,7 +880,7 @@ draw_info(void)
 		return;
 
 	werase(w_info);
-	if (current < n_ptarray - 1) {
+	if (current < n_ptarray) {
 		/* XXX: Word wrapping, and overflow handling? */
 		mvwprintw(w_info, 0, 0, "%s:",
 		    ptarray[current]->vpt->sdesc);

From phk at FreeBSD.org  Wed Jan 24 08:35:08 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2024 08:35:08 +0000 (UTC)
Subject: [master] 81c3498fb Polish VCLS code.
Message-ID: <20240124083508.0A92694749@lists.varnish-cache.org>


commit 81c3498fbe10d1e6015dde2bee379db18a87df0d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 16 08:49:28 2024 +0000

    Polish VCLS code.

diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c
index b94601507..376050b44 100644
--- a/lib/libvarnish/vav.c
+++ b/lib/libvarnish/vav.c
@@ -249,7 +249,10 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag)
 			argv[0] = err_missing_quote;
 			return (argv);
 		}
-		/* Ensure slots for 1 new arg plus 1 trailing arg */
+		/*
+		 * Ensure slots for 1 new arg plus 1 trailing arg,
+		 * vcli_serve.c depends on this extra slot.
+		 */
 		if (nargv + 2 >= largv) {
 			argv = realloc(argv, sizeof (*argv) * (largv += largv));
 			assert(argv != NULL);
diff --git a/lib/libvarnish/vcli_serve.c b/lib/libvarnish/vcli_serve.c
index 3f3c71dc9..4e8ff8ef6 100644
--- a/lib/libvarnish/vcli_serve.c
+++ b/lib/libvarnish/vcli_serve.c
@@ -219,6 +219,8 @@ cls_dispatch(struct cli *cli, struct VCLS *cs, char * const * av, int ac)
 
 	AN(av);
 	assert(ac >= 0);
+	AZ(av[0]);
+	AN(av[1]);
 
 	VTAILQ_FOREACH(cp, &cs->funcs, list) {
 		if (cp->auth > cli->auth)
@@ -230,8 +232,10 @@ cls_dispatch(struct cli *cli, struct VCLS *cs, char * const * av, int ac)
 	if (cp == NULL && cs->wildcard && cs->wildcard->auth <= cli->auth)
 		cp = cs->wildcard;
 
-	if (cp == NULL)
+	if (cp == NULL) {
+		VCLI_Out(cli, "Unknown request.\nType 'help' for more info.\n");
 		return;
+	}
 
 	VSB_clear(cli->sb);
 
@@ -262,10 +266,12 @@ cls_dispatch(struct cli *cli, struct VCLS *cs, char * const * av, int ac)
 	}
 
 	cli->result = CLIS_OK;
+	cli->cls = cs;
 	if (json)
 		cp->jsonfunc(cli, (const char * const *)av, cp->priv);
 	else
 		cp->func(cli, (const char * const *)av, cp->priv);
+	cli->cls = NULL;
 }
 
 /*--------------------------------------------------------------------
@@ -273,11 +279,10 @@ cls_dispatch(struct cli *cli, struct VCLS *cs, char * const * av, int ac)
  */
 
 static int
-cls_exec(struct VCLS_fd *cfd, char * const *av)
+cls_exec(struct VCLS_fd *cfd, char * const *av, int ac)
 {
 	struct VCLS *cs;
 	struct cli *cli;
-	int na;
 	ssize_t len;
 	char *s;
 	unsigned lim;
@@ -291,11 +296,8 @@ cls_exec(struct VCLS_fd *cfd, char * const *av)
 	CHECK_OBJ_NOTNULL(cli, CLI_MAGIC);
 	AN(cli->cmd);
 
-	cli->cls = cs;
-
 	cli->result = CLIS_UNKNOWN;
 	VSB_clear(cli->sb);
-	VCLI_Out(cli, "Unknown request.\nType 'help' for more info.\n");
 
 	if (cs->before != NULL)
 		cs->before(cli);
@@ -313,19 +315,13 @@ cls_exec(struct VCLS_fd *cfd, char * const *av)
 			break;
 		}
 
-		if (isupper(av[1][0])) {
-			VCLI_Out(cli, "all commands are in lower-case.\n");
+		if (!islower(av[1][0])) {
+			VCLI_Out(cli, "All commands are in lower-case.\n");
 			VCLI_SetResult(cli, CLIS_UNKNOWN);
 			break;
 		}
 
-		if (!islower(av[1][0]))
-			break;
-
-		for (na = 0; av[na + 1] != NULL; na++)
-			continue;
-
-		cls_dispatch(cli, cs, av, na);
+		cls_dispatch(cli, cs, av, ac);
 
 	} while (0);
 
@@ -334,8 +330,6 @@ cls_exec(struct VCLS_fd *cfd, char * const *av)
 	if (cs->after != NULL)
 		cs->after(cli);
 
-	cli->cls = NULL;
-
 	s = VSB_data(cli->sb);
 	len = VSB_len(cli->sb);
 	lim = *cs->limit;
@@ -415,7 +409,7 @@ cls_feed(struct VCLS_fd *cfd, const char *p, const char *e)
 				AN(cfd->last_arg);
 			} else {
 				/* Plain command */
-				i = cls_exec(cfd, av);
+				i = cls_exec(cfd, av, ac - 1);
 				VAV_Free(av);
 				VSB_destroy(&cli->cmd);
 				if (i)
@@ -435,7 +429,7 @@ cls_feed(struct VCLS_fd *cfd, const char *p, const char *e)
 				REPLACE(cfd->argv[cfd->argc - 2], NULL);
 				cfd->argv[cfd->argc - 2] =
 				    VSB_data(cfd->last_arg);
-				i = cls_exec(cfd, cfd->argv);
+				i = cls_exec(cfd, cfd->argv, cfd->argc - 2);
 				cfd->argv[cfd->argc - 2] = NULL;
 				VAV_Free(cfd->argv);
 				cfd->argv = NULL;

From phk at FreeBSD.org  Wed Jan 24 08:35:08 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2024 08:35:08 +0000 (UTC)
Subject: [master] 796a645b1 Move the initial cli-pipe handshake to mgt_cli.c
Message-ID: <20240124083508.2CD1C9474C@lists.varnish-cache.org>


commit 796a645b1fd4ecff58205ed7409463aacb28c29d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 08:33:53 2024 +0000

    Move the initial cli-pipe handshake to mgt_cli.c

diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 3d133db34..5b22eb85c 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -89,7 +89,7 @@ void mgt_cli_setup(int fdi, int fdo, int auth, const char *ident,
     mgt_cli_close_f *close_func, void *priv);
 int mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...)
     v_printflike_(3, 4);
-void mgt_cli_start_child(int fdi, int fdo);
+unsigned mgt_cli_start_child(int fd, double tmo);
 void mgt_cli_stop_child(void);
 void mgt_cli_telnet(const char *T_arg);
 void mgt_cli_master(const char *M_arg);
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 697b05826..4ab09253c 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -430,7 +430,8 @@ mgt_launch_child(struct cli *cli)
 	bstart = mgt_param.startup_timeout >= mgt_param.cli_timeout;
 	dstart = bstart ? mgt_param.startup_timeout : mgt_param.cli_timeout;
 	t0 = VTIM_mono();
-	if (VCLI_ReadResult(child_cli_fd, &u, NULL, dstart)) {
+	u = mgt_cli_start_child(child_cli_fd, dstart);
+	if (u != CLIS_OK) {
 		assert(u == CLIS_COMMS);
 		if (VTIM_mono() - t0 < dstart)
 			mgt_launch_err(cli, u, "Child failed on launch ");
@@ -470,7 +471,6 @@ mgt_launch_child(struct cli *cli)
 		ev_poker = e;
 	}
 
-	mgt_cli_start_child(child_cli_fd, child_cli_fd);
 	child_pid = pid;
 
 	if (mgt_push_vcls(cli, &u, &p)) {
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index c3cada3d9..6693af8c3 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -213,12 +213,17 @@ mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...)
 
 /*--------------------------------------------------------------------*/
 
-void
-mgt_cli_start_child(int fdi, int fdo)
+unsigned
+mgt_cli_start_child(int fd, double tmo)
 {
+	unsigned u;
 
-	cli_i = fdi;
-	cli_o = fdo;
+	cli_i = fd;
+	cli_o = fd;
+	if (VCLI_ReadResult(cli_i, &u, NULL, tmo)) {
+		return (CLIS_COMMS);
+	}
+	return (u);
 }
 
 /*--------------------------------------------------------------------*/

From dridi.boukelmoune at gmail.com  Fri Jan 26 16:55:08 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 26 Jan 2024 16:55:08 +0000 (UTC)
Subject: [master] ea640d472 expire: Add hits to EXP_Expired log statement
Message-ID: <20240126165508.7C33663B7E@lists.varnish-cache.org>


commit ea640d4720451f9747dcf55b9454454847fa1f02
Author: AlveElde <alve_elde at hotmail.com>
Date:   Tue Jan 3 02:59:02 2023 +0100

    expire: Add hits to EXP_Expired log statement
    
    This makes it easier to estimate the popularity of expired objects.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 0392d191d..c179f94f8 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -354,8 +354,9 @@ exp_expire(struct exp_priv *ep, vtim_real now)
 		assert(oc->timer_idx == VBH_NOIDX);
 
 		CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
-		VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired xid=%ju t=%.0f",
-		    VXID(ObjGetXID(ep->wrk, oc)), EXP_Ttl(NULL, oc) - now);
+		VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired x=%ju t=%.0f h=%ld",
+		    VXID(ObjGetXID(ep->wrk, oc)), EXP_Ttl(NULL, oc) - now,
+		    oc->hits);
 		ObjSendEvent(ep->wrk, oc, OEV_EXPIRE);
 		(void)HSH_DerefObjCore(ep->wrk, &oc, 0);
 	}
diff --git a/bin/varnishtest/tests/r02339.vtc b/bin/varnishtest/tests/r02339.vtc
index 60848137d..a19c22ce7 100644
--- a/bin/varnishtest/tests/r02339.vtc
+++ b/bin/varnishtest/tests/r02339.vtc
@@ -21,7 +21,7 @@ varnish v1 -vcl+backend {
 varnish v1 -cliok "param.set timeout_idle 2"
 
 logexpect l0 -v v1 -g raw {
-	expect * 0 ExpKill	"EXP_Expired xid=1002"
+	expect * 0 ExpKill	"EXP_Expired x=1002"
 } -start
 
 logexpect l2 -v v1 -g raw {
@@ -109,4 +109,4 @@ varnish v1 -errvcl "Not available in subroutine 'vcl_backend_response'" {
 	sub vcl_backend_response {
 		if (bereq.url == "response") { purge.hard(); }
 	}
-}
\ No newline at end of file
+}
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index ebcb3ac36..c65257dcc 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -367,7 +367,7 @@ SLTM(ExpKill, 0, "Object expiry event",
 	"\tEXP_Inbox p=%p e=%f f=0x%x\n"
 	"\tEXP_Kill p=%p e=%f f=0x%x\n"
 	"\tEXP_When p=%p e=%f f=0x%x\n"
-	"\tEXP_Expired x=%u t=%f\n"
+	"\tEXP_Expired x=%u t=%f h=%u\n"
 	"\tLRU_Cand p=%p f=0x%x r=%d\n"
 	"\tLRU x=%u\n"
 	"\tLRU_Fail\n"
@@ -380,6 +380,7 @@ SLTM(ExpKill, 0, "Object expiry event",
 	"\tf=0x%x       Objcore flags\n"
 	"\tr=%d         Objcore refcount\n"
 	"\tx=%u         Object VXID\n"
+	"\th=%u         Objcore hits\n"
 	"\n"
 )
 
diff --git a/vmod/tests/purge_c00000.vtc b/vmod/tests/purge_c00000.vtc
index 8cf199a12..267f4b882 100644
--- a/vmod/tests/purge_c00000.vtc
+++ b/vmod/tests/purge_c00000.vtc
@@ -68,7 +68,7 @@ logexpect l2 -v v1 -q "Begin ~ bgfetch" {
 } -start
 
 logexpect l3 -v v1 -g raw -q "vxid == 0" {
-	expect * * ExpKill xid=1008
+	expect * * ExpKill x=1008
 } -start
 
 client c1 {

From dridi.boukelmoune at gmail.com  Fri Jan 26 16:55:08 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 26 Jan 2024 16:55:08 +0000 (UTC)
Subject: [master] 5963b6dc0 expire: New EXP_Removed log statement
Message-ID: <20240126165508.9B86863B80@lists.varnish-cache.org>


commit 5963b6dc0e146d03bdfc7ed0d4cc39fb1053f122
Author: AlveElde <alve_elde at hotmail.com>
Date:   Tue Jan 3 03:04:08 2023 +0100

    expire: New EXP_Removed log statement
    
    It is generally useful to know when an object leaves the cache and while
    it may sometimes be logged by call sites (like bans) it is more helpful
    centralize it in ExpKill records to get a (more) complete view of what
    goes in and out of (durable) storage.
    
    The hits attribute is especially useful, as it can give an indication of
    the popularity of objects being removed before they expire.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index c179f94f8..240dc99f0 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -261,7 +261,7 @@ EXP_Rearm(struct objcore *oc, vtim_real now,
  */
 
 static void
-exp_inbox(struct exp_priv *ep, struct objcore *oc, unsigned flags)
+exp_inbox(struct exp_priv *ep, struct objcore *oc, unsigned flags, double now)
 {
 
 	CHECK_OBJ_NOTNULL(ep, EXP_PRIV_MAGIC);
@@ -279,6 +279,9 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, unsigned flags)
 		assert(oc->timer_idx == VBH_NOIDX);
 		assert(oc->refcnt > 0);
 		AZ(oc->exp_flags);
+		VSLb(&ep->vsl, SLT_ExpKill, "EXP_Removed x=%ju t=%.0f h=%ld",
+		    VXID(ObjGetXID(ep->wrk, oc)), EXP_Ttl(NULL, oc) - now,
+		    oc->hits);
 		ObjSendEvent(ep->wrk, oc, OEV_EXPIRE);
 		(void)HSH_DerefObjCore(ep->wrk, &oc, 0);
 		return;
@@ -427,7 +430,7 @@ exp_thread(struct worker *wrk, void *priv)
 		t = VTIM_real();
 
 		if (oc != NULL)
-			exp_inbox(ep, oc, flags);
+			exp_inbox(ep, oc, flags, t);
 		else
 			tnext = exp_expire(ep, t);
 	}
diff --git a/bin/varnishtest/tests/c00126.vtc b/bin/varnishtest/tests/c00126.vtc
new file mode 100644
index 000000000..7403f7585
--- /dev/null
+++ b/bin/varnishtest/tests/c00126.vtc
@@ -0,0 +1,42 @@
+varnishtest "Make sure EXP_Removed is logged correctly"
+
+server s1 -repeat 4 {
+	rxreq
+	txresp -bodylen 500000
+} -start
+
+varnish v1 -arg "-ss1=default,1m" -vcl+backend { } -start
+varnish v1 -cliok "param.set vsl_mask +ExpKill"
+
+logexpect l1 -v v1 -g raw {
+	expect * 0 ExpKill "EXP_Removed x=1002 t=.* h=1"
+	expect * 0 ExpKill "EXP_Removed x=1005 t=.* h=0"
+} -start
+
+client c1 {
+	loop 2 {
+		txreq -url "/1"
+		rxresp
+		expect resp.status == 200
+	}
+
+	txreq -url "/2"
+	rxresp
+	expect resp.status == 200
+
+	txreq -url "/3"
+	rxresp
+	expect resp.status == 200
+
+	txreq -url "/4"
+	rxresp
+	expect resp.status == 200
+} -run
+
+# NOTE: Nuked objects are mailed twice
+varnish v1 -expect n_lru_nuked == 2
+varnish v1 -expect MAIN.n_object == 2
+varnish v1 -expect MAIN.exp_mailed == 6
+varnish v1 -expect MAIN.exp_received == 6
+
+logexpect l1 -wait
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index c65257dcc..84632b861 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -355,6 +355,8 @@ SLTM(ExpKill, 0, "Object expiry event",
 	"\tLogged when the expiry thread moves an object on the binheap.\n\n"
 	"EXP_Expired\n"
 	"\tLogged when the expiry thread expires an object.\n\n"
+	"EXP_Removed\n"
+	"\tLogged when the expiry thread removes an object before expiry.\n\n"
 	"LRU_Cand\n"
 	"\tLogged when an object is evaluated for LRU force expiry.\n\n"
 	"LRU\n"
@@ -368,6 +370,7 @@ SLTM(ExpKill, 0, "Object expiry event",
 	"\tEXP_Kill p=%p e=%f f=0x%x\n"
 	"\tEXP_When p=%p e=%f f=0x%x\n"
 	"\tEXP_Expired x=%u t=%f h=%u\n"
+	"\tEXP_Removed x=%u t=%f h=%u\n"
 	"\tLRU_Cand p=%p f=0x%x r=%d\n"
 	"\tLRU x=%u\n"
 	"\tLRU_Fail\n"

From dridi.boukelmoune at gmail.com  Fri Jan 26 16:55:08 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 26 Jan 2024 16:55:08 +0000 (UTC)
Subject: [master] e3303ad3b expire: Rename ExpKill's EXP_expire to EXP_Inspect
Message-ID: <20240126165508.C2B7863B83@lists.varnish-cache.org>


commit e3303ad3ba1c4c790147cdf18c6eee02acd352b1
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jan 10 13:27:40 2024 +0100

    expire: Rename ExpKill's EXP_expire to EXP_Inspect
    
    And while at it document it.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 240dc99f0..69fe24e07 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -328,7 +328,7 @@ exp_expire(struct exp_priv *ep, vtim_real now)
 	oc = VBH_root(ep->heap);
 	if (oc == NULL)
 		return (now + 355. / 113.);
-	VSLb(&ep->vsl, SLT_ExpKill, "EXP_expire p=%p e=%.6f f=0x%x", oc,
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_Inspect p=%p e=%.6f f=0x%x", oc,
 	    oc->timer_when - now, oc->flags);
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
diff --git a/bin/varnishtest/tests/b00063.vtc b/bin/varnishtest/tests/b00063.vtc
index 6879968e8..d88b38cd0 100644
--- a/bin/varnishtest/tests/b00063.vtc
+++ b/bin/varnishtest/tests/b00063.vtc
@@ -47,9 +47,9 @@ varnish v1 -vcl+backend {
 } -start
 
 logexpect l1 -v v1 -g raw {
-	expect * *	ExpKill		EXP_expire
+	expect * *	ExpKill		EXP_Inspect
 	expect * *	ExpKill		EXP_When
-	expect * *	ExpKill		EXP_expire
+	expect * *	ExpKill		EXP_Inspect
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/c00105.vtc b/bin/varnishtest/tests/c00105.vtc
index 1c6005900..a5c198d5e 100644
--- a/bin/varnishtest/tests/c00105.vtc
+++ b/bin/varnishtest/tests/c00105.vtc
@@ -35,7 +35,7 @@ varnish v1 -vcl+backend {
 	}
 } -start
 
-logexpect l1 -v v1 -g raw -q "ExpKill ~ EXP_expire" {
+logexpect l1 -v v1 -g raw -q "ExpKill ~ EXP_Inspect" {
 	expect 0 0 ExpKill
 } -start
 
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 84632b861..8fd68ac89 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -353,6 +353,9 @@ SLTM(ExpKill, 0, "Object expiry event",
 	"\tLogged when the expiry thread kills an object from the inbox.\n\n"
 	"EXP_When\n"
 	"\tLogged when the expiry thread moves an object on the binheap.\n\n"
+	"EXP_Inspect\n"
+	"\tLogged when the expiry thread inspects the next object scheduled"
+	" to expire.\n\n"
 	"EXP_Expired\n"
 	"\tLogged when the expiry thread expires an object.\n\n"
 	"EXP_Removed\n"
@@ -369,6 +372,7 @@ SLTM(ExpKill, 0, "Object expiry event",
 	"\tEXP_Inbox p=%p e=%f f=0x%x\n"
 	"\tEXP_Kill p=%p e=%f f=0x%x\n"
 	"\tEXP_When p=%p e=%f f=0x%x\n"
+	"\tEXP_Inspect p=%p e=%f f=0x%x\n"
 	"\tEXP_Expired x=%u t=%f h=%u\n"
 	"\tEXP_Removed x=%u t=%f h=%u\n"
 	"\tLRU_Cand p=%p f=0x%x r=%d\n"

From dridi.boukelmoune at gmail.com  Fri Jan 26 16:55:08 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 26 Jan 2024 16:55:08 +0000 (UTC)
Subject: [master] 709f71ab7 hash: A purged object didn't get a chance to expire
Message-ID: <20240126165508.E9FB963B88@lists.varnish-cache.org>


commit 709f71ab731298fa4d30d4c8b2b61dae04002ea6
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Jan 11 17:40:52 2024 +0100

    hash: A purged object didn't get a chance to expire
    
    So the n_expired, n_purges and n_obj_purged counters should reflect
    that. Likewise, it shouldn't be logged as "EXP_Expired".

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 4b3799888..6d982ddec 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -674,10 +674,12 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, vtim_real ttl_now,
 {
 	struct objcore *oc, *oc_nows[2], **ocp;
 	unsigned i, j, n, n_max, total = 0;
+	int is_purge;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
+	is_purge = (ttl == 0 && grace == 0 && keep == 0);
 	n_max = WS_ReserveLumps(wrk->aws, sizeof *ocp);
 	if (n_max < 2) {
 		/* No space on the workspace. Give it a stack buffer of 2
@@ -740,7 +742,10 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, vtim_real ttl_now,
 		}
 		for (i = 0; i < j; i++) {
 			CHECK_OBJ_NOTNULL(ocp[i], OBJCORE_MAGIC);
-			EXP_Rearm(ocp[i], ttl_now, ttl, grace, keep);
+			if (is_purge)
+				EXP_Remove(ocp[i]);
+			else
+				EXP_Rearm(ocp[i], ttl_now, ttl, grace, keep);
 			(void)HSH_DerefObjCore(wrk, &ocp[i], 0);
 			AZ(ocp[i]);
 			total++;
@@ -765,7 +770,8 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, vtim_real ttl_now,
 	}
 
 	WS_Release(wrk->aws, 0);
-	Pool_PurgeStat(total);
+	if (is_purge)
+		Pool_PurgeStat(total);
 	return (total);
 }
 
diff --git a/bin/varnishtest/tests/b00036.vtc b/bin/varnishtest/tests/b00036.vtc
index 97902ecb2..fc1ad0519 100644
--- a/bin/varnishtest/tests/b00036.vtc
+++ b/bin/varnishtest/tests/b00036.vtc
@@ -33,3 +33,11 @@ client c1 {
 	expect resp.http.foo == 2
 
 } -run
+
+varnish v1 -vsl_catchup
+
+varnish v1 -expect MAIN.n_purges == 1
+varnish v1 -expect MAIN.n_obj_purged == 1
+
+# NB: a purge used to increase n_expired
+varnish v1 -expect MAIN.n_expired == 0
diff --git a/bin/varnishtest/tests/r02339.vtc b/bin/varnishtest/tests/r02339.vtc
index a19c22ce7..9ff18632d 100644
--- a/bin/varnishtest/tests/r02339.vtc
+++ b/bin/varnishtest/tests/r02339.vtc
@@ -21,7 +21,7 @@ varnish v1 -vcl+backend {
 varnish v1 -cliok "param.set timeout_idle 2"
 
 logexpect l0 -v v1 -g raw {
-	expect * 0 ExpKill	"EXP_Expired x=1002"
+	expect * 0 ExpKill	"EXP_Removed x=1002"
 } -start
 
 logexpect l2 -v v1 -g raw {

From phk at FreeBSD.org  Fri Jan 26 18:26:05 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 26 Jan 2024 18:26:05 +0000 (UTC)
Subject: [master] 58c714eeb White-space OCD
Message-ID: <20240126182605.334B293384@lists.varnish-cache.org>


commit 58c714eeb0c2fadf34f5e197c1dee4b80ad57fa0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 08:50:47 2024 +0000

    White-space OCD

diff --git a/include/vas.h b/include/vas.h
index c7ba06577..2e6a9ee2a 100644
--- a/include/vas.h
+++ b/include/vas.h
@@ -91,7 +91,7 @@ do {									\
 } while (0)
 
 #define _PTOK(call, var)						\
-do { 									\
+do {									\
 	int var = call;							\
 	if (!var)							\
 		break;							\

From phk at FreeBSD.org  Fri Jan 26 18:26:05 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 26 Jan 2024 18:26:05 +0000 (UTC)
Subject: [master] 7ba00a3f9 Fix printf formats
Message-ID: <20240126182605.4A1C093388@lists.varnish-cache.org>


commit 7ba00a3f9a7b5452ea126d86fdb0dde0eea41810
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 26 18:25:39 2024 +0000

    Fix printf formats

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 69fe24e07..60aafb098 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -279,9 +279,9 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, unsigned flags, double now)
 		assert(oc->timer_idx == VBH_NOIDX);
 		assert(oc->refcnt > 0);
 		AZ(oc->exp_flags);
-		VSLb(&ep->vsl, SLT_ExpKill, "EXP_Removed x=%ju t=%.0f h=%ld",
+		VSLb(&ep->vsl, SLT_ExpKill, "EXP_Removed x=%ju t=%.0f h=%jd",
 		    VXID(ObjGetXID(ep->wrk, oc)), EXP_Ttl(NULL, oc) - now,
-		    oc->hits);
+		    (intmax_t)oc->hits);
 		ObjSendEvent(ep->wrk, oc, OEV_EXPIRE);
 		(void)HSH_DerefObjCore(ep->wrk, &oc, 0);
 		return;
@@ -357,9 +357,9 @@ exp_expire(struct exp_priv *ep, vtim_real now)
 		assert(oc->timer_idx == VBH_NOIDX);
 
 		CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
-		VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired x=%ju t=%.0f h=%ld",
+		VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired x=%ju t=%.0f h=%jd",
 		    VXID(ObjGetXID(ep->wrk, oc)), EXP_Ttl(NULL, oc) - now,
-		    oc->hits);
+		    (intmax_t)oc->hits);
 		ObjSendEvent(ep->wrk, oc, OEV_EXPIRE);
 		(void)HSH_DerefObjCore(ep->wrk, &oc, 0);
 	}

From nils.goroll at uplex.de  Mon Jan 29 15:46:05 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 29 Jan 2024 15:46:05 +0000 (UTC)
Subject: [master] 10f1e5cf1 Add SessClose reasons to documentation
Message-ID: <20240129154605.C528E65754@lists.varnish-cache.org>


commit 10f1e5cf1dd1042a1f72f49259900888a92bb91d
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jan 29 16:44:54 2024 +0100

    Add SessClose reasons to documentation
    
    Motivated by #4042

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 8fd68ac89..24fca6f9d 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -91,6 +91,16 @@ SLTM(SessOpen, 0, "Client connection opened",
 	"\t+------------------- Remote IPv4/6 address / 0.0.0.0 for UDS\n"
 	"\n"
 )
+/*
+ * XXX generate the list of SC_* reasons (see also HTC info):
+ *
+ * #include <stdio.h>
+ * int main(void) {
+ * #define SESS_CLOSE(e, sc, err, desc)			\
+ *	printf("\t\"\\t* ``%s``: %s\\n\"\n", #e, desc);
+ * #include "include/tbl/sess_close.h"
+ *	return (0);
+ */
 
 SLTM(SessClose, 0, "Client connection closed",
 	"SessClose is the last record for any client connection.\n\n"
@@ -99,6 +109,26 @@ SLTM(SessClose, 0, "Client connection closed",
 	"\t|  |\n"
 	"\t|  +- How long the session was open\n"
 	"\t+---- Why the connection closed\n"
+	"\nExplanation of reasons (first column):\n"
+	"\t* ``REM_CLOSE``: Client Closed\n"
+	"\t* ``REQ_CLOSE``: Client requested close\n"
+	"\t* ``REQ_HTTP10``: Proto < HTTP/1.1\n"
+	"\t* ``RX_BAD``: Received bad req/resp\n"
+	"\t* ``RX_BODY``: Failure receiving body\n"
+	"\t* ``RX_JUNK``: Received junk data\n"
+	"\t* ``RX_OVERFLOW``: Received buffer overflow\n"
+	"\t* ``RX_TIMEOUT``: Receive timeout\n"
+	"\t* ``RX_CLOSE_IDLE``: timeout_idle reached\n"
+	"\t* ``TX_PIPE``: Piped transaction\n"
+	"\t* ``TX_ERROR``: Error transaction\n"
+	"\t* ``TX_EOF``: EOF transmission\n"
+	"\t* ``RESP_CLOSE``: Backend/VCL requested close\n"
+	"\t* ``OVERLOAD``: Out of some resource\n"
+	"\t* ``PIPE_OVERFLOW``: Session pipe overflow\n"
+	"\t* ``RANGE_SHORT``: Insufficient data for range\n"
+	"\t* ``REQ_HTTP20``: HTTP2 not accepted\n"
+	"\t* ``VCL_FAILURE``: VCL failure\n"
+	"\t* ``RAPID_RESET``: HTTP2 rapid reset\n"
 	"\n"
 )
 

From nils.goroll at uplex.de  Mon Jan 29 15:59:05 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 29 Jan 2024 15:59:05 +0000 (UTC)
Subject: [master] 2511f5364 Polish HTC error documentation
Message-ID: <20240129155905.236D16CED3@lists.varnish-cache.org>


commit 2511f5364489c8345b3879a5eee21179bf71e9b2
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jan 29 16:48:22 2024 +0100

    Polish HTC error documentation

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 24fca6f9d..c94af34f2 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -194,7 +194,7 @@ SLTM(Length, 0, "Size of object body",
  * #include <stdio.h>
  * int main(void) {
  * #define HTC_STATUS(e, n, s, l) \
- *	printf("\t\"\\t* %s (%d): %s\\n\"\n", s, n, l);
+ *	printf("\t\"\\t* ``%s`` (%d): %s\\n\"\n", s, n, l);
  * #include "include/tbl/htc.h"
  *	return (0);
  * }
@@ -204,15 +204,15 @@ SLTM(FetchError, 0, "Error while fetching object",
 	"Logs the error message of a failed fetch operation.\n\n"
 	"Error messages should be self-explanatory, yet the http connection\n"
 	"(HTC) class of errors is reported with these symbols:\n\n"
-	"\t* junk (-5): Received unexpected data\n"
-	"\t* close (-4): Connection closed\n"
-	"\t* timeout (-3): Timed out\n"
-	"\t* overflow (-2): Buffer/workspace too small\n"
-	"\t* eof (-1): Unexpected end of input\n"
-	"\t* empty (0): Empty response\n"
-	"\t* more (1): More data required\n"
-	"\t* complete (2): Data complete (no error)\n"
-	"\t* idle (3): Connection was closed while idle\n"
+	"\t* ``junk`` (-5): Received unexpected data\n"
+	"\t* ``close`` (-4): Connection closed\n"
+	"\t* ``timeout`` (-3): Timed out\n"
+	"\t* ``overflow`` (-2): Buffer/workspace too small\n"
+	"\t* ``eof`` (-1): Unexpected end of input\n"
+	"\t* ``empty`` (0): Empty response\n"
+	"\t* ``more`` (1): More data required\n"
+	"\t* ``complete`` (2): Data complete (no error)\n"
+	"\t* ``idle`` (3): Connection was closed while idle\n"
 	"\nNotice that some HTC errors are never emitted."
 	)
 

From nils.goroll at uplex.de  Wed Jan 31 10:06:07 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 31 Jan 2024 10:06:07 +0000 (UTC)
Subject: [master] e0b164f9f Polish
Message-ID: <20240131100607.95E27962F3@lists.varnish-cache.org>


commit e0b164f9f134c0e594df8cb543036a441836da25
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jan 31 11:05:12 2024 +0100

    Polish

diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c
index fb7043963..a454b14d9 100644
--- a/bin/varnishd/cache/cache_vrt_vcl.c
+++ b/bin/varnishd/cache/cache_vrt_vcl.c
@@ -154,6 +154,17 @@ vcldir_free(struct vcldir *vdir)
 	FREE_OBJ(vdir);
 }
 
+static VCL_BACKEND
+vcldir_surplus(struct vcldir *vdir)
+{
+
+	CHECK_OBJ_NOTNULL(vdir, VCLDIR_MAGIC);
+	assert(vdir->refcnt == 1);
+	vdir->refcnt = 0;
+	vcldir_free(vdir);
+	return (NULL);
+}
+
 VCL_BACKEND
 VRT_AddDirector(VRT_CTX, const struct vdi_methods *m, void *priv,
     const char *fmt, ...)
@@ -217,12 +228,9 @@ VRT_AddDirector(VRT_CTX, const struct vdi_methods *m, void *priv,
 		VDI_Event(vdir->dir, VCL_EVENT_WARM);
 	Lck_Unlock(&vcl_mtx);
 
-	if (temp == VCL_TEMP_COOLING) {
-		assert(vdir->refcnt == 1);
-		vdir->refcnt = 0;
-		vcldir_free(vdir);
-		return (NULL);
-	}
+	if (temp == VCL_TEMP_COOLING)
+		return (vcldir_surplus(vdir));
+
 	if (!temp->is_warm && temp != VCL_TEMP_INIT)
 		WRONG("Dynamic Backends can only be added to warm VCLs");
 

From nils.goroll at uplex.de  Wed Jan 31 10:06:07 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 31 Jan 2024 10:06:07 +0000 (UTC)
Subject: [master] c096659dd Fix surplus vcldir cleanup
Message-ID: <20240131100607.85BE5962F1@lists.varnish-cache.org>


commit c096659dd7b7932ead816d0389c9b24911e25dae
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jan 31 09:28:13 2024 +0100

    Fix surplus vcldir cleanup
    
    in VRT_AddDirector, we create the new vcldir with an initial
    reference, which we need to drop if we can not add it.
    
    Compare:
    
            VRT_AddDirector()
                    ...
                    vdir->refcnt++;
    
            vcldir_free()
                    ...
                    AZ(vdir->refcnt);
    
    Noticed when testing other experimental changes while working on
    https://github.com/nigoroll/libvmod-dynamic/issues/110
    
     #5  0x000055820c8cb845 in VAS_Fail (func=0x55820c904559 "vcldir_free", file=0x55820c903a47 "cache/cache_vrt_vcl.c",
         line=150, cond=0x55820c90459a "(vdir->refcnt) == 0", kind=VAS_ASSERT) at vas.c:67
     #6  0x000055820c83a442 in vcldir_free (vdir=0x7f662aa53140) at cache/cache_vrt_vcl.c:150
     #7  0x000055820c839fe1 in VRT_AddDirector (ctx=0x7f662befe250, m=0x55820c965260 <vbe_methods_noprobe>,
         priv=0x7f662aa20780, fmt=0x55820c900f7f "%s") at cache/cache_vrt_vcl.c:219
     #8  0x000055820c7c7c4d in VRT_new_backend_clustered (ctx=0x7f662befe250, vc=0x0, vrt=0x7f662befdd10, via=0x0)
        at cache/cache_backend.c:737
     #9  0x000055820c7c8632 in VRT_new_backend (ctx=0x7f662befe250, vrt=0x7f662befdd10, via=0x0)
         at cache/cache_backend.c:755

diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c
index 85b594b7d..fb7043963 100644
--- a/bin/varnishd/cache/cache_vrt_vcl.c
+++ b/bin/varnishd/cache/cache_vrt_vcl.c
@@ -218,6 +218,8 @@ VRT_AddDirector(VRT_CTX, const struct vdi_methods *m, void *priv,
 	Lck_Unlock(&vcl_mtx);
 
 	if (temp == VCL_TEMP_COOLING) {
+		assert(vdir->refcnt == 1);
+		vdir->refcnt = 0;
 		vcldir_free(vdir);
 		return (NULL);
 	}

From dridi.boukelmoune at gmail.com  Wed Jan 31 16:37:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Jan 2024 16:37:05 +0000 (UTC)
Subject: [master] 660fedef8 v1p: No timeout when timeout_pipe is zero
Message-ID: <20240131163705.6DDCEA99B8@lists.varnish-cache.org>


commit 660fedef850c8f2ff7d372ef8c037412092c57c3
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Jan 19 16:27:58 2024 +0100

    v1p: No timeout when timeout_pipe is zero

diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index d273ae4f4..0c4a69a03 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -117,6 +117,7 @@ void
 V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
 {
 	struct pollfd fds[2];
+	vtim_dur tmo;
 	int i, j;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -142,8 +143,10 @@ V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
 	while (fds[0].fd > -1 || fds[1].fd > -1) {
 		fds[0].revents = 0;
 		fds[1].revents = 0;
-		i = poll(fds, 2,
-		    (int)(cache_param->pipe_timeout * 1e3));
+		tmo = cache_param->pipe_timeout;
+		if (tmo == 0.)
+			tmo = -1.;
+		i = poll(fds, 2, (int)(tmo * 1e3));
 		if (i < 1)
 			break;
 		if (fds[0].revents &&
diff --git a/bin/varnishtest/tests/s00013.vtc b/bin/varnishtest/tests/s00013.vtc
new file mode 100644
index 000000000..ac7168bb1
--- /dev/null
+++ b/bin/varnishtest/tests/s00013.vtc
@@ -0,0 +1,22 @@
+varnishtest "pipe timeouts"
+
+server s1 {
+	rxreq
+	txresp -hdr "transfer-encoding: chunked"
+	delay 1.1
+} -start
+
+varnish v1 -cliok "param.set pipe_timeout 0s"
+varnish v1 -vcl+backend "" -start
+
+logexpect l1 -v v1 -q "Timestamp:PipeSess[2] > 1.0" {
+	expect 1001 * ReqMethod PIPE
+} -start
+
+client c1 {
+	non_fatal
+	txreq -method PIPE
+	rxresp
+} -run
+
+logexpect l1 -wait

From dridi.boukelmoune at gmail.com  Wed Jan 31 16:37:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Jan 2024 16:37:05 +0000 (UTC)
Subject: [master] 7eb20abe6 v1p: Refine stream_close_t outcome
Message-ID: <20240131163705.7BF37A99BA@lists.varnish-cache.org>


commit 7eb20abe671046fc253f4bf5731dd94f3b8ba86f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Jan 19 16:37:49 2024 +0100

    v1p: Refine stream_close_t outcome

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 2a9e31edc..6b378aede 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -391,11 +391,10 @@ vbe_dir_http1pipe(VRT_CTX, VCL_BACKEND d)
 		    &v1a.bereq, &v1a.out);
 		VSLb_ts_req(ctx->req, "Pipe", W_TIM_real(ctx->req->wrk));
 		if (i == 0)
-			V1P_Process(ctx->req, *PFD_Fd(pfd), &v1a);
+			retval = V1P_Process(ctx->req, *PFD_Fd(pfd), &v1a);
 		VSLb_ts_req(ctx->req, "PipeSess", W_TIM_real(ctx->req->wrk));
-		ctx->bo->htc->doclose = SC_TX_PIPE;
+		ctx->bo->htc->doclose = retval;
 		vbe_dir_finish(ctx, d);
-		retval = SC_TX_PIPE;
 	}
 	V1P_Charge(ctx->req, &v1a, bp->vsc);
 	CHECK_OBJ_NOTNULL(retval, STREAM_CLOSE_MAGIC);
diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h
index dc454bcab..b5ab02446 100644
--- a/bin/varnishd/http1/cache_http1.h
+++ b/bin/varnishd/http1/cache_http1.h
@@ -54,7 +54,7 @@ struct v1p_acct {
 
 int V1P_Enter(void);
 void V1P_Leave(void);
-void V1P_Process(const struct req *, int fd, struct v1p_acct *);
+stream_close_t V1P_Process(const struct req *, int fd, struct v1p_acct *);
 void V1P_Charge(struct req *, const struct v1p_acct *, struct VSC_vbe *);
 
 /* cache_http1_line.c */
diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index 0c4a69a03..c55f68803 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -113,11 +113,12 @@ V1P_Charge(struct req *req, const struct v1p_acct *a, struct VSC_vbe *b)
 	Lck_Unlock(&pipestat_mtx);
 }
 
-void
+stream_close_t
 V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
 {
 	struct pollfd fds[2];
 	vtim_dur tmo;
+	stream_close_t sc;
 	int i, j;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -129,7 +130,7 @@ V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
 		    req->htc->pipeline_e - req->htc->pipeline_b);
 		VTCP_Assert(j);
 		if (j < 0)
-			return;
+			return (SC_OVERLOAD);
 		req->htc->pipeline_b = NULL;
 		req->htc->pipeline_e = NULL;
 		v1a->in += j;
@@ -140,6 +141,7 @@ V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
 	fds[1].fd = req->sp->fd;
 	fds[1].events = POLLIN;
 
+	sc = SC_TX_PIPE;
 	while (fds[0].fd > -1 || fds[1].fd > -1) {
 		fds[0].revents = 0;
 		fds[1].revents = 0;
@@ -147,6 +149,8 @@ V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
 		if (tmo == 0.)
 			tmo = -1.;
 		i = poll(fds, 2, (int)(tmo * 1e3));
+		if (i == 0)
+			sc = SC_RX_TIMEOUT;
 		if (i < 1)
 			break;
 		if (fds[0].revents &&
@@ -168,6 +172,8 @@ V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
 			fds[1].fd = -1;
 		}
 	}
+
+	return (sc);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishtest/tests/s00013.vtc b/bin/varnishtest/tests/s00013.vtc
index ac7168bb1..06e89871b 100644
--- a/bin/varnishtest/tests/s00013.vtc
+++ b/bin/varnishtest/tests/s00013.vtc
@@ -4,6 +4,12 @@ server s1 {
 	rxreq
 	txresp -hdr "transfer-encoding: chunked"
 	delay 1.1
+	close
+
+	accept
+	rxreq
+	txresp -hdr "transfer-encoding: chunked"
+	expect_close
 } -start
 
 varnish v1 -cliok "param.set pipe_timeout 0s"
@@ -20,3 +26,10 @@ client c1 {
 } -run
 
 logexpect l1 -wait
+
+varnish v1 -cliok "param.set pipe_timeout 500ms"
+client c1 -run
+
+varnish v1 -expect MAIN.s_pipe == 2
+varnish v1 -expect MAIN.sc_tx_pipe == 1
+varnish v1 -expect MAIN.sc_rx_timeout == 1

From dridi.boukelmoune at gmail.com  Wed Jan 31 16:37:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Jan 2024 16:37:05 +0000 (UTC)
Subject: [master] 32215660c param: New pipe_task_deadline parameter
Message-ID: <20240131163705.9889BA99BD@lists.varnish-cache.org>


commit 32215660c1c4851232fc4127c3862bbee7848365
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Jan 19 16:38:43 2024 +0100

    param: New pipe_task_deadline parameter

diff --git a/include/tbl/params.h b/include/tbl/params.h
index c88809da7..d8f485c71 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -766,6 +766,18 @@ PARAM_SIMPLE(
 	"Maximum number of sessions dedicated to pipe transactions."
 )
 
+PARAM_SIMPLE(
+	/* name */	pipe_task_deadline,
+	/* type */	timeout,
+	/* min */	"0.000",
+	/* max */	NULL,
+	/* def */	"0s",
+	/* units */	"seconds",
+	/* descr */
+	"Deadline for PIPE sessions. Regardless of activity in either "
+	"direction after this many seconds, the session is closed."
+)
+
 PARAM_SIMPLE(
 	/* name */	pipe_timeout,
 	/* type */	timeout,

From dridi.boukelmoune at gmail.com  Wed Jan 31 16:37:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Jan 2024 16:37:05 +0000 (UTC)
Subject: [master] bb6441474 v1p: Honor pipe_task_deadline parameter
Message-ID: <20240131163705.B38CBA99C4@lists.varnish-cache.org>


commit bb6441474971c975bee50e1aa919b2387a934562
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Jan 19 18:04:32 2024 +0100

    v1p: Honor pipe_task_deadline parameter

diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index c55f68803..d0b513c7c 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -40,6 +40,7 @@
 
 #include "cache_http1.h"
 #include "vtcp.h"
+#include "vtim.h"
 
 #include "VSC_vbe.h"
 
@@ -117,7 +118,8 @@ stream_close_t
 V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
 {
 	struct pollfd fds[2];
-	vtim_dur tmo;
+	vtim_dur tmo, tmo_task;
+	vtim_real deadline;
 	stream_close_t sc;
 	int i, j;
 
@@ -141,6 +143,10 @@ V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
 	fds[1].fd = req->sp->fd;
 	fds[1].events = POLLIN;
 
+	deadline = cache_param->pipe_task_deadline;
+	if (deadline > 0.)
+		deadline += req->sp->t_idle;
+
 	sc = SC_TX_PIPE;
 	while (fds[0].fd > -1 || fds[1].fd > -1) {
 		fds[0].revents = 0;
@@ -148,6 +154,11 @@ V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
 		tmo = cache_param->pipe_timeout;
 		if (tmo == 0.)
 			tmo = -1.;
+		if (deadline > 0.) {
+			tmo_task = deadline - VTIM_real();
+			tmo = (tmo > 0.) ? vmin(tmo, tmo_task) : tmo_task;
+			tmo = vmax(tmo, 0.);
+		}
 		i = poll(fds, 2, (int)(tmo * 1e3));
 		if (i == 0)
 			sc = SC_RX_TIMEOUT;
diff --git a/bin/varnishtest/tests/s00013.vtc b/bin/varnishtest/tests/s00013.vtc
index 06e89871b..e27838df5 100644
--- a/bin/varnishtest/tests/s00013.vtc
+++ b/bin/varnishtest/tests/s00013.vtc
@@ -10,13 +10,31 @@ server s1 {
 	rxreq
 	txresp -hdr "transfer-encoding: chunked"
 	expect_close
+
+	accept
+	rxreq
+	txresp -hdr "transfer-encoding: chunked"
+	expect_close
+
+	accept
+	non_fatal
+	rxreq
+	txresp -hdr "transfer-encoding: chunked"
+	loop 20 {
+		chunkedlen 1
+		delay 0.1
+	}
 } -start
 
 varnish v1 -cliok "param.set pipe_timeout 0s"
+varnish v1 -cliok "param.set pipe_task_deadline 0s"
 varnish v1 -vcl+backend "" -start
 
-logexpect l1 -v v1 -q "Timestamp:PipeSess[2] > 1.0" {
-	expect 1001 * ReqMethod PIPE
+logexpect l1 -v v1 -g raw -q SessClose {
+	expect 1000 * SessClose {^TX_PIPE 1\.}
+	expect 1003 * SessClose {^RX_TIMEOUT 0\.}
+	expect 1006 * SessClose {^RX_TIMEOUT 1\.}
+	expect 1009 * SessClose {^RX_TIMEOUT 1\.}
 } -start
 
 client c1 {
@@ -25,11 +43,20 @@ client c1 {
 	rxresp
 } -run
 
-logexpect l1 -wait
+varnish v1 -cliok "param.set pipe_timeout 500ms"
+varnish v1 -cliok "param.set pipe_task_deadline 0s"
+client c1 -run
+
+varnish v1 -cliok "param.set pipe_timeout 0s"
+varnish v1 -cliok "param.set pipe_task_deadline 1.1s"
+client c1 -run
 
 varnish v1 -cliok "param.set pipe_timeout 500ms"
+varnish v1 -cliok "param.set pipe_task_deadline 1.1s"
 client c1 -run
 
-varnish v1 -expect MAIN.s_pipe == 2
+logexpect l1 -wait
+
+varnish v1 -expect MAIN.s_pipe == 4
 varnish v1 -expect MAIN.sc_tx_pipe == 1
-varnish v1 -expect MAIN.sc_rx_timeout == 1
+varnish v1 -expect MAIN.sc_rx_timeout == 3

From dridi.boukelmoune at gmail.com  Wed Jan 31 16:37:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Jan 2024 16:37:05 +0000 (UTC)
Subject: [master] 490553b00 doc: Succinctly describe parameters units
Message-ID: <20240131163705.CEDC9A99C8@lists.varnish-cache.org>


commit 490553b006251ca2c421d150eed3f50bbe8b9cfc
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Jan 25 16:00:00 2024 +0100

    doc: Succinctly describe parameters units
    
    This takes care of documenting the meaning of a zero timeout value in
    a central location, since none of the timeout parameters have a mention
    for the effect it produces.
    
    This is written as if there was a separate varnish-param(7) manual,
    which should eventually be enacted. This manual extraction was initially
    implemented in the adjacent #3817 timeout nomenclature draft.

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 16aa63260..002de7d73 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -540,6 +540,59 @@ to load the configurations at startup.
 RUN TIME PARAMETERS
 ===================
 
+Runtime parameters can either be set during startup with the ``-p`` command
+line option for ``varnishd(1)`` or through the CLI using the ``param.set`` or
+``param.reset`` commands. They can be locked during startup with the ``-r``
+command line option.
+
+Run Time Parameter Units
+------------------------
+
+There are different types of parameters that may accept a list of specific
+values, or optionally take a unit suffix.
+
+bool
+~~~~
+
+A boolean parameter accepts the values ``on`` and ``off``.
+
+It will also recognize the following values:
+
+- ``yes`` and ``no``
+- ``true`` and ``false``
+- ``enable`` and ``disable``
+
+bytes
+~~~~~
+
+A bytes parameter requires one of the following units suffixes:
+
+- ``b`` (bytes)
+- ``k`` (kibibytes, 1024 bytes)
+- ``m`` (mebibytes, 1024 kibibytes)
+- ``g`` (gibibytes, 1024 mebibytes)
+- ``t`` (tebibytes, 1024 gibibytes)
+- ``p`` (pebibytes, 1024 tebibytes)
+
+Multiplicator units may be appended with an extra ``b``. For example ``32k``
+is equivalent to ``32kb``. Bytes units are case-insensitive.
+
+seconds
+~~~~~~~
+
+A duration parameter may accept the following units suffixes:
+
+- ``ms`` (milliseconds)
+- ``s`` (seconds)
+- ``m`` (minutes)
+- ``h`` (hours)
+- ``d`` (days)
+- ``w`` (weeks)
+- ``y`` (years)
+
+If the parameter is a timeout or a deadline, a value of zero (when allowed)
+disables the effect of the parameter.
+
 Run Time Parameter Flags
 ------------------------
 

From dridi.boukelmoune at gmail.com  Wed Jan 31 16:37:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Jan 2024 16:37:05 +0000 (UTC)
Subject: [master] 481707676 vrt_var: Expose bereq.task_deadline in vcl_pipe
Message-ID: <20240131163705.F0CE9A99CC@lists.varnish-cache.org>


commit 481707676c63ab9743fd5c796bff347bace327a2
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Jan 25 16:54:20 2024 +0100

    vrt_var: Expose bereq.task_deadline in vcl_pipe
    
    The [be]req.task_deadline should eventually be generalized to all
    subroutines with read-write access to [be]req. When that happens,
    bereq.task_deadline will inherit req.task_deadline during the pipe
    transition.
    
    Related tasks should otherwise have independent deadlines, except
    sub-request tasks like ESI where there is a hierarchical dependency.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a7fcac6bf..7d6b68457 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -419,6 +419,7 @@ struct busyobj {
 	vtim_dur		connect_timeout;
 	vtim_dur		first_byte_timeout;
 	vtim_dur		between_bytes_timeout;
+	vtim_dur		task_deadline;
 
 	/* Timers */
 	vtim_real		t_first;	/* First timestamp logged */
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 6b378aede..c24ae96a2 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -366,6 +366,7 @@ vbe_dir_http1pipe(VRT_CTX, VCL_BACKEND d)
 	struct backend *bp;
 	struct v1p_acct v1a;
 	struct pfd *pfd;
+	vtim_real deadline;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
@@ -390,8 +391,15 @@ vbe_dir_http1pipe(VRT_CTX, VCL_BACKEND d)
 		i = V1F_SendReq(ctx->req->wrk, ctx->bo,
 		    &v1a.bereq, &v1a.out);
 		VSLb_ts_req(ctx->req, "Pipe", W_TIM_real(ctx->req->wrk));
-		if (i == 0)
-			retval = V1P_Process(ctx->req, *PFD_Fd(pfd), &v1a);
+		if (i == 0) {
+			deadline = ctx->bo->task_deadline;
+			if (isnan(deadline))
+				deadline = cache_param->pipe_task_deadline;
+			if (deadline > 0.)
+				deadline += ctx->req->sp->t_idle;
+			retval = V1P_Process(ctx->req, *PFD_Fd(pfd), &v1a,
+			    deadline);
+		}
 		VSLb_ts_req(ctx->req, "PipeSess", W_TIM_real(ctx->req->wrk));
 		ctx->bo->htc->doclose = retval;
 		vbe_dir_finish(ctx, d);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c7469f5eb..3f92d91aa 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -795,6 +795,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 	}
 
 	bo->wrk = wrk;
+	bo->task_deadline = NAN; /* XXX: copy req->task_deadline */
 	if (WS_Overflowed(req->ws))
 		wrk->vpi->handling = VCL_RET_FAIL;
 	else
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index c1282c149..b69005cd0 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -409,6 +409,8 @@ VRT_r_bereq_##which(VRT_CTX)					\
 BEREQ_TIMEOUT(connect_timeout)
 BEREQ_TIMEOUT(first_byte_timeout)
 BEREQ_TIMEOUT(between_bytes_timeout)
+BEREQ_TIMEOUT(task_deadline)
+
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h
index b5ab02446..f758a74c9 100644
--- a/bin/varnishd/http1/cache_http1.h
+++ b/bin/varnishd/http1/cache_http1.h
@@ -54,7 +54,8 @@ struct v1p_acct {
 
 int V1P_Enter(void);
 void V1P_Leave(void);
-stream_close_t V1P_Process(const struct req *, int fd, struct v1p_acct *);
+stream_close_t V1P_Process(const struct req *, int fd, struct v1p_acct *,
+    vtim_real deadline);
 void V1P_Charge(struct req *, const struct v1p_acct *, struct VSC_vbe *);
 
 /* cache_http1_line.c */
diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index d0b513c7c..7ffe4b68c 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -115,11 +115,11 @@ V1P_Charge(struct req *req, const struct v1p_acct *a, struct VSC_vbe *b)
 }
 
 stream_close_t
-V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
+V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a,
+    vtim_real deadline)
 {
 	struct pollfd fds[2];
 	vtim_dur tmo, tmo_task;
-	vtim_real deadline;
 	stream_close_t sc;
 	int i, j;
 
@@ -143,10 +143,6 @@ V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
 	fds[1].fd = req->sp->fd;
 	fds[1].events = POLLIN;
 
-	deadline = cache_param->pipe_task_deadline;
-	if (deadline > 0.)
-		deadline += req->sp->t_idle;
-
 	sc = SC_TX_PIPE;
 	while (fds[0].fd > -1 || fds[1].fd > -1) {
 		fds[0].revents = 0;
diff --git a/bin/varnishtest/tests/s00013.vtc b/bin/varnishtest/tests/s00013.vtc
index e27838df5..2898ec87f 100644
--- a/bin/varnishtest/tests/s00013.vtc
+++ b/bin/varnishtest/tests/s00013.vtc
@@ -6,15 +6,12 @@ server s1 {
 	delay 1.1
 	close
 
-	accept
-	rxreq
-	txresp -hdr "transfer-encoding: chunked"
-	expect_close
-
-	accept
-	rxreq
-	txresp -hdr "transfer-encoding: chunked"
-	expect_close
+	loop 3 {
+		accept
+		rxreq
+		txresp -hdr "transfer-encoding: chunked"
+		expect_close
+	}
 
 	accept
 	non_fatal
@@ -28,13 +25,20 @@ server s1 {
 
 varnish v1 -cliok "param.set pipe_timeout 0s"
 varnish v1 -cliok "param.set pipe_task_deadline 0s"
-varnish v1 -vcl+backend "" -start
+varnish v1 -vcl+backend {
+	sub vcl_pipe {
+		if (req.method == "TMO") {
+			set bereq.task_deadline = 1.1s;
+		}
+	}
+} -start
 
 logexpect l1 -v v1 -g raw -q SessClose {
 	expect 1000 * SessClose {^TX_PIPE 1\.}
 	expect 1003 * SessClose {^RX_TIMEOUT 0\.}
 	expect 1006 * SessClose {^RX_TIMEOUT 1\.}
 	expect 1009 * SessClose {^RX_TIMEOUT 1\.}
+	expect 1012 * SessClose {^RX_TIMEOUT 1\.}
 } -start
 
 client c1 {
@@ -51,12 +55,21 @@ varnish v1 -cliok "param.set pipe_timeout 0s"
 varnish v1 -cliok "param.set pipe_task_deadline 1.1s"
 client c1 -run
 
+varnish v1 -cliok "param.set pipe_timeout 0s"
+varnish v1 -cliok "param.set pipe_task_deadline 0s"
+
+client c2 {
+	non_fatal
+	txreq -method TMO
+	rxresp
+} -run
+
 varnish v1 -cliok "param.set pipe_timeout 500ms"
 varnish v1 -cliok "param.set pipe_task_deadline 1.1s"
 client c1 -run
 
 logexpect l1 -wait
 
-varnish v1 -expect MAIN.s_pipe == 4
+varnish v1 -expect MAIN.s_pipe == 5
 varnish v1 -expect MAIN.sc_tx_pipe == 1
-varnish v1 -expect MAIN.sc_rx_timeout == 3
+varnish v1 -expect MAIN.sc_rx_timeout == 4
diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index 23c169019..510d13950 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -840,6 +840,20 @@ bereq.retries
 	A count of how many times this request has been retried.
 
 
+.. _bereq.task_deadline:
+
+bereq.task_deadline
+
+	Type: DURATION
+
+	Readable from: vcl_pipe
+
+	Writable from: vcl_pipe
+
+	Deadline for pipe sessions, defaults ``0s``, which falls back to the
+        ``pipe_task_deadline`` parameter, see :ref:`varnishd(1)`
+
+
 .. _bereq.time:
 
 bereq.time

From dridi.boukelmoune at gmail.com  Wed Jan 31 21:44:07 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Jan 2024 21:44:07 +0000 (UTC)
Subject: [master] 0959f3559 backend: Assume TX_ERROR by default for pipe
Message-ID: <20240131214407.D6A93BEA1C@lists.varnish-cache.org>


commit 0959f3559647fbb950d3b97d8f0dd8cb10101ca5
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jan 31 22:39:54 2024 +0100

    backend: Assume TX_ERROR by default for pipe

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index c24ae96a2..61be0aa27 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -382,11 +382,10 @@ vbe_dir_http1pipe(VRT_CTX, VCL_BACKEND d)
 
 	ctx->req->res_mode = RES_PIPE;
 
+	retval = SC_TX_ERROR;
 	pfd = vbe_dir_getfd(ctx, ctx->req->wrk, d, bp, 0);
 
-	if (pfd == NULL) {
-		retval = SC_TX_ERROR;
-	} else {
+	if (pfd != NULL) {
 		CHECK_OBJ_NOTNULL(ctx->bo->htc, HTTP_CONN_MAGIC);
 		i = V1F_SendReq(ctx->req->wrk, ctx->bo,
 		    &v1a.bereq, &v1a.out);