Child panics on OpenSolaris
Paul Wright
wrighty+varnishmisc at gmail.com
Wed Feb 10 12:05:30 CET 2010
Hello list,
Using the Letsgetdugg[1] article I've installed Varnish on an
OpenSolaris zone. During testing it works as expected but when it
receives production traffic I'm seeing children die with three
different types of panics[2][3][4] that look like this:
Panic message: Assert error in TCP_nonblocking(), tcp.c line 172:
Panic message: Assert error in TCP_blocking(), tcp.c line 163:
Assert error in VCA_Prep(), cache_acceptor.c line 163:
I've tried both enabling and disabling KeepAlive on the backend server
but doesn't seem to have any effect. I've also tried a 2GB and 1GB
malloc cache just in case it was a 32bit issue (it's not and I've
since confirmed it's running as a 64bit process).
The VCL I'm using is pretty simple[5], it normalises the host header
and unsets the cookie header if the request is for a static asset.
This is how I'm starting up Varnish at the moment:
newtask -p highfile /opt/sbin/varnishd -f /opt/etc/varnish/firebox.vcl -F \
-p cc_command='/opt/SunStudioExpress/bin/cc -Kpic -G -m64 -o %o %s' \
-T 127.0.0.1:9001 \
-s malloc,1G \
-p sess_timeout=5s \
-p max_restarts=12 \
-p waiter=poll \
-p connect_timeout=0s \
-p sess_workspace=65536
Is there anything that jumps out as incorrect? Is there some
additional configuration required for Solaris or are these panics to
be expected?
Cheers,
Paul.
[1] - http://letsgetdugg.com/2009/12/04/varnish-on-solaris/
[2] First panic type:
Child (18997) died signal=6
Child (18997) Panic message: Assert error in TCP_nonblocking(), tcp.c line 172:
Condition((ioctl(sock, ((int)((uint32_t)(0x80000000|(((sizeof
(int))&0xff)<<16)| ('f'<<8)|126))), &i)) == 0) not true.
errno = 9 (Bad file number)
thread = (cache-worker)
ident = -smalloc,-hcritbit,poll
Backtrace:
44548b: /opt/sbin/varnishd'pan_backtrace+0x1b [0x44548b]
445795: /opt/sbin/varnishd'pan_ic+0x1c5 [0x445795]
fffffd7ff3e5dfec: /opt/lib/libvarnish.so.1.0.0'TCP_nonblocking+0x7c
[0xfffffd7ff3e5dfec]
419091: /opt/sbin/varnishd'vca_return_session+0x1b1 [0x419091]
42675d: /opt/sbin/varnishd'cnt_wait+0x2bd [0x42675d]
42b94a: /opt/sbin/varnishd'CNT_Session+0x4ba [0x42b94a]
44801b: /opt/sbin/varnishd'wrk_do_cnt_sess+0x19b [0x44801b]
447614: /opt/sbin/varnishd'wrk_thread_real+0x854 [0x447614]
447b73: /opt/sbin/varnishd'wrk_thread+0x123 [0x447b73]
fffffd7ff653acf5: /lib/amd64/libc.so.1'_thrp_setup+0x8d [0xfffffd7ff653acf5]
sp = 866548 {
fd = 25, id = 25, xid = 0,
client = 92.41.40.169:2589,
step = STP_WAIT,
handling = deliver,
restarts = 0, esis = 0
ws = 8665b8 {
id = "sess",
{s,f,r,e} = {8672c0,+18,+32786,+65536},
},
http[req] = {
ws = 8665b8[sess]
"",
"/i/template/2009/search_icon_1.gif",
"HTTP/1.1",
"Accept: */*",
"Referer:
http://www.firebox.com/product/2579/Yurakoro-Lucky-Cats?aff=1781",
"Accept-Language: en-gb",
"User-Agent: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1;
Trident/4.0; GTB6.4; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR
3.0.4506.2152; .NET CLR 3.5.30729; OfficeLiveConnector.1.3;
OfficeLivePatch.0.0)",
"Accept-Encoding: gzip, deflate",
"Connection: Keep-Alive",
"host: media.firebox.com",
"X-Forwarded-For: 92.41.40.169",
},
},
[3] Second panic type:
Child (12024) said Child starts
Child (12024) died signal=6
Child (12024) Panic message: Assert error in TCP_blocking(), tcp.c line 163:
Condition((ioctl(sock, ((int)((uint32_t)(0x80000000|(((sizeof
(int))&0xff)<<16)| ('f'<<8)|126))), &i)) == 0) not true.
errno = 9 (Bad file number)
thread = (cache-worker)
ident = -smalloc,-hcritbit,poll
Backtrace:
44548b: /opt/sbin/varnishd'pan_backtrace+0x1b [0x44548b]
445795: /opt/sbin/varnishd'pan_ic+0x1c5 [0x445795]
fffffd7ff3e5df5c: /opt/lib/libvarnish.so.1.0.0'TCP_blocking+0x7c
[0xfffffd7ff3e5df5c]
42b686: /opt/sbin/varnishd'CNT_Session+0x1f6 [0x42b686]
44801b: /opt/sbin/varnishd'wrk_do_cnt_sess+0x19b [0x44801b]
447614: /opt/sbin/varnishd'wrk_thread_real+0x854 [0x447614]
447b73: /opt/sbin/varnishd'wrk_thread+0x123 [0x447b73]
fffffd7ff653acf5: /lib/amd64/libc.so.1'_thrp_setup+0x8d [0xfffffd7ff653acf5]
fffffd7ff653afb0: /lib/amd64/libc.so.1'_lwp_start+0x0 [0xfffffd7ff653afb0]
sp = 3491f88 {
fd = 156, id = 156, xid = 0,
client = ?.?.?.?:?,
step = STP_FIRST,
handling = deliver,
restarts = 0, esis = 0
ws = 3491ff8 {
id = "sess",
{s,f,r,e} = {3492d00,3492d00,0,+65536},
},
http[req] = {
ws = 3491ff8[sess]
"",
"/pic/p2387_search.jpg",
"HTTP/1.1",
"User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB;
rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20 (.NET CLR 3.5.30729)",
"Accept: image/png,*/*;q=0.5",
"Accept-Language: en-gb,en;q=0.5",
"Accept-Encoding: gzip,deflate",
"Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7",
"Keep-Alive: 300",
"Connection: keep-alive",
"Referer: http://www.firebox.com/admin/allproducts",
"host: media.firebox.com",
"X-Forwarded-For: 94.196.164.41",
},
worker = fffffd7ff8e08d30 {
ws = fffffd7ff8e08e78 {
id = "wrk",
{s,f,r,e} = {fffffd7ff8df6c40,fffffd7ff8df6c40,0,+65536},
},
},
},
[4] Third panic type:
Child (14402) died signal=6
Child (14402) Panic message: Assert error in VCA_Prep(),
cache_acceptor.c line 163:
Condition((setsockopt(sp->fd, 0xffff, 0x0080, &linger, sizeof
linger)) == 0) not true.
errno = 9 (Bad file number)
thread = (cache-worker)
ident = -smalloc,-hcritbit,poll
Backtrace:
44548b: /opt/sbin/varnishd'pan_backtrace+0x1b [0x44548b]
445795: /opt/sbin/varnishd'pan_ic+0x1c5 [0x445795]
418495: /opt/sbin/varnishd'VCA_Prep+0x255 [0x418495]
429284: /opt/sbin/varnishd'cnt_first+0xa4 [0x429284]
42b98e: /opt/sbin/varnishd'CNT_Session+0x4fe [0x42b98e]
44801b: /opt/sbin/varnishd'wrk_do_cnt_sess+0x19b [0x44801b]
447614: /opt/sbin/varnishd'wrk_thread_real+0x854 [0x447614]
447b73: /opt/sbin/varnishd'wrk_thread+0x123 [0x447b73]
fffffd7ff653acf5: /lib/amd64/libc.so.1'_thrp_setup+0x8d [0xfffffd7ff653acf5]
fffffd7ff653afb0: /lib/amd64/libc.so.1'_lwp_start+0x0 [0xfffffd7ff653afb0]
sp = 67ef48 {
fd = 131, id = 131, xid = 0,
client = 90.196.3.202:52874,
step = STP_FIRST,
handling = deliver,
restarts = 0, esis = 0
ws = 67efb8 {
id = "sess",
{s,f,r,e} = {67fcc0,+19,0,+65536},
},
http[req] = {
ws = 67efb8[sess]
"",
"/i/video_graphics/blankpix.gif",
"HTTP/1.1",
"Accept: */*",
"Referer: http://www.firebox.com/product/2277/Naughty-Knot",
"Accept-Language: en-us",
"Accept-Encoding: gzip, deflate",
"User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;
SV1; GTB0.0; .NET CLR 2.0.50727)",
"Connection: Keep-Alive",
"host: media.firebox.com",
"X-Forwarded-For: 86.156.9.150",
},
worker = fffffd7ff87ebd30 {
ws = fffffd7ff87ebe78 {
id = "wrk",
{s,f,r,e} = {fffffd7ff87d9c40,fffffd7ff87d9c40,0,+65536},
},
},
},
[5] firebox.vcl
backend martin {
.host = "10.0.0.21";
.port = "80";
}
director dynamic_director round-robin {
{
.backend = martin;
}
}
director static_director round-robin {
{
.backend = martin;
}
}
sub vcl_recv {
// normalise static requests
if ( req.http.host ~ "media([0-9]+).firebox.com" ) {
set req.http.host = "media.firebox.com";
}
//catch any relative image URLs that haven't been repointed to media
if ( req.http.host != "media.firebox.com" &&
( req.url ~ "^/pic/.+" || req.url ~ "^/i/.+" ) ) {
set req.http.host = "media.firebox.com";
}
// split traffic based on host name
if ( req.http.host == "media.firebox.com" ) {
remove req.http.cookie;
set req.backend = static_director;
} else {
// dynamic content that should be cached (ie no cookies required)
// these patterns should match up with settings.inc on the PHP side
if ( req.url ~ "^/styles/(.+).css$" ||
req.url ~ "^/js/(.+).js$" ) {
remove req.http.cookie;
}
//default all requests to the dynamic backend
set req.backend = dynamic_director;
}
}
More information about the varnish-misc
mailing list