varnish is sticking too much to web nodes
Boris Guéry
guery.b at gmail.com
Tue Jun 3 10:08:54 CEST 2014
Maybe you can start to check the request counters of Apache using its
status module.
You should be able to see how much requests your webserver is handling
at a given time.
Check http://httpd.apache.org/docs/2.2/mod/mod_status.html
It should also be easy to create a script which check every n seconds
the apache status of each servers
and aggregate data for further analysis.
Le 02/06/14 18:08, Tim Dunphy a écrit :
>
> We have two out of six web nodes showing extremely high CPU usage
> compared to the others in the farm.
>
> We're using the round-robin load balancing algorithm. Using a tool
> like 'htop' shows us that all the hungry processes belong to apache.
> Yet diffing the configuration files between the nodes with low cpu
> usage and the ones with high cpu usage shows no difference at all.
>
> The theory is that varnish is throwing traffic at these two hosts too
> often.
>
> So I was just wondering if there could be any way to diagnose this
> from a Varnish perspective. What clues could tools like varnishtop and
> varnishstat or varnishlog yield in investigating this type of problem?
>
> Here's my varnish vcl (cleaned of any too-identifying info) in case
> anyone can offer an opinion as to why this is going on:
>
> backend web1 {
> .host = "10.10.1.98";
> .port = "80";
> .connect_timeout = 90s;
> .first_byte_timeout = 90s;
> .between_bytes_timeout = 90s;
> .max_connections = 70;
> .probe = {
> .url = "/healthcheck.php";
> .timeout = 5s;
> .interval = 30s;
> .window = 10;
> .threshold = 1;
> }
> }
>
> backend web2 {
> .host = "10.10.1.99";
> .port = "80";
> .connect_timeout = 90s;
> .first_byte_timeout = 90s;
> .between_bytes_timeout = 90s;
> .max_connections = 70;
> .probe = {
> .url = "/healthcheck.php";
> .timeout = 5s;
> .interval = 30s;
> .window = 10;
> .threshold = 1;
> }
> }
>
> backend web3 {
> .host = "10.10.1.100";
> .port = "80";
> .connect_timeout = 90s;
> .first_byte_timeout = 90s;
> .between_bytes_timeout = 90s;
> .max_connections = 70;
> .probe = {
> .url = "/healthcheck.php";
> .timeout = 5s;
> .interval = 30s;
> .window = 10;
> .threshold = 1;
> }
> }
>
> backend web4 {
> .host = "10.10.1.101";
> .port = "80";
> .connect_timeout = 90s;
> .first_byte_timeout = 90s;
> .between_bytes_timeout = 90s;
> .max_connections = 70;
> .probe = {
> .url = "/healthcheck.php";
> .timeout = 5s;
> .interval = 30s;
> .window = 10;
> .threshold = 1;
> }
> }
>
> backend web5 {
> .host = "10.10.1.235";
> .port = "80";
> .connect_timeout = 90s;
> .first_byte_timeout = 90s;
> .between_bytes_timeout = 90s;
> .max_connections = 70;
> .probe = {
> .url = "/healthcheck.php";
> .timeout = 5s;
> .interval = 30s;
> .window = 10;
> .threshold = 1;
> }
> }
>
> backend web6 {
> .host = "10.10.1.236";
> .port = "80";
> .connect_timeout = 90s;
> .first_byte_timeout = 90s;
> .between_bytes_timeout = 90s;
> .max_connections = 70;
> .probe = {
> .url = "/healthcheck.php";
> .timeout = 5s;
> .interval = 30s;
> .window = 10;
> .threshold = 1;
> }
> }
>
> backend varnish1 {
> .host = "10.10.1.96";
> .port = "80";
> .connect_timeout = 90s;
> .first_byte_timeout = 90s;
> .between_bytes_timeout = 90s;
> .max_connections = 1000;
> }
>
> backend varnish2 {
> .host = "10.10.1.97";
> .port = "80";
> .connect_timeout = 90s;
> .first_byte_timeout = 90s;
> .between_bytes_timeout = 90s;
> .max_connections = 1000;
> }
>
> acl purge {
> "localhost";
> "127.0.0.1";
> "10.10.1.96";
> "10.10.1.97";
> }
>
> director www round-robin {
> { .backend = web1; }
> { .backend = web2; }
> { .backend = web3; }
> { .backend = web4; }
> { .backend = web5; }
> { .backend = web6; }
>
>
> }
>
> director cache round-robin {
> { .backend = varnish1; }
> { .backend = varnish2; }
> }
>
> sub vcl_recv {
> set req.backend = www;
> if (req.http.User-Agent ~ "typhoeus") {
> error 403 "Temporarily disabled";
> }
> set req.grace = 6h;
> if (! req.backend.healthy) {
> set req.grace = 24h;
> }
> set req.http.X-Forwarded-For = req.http.X-Forwarded-For ", "
> client.ip;
> /*if (req.http.host ~ "^origin\.test\.(.+\.|)mywebsite.com
> <http://mywebsite.com>$") {
> return(pass);
> }*/
> if (req.http.host ~ "^origin\.(.+\.|)mywebsite\.com$") {
> return(pass);
> }
> if (req.http.host ~ "test.cms.mywebsite.com
> <http://test.cms.mywebsite.com>|test.cms-2.mywebsite.com
> <http://test.cms-2.mywebsite.com>|test.facebook.mywebsite.com
> <http://test.facebook.mywebsite.com>") {
> return (pass);
> }
> if (req.http.host ~ "test.cache-only.mywebsite.com
> <http://test.cache-only.mywebsite.com>") {
> set req.backend = cache;
> }
> /* if (req.http.host ~ "test\.*\.mywebsite.com
> <http://mywebsite.com>") {
> if (req.http.host !~ "test\.(.+\.|)m\.mywebsite\.com$") {
> set req.http.host = "test.mywebsite.com
> <http://test.mywebsite.com>";
> }
> } else {
> return (pass);
> }*/
>
> if (req.http.host ~ ".*\.mywebsite.com <http://mywebsite.com>") {
> if (req.http.host !~ "(.+\.|)m\.mywebsite\.com$") {
> set req.http.host = "www.mywebsite.com
> <http://www.mywebsite.com>";
> }
> } else {
> return (pass);
> }
>
> /*
> if (req.url ~ "^/node/75402/tweettracker_brand/config/xml") {
> error 403 "Temporarily disabled";
> }
> */
> if (req.request == "PURGE") {
> if (!client.ip ~ purge) {
> error 405 "Not allowed.";
> }
> return (lookup);
> }
> if (req.request != "GET" &&
> req.request != "HEAD" &&
> req.request != "PUT" &&
> req.request != "POST" &&
> req.request != "TRACE" &&
> req.request != "OPTIONS" &&
> req.request != "DELETE") {
> return (pipe);
> }
> if (req.request != "GET" && req.request != "HEAD") {
> return (pass);
> }
> if (req.url ~ "eyeblaster/addineyeV2.html\?.*") {
> set req.url = "/eyeblaster/addineyeV2.html";
> }
> if (req.url ~ "sites/all/modules/custom/bravo_ad/ads.html\?.*") {
> set req.url = "/sites/all/modules/custom/bravo_ad/ads.html";
> }
> if (req.url ~
> "sites/m.mywebsite.com/modules/custom/bravo_ad/ads.html\?.*
> <http://m.mywebsite.com/modules/custom/bravo_ad/ads.html%5C?.*>") {
> set req.url =
> "/sites/m.mywebsite.com/modules/custom/bravo_ad/ads.html
> <http://m.mywebsite.com/modules/custom/bravo_ad/ads.html>";
> }
> if (req.url ~
> "ahah_helper\.php|bravo_points\.php|install\.php|update\.php|cron\.php|/json(:?\?.*)?$")
> {
> return (pass);
> }
> if (req.http.Authorization) {
> return (pass);
> }
> if (req.url ~ "login" || req.url ~ "logout") {
> return (pass);
> }
> if (req.url ~ "^/admin/" || req.url ~ "^/node/add/") {
> return (pass);
> }
> if (req.http.Cache-Control ~ "no-cache") {
> //return (pass);
> }
> if (req.http.Cookie ~
> "(VARNISH|DRUPAL_UID|LOGGED_IN|SESS|_twitter_sess)") {
> set req.http.Cookie = regsuball(req.http.Cookie,
> "(^|;\s*)(__[a-z]+|has_js)=[^;]*", "");
> set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
> } else {
> unset req.http.Cookie;
> }
> if (req.restarts == 0) {
> if (client.ip == "10.10.1.96" || client.ip == "10.10.1.97") {
> set req.backend = www;
> } elsif (server.ip == "10.10.1.96") {
> set req.backend = varnish2;
> } else {
> set req.backend = varnish1;
> }
> } elsif (req.restarts >= 2) {
> return (pass);
> }
> if (req.restarts >= 2) {
> return (pass);
> }
> if (req.url ~
> "\.(ico|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|ICO|JPG|JPEG|PNG|GIF|GZ|TGZ|BZ2|TBZ|MP3|OOG|SWF)")
> {
> unset req.http.Accept-Encoding;
> }
> if (req.url ~
> "^/(sites/all/modules/mywebsite_admanager/includes/ads.php|doubleclick/DARTIframe.html)(\?.*|)$")
> {
> set req.url = regsub(req.url, "\?.*$", "");
> }
> /* keep user-agents
> if (! req.http.User-Agent ~ "MSIE") {
> set req.http.User-Agent = "Mozilla";
> }
> */
> if (req.http.Accept-Encoding ~ "gzip") {
> set req.http.Accept-Encoding = "gzip";
> } elsif (req.http.Accept-Encoding ~ "deflate") {
> set req.http.Accept-Encoding = "deflate";
> } else {
> unset req.http.Accept-Encoding;
> }
> return (lookup);
> }
>
> sub vcl_pipe {
> set bereq.http.connection = "close";
> return (pipe);
> }
>
> sub vcl_pass {
> return (pass);
> }
>
> sub vcl_hash {
> set req.hash += req.url;
> set req.hash += req.http.host;
> if (req.http.Cookie ~ "VARNISH|DRUPAL_UID|LOGGED_IN") {
> set req.hash += req.http.Cookie;
> }
> return (hash);
> }
>
> sub vcl_hit {
> if (req.request == "PURGE") {
> set obj.ttl = 0s;
> error 200 "Purged.";
> }
> /*
> if (!obj.cacheable) {
> return (pass);
> }
> return (deliver);
> */
> }
>
> sub vcl_fetch {
> if (beresp.status == 500) {
> set req.http.X-Varnish-Error = "1";
> restart;
> }
> set beresp.grace = 6h;
> # Set a short circuit cache lifetime for resp codes above 302
> if (beresp.status > 302) {
> set beresp.ttl = 60s;
> set beresp.http.Cache-Control = "max-age = 60";
> }
> if (beresp.http.Edge-control ~ "no-store") {
> set beresp.http.storage = "1";
> set beresp.cacheable = false;
> return (pass);
> }
> if (beresp.status >= 300 || !beresp.cacheable) {
> set beresp.http.Varnish-X-Cacheable = "Not Cacheable";
> set beresp.http.storage = "1";
> return (pass);
> }
> if (beresp.http.Set-Cookie) {
> return (pass);
> }
> if (beresp.cacheable) {
> unset beresp.http.expires;
> set beresp.ttl = 600s;
> set beresp.http.Cache-Control = "max-age = 600";
> if (req.url ~
> "\.(ico|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|ICO|JPG|JPEG|PNG|GIF|GZ|TGZ|BZ2|TBZ|MP3|OOG|SWF)")
> {
> set beresp.ttl = 43829m;
> set beresp.http.Cache-Control = "max-age = 1000000";
> }
> /*
> if (req.url ~ "json(:?\??.*)?$") {
> set beresp.ttl = 30s;
> set beresp.http.Cache-Control = "max-age = 30";
> }
> */
> }
> return (deliver);
> }
>
> sub vcl_deliver {
> if (obj.hits > 0) {
> set resp.http.Varnish-X-Cache = "HIT";
> set resp.http.Varnish-X-Cache-Hits = obj.hits;
> } else {
> set resp.http.Varnish-X-Cache = "MISS";
> }
> return (deliver);
> }
>
> sub vcl_error {
> if (req.restarts == 0) {
> return (restart);
> }
> if (req.http.X-Varnish-Error != "1") {
> set req.http.X-Varnish-Error = "1";
> return (restart);
> }
> set obj.http.Content-Type = "text/html; charset=utf-8";
> synthetic {"
> <?xml version="1.0" encoding="utf-8"?>
> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
> "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"
> <http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd%22> >
> <html>
> <head>
> <title>mywebsite.com <http://mywebsite.com> "} obj.status " "
> obj.response {"</title>
> <style type="text/css">
> body {background-color: #fff; margin: auto;}
> #page {width: 1000px; margin: 0 auto; background-color: #fff;}
> </style>
> </head>
> <body>
> <div id="page">
> <img
> src="http://www.mywebsite.com/media/site-unavailable/site-unavailable.jpg"
> width="1000" height="700" >
> <!--
> <h4>Debug Info:</h4>
> <pre>
> Status: "} obj.status {"
> Response: "} obj.response {"
> XID: "} req.xid {"
> </pre>
> <address><a
> href="http://www.varnish-cache.org/">Varnish</a></address<http://www.varnish-cache.org/%22%3eVarnish%3c/a%3e%3c/address>
> <http://www.varnish-cache.org/%22%3eVarnish%3c/a%3e%3c/address> >
> -->
> </div>
> </body>
> </html>
> "};
> return (deliver);
> }
>
>
> Thanks
> Tim
> --
> GPG me!!
>
> gpg --keyserver pool.sks-keyservers.net
> <http://pool.sks-keyservers.net> --recv-keys F186197B
>
>
>
> _______________________________________________
> varnish-misc mailing list
> varnish-misc at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-misc
--
Boris Guéry
twitter: @borisguery
skype: borisguery
mobile: +33 6 86 83 03 12
pgp: 0x034C6265
irc: bgy @ freenode
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-misc/attachments/20140603/ea0d6253/attachment-0001.html>
More information about the varnish-misc
mailing list