skip to Main Content

I have 2 servers, one which has magento 2 installed (ip – 129.89.188.244 port 80) and Varnish on another (ip – 129.89.188.245 port 80)

My Varnish Configuration:

File /etc/default/varnish:-

DAEMON_OPTS="-a :80 
             -T 127.0.0.1:6082 
             -b 129.89.188.244:80 
             -f /etc/varnish/default.vcl 
             -S /etc/varnish/secret 
             -s malloc,256m"

netstat -tulpn :-

tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      1288/sshd       
tcp        0      0 127.0.0.1:6082          0.0.0.0:*               LISTEN      11115/varnishd  
tcp        0      0 0.0.0.0:80              0.0.0.0:*               LISTEN      11115/varnishd  
tcp6       0      0 :::22                   :::*                    LISTEN      1288/sshd       
tcp6       0      0 :::80                   :::*                    LISTEN      11115/varnishd  

/etc/varnish/default.vcl : –

# VCL version 5.0 is not supported so it should be 4.0 even though actually used Varnish version is 5
vcl 4.0;

import std;
# The minimal Varnish version is 5.0
# For SSL offloading, pass the following header in your proxy server or load balancer: 'X-Forwarded-Proto: https'

backend default {
    .host = "129.89.188.244";
    .port = "80";
    .first_byte_timeout = 600s;
    .probe = {
        .url = "/pub/health_check.php";
        .timeout = 2s;
        .interval = 5s;
        .window = 10;
        .threshold = 5;
   }
}

acl purge {
    "129.89.188.245";
    "127.0.0.1";
    "localhost";
}

sub vcl_recv {
    if (req.method == "PURGE") {
        if (client.ip !~ purge) {
            return (synth(405, "Method not allowed"));
        }
        # To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header
        # has been added to the response in your backend server config. This is used, for example, by the
        # capistrano-magento2 gem for purging old content from varnish during it's deploy routine.
        if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) {
            return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required"));
        }
        if (req.http.X-Magento-Tags-Pattern) {
          ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
        }
        if (req.http.X-Pool) {
          ban("obj.http.X-Pool ~ " + req.http.X-Pool);
        }
        return (synth(200, "Purged"));
    }

    if (req.method != "GET" &&
        req.method != "HEAD" &&
        req.method != "PUT" &&
        req.method != "POST" &&
        req.method != "TRACE" &&
        req.method != "OPTIONS" &&
        req.method != "DELETE") {
          /* Non-RFC2616 or CONNECT which is weird. */
          return (pipe);
    }

    # We only deal with GET and HEAD by default
    if (req.method != "GET" && req.method != "HEAD") {
        return (pass);
    }

    # Bypass shopping cart, checkout and search requests
    if (req.url ~ "/checkout" || req.url ~ "/catalogsearch") {
        return (pass);
    }

    # Bypass health check requests
    if (req.url ~ "/pub/health_check.php") {
        return (pass);
    }

    # Set initial grace period usage status
    set req.http.grace = "none";

    # normalize url in case of leading HTTP scheme and domain
    set req.url = regsub(req.url, "^http[s]?://", "");

    # collect all cookies
    std.collect(req.http.Cookie);

    # Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression
    if (req.http.Accept-Encoding) {
        if (req.url ~ ".(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
            # No point in compressing these
            unset req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            # unkown algorithm
            unset req.http.Accept-Encoding;
        }
    }

    # Remove Google gclid parameters to minimize the cache objects
    set req.url = regsuball(req.url,"?gclid=[^&]+$",""); # strips when QS = "?gclid=AAA"
    set req.url = regsuball(req.url,"?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar"
    set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz"

    # Static files caching
    if (req.url ~ "^/(pub/)?(media|static)/") {
        # Static files should not be cached by default
        return (pass);

        # But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
        #unset req.http.Https;
        #unset req.http.X-Forwarded-Proto;
        #unset req.http.Cookie;
    }

    return (hash);
}

sub vcl_hash {
    if (req.http.cookie ~ "X-Magento-Vary=") {
        hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "1"));
    }

    # For multi site configurations to not cache each other's content
    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }

    # To make sure http users don't see ssl warning
    if (req.http.X-Forwarded-Proto) {
        hash_data(req.http.X-Forwarded-Proto);
    }

}

sub vcl_backend_response {

    set beresp.grace = 3d;

    if (beresp.http.content-type ~ "text") {
        set beresp.do_esi = true;
    }

    if (bereq.url ~ ".js$" || beresp.http.content-type ~ "text") {
        set beresp.do_gzip = true;
    }

    if (beresp.http.X-Magento-Debug) {
        set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
    }

    # cache only successfully responses and 404s
    if (beresp.status != 200 && beresp.status != 404) {
        set beresp.ttl = 0s;
        set beresp.uncacheable = true;
        return (deliver);
    } elsif (beresp.http.Cache-Control ~ "private") {
        set beresp.uncacheable = true;
        set beresp.ttl = 86400s;
        return (deliver);
    }

    # validate if we need to cache it and prevent from setting cookie
    # images, css and js are cacheable by default so we have to remove cookie also
    if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
        unset beresp.http.set-cookie;
    }

   # If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass
   if (beresp.ttl <= 0s ||
       beresp.http.Surrogate-control ~ "no-store" ||
       (!beresp.http.Surrogate-Control &&
       beresp.http.Cache-Control ~ "no-cache|no-store") ||
       beresp.http.Vary == "*") {
        # Mark as Hit-For-Pass for the next 2 minutes
        set beresp.ttl = 120s;
        set beresp.uncacheable = true;
    }

    return (deliver);
}

sub vcl_deliver {
    if (resp.http.X-Magento-Debug) {
        if (resp.http.x-varnish ~ " ") {
            set resp.http.X-Magento-Cache-Debug = "HIT";
            set resp.http.Grace = req.http.grace;
        } else {
            set resp.http.X-Magento-Cache-Debug = "MISS";
        }
    } else {
        unset resp.http.Age;
    }

    # Not letting browser to cache non-static files.
    if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") {
        set resp.http.Pragma = "no-cache";
        set resp.http.Expires = "-1";
        set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
    }

    unset resp.http.X-Magento-Debug;
    unset resp.http.X-Magento-Tags;
    unset resp.http.X-Powered-By;
    unset resp.http.Server;
    unset resp.http.X-Varnish;
    unset resp.http.Via;
    unset resp.http.Link;
}

sub vcl_hit {
    if (obj.ttl >= 0s) {
        # Hit within TTL period
        return (deliver);
    }
    if (std.healthy(req.backend_hint)) {
        if (obj.ttl + 300s > 0s) {
            # Hit after TTL expiration, but within grace period
            set req.http.grace = "normal (healthy server)";
            return (deliver);
        } else {
            # Hit after TTL and grace expiration
            return (miss);
        }
    } else {
        # server is not healthy, retrieve from cache
        set req.http.grace = "unlimited (unhealthy server)";
        return (deliver);
    }
}

Now the issue is, when I open the URL 129.89.188.244 , magento opens but it’s not getting cached in varnish. But when I call varnish URL 129.89.188.245, it will redirect to my magento url 129.89.188.244. In my varnish log, it shows that the page is cached already but magento is not getting served from that varnish cache.

3

Answers


  1. this is the expected behaviour.
    Varnish sits in front of you Magento server and if you ping your Magento server bypassing Varnish, opening 129.89.188.244, then Magento will send you a response without involving Varnish, while if you ping varnish, Varnish will then call Magento and cache. The other way around is not possible and makes no sense.

    Login or Signup to reply.
  2. In order for the cache to work, your requests always have to go through the proxy/cache (Varnish).

    Varnish evaluates your requests and decides if the request is already cached (then returns it from the cache) or not – then it will redirect it to Magento and cache the response, before returning it to the client.

    That’s how most caches work. You can read in detail how Varnish cache mechanism works here.

    If you want to hit your cache, you always have to go through Varnish (129.89.188.245:80).

    Consider this diagram form the official documentation:

    enter image description here

    Login or Signup to reply.
  3. This solution worked for me. To configure Varnish and Magento on a different server.
    
    Varnish server: xxx.xxx.xxx.xxx port 80
    
    Magento server: yyy.yyy.yyy.yyy port 80
    
    Changes need to be made on the varnish server: 
    
     1. login to varnish server
     2. go to file /etc/varnish/default.vcl
     3. under the "backend default" update
    
        .host = "yyy.yyy.yyy.yyy";//(use Magento server IP for better network)
        .port = "80";//(Magento web server port) 
     4. Restart the Varnish (systemctl restart varnish)
    
    Note: Kindly use the default VCL that is generated during the varnish installation and don't update it with Magento generated VCL for varnish ( available from Magento Admin)
    
    Changes need to be made on the Magento server:
    1. Log in to the Magento server
    2. Go to the env.php file located in the app/etc directory
    3. Update the values in     'http_cache_hosts' => [
            [
                'host' => 'xxx.xxx.xxx.xxx', //(varnish server public ip)
                'port' => '80' // ( varnish server port)
            ]
        ]
    Now update your base URL's on the core_config_data table to your varnish public  Ip (http://xxx.xxx.xxx.xxx/)
    flush the Magento caches ( bin/magento ca:fl)
    
    Login or Signup to reply.
Please signup or login to give your own answer.
Back To Top
Search