Question 39086 in Ubuntu Question posted in
The official Ubuntu documentation can be found here.

Varnish Xkey softpurge – Ubuntu

I’m trying to manage a Xkey soft-purge on objects stored from different hosts, using Xkey as a sort of tag to purge all objects that match the Xkey tag disregarding the hashing.
First of all, is it possible? Or is hashing in the loop in any case?

In vcl_recv I manually set a Xkey key with set req.http.xkey = req.url;
Then during PURGE call I use set req.http.x-purges = xkey.purge(req.url); to find objects, here non objects are found.

Full VCL:

vcl 4.0;

import std;
import directors;
import purge;
import xkey;


backend server1 {
    .host = "ecommerce-node1-prod";
    .port = "80";
    .probe = {
        .url = "/admin";
        .timeout = 1s;
        .interval = 5s;
        .window = 5;
        .threshold = 3;
    }
}

backend server2 {
    .host = "ecommerce-node2-prod";
    .port = "80";
    .probe = {
       .url = "/admin";
       .timeout = 1s;
       .interval = 5s;
       .window = 5;
       .threshold = 3;
   }
}

sub vcl_init {
    new bar = directors.round_robin();
    bar.add_backend(server1);
    bar.add_backend(server2);
}

# ACL for purgers IP. (This needs to contain app server ips)
acl purgers {
    "127.0.0.1";
    "localhost";
    "::1";
    "ecommerce-node1-prod";
    "ecommerce-node2-prod";
}

sub vcl_recv {
    # Mitigate httpoxy application vulnerability, see: https://httpoxy.org/
    unset req.http.Proxy;
    set req.http.xkey = req.url;

    # Strip query strings only needed by browser javascript. Customize to used tags.
    if (req.url ~ "(?|&)(pk_campaign|piwik_campaign|pk_kwd|piwik_kwd|pk_keyword|pixelId|kwid|kw|adid|chl|dv|nk|pa|camid|adgid|cx|ie|cof|siteurl|utm_[a-z]+|_ga|gclid)=") {
        # see rfc3986#section-2.3 "Unreserved Characters" for regex
        set req.url = regsuball(req.url, "(pk_campaign|piwik_campaign|pk_kwd|piwik_kwd|pk_keyword|pixelId|kwid|kw|adid|chl|dv|nk|pa|camid|adgid|cx|ie|cof|siteurl|utm_[a-z]+|_ga|gclid)=[A-Za-z0-9-_.~]+&?", "");
    }
    set req.url = regsub(req.url, "(?|?&|&)$", "");

    # Normalize query arguments
    set req.url = std.querysort(req.url);

    # Normalize Accept-Encoding header
    # straight from the manual: https://www.varnish-cache.org/docs/3.0/tutorial/vary.html
    if (req.http.Accept-Encoding) {
        if (req.url ~ ".(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
            # No point in compressing these
            unset req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elsif (req.http.Accept-Encoding ~ "deflate") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            # unkown algorithm
            unset req.http.Accept-Encoding;
        }
    }


    # Handle PURGE
    if (req.method == "PURGE") {
        if (!client.ip ~ purgers) {
            return (synth(405, "Method not allowed"));
        }


          #set req.http.n-gone = xkey.softpurge(req.http.xkey);
          set req.http.x-purges = xkey.purge(req.url);

          return (synth(200, "Invalidated "+ req.http.x-purges +" objects"));
    }

    if (req.method != "GET" &&
        req.method != "HEAD" &&
        req.method != "PUT" &&
        req.method != "POST" &&
        req.method != "TRACE" &&
        req.method != "OPTIONS" &&
        req.method != "PATCH" &&
        req.method != "DELETE") {
        /* Non-RFC2616 or CONNECT which is weird. */
        return (pipe);
    }

    # We only deal with GET and HEAD by default
    if (req.method != "GET" && req.method != "HEAD") {
        return (pass);
    }

    #pass media to backend (already cached by CloudFront)
    if (req.url ~ "^/(media|bundles|css|fonts|js|theme|thumbnail)(/.*)?$") {
        return (pass);
    }

    # Don't cache Authenticate & Authorization
    if (req.http.Authenticate || req.http.Authorization) {
        return (pass);
    }

    # Always pass these paths directly to php without caching
    # Note: virtual URLs might bypass this rule (e.g. /en/checkout)
    if (req.url ~ "^/(checkout|account|admin|api)(/.*)?$") {
        return (pass);
    }

    return (hash);
}

sub vcl_hash {
       # Consider Shopware http cache cookies
    if (req.http.cookie ~ "sw-cache-hash=") {
        hash_data("+context=" + regsub(req.http.cookie, "^.*?sw-cache-hash=([^;]*);*.*$", "1"));
    } elseif (req.http.cookie ~ "sw-currency=") {
        hash_data("+currency=" + regsub(req.http.cookie, "^.*?sw-currency=([^;]*);*.*$", "1"));
    }

    #return(lookup);
}

sub vcl_hit {
  
  # Consider client states for response headers
  if (req.http.cookie ~ "sw-states=") {
     set req.http.states = regsub(req.http.cookie, "^.*?sw-states=([^;]*);*.*$", "1");

     if (req.http.states ~ "logged-in" && obj.http.sw-invalidation-states ~ "logged-in" ) {
        return (pass);
     }

     if (req.http.states ~ "cart-filled" && obj.http.sw-invalidation-states ~ "cart-filled" ) {
        return (pass);
     }
  }



}

sub vcl_backend_response {
    # Fix Vary Header in some cases
    if (beresp.http.Vary ~ "User-Agent") {
        set beresp.http.Vary = regsub(beresp.http.Vary, ",? *User-Agent *", "");
        set beresp.http.Vary = regsub(beresp.http.Vary, "^, *", "");
        if (beresp.http.Vary == "") {
            unset beresp.http.Vary;
        }
    }

    # Respect the Cache-Control=private header from the backend
    if (
        beresp.http.Pragma        ~ "no-cache" ||
        beresp.http.Cache-Control ~ "no-cache" ||
        beresp.http.Cache-Control ~ "private"
    ) {
        set beresp.ttl = 0s;
        set beresp.http.X-Cacheable = "NO:Cache-Control=private";
        set beresp.uncacheable = true;
        return (deliver);
    }

    # strip the cookie before the image is inserted into cache.
    if (bereq.url ~ ".(png|gif|jpg|swf|css|js|webp)$") {
        unset beresp.http.set-cookie;
    }

    # Allow items to be stale if needed.
    set beresp.grace = 60s;


    # Save the bereq.url so purges work efficiently
    set beresp.http.x-url = bereq.url;
    set beresp.http.X-Cacheable = "YES";

    # Remove the exact PHP Version from the response for more security
    unset beresp.http.x-powered-by;


    return (deliver);
}
   
sub vcl_deliver {
    ## we don't want the client to cache
    set resp.http.Cache-Control = "max-age=0, private";

    # remove link header, if session is already started to save client resources
    if (req.http.cookie ~ "session-") {
        unset resp.http.Link;
    }

    # Set a cache header to allow us to inspect the response headers during testing
    if (obj.hits > 0) {
        unset resp.http.set-cookie;
        set resp.http.X-Cache = "HIT";
    }  else {
        set resp.http.X-Cache = "MISS";
    }

    # Remove the exact PHP Version from the response for more security (e.g. 404 pages)
    unset resp.http.x-powered-by;

    # invalidation headers are only for internal use
    unset resp.http.sw-invalidation-states;

    set resp.http.X-Cache-Hits = obj.hits;
}

Request where xkey is added

*   << Request  >> 32796
-   Begin          req 32795 rxreq
-   Timestamp      Start: 1664372821.615382 0.000000 0.000000
-   Timestamp      Req: 1664372821.615382 0.000000 0.000000
-   VCL_use        boot
-   ReqStart       172.16.0.80 20244 a0
-   ReqMethod      GET
-   ReqURL         /it-it/prodotti/catene/catene-di-luci/
-   ReqProtocol    HTTP/1.1
-   ReqHeader      X-Forwarded-For: 84.247.245.84, 130.176.89.143
-   ReqHeader      X-Forwarded-Proto: https
-   ReqHeader      X-Forwarded-Port: 443
-   ReqHeader      Host: test-prod.luminalpark.com
-   ReqHeader      X-Amzn-Trace-Id: Root=1-63345055-76f876fd27d8613e28b7c9db
-   ReqHeader      User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36 Edg/105.0.1343.53
-   ReqHeader      X-Amz-Cf-Id: JA7XrQuv5k-jhdhb3uKmlf88YjlJVvcDcw6udobH5vL-YkSMDT2flw==
-   ReqHeader      Via: 2.0 6c61cea6f371b1744d3b5315a0029062.cloudfront.net (CloudFront)
-   ReqHeader      Cookie: _dc_gtm_UA-830149-18=1; _ga=GA1.1.1015042543.1664369859; _ga_499EG6CXZD=GS1.1.1664371930.2.1.1664372814.0.0.0; _gcl_au=1.1.1616057317.1664369858; _gid=GA1.2.1284203691.1664369859; lp-platform=it-it; lp-state=IT; newUser=1; poll_manager=1; session
-   ReqHeader      Accept-Language: it,it-IT;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6,fr;q=0.5
-   ReqHeader      Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
-   ReqHeader      Accept-Encoding: gzip, deflate, br
-   ReqHeader      cache-control: max-age=0
-   ReqHeader      sec-ch-ua: "Microsoft Edge";v="105", "Not)A;Brand";v="8", "Chromium";v="105"
-   ReqHeader      sec-ch-ua-mobile: ?0
-   ReqHeader      sec-ch-ua-platform: "Windows"
-   ReqHeader      dnt: 1
-   ReqHeader      upgrade-insecure-requests: 1
-   ReqHeader      sec-fetch-site: none
-   ReqHeader      sec-fetch-mode: navigate
-   ReqHeader      sec-fetch-user: ?1
-   ReqHeader      sec-fetch-dest: document
-   ReqHeader      CloudFront-Viewer-HTTP-Version: 2.0
-   ReqHeader      CloudFront-Forwarded-Proto: https
-   ReqHeader      CloudFront-Viewer-Address: 84.247.245.84:59146
-   ReqHeader      CloudFront-Viewer-TLS: TLSv1.3:TLS_AES_128_GCM_SHA256:connectionReused
-   ReqHeader      X-Cloudfront-Origin: VC5ZNQ588QNE3S
-   ReqUnset       X-Forwarded-For: 84.247.245.84, 130.176.89.143
-   ReqHeader      X-Forwarded-For: 84.247.245.84, 130.176.89.143, 172.16.0.80
-   ReqUnset       Via: 2.0 6c61cea6f371b1744d3b5315a0029062.cloudfront.net (CloudFront)
-   ReqHeader      Via: 2.0 6c61cea6f371b1744d3b5315a0029062.cloudfront.net (CloudFront), 1.1 ip-172-16-3-107 (Varnish/trunk)
-   VCL_call       RECV
-   ReqHeader      xkey: /it-it/prodotti/catene/catene-di-luci/
-   ReqURL         /it-it/prodotti/catene/catene-di-luci/
-   ReqURL         /it-it/prodotti/catene/catene-di-luci/
-   ReqUnset       Accept-Encoding: gzip, deflate, br
-   ReqHeader      Accept-Encoding: gzip
-   VCL_return     hash
-   VCL_call       HASH
-   VCL_return     lookup
-   Hit            32771 7190.679091 60.000000 0.000000
-   VCL_call       HIT
-   VCL_return     deliver
-   RespProtocol   HTTP/1.1
-   RespStatus     200
-   RespReason     OK
-   RespHeader     Server: nginx/1.18.0 (Ubuntu)
-   RespHeader     Content-Type: text/html; charset=UTF-8
-   RespHeader     Cache-Control: must-revalidate, public, s-maxage=7200
-   RespHeader     Date: Wed, 28 Sep 2022 13:46:52 GMT
-   RespHeader     Strict-Transport-Security: max-age=31536000; includeSubDomains
-   RespHeader     X-Frame-Options: deny
-   RespHeader     X-Content-Type-Options: nosniff
-   RespHeader     Referrer-Policy: strict-origin-when-cross-origin
-   RespHeader     sw-invalidation-states:
-   RespHeader     Set-Cookie: sw-states=deleted; expires=Tue, 28-Sep-2021 13:46:51 GMT; Max-Age=0; path=/; secure; httponly; samesite=lax
-   RespHeader     Set-Cookie: sw-cache-hash=deleted; expires=Tue, 28-Sep-2021 13:46:51 GMT; Max-Age=0; path=/; httponly
-   RespHeader     Content-Encoding: gzip
-   RespHeader     Vary: Accept-Encoding
-   RespHeader     x-url: /it-it/prodotti/catene/catene-di-luci/
-   RespHeader     X-Cacheable: YES
-   RespHeader     X-Varnish: 32796 32771
-   RespHeader     Age: 9
-   RespHeader     Via: 1.1 ip-172-16-3-107 (Varnish/trunk)
-   RespHeader     Accept-Ranges: bytes
-   VCL_call       DELIVER
-   RespUnset      Cache-Control: must-revalidate, public, s-maxage=7200
-   RespHeader     Cache-Control: max-age=0, private
-   RespUnset      Set-Cookie: sw-states=deleted; expires=Tue, 28-Sep-2021 13:46:51 GMT; Max-Age=0; path=/; secure; httponly; samesite=lax
-   RespUnset      Set-Cookie: sw-cache-hash=deleted; expires=Tue, 28-Sep-2021 13:46:51 GMT; Max-Age=0; path=/; httponly
-   RespHeader     X-Cache: HIT
-   RespUnset      sw-invalidation-states:
-   RespHeader     X-Cache-Hits: 1
-   VCL_return     deliver
-   Timestamp      Process: 1664372821.615457 0.000074 0.000074
-   Filters
-   RespHeader     Content-Length: 110820
-   RespHeader     Connection: keep-alive
-   Timestamp      Resp: 1664372821.615548 0.000166 0.000091
-   ReqAcct        1598 0 1598 619 110820 111439
-   End

Purge request log, showing no objects found:

*   << Request  >> 1277962
-   Begin          req 1277960 rxreq
-   Timestamp      Start: 1664372901.809064 0.000000 0.000000
-   Timestamp      Req: 1664372901.809064 0.000000 0.000000
-   VCL_use        boot
-   ReqStart       172.16.2.136 51214 a0
-   ReqMethod      PURGE
-   ReqURL         /it-it/prodotti/catene/catene-di-luci/
-   ReqProtocol    HTTP/1.1
-   ReqHeader      Host: 172.16.3.107
-   ReqHeader      User-Agent: GuzzleHttp/7
-   ReqHeader      X-Forwarded-For: 172.16.2.136
-   ReqHeader      Via: 1.1 ip-172-16-3-107 (Varnish/trunk)
-   VCL_call       RECV
-   ReqHeader      xkey: /it-it/prodotti/catene/catene-di-luci/
-   ReqURL         /it-it/prodotti/catene/catene-di-luci/
-   ReqURL         /it-it/prodotti/catene/catene-di-luci/
-   ReqHeader      x-purges: 0
-   VCL_return     synth
-   VCL_call       HASH
-   VCL_Log        hashing for BAN request
-   VCL_return     lookup
-   RespProtocol   HTTP/1.1
-   RespStatus     200
-   RespReason     Invalidated 0 objects
-   RespHeader     Date: Wed, 28 Sep 2022 13:48:21 GMT
-   RespHeader     Server: Varnish
-   RespHeader     X-Varnish: 1277962
-   VCL_call       SYNTH
-   RespHeader     Content-Type: text/html; charset=utf-8
-   RespHeader     Retry-After: 5
-   VCL_return     deliver
-   Timestamp      Process: 1664372901.809126 0.000061 0.000061
-   RespHeader     Content-Length: 287
-   Storage        malloc Transient
-   Filters
-   RespHeader     Connection: keep-alive
-   Timestamp      Resp: 1664372901.809155 0.000090 0.000029
-   ReqAcct        101 0 101 213 287 500
-   End
Tags:

2

Answers


  1. 0

    It’s perfectly possible, but you need to add the header to beresp object (in vcl_backend_response) to have the objects properly tagged

    Login or Signup to reply.
  2. 0

    Setting keys

    vmod_xkey doesn’t offer you a lot of flexibility when it comes to setting keys. The only way to do this is by adding one or more xkey headers to your response containing the tags you want to assign to this object:

    Here’s an example where we set 3 tags:

    HTTP/1.1 200 OK
xkey: 8155054
xkey: 166412
xkey: 234323

    It’s also possible to set multiple keys in a single xkey header as illustrated below:

    HTTP/1.1 200 OK
xkey: 8155054 166412 234323

    Unlike Varnish Enterprise’s vmod_ykey, vmod_xkey doesn’t allow you to set tags in vcl_backend_response. vmod_ykey also allows you to choose the header you want to use to tag content. vmod_xkey only allows the xkey header.

    Invalidating keys

    Once you have tagged the content, you can remove content based on those tags. Here’s a quick example of a tag that was passed to Varnish through the xkey request header:

    sub vcl_recv {
    if (req.method == "PURGE") {
        if (client.ip !~ purgers) {
            return (synth(403, "Forbidden"));
        }
        if (req.http.xkey) {
            set req.http.n-gone = xkey.softpurge(req.http.xkey)
            return (synth(200, "Invalidated "+req.http.n-gone+" objects"));
        } else {
            return (purge);
        }       
    }
}

    The purgers ACL isn’t included in the VCL example, but should be defined in your VCL code and contain the allowed IP addresses, hostnames or subnets.

    Here’s an example of how you would execute the PURGE using cURL :

    curl -XPURGE -H "xkey: 8155054" http://test-prod.luminalpark.com/
    Login or Signup to reply.
Please signup or login to give your own answer.

Back To Top
Search