<commit_before>/*
 * Varnish 4 example config for Drupal 7 & 8 / Pressflow 6 & 7
 */

# Original source: https://github.com/NITEMAN/varnish-bites/varnish4/drupal-base.vcl
# Copyright (c) 2015 <NAME> and individual contributors.
# MIT License

# Intended to be used both in simple production environments and with
# learning/teaching purposes.
# Some references may not be yet updated to VCL 4.0, we use SeeV3 then.

# WARNING:
# Note that the built-in logic will be appended to our code if no return is
# performed before.
# Built-in logic is included commented out right after our's for reference
# purposes in that cases.
# See https://www.varnish-cache.org/trac/wiki/VCLExampleDefault

#######################################################################
# Initialization: Version & imports;

/* Version statement */
# Since Varnish 4.0 it's mandatory to declare VCL version on first line.
vcl 4.0;

/* Module (VMOD) imports */
# Standard module
# See https://www.varnish-cache.org/docs/4.0/reference/vmod_std.generated.html
import std;

# Directors module
# See https://www.varnish-cache.org/docs/4.0/reference/vmod_directors.generated.html
# Unused in simple configs.
#import directors;


#######################################################################
# Probe, backend, ACL and subroutine definitions

/* Backend probes / healthchecks */
# See https://www.varnish-cache.org/docs/4.0/reference/vcl.html#probes
probe basic {
  /* Only test that backend's IP serves content for '/' */
  # This might be a too heavy probe
  # .url = "/";

  /* Only test that backend's IP has apache working */
  # Nginx would fail this probe with a default config
  .request =
    "OPTIONS * HTTP/1.1"
    "Host: *"
    "Connection: close";

  /* Common options */
  .interval = 10s;
  .timeout = 2s;
  .window = 8;
  .threshold = 6;
}

/* Backend definitions.*/
# See https://www.varnish-cache.org/docs/4.0/reference/vcl.html#backend-definition
backend default {
  /* Default backend on the same machine. */
  # WARNING: timeouts could be not big enought for certain POST requests.
  .host = "127.0.0.1";
  .port = "8008";
  .max_connections = 100;
  .connect_timeout = 60s;
  .first_byte_timeout = 60s;
  .between_bytes_timeout = 60s;
  .probe = basic;
}

/* Access Control Lists */
# See https://www.varnish-cache.org/docs/4.0/reference/vcl.html#access-control-list-acl
acl purge_ban {
  /* Simple access control list for allowing item purge for the self machine */
  "127.0.0.1"/32; // We can use '"localhost";' instead
}
acl allowed_monitors {
  /* Simple access control list for allowing item purge for the self machine */
  "127.0.0.1"/32; // We can use'"localhost";' instead
}
# acl own_proxys {
#   "127.0.0.1"/32; // We can use'"localhost";' instead
# }

/* Custom subroutines */
# See https://www.varnish-cache.org/docs/4.0/reference/vcl.html#subroutines
#TODO# Test in Varnihs 4
# Empty in simple configs.
# The only restriction naming subs is that the 'vlc_' prefix is reserverd for
# Varnish use. As a task can need several chunks of code in diferent states,
# it's a good idea to identify what main sub will call each with a suffix.
# /* Example 301 client redirection removing "www" prefix from request */
# sub perm_redirections_recv {
#   if ( req.http.host ~ "^www.*$" ) {
#     return (
#       synth(751, "http://" + regsub(req.http.host, "^www\.", "") + req.url)
#     );
#   }
# }
# sub perm_redirections_synth {
#   if ( resp.status == 751 ) {
#     /* Get new URL from the response */
#     set resp.http.Location = resp.reason;
#     /* Set HTTP 301 for permanent redirect */
#     set resp.status = 301;
#     set resp.reason = "Moved Permanently";
#     return (deliver);
#   }
# }


#######################################################################
# Client side

# vcl_recv: Called at the beginning of a request, after the complete request
# has been received and parsed. Its purpose is to decide whether or not to
# serve the request, how to do it, and, if applicable, which backend to use.
# See https://www.varnish-cache.org/docs/4.0/users-guide/vcl-built-in-subs.html#vcl-recv
sub vcl_recv {
  /* 0th: general bypass, general return & authorization checks */
  # Empty in simple configs.
  # Useful for debugging we can pipe or pass the request to default backend
  # here to bypass completely Varnish.
  # return (pipe);
  # return (pass);
  # We can also return here a 200 Ok for network performance benchmarking.
  # return (synth(200, "Ok"));
  # Finally we can perform basic HTTP authentification here, by example.
  # SeeV3 http://blog.tenya.me/blog/2011/12/14/varnish-http-authentication/

  /* 1st: Check for Varnish special requests */
  # Custom response implementation example in order to check that Varnish is
  # working properly.
  # This is usefull for automatic monitoring with monit or when Varnish is
  # behind another proxies like HAProxy.
  if ( ( req.http.host == "monitor.server.health"
      || req.http.host == "health.varnish" )
    && client.ip ~ allowed_monitors
    && ( req.method == "OPTIONS" || req.method == "GET" )
  ) {
    return (synth(200, "OK"));
  }
  # Purge logic
  # See https://www.varnish-cache.org/docs/4.0/users-guide/purging.html#http-purging
  # SeeV3 https://www.varnish-software.com/static/book/Cache_invalidation.html#removing-a-single-object
  if ( req.method == "PURGE" ) {
    if ( client.ip !~ purge_ban ) {
      return (synth(405, "Not allowed."));
    }
    return (purge);
  }
  # Ban logic
  # See https://www.varnish-cache.org/docs/4.0/users-guide/purging.html#bans
  if ( req.method == "BAN" ) {
    if ( client.ip !~ purge_ban ) {
      return (synth(405, "Not allowed."));
    }
    if (req.http.Purge-Cache-Tags) {
      ban(  "obj.http.X-Host == " + req.http.host +
        " && obj.http.Purge-Cache-Tags ~ " + req.http.Purge-Cache-Tags
        );
    }
    else {
      # Assumes req.url is a regex. This might be a bit too simple
      ban(  "obj.http.X-Host == " + req.http.host +
        " && obj.http.X-Url ~ " + req.url
        );
    }
    return (synth(200, "Ban added"));
  }

  /* 2nd: Do some Varnish black magic such as custom client redirections */
  # Empty in simple configs.
  # call perm_redirections_recv;
  # Here we can also enforce SSL when Varnish run behind some SSL termination
  # point.

  /* 3rd: Time for backend choice */
  # Empty in simple configs.

  /* 4th: Prepare request for the backend */
  # Empty in simple configs.
  # Example remove own_proxys from X-Forwarded-For
  # See https://www.varnish-cache.org/docs/4.0/whats-new/upgrading.html#x-forwarded-for-is-now-set-before-vcl-recv
  # Varnish 4 regsub doesn't accept anything but plain regexp, so we can't use
  # client.ip to exclude the proxy ips from the request:
  #   set req.http.X-Forwarded-For
  #     = regsub(req.http.X-Forwarded-For, ",( )?" + client.ip, "");
  # Instead, we need to add the proxy ips manually in the exclude list:
  # if ( req.restarts == 0
  #   && client.ip ~ own_proxys
  #   && req.http.X-Forwarded-For
  # ) {
  #   set req.http.X-Forwarded-For
  #     = regsub(req.http.X-Forwarded-For,
  #         "(, )?(10\.10\.10\.10|10\.11\.11\.11)", "");
  # }
  # An alternative could be to skip all this and try to modify the header
  # manually so Varnish doesn't touch it.
  # set req.http.X-Forwarded-For = req.http.X-Forwarded-For + "";
  #
  # Example normalize the host header, remove the port (in case you're testing
  # this on various TCP ports)
  # set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");

  /* 5th: Bypass breakpoint 1 */
  # Useful for debugging we can now pipe or pass the request to backend with
  # headers setted.
  # return (pipe);
  # return (pass);

  /* 6th: Decide if we should deal with a request (mostly from built-in logic) */
  if ( req.method == "PRI" ) {
    /* We do not support SPDY or HTTP/2.0 */
    return (synth(405));
  }
  if ( req.method != "GET"
    && req.method != "HEAD"
    && req.method != "PUT"
    && req.method != "POST"
    && req.method != "TRACE"
    && req.method != "OPTIONS"
    && req.method != "DELETE"
  ) {
    /* Non-RFC2616 or CONNECT which is weird. */
    return (pipe);
  }
  if ( req.method != "GET"
    && req.method != "HEAD"
  ) {
    /* We only deal with GET and HEAD by default */
    return (pass);
  }
  if ( req.http.Authorization ) {
    /* Not cacheable by default */
    return (pass);
  }
  # Websocket support
  # See https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html
  if ( req.http.Upgrade ~ "(?i)websocket" ) {
    return (pipe);
  }

  /* 7th: Access control for some URLs by ACL */
  # Empty in simple configs.
  # By example denial some URLs depending on client-ip, we'll need to define
  # corresponding ACL 'internal'.
  # if ( req.url ~ "^/(cron|install)\.php"
  #   && client.ip !~ internal
  # ) {
  #   # Have Varnish throw the error directly.
  #   return (synth(403, "Forbidden."));
  #   # Use a custom error page that you've defined in Drupal at the path "404".
  #   # set req.url = "/403";
  # }

  /* 8th: Custom exceptions */
  # Host exception example:
  # if ( req.http.host == "ejemplo.exception.com" ) {
  #     return (pass);
  # }
  # Drupal exceptions, edit if we want to cache some AJAX/AHAH request.
  # Add here filters for never cache URLs such as Payment Gateway's callbacks.
  if ( req.url ~ "^/status\.php$"
    || req.url ~ "^/update\.php$"
    || req.url ~ "^/ooyala/ping$"
    || req.url ~ "^/admin/build/features"
    || req.url ~ "^/info/.*$"
    || req.url ~ "^/flag/.*$"
    || req.url ~ "^.*/ajax/.*$"
    || req.url ~ "^.*/ahah/.*$"
  ) {
    /* Do not cache these paths */
    return (pass);
  }
  # Pipe these paths directly to backend for streaming.
  if ( req.url ~ "^/admin/content/backup_migrate/export"
    || req.url ~ "^/admin/config/system/backup_migrate"
  ) {
    return (pipe);
  }
  if ( req.url ~ "^/system/files" ) {
    return (pipe);
  }

  /* 9th: Graced objets & Serve from anonymous cahe if all backends are down */
  # See https://www.varnish-software.com/blog/grace-varnish-4-stale-while-revalidate-semantics-varnish
  # set req.http.X-Varnish-Grace = "none";
  if ( ! std.healthy(req.backend_hint) ) {
    # We must do this here since cookie hashing
    unset req.http.Cookie;
    #TODO# Add sick marker
  }

  /* 10th: Deal with compression and the Accept-Encoding header */
  # Althought Varnish 4 handles gziped content itself by default, just to be
  # sure we want to remove Accept-Encoding for some compressed formats.
  # See https://www.varnish-cache.org/docs/4.0/phk/gzip.html#what-does-http-gzip-support-do
  # See https://www.varnish-cache.org/docs/4.0/users-guide/compression.html
  # See https://www.varnish-cache.org/docs/4.0/reference/varnishd.html?highlight=http_gzip_support
  # See (for older configs) https://www.varnish-cache.org/trac/wiki/VCLExampleNormalizeAcceptEncoding
  if ( req.http.Accept-Encoding ) {
    if ( req.url ~ "(?i)\.(7z|avi|bz2|flv|gif|gz|jpe?g|mpe?g|mk[av]|mov|mp[34]|og[gm]|pdf|png|rar|swf|tar|tbz|tgz|woff2?|zip|xz)(\?.*)?$"
    ) {
      /* Already compressed formats, no sense trying to compress again */
      unset req.http.Accept-Encoding;
    }
  }

  /* 11th: Further request manipulation */
  # Empty in simple configs.
  # We could add here a custom header grouping User-agent families.
  # Generic URL manipulation.
  # Remove Google Analytics added parameters, useless for our backends.
  if ( req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=" ) {
    set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
    set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
    set req.url = regsub(req.url, "\?&", "?");
    set req.url = regsub(req.url, "\?$", "");
  }
  # Strip anchors, server doesn't need it.
  if ( req.url ~ "\#" ) {
    set req.url = regsub(req.url, "\#.*$", "");
  }
  # Strip a trailing ? if it exists
  if ( req.url ~ "\?$" ) {
    set req.url = regsub(req.url, "\?$", "");
  }
  # Normalize the querystring arguments
  set req.url = std.querysort(req.url);

  /* 12th: Cookie removal */
  # Always cache the following static file types for all users.
  # Use with care if we control certain downloads depending on cookies.
  # Be carefull also if appending .htm[l] via Drupal's clean URLs.
  if ( req.url ~ "(?i)\.(bz2|css|eot|gif|gz|html?|ico|jpe?g|js|mp3|ogg|otf|pdf|png|rar|svg|swf|tbz|tgz|ttf|woff2?|zip)(\?(itok=)?[a-z0-9_=\.\-]+)?$"
    && req.url !~ "/system/storage/serve"
  ) {
      unset req.http.Cookie;
  }
  # Remove all cookies that backend doesn't need to know about.
  # See https://www.varnish-cache.org/trac/wiki/VCLExampleRemovingSomeCookies
  if ( req.http.Cookie ) {
    /* Warning: Not a pretty solution */
    # Prefix header containing cookies with ';'
    set req.http.Cookie = ";" + req.http.Cookie;
    # Remove any spaces after ';' in header containing cookies
    set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
    # Prefix cookies we want to preserve with one space:
    #   'S{1,2}ESS[a-z0-9]+' is the regular expression matching a Drupal session
    #   cookie ({1,2} added for HTTPS support).
    #   'NO_CACHE' is usually set after a POST request to make sure issuing user
    #   see the results of his post.
    #   'OATMEAL' & 'CHOCOLATECHIP' are special cookies used by Drupal's Bakery
    #   module to provide Single Sign On.
    # Keep in mind we should add here any cookie that should reach the backend
    # such as splash avoiding cookies.
    set req.http.Cookie
      = regsuball(
          req.http.Cookie,
          ";(S{1,2}ESS[a-z0-9]+|NO_CACHE|OATMEAL|CHOCOLATECHIP)=",
          "; \1="
        );
    # Remove from the header any single Cookie not prefixed with a space until
    # next ';' separator.
    set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
    # Remove any '; ' at the start or the end of the header.
    set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
    #If there are no remaining cookies, remove the cookie header.
    if ( req.http.Cookie == "" ) {
      unset req.http.Cookie;
    }
  }

  /* 13th: Session cookie & special cookies bypass caching stage */
  # As we might want to cache some requests, hashed with its cookies, we don't
  # simply pass when some cookies remain present at this point.
  # Instead we look for request that must be passed due to the cookie header.
  if ( req.http.Cookie ~ "SESS"
    || req.http.Cookie ~ "SSESS"
    || req.http.Cookie ~ "NO_CACHE"
    || req.http.Cookie ~ "OATMEAL"
    || req.http.Cookie ~ "CHOCOLATECHIP"
  ) {
    return (pass);
  }<commit_msg>improve BigPipe support, take 2<commit_after>/*
 * Varnish 4 example config for Drupal 7 & 8 / Pressflow 6 & 7
 */

# Original source: https://github.com/NITEMAN/varnish-bites/varnish4/drupal-base.vcl
# Copyright (c) 2015 <NAME> and individual contributors.
# MIT License

# Intended to be used both in simple production environments and with
# learning/teaching purposes.
# Some references may not be yet updated to VCL 4.0, we use SeeV3 then.

# WARNING:
# Note that the built-in logic will be appended to our code if no return is
# performed before.
# Built-in logic is included commented out right after our's for reference
# purposes in that cases.
# See https://www.varnish-cache.org/trac/wiki/VCLExampleDefault

#######################################################################
# Initialization: Version & imports;

/* Version statement */
# Since Varnish 4.0 it's mandatory to declare VCL version on first line.
vcl 4.0;

/* Module (VMOD) imports */
# Standard module
# See https://www.varnish-cache.org/docs/4.0/reference/vmod_std.generated.html
import std;

# Directors module
# See https://www.varnish-cache.org/docs/4.0/reference/vmod_directors.generated.html
# Unused in simple configs.
#import directors;


#######################################################################
# Probe, backend, ACL and subroutine definitions

/* Backend probes / healthchecks */
# See https://www.varnish-cache.org/docs/4.0/reference/vcl.html#probes
probe basic {
  /* Only test that backend's IP serves content for '/' */
  # This might be a too heavy probe
  # .url = "/";

  /* Only test that backend's IP has apache working */
  # Nginx would fail this probe with a default config
  .request =
    "OPTIONS * HTTP/1.1"
    "Host: *"
    "Connection: close";

  /* Common options */
  .interval = 10s;
  .timeout = 2s;
  .window = 8;
  .threshold = 6;
}

/* Backend definitions.*/
# See https://www.varnish-cache.org/docs/4.0/reference/vcl.html#backend-definition
backend default {
  /* Default backend on the same machine. */
  # WARNING: timeouts could be not big enought for certain POST requests.
  .host = "127.0.0.1";
  .port = "8008";
  .max_connections = 100;
  .connect_timeout = 60s;
  .first_byte_timeout = 60s;
  .between_bytes_timeout = 60s;
  .probe = basic;
}

/* Access Control Lists */
# See https://www.varnish-cache.org/docs/4.0/reference/vcl.html#access-control-list-acl
acl purge_ban {
  /* Simple access control list for allowing item purge for the self machine */
  "127.0.0.1"/32; // We can use '"localhost";' instead
}
acl allowed_monitors {
  /* Simple access control list for allowing item purge for the self machine */
  "127.0.0.1"/32; // We can use'"localhost";' instead
}
# acl own_proxys {
#   "127.0.0.1"/32; // We can use'"localhost";' instead
# }

/* Custom subroutines */
# See https://www.varnish-cache.org/docs/4.0/reference/vcl.html#subroutines
#TODO# Test in Varnihs 4
# Empty in simple configs.
# The only restriction naming subs is that the 'vlc_' prefix is reserverd for
# Varnish use. As a task can need several chunks of code in diferent states,
# it's a good idea to identify what main sub will call each with a suffix.
# /* Example 301 client redirection removing "www" prefix from request */
# sub perm_redirections_recv {
#   if ( req.http.host ~ "^www.*$" ) {
#     return (
#       synth(751, "http://" + regsub(req.http.host, "^www\.", "") + req.url)
#     );
#   }
# }
# sub perm_redirections_synth {
#   if ( resp.status == 751 ) {
#     /* Get new URL from the response */
#     set resp.http.Location = resp.reason;
#     /* Set HTTP 301 for permanent redirect */
#     set resp.status = 301;
#     set resp.reason = "Moved Permanently";
#     return (deliver);
#   }
# }


#######################################################################
# Client side

# vcl_recv: Called at the beginning of a request, after the complete request
# has been received and parsed. Its purpose is to decide whether or not to
# serve the request, how to do it, and, if applicable, which backend to use.
# See https://www.varnish-cache.org/docs/4.0/users-guide/vcl-built-in-subs.html#vcl-recv
sub vcl_recv {
  /* 0th: general bypass, general return & authorization checks */
  # Empty in simple configs.
  # Useful for debugging we can pipe or pass the request to default backend
  # here to bypass completely Varnish.
  # return (pipe);
  # return (pass);
  # We can also return here a 200 Ok for network performance benchmarking.
  # return (synth(200, "Ok"));
  # Finally we can perform basic HTTP authentification here, by example.
  # SeeV3 http://blog.tenya.me/blog/2011/12/14/varnish-http-authentication/

  /* 1st: Check for Varnish special requests */
  # Custom response implementation example in order to check that Varnish is
  # working properly.
  # This is usefull for automatic monitoring with monit or when Varnish is
  # behind another proxies like HAProxy.
  if ( ( req.http.host == "monitor.server.health"
      || req.http.host == "health.varnish" )
    && client.ip ~ allowed_monitors
    && ( req.method == "OPTIONS" || req.method == "GET" )
  ) {
    return (synth(200, "OK"));
  }
  # Purge logic
  # See https://www.varnish-cache.org/docs/4.0/users-guide/purging.html#http-purging
  # SeeV3 https://www.varnish-software.com/static/book/Cache_invalidation.html#removing-a-single-object
  if ( req.method == "PURGE" ) {
    if ( client.ip !~ purge_ban ) {
      return (synth(405, "Not allowed."));
    }
    return (purge);
  }
  # Ban logic
  # See https://www.varnish-cache.org/docs/4.0/users-guide/purging.html#bans
  if ( req.method == "BAN" ) {
    if ( client.ip !~ purge_ban ) {
      return (synth(405, "Not allowed."));
    }
    if (req.http.Purge-Cache-Tags) {
      ban(  "obj.http.X-Host == " + req.http.host +
        " && obj.http.Purge-Cache-Tags ~ " + req.http.Purge-Cache-Tags
        );
    }
    else {
      # Assumes req.url is a regex. This might be a bit too simple
      ban(  "obj.http.X-Host == " + req.http.host +
        " && obj.http.X-Url ~ " + req.url
        );
    }
    return (synth(200, "Ban added"));
  }

  /* 2nd: Do some Varnish black magic such as custom client redirections */
  # Empty in simple configs.
  # call perm_redirections_recv;
  # Here we can also enforce SSL when Varnish run behind some SSL termination
  # point.

  /* 3rd: Time for backend choice */
  # Empty in simple configs.

  /* 4th: Prepare request for the backend */
  # Empty in simple configs.
  # Example remove own_proxys from X-Forwarded-For
  # See https://www.varnish-cache.org/docs/4.0/whats-new/upgrading.html#x-forwarded-for-is-now-set-before-vcl-recv
  # Varnish 4 regsub doesn't accept anything but plain regexp, so we can't use
  # client.ip to exclude the proxy ips from the request:
  #   set req.http.X-Forwarded-For
  #     = regsub(req.http.X-Forwarded-For, ",( )?" + client.ip, "");
  # Instead, we need to add the proxy ips manually in the exclude list:
  # if ( req.restarts == 0
  #   && client.ip ~ own_proxys
  #   && req.http.X-Forwarded-For
  # ) {
  #   set req.http.X-Forwarded-For
  #     = regsub(req.http.X-Forwarded-For,
  #         "(, )?(10\.10\.10\.10|10\.11\.11\.11)", "");
  # }
  # An alternative could be to skip all this and try to modify the header
  # manually so Varnish doesn't touch it.
  # set req.http.X-Forwarded-For = req.http.X-Forwarded-For + "";
  #
  # Example normalize the host header, remove the port (in case you're testing
  # this on various TCP ports)
  # set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");

  /* 5th: Bypass breakpoint 1 */
  # Useful for debugging we can now pipe or pass the request to backend with
  # headers setted.
  # return (pipe);
  # return (pass);

  /* 6th: Decide if we should deal with a request (mostly from built-in logic) */
  if ( req.method == "PRI" ) {
    /* We do not support SPDY or HTTP/2.0 */
    return (synth(405));
  }
  if ( req.method != "GET"
    && req.method != "HEAD"
    && req.method != "PUT"
    && req.method != "POST"
    && req.method != "TRACE"
    && req.method != "OPTIONS"
    && req.method != "DELETE"
  ) {
    /* Non-RFC2616 or CONNECT which is weird. */
    return (pipe);
  }
  if ( req.method != "GET"
    && req.method != "HEAD"
  ) {
    /* We only deal with GET and HEAD by default */
    return (pass);
  }
  if ( req.http.Authorization ) {
    /* Not cacheable by default */
    return (pass);
  }
  # Websocket support
  # See https://www.varnish-cache.org/docs/4.0/users-guide/vcl-example-websockets.html
  if ( req.http.Upgrade ~ "(?i)websocket" ) {
    return (pipe);
  }

  /* 7th: Access control for some URLs by ACL */
  # Empty in simple configs.
  # By example denial some URLs depending on client-ip, we'll need to define
  # corresponding ACL 'internal'.
  # if ( req.url ~ "^/(cron|install)\.php"
  #   && client.ip !~ internal
  # ) {
  #   # Have Varnish throw the error directly.
  #   return (synth(403, "Forbidden."));
  #   # Use a custom error page that you've defined in Drupal at the path "404".
  #   # set req.url = "/403";
  # }

  /* 8th: Custom exceptions */
  # Host exception example:
  # if ( req.http.host == "ejemplo.exception.com" ) {
  #     return (pass);
  # }
  # Drupal exceptions, edit if we want to cache some AJAX/AHAH request.
  # Add here filters for never cache URLs such as Payment Gateway's callbacks.
  if ( req.url ~ "^/status\.php$"
    || req.url ~ "^/update\.php$"
    || req.url ~ "^/ooyala/ping$"
    || req.url ~ "^/admin/build/features"
    || req.url ~ "^/info/.*$"
    || req.url ~ "^/flag/.*$"
    || req.url ~ "^.*/ajax/.*$"
    || req.url ~ "^.*/ahah/.*$"
  ) {
    /* Do not cache these paths */
    return (pass);
  }
  # Pipe these paths directly to backend for streaming.
  if ( req.url ~ "^/admin/content/backup_migrate/export"
    || req.url ~ "^/admin/config/system/backup_migrate"
  ) {
    return (pipe);
  }
  if ( req.url ~ "^/system/files" ) {
    return (pipe);
  }

  /* 9th: Graced objets & Serve from anonymous cahe if all backends are down */
  # See https://www.varnish-software.com/blog/grace-varnish-4-stale-while-revalidate-semantics-varnish
  # set req.http.X-Varnish-Grace = "none";
  if ( ! std.healthy(req.backend_hint) ) {
    # We must do this here since cookie hashing
    unset req.http.Cookie;
    #TODO# Add sick marker
  }

  /* 10th: Deal with compression and the Accept-Encoding header */
  # Althought Varnish 4 handles gziped content itself by default, just to be
  # sure we want to remove Accept-Encoding for some compressed formats.
  # See https://www.varnish-cache.org/docs/4.0/phk/gzip.html#what-does-http-gzip-support-do
  # See https://www.varnish-cache.org/docs/4.0/users-guide/compression.html
  # See https://www.varnish-cache.org/docs/4.0/reference/varnishd.html?highlight=http_gzip_support
  # See (for older configs) https://www.varnish-cache.org/trac/wiki/VCLExampleNormalizeAcceptEncoding
  if ( req.http.Accept-Encoding ) {
    if ( req.url ~ "(?i)\.(7z|avi|bz2|flv|gif|gz|jpe?g|mpe?g|mk[av]|mov|mp[34]|og[gm]|pdf|png|rar|swf|tar|tbz|tgz|woff2?|zip|xz)(\?.*)?$"
    ) {
      /* Already compressed formats, no sense trying to compress again */
      unset req.http.Accept-Encoding;
    }
  }

  /* 11th: Further request manipulation */
  # Empty in simple configs.
  # We could add here a custom header grouping User-agent families.
  # Generic URL manipulation.
  # Remove Google Analytics added parameters, useless for our backends.
  if ( req.url ~ "(\?|&)(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=" ) {
    set req.url = regsuball(req.url, "&(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "");
    set req.url = regsuball(req.url, "\?(utm_source|utm_medium|utm_campaign|utm_content|gclid|cx|ie|cof|siteurl)=([A-z0-9_\-\.%25]+)", "?");
    set req.url = regsub(req.url, "\?&", "?");
    set req.url = regsub(req.url, "\?$", "");
  }
  # Strip anchors, server doesn't need it.
  if ( req.url ~ "\#" ) {
    set req.url = regsub(req.url, "\#.*$", "");
  }
  # Strip a trailing ? if it exists
  if ( req.url ~ "\?$" ) {
    set req.url = regsub(req.url, "\?$", "");
  }
  # Normalize the querystring arguments
  set req.url = std.querysort(req.url);

  /* 12th: Cookie removal */
  # Always cache the following static file types for all users.
  # Use with care if we control certain downloads depending on cookies.
  # Be carefull also if appending .htm[l] via Drupal's clean URLs.
  if ( req.url ~ "(?i)\.(bz2|css|eot|gif|gz|html?|ico|jpe?g|js|mp3|ogg|otf|pdf|png|rar|svg|swf|tbz|tgz|ttf|woff2?|zip)(\?(itok=)?[a-z0-9_=\.\-]+)?$"
    && req.url !~ "/system/storage/serve"
  ) {
      unset req.http.Cookie;
  }
  # Remove all cookies that backend doesn't need to know about.
  # See https://www.varnish-cache.org/trac/wiki/VCLExampleRemovingSomeCookies
  if ( req.http.Cookie ) {
    /* Warning: Not a pretty solution */
    # Prefix header containing cookies with ';'
    set req.http.Cookie = ";" + req.http.Cookie;
    # Remove any spaces after ';' in header containing cookies
    set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
    # Prefix cookies we want to preserve with one space:
    #   'S{1,2}ESS[a-z0-9]+' is the regular expression matching a Drupal session
    #   cookie ({1,2} added for HTTPS support).
    #   'NO_CACHE' is usually set after a POST request to make sure issuing user
    #   see the results of his post.
    #   'OATMEAL' & 'CHOCOLATECHIP' are special cookies used by Drupal's Bakery
    #   module to provide Single Sign On.
    # Keep in mind we should add here any cookie that should reach the backend
    # such as splash avoiding cookies.
    set req.http.Cookie
      = regsuball(
          req.http.Cookie,
          ";(S{1,2}ESS[a-z0-9]+|NO_CACHE|OATMEAL|CHOCOLATECHIP|big_pipe_nojs)=",
          "; \1="
        );
    # Remove from the header any single Cookie not prefixed with a space until
    # next ';' separator.
    set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
    # Remove any '; ' at the start or the end of the header.
    set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
    #If there are no remaining cookies, remove the cookie header.
    if ( req.http.Cookie == "" ) {
      unset req.http.Cookie;
    }
  }

  /* 13th: Session cookie & special cookies bypass caching stage */
  # As we might want to cache some requests, hashed with its cookies, we don't
  # simply pass when some cookies remain present at this point.
  # Instead we look for request that must be passed due to the cookie header.
  if ( req.http.Cookie ~ "SESS"
    || req.http.Cookie ~ "SSESS"
    || req.http.Cookie ~ "NO_CACHE"
    || req.http.Cookie ~ "OATMEAL"
    || req.http.Cookie ~ "CHOCOLATECHIP"
  ) {
    return (pass);
  }
{'fraction_non_alphanumeric': 0.15801186943620177, 'fraction_numerical': 0.015342595090369572, 'mean_word_length': 3.0082443573455873, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 40, 'lorem ipsum': 0, 'www.': 44, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4592782', 'n_tokens_mistral': 10624, 'n_tokens_neox': 10033, 'n_words': 2842}