<commit_before># NOTE: this file is managed by puppet
#

import std;

#Import file with probe definitions
include "includes/probes.vcl";

#Import file with backend definitions
include "includes/backends.vcl";

#Import file with director definitions;
include "includes/directors.vcl";

#Import file with acl definitions
include "includes/acls.vcl";

sub vcl_recv {
  
  # Force look-up if request is a no-cache request.     
  if (req.http.Cache-Control ~ "no-cache") {
    return (pass);
  }

<%- if @defaultgrace -%>
  # Default grace period
  set req.grace = <%= @defaultgrace %>;
<%- end -%>

  unset req.http.If-Modified-Since;
  unset req.http.If-None-Match;
 
  # Modify HTTP X-Forwarded-For header.
  # This will replace Varnish's IP with actual client's.
  remove req.http.X-Forwarded-For;
  set    req.http.X-Forwarded-For = client.ip;
 
  # Remove all cookies that Drupal doesn't need to know about. ANY remaining
  # cookie will cause the request to pass-through to a backend. For the most part
  # we always set the NO_CACHE cookie after any POST request, disabling the
  # Varnish cache temporarily. The session cookie allows all authenticated users
  # to pass through as long as they're logged in.
  #
  # 1. Append a semi-colon to the front of the cookie string.
  # 2. Remove all spaces that appear after semi-colons.
  # 3. Match the cookies we want to keep, adding the space we removed
  #    previously, back. (\1) is first matching group in the regsuball.
  # 4. Remove all other cookies, identifying them by the fact that they have
  #    no space after the preceding semi-colon.
  # 5. Remove all spaces and semi-colons from the beginning and end of the
  #    cookie string.
  if (!(req.http.Content-Type ~ "multipart/form-data")) {
    if (req.http.Cookie) {
      set req.http.Cookie = ";" + req.http.Cookie;
      set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
      set req.http.Cookie = regsuball(req.http.Cookie, ";(<%- @cookiekeeps.each do |cookie| -%><%= cookie %><%- if cookie != @cookiekeeps[-1] -%>|<%-end-%><%-end-%>)=", "; \1=");
      set req.http.Cookie = regsuball(req.http.Cookie, ";(S{1,2}ESS[a-z0-9]+|NO_CACHE)=", "; \1=");
      set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
      set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");

      if (req.http.Cookie == "") {
        # If there are no remaining cookies, remove the cookie header. If there
        # aren't any cookie headers, Varnish's default behavior will be to cache
        # the page.
        unset req.http.Cookie;
      }
    }

    # Always cache the following file types for all users.
    if (req.url ~ "(?i)\.(html|htm|jpg|jpeg|gif|png|tiff|tif|svg|swf|ico|css|kss|js|vsd|doc|ppt|pps|xls|pdf|mp3|mp4|m4a|ogg|mov|avi|wmv|sxw|zip|gz|bz2|tar|rar|odc|odb|odf|odg|odi|odp|ods|odt|sxc|sxd|sxi|sxw|dmg|torrent|deb|msi|iso|rpm|jar|class|flv|exe)(\?[a-z0-9]+)?$") {
      remove req.http.Cookie;
    }
  }
    
  # Handling of different encoding types.
  if (req.http.Accept-Encoding) {
    if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
      remove req.http.Accept-Encoding;
    }
    elsif (req.http.Accept-Encoding ~ "gzip") {
      set req.http.Accept-Encoding = "gzip";
    }
    elsif (req.http.Accept-Encoding ~ "deflate") {
      set req.http.Accept-Encoding = "deflate";
    }
    else {
      remove req.http.Accept-Encoding;
    }
  }  

  # Blocked list
  if ((client.ip ~ blockedips)<%- if @blockedbots.length > 0 -%>||(req.http.User-Agent ~ "(?i)<%- @blockedbots.each do |bot| -%><%= bot %><%- if cookie != @blockedbots[-1] -%>|<%-end-%><%-end-%>")<%-end-%>) {
    error 403 "Access forbidden.";
  }

  # Web Application Firewall
  # These rules are defined by a number between semicolons. Example: ;48;
  # All expections must be defined at the req.http.EXCE variable
  # DO NOT MODIFY waf.vcl FILE!
  
  set req.http.EXCE = ";<%- @wafexceptions.each do |exce| -%><%= exce %>;<%-end-%>";
  set req.http.RULE = "";

  include "includes/waf.vcl";
  if (!(req.http.RULE == "")) {
    std.log( "ATTACK SIGNATURE rules:"+req.http.RULE+" url:"+req.url+" COOKIE:"+req.http.Cookie+"   IP:"+client.ip);
    error 403 "ATTACK DETECTED - IP LOGGED";
  }
  remove req.http.EXCE;
  remove req.http.RULE;

  # backend selection logic
  include "includes/backendselection.vcl";

  # Allows purge for the IPs in purge ACL
  if (req.request == "PURGE") {
    if (!(client.ip ~ purge)) {
        error 405 "Not allowed.";
    }
    ban(" req.url == " + req.url);
    set req.url = regsuball(req.url,"80","443"); 
    set req.url = regsuball(req.url,"http","https");
    ban(" req.url == " + req.url);
    error 200 "Added ban.";      
  }

  # Ignore anything but GET and HEAD requests.
  if (req.request != "GET" && req.request != "HEAD") {
    return (pass);
  }
 
  #-----------------------------------------------------------------------------
  # Drupal specific settings

  # Modify (remove) progress.js request parameters.
  if (req.url ~ "^/misc/progress\.js\?[0-9]+$") {
    set req.url = "/misc/progress.js";
  }

  # Do not allow outside access to cron.php, update.php, or install.php.  
  if (req.url ~ "^/(cron|install|update)\.php$" && !client.ip ~ internal) {        
    error 404 "Page not found.";
  }
  
  # Cache static content of themes and files.  
  if (req.url ~ "^/themes/" && req.url ~ ".(css|js|png|gif|jp(e)?g)") {
    unset req.http.Cookie;
  }
  if (req.url ~ "^/sites/") {
    unset req.http.Cookie;
  }
  
  # Catch Drupal misc files (like drupal.js and jquery.js)
  if (req.url ~ "^/misc/") {
    unset req.http.Cookie;
  } 
  # Drupal js/css doesn’t need cookies, cache them
  if (req.url ~ "^/modules/.*\.(js|css)\?") {
    unset req.http.Cookie;
  }
  
  # Do not cache these URL paths.
  if (req.url ~ "^/status\.php$" ||
      req.url ~ "^/update\.php$" ||
      req.url ~ "^/admin/build/features" ||
      req.url ~ "^/cron\.php$" ||
      req.url ~ "/edit/.*$" ||
      req.url ~ "^/admin" ||
      req.url ~ "^/admin/.*$" ||
      req.url ~ "^/user/.*$" ||
      req.url ~ "^/info/.*$" ||
      req.url ~ "^/flag/.*$" ||
      req.url ~ "^.*/ajax/.*$" ||
      req.url ~ "^.*/ahah/.*$") {
      
       return (pass);
  }

  # End Drupal
  #-----------------------------------------------------------------------------
  <commit_msg>Fixing a few issues in the varnish VCL template.<commit_after># NOTE: this file is managed by puppet
#
import std;

#Import file with probe definitions
include "includes/probes.vcl";

#Import file with backend definitions
include "includes/backends.vcl";

#Import file with director definitions;
include "includes/directors.vcl";

#Import file with acl definitions
include "includes/acls.vcl";

sub vcl_recv {

<%- if @defaultgrace -%>
  # Default grace period
  set req.grace = <%= @defaultgrace %>;
<%- end -%>

  unset req.http.If-Modified-Since;
  unset req.http.If-None-Match;
 
  # Modify HTTP X-Forwarded-For header.
  # This will replace Varnish's IP with actual client's.
  remove req.http.X-Forwarded-For;
  set    req.http.X-Forwarded-For = client.ip;
 
  # Remove all cookies that Drupal doesn't need to know about. ANY remaining
  # cookie will cause the request to pass-through to a backend. For the most part
  # we always set the NO_CACHE cookie after any POST request, disabling the
  # Varnish cache temporarily. The session cookie allows all authenticated users
  # to pass through as long as they're logged in.
  #
  # 1. Append a semi-colon to the front of the cookie string.
  # 2. Remove all spaces that appear after semi-colons.
  # 3. Match the cookies we want to keep, adding the space we removed
  #    previously, back. (\1) is first matching group in the regsuball.
  # 4. Remove all other cookies, identifying them by the fact that they have
  #    no space after the preceding semi-colon.
  # 5. Remove all spaces and semi-colons from the beginning and end of the
  #    cookie string.
  if (!(req.http.Content-Type ~ "multipart/form-data")) {
    if (req.http.Cookie) {
      set req.http.Cookie = ";" + req.http.Cookie;
      set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
      set req.http.Cookie = regsuball(req.http.Cookie, ";(<%- @cookiekeeps.each do |cookie| -%><%= cookie %><%- if cookie != @cookiekeeps[-1] -%>|<%-end-%><%-end-%>)=", "; \1=");
      set req.http.Cookie = regsuball(req.http.Cookie, ";(S{1,2}ESS[a-z0-9]+|NO_CACHE)=", "; \1=");
      set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
      set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");

      if (req.http.Cookie == "") {
        # If there are no remaining cookies, remove the cookie header. If there
        # aren't any cookie headers, Varnish's default behavior will be to cache
        # the page.
        unset req.http.Cookie;
      }
    }

    # Always cache the following file types for all users.
    if (req.url ~ "(?i)\.(html|htm|jpg|jpeg|gif|png|tiff|tif|svg|swf|ico|css|kss|js|vsd|doc|ppt|pps|xls|pdf|mp3|mp4|m4a|ogg|mov|avi|wmv|sxw|zip|gz|bz2|tar|rar|odc|odb|odf|odg|odi|odp|ods|odt|sxc|sxd|sxi|sxw|dmg|torrent|deb|msi|iso|rpm|jar|class|flv|exe)(\?[a-z0-9]+)?$") {
      remove req.http.Cookie;
    }
  }
    
  # Handling of different encoding types.
  if (req.http.Accept-Encoding) {
    if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
      remove req.http.Accept-Encoding;
    }
    elsif (req.http.Accept-Encoding ~ "gzip") {
      set req.http.Accept-Encoding = "gzip";
    }
    elsif (req.http.Accept-Encoding ~ "deflate") {
      set req.http.Accept-Encoding = "deflate";
    }
    else {
      remove req.http.Accept-Encoding;
    }
  }  

  # Blocked list
  if ((client.ip ~ blockedips)<%- if @blockedbots.length > 0 -%>||(req.http.User-Agent ~ "(?i)<%- @blockedbots.each do |bot| -%><%= bot %><%- if cookie != @blockedbots[-1] -%>|<%-end-%><%-end-%>")<%-end-%>) {
    error 403 "Access forbidden.";
  }

  # Web Application Firewall
  # These rules are defined by a number between semicolons. Example: ;48;
  # All expections must be defined at the req.http.EXCE variable
  # DO NOT MODIFY waf.vcl FILE!
  
  set req.http.EXCE = ";<%- @wafexceptions.each do |exce| -%><%= exce %>;<%-end-%>";
  set req.http.RULE = "";

  include "includes/waf.vcl";
  if (!(req.http.RULE == "")) {
    std.log( "ATTACK SIGNATURE rules:"+req.http.RULE+" url:"+req.url+" COOKIE:"+req.http.Cookie+"   IP:"+client.ip);
    error 403 "ATTACK DETECTED - IP LOGGED";
  }
  remove req.http.EXCE;
  remove req.http.RULE;

  # backend selection logic
  include "includes/backendselection.vcl";

  # Allows purge for the IPs in purge ACL
  if (req.request == "PURGE") {
    if (!(client.ip ~ purge)) {
        error 405 "Not allowed.";
    }
    ban(" req.url == " + req.url);
    set req.url = regsuball(req.url,"80","443"); 
    set req.url = regsuball(req.url,"http","https");
    ban(" req.url == " + req.url);
    error 200 "Added ban.";      
  }

  # Ignore anything but GET and HEAD requests.
  if (req.request != "GET" && req.request != "HEAD") {
    return (pass);
  }
 
  #-----------------------------------------------------------------------------
  # Drupal specific settings

  # Modify (remove) progress.js request parameters.
  if (req.url ~ "^/misc/progress\.js\?[0-9]+$") {
    set req.url = "/misc/progress.js";
  }

  # Do not allow outside access to cron.php, update.php, or install.php.  
  if (req.url ~ "^/(cron|install|update)\.php$" && !client.ip ~ internal) {        
    error 404 "Page not found.";
  }
  
  # Cache static content of themes and files.  
  if (req.url ~ "^/themes/" && req.url ~ ".(css|js|png|gif|jp(e)?g)") {
    unset req.http.Cookie;
  }
  if (req.url ~ "^/sites/") {
    unset req.http.Cookie;
  }
  
  # Catch Drupal misc files (like drupal.js and jquery.js)
  if (req.url ~ "^/misc/") {
    unset req.http.Cookie;
  } 
  # Drupal js/css doesn’t need cookies, cache them
  if (req.url ~ "^/modules/.*\.(js|css)\?") {
    unset req.http.Cookie;
  }
  
  # Do not cache these URL paths.
  if (req.url ~ "^/status\.php$" ||
      req.url ~ "^/update\.php$" ||
      req.url ~ "^/admin/build/features" ||
      req.url ~ "^/cron\.php$" ||
      req.url ~ "/edit/.*$" ||
      req.url ~ "^/admin" ||
      req.url ~ "^/admin/.*$" ||
      req.url ~ "^/user/.*$" ||
      req.url ~ "^/info/.*$" ||
      req.url ~ "^/flag/.*$" ||
      req.url ~ "^.*/ajax/?.*$" ||
      req.url ~ "^.*/ahah/?.*$") {
      
       return (pass);
  }

  # End Drupal
  #-----------------------------------------------------------------------------
  
{'fraction_non_alphanumeric': 0.19865079365079366, 'fraction_numerical': 0.0074603174603174605, 'mean_word_length': 2.55859926574414, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7455396', 'n_tokens_mistral': 4672, 'n_tokens_neox': 4425, 'n_words': 1127}