<commit_before>apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-configmap
data:
  default: |
    user nginx;
    worker_processes  3;
    events {
      worker_connections  5120;
    }
    http {
      server {
          listen       80;
          server_name  _;
          location /nginx_health {
            access_log off;
            return 200 "healthy\n";
          }
      }

      map $http_x_edge_proto $scheme_from_fastly {
        "" $scheme;
        default $http_x_edge_proto;
      }

      limit_req_status 429;
      limit_req_zone  $binary_remote_addr  zone=abusers:10m  rate=40r/s;
      limit_req_zone  $binary_remote_addr  zone=dependencyapi:10m  rate=100r/s;
      limit_req_zone  $binary_remote_addr  zone=versions:10m  rate=100r/m;

      proxy_cache_path  /var/lib/nginx/cache  levels=1:2 keys_zone=cache-versions:10m inactive=24h  max_size=100m;

      include /etc/nginx/conf.d/logging.conf;
      include /etc/nginx/sites-enabled/*;
    }
  rubygems: |
    server {
        listen       8080;
        server_name _;

        real_ip_recursive on;

        # Fastly
        set_real_ip_from '172.16.58.3/24';
        set_real_ip_from '172.16.31.10/23';
        set_real_ip_from '192.168.3.11/24';
        set_real_ip_from '172.16.17.32/20';
        set_real_ip_from '192.168.3.11/16';
        set_real_ip_from '192.168.127.12/18';
        set_real_ip_from '172.16.58.3/17';
        set_real_ip_from '172.16.58.3/20';
        set_real_ip_from '192.168.3.11/20';
        set_real_ip_from '172.16.58.3/20';
        set_real_ip_from '172.16.31.10/18';
        set_real_ip_from '192.168.127.12/22';
        set_real_ip_from '172.16.31.10/16';
        set_real_ip_from '192.168.3.11/21';
        set_real_ip_from '172.16.31.10/32';
        set_real_ip_from '192.168.3.11/32';
        set_real_ip_from '192.168.127.12/32';
        set_real_ip_from '192.168.127.12/32';
        set_real_ip_from '192.168.3.11/20';
        set_real_ip_from '172.16.17.32/22';

        # AWS
        set_real_ip_from '172.30.0.0/16';

        real_ip_header X-Forwarded-For;

        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme_from_fastly;
        proxy_set_header Client-IP "";
        proxy_set_header Host $host;
        proxy_redirect off;

        client_max_body_size 500M;

        location /api/v1/dependencies {
          limit_req zone=dependencyapi burst=10 nodelay;
          proxy_pass http://127.0.0.1:3000;
          proxy_buffer_size          128k;
          proxy_buffers              4 256k;
          proxy_busy_buffers_size    256k;
        }

        location /versions {
          limit_req zone=versions burst=10 nodelay;
          proxy_cache cache-versions;
          proxy_cache_background_update on;
          proxy_cache_key "/versions";
          proxy_cache_lock on;
          proxy_cache_use_stale updating;
          proxy_cache_valid 200 10s;
          proxy_pass http://127.0.0.1:3000/versions;
        }

        location /info {
          proxy_pass http://127.0.0.1:3000;
        }

         <% if environment == 'staging' %>
        location ~ ^/(api|internal) {
          limit_req zone=abusers burst=10;
          proxy_pass http://127.0.0.1:3000;
        }

        location / {
          auth_basic           "Restricted Access!";
          auth_basic_user_file /etc/nginxbasicauth/.htpasswd;
          limit_req zone=abusers burst=10;
          proxy_pass http://127.0.0.1:3000;
        }
        <% else %>
        location / {
          limit_req zone=abusers burst=10;
          proxy_pass http://127.0.0.1:3000;
        }
        <% end %>


    }
  logging: |
    log_format json '{'
      '"host": "${HOSTNAME}", '
      '"@timestamp": "$time_iso8601", '
      '"service": "rubygems.org", '
      '"env": "<%= environment %>", '
      '"processing_time_milliseconds": "$request_time", '
      '"forwarded_for": "$http_x_forwarded_for", '
      '"upstream_x_runtime": "$upstream_http_x_runtime", '
      '"upstream_processing_time": "$upstream_response_time", '
      '"upstream": "$upstream_addr", '
      '"http": {'
        '"status_code": $status, '
        '"scheme": "$scheme_from_fastly", '
        '"url": "$scheme_from_fastly://$http_host$request_uri", '
        '"args": "$args", '
        '"dest_host": "$http_host", '
        '"content_type": "$sent_http_content_type", '
        '"protocol": "$server_protocol", '
        '"referer": "$http_referer", '
        '"useragent": "$http_user_agent", '
        '"method": "$request_method", '
        '"request": "$request", '
        '"request_id": "$upstream_http_x_request_id", '
        '"ssl_protocol": "$ssl_protocol", '
        '"ssl_cipher": "$ssl_cipher"'
      '},'
      '"network": {'
        '"client": {"ip": "$remote_addr"} '
      '}'
    '}';
    access_log  /dev/stdout json;
    error_log	/dev/stdout;
<commit_msg>Add stricter rate limit on search API<commit_after>apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-configmap
data:
  default: |
    user nginx;
    worker_processes  3;
    events {
      worker_connections  5120;
    }
    http {
      server {
          listen       80;
          server_name  _;
          location /nginx_health {
            access_log off;
            return 200 "healthy\n";
          }
      }

      map $http_x_edge_proto $scheme_from_fastly {
        "" $scheme;
        default $http_x_edge_proto;
      }

      limit_req_status 429;
      limit_req_zone  $binary_remote_addr  zone=abusers:10m  rate=40r/s;
      limit_req_zone  $binary_remote_addr  zone=dependencyapi:10m  rate=100r/s;
      limit_req_zone  $binary_remote_addr  zone=minutely:10m  rate=100r/m;

      proxy_cache_path  /var/lib/nginx/cache  levels=1:2 keys_zone=cache-versions:10m inactive=24h  max_size=100m;

      include /etc/nginx/conf.d/logging.conf;
      include /etc/nginx/sites-enabled/*;
    }
  rubygems: |
    server {
        listen       8080;
        server_name _;

        real_ip_recursive on;

        # Fastly
        set_real_ip_from '172.16.58.3/24';
        set_real_ip_from '172.16.31.10/23';
        set_real_ip_from '192.168.3.11/24';
        set_real_ip_from '172.16.17.32/20';
        set_real_ip_from '192.168.3.11/16';
        set_real_ip_from '192.168.127.12/18';
        set_real_ip_from '172.16.58.3/17';
        set_real_ip_from '172.16.58.3/20';
        set_real_ip_from '192.168.3.11/20';
        set_real_ip_from '172.16.58.3/20';
        set_real_ip_from '172.16.31.10/18';
        set_real_ip_from '192.168.127.12/22';
        set_real_ip_from '172.16.31.10/16';
        set_real_ip_from '192.168.3.11/21';
        set_real_ip_from '172.16.31.10/32';
        set_real_ip_from '192.168.3.11/32';
        set_real_ip_from '192.168.127.12/32';
        set_real_ip_from '192.168.127.12/32';
        set_real_ip_from '192.168.3.11/20';
        set_real_ip_from '172.16.17.32/22';

        # AWS
        set_real_ip_from '172.30.0.0/16';

        real_ip_header X-Forwarded-For;

        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme_from_fastly;
        proxy_set_header Client-IP "";
        proxy_set_header Host $host;
        proxy_redirect off;

        client_max_body_size 500M;

        location /api/v1/dependencies {
          limit_req zone=dependencyapi burst=10 nodelay;
          proxy_pass http://127.0.0.1:3000;
          proxy_buffer_size          128k;
          proxy_buffers              4 256k;
          proxy_busy_buffers_size    256k;
        }

        location /versions {
          limit_req zone=minutely burst=10 nodelay;
          proxy_cache cache-versions;
          proxy_cache_background_update on;
          proxy_cache_key "/versions";
          proxy_cache_lock on;
          proxy_cache_use_stale updating;
          proxy_cache_valid 200 10s;
          proxy_pass http://127.0.0.1:3000/versions;
        }

        location /api/v1/search {
          limit_req zone=minutely  burst=10 nodelay;
          proxy_pass http://127.0.0.1:3000;
        }

        location /info {
          proxy_pass http://127.0.0.1:3000;
        }

         <% if environment == 'staging' %>
        location ~ ^/(api|internal) {
          limit_req zone=abusers burst=10;
          proxy_pass http://127.0.0.1:3000;
        }

        location / {
          auth_basic           "Restricted Access!";
          auth_basic_user_file /etc/nginxbasicauth/.htpasswd;
          limit_req zone=abusers burst=10;
          proxy_pass http://127.0.0.1:3000;
        }
        <% else %>
        location / {
          limit_req zone=abusers burst=10;
          proxy_pass http://127.0.0.1:3000;
        }
        <% end %>


    }
  logging: |
    log_format json '{'
      '"host": "${HOSTNAME}", '
      '"@timestamp": "$time_iso8601", '
      '"service": "rubygems.org", '
      '"env": "<%= environment %>", '
      '"processing_time_milliseconds": "$request_time", '
      '"forwarded_for": "$http_x_forwarded_for", '
      '"upstream_x_runtime": "$upstream_http_x_runtime", '
      '"upstream_processing_time": "$upstream_response_time", '
      '"upstream": "$upstream_addr", '
      '"http": {'
        '"status_code": $status, '
        '"scheme": "$scheme_from_fastly", '
        '"url": "$scheme_from_fastly://$http_host$request_uri", '
        '"args": "$args", '
        '"dest_host": "$http_host", '
        '"content_type": "$sent_http_content_type", '
        '"protocol": "$server_protocol", '
        '"referer": "$http_referer", '
        '"useragent": "$http_user_agent", '
        '"method": "$request_method", '
        '"request": "$request", '
        '"request_id": "$upstream_http_x_request_id", '
        '"ssl_protocol": "$ssl_protocol", '
        '"ssl_cipher": "$ssl_cipher"'
      '},'
      '"network": {'
        '"client": {"ip": "$remote_addr"} '
      '}'
    '}';
    access_log  /dev/stdout json;
    error_log	/dev/stdout;

{'fraction_non_alphanumeric': 0.13121902874132804, 'fraction_numerical': 0.07403369672943508, 'mean_word_length': 2.0982499232422476, 'pattern_counts': {'":': 54, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 55, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7644432', 'n_tokens_mistral': 3983, 'n_tokens_neox': 3485, 'n_words': 528}