<commit_before>{% set production_auth_key = salt.vault.read('secret-micromasters/production/elasticsearch-auth-key').data.value %}
{% set rc_auth_key = salt.vault.read('secret-micromasters/rc/elasticsearch-auth-key').data.value %}
{% set ci_auth_key = salt.vault.read('secret-micromasters/ci/elasticsearch-auth-key').data.value %}

elasticsearch:
  lookup:
    elastic_stack: True
    configuration_settings:
      discovery:
        zen.hosts_provider: ec2
      cluster.name: micromasters
      discovery.ec2.tag.escluster: micromasters
      rest.action.multi.allow_explicit_index: 'false'
      network.host: [_eth0_, _lo_]
  plugin_settings:
    readonlyrest:
      readonlyrest:
        enable: 'true'
        response_if_req_forbidden: Acess Denied
        access_control_rules:
          - name: Cluster access within VPC
            type: allow
            accept_x-forwarded-for_header: 'true'
            indices:
              - <no-index>
            actions:
              - 'cluster:*'
            hosts:
              - localhost
              - 127.0.0.1
              - 10.10.0.0/16
          - name: Access for micromasters production index with HTTP Auth
            type: allow
            indices:
              - micromasters
              - 'micromasters_*'
            accept_x-forwarded-for_header: 'true'
            actions:
              - 'indices:*'
            auth_key: {{ production_auth_key }}
          - name: Access for micromasters RC index with HTTP Auth
            type: allow
            indices:
              - 'micromasters-rc*'
            accept_x-forwarded-for_header: 'true'
            actions:
              - 'indices:*'
            auth_key: {{ rc_auth_key }}
          - name: Access for micromasters CI index with HTTP Auth
            type: allow
            indices:
              - 'micromasters-ci*'
            accept_x-forwarded-for_header: 'true'
            actions:
              - 'indices:*'
            auth_key: {{ ci_auth_key }}
          - name: View existence of indices with RC Auth
            type: allow
            accept_x-forwarded-for_header: 'true'
            methods:
              - GET
              - HEAD
              - OPTIONS
              - POST
            indices:
              - '_all'
              - 'micromasters*'
            actions:
              - 'indices:admin/get'
              - 'indices:admin/exists'
              - 'indices:admin/refresh[s]'
            auth_key: {{ rc_auth_key }}
          - name: View existence of indices with CI Auth
            type: allow
            accept_x-forwarded-for_header: 'true'
            methods:
              - GET
              - HEAD
              - OPTIONS
              - POST
            indices:
              - '_all'
              - 'micromasters*'
            actions:
              - 'indices:admin/get'
              - 'indices:admin/exists'
              - 'indices:admin/refresh[s]'
            auth_key: {{ ci_auth_key }}
          - name: View existence of indices with Production Auth
            type: allow
            accept_x-forwarded-for_header: 'true'
            methods:
              - GET
              - HEAD
              - OPTIONS
              - POST
            indices:
              - '_all'
              - 'micromasters*'
            actions:
              - 'indices:admin/get'
              - 'indices:admin/exists'
              - 'indices:admin/refresh[s]'
            auth_key: {{ production_auth_key }}
    products:
      elasticsearch: '5.x'
  plugins:
    - name: discovery-ec2
    - name: readonlyrest
      location: https://raw.githubusercontent.com/mitodl/salt-ops/master/salt/artifacts/readonlyrest-1.16.15_es5.6.6.zip<commit_msg>Added the scroll permission to the _all index<commit_after>{% set production_auth_key = salt.vault.read('secret-micromasters/production/elasticsearch-auth-key').data.value %}
{% set rc_auth_key = salt.vault.read('secret-micromasters/rc/elasticsearch-auth-key').data.value %}
{% set ci_auth_key = salt.vault.read('secret-micromasters/ci/elasticsearch-auth-key').data.value %}

elasticsearch:
  lookup:
    elastic_stack: True
    configuration_settings:
      discovery:
        zen.hosts_provider: ec2
      cluster.name: micromasters
      discovery.ec2.tag.escluster: micromasters
      rest.action.multi.allow_explicit_index: 'false'
      network.host: [_eth0_, _lo_]
  plugin_settings:
    readonlyrest:
      readonlyrest:
        enable: 'true'
        response_if_req_forbidden: Acess Denied
        access_control_rules:
          - name: Cluster access within VPC
            type: allow
            accept_x-forwarded-for_header: 'true'
            indices:
              - <no-index>
            actions:
              - 'cluster:*'
            hosts:
              - localhost
              - 127.0.0.1
              - 10.10.0.0/16
          - name: Access for micromasters production index with HTTP Auth
            type: allow
            indices:
              - micromasters
              - 'micromasters_*'
            accept_x-forwarded-for_header: 'true'
            actions:
              - 'indices:*'
            auth_key: {{ production_auth_key }}
          - name: Access for micromasters RC index with HTTP Auth
            type: allow
            indices:
              - 'micromasters-rc*'
            accept_x-forwarded-for_header: 'true'
            actions:
              - 'indices:*'
            auth_key: {{ rc_auth_key }}
          - name: Access for micromasters CI index with HTTP Auth
            type: allow
            indices:
              - 'micromasters-ci*'
            accept_x-forwarded-for_header: 'true'
            actions:
              - 'indices:*'
            auth_key: {{ ci_auth_key }}
          - name: View existence of indices with RC Auth
            type: allow
            accept_x-forwarded-for_header: 'true'
            methods:
              - GET
              - HEAD
              - OPTIONS
              - POST
            indices:
              - '_all'
              - 'micromasters*'
            actions:
              - 'indices:admin/get'
              - 'indices:admin/exists'
              - 'indices:admin/refresh[s]'
              - 'indices:data/read/scroll'
            auth_key: {{ rc_auth_key }}
          - name: View existence of indices with CI Auth
            type: allow
            accept_x-forwarded-for_header: 'true'
            methods:
              - GET
              - HEAD
              - OPTIONS
              - POST
            indices:
              - '_all'
              - 'micromasters*'
            actions:
              - 'indices:admin/get'
              - 'indices:admin/exists'
              - 'indices:admin/refresh[s]'
              - 'indices:data/read/scroll'
            auth_key: {{ ci_auth_key }}
          - name: View existence of indices with Production Auth
            type: allow
            accept_x-forwarded-for_header: 'true'
            methods:
              - GET
              - HEAD
              - OPTIONS
              - POST
            indices:
              - '_all'
              - 'micromasters*'
            actions:
              - 'indices:admin/get'
              - 'indices:admin/exists'
              - 'indices:admin/refresh[s]'
              - 'indices:data/read/scroll'
            auth_key: {{ production_auth_key }}
    products:
      elasticsearch: '5.x'
  plugins:
    - name: discovery-ec2
    - name: readonlyrest
      location: https://raw.githubusercontent.com/mitodl/salt-ops/master/salt/artifacts/readonlyrest-1.16.15_es5.6.6.zip
{'fraction_non_alphanumeric': 0.0958130988318677, 'fraction_numerical': 0.007087544297151857, 'mean_word_length': 1.3554868624420402, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '233692', 'n_tokens_mistral': 2129, 'n_tokens_neox': 2034, 'n_words': 407}