<commit_before>#!/bin/bash
# init-es.sh is the script to be run to properly setup the es cluster.  The main things
#  it does is set up the aliases and mappings as required by BF.  The mappings
#  are required to properly tokenize the incoming data.
# You can pass it a url if your elastic search doesn't reside on localhost:9092
#WARNING: this script will destroy existing ES data and reset it to the proper init state

ELASTICSEARCH_URL=${1:-"http://localhost:9200"}

function checkFile
{
  echo checking $1.
  if [ ! -f $1 ]; then 
    echo $1 not found
    exit 2
  fi
}

checkFile index_settings.json
checkFile metrics_mapping.json
checkFile metrics_mapping_enums.json
checkFile events_mapping.json

#delete the old indices
curl -XDELETE $ELASTICSEARCH_URL'/enums/' >& /dev/null
curl -XDELETE $ELASTICSEARCH_URL'/events/' >& /dev/null
curl -XDELETE $ELASTICSEARCH_URL'/metric_metadata/' >& /dev/null
curl -XDELETE $ELASTICSEARCH_URL'/metric_metadata_v2/' >& /dev/null
curl -XDELETE $ELASTICSEARCH_URL'/metric_metadata_write/' >& /dev/null


#create the new indices
curl -XPUT $ELASTICSEARCH_URL'/metric_metadata'
curl -XPUT $ELASTICSEARCH_URL'/enums'
curl -XPUT $ELASTICSEARCH_URL'/events'

#create the aliases
curl -XPOST $ELASTICSEARCH_URL'/_aliases' -d '
{
    "actions" : [
        { "add" : { "alias" : "metric_metadata_write", "index" : "metric_metadata" } }
    ]
}'
curl -XPOST $ELASTICSEARCH_URL'/_aliases' -d '
{
    "actions" : [
        { "add" : { "alias" : "metric_metadata_read", "index" : "metric_metadata" } }
    ]
}'
curl -XPOST $ELASTICSEARCH_URL'/_aliases' -d '
{
    "actions" : [
        { "add" : { "alias" : "metric_metadata_read", "index" : "enums" } }
    ]
}'

#add index settings to metric_metadata index
curl -XPOST $ELASTICSEARCH_URL'/metric_metadata/_close'
curl -XPUT $ELASTICSEARCH_URL'/metric_metadata/_settings' -d @index_settings.json
curl -XPOST $ELASTICSEARCH_URL'/metric_metadata/_open'


#add mappings to metric_metadata index
curl -XDELETE $ELASTICSEARCH_URL'/metric_metadata/_mapping/metrics' >& /dev/null
curl -XPUT $ELASTICSEARCH_URL'/metric_metadata/_mapping/metrics' -d @metrics_mapping.json

#add index settings to enums index
curl -XPOST $ELASTICSEARCH_URL'/enums/_close'
curl -XPUT $ELASTICSEARCH_URL'/enums/_settings' -d  @index_settings.json
curl -XPOST $ELASTICSEARCH_URL'/enums/_open'

#add mappings to enums index
curl -XDELETE $ELASTICSEARCH_URL'/enums/_mapping/metrics'  >& /dev/null
curl -XPUT $ELASTICSEARCH_URL'/enums/_mapping/metrics' -d @metrics_mapping_enums.json

#add mappings to graphite_event index
curl -XDELETE $ELASTICSEARCH_URL'/events/_mapping/graphite_event' >& /dev/null
curl -XPUT $ELASTICSEARCH_URL'/events/_mapping/graphite_event' -d @events_mapping.json
<commit_msg>Add username & password & auth & absolute path to allow it to run from any directory<commit_after>#!/bin/bash
# init-es.sh is the script to be run to properly setup the es cluster.  The main things
#  it does is set up the aliases and mappings as required by BF.  The mappings
#  are required to properly tokenize the incoming data.
# You can pass it a url if your elastic search doesn't reside on localhost:9092
#WARNING: this script will destroy existing ES data and reset it to the proper init state
# Example:
#    Local ES: ./init-es.sh
#    ES on OR: ./init-es.sh <url for OR:ES> <username> <password>
ELASTICSEARCH_URL=${1:-"http://localhost:9200"}
ES_USERNAME=${2:-}
ES_PASSWD=${3:-}

# Set a auth header for curl if username and passwd is supplied
# Allows initing ES when ES is supplied by a SaaS provider like ObjectRocket
if [ $ES_USERNAME ] && [ $ES_PASSWD ]; then
    AUTH="-u ${ES_USERNAME}:${ES_PASSWD}"
fi

# Using this in the checkfile function allows us to run this script from any directory
ABSOLUTE_PATH=$(cd `dirname "${BASH_SOURCE[0]}"` && pwd)
function checkFile
{
  echo checking $1.
  if [ ! -f $ABSOLUTE_PATH/$1 ]; then
    echo $1 not found
    exit 2
  fi
}

checkFile index_settings.json
checkFile metrics_mapping.json
checkFile metrics_mapping_enums.json
checkFile events_mapping.json

#delete the old indices
curl $AUTH -XDELETE $ELASTICSEARCH_URL'/enums/' >& /dev/null
curl $AUTH -XDELETE $ELASTICSEARCH_URL'/events/' >& /dev/null
curl $AUTH -XDELETE $ELASTICSEARCH_URL'/metric_metadata/' >& /dev/null
curl $AUTH -XDELETE $ELASTICSEARCH_URL'/metric_metadata_v2/' >& /dev/null
curl $AUTH -XDELETE $ELASTICSEARCH_URL'/metric_metadata_write/' >& /dev/null


#create the new indices
curl $AUTH -XPUT $ELASTICSEARCH_URL'/metric_metadata'
curl $AUTH -XPUT $ELASTICSEARCH_URL'/enums'
curl $AUTH -XPUT $ELASTICSEARCH_URL'/events'

#create the aliases
curl $AUTH -XPOST $ELASTICSEARCH_URL'/_aliases' -d '
{
    "actions" : [
        { "add" : { "alias" : "metric_metadata_write", "index" : "metric_metadata" } }
    ]
}'
curl $AUTH -XPOST $ELASTICSEARCH_URL'/_aliases' -d '
{
    "actions" : [
        { "add" : { "alias" : "metric_metadata_read", "index" : "metric_metadata" } }
    ]
}'
curl $AUTH -XPOST $ELASTICSEARCH_URL'/_aliases' -d '
{
    "actions" : [
        { "add" : { "alias" : "metric_metadata_read", "index" : "enums" } }
    ]
}'

#add index settings to metric_metadata index
curl $AUTH -XPOST $ELASTICSEARCH_URL'/metric_metadata/_close'
curl $AUTH -XPUT $ELASTICSEARCH_URL'/metric_metadata/_settings' -d @index_settings.json
curl $AUTH -XPOST $ELASTICSEARCH_URL'/metric_metadata/_open'


#add mappings to metric_metadata index
curl $AUTH -XDELETE $ELASTICSEARCH_URL'/metric_metadata/_mapping/metrics' >& /dev/null
curl $AUTH -XPUT $ELASTICSEARCH_URL'/metric_metadata/_mapping/metrics' -d @metrics_mapping.json

#add index settings to enums index
curl $AUTH -XPOST $ELASTICSEARCH_URL'/enums/_close'
curl $AUTH -XPUT $ELASTICSEARCH_URL'/enums/_settings' -d  @index_settings.json
curl $AUTH -XPOST $ELASTICSEARCH_URL'/enums/_open'

#add mappings to enums index
curl $AUTH -XDELETE $ELASTICSEARCH_URL'/enums/_mapping/metrics'  >& /dev/null
curl $AUTH -XPUT $ELASTICSEARCH_URL'/enums/_mapping/metrics' -d @metrics_mapping_enums.json

#add mappings to graphite_event index
curl $AUTH -XDELETE $ELASTICSEARCH_URL'/events/_mapping/graphite_event' >& /dev/null
curl $AUTH -XPUT $ELASTICSEARCH_URL'/events/_mapping/graphite_event' -d @events_mapping.json

{'fraction_non_alphanumeric': 0.11978832584990379, 'fraction_numerical': 0.004971135343168698, 'mean_word_length': 4.544, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4038783', 'n_tokens_mistral': 2289, 'n_tokens_neox': 2157, 'n_words': 584}