<commit_before>#!/bin/bash
# Address the problem  of lacking "DELETE without COMMIT" in elasticsearch:
# Create temporary indices and alias them to enable importing from scratch 
# while keeping read access to the old index.
# Notice that you might obtain duplicate results during indexing that must be handled on client side


host_and_port=$(inifile_lookup /usr/local/var/lib/tuelib/Elasticsearch.conf Elasticsearch host)


function selectExisting {
    for schema in *_schema.json; do
        index="${schema%_schema.json}"
        http_code=$(curl -s --silent --output /dev/null --head --write "%{http_code}" ${host_and_port}/${index})
        if [ ${http_code} == "200" ]; then
            existing_indices+=("${index}")
        fi
     done

}
   

existing_indices=()
selectExisting
for index in ${existing_indices[@]}; do
   echo $index "present"
   curl --request GET "${host_and_port}/_cat/aliases/${index}" --header 'Content-Type: application/json' 
done


   #curl -XPUT "${host_and_port}/${index}/_settings" --header 'Content-Type: application/json' --data'{ "settings": { "index.blocks.write": true } }'<commit_msg>Import alias creation<commit_after>#!/bin/bash
# Address the problem  of lacking "DELETE without COMMIT" in elasticsearch:
# Create temporary indices and alias them to enable importing from scratch
# while keeping read access to the old index.
# Notice that you might obtain duplicate results during indexing that must be handled on client side


host_and_port=$(inifile_lookup /usr/local/var/lib/tuelib/Elasticsearch.conf Elasticsearch host)


function GetExistingFulltextIndices {
    existing_fulltext_indices=()
    for schema in *_schema.json; do
        index="${schema%_schema.json}"
        http_code=$(curl --silent --output /dev/null --head --write "%{http_code}" ${host_and_port}/${index})
        if [ ${http_code} == "200" ]; then
            existing_fulltext_indices+=("${index}")
        else
            echo "Index ${index} not present -- Aborting"
            exit 1
        fi
    done
    echo ${existing_fulltext_indices[@]}
}


function GetExistingFulltextAliases {
    existing_fulltext_indices=$@
    check_output=()
    for index in ${existing_fulltext_indices[@]}; do
        check_output+=($(curl --silent --request GET "${host_and_port}/_cat/aliases/${index}" --header 'Content-Type: application/json'))
    done
    echo ${check_output}
}


function CreateReadWriteIndicesAndAliases {
    indices=$@
    for index in ${indices[@]}; do
        curl --silent --request PUT "${host_and_port}/${index}/_settings" --header 'Content-Type: application/json'  \
            --data '{ "settings": { "index.blocks.write": true } }'

        curl --silent --request POST "${host_and_port}/${index}/_clone/${index}_read" --header 'Content-Type: application/json'
        curl --silent --request GET "${host_and_port}/_cluster/health/${index}_read?wait_for_status=yellow&timeout=30s" --header 'Content-Type: application/json'

        curl --silent --request POST "${host_and_port}/${index}/_clone/${index}_write" --header 'Content-Type: application/json'
        curl --silent --request GET "${host_and_port}/_cluster/health/${index}_write?wait_for_status=yellow&timeout=30s" --header 'Content-Type: application/json'
        curl --request POST "${host_and_port}/_aliases" --header 'Content-Type: application/json' --data-binary @- << ENDJSON
                       { "actions" :
                          [
                             { "remove_index" : { "index" : "${index}" } },
                             { "add" : { "index" : "${index}_write",
                                         "alias" : "${index}",
                                         "is_write_index" : true
                                       }
                             },
                             { "add" : { "index" : "${index}_read",
                                         "alias" : "${index}"
                                       }
                             }
                          ]
                       }
ENDJSON
    done
}

#Check whether all necessary indices exist = Abort if not
existing_fulltext_indices=($(GetExistingFulltextIndices))


#Check whether aliases exist => Abort if they do
existing_fulltext_aliases=($(GetExistingFulltextAliases ${existing_fulltext_indices[@]}))
if [ ${#existing_fulltext_aliases[@]} -ne 0 ]; then
    echo "Don't know how to handle previously existing aliases (${#existing_fulltext_aliases[@]})"
    exit 1
fi

#Clone indices to a temporary name, set the aliases and select the write alias
CreateReadWriteIndicesAndAliases ${existing_fulltext_indices[@]}
{'fraction_non_alphanumeric': 0.13008830497523152, 'fraction_numerical': 0.0027999138488046522, 'mean_word_length': 2.5944272445820435, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6190418', 'n_tokens_mistral': 1305, 'n_tokens_neox': 1276, 'n_words': 342}