<commit_before>#!/bin/bash
#curl -XDELETE http://131.111.22.70:9200/test-sedgwick/
# curl -XPUT http://172.16.17.32:9200/test-sedgwick/
# curl -XPUT http://172.16.31.10:9200/sedgwick/id/_mapping --data-binary @esobjectmappings.json
curl -XDELETE http://192.168.16.67:9200/sedgwick/$1/
CURL -XPUT http://192.168.16.67:9200/sedgwick/$1/_mapping --data-binary @esmapping2.txt
CURL -XPUT http://192.168.16.67:9200/sedgwick/_bulk --data-binary @$1-esbulk.txt<commit_msg>esimport tools added<commit_after>#!/bin/bash
#curl -XDELETE http://172.16.31.10:9200/test-sedgwick/
# curl -XPUT http://172.16.17.32:9200/test-sedgwick/
# curl -XPUT http://172.16.31.10:9200/sedgwick/id/_mapping --data-binary @esobjectmappings.json
curl -XDELETE http://192.168.16.67:9200/sedgwick/$1/
curl -XPUT http://192.168.16.67:9200/sedgwick/$1/_mapping --data-binary @esmapping2.txt
curl -XPUT http://192.168.16.67:9200/sedgwick/_bulk --data-binary @$1-esbulk.txt
{'fraction_non_alphanumeric': 0.20085470085470086, 'fraction_numerical': 0.18269230769230768, 'mean_word_length': 3.905759162303665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 12, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1241413', 'n_tokens_mistral': 499, 'n_tokens_neox': 397, 'n_words': 51}