<commit_before>#!/bin/bash
# Hyper Dumb Cron helper to Drop/Rotate data in ES sandbox @facetflow
# If you need something smart, secure and reliable, please use the official helper scripts from ES

# m h  dom mon dow   command
# */5 * * * * /opt/rotate_stash.sh

# MANDATORY: Your Facetflow API Key
API_KEY=""

# Use local nginx https proxy or actual ES url
ES_HOST='http://localhost:19200'

# Document Limit per index being checked
LIMIT=4500

# For Sandbox only, chop without mercy
INDEXS=$(date +%Y.%m.*)

# Count documents in index
COUNT=$(curl -XGET "$ES_HOST/logstash-$INDEXS/_count" -u $API_KEY: -d '{"query": {"match_all": {} } }' -s | sed 's,.*count":\([^<]*\)}.*,\1,g' )
echo "Current index count is: $COUNT"

# Remove if count is higher than $LIMIT
if [ "$COUNT" -gt "$LIMIT" ]; then
	echo "Deleting logstash-$INDEXS"
	curl -XDELETE "$ES_HOST/logstash-$INDEXS/" -u $API_KEY:
	echo
	exit 1;
fi
exit 0;<commit_msg>Update rotate_sandbox.sh<commit_after>#!/bin/bash
# Hyper Dumb Cron helper to Drop/Rotate data in ES sandbox @facetflow
# If you need something smart, secure and reliable, please use the official helper scripts from ES

# m h  dom mon dow   command
# */5 * * * * /opt/rotate_stash.sh

# MANDATORY: Your Facetflow API Key
API_KEY=""

# Use local nginx https proxy or actual ES url
ES_HOST='http://localhost:19200'

# Document Limit per index being checked
LIMIT=4500

# For Sandbox only, chop all Logstash indexes without mercy or use date ranger
INDEXS='*'
# INDEXS=$(date +%Y.%m.*)

# Count documents in index
COUNT=$(curl -XGET "$ES_HOST/logstash-$INDEXS/_count" -u $API_KEY: -d '{"query": {"match_all": {} } }' -s | sed 's,.*count":\([^<]*\)}.*,\1,g' )
echo "Current index count is: $COUNT"

# Remove if count is higher than $LIMIT
if [ "$COUNT" -gt "$LIMIT" ]; then
	echo "Deleting logstash-$INDEXS"
	curl -XDELETE "$ES_HOST/logstash-$INDEXS/" -u $API_KEY:
	echo
	exit 1;
fi
exit 0;
{'fraction_non_alphanumeric': 0.1488469601677149, 'fraction_numerical': 0.013626834381551363, 'mean_word_length': 4.1875, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6507543', 'n_tokens_mistral': 712, 'n_tokens_neox': 677, 'n_words': 236}