<commit_before>#!/bin/bash

DELETE_COUNT=5

ls /var/lib/scylla/data| grep -v system |sort -R |tail -$DELETE_COUNT |while read file; do
    rm -rf $file
done<commit_msg>changing the way we corrupt the data before repair and rebuild, instead of deleting the entire keyspace, we will delete only several sstables<commit_after>#! /bin/bash

n=3    # Number of file sets to delete
s=1000 # Size of Data files to delete
c=0    # Counter to avoid infinite loop

files=`find /var/lib/scylla/data/keyspace1/*/ -maxdepth 1 -type f -size "+"$s"M" -exec ls -latr {} + | head -$n | awk '{print$9}' | cut -f 1,2,3,4,5 -d '-'`
while [ "$files" = "" ] && [ $c -lt 10 ]
do
        echo "Empty!, looking for files half the size"
        s=$((s / 2))
        echo "size to find is:" $s
        files=`find /var/lib/scylla/data/keyspace1/*/ -maxdepth 1 -type f -size "+"$s"M" -exec ls -latr {} + | head -$n | awk '{print$9}' | cut -f 1,2,3,4,5 -d '-'`
        c=$((c+1))
        if [ $c -eq 10 ]
        then
                echo "Max retries reached - exiting"
        fi
done

for f in $files
do
        f="$f-*"
        rm -f $f
        echo "Deletion exit code is" $?
done
{'fraction_non_alphanumeric': 0.157439446366782, 'fraction_numerical': 0.025086505190311418, 'mean_word_length': 2.3439306358381504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5361432', 'n_tokens_mistral': 445, 'n_tokens_neox': 417, 'n_words': 150}