<commit_before>#!/bin/bash
set -e

SOURCES_DIR=src/main/java

echo "Testing Checkstyle started"

export EXCLUDES_ACCUM=""

while read line ; do
    [[ "$line" == \#* ]] && continue # Skip lines with comments
    [[ -z "$line" ]] && continue     # Skip empty lines
    
    REPO_NAME=`echo $line | cut -d '|' -f 1`
    REPO_TYPE=`echo $line | cut -d '|' -f 2`
    REPO_URL=` echo $line | cut -d '|' -f 3`
    COMMIT_ID=`echo $line | cut -d '|' -f 4`
    EXCLUDES=` echo $line | cut -d '|' -f 5`
    
    EXCLUDES_ACCUM+=",$EXCLUDES"

    REPO_SOURCES_DIR=$SOURCES_DIR/$REPO_NAME
    
    if [ "$REPO_TYPE" == "github" ]; then
        rm -rf $REPO_SOURCES_DIR
        if [ ! -d "$REPO_SOURCES_DIR" ]; then
            TARNAME=$(echo $REPO_URL | sed -r 's/\//-/')
            TARPATH=$SOURCES_DIR/$TARNAME".tar.gz"
            if [ ! -f "$TARPATH" ]; then
                echo "Requesting a tar: 'wget https://api.github.com/repos/$REPO_URL/tarball/$COMMIT_ID -O $TARPATH'"
                wget https://api.github.com/repos/$REPO_URL/tarball/$COMMIT_ID -O $TARPATH
                fi
<commit_msg>checkstyle-tester launcher has been updated to detect OS and use appropriate sed flag<commit_after>#!/bin/bash
set -e

SOURCES_DIR=src/main/java

echo "Testing Checkstyle started"

export EXCLUDES_ACCUM=""

while read line ; do
    [[ "$line" == \#* ]] && continue # Skip lines with comments
    [[ -z "$line" ]] && continue     # Skip empty lines
    
    REPO_NAME=`echo $line | cut -d '|' -f 1`
    REPO_TYPE=`echo $line | cut -d '|' -f 2`
    REPO_URL=` echo $line | cut -d '|' -f 3`
    COMMIT_ID=`echo $line | cut -d '|' -f 4`
    EXCLUDES=` echo $line | cut -d '|' -f 5`
    
    EXCLUDES_ACCUM+=",$EXCLUDES"

    REPO_SOURCES_DIR=$SOURCES_DIR/$REPO_NAME
    
    if [ "$REPO_TYPE" == "github" ]; then
        rm -rf $REPO_SOURCES_DIR
        if [ ! -d "$REPO_SOURCES_DIR" ]; then
            OS=`uname`
            if [ $OS="Darwin" ]; then
                TARNAME=$(echo $REPO_URL | sed -E 's/\//-/')
            else
                TARNAME=$(echo $REPO_URL | sed -r 's/\//-/')
            fi
            TARPATH=$SOURCES_DIR/$TARNAME".tar.gz"
            if [ ! -f "$TARPATH" ]; then
                echo "Requesting a tar: 'wget https://api.github.com/repos/$REPO_URL/tarball/$COMMIT_ID -O $TARPATH'"
                wget https://api.github.com/repos/$REPO_URL/tarball/$COMMIT_ID -O $TARPATH
                fi

{'fraction_non_alphanumeric': 0.16390728476821192, 'fraction_numerical': 0.0041390728476821195, 'mean_word_length': 2.102695763799743, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6689402', 'n_tokens_mistral': 968, 'n_tokens_neox': 893, 'n_words': 211}