<commit_before>#!/bin/bash

# Get directory where the current script is located:
MY_DIR="$( cd "$( dirname "$0" )" && pwd )"

# Find protoc and check version:
PROTOC_CMD=$(which protoc)

if [[ -z ${PROTOC_CMD} ]]; then
    echo "Could NOT find protobuf compiler (protoc) - big fail"
    exit -1
fi

PROTOC_VERSION="$(${PROTOC_CMD} --version | cut -f 2 -d ' ')"
PROTOC_VERSION_REQUIRED="2.5.0"

if [[ ${PROTOC_VERSION} != ${PROTOC_VERSION_REQUIRED} ]]; then
    echo "Protoc available, but version is - ${PROTOC_VERSION} - requires ${PROTOC_VERSION_REQUIRED}"
    exit -1
fi

echo "Protobuf compiler (protoc) available - ${PROTOC_CMD} - version ${PROTOC_VERSION}"

# Get proto files:
PROTO_FILES_DIR="${MY_DIR}/../protos"
PROTO_FILES="$(ls  ${PROTO_FILES_DIR}/*.proto)"

echo "Proto files:"
for file in ${PROTO_FILES}; do
    echo -e "\t* $(basename $file)"
done

# Compile protos
PROTO_OUT_DIR="${MY_DIR}/../hbite/protobuf"

mkdir -p ${PROTO_OUT_DIR}
touch ${PROTO_OUT_DIR}/__init__.py

${PROTOC_CMD} --proto_path=${PROTO_FILES_DIR} --python_out=${PROTO_OUT_DIR} ${PROTO_FILES}

if [[ $? != 0 ]]; then
    echo "Proto files compilation failed!"
    exit -1
else
    echo "Success - proto files compiled to:"
    for file in $(ls ${PROTO_OUT_DIR}); do
        echo -e "\t* $(basename $file)"
    done
fi<commit_msg>Fix whitespaces/style in compile_protos script<commit_after>#!/bin/bash

# Get directory where the current script is located:
MY_DIR="$( cd "$( dirname "$0" )" && pwd )"

# Find protoc and check version:
PROTOC_CMD=$(which protoc)

if [[ -z ${PROTOC_CMD} ]]; then
    echo "Could NOT find protobuf compiler (protoc) - big fail"
    exit -1
fi

PROTOC_VERSION="$( ${PROTOC_CMD} --version | cut -f 2 -d ' ' )"
PROTOC_VERSION_REQUIRED="2.5.0"

if [[ ${PROTOC_VERSION} != ${PROTOC_VERSION_REQUIRED} ]]; then
    echo "Protoc available, but version is - ${PROTOC_VERSION} - requires ${PROTOC_VERSION_REQUIRED}"
    exit -1
fi

echo "Protobuf compiler (protoc) available - ${PROTOC_CMD} - version ${PROTOC_VERSION}"

# Get proto files:
PROTO_FILES_DIR="${MY_DIR}/../protos"
PROTO_FILES="$( ls  ${PROTO_FILES_DIR}/*.proto )"

echo "Proto files:"
for file in ${PROTO_FILES}; do
    echo -e "\t* $( basename $file )"
done

# Compile protos
PROTO_OUT_DIR="${MY_DIR}/../hbite/protobuf"

mkdir -p ${PROTO_OUT_DIR}
touch ${PROTO_OUT_DIR}/__init__.py

${PROTOC_CMD} --proto_path=${PROTO_FILES_DIR} --python_out=${PROTO_OUT_DIR} ${PROTO_FILES}

if [[ $? != 0 ]]; then
    echo "Proto files compilation failed!"
    exit -1
else
    echo "Success - proto files compiled to:"
    for file in $( ls ${PROTO_OUT_DIR} ); do
        echo -e "\t* $( basename $file )"
    done
fi
{'fraction_non_alphanumeric': 0.1621115687008611, 'fraction_numerical': 0.0067390490453013855, 'mean_word_length': 4.0606060606060606, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1847251', 'n_tokens_mistral': 970, 'n_tokens_neox': 932, 'n_words': 265}