<commit_before>#! /bin/bash -ex

# This script downloads a bundle from Mozilla's Mercurial repositories and extracts them.
#
# Arguments: <script> <URL connector> <repository name> <absolute base dir path>
#
# Sample commands:
#
# $ bash <script> / mozilla-central /home/ubuntu/trees
# Note that mozilla-central is in https://hg.mozilla.org/mozilla-central/
#
# $ bash <script> /releases/ mozilla-beta /home/ubuntu/trees
# Note that mozilla-beta is in https://hg.mozilla.org/releases/mozilla-beta/

mkdir -p "$3"
pushd "$3"
date
# The clone process hangs quite so often, but less when downloaded standalone
if [ ! -d "$3"/"$2" ]; then
    timeout 10 hg clone --stream https://hg.mozilla.org"$1""$2" "$3"/"$2" \
        > "$3"/"$2"_url_raw.txt 2>&1
    # Ensure that the cloning process did not create any repo directories
    rm -rf "$3"/"$2"
else
    echo "$3/$2 currently exists, which it shouldn't. Exiting..."
    exit 1<commit_msg>Fix shellcheck SC2115 error in get_hg_repo.sh<commit_after>#! /bin/bash -ex

# This script downloads a bundle from Mozilla's Mercurial repositories and extracts them.
#
# Arguments: <script> <URL connector> <repository name> <absolute base dir path>
#
# Sample commands:
#
# $ bash <script> / mozilla-central /home/ubuntu/trees
# Note that mozilla-central is in https://hg.mozilla.org/mozilla-central/
#
# $ bash <script> /releases/ mozilla-beta /home/ubuntu/trees
# Note that mozilla-beta is in https://hg.mozilla.org/releases/mozilla-beta/

mkdir -p "$3"
pushd "$3"
date
# The clone process hangs quite so often, but less when downloaded standalone
if [ ! -d "$3"/"$2" ]; then
    timeout 10 hg clone --stream https://hg.mozilla.org"$1""$2" "$3"/"$2" \
        > "$3"/"$2"_url_raw.txt 2>&1
    # Ensure that the cloning process did not create any repo directories
    rm -rf "${3:?}"/"$2"
else
    echo "$3/$2 currently exists, which it shouldn't. Exiting..."
    exit 1
{'fraction_non_alphanumeric': 0.14937106918238993, 'fraction_numerical': 0.0220125786163522, 'mean_word_length': 3.556085918854415, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5301809', 'n_tokens_mistral': 712, 'n_tokens_neox': 616, 'n_words': 212}