<commit_before>#!/bin/bash

set -e
set -u
set -o pipefail

normalize_boolean() {
    [[ $1 =~ 1|yes|true ]] && echo 'true' || echo 'false'
}

fetch_origin() {
    # List of popular "what is my IP?" providers.
    local hosts=(
        'http://checkip.amazonaws.com/'
        'http://icanhazip.com/'
        'http://ident.me/'
        'http://ifconfig.io/'
        'http://ipecho.net/plain'
        'http://whatismyip.akamai.com/'
    )

    local tries=0
    local origin=''
    local host="${hosts[$(( RANDOM % ${#hosts[@]} ))]}"

    # Try to get an IP address but give up after trying 5 times.
    while [[ -z $origin ]] && (( tries < 5 )); do
        set +e
        if which curl &>/dev/null; then
            origin=$(curl --max-time 5 --user-agent 'curl/1.0.0' -L "$host" 2>/dev/null)
        else
            origin=$(wget --timeout 5 --user-agent 'Wget/1.0.0' -O- "$host" 2>/dev/null)<commit_msg>Update origin module.<commit_after>#!/bin/bash

set -e
set -u
set -o pipefail

normalize_boolean() {
    local boolean="$1"
    shift

    local status='true'
    [[ "$boolean" =~ (1|yes|true) ]] || status='false'

    echo "$status"
}

fetch_origin() {
    # List of popular "what is my IP?" providers.
    local hosts=(
        'http://checkip.amazonaws.com/'
        'http://icanhazip.com/'
        'http://ident.me/'
        'http://ifconfig.io/'
        'http://ipecho.net/plain'
        'http://whatismyip.akamai.com/'
    )

    local tries=0
    local origin=
    local host="${hosts[$(( RANDOM % ${#hosts[@]} ))]}"

    # Try to get an IP address but give up after trying 5 times.
    while [[ -z $origin ]] && (( tries < 5 )); do
        set +e
        if which curl &>/dev/null; then
            origin=$(curl --max-time 5 --user-agent 'curl/1.0.0' -L "$host" 2>/dev/null)
        else
            origin=$(wget --timeout 5 --user-agent 'Wget/1.0.0' -O- "$host" 2>/dev/null)
{'fraction_non_alphanumeric': 0.1926945473795659, 'fraction_numerical': 0.015881418740074114, 'mean_word_length': 2.0434782608695654, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2635495', 'n_tokens_mistral': 690, 'n_tokens_neox': 659, 'n_words': 171}