<commit_before>#!/usr/bin/env bash
#
# Usage: ./wu.sh [years]
#
# If years is not provided as an argument, a default value of 1 is used.
YEARS=${1:-1}

for airport in `cat airports.txt`
do
    mkdir -pv csv/$airport
    for dt in `python -c "from datetime import datetime, timedelta; \
        now = datetime.now(); \
        print '\n'.join(((now - timedelta(days=day)).strftime('%Y-%m-%d')) \
        for day in range((now - datetime(now.year - $YEARS, now.month, \
            now.day)).days, 0, -1))"`
    do
        URL=`echo $airport $dt \
            | sed 's/\-/ /g' \
            | awk '{printf("https://www.wunderground.com/history/airport/%s/%d/%d/%d/DailyHistory.html?format=1\n", $1, $2, $3, $4)}'`
        if [[ ! -f csv/$airport/$dt.csv ]]
        then
            sleep 1
            curl -s $URL | sed 's/<br \/>$//g' > csv/$airport/$dt.csv
        fi
    done
done<commit_msg>Basic output messages for skipped, downloaded<commit_after>#!/usr/bin/env bash
#
# Usage: ./wu.sh [years]
#
# If years is not provided as an argument, a default value of 1 is used.
YEARS=${1:-1}

for airport in `cat airports.txt`
do
    mkdir -pv csv/$airport
    for dt in `python -c "from datetime import datetime, timedelta; \
        now = datetime.now(); \
        print '\n'.join(((now - timedelta(days=day)).strftime('%Y-%m-%d')) \
        for day in range((now - datetime(now.year - $YEARS, now.month, \
            now.day)).days, 0, -1))"`
    do
        URL=`echo $airport $dt \
            | sed 's/\-/ /g' \
            | awk '{printf("https://www.wunderground.com/history/airport/%s/%d/%d/%d/DailyHistory.html?format=1\n", $1, $2, $3, $4)}'`
        echo -n "$airport on $dt... "
        if [[ ! -f csv/$airport/$dt.csv ]]
        then
            sleep 1
            curl -s $URL | sed 's/<br \/>$//g' > csv/$airport/$dt.csv
            echo "Done"
        else
            echo "Skipped"
        fi
    done
done
{'fraction_non_alphanumeric': 0.18574401664932363, 'fraction_numerical': 0.011446409989594173, 'mean_word_length': 1.9092284417549168, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5569642', 'n_tokens_mistral': 712, 'n_tokens_neox': 670, 'n_words': 182}