<commit_before>#!/bin/bash

# Set up strict bash
# See http://redsymbol.net/articles/unofficial-bash-strict-mode/
set -euo pipefail
IFS=$'\n\t'

input_dir="/mnt/fluiddata/cconsonni/pagecounts/data/output"

for year in {2008..2016}
do
    all_files="$input_dir/$year-*/output-*"
    ./parse_dumps.sh -k "./../data/keywords.txt" -i "$all_files" -o "./output/$year" --debug
    ./generate_weekly_data.py ./output/$year/result.csv ./output/$year/$year.csv
done
<commit_msg>Fix wrong file extension.<commit_after>#!/bin/bash

# Set up strict bash
# See http://redsymbol.net/articles/unofficial-bash-strict-mode/
set -euo pipefail
IFS=$'\n\t'

input_dir="/mnt/fluiddata/cconsonni/pagecounts/data/output"

for year in {2009..2016}
do
    all_files="$input_dir/$year-*/output-*"
    ./parse_dumps.sh -k "./../data/keywords.txt" -i "$all_files" -o "./output/$year" --debug
    ./generate_weekly_data.py ./output/$year/result.output ./output/$year/$year.csv
done

{'fraction_non_alphanumeric': 0.19412381951731375, 'fraction_numerical': 0.016789087093389297, 'mean_word_length': 3.4788732394366195, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5687770', 'n_tokens_mistral': 382, 'n_tokens_neox': 353, 'n_words': 63}