<commit_before>#!/bin/bash

set -exo pipefail

LATEST_CRAWL=$(basename $(ls stats/CC-MAIN-201*.gz | tail -n 1) .gz)

function update_json() {
    regex="$1"
    excerpt="$2"
    if [ -e "$excerpt" ] && grep -qF "$LATEST_CRAWL" $excerpt; then
        zgrep -Eh "$regex" stats/$LATEST_CRAWL.gz  >>$excerpt
    else
        zcat stats/CC-MAIN-*.gz | grep -Eh "$regex" >$excerpt
    fi
}

# filter data to speed-up reading while plotting
update_json '^\["size'           stats/excerpt/size.json
update_json '^\["histogram"'     stats/excerpt/histogram.json
update_json '^\["tld"'           stats/excerpt/tld.json
update_json '^\["(size|mimetype)"'      stats/excerpt/mimetype.json
update_json '^\["(size|charset)"'       stats/excerpt/charset.json
update_json '^\["(size|primary_language|languages)"' stats/excerpt/language.json

python3 plot/crawl_size.py <stats/excerpt/size.json

python3 plot/overlap.py    <stats/excerpt/size.json

python3 plot/histogram.py  <stats/excerpt/histogram.json

(cat stats/crawler/CC-MAIN-*.json; grep -E '"CC-MAIN-201(6-[^0][0-9]|[789]-)' stats/excerpt/size.json) \
	| python3 plot/crawler_metrics.py

python3 plot/tld.py CC-MAIN-2016-07 CC-MAIN-2017-04 CC-MAIN-2018-05 $LATEST_CRAWL <stats/excerpt/tld.json

python3 plot/mimetype.py <stats/excerpt/mimetype.json

python3 plot/charset.py  <stats/excerpt/charset.json
python3 plot/language.py <stats/excerpt/language.json<commit_msg>Improve plotting over multiple crawls: store excerpts compressed<commit_after>#!/bin/bash

set -exo pipefail

LATEST_CRAWL=$(basename $(ls stats/CC-MAIN-201*.gz | tail -n 1) .gz)

function update_json() {
    regex="$1"
    excerpt="$2"
    if [ -e "$excerpt" ]; then
        if ! zgrep -qF "$LATEST_CRAWL" $excerpt; then
            zgrep -Eh "$regex" stats/$LATEST_CRAWL.gz | gzip >>$excerpt
        fi
    else
        zcat stats/CC-MAIN-*.gz | grep -Eh "$regex" | gzip  >$excerpt
    fi
}

# filter data to speed-up reading while plotting
update_json '^\["size'                               stats/excerpt/size.json.gz
update_json '^\["histogram"'                         stats/excerpt/histogram.json.gz
update_json '^\["tld"'                               stats/excerpt/tld.json.gz
update_json '^\["(size|mimetype)"'                   stats/excerpt/mimetype.json.gz
update_json '^\["(size|charset)"'                    stats/excerpt/charset.json.gz
update_json '^\["(size|primary_language|languages)"' stats/excerpt/language.json.gz

zcat stats/excerpt/size.json.gz \
     | python3 plot/crawl_size.py

zcat stats/excerpt/size.json.gz \
     | python3 plot/overlap.py

zcat stats/excerpt/histogram.json.gz \
    | python3 plot/histogram.py

(cat stats/crawler/CC-MAIN-*.json;
 zgrep -E '"CC-MAIN-201(6-[^0][0-9]|[789]-)' stats/excerpt/size.json.gz) \
	| python3 plot/crawler_metrics.py

zcat stats/excerpt/tld.json.gz \
    | python3 plot/tld.py CC-MAIN-2016-07 CC-MAIN-2017-04 CC-MAIN-2018-05 $LATEST_CRAWL

zcat stats/excerpt/mimetype.json.gz \
    | python3 plot/mimetype.py

zcat stats/excerpt/charset.json.gz \
    | python3 plot/charset.py

zcat stats/excerpt/language.json.gz \
    | python3 plot/language.py
{'fraction_non_alphanumeric': 0.15394442670073458, 'fraction_numerical': 0.02682848930054296, 'mean_word_length': 2.9948979591836733, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4051891', 'n_tokens_mistral': 1286, 'n_tokens_neox': 1167, 'n_words': 206}