<commit_before>#!/bin/bash
# Returns total responses on stdout
# Returns total bytes read on stderr
# Naive testing showed wc --bytes faster than dd or pv (with pv not being 100% accurate)
#
zcat /dev/stdin | tee >(wc --bytes | awk '{ printf "Bytes:\t" $1 "\n" }' > /dev/stderr) | grep "WARC-Type: response" | wc -l | awk '{ printf "Responses:\t" $1 "\n" }' > /dev/stdout<commit_msg>`count_responses` update: retrieves and processes a single file<commit_after>#!/bin/bash
# Saves total response count to responses folder
# Returns total byte count to size folder
# Naive testing showed wc --bytes faster than dd or pv (with pv not being 100% accurate)
#
mkdir -p responses
mkdir -p size
s3cmd get s3://aws-publicdatasets/$1 - | zcat | tee >(wc --bytes > size/`basename $1`) | grep "WARC-Type: response" | wc -l > responses/`basename $1`
# Helpful:
# As there's no good error checking, errors sometimes pop up leaving empty or tiny results
# cat * | grep -E '[0-9]{6,10}' | wc -l
# cat * | grep -E '[0-9]{6,10}' | awk '{ SUM += $1} END { print SUM }'
###
# cat * | grep -E '[0-9]{4,10}' | wc -l
# cat * | grep -E '[0-9]{4,10}' | awk '{ SUM += $1} END { print SUM }'
{'fraction_non_alphanumeric': 0.16752577319587628, 'fraction_numerical': 0.03006872852233677, 'mean_word_length': 3.5686274509803924, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6494265', 'n_tokens_mistral': 440, 'n_tokens_neox': 407, 'n_words': 157}