<commit_before>#!/bin/bash -eu
#
# This script checks the MD5 sum of /tmp/linux-3.0.tar.gz and downloads
# a fresh copy if its incorrect or the file is missing.

TGZ=/tmp/linux-3.0.tar.gz

MD5_ACTUAL="$(md5sum $TGZ | cut -f1 -d' ')"
MD5_WANT="f7e6591d86a9dbe123dfd1a0be054e7f"

if [[ $MD5_ACTUAL != $MD5_WANT ]]; then
	echo "Downloading linux-3.0.tar.gz"
	wget -nv --show-progress -c -O $TGZ \
		https://cdn.kernel.org/pub/linux/kernel/v3.0/linux-3.0.tar.gz
fi
<commit_msg>tests: only check the size in dl-linux-tarball.bash<commit_after>#!/bin/bash -eu
#
# This script checks the size of /tmp/linux-3.0.tar.gz and downloads
# a fresh copy if the size is incorrect or the file is missing.

TGZ=/tmp/linux-3.0.tar.gz

SIZE_WANT=96675825
SIZE_ACTUAL=0
if [[ -e $TGZ ]]; then
	SIZE_ACTUAL=$(stat -c %s $TGZ)
fi

if [[ $SIZE_ACTUAL -ne $SIZE_WANT ]]; then
	echo "Downloading linux-3.0.tar.gz"
	wget -nv --show-progress -c -O $TGZ \
		https://cdn.kernel.org/pub/linux/kernel/v3.0/linux-3.0.tar.gz
fi

{'fraction_non_alphanumeric': 0.16080402010050251, 'fraction_numerical': 0.053266331658291456, 'mean_word_length': 3.4070796460176993, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5296204', 'n_tokens_mistral': 465, 'n_tokens_neox': 430, 'n_words': 103}