<commit_before>#!/bin/bash
## Notes to go here...
run=5
batch=10
pause=2

internet_wait ()
{
	nc -z -w 1 -4 8.8.8.8 53 > /dev/null 2>&1
	result=$?
	while [ $result -ne 0 ]; do
		sleep $pause
		echo "Offline"
		nc -z -w 1 -4 8.8.8.8 53 > /dev/null 2>&1
		result=$?
	done
}

wget_wait ()
{
	count=$( ps -A | grep -o wget | wc -l )
	while [ $count -ge $run ]; do
		sleep $pause
		count=$( ps -A | grep -o wget | wc -l )
		#echo $count
	done
}

while :
do 
	internet_wait
	
	if [ -f top-1m.csv.zip ]; then
		wget -N https://s3.amazonaws.com/alexa-static/top-1m.csv.zip 
	else
		wget -S https://s3.amazonaws.com/alexa-static/top-1m.csv.zip 
	fi

	if [ -f top-1million-sites.csv.zip ]; then
		wget -N https://statvoo.com/dl/top-1million-sites.csv.zip
	else
		wget -S https://statvoo.com/dl/top-1million-sites.csv.zip
	fi

	gzip -dc  top-1m.csv.zip | pv -s 22000000 | sed 's/[^,]*,//' > /tmp/test1.txt
	gzip -dc  top-1million-sites.csv.zip | pv -s 22000000 | sed 's/[^,]*,//' > /tmp/test2.txt

	cat /tmp/test1.txt /tmp/test2.txt | pv | sort -uR > sites.txt

	rm /tmp/test1.txt
	rm /tmp/test2.txt
	
	sites=0
	time=$SECONDS
	while read p; do
		sites=$((sites+1))
		printf "$sites -> $p\n"
        wget -b -4 -T 8 -t 1 -qO /dev/null --https-only --no-check-certificate https://$p > /dev/null
		if [ $((sites % batch)) -eq 0 ] ; then
			speed=$(( sites / ((SECONDS - time)+1)))
			printf "$speed websites per second.\n"
			internet_wait
			wget_wait
		fi
	done < sites.txt
done<commit_msg>more wget updates<commit_after>#!/bin/bash
## Notes to go here...
run=5
batch=10
pause=2

internet_wait ()
{
	nc -z -w 1 -4 8.8.8.8 53 > /dev/null 2>&1
	result=$?
	while [ $result -ne 0 ]; do
		sleep $pause
		echo "Offline"
		nc -z -w 1 -4 8.8.8.8 53 > /dev/null 2>&1
		result=$?
	done
}

wget_wait ()
{
	count=$( ps -A | grep -o wget | wc -l )
	while [ $count -ge $run ]; do
		sleep $pause
		count=$( ps -A | grep -o wget | wc -l )
		#echo $count
	done
}

while :
do 
	internet_wait
	
	if [ -f top-1m.csv.zip ]; then
		wget -N https://s3.amazonaws.com/alexa-static/top-1m.csv.zip 
	else
		wget -S https://s3.amazonaws.com/alexa-static/top-1m.csv.zip 
	fi

	if [ -f top-1million-sites.csv.zip ]; then
		wget -N https://statvoo.com/dl/top-1million-sites.csv.zip
	else
		wget -S https://statvoo.com/dl/top-1million-sites.csv.zip
	fi

	gzip -dc  top-1m.csv.zip | pv -s 22000000 | sed 's/[^,]*,//' > /tmp/test1.txt
	gzip -dc  top-1million-sites.csv.zip | pv -s 22000000 | sed 's/[^,]*,//' > /tmp/test2.txt

	cat /tmp/test1.txt /tmp/test2.txt | pv | sort -uR > sites.txt

	rm /tmp/test1.txt
	rm /tmp/test2.txt
	
	sites=0
	time=$SECONDS
	while read p; do
		sites=$((sites+1))
		printf "$sites -> $p\n"
        wget -b -4 -T 8 -t 1 -qO /dev/null --limit-rate=20k --https-only --no-check-certificate https://$p > /dev/null
		if [ $((sites % batch)) -eq 0 ] ; then
			speed=$(( sites / ((SECONDS - time)+1)))
			printf "$speed websites per second.\n"
			internet_wait
			wget_wait
		fi
	done < sites.txt
done
{'fraction_non_alphanumeric': 0.18878065166274774, 'fraction_numerical': 0.043668122270742356, 'mean_word_length': 2.3649717514124293, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1291679', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1318, 'n_words': 334}