<commit_before>#!/bin/bash

echo '<html>' > index.html
echo '<head>' >> index.html
echo '<title>NBA ESPN Stats & Info</title>' >> index.html
echo '<meta charset="UTF-8">' >> index.html
echo '</head>' >> index.html
echo '<body style="background-color: #efefef;">' >> index.html
echo '<center>' >> index.html
echo '<img id="loading" src="loading.gif" />' >> index.html
echo '<script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>' >> index.html
echo '<div id="content" style="display: none;">' >> index.html


newest_id_to_get=$(\
	curl -s --compressed 'https://twitter.com/ESPNStatsInfo' | \
	tr -d '\n' | \
	sed 's/data-tweet-id/\ndata-tweet-id/g' | \
	grep -F TweetText | \
	head -1 | \
	awk -F '"' '{print $2}' \
)


#get last max tweets
max=25
count=0
page_count=0
found_ids=''
while [ $count -lt $max ]; do 
	tweets=$( \
		curl -s --compressed "https://mobile.twitter.com/i/rw/profile/timeline?max_id=$newest_id_to_get&screen_name=ESPNStatsInfo&type=tweets" | \
		tr -d '\n' | \
		sed 's/data-tweet-id/\ndata-tweet-id/g' | \
		grep -F TweetText \
	)

	page=$( \
		echo "$tweets" | \
		grep -F -f keywords.list | \
		awk -F '"' '{print $2}' \
	)

	found_ids=$(printf "$found_ids\n$page" | sort -r -n | uniq)
	
	count=$(echo "$found_ids" | wc -l)
	page_count=$((page_count+1))
	
	echo "$count $max $page_count" | awk '{print "found=" $1, "looking_for=" $2, "page_count=" $3, "progress=" 100*$1/$2 "%"}'

	newest_id_to_get=$(echo "$tweets" | tail -1 | awk -F '"' '{print $2}')
done

echo "$found_ids" | head -$max | \
xargs -n 1 -I {} curl -s --compressed "https://api.twitter.com/1.1/statuses/oembed.json?id={}" | \
jq --raw-output '.html' | \
sed 's/<script.*script>/ /g' >> index.html

echo '</div>' >> index.html
echo '</center>' >> index.html
echo '<script src="index.js" charset="utf-8"></script>' >> index.html
echo '</body>' >> index.html
echo '</html>' >> index.html<commit_msg>smoother reload X2<commit_after>#!/bin/bash

echo '<html>' > index.html
echo '<head>' >> index.html
echo '<title>NBA ESPN Stats & Info</title>' >> index.html
echo '<meta charset="UTF-8">' >> index.html
echo '</head>' >> index.html
echo '<body style="background-color: #efefef;">' >> index.html
echo '<center>' >> index.html
echo '<img id="loading" src="loading.gif" />' >> index.html
echo '<div id="content" style="display: none;">' >> index.html


newest_id_to_get=$(\
	curl -s --compressed 'https://twitter.com/ESPNStatsInfo' | \
	tr -d '\n' | \
	sed 's/data-tweet-id/\ndata-tweet-id/g' | \
	grep -F TweetText | \
	head -1 | \
	awk -F '"' '{print $2}' \
)


#get last max tweets
max=25
count=0
page_count=0
found_ids=''
while [ $count -lt $max ]; do 
	tweets=$( \
		curl -s --compressed "https://mobile.twitter.com/i/rw/profile/timeline?max_id=$newest_id_to_get&screen_name=ESPNStatsInfo&type=tweets" | \
		tr -d '\n' | \
		sed 's/data-tweet-id/\ndata-tweet-id/g' | \
		grep -F TweetText \
	)

	page=$( \
		echo "$tweets" | \
		grep -F -f keywords.list | \
		awk -F '"' '{print $2}' \
	)

	found_ids=$(printf "$found_ids\n$page" | sort -r -n | uniq)
	
	count=$(echo "$found_ids" | wc -l)
	page_count=$((page_count+1))
	
	echo "$count $max $page_count" | awk '{print "found=" $1, "looking_for=" $2, "page_count=" $3, "progress=" 100*$1/$2 "%"}'

	newest_id_to_get=$(echo "$tweets" | tail -1 | awk -F '"' '{print $2}')
done

echo "$found_ids" | head -$max | \
xargs -n 1 -I {} curl -s --compressed "https://api.twitter.com/1.1/statuses/oembed.json?id={}" | \
jq --raw-output '.html' | \
sed 's/<script.*script>/ /g' >> index.html

echo '</div>' >> index.html
echo '</center>' >> index.html
echo '<script src="index.js" charset="utf-8"></script>' >> index.html
echo '<script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>' >> index.html
echo '</body>' >> index.html
echo '</html>' >> index.html

{'fraction_non_alphanumeric': 0.23309053069719043, 'fraction_numerical': 0.012747138397502602, 'mean_word_length': 3.6103117505995206, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 103, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2792621', 'n_tokens_mistral': 1582, 'n_tokens_neox': 1508, 'n_words': 343}