<commit_before>#!/bin/sh

# This fetches all google code source pages and reports those that have moved
# to github.

set -e

all=$(sed -n 's@Source Code:\(.*code\.google\.com.*/source\)@\1@p' metadata/*.txt | sort -u)
len=$(echo "$all" | wc -l)

echo "$len apps left"

echo "$all" | sort -u | shuf | while read url; do
	printf "."
	
	found=$(curl -s $url/checkout | sed -n 's/.*<A HREF="\(.*\)">here<\/A>.*/\1/p')
	if [ -n "$found" ]; then
		printf "\n%s\n" $url
		printf "\tProject moved: %s\n" $found
	fi
done<commit_msg>google-code-moved: support pages that directly redirect<commit_after>#!/bin/sh

# This fetches all google code source pages and reports those that have moved
# to github.

set -e

all=$(sed -n 's@Source Code:\(.*code\.google\.com.*/source\)@\1@p' metadata/*.txt | sort -u)
len=$(echo "$all" | wc -l)

echo "$len apps left"

moved() {
	printf "\n%s\n" $1
	printf "\tProject moved: %s\n" $2
}

echo "$all" | sort -u | shuf | while read url; do
	printf "."

	redurl=$(curl -s -I $url/checkout | perl -n -e '/^Location: (.*)$/ && print "$1\n"')

	if echo $redurl | grep -q hosting/moved; then
		found=$(curl -L -s $url/checkout | sed -n 's/.*View the project at: <a href="\(.*\)">.*/\1/p')
		moved $url $found
		continue
	fi
	if [ -n "$redurl" ] && [ "$redurl" != "$url" ]; then
		moved $url $redurl
		continue
	fi
	
	found=$(curl -L -s $url/checkout | sed -n 's/.*<A HREF="\(.*\)">here<\/A>.*/\1/p')
	if [ -n "$found" ]; then
		moved $url $found
		continue
	fi
done

echo
{'fraction_non_alphanumeric': 0.22386058981233245, 'fraction_numerical': 0.005361930294906166, 'mean_word_length': 2.9497354497354498, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1975122', 'n_tokens_mistral': 635, 'n_tokens_neox': 593, 'n_words': 184}