<commit_before>#!/bin/bash

# TODO: figure out how to automatically update these:
export token='wikidot_token7=<KEY>';
export cookie='WIKIDOT_SESSION_ID_b18d85b5=_domain_cookie_2238949_38702de0c8db6cbf29c4d41cc092d7a7';

#gets records of all user votes on a page
getVotes(){

	pid=$2;
	pname=$1;

	response=$(mktemp);

	vote=$(mktemp);
	uid=$(mktemp);
	uname=$(mktemp);

	# get author vote data (since author's can't upvote their own work)

	curl 'http://www.scp-wiki.net/ajax-module-connector.php' \
	  -H "Cookie: wikidot_udsession=1; $cookie; $token;" \
	  -H 'Origin: http://www.scp-wiki.net' \
	  -H "Referer: http://www.scp-wiki.net/$pname" \
	  -H 'Accept-Encoding: gzip, deflate' \
	  -H 'Accept-Language: en-US,en;q=0.8' \
	  -H 'User-Agent: scpRank' \
	  -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' \
	  -H 'Accept: */*' \
	  -H 'X-Requested-With: XMLHttpRequest' \
	  -H 'Connection: keep-alive' \
	  --data "page=1&perpage=20&page_id=$pid&options=%7B%22new%22%3Atrue%7D&moduleName=history%2FPageRevisionListModule&callbackIndex=3&$token" \
	  --compressed \
	  --silent \
	| python2.7 -c "import json,sys;obj=json.load(sys.stdin);print obj['body'];" 2>/dev/null \
	> $response;

	if test $? -ne 0;
	then
		rm $response $vote $uid $uname;
		echo "$pname author" >> errors.log;
		exit 1;
	fi

	#extract user data from response
	echo '+1' \
	> $vote;

	cat $response \
	| hxselect -ci -s '\n' 'a' 2>/dev/null \
	| grep -oP '(?<=userid=)[0-9]*' \
	> $uid;

	cat $response \
	| hxselect -ci -s '\n' 'a' 2>/dev/null \
	| grep -oP '(?<=alt=")[^"]*' \
	| tr "[:upper:]" "[:lower:]" \
	> $uname;

	#requet user vote data
	curl 'http://www.scp-wiki.net/ajax-module-connector.php' \
	  -H "Cookie: wikidot_udsession=1; $cookie; $token;" \
	  -H 'Origin: http://www.scp-wiki.net' \
	  -H "Referer: http://www.scp-wiki.net/$pname" \
	  -H 'Accept-Encoding: gzip, deflate' \
	  -H 'Accept-Language: en-US,en;q=0.8' \
	  -H 'User-Agent: scpRank' \
	  -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' \
	  -H 'Accept: */*' \
	  -H 'X-Requested-With: XMLHttpRequest' \
	  -H 'Connection: keep-alive' \
	  --data "pageId=$pid&moduleName=pagerate%2FWhoRatedPageModule&callbackIndex=1&$token" \
	  --compressed \
	  --silent \
	| python2.7 -c "import json,sys;obj=json.load(sys.stdin);print obj['body'];" 2>/dev/null \
	> $response;

	if test $? -ne 0;
	then
		rm $response $vote $uid $uname;
		echo "$pname user" >> errors.log;
		exit 1;
	fi
	#extract vote data from response
	cat $response \
	| hxselect -ci -s '\n' 'span[style*="color:#777"]' \
	| tr -d ' ' \
	| tr -s '\n' \
	| sed -e 's/$/1/' \
	| tail -n +2 \
	>> $vote;

	cat $response \
	| hxselect -ci -s '\n' 'a' \
	| grep -oP '(?<=userid=)[0-9]*' \
	>> $uid;

	cat $response \
	| hxselect -ci -s '\n' 'a' \
	| grep -oP '(?<=alt=")[^"]*' \
	| tr "[:upper:]" "[:lower:]" \
	>> $uname;

	#zip data together into output format
	paste $uid $vote \
	| sed -e "s/^/$pid\t/" \
	| awk -F'\t+' 'NF == 3';

	paste $uid $uname \
	| awk -F'\t+' 'NF == 2' \
	>> uids2.tsv;

	#clean up temporary files
	rm $response $vote $uid $uname;
}
export -f getVotes;



#retrieves a list of page names
getPages(){
	python2.7 -c 'from whiffle import wikidotapi; print "\n".join(str(p) for p in wikidotapi.connection().Pages)';
}



#retrieves the pid number for a given page name
getPid(){
	echo -ne $1 '\t';
	curl --silent "http://www.scp-wiki.net/$1" \
	| grep -oP '(?<=pageId = )[0-9]*' ;
	exit $?;
}
export -f getPid;



# check robots.txt to make sure we are allowed in
# Currently, there are no restrictions at all, so this just checks in case
# a new rule is added that applies to this bot. If a rule gets added, we
# will need to add something else to make sure we are still allowed.
curl 'http://www.scp-wiki.net/robots.txt' \
  -H 'User-Agent: scpRank' \
  --silent \
| grep -i 'scpRank|User-agent: \*' \
> /dev/null

if [ ! $? ];
then<commit_msg>Removed extra space in tsv output<commit_after>#!/bin/bash

# TODO: figure out how to automatically update these:
export token='wikidot_token<PASSWORD>';
export cookie='WIKIDOT_SESSION_ID_b18d85b5=_domain_cookie_2238949_38702de0c8db6cbf29c4d41cc092d7a7';

#gets records of all user votes on a page
getVotes(){

	pid=$2;
	pname=$1;

	response=$(mktemp);

	vote=$(mktemp);
	uid=$(mktemp);
	uname=$(mktemp);

	# get author vote data (since author's can't upvote their own work)

	curl 'http://www.scp-wiki.net/ajax-module-connector.php' \
	  -H "Cookie: wikidot_udsession=1; $cookie; $token;" \
	  -H 'Origin: http://www.scp-wiki.net' \
	  -H "Referer: http://www.scp-wiki.net/$pname" \
	  -H 'Accept-Encoding: gzip, deflate' \
	  -H 'Accept-Language: en-US,en;q=0.8' \
	  -H 'User-Agent: scpRank' \
	  -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' \
	  -H 'Accept: */*' \
	  -H 'X-Requested-With: XMLHttpRequest' \
	  -H 'Connection: keep-alive' \
	  --data "page=1&perpage=20&page_id=$pid&options=%7B%22new%22%3Atrue%7D&moduleName=history%2FPageRevisionListModule&callbackIndex=3&$token" \
	  --compressed \
	  --silent \
	| python2.7 -c "import json,sys;obj=json.load(sys.stdin);print obj['body'];" 2>/dev/null \
	> $response;

	if test $? -ne 0;
	then
		rm $response $vote $uid $uname;
		echo "$pname author" >> errors.log;
		exit 1;
	fi

	#extract user data from response
	echo '+1' \
	> $vote;

	cat $response \
	| hxselect -ci -s '\n' 'a' 2>/dev/null \
	| grep -oP '(?<=userid=)[0-9]*' \
	> $uid;

	cat $response \
	| hxselect -ci -s '\n' 'a' 2>/dev/null \
	| grep -oP '(?<=alt=")[^"]*' \
	| tr "[:upper:]" "[:lower:]" \
	> $uname;

	#requet user vote data
	curl 'http://www.scp-wiki.net/ajax-module-connector.php' \
	  -H "Cookie: wikidot_udsession=1; $cookie; $token;" \
	  -H 'Origin: http://www.scp-wiki.net' \
	  -H "Referer: http://www.scp-wiki.net/$pname" \
	  -H 'Accept-Encoding: gzip, deflate' \
	  -H 'Accept-Language: en-US,en;q=0.8' \
	  -H 'User-Agent: scpRank' \
	  -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' \
	  -H 'Accept: */*' \
	  -H 'X-Requested-With: XMLHttpRequest' \
	  -H 'Connection: keep-alive' \
	  --data "pageId=$pid&moduleName=pagerate%2FWhoRatedPageModule&callbackIndex=1&$token" \
	  --compressed \
	  --silent \
	| python2.7 -c "import json,sys;obj=json.load(sys.stdin);print obj['body'];" 2>/dev/null \
	> $response;

	if test $? -ne 0;
	then
		rm $response $vote $uid $uname;
		echo "$pname user" >> errors.log;
		exit 1;
	fi
	#extract vote data from response
	cat $response \
	| hxselect -ci -s '\n' 'span[style*="color:#777"]' \
	| tr -d ' ' \
	| tr -s '\n' \
	| sed -e 's/$/1/' \
	| tail -n +2 \
	>> $vote;

	cat $response \
	| hxselect -ci -s '\n' 'a' \
	| grep -oP '(?<=userid=)[0-9]*' \
	>> $uid;

	cat $response \
	| hxselect -ci -s '\n' 'a' \
	| grep -oP '(?<=alt=")[^"]*' \
	| tr "[:upper:]" "[:lower:]" \
	>> $uname;

	#zip data together into output format
	paste $uid $vote \
	| sed -e "s/^/$pid\t/" \
	| awk -F'\t+' 'NF == 3';

	paste $uid $uname \
	| awk -F'\t+' 'NF == 2' \
	>> uids2.tsv;

	#clean up temporary files
	rm $response $vote $uid $uname;
}
export -f getVotes;



#retrieves a list of page names
getPages(){
	python2.7 -c 'from whiffle import wikidotapi; print "\n".join(str(p) for p in wikidotapi.connection().Pages)';
}



#retrieves the pid number for a given page name
getPid(){
	echo -ne $1'\t';
	curl --silent "http://www.scp-wiki.net/$1" \
	| grep -oP '(?<=pageId = )[0-9]*' ;
	exit $?;
}
export -f getPid;



# check robots.txt to make sure we are allowed in
# Currently, there are no restrictions at all, so this just checks in case
# a new rule is added that applies to this bot. If a rule gets added, we
# will need to add something else to make sure we are still allowed.
curl 'http://www.scp-wiki.net/robots.txt' \
  -H 'User-Agent: scpRank' \
  --silent \
| grep -i 'scpRank|User-agent: \*' \
> /dev/null

if [ ! $? ];
then
{'fraction_non_alphanumeric': 0.1994667343829355, 'fraction_numerical': 0.021711528694768917, 'mean_word_length': 3.0834629341627786, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 20, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1485652', 'n_tokens_mistral': 3513, 'n_tokens_neox': 3280, 'n_words': 797}