<commit_before>#!/bin/sh
# Copyright (c) 2009 <NAME> <<EMAIL>>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

home_timeline="http://api.twitter.com/1/statuses/home_timeline.xml"
if [ ! -d ~/.twist ]; then
	mkdir ~/.twist
fi

# Transforms XML from the Twitter API status methods and makes them
# more awk-friendly. Takes the XML from stdin.
# Usage:
#   some-command | format_statuses
format_statuses()
{
	xmlstarlet sel -T -t -m "/statuses/status" -s D:N:- "id" -v "concat(id,'&#9;',user/screen_name,'&#9;',normalize-space(text))" -n
}

# Returns the number of <status> elements in the given XML. Takes
# the XML from stdin.
# Usage:
#   some-command | count_statuses
count_statuses()
{
	xmlstarlet sel -t -v "count(/statuses/status)"
}

# Builds and returns a URL used to fetch new tweets. Mostly used internally
# by the new_tweets function.
# Usage:
#   new_tweets_paged_url [page]
new_tweets_paged_url()
{
	local page base_url lastid
	page=${1:-0}
	base_url="${home_timeline}?page=${page}"
	if [ -f ~/.twist/lastid ]; then
		lastid=`cat ~/.twist/lastid`
	fi
	if [ -n "$lastid" ]; then<commit_msg>default page value should be 1, not 0. whoops<commit_after>#!/bin/sh
# Copyright (c) 2009 <NAME> <<EMAIL>>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

home_timeline="http://api.twitter.com/1/statuses/home_timeline.xml"
if [ ! -d ~/.twist ]; then
	mkdir ~/.twist
fi

# Transforms XML from the Twitter API status methods and makes them
# more awk-friendly. Takes the XML from stdin.
# Usage:
#   some-command | format_statuses
format_statuses()
{
	xmlstarlet sel -T -t -m "/statuses/status" -s D:N:- "id" -v "concat(id,'&#9;',user/screen_name,'&#9;',normalize-space(text))" -n
}

# Returns the number of <status> elements in the given XML. Takes
# the XML from stdin.
# Usage:
#   some-command | count_statuses
count_statuses()
{
	xmlstarlet sel -t -v "count(/statuses/status)"
}

# Builds and returns a URL used to fetch new tweets. Mostly used internally
# by the new_tweets function.
# Usage:
#   new_tweets_paged_url [page]
new_tweets_paged_url()
{
	local base_url lastid
	base_url="${home_timeline}?page=${1:-1}"
	if [ -f ~/.twist/lastid ]; then
		lastid=`cat ~/.twist/lastid`
	fi
	if [ -n "$lastid" ]; then
{'fraction_non_alphanumeric': 0.10783487784330244, 'fraction_numerical': 0.005616399887672002, 'mean_word_length': 4.30849478390462, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6897637', 'n_tokens_mistral': 1201, 'n_tokens_neox': 1128, 'n_words': 447}