<commit_before>#!/usr/bin/ruby

ENV['GEM_PATH'] = '/home/singpolyma/.gems:/usr/lib/ruby/gems/1.8'

require 'rubygems'
require 'open-uri'
require 'json'
require 'digest/sha1'
require 'ombpost'

config_path = 'tokens/' + Digest::SHA1.new("http://tw2omb.singpolyma.net/profile.php?user=#{ARGV[0]}").to_s
config = open(config_path).read.split(/\n/)

statuses = if config[4]
	JSON::parse(open("http://search.twitter.com/search.json?q=from%3A#{ARGV[0]}&since_id=#{config[4]}").read)['results']
else
	JSON::parse(open("http://search.twitter.com/search.json?q=from%3A#{ARGV[0]}").read)['results']
end

statuses.reverse_each do |el|
	omb_post(config[2], config[3], config[0], config[1], el['id'], el['text'])
end

if statuses.length > 0
	config[4] = statuses.shift['id']
	#File.new(config_path, 'w').write(config.join("\n"))
end
<commit_msg>write out the last id again<commit_after>#!/usr/bin/ruby

ENV['GEM_PATH'] = '/home/singpolyma/.gems:/usr/lib/ruby/gems/1.8'

require 'rubygems'
require 'open-uri'
require 'json'
require 'digest/sha1'
require 'ombpost'

config_path = 'tokens/' + Digest::SHA1.new("http://tw2omb.singpolyma.net/profile.php?user=#{ARGV[0]}").to_s
config = open(config_path).read.split(/\n/)

statuses = if config[4]
	JSON::parse(open("http://search.twitter.com/search.json?q=from%3A#{ARGV[0]}&since_id=#{config[4]}").read)['results']
else
	JSON::parse(open("http://search.twitter.com/search.json?q=from%3A#{ARGV[0]}").read)['results']
end

statuses.reverse_each do |el|
	omb_post(config[2], config[3], config[0], config[1], el['id'], el['text'])
end

if statuses.length > 0
	config[4] = statuses.shift['id']
	File.new(config_path, 'w').write(config.join("\n"))
end

{'fraction_non_alphanumeric': 0.22957662492546213, 'fraction_numerical': 0.02146690518783542, 'mean_word_length': 6.359649122807017, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3988586', 'n_tokens_mistral': 689, 'n_tokens_neox': 676, 'n_words': 91}