<commit_before>#!/usr/local/bin/ruby
require 'rubygems'
gem 'twitter'
require 'twitter'
require 'builder'
require 'fileutils'

TINYURLS_YAML = File.join(File.dirname(__FILE__), 'tinyurls.yml')
TINYURLS = YAML.load_file(TINYURLS_YAML) rescue {}

TINYURL_REGEX = %r{(http://(?:tinyurl\.com|bit\.ly|jm\.p)/[a-zA-Z0-9]+)}
HTTP_REGEX = URI.regexp(%w(http))

def untinyurl(url)
  if url =~ TINYURL_REGEX
    return TINYURLS[$1] if TINYURLS[$1]

    TINYURLS[$1] = Net::HTTP.get_response( URI.parse($1) )["location"]
  else
    url
  end
end

def mp3_length(url)
  uri = URI.parse(url)
  resp = nil
  Net::HTTP.start(uri.host, uri.port) {|http|
    resp = http.head(uri.path)
  }
  resp["content-length"]
end

PODCAST_FILE = File.expand_path("~/www/nuclearsquid.com/djgoatboy.rss")

begin
  config = YAML.load_file(File.expand_path("~/.twitter_config.yml"))
  httpauth = Twitter::HTTPAuth.new(config['username'], config['password'])

  client = Twitter::Base.new(httpauth)
  tweets = client.user_timeline(:id => "djgoatboy")

  File.open(PODCAST_FILE + '.new', 'w+') do |file|
    builder = Builder::XmlMarkup.new(:target => file, :indent => 4)
    builder.instruct!
    # There's gotta be a better way than this
    podcast = builder.rss("version" => "2.0") { |rss|
      rss.channel { |channel|
        channel.title "@djgoatboy's mp3 feed"
        channel.link "http://twitter.com/djgoatboy"
        channel.language 'en-US'

        tweets.each do |tweet|
          if tweet.text =~ HTTP_REGEX
            url = untinyurl(tweet.text.slice(HTTP_REGEX))

            # Make sure we're getting a mp3
            if url =~ /\.mp3$/
              length = mp3_length(url)

              channel.item {|item|<commit_msg>Switch to SuperTweet.Net for Twitter API Basic Auth<commit_after>#!/usr/local/bin/ruby
require 'rubygems'
gem 'twitter'
require 'twitter'
require 'builder'
require 'fileutils'

TINYURLS_YAML = File.join(File.dirname(__FILE__), 'tinyurls.yml')
TINYURLS = YAML.load_file(TINYURLS_YAML) rescue {}

TINYURL_REGEX = %r{(http://(?:tinyurl\.com|bit\.ly|jm\.p)/[a-zA-Z0-9]+)}
HTTP_REGEX = URI.regexp(%w(http))

def untinyurl(url)
  if url =~ TINYURL_REGEX
    return TINYURLS[$1] if TINYURLS[$1]

    TINYURLS[$1] = Net::HTTP.get_response( URI.parse($1) )["location"]
  else
    url
  end
end

def mp3_length(url)
  uri = URI.parse(url)
  resp = nil
  Net::HTTP.start(uri.host, uri.port) {|http|
    resp = http.head(uri.path)
  }
  resp["content-length"]
end

PODCAST_FILE = File.expand_path("~/www/nuclearsquid.com/djgoatboy.rss")

begin
  config = YAML.load_file(File.expand_path("~/.twitter_config.yml"))
  # Use SuperTweet.Net to get to twitter via Basic Auth
  httpauth = Twitter::HTTPAuth.new config['user'], config['password'], :api_endpoint => 'api.supertweet.net' #, :ssl => true

  client = Twitter::Base.new(httpauth)
  tweets = client.user_timeline(:id => "djgoatboy")

  File.open(PODCAST_FILE + '.new', 'w+') do |file|
    builder = Builder::XmlMarkup.new(:target => file, :indent => 4)
    builder.instruct!
    # There's gotta be a better way than this
    podcast = builder.rss("version" => "2.0") { |rss|
      rss.channel { |channel|
        channel.title "@djgoatboy's mp3 feed"
        channel.link "http://twitter.com/djgoatboy"
        channel.language 'en-US'

        tweets.each do |tweet|
          if tweet.text =~ HTTP_REGEX
            url = untinyurl(tweet.text.slice(HTTP_REGEX))

            # Make sure we're getting a mp3
            if url =~ /\.mp3$/
              length = mp3_length(url)

              channel.item {|item|
{'fraction_non_alphanumeric': 0.15872125630959058, 'fraction_numerical': 0.007851934941110488, 'mean_word_length': 3.3079710144927534, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1507525', 'n_tokens_mistral': 1388, 'n_tokens_neox': 1329, 'n_words': 258}