<commit_before># encode: UTF-8
require 'processors/twitter'
require 'processors/http'
require 'processors/photo_service'
require 'processors/tumblr'
require 'combinators/twitter_tumblr'
require 'log_aware'

class Tweetlr

  VERSION = '0.1.10'
  
  API_ENDPOINT_TWITTER = 'http://search.twitter.com/search.json'
  API_ENDPOINT_TUMBLR = 'http://www.tumblr.com'
  TWITTER_RESULTS_PER_PAGE = 100
  TWITTER_RESULTS_TYPE = 'recent'
  UPDATE_PERIOD = 600 #10 minutes
  
  include LogAware
  def self.log
    LogAware.log #TODO why doesn't the include make the log method accessible?
  end
  
  def initialize(args)
    log = Logger.new(STDOUT)
    if (Logger::DEBUG..Logger::UNKNOWN).to_a.index(args[:loglevel])
      log.level = args[:loglevel] 
    else
      log.level = Logger::INFO
    end
    log.debug "log level set to #{log.level}"
    LogAware.log=log
    
    @email = args[:tumblr_email]
    @password = args[:<PASSWORD>]
    @cookie = args[:cookie]
    @api_endpoint_twitter = 
    @api_endpoint_tumblr = args[:api_endpoint_tumblr] || API_ENDPOINT_TUMBLR
    @whitelist = args[:whitelist]
    @shouts = args[:shouts]
    @update_period = args[:update_period] || UPDATE_PERIOD
    @whitelist.each {|entry| entry.downcase!} if @whitelist
  end
  
  def self.crawl(config)
    log.debug "#{self}.crawl() using config: #{config.inspect}"
    twitter_config = {
      :since_id => config[:since_id] || config[:start_at_tweet_id],
      :search_term => config[:terms] || config[:search_term] ,
      :results_per_page => config[:results_per_page] || TWITTER_RESULTS_PER_PAGE,<commit_msg>glitch that did not take effect since empty twitter endpoint is taken care of by crawl action. elsewhise: NUKE<commit_after># encode: UTF-8
require 'processors/twitter'
require 'processors/http'
require 'processors/photo_service'
require 'processors/tumblr'
require 'combinators/twitter_tumblr'
require 'log_aware'

class Tweetlr

  VERSION = '0.1.10'
  
  API_ENDPOINT_TWITTER = 'http://search.twitter.com/search.json'
  API_ENDPOINT_TUMBLR = 'http://www.tumblr.com'
  TWITTER_RESULTS_PER_PAGE = 100
  TWITTER_RESULTS_TYPE = 'recent'
  UPDATE_PERIOD = 600 #10 minutes
  
  include LogAware
  def self.log
    LogAware.log #TODO why doesn't the include make the log method accessible?
  end
  
  def initialize(args)
    log = Logger.new(STDOUT)
    if (Logger::DEBUG..Logger::UNKNOWN).to_a.index(args[:loglevel])
      log.level = args[:loglevel] 
    else
      log.level = Logger::INFO
    end
    log.debug "log level set to #{log.level}"
    LogAware.log=log
    
    @email = args[:tumblr_email]
    @password = args[:<PASSWORD>]
    @cookie = args[:cookie]
    @api_endpoint_twitter = args[:api_endpoint_twitter] || API_ENDPOINT_TWITTER
    @api_endpoint_tumblr = args[:api_endpoint_tumblr] || API_ENDPOINT_TUMBLR
    @whitelist = args[:whitelist]
    @shouts = args[:shouts]
    @update_period = args[:update_period] || UPDATE_PERIOD
    @whitelist.each {|entry| entry.downcase!} if @whitelist
  end
  
  def self.crawl(config)
    log.debug "#{self}.crawl() using config: #{config.inspect}"
    twitter_config = {
      :since_id => config[:since_id] || config[:start_at_tweet_id],
      :search_term => config[:terms] || config[:search_term] ,
      :results_per_page => config[:results_per_page] || TWITTER_RESULTS_PER_PAGE,
{'fraction_non_alphanumeric': 0.12251655629139073, 'fraction_numerical': 0.007826610475617099, 'mean_word_length': 4.081039755351682, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2523727', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1150, 'n_words': 231}