<commit_before>require "em-twitter"
require "twitter"
require "json"
require "./tokens"

def twitter
  @_twitter ||= Twitter::REST::Client.new(consumer_key: CONSUMER_KEY, consumer_secret: CONSUMER_SECRET, oauth_token: OAUTH_TOKEN, oauth_token_secret: OAUTH_TOKEN_SECRET)
end

def user(update: false)
  if !@_user || update
    @_user = twitter.user
  end

  @_user
end

def serve(event, json)
  @events ||= {}
  @events[event] && @events[event].each do |c|
    EM.defer do
      begin
        c.call(json)
      rescue
        puts "Failed to execute callback: #{event}: #{$!}"
        puts $@
      end
    end
  end
end

def on_event(event, &blk)
  @events ||= {}
  @events[event] ||= []
  @events[event] << blk
end

def on_tweet(regexp, &blk)
  on_event(:tweet) do |json|
    match = regexp.match(json[:text])
    if match
      blk.call(json, *match)
    end
  end
end

def mainloop
  EM.run do
    client = EM::Twitter::Client.new(
      host: "userstream.twitter.com",
      path: "/1.1/user.json",
      oauth: twitter.credentials,
      method: :get
    )

    client.each do |str|
      json = JSON.parse(str, symbolize_names: true) rescue next

      if json[:text]
        serve(:tweet, json)
      elsif json[:event]
        serve(json[:event].to_sym, json)
      end
    end

    client.connect
  end
end

def load_commands
  Dir.glob(File.expand_path("../commands/*.rb", __FILE__)) do |file|
    begin
      load file
    rescue
      puts "Failed to load command: #{File.basename(file)}: #{$!}"
      puts $@
    end
  end
end

load_commands
mainloop

<commit_msg>worker: ignore retweets<commit_after>require "em-twitter"
require "twitter"
require "json"
require "./tokens"

def twitter
  @_twitter ||= Twitter::REST::Client.new(consumer_key: CONSUMER_KEY, consumer_secret: CONSUMER_SECRET, oauth_token: OAUTH_TOKEN, oauth_token_secret: OAUTH_TOKEN_SECRET)
end

def user(update: false)
  if !@_user || update
    @_user = twitter.user
  end

  @_user
end

def serve(event, json)
  @events ||= {}
  @events[event] && @events[event].each do |c|
    EM.defer do
      begin
        c.call(json)
      rescue
        puts "Failed to execute callback: #{event}: #{$!}"
        puts $@
      end
    end
  end
end

def on_event(event, &blk)
  @events ||= {}
  @events[event] ||= []
  @events[event] << blk
end

def on_tweet(regexp, &blk)
  on_event(:tweet) do |json|
    match = regexp.match(json[:text])
    if match
      blk.call(json, *match)
    end
  end
end

def mainloop
  EM.run do
    client = EM::Twitter::Client.new(
      host: "userstream.twitter.com",
      path: "/1.1/user.json",
      oauth: twitter.credentials,
      method: :get
    )

    client.each do |str|
      json = JSON.parse(str, symbolize_names: true) rescue next

      if json[:text] && !json[:retweeted_status]
        serve(:tweet, json)
      elsif json[:event]
        serve(json[:event].to_sym, json)
      end
    end

    client.connect
  end
end

def load_commands
  Dir.glob(File.expand_path("../commands/*.rb", __FILE__)) do |file|
    begin
      load file
    rescue
      puts "Failed to load command: #{File.basename(file)}: #{$!}"
      puts $@
    end
  end
end

load_commands
mainloop


{'fraction_non_alphanumeric': 0.13610763454317898, 'fraction_numerical': 0.0012515644555694619, 'mean_word_length': 2.810488676996424, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1134472', 'n_tokens_mistral': 1164, 'n_tokens_neox': 1148, 'n_words': 275}