<commit_before>require "logstash/inputs/base"
require "logstash/namespace"
require "tweetstream" # rubygem 'tweetstream'

# TODO(sissel): This could use some refactoring.
class LogStash::Inputs::Twitter < LogStash::Inputs::Base

  config_name "twitter"
  config :user, :validate => :string, :required => true
  config :password, :validate => :password, :required => true
  config :debug, :validate => :boolean

  config :keywords, :validate => :array, :required => true

  public
  def initialize(params)
    super
    @debug ||= false
  end # def initialize

  def register
    # nothing to do
  end

  public
  def run(queue)
    loop do
      stream = TweetStream::Client.new(@user, @password.value)
      stream.track(@keywords) do |status|
        @logger.debug("Got twitter status from @#{status["user"]["source"]}")
        event = LogStash::Event.new(
          "@message" => status["text"],
          "@type" => @type,
          "@tags" => @tags.clone
        )

        event.fields.merge!(
          "user" => (status["user"]["screen_name"] rescue nil), 
          "client" => (status["user"]["source"] rescue nil),
          "retweeted" => (status["retweeted"] rescue nil)
        )

        event.fields["in-reply-to"] = status["in_reply_to_status_id"] if status["in_reply_to_status_id"]

        urls = status["entities"]["urls"] rescue []
        if urls.size > 0
          event.fields["urls"] = urls.collect { |u| u["url"] }
        end

        event.source = source
        @logger.debug(["Got event", event])
        queue << event
      end # stream.track

      # Some closure or error occured, sleep and try again.
      sleep 30
    end
  end
end # class LogStash::Inputs::Twitter
<commit_msg>remove comment<commit_after>require "logstash/inputs/base"
require "logstash/namespace"
require "tweetstream" # rubygem 'tweetstream'

class LogStash::Inputs::Twitter < LogStash::Inputs::Base

  config_name "twitter"
  config :user, :validate => :string, :required => true
  config :password, :validate => :password, :required => true
  config :debug, :validate => :boolean

  config :keywords, :validate => :array, :required => true

  public
  def initialize(params)
    super
    @debug ||= false
  end # def initialize

  def register
    # nothing to do
  end

  public
  def run(queue)
    loop do
      stream = TweetStream::Client.new(@user, @password.value)
      stream.track(@keywords) do |status|
        @logger.debug("Got twitter status from @#{status["user"]["source"]}")
        event = LogStash::Event.new(
          "@message" => status["text"],
          "@type" => @type,
          "@tags" => @tags.clone
        )

        event.fields.merge!(
          "user" => (status["user"]["screen_name"] rescue nil), 
          "client" => (status["user"]["source"] rescue nil),
          "retweeted" => (status["retweeted"] rescue nil)
        )

        event.fields["in-reply-to"] = status["in_reply_to_status_id"] if status["in_reply_to_status_id"]

        urls = status["entities"]["urls"] rescue []
        if urls.size > 0
          event.fields["urls"] = urls.collect { |u| u["url"] }
        end

        event.source = source
        @logger.debug(["Got event", event])
        queue << event
      end # stream.track

      # Some closure or error occured, sleep and try again.
      sleep 30
    end
  end
end # class LogStash::Inputs::Twitter

{'fraction_non_alphanumeric': 0.1479621972829297, 'fraction_numerical': 0.0017720023626698169, 'mean_word_length': 2.8401360544217686, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5084218', 'n_tokens_mistral': 1046, 'n_tokens_neox': 1041, 'n_words': 261}