<commit_before>class Pupil
  class Stream
    attr_reader :screen_name
    class StreamError < StandardError ; end
    STREAM_APIS = {
      :userstream => "https://userstream.twitter.com/2/user.json",
      :filter => "https://stream.twitter.com/1/statuses/filter.json%s"
    }

    def initialize key
      @screen_name = key[:screen_name]

      @consumer = OAuth::Consumer.new(
      key[:consumer_key],
      key[:consumer_secret],
      :site => TWITTER_API_URL
      )
      @access_token = OAuth::AccessToken.new(
      @consumer,
      key[:access_token],
      key[:access_token_secret]
      )
    end

    def start(type, param=nil, &block)
      raise ArgumentError unless block_given?
      
      run_get_stream type, param, &block
    end
    
    def run_get_stream(type, param=nil, &block)
      uri = URI.parse(STREAM_APIS[type] % Pupil.param_serializer(param))
      https = Net::HTTP.new(uri.host, uri.port)
      https.use_ssl = true
      https.verify_mode = OpenSSL::SSL::VERIFY_NONE
      https.verify_depth = 5

      while true do
        begin
          https.start do |https|
            request = Net::HTTP::Get.new(uri.request_uri)
            request["User-Agent"] = "Ruby/#{RUBY_VERSION} Pupil::Stream"
            request.oauth!(https, @consumer, @access_token)
            buf = ""
            https.request(request) do |response|
              response.read_body do |chunk|
                buf << chunk
                while (line = buf[/.+?(\r\n)+/m]) != nil
                  begin
                    buf.sub!(line,"")
                    line.strip!
                    status = JSON.parse(line)
                  rescue
                    break
                  end
                  
                  block.call StreamEvent.new(status)
                end
              end
            end
          end
        rescue => vars
          raise StreamError, "StreamError: #{vars}"
        end
      end
    end
    
    class StreamEvent
      attr_reader :type
      attr_reader :content
      
      def initialize status
        @type, @content = self.guess_event status
      end
      
      def guess_event status
        if status["delete"]
          return :delete, status["delete"]["status"]
        elsif status["friends"]
          return :friends, status["friends"]
        elsif status["event"] == "favorite"
          return :favorite, status
        elsif status["retweeted_status"]
          return :retweet, status
        elsif status["text"]
          return :status, status#Pupil::Status.new(status)
        else
          return :unknown, status
        end
      end
    end
  end
end<commit_msg>New classes, Pupil::Stream::Status and Pupil::Stream::Shash<commit_after>class Pupil
  class Stream
    attr_reader :screen_name
    class StreamError < StandardError ; end
    STREAM_APIS = {
      :userstream => "https://userstream.twitter.com/2/user.json",
      :filter => "https://stream.twitter.com/1/statuses/filter.json%s"
    }

    def initialize key
      @screen_name = key[:screen_name]

      @consumer = OAuth::Consumer.new(
      key[:consumer_key],
      key[:consumer_secret],
      :site => TWITTER_API_URL
      )
      @access_token = OAuth::AccessToken.new(
      @consumer,
      key[:access_token],
      key[:access_token_secret]
      )
    end

    def start(type, param=nil, &block)
      raise ArgumentError unless block_given?
      
      run_get_stream type, param, &block
    end
    
    def run_get_stream(type, param=nil, &block)
      uri = URI.parse(STREAM_APIS[type] % Pupil.param_serializer(param))
      https = Net::HTTP.new(uri.host, uri.port)
      https.use_ssl = true
      https.verify_mode = OpenSSL::SSL::VERIFY_NONE
      https.verify_depth = 5

      while true do
        begin
          https.start do |https|
            request = Net::HTTP::Get.new(uri.request_uri)
            request["User-Agent"] = "Ruby/#{RUBY_VERSION} Pupil::Stream"
            request.oauth!(https, @consumer, @access_token)
            buf = ""
            https.request(request) do |response|
              response.read_body do |chunk|
                buf << chunk
                while (line = buf[/.+?(\r\n)+/m]) != nil
                  begin
                    buf.sub!(line,"")
                    line.strip!
                    status = JSON.parse(line)
                  rescue
                    break
                  end
                  
                  event = self.guess_event status
                  block.call event
                end
              end
            end
          end
        rescue => vars
          raise StreamError, "StreamError: #{vars}"
        end
      end
    end
    
    def guess_event status
      if status["delete"]
        return Shash.new(:delete, status["delete"]["status"])
      elsif status["friends"]
        return Shash.new(:friends, status["friends"])
      elsif status["event"] == "favorite"
        return Shash.new(:favorite, status)
      elsif status["retweeted_status"]
        return Status.new(status, :retweeted)
      elsif status["text"]
        return Status.new(status)
      else
        return Shash.new(:unknown, status)
      end
    end
    
    # Stream Status
    class Status < Pupil::Status
      attr_reader :event
      attr_reader :retweeted_status
      
      def initialize status, event=nil
        super(status)
        @event = (event)? event : :status
        @retweeted_status = status["retweeted_status"]
      end
    end
    
    # Stream Hash
    class Shash
      attr_reader :event
      
      def initialize event, status
        @hash = status
        @event = event
      end
      
      def [] param
        @hash[param]
      end
      
      def size
        @hash.size
      end
      
      method_alias :length, :size
    end
  end
end
{'fraction_non_alphanumeric': 0.10217354098921787, 'fraction_numerical': 0.0010268697586856067, 'mean_word_length': 1.6980609418282548, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7625591', 'n_tokens_mistral': 1716, 'n_tokens_neox': 1660, 'n_words': 367}