<commit_before># -*- encoding: utf-8 -*-
require 'stringex'

module OpenCalais
  class Response
    attr_accessor :raw, :language, :topics, :tags, :entities, :relations, :locations

    def initialize(response)
      @raw  = response

      @language = 'English'
      @topics    = []
      @tags      = []
      @entities  = []
      @relations = []
      @locations = []

      parse(response)
    end

    def humanize_topic(topic)
      begin
        topic.gsub('_', ' & ').titleize.remove_formatting
      rescue

      end
    end

    def importance_to_score(imp)
      case imp
      when 1 then 0.9
      when 2 then 0.7
      else 0.5
      end
    end

    def parse(response)
      begin
        r = response.body
        @language = r.doc.meta.language rescue nil
        @language = nil if @language == 'InputTextTooShort'

        r.each do |k,v|
          case v._typeGroup
          when 'topics'
            self.topics << {:name => humanize_topic(v.categoryName), :score => v.score.to_f, :original => v.categoryName}
          when 'socialTag'
            self.tags << {:name => v.name.gsub('_', ' and ').downcase, :score => importance_to_score(v.importance)}
          when 'entities'
            item = {:guid => k, :name => v.name, :type => v._type.remove_formatting.titleize, :score => 1.0}

            instances = Array(v.instances).select{|i| i.exact.downcase != item[:name].downcase }
            item[:matches] = instances if instances && instances.size > 0

            if OpenCalais::GEO_TYPES.include?(v._type)
              if (v.resolutions && v.resolutions.size > 0)
                r = v.resolutions.first
                item[:name]      = r.shortname || r.name
                item[:latitude]  = r.latitude
                item[:longitude] = r.longitude
                item[:country]   = r.containedbycountry if r.containedbycountry
                item[:state]     = r.containedbystate if r.containedbystate
              end
              self.locations << item
            else
              self.entities << item
            end
          when 'relations'
            item = v.reject{|k,v| k[0] == '_' || k == 'instances'} || {}
            item[:type] = v._type.remove_formatting.titleize
            self.relations << item
          end
        end

        # remove social tags which are in the topics list already
        topic_names = self.topics.collect{|topic| topic[:name].downcase}
        self.tags.delete_if{|tag| topic_names.include?(tag[:name]) }
      rescue Exception => e
        Rails.logger.info "Parse Error: #{e.message}"
      end
    end
  end
end<commit_msg>better error handling<commit_after># -*- encoding: utf-8 -*-
require 'stringex'

module OpenCalais
  class Response
    attr_accessor :raw, :language, :topics, :tags, :entities, :relations, :locations

    def initialize(response)
      @raw  = response

      @language = 'English'
      @topics    = []
      @tags      = []
      @entities  = []
      @relations = []
      @locations = []

      parse(response)
    end

    def humanize_topic(topic)
      begin
        topic.gsub('_', ' & ').titleize.remove_formatting
      rescue

      end
    end

    def importance_to_score(imp)
      case imp
      when 1 then 0.9
      when 2 then 0.7
      else 0.5
      end
    end

    def parse(response)
      begin
        r = response.body
        @language = r.doc.meta.language rescue nil
        @language = nil if @language == 'InputTextTooShort'

        r.each do |k,v|
          case v._typeGroup
          when 'topics'
            self.topics << {:name => humanize_topic(v.categoryName), :score => v.score.to_f, :original => v.categoryName}
          when 'socialTag'
            self.tags << {:name => v.name.gsub('_', ' and ').downcase, :score => importance_to_score(v.importance)}
          when 'entities'
            item = {:guid => k, :name => v.name, :type => v._type.remove_formatting.titleize, :score => 1.0}

            instances = Array(v.instances).select{|i| i.exact.downcase != item[:name].downcase }
            item[:matches] = instances if instances && instances.size > 0

            if OpenCalais::GEO_TYPES.include?(v._type)
              if (v.resolutions && v.resolutions.size > 0)
                r = v.resolutions.first
                item[:name]      = r.shortname || r.name
                item[:latitude]  = r.latitude
                item[:longitude] = r.longitude
                item[:country]   = r.containedbycountry if r.containedbycountry
                item[:state]     = r.containedbystate if r.containedbystate
              end
              self.locations << item
            else
              self.entities << item
            end
          when 'relations'
            item = v.reject{|k,v| k[0] == '_' || k == 'instances'} || {}
            item[:type] = v._type.remove_formatting.titleize
            self.relations << item
          end
        end

        # remove social tags which are in the topics list already
        if ! self.topics.nil?
          topic_names = self.topics.collect{|topic| topic[:name].downcase}
          self.tags.delete_if{|tag| topic_names.include?(tag[:name]) }
        end
      rescue Exception => e
        Rails.logger.info "OpenCalais Parse Error: #{e.message}"
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.12201092073055922, 'fraction_numerical': 0.0052720768216908305, 'mean_word_length': 1.8285410010649628, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '212945', 'n_tokens_mistral': 1635, 'n_tokens_neox': 1561, 'n_words': 367}