<commit_before># An **Entity**, for our purposes, is a piece of information either about or
# contained by a document (i.e., it may have a location within the document's
# text.) Each valid kind of entity must be whitelisted.
class Entity < ActiveRecord::Base

  include DC::Store::DocumentResource
  include DC::Store::EntityResource

  DEFAULT_RELEVANCE = 0.0

  ALL_CAPS = /\A[A-Z0-9\s.]+\Z/

  belongs_to :document
  belongs_to :account
  belongs_to :organization
  validates  :kind, :inclusion=>{ :in => DC::VALID_KINDS }
  validates  :value, :presence=>true
  text_attr  :value

  scope :kind, lambda {|kind| where( :kind => kind ) }

  def self.normalize_kind(string)
    DC::CALAIS_MAP[string.underscore.to_sym]
  end

  # Truncate and titlize the value, if necessary.
  def self.normalize_value(string)
    string = string[0...255]
    return string unless string.length > 5 && string.match(ALL_CAPS)
    string.titleize
  end

  def self.search_in_documents(kind, value, ids)
    return [] if value.nil?
    self.includes(:document).references(:document).where([
      "document_id in (?) and kind = ? and lower(value) like lower('%#{Entity.connection.quote_string(value)}%')", ids, kind
    ])
  end

  # Merge this entity with another of the "same" entity for the same document.
  # Needs to happen when the pass the document's text to Calais in chunks.
  def merge(entity)
    self.relevance = (self.relevance + entity.relevance) / 2.0
    @split_occurrences = self.split_occurrences + entity.split_occurrences
    self.occurrences = Occurrence.to_csv(@split_occurrences)<commit_msg>Strip spaces to ensure that new calais gem keys work.<commit_after># An **Entity**, for our purposes, is a piece of information either about or
# contained by a document (i.e., it may have a location within the document's
# text.) Each valid kind of entity must be whitelisted.
class Entity < ActiveRecord::Base

  include DC::Store::DocumentResource
  include DC::Store::EntityResource

  DEFAULT_RELEVANCE = 0.0

  ALL_CAPS = /\A[A-Z0-9\s.]+\Z/

  belongs_to :document
  belongs_to :account
  belongs_to :organization
  validates  :kind, :inclusion=>{ :in => DC::VALID_KINDS }
  validates  :value, :presence=>true
  text_attr  :value

  scope :kind, lambda {|kind| where( :kind => kind ) }

  def self.normalize_kind(string)
    DC::CALAIS_MAP[string.gsub(/\s+/,'').underscore.to_sym]
  end

  # Truncate and titlize the value, if necessary.
  def self.normalize_value(string)
    string = string[0...255]
    return string unless string.length > 5 && string.match(ALL_CAPS)
    string.titleize
  end

  def self.search_in_documents(kind, value, ids)
    return [] if value.nil?
    self.includes(:document).references(:document).where([
      "document_id in (?) and kind = ? and lower(value) like lower('%#{Entity.connection.quote_string(value)}%')", ids, kind
    ])
  end

  # Merge this entity with another of the "same" entity for the same document.
  # Needs to happen when the pass the document's text to Calais in chunks.
  def merge(entity)
    self.relevance = (self.relevance + entity.relevance) / 2.0
    @split_occurrences = self.split_occurrences + entity.split_occurrences
    self.occurrences = Occurrence.to_csv(@split_occurrences)
{'fraction_non_alphanumeric': 0.11487526947951955, 'fraction_numerical': 0.006775485063135201, 'mean_word_length': 4.0046224961479195, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4078280', 'n_tokens_mistral': 1090, 'n_tokens_neox': 1028, 'n_words': 306}