<commit_before># An **Entity**, for our purposes, is a piece of information either about or
# contained by a document (i.e., it may have a location within the document's
# text.) Each valid kind of entity must be whitelisted.
class Entity < ActiveRecord::Base

  include DC::Store::DocumentResource
  include DC::Store::EntityResource

  DEFAULT_RELEVANCE = 0.0

  ALL_CAPS = /\A[A-Z\s]+\Z/

  belongs_to :document

  validates_inclusion_of :kind, :in => DC::VALID_KINDS

  named_scope :kind, lambda {|kind| {:conditions => {:kind => kind}} }

  # We don't pay attention to all kinds of Calais-generated entities, just
  # most of them. Checks the whitelist.
  def self.acceptable_kind?(kind)
    !!DC::CALAIS_MAP[kind.underscore.to_sym]
  end

  def self.normalize_value(string)
    return string unless string.length > 5 && string.match(ALL_CAPS)
    string.titleize
  end

  def self.search_in_documents(kind, value, ids)
    entities = self.all(:conditions => {:document_id => ids, :kind => kind, :value => value})
    if entities.empty?<commit_msg>little more lenient on Entity.normalize_vlue<commit_after># An **Entity**, for our purposes, is a piece of information either about or
# contained by a document (i.e., it may have a location within the document's
# text.) Each valid kind of entity must be whitelisted.
class Entity < ActiveRecord::Base

  include DC::Store::DocumentResource
  include DC::Store::EntityResource

  DEFAULT_RELEVANCE = 0.0

  ALL_CAPS = /\A[A-Z0-9\s.]+\Z/

  belongs_to :document

  validates_inclusion_of :kind, :in => DC::VALID_KINDS

  named_scope :kind, lambda {|kind| {:conditions => {:kind => kind}} }

  # We don't pay attention to all kinds of Calais-generated entities, just
  # most of them. Checks the whitelist.
  def self.acceptable_kind?(kind)
    !!DC::CALAIS_MAP[kind.underscore.to_sym]
  end

  def self.normalize_value(string)
    return string unless string.length > 5 && string.match(ALL_CAPS)
    string.titleize
  end

  def self.search_in_documents(kind, value, ids)
    entities = self.all(:conditions => {:document_id => ids, :kind => kind, :value => value})
    if entities.empty?
{'fraction_non_alphanumeric': 0.11723493694535264, 'fraction_numerical': 0.0037365716954694066, 'mean_word_length': 4.173913043478261, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '877328', 'n_tokens_mistral': 710, 'n_tokens_neox': 670, 'n_words': 213}