<commit_before>module ErrorStore::Interfaces
  class Breadcrumbs < ErrorStore::BaseInterface
    def self.display_name
      'Breadcrumbs'
    end

    def type
      :breadcrumbs
    end

    def sanitize_data(data)
      values = []
      data[:values].each do |value|
        values << Breadcrumbs.new(@error).normalize_crumb(value)
      end
      self._data[:values] = values

      self
    end

    def normalize_crumb(crumb)
      type = crumb[:type] || 'default'
      timestamp = @error.process_timestamp!(crumb)[:timestamp]

      hash =
      {
        type: type,
        timestamp: timestamp
      }

      level = crumb[:level]
      unless level.blank? || level == 'info'
        hash[:level] = level
      end

      message = crumb[:message]
      unless message.blank?
         hash[:message] = trim(message, max_size: 4096)
      end

      category = crumb[:category]
      unless category.blank?
         hash[:category] = trim(category, max_size: 256)
      end

      event_id = crumb[:event_id]
      unless event_id.blank?
         hash[:event_id] = event_id
      end

      if crumb[:data].present?
        hash[:data] = trim(crumb[:data], max_size: 4096)
      end

      return hash
    end
  end
end<commit_msg>update breadcrumbs code to normalize_crumb and process_timestamp<commit_after>module ErrorStore::Interfaces
  class Breadcrumbs < ErrorStore::BaseInterface
    def self.display_name
      'Breadcrumbs'
    end

    def type
      :breadcrumbs
    end

    def sanitize_data(data)
      values = []
      data[:values].each do |value|
        values << normalize_crumb(value)
      end
      self._data[:values] = values

      self
    end

    def normalize_crumb(crumb)
      type = crumb[:type] || 'default'
      timestamp = process_timestamp!(crumb[:timestamp])

      hash =
      {
        type: type,
        timestamp: timestamp
      }

      level = crumb[:level]
      unless level.blank? || level == 'info'
        hash[:level] = level
      end

      message = fix_encoding(crumb[:message])
      unless message.blank?
         hash[:message] = trim(message, max_size: 4096)
      end

      category = fix_encoding(crumb[:category])
      unless category.blank?
         hash[:category] = trim(category, max_size: 256)
      end

      event_id = crumb[:event_id]
      unless event_id.blank?
         hash[:event_id] = event_id
      end

      if crumb[:data].present?
        hash[:data] = trim(crumb[:data], max_size: 4096)
      end

      return hash
    end
  end
end
{'fraction_non_alphanumeric': 0.10308056872037914, 'fraction_numerical': 0.008688783570300158, 'mean_word_length': 2.214467005076142, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4029873', 'n_tokens_mistral': 804, 'n_tokens_neox': 764, 'n_words': 173}