<commit_before>require 'memory_profiler'


module Api::ActionNetwork::People
  extend Api::ActionNetwork::Import

  def self.resource
    'person'
  end

  def self.import!(group)
    existing_count = 0
    new_count = 0
    updated_count = 0
    next_uri = first_uri(synced_at: group.synced_at)

    logger.info "Api::ActionNetwork::People#import! from #{next_uri}"

    logs = []
    report = MemoryProfiler.report do
      while next_uri
        #::Person.transaction do
          people, next_uri = request_resources_from_action_network(next_uri, group)

          people.each(&:sanitize_email_addresses)

          existing_people, new_people = partition(people)

          new_count += new_people.size
          existing_count += existing_count.size

          people.each do |new_person|
            logs << Api::ActionNetwork::Person.after_import(new_person, group)
          end
        #end
      end
      logger.debug "Api::ActionNetwork::People#import! new: #{new_count} existing: #{existing_count} updated: #{updated_count}"
    end

    logger.debug report.pretty_print


    {
      created: logs.map { |log| log[:created] || 0 }.sum,
      updated: logs.map { |log| log[:updated] || 0 }.sum,
      errors: logs.map { |log| log[:errors] || 0 }.sum
    }
  end

  def self.errors_count
    0
  end
end<commit_msg>reducing memory footprint of import scripts<commit_after>require 'memory_profiler'


module Api::ActionNetwork::People
  extend Api::ActionNetwork::Import

  def self.resource
    'person'
  end

  def self.import!(group)
    existing_count = 0
    new_count = 0
    updated_count = 0
    next_uri = first_uri(synced_at: group.synced_at)

    logger.info "Api::ActionNetwork::People#import! from #{next_uri}"

    logs = []
    report = MemoryProfiler.report do
      while next_uri
        #::Person.transaction do
          people, next_uri = request_resources_from_action_network(next_uri, group)

          people.each(&:sanitize_email_addresses)

          #existing_people, new_people = partition(people)

          #new_count += new_people.size
          #existing_count += existing_count.size

          people.each do |new_person|
            logs << Api::ActionNetwork::Person.after_import(new_person, group)
          end
        #end
      end
      #logger.debug "Api::ActionNetwork::People#import! new: #{new_count} existing: #{existing_count} updated: #{updated_count}"
    end

    logger.debug report.pretty_print


    {
      created: logs.map { |log| log[:created] || 0 }.sum,
      updated: logs.map { |log| log[:updated] || 0 }.sum,
      errors: logs.map { |log| log[:errors] || 0 }.sum
    }
  end

  def self.errors_count
    0
  end
end
{'fraction_non_alphanumeric': 0.11078066914498141, 'fraction_numerical': 0.0052044609665427505, 'mean_word_length': 2.888728323699422, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '917398', 'n_tokens_mistral': 856, 'n_tokens_neox': 826, 'n_words': 182}