<commit_before># encoding: utf-8

require 'time_difference'

module Anon
  # Anonymiser base class
  class Base

    def self.anonymise!(*args)
      new(*args).anonymise!
    end

    protected

    def anonymous_email(personal_email)
      @anonymised_emails ||= {}
      
      unless @anonymised_emails.has_key? personal_email
        next_count = @anonymised_emails.count + 1
        @anonymised_emails[personal_email] = "<EMAIL>"
      end

      @anonymised_emails[personal_email]     
    end

    def start_progress
      @progress = 0
      @started = Time.now
      update_progress
    end

    def increment_progress
      @progress += 1
      update_progress
    end

    def complete_progress
      stopped = Time.now
      duration = TimeDifference.between(@started, stopped).in_seconds
      average = (@progress.to_f / duration.to_f).round
      puts "Read #{@progress} lines in #{duration} seconds (#{average} lines/s)"
      puts "#{@anonymised_emails.count} unique e-mails replaced"
    end

    private

    def update_progress
      if @progress % 100 == 0
        print "Working... #{@progress}\r"
        $stdout.flush
      end
    end
  end
end<commit_msg>#5 - Fix Infinity error when duration was 0<commit_after># encoding: utf-8

require 'time_difference'

module Anon
  # Anonymiser base class
  class Base

    def self.anonymise!(*args)
      new(*args).anonymise!
    end

    protected

    def anonymous_email(personal_email)
      @anonymised_emails ||= {}
      
      unless @anonymised_emails.has_key? personal_email
        next_count = @anonymised_emails.count + 1
        @anonymised_emails[personal_email] = "<EMAIL>"
      end

      @anonymised_emails[personal_email]     
    end

    def start_progress
      @progress = 0
      @started = Time.now
      update_progress
    end

    def increment_progress
      @progress += 1
      update_progress
    end

    def complete_progress
      stopped = Time.now
      duration = TimeDifference.between(@started, stopped).in_seconds
      if duration == 0
        average = @progress
      else
        average = (@progress.to_f / duration.to_f).round
      end
      
      puts "Read #{@progress} lines in #{duration} seconds (#{average} lines/s)"
      puts "#{@anonymised_emails.count} unique e-mails replaced"
    end

    private

    def update_progress
      if @progress % 100 == 0
        print "Working... #{@progress}\r"
        $stdout.flush
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.0905050505050505, 'fraction_numerical': 0.007676767676767677, 'mean_word_length': 2.3825136612021858, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '181742', 'n_tokens_mistral': 791, 'n_tokens_neox': 750, 'n_words': 183}