<commit_before>require 'mail'

class FakeEmailService

  def initialize
    fake_domains_file = File.expand_path('../../config/fake_domains.list', __FILE__)
    @fake_domains = File.readlines(fake_domains_file).map {|fd| fd.strip.downcase }
  end

  def is_fake_email?(email)
    email_address = Mail::Address.new(email)

    domain = email_address.domain.strip.downcase
    second_level_domain = domain.split('.')[-2..-1].join('.')

    domains = [domain, second_level_domain]

    @fake_domains.any? {|fake_domain| domains.include?(fake_domain) }
  end

end<commit_msg>Check third level domains for any case<commit_after>require 'mail'

class FakeEmailService

  def initialize
    fake_domains_file = File.expand_path('../../config/fake_domains.list', __FILE__)
    @fake_domains = File.readlines(fake_domains_file).map {|fd| fd.strip.downcase }
  end

  def is_fake_email?(email)
    email_address = Mail::Address.new(email)

    domain = email_address.domain.strip.downcase
    domain_parts = domain.split('.')

    second_level_domain = Array(domain_parts[-2..-1]).join('.')
    third_level_domain = Array(domain_parts[-3..-1]).join('.')

    domains = [domain, second_level_domain, third_level_domain].compact

    @fake_domains.any? {|fake_domain| domains.include?(fake_domain) }
  end

end
{'fraction_non_alphanumeric': 0.13713405238828968, 'fraction_numerical': 0.004622496147919877, 'mean_word_length': 4.323770491803279, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3275908', 'n_tokens_mistral': 441, 'n_tokens_neox': 427, 'n_words': 70}