<commit_before>class EmailRetrievalService
  def initialize(user)
    @user = user
    @gmail = Gmail.connect(:xoauth2, user.email, user.oauth_token)
  end

  def retrieve_emails(partner)
    @emails = @gmail.inbox.emails(from: partner.email).first(10)
    save_emails(partner)
    @email_count = @emails.count
  end

  def save_emails(partner)
    @emails.each do |email|
      new_email = Email.new(user_id: @user.id, partner_id: partner.id)
      new_email.update(subject: email.subject, text: clean_email_text(email), sent_at: email.date)
    end
  end

  def clean_email_text(email)
    text = ActionView::Base.full_sanitizer.sanitize(email.body.decoded)
    text.gsub(/^.+text\/plain; charset=UTF-8/,'').gsub(/\\n/,'')
    text = RbLibText.tokens(text).join(" ")
  end

end<commit_msg>Add more regex for clean up of text<commit_after>class EmailRetrievalService
  def initialize(user)
    @user = user
    @gmail = Gmail.connect(:xoauth2, user.email, user.oauth_token)
  end

  def retrieve_emails(partner)
    @emails = @gmail.inbox.emails(from: partner.email).last(10)
    save_emails(partner)
    @email_count = @emails.count
  end

  def save_emails(partner)
    @emails.each do |email|
      new_email = Email.new(user_id: @user.id, partner_id: partner.id)
      new_email.update(subject: email.subject, text: clean_email_text(email), sent_at: email.date)
    end
  end

  def clean_email_text(email)
    text = ActionView::Base.full_sanitizer.sanitize(email.body.decoded)
    cleaned_headers = remove_headings(text)
    cleaned_spaces = remove_spaces(cleaned_headers)
    cleaned_text = remove_numbers(cleaned_spaces)
    return RbLibText.tokens(cleaned_text).join(" ")
  end

  def remove_headings(text)
    text.gsub(/.*Content-Transfer-Encoding/,'').gsub(/.*Content-Type.*[uUtTfF]-8/,'')
  end

  def remove_spaces(text)
    text.gsub(/\\n/,'')
  end

  def remove_numbers(text)
    text.gsub(/--.?\d*/,'')
  end

end
{'fraction_non_alphanumeric': 0.13561076604554864, 'fraction_numerical': 0.004140786749482402, 'mean_word_length': 3.9060913705583755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5549549', 'n_tokens_mistral': 725, 'n_tokens_neox': 708, 'n_words': 118}