<commit_before>require 'email/html_cleaner'
#
# Handles an incoming message
#

module Email

  class Receiver

    include ActionView::Helpers::NumberHelper

    class ProcessingError < StandardError; end
    class EmailUnparsableError < ProcessingError; end
    class EmptyEmailError < ProcessingError; end
    class UserNotFoundError < ProcessingError; end
    class UserNotSufficientTrustLevelError < ProcessingError; end
    class BadDestinationAddress < ProcessingError; end
    class TopicNotFoundError < ProcessingError; end
    class TopicClosedError < ProcessingError; end
    class EmailLogNotFound < ProcessingError; end
    class InvalidPost < ProcessingError; end

    attr_reader :body, :email_log

    def initialize(raw)
      @raw = raw
    end

    def process
      raise EmptyEmailError if @raw.blank?

      message = Mail.new(@raw)

      body = parse_body message

      dest_info = {type: :invalid, obj: nil}
      message.to.each do |to_address|
        if dest_info[:type] == :invalid
          dest_info = check_address to_address
        end
      end

      raise BadDestinationAddress if dest_info[:type] == :invalid
      raise TopicNotFoundError if message.header.to_s =~ /auto-generated/ || message.header.to_s =~ /auto-replied/

      # TODO get to a state where we can remove this
      @message = message
      @body = body

      if dest_info[:type] == :category
        raise BadDestinationAddress unless SiteSetting.email_in
        category = dest_info[:obj]
        @category_id = category.id
        @allow_strangers = category.email_in_allow_strangers

        user_email = @message.from.first
        @user = User.find_by_email(user_email)
        if @user.blank? && @allow_strangers

          wrap_body_in_quote user_email
          # TODO This is WRONG it should register an account
          # and email the user details on how to log in / activate
          @user = Discourse.system_user
        end

        raise UserNotFoundError if @user.blank?
        raise UserNotSufficientTrustLevelError.new @user unless @allow_strangers || @user.has_trust_level?(TrustLevel[SiteSetting.email_in_min_trust.to_i])

        create_new_topic
      else
        @email_log = dest_info[:obj]

        raise EmailLogNotFound if @email_log.blank?
        raise TopicNotFoundError if Topic.find_by_id(@email_log.topic_id).nil?
        raise TopicClosedError if Topic.find_by_id(@email_log.topic_id).closed?

        create_reply
      end
    rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e
      raise EmailUnparsableError.new(e)
    end

    def check_address(address)
      category = Category.find_by_email(address)
      return {type: :category, obj: category} if category

      regex = Regexp.escape SiteSetting.reply_by_email_address
      regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.*)")
      regex = Regexp.new regex
      match = regex.match address
      if match && match[1].present?
        reply_key = match[1]
        email_log = EmailLog.for(reply_key)

        return {type: :reply, obj: email_log}
      end

      {type: :invalid, obj: nil}
    end

    def parse_body(message)
      body = select_body message
      encoding = body.encoding
      raise EmptyEmailError if body.strip.blank?

      body = discourse_email_trimmer body
      raise EmptyEmailError if body.strip.blank?

      body = EmailReplyParser.parse_reply body
      raise EmptyEmailError if body.strip.blank?

      body.force_encoding(encoding).encode("UTF-8")
    end

    def select_body(message)
      html = nil
      # If the message is multipart, return that part (favor html)
      if message.multipart?
        html = fix_charset message.html_part
        text = fix_charset message.text_part

        # prefer plain text
        if text
          return text
        end
      elsif message.content_type =~ /text\/html/
        html = fix_charset message
      end

      if html
        body = HtmlCleaner.new(html).output_html
      else
        body = fix_charset message
      end

      # Certain trigger phrases that means we didn't parse correctly
      if body =~ /Content\-Type\:/ || body =~ /multipart\/alternative/ || body =~ /text\/plain/
        raise EmptyEmailError
      end

      body
    end

    # Force encoding to UTF-8 on a Mail::Message or Mail::Part
    def fix_charset(object)
      return nil if object.nil?

      if object.charset
        object.body.decoded.force_encoding(object.charset).encode("UTF-8").to_s
      else
        object.body.to_s
      end
    end

    REPLYING_HEADER_LABELS = ['From', 'Sent', 'To', 'Subject', 'Reply To', 'Cc', 'Bcc', 'Date']
    REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |lbl| "#{lbl}:" })

    def discourse_email_trimmer(body)
      lines = body.scrub.lines.to_a
      range_end = 0

      lines.each_with_index do |l, idx|
        break if l =~ /\A\s*\-{3,80}\s*\z/ ||<commit_msg>FIX: convert UTF8 charset to UTF-8<commit_after>require 'email/html_cleaner'
#
# Handles an incoming message
#

module Email

  class Receiver

    include ActionView::Helpers::NumberHelper

    class ProcessingError < StandardError; end
    class EmailUnparsableError < ProcessingError; end
    class EmptyEmailError < ProcessingError; end
    class UserNotFoundError < ProcessingError; end
    class UserNotSufficientTrustLevelError < ProcessingError; end
    class BadDestinationAddress < ProcessingError; end
    class TopicNotFoundError < ProcessingError; end
    class TopicClosedError < ProcessingError; end
    class EmailLogNotFound < ProcessingError; end
    class InvalidPost < ProcessingError; end

    attr_reader :body, :email_log

    def initialize(raw)
      @raw = raw
    end

    def process
      raise EmptyEmailError if @raw.blank?

      message = Mail.new(@raw)

      body = parse_body message

      dest_info = {type: :invalid, obj: nil}
      message.to.each do |to_address|
        if dest_info[:type] == :invalid
          dest_info = check_address to_address
        end
      end

      raise BadDestinationAddress if dest_info[:type] == :invalid
      raise TopicNotFoundError if message.header.to_s =~ /auto-generated/ || message.header.to_s =~ /auto-replied/

      # TODO get to a state where we can remove this
      @message = message
      @body = body

      if dest_info[:type] == :category
        raise BadDestinationAddress unless SiteSetting.email_in
        category = dest_info[:obj]
        @category_id = category.id
        @allow_strangers = category.email_in_allow_strangers

        user_email = @message.from.first
        @user = User.find_by_email(user_email)
        if @user.blank? && @allow_strangers

          wrap_body_in_quote user_email
          # TODO This is WRONG it should register an account
          # and email the user details on how to log in / activate
          @user = Discourse.system_user
        end

        raise UserNotFoundError if @user.blank?
        raise UserNotSufficientTrustLevelError.new @user unless @allow_strangers || @user.has_trust_level?(TrustLevel[SiteSetting.email_in_min_trust.to_i])

        create_new_topic
      else
        @email_log = dest_info[:obj]

        raise EmailLogNotFound if @email_log.blank?
        raise TopicNotFoundError if Topic.find_by_id(@email_log.topic_id).nil?
        raise TopicClosedError if Topic.find_by_id(@email_log.topic_id).closed?

        create_reply
      end
    rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e
      raise EmailUnparsableError.new(e)
    end

    def check_address(address)
      category = Category.find_by_email(address)
      return {type: :category, obj: category} if category

      regex = Regexp.escape SiteSetting.reply_by_email_address
      regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.*)")
      regex = Regexp.new regex
      match = regex.match address
      if match && match[1].present?
        reply_key = match[1]
        email_log = EmailLog.for(reply_key)

        return {type: :reply, obj: email_log}
      end

      {type: :invalid, obj: nil}
    end

    def parse_body(message)
      body = select_body message
      encoding = body.encoding
      raise EmptyEmailError if body.strip.blank?

      body = discourse_email_trimmer body
      raise EmptyEmailError if body.strip.blank?

      body = EmailReplyParser.parse_reply body
      raise EmptyEmailError if body.strip.blank?

      body.force_encoding(encoding).encode("UTF-8")
    end

    def select_body(message)
      html = nil
      # If the message is multipart, return that part (favor html)
      if message.multipart?
        html = fix_charset message.html_part
        text = fix_charset message.text_part

        # prefer plain text
        if text
          return text
        end
      elsif message.content_type =~ /text\/html/
        html = fix_charset message
      end

      if html
        body = HtmlCleaner.new(html).output_html
      else
        body = fix_charset message
      end

      # Certain trigger phrases that means we didn't parse correctly
      if body =~ /Content\-Type\:/ || body =~ /multipart\/alternative/ || body =~ /text\/plain/
        raise EmptyEmailError
      end

      body
    end

    # Force encoding to UTF-8 on a Mail::Message or Mail::Part
    def fix_charset(object)
      return nil if object.nil?

      if object.charset
        # convert UTF8 charset to UTF-8
        object.charset = object.charset.gsub(/utf8/i, "UTF-8") if object.charset.downcase == "utf8"
        object.body.decoded.force_encoding(object.charset).encode("UTF-8").to_s
      else
        object.body.to_s
      end
    end

    REPLYING_HEADER_LABELS = ['From', 'Sent', 'To', 'Subject', 'Reply To', 'Cc', 'Bcc', 'Date']
    REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |lbl| "#{lbl}:" })

    def discourse_email_trimmer(body)
      lines = body.scrub.lines.to_a
      range_end = 0

      lines.each_with_index do |l, idx|
        break if l =~ /\A\s*\-{3,80}\s*\z/ ||
{'fraction_non_alphanumeric': 0.08544461003477397, 'fraction_numerical': 0.0024838549428713363, 'mean_word_length': 2.663027656477438, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '473573', 'n_tokens_mistral': 3067, 'n_tokens_neox': 2931, 'n_words': 770}