<commit_before># encoding: utf-8
module Mail
  
  # Provides access to a header object.
  # 
  # ===Per RFC2822
  # 
  #  2.2. Header Fields
  # 
  #   Header fields are lines composed of a field name, followed by a colon
  #   (":"), followed by a field body, and terminated by CRLF.  A field
  #   name MUST be composed of printable US-ASCII characters (i.e.,
  #   characters that have values between 33 and 126, inclusive), except
  #   colon.  A field body may be composed of any US-ASCII characters,
  #   except for CR and LF.  However, a field body may contain CRLF when
  #   used in header "folding" and  "unfolding" as described in section
  #   2.2.3.  All field bodies MUST conform to the syntax described in
  #   sections 3 and 4 of this standard.
  class Header
    include Patterns
    include Utilities
    include Enumerable
    
    # Creates a new header object.
    # 
    # Accepts raw text or nothing.  If given raw text will attempt to parse
    # it and split it into the various fields, instantiating each field as
    # it goes.
    # 
    # If it finds a field that should be a structured field (such as content
    # type), but it fails to parse it, it will simply make it an unstructured
    # field and leave it alone.  This will mean that the data is preserved but
    # no automatic processing of that field will happen.  If you find one of
    # these cases, please make a patch and send it in, or at the least, send
    # me the example so we can fix it.
    def initialize(header_text = nil, charset = nil)
      @errors = []
      @charset = charset
      self.raw_source = header_text.to_crlf
      split_header if header_text
    end
    
    # The preserved raw source of the header as you passed it in, untouched
    # for your Regexing glory.
    def raw_source
      @raw_source
    end
    
    # Returns an array of all the fields in the header in order that they
    # were read in.
    def fields
      @fields ||= FieldList.new
    end
    
    #  3.6. Field definitions
    #  
    #   It is important to note that the header fields are not guaranteed to
    #   be in a particular order.  They may appear in any order, and they
    #   have been known to be reordered occasionally when transported over
    #   the Internet.  However, for the purposes of this standard, header
    #   fields SHOULD NOT be reordered when a message is transported or
    #   transformed.  More importantly, the trace header fields and resent
    #   header fields MUST NOT be reordered, and SHOULD be kept in blocks
    #   prepended to the message.  See sections 3.6.6 and 3.6.7 for more
    #   information.
    # 
    # Populates the fields container with Field objects in the order it
    # receives them in.
    #
    # Acceps an array of field string values, for example:
    # 
    #  h = Header.new
    #  h.fields = ['From: <EMAIL>', 'To: <EMAIL>']
    def fields=(unfolded_fields)
      @fields = Mail::FieldList.new
      unfolded_fields.each do |field|<commit_msg>Only use the a maximum of 1000 lines when parsing the headers. It's likely there was an error and causes severe performance degradation<commit_after># encoding: utf-8
module Mail
  
  # Provides access to a header object.
  # 
  # ===Per RFC2822
  # 
  #  2.2. Header Fields
  # 
  #   Header fields are lines composed of a field name, followed by a colon
  #   (":"), followed by a field body, and terminated by CRLF.  A field
  #   name MUST be composed of printable US-ASCII characters (i.e.,
  #   characters that have values between 33 and 126, inclusive), except
  #   colon.  A field body may be composed of any US-ASCII characters,
  #   except for CR and LF.  However, a field body may contain CRLF when
  #   used in header "folding" and  "unfolding" as described in section
  #   2.2.3.  All field bodies MUST conform to the syntax described in
  #   sections 3 and 4 of this standard.
  class Header
    include Patterns
    include Utilities
    include Enumerable
    
    # Creates a new header object.
    # 
    # Accepts raw text or nothing.  If given raw text will attempt to parse
    # it and split it into the various fields, instantiating each field as
    # it goes.
    # 
    # If it finds a field that should be a structured field (such as content
    # type), but it fails to parse it, it will simply make it an unstructured
    # field and leave it alone.  This will mean that the data is preserved but
    # no automatic processing of that field will happen.  If you find one of
    # these cases, please make a patch and send it in, or at the least, send
    # me the example so we can fix it.
    def initialize(header_text = nil, charset = nil)
      @errors = []
      @charset = charset
      self.raw_source = header_text.to_crlf
      split_header if header_text
    end
    
    # The preserved raw source of the header as you passed it in, untouched
    # for your Regexing glory.
    def raw_source
      @raw_source
    end
    
    # Returns an array of all the fields in the header in order that they
    # were read in.
    def fields
      @fields ||= FieldList.new
    end
    
    #  3.6. Field definitions
    #  
    #   It is important to note that the header fields are not guaranteed to
    #   be in a particular order.  They may appear in any order, and they
    #   have been known to be reordered occasionally when transported over
    #   the Internet.  However, for the purposes of this standard, header
    #   fields SHOULD NOT be reordered when a message is transported or
    #   transformed.  More importantly, the trace header fields and resent
    #   header fields MUST NOT be reordered, and SHOULD be kept in blocks
    #   prepended to the message.  See sections 3.6.6 and 3.6.7 for more
    #   information.
    # 
    # Populates the fields container with Field objects in the order it
    # receives them in.
    #
    # Acceps an array of field string values, for example:
    # 
    #  h = Header.new
    #  h.fields = ['From: <EMAIL>', 'To: <EMAIL>']
    def fields=(unfolded_fields)
      @fields = Mail::FieldList.new
      warn "Warning: more than 1000 header fields only using the first 1000" if unfolded_fields.length > 1000
      unfolded_fields[0..1000].each do |field|
{'fraction_non_alphanumeric': 0.05799648506151142, 'fraction_numerical': 0.011343665122223998, 'mean_word_length': 2.5307388606880994, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '464865', 'n_tokens_mistral': 1851, 'n_tokens_neox': 1742, 'n_words': 868}