<commit_before>require "tempfile"

#This class parses and handels post-multipart requests.
class Hayabusa::Http_session::Post_multipart
  #This hash contains all the data read from the post-request.
  attr_reader :return
  
  def initialize(args)
    @args = args
    crlf = args[:crlf]
    
    boundary_regexp = /\A--#{Regexp.escape(@args[:boundary])}(--)?(\r\n|\n|$)\z/
    @return = {}
    @data = nil
    @mode = nil
    @clength = 0
    @headers = {}
    @counts = {}
    str_crlf = nil
    
    @args[:io].each do |line|
      if boundary_regexp =~ line
        #Finish the data we were writing.
        self.finish_data if @data
        
        @data_first = true
        @data_written = 0
        @clength = nil
        @name = nil
        @mode = "headers"
        str_crlf = nil
      elsif @mode == "headers"
        if match = line.match(/^(.+?):\s+(.+)#{crlf}$/)
          key = match[1].to_s.downcase
          val = match[2]
          
          @headers[key] = val
          
          if key == "content-length"<commit_msg>Fixed 'invalid byte sequence' bug when uploading files with special characters in something else than UTF-8.<commit_after>require "tempfile"

#This class parses and handels post-multipart requests.
class Hayabusa::Http_session::Post_multipart
  #This hash contains all the data read from the post-request.
  attr_reader :return
  
  def initialize(args)
    @args = args
    crlf = args[:crlf]
    
    boundary_regexp = /\A--#{Regexp.escape(@args[:boundary])}(--)?(\r\n|\n|$)\z/
    @return = {}
    @data = nil
    @mode = nil
    @clength = 0
    @headers = {}
    @counts = {}
    str_crlf = nil
    
    @args[:io].each do |line|
      begin
        boundary_match = line.match(boundary_regexp)
      rescue ArgumentError
        #Happens when "invalid byte sequence in UTF-8" - the boundary-line will be UTF-8-valid and match the 'boundary_regexp'.
        boundary_match = false
      end
      
      if boundary_match
        #Finish the data we were writing.
        self.finish_data if @data
        
        @data_first = true
        @data_written = 0
        @clength = nil
        @name = nil
        @mode = "headers"
        str_crlf = nil
      elsif @mode == "headers"
        if match = line.match(/^(.+?):\s+(.+)#{crlf}$/)
          key = match[1].to_s.downcase
          val = match[2]
          
          @headers[key] = val
          
          if key == "content-length"
{'fraction_non_alphanumeric': 0.12551271534044298, 'fraction_numerical': 0.004511894995898278, 'mean_word_length': 2.0073982737361282, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4616035', 'n_tokens_mistral': 804, 'n_tokens_neox': 774, 'n_words': 205}