<commit_before>class Curlser
  
  class Response
    
    attr_reader :status, :connections, :redirects, :url_effective, :content_type
    attr_accessor :method, :path, :for_request_number, :body
    
    
    def initialize(opts)
      @status = opts[:status]
      @connections = opts[:connections]
      @redirects = opts[:redirects]
      @url_effective = opts[:url_effective]
      @content_type = opts[:content_type]
    end
    
    def self.parse_from(output)
      matches = output.match(/^(\d*) (\d*) (\d*) (\S*) (.*)$/)
      
      new({:status => matches[1],
           :connections => matches[2],
           :redirects => matches[3],
           :url_effective => matches[4],
           :content_type => matches[5]})
    end
  end
  
  class Cookie
  
    attr_reader :httponly, :domain, :tailmatch, :path, :secure, :expires, :name, :value

    def initialize(opts)
      @httponly = opts[:httponly]
      @domain = opts[:domain]
      @tailmatch = opts[:tailmatch]
      @path = opts[:path]
      @secure = opts[:secure]
      @expires = opts[:expires]
      @name = opts[:name]
      @value = opts[:value]     
    end

    
    # static char *get_netscape_format(const struct Cookie *co)
    # {
    #   return aprintf(
    #     "%s"     /* httponly preamble */
    #     "%s%s\t" /* domain */
    #     "%s\t"   /* tailmatch */
    #     "%s\t"   /* path */
    #     "%s\t"   /* secure */
    #     "%" FORMAT_OFF_T "\t"   /* expires */
    #     "%s\t"   /* name */
    #     "%s",    /* value */
    
    def self.parse(netscape_format_line)
      matches = netscape_format_line.scan(/(\S+)/)
      matches.flatten!
      
      httponly_preamble, domain = matches[0].split("_")
      
      new({:httponly => httponly_preamble,
           :domain => domain,
           :tailmatch => matches[1],
           :path => matches[2],
           :secure => matches[3],
           :expires => matches[4],
           :name => matches[5],
           :value => matches[6]
           })    
    end
  end
  
  class CookieJar
    
    attr_reader :cookies, :cookie_jar_file_path
    
    def initialize(cookie_jar_file_path)
      @cookies = {}
      @cookie_jar_file_path = cookie_jar_file_path
      
      cookiejar_contents = if File.exists?(@cookie_jar_file_path)
        File.read(@cookie_jar_file_path)
      else
        ""
      end
      
      lines = cookiejar_contents.split("\n")

      # skip header
      4.times { lines.shift }
      
      lines.each do |line|
        cookie = Cookie.parse(line)
        @cookies[cookie.name] = cookie
      end
          
    end
    
    def delete_cookies!
      if File.exists? @cookie_jar_file_path
        FileUtils.rm(@cookie_jar_file_path)
        return true
      else
        return false
      end
      
      @cookies = {}
    end
    
  end
  
  
  require 'fileutils'

  attr_reader :responses, :cookie_jar
  
  def initialize(base_url, opts={})
    
    @request_counter = 0
    @csrf_token = nil
    @responses = []    
    
    @base_url = base_url    
    @working_dir = opts[:working_dir] ? opts[:working_dir] : "curlser"
    @follow_redirects = opts[:follow_redirects] ? true : false
    @debug = opts[:debug] ? true : false
    @insecure = opts[:insecure] ? true : false

    FileUtils.mkdir_p @working_dir
    <commit_msg>Too repetative naming<commit_after>class Curlser
  
  class Response
    
    attr_reader :status, :connections, :redirects, :url_effective, :content_type
    attr_accessor :method, :path, :for_request_number, :body
    
    
    def initialize(opts)
      @status = opts[:status]
      @connections = opts[:connections]
      @redirects = opts[:redirects]
      @url_effective = opts[:url_effective]
      @content_type = opts[:content_type]
    end
    
    def self.parse_from(output)
      matches = output.match(/^(\d*) (\d*) (\d*) (\S*) (.*)$/)
      
      new({:status => matches[1],
           :connections => matches[2],
           :redirects => matches[3],
           :url_effective => matches[4],
           :content_type => matches[5]})
    end
  end
  
  class Cookie
  
    attr_reader :httponly, :domain, :tailmatch, :path, :secure, :expires, :name, :value

    def initialize(opts)
      @httponly = opts[:httponly]
      @domain = opts[:domain]
      @tailmatch = opts[:tailmatch]
      @path = opts[:path]
      @secure = opts[:secure]
      @expires = opts[:expires]
      @name = opts[:name]
      @value = opts[:value]     
    end

    
    # static char *get_netscape_format(const struct Cookie *co)
    # {
    #   return aprintf(
    #     "%s"     /* httponly preamble */
    #     "%s%s\t" /* domain */
    #     "%s\t"   /* tailmatch */
    #     "%s\t"   /* path */
    #     "%s\t"   /* secure */
    #     "%" FORMAT_OFF_T "\t"   /* expires */
    #     "%s\t"   /* name */
    #     "%s",    /* value */
    
    def self.parse(netscape_format_line)
      matches = netscape_format_line.scan(/(\S+)/)
      matches.flatten!
      
      httponly_preamble, domain = matches[0].split("_")
      
      new({:httponly => httponly_preamble,
           :domain => domain,
           :tailmatch => matches[1],
           :path => matches[2],
           :secure => matches[3],
           :expires => matches[4],
           :name => matches[5],
           :value => matches[6]
           })    
    end
  end
  
  class CookieJar
    
    attr_reader :cookies, :cookie_jar_file_path
    
    def initialize(cookie_jar_file_path)
      @cookies = {}
      @cookie_jar_file_path = cookie_jar_file_path
      
      cookiejar_contents = if File.exists?(@cookie_jar_file_path)
        File.read(@cookie_jar_file_path)
      else
        ""
      end
      
      lines = cookiejar_contents.split("\n")

      # skip header
      4.times { lines.shift }
      
      lines.each do |line|
        cookie = Cookie.parse(line)
        @cookies[cookie.name] = cookie
      end
          
    end
    
    def delete!
      if File.exists? @cookie_jar_file_path
        FileUtils.rm(@cookie_jar_file_path)
        return true
      else
        return false
      end
      
      @cookies = {}
    end
    
  end
  
  
  require 'fileutils'

  attr_reader :responses, :cookie_jar
  
  def initialize(base_url, opts={})
    
    @request_counter = 0
    @csrf_token = nil
    @responses = []    
    
    @base_url = base_url    
    @working_dir = opts[:working_dir] ? opts[:working_dir] : "curlser"
    @follow_redirects = opts[:follow_redirects] ? true : false
    @debug = opts[:debug] ? true : false
    @insecure = opts[:insecure] ? true : false

    FileUtils.mkdir_p @working_dir
    
{'fraction_non_alphanumeric': 0.1318415482310251, 'fraction_numerical': 0.0042334442092530995, 'mean_word_length': 1.9650380995069476, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2918047', 'n_tokens_mistral': 2208, 'n_tokens_neox': 2169, 'n_words': 418}