<commit_before># frozen_string_literal: true
module Excon
  module Utils
    extend self

    CONTROL   = (0x0..0x1f).map {|c| c.chr }.join + "\x7f"
    DELIMS    = '<>#%"'
    UNWISE    = '{}|\\^[]`'
    NONASCII  = (0x80..0xff).map {|c| c.chr }.join
    UNESCAPED = /([#{ Regexp.escape(CONTROL + ' ' + DELIMS + UNWISE + NONASCII) }])/
    ESCAPED   = /%([0-9a-fA-F]{2})/

    def binary_encode(string)
      if FORCE_ENC && string.encoding != Encoding::ASCII_8BIT
        if string.frozen?
          string.dup.force_encoding('BINARY')
        else
          string.force_encoding('BINARY')
        end
      else
        string
      end
    end

    def connection_uri(datum = @data)
      unless datum
        raise ArgumentError, '`datum` must be given unless called on a Connection'
      end
      if datum[:scheme] == UNIX
        "#{datum[:scheme]}://#{datum[:socket]}"
      else
        "#{datum[:scheme]}://#{datum[:host]}#{port_string(datum)}"
      end
    end

    # Redact sensitive info from provided data
    def redact(datum)
      datum = datum.dup
      if datum.has_key?(:headers)
        if datum[:headers].has_key?('Authorization') || datum[:headers].has_key?('Proxy-Authorization')
          datum[:headers] = datum[:headers].dup
        end
        if datum[:headers].has_key?('Authorization')
          datum[:headers]['Authorization'] = REDACTED
        end
        if datum[:headers].has_key?('Proxy-Authorization')
          datum[:headers]['Proxy-Authorization'] = REDACTED
        end
      end
      if datum.has_key?(:password)
        datum[:password] = REDACTED
      end
      if datum.has_key?(:proxy) && datum[:proxy]&.has_key?(:password)
        datum[:proxy] = datum[:proxy].dup
        datum[:proxy][:password] = REDACTED
      end
      datum<commit_msg>&. is a Ruby 2.3 feature, thus breaking gem for users using lower versions<commit_after># frozen_string_literal: true
module Excon
  module Utils
    extend self

    CONTROL   = (0x0..0x1f).map {|c| c.chr }.join + "\x7f"
    DELIMS    = '<>#%"'
    UNWISE    = '{}|\\^[]`'
    NONASCII  = (0x80..0xff).map {|c| c.chr }.join
    UNESCAPED = /([#{ Regexp.escape(CONTROL + ' ' + DELIMS + UNWISE + NONASCII) }])/
    ESCAPED   = /%([0-9a-fA-F]{2})/

    def binary_encode(string)
      if FORCE_ENC && string.encoding != Encoding::ASCII_8BIT
        if string.frozen?
          string.dup.force_encoding('BINARY')
        else
          string.force_encoding('BINARY')
        end
      else
        string
      end
    end

    def connection_uri(datum = @data)
      unless datum
        raise ArgumentError, '`datum` must be given unless called on a Connection'
      end
      if datum[:scheme] == UNIX
        "#{datum[:scheme]}://#{datum[:socket]}"
      else
        "#{datum[:scheme]}://#{datum[:host]}#{port_string(datum)}"
      end
    end

    # Redact sensitive info from provided data
    def redact(datum)
      datum = datum.dup
      if datum.has_key?(:headers)
        if datum[:headers].has_key?('Authorization') || datum[:headers].has_key?('Proxy-Authorization')
          datum[:headers] = datum[:headers].dup
        end
        if datum[:headers].has_key?('Authorization')
          datum[:headers]['Authorization'] = REDACTED
        end
        if datum[:headers].has_key?('Proxy-Authorization')
          datum[:headers]['Proxy-Authorization'] = REDACTED
        end
      end
      if datum.has_key?(:password)
        datum[:password] = REDACTED
      end
      if datum.has_key?(:proxy) && datum[:proxy] && datum[:proxy].has_key?(:password)
        datum[:proxy] = datum[:proxy].dup
        datum[:proxy][:password] = REDACTED
      end
      datum
{'fraction_non_alphanumeric': 0.15191941192485706, 'fraction_numerical': 0.007623196297304655, 'mean_word_length': 2.4081632653061225, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4695036', 'n_tokens_mistral': 1262, 'n_tokens_neox': 1242, 'n_words': 239}