<commit_before># coding: utf-8
#
# Why would we ever use Ruby 1.8.7 when we can backport with something
# as simple as this?
#
# copied from prawn.
# modified by <NAME>
#
unless defined?(Encoding)
  class Encoding # :nodoc:
    class ConverterNotFoundError < StandardError; end
    class UndefinedConversionError < StandardError; end

    ASCII    = 0
    BINARY   = 1
    UTF_8    = 2
    EUCJP    = 3
    SJIS     = 4
    UTF_16LE = 5
    UTF_16BE = 6
  end
end

class String #:nodoc:
  def first_line
    self.each_line { |line| return line }
  end
  unless "".respond_to?(:lines)
    alias_method :lines, :to_a
  end
  unless "".respond_to?(:each_char)
    def each_char #:nodoc:
      # copied from jcode
      if block_given?
        scan(/./m) { |x| yield x }
      else
        scan(/./m)
      end
    end
  end

  unless "".respond_to?(:encode)
    @encoding = Encoding::UTF_8

    def encode(encoding) # :nodoc:
      require 'nkf'
      @encoding ||= Encoding::UTF_8
      if @encoding == Encoding::UTF_8
        # supported only $KCODE = 'u'. so @encoding.nil? means UTF_8.
        case encoding
        when /ASCII/i
          if self.mbchar?('UTF8')
            raise Encoding::UndefinedConversionError
          else
            str = String.new(self)
            str.force_encoding(encoding)
            str
          end
        when /BINARY/i
          if self.mbchar?('UTF8')
            raise Encoding::UndefinedConversionError
          else
            str = String.new(self)
            str.force_encoding(encoding)
            str
          end
        when /UTF_8/i
          raise Encoding::ConverterNotFoundError
        when /EUCJP/i, /SJIS/i
          enc = encoding =~ /EUCJP/i ? 'e' : 's'
          str = NKF.nkf("-#{enc} -m0 -W", self)
          str.force_encoding(encoding)
          str
        when /UTF_16LE/i, /UTF_16BE/i
          raise Encoding::ConverterNotFoundError
        else
          raise "Sorry, encoding #{encoding} is not supported by WriteExcel."
        end
      elsif @encoding == Encoding::ASCII
        case encoding
        when /ASCII/i, /BINARY/i, /EUCJP/i, /SJIS/i
          str = String.new(self)
          str.force_encoding(encoding)
          str
        when /UTF_8/i, /UTF_16LE/i, /UTF_16BE/i
          raise Encoding::ConverterNotFoundError
        else
          raise "Sorry, encoding #{encoding} is not supported by WriteExcel."
        end
      elsif @encoding == Encoding::BINARY
        case encoding
        when /ASCII/i
          if self.ascii_only? || self.mbchar?('UTF8') || self.mbchar?('EUCJP') || self.mbchar?('SJIS')
            raise Encoding::UndefinedConversionError
          else
            str = String.new(self)
            str.force_encoding(encoding)
            str
          end
        when /BINARY/i
          self
        when /EUCJP/i, /SJIS/i
          if self.ascii_only? || self.mbchar?('UTF8') || self.mbchar?('EUCJP') || self.mbchar?('SJIS')
            raise Encoding::UndefinedConversionError
          else
            str = String.new(self)
            str.force_encoding(encoding)
            str
          end
        when /UTF_8/i, /UTF_16LE/i, /UTF_16BE/i
          raise Encoding::ConverterNotFoundError
        else
          raise "Sorry, encoding #{encoding} is not supported by WriteExcel."
        end<commit_msg>* refactoring : same code in compatibility.rb<commit_after># coding: utf-8
#
# Why would we ever use Ruby 1.8.7 when we can backport with something
# as simple as this?
#
# copied from prawn.
# modified by <NAME>
#
unless defined?(Encoding)
  class Encoding # :nodoc:
    class ConverterNotFoundError < StandardError; end
    class UndefinedConversionError < StandardError; end

    ASCII    = 0
    BINARY   = 1
    UTF_8    = 2
    EUCJP    = 3
    SJIS     = 4
    UTF_16LE = 5
    UTF_16BE = 6
  end
end

class String #:nodoc:
  def first_line
    self.each_line { |line| return line }
  end
  unless "".respond_to?(:lines)
    alias_method :lines, :to_a
  end
  unless "".respond_to?(:each_char)
    def each_char #:nodoc:
      # copied from jcode
      if block_given?
        scan(/./m) { |x| yield x }
      else
        scan(/./m)
      end
    end
  end

  unless "".respond_to?(:encode)
    @encoding = Encoding::UTF_8

    def encode(encoding) # :nodoc:
      require 'nkf'
      @encoding ||= Encoding::UTF_8
      if @encoding == Encoding::UTF_8
        # supported only $KCODE = 'u'. so @encoding.nil? means UTF_8.
        case encoding
        when /ASCII/i
          if self.mbchar?('UTF8')
            raise Encoding::UndefinedConversionError
          else
            str = String.new(self)
            str.force_encoding(encoding)
            str
          end
        when /BINARY/i
          if self.mbchar?('UTF8')
            raise Encoding::UndefinedConversionError
          else
            str = String.new(self)
            str.force_encoding(encoding)
            str
          end
        when /UTF_8/i
          raise Encoding::ConverterNotFoundError
        when /EUCJP/i, /SJIS/i
          enc = encoding =~ /EUCJP/i ? 'e' : 's'
          str = NKF.nkf("-#{enc} -m0 -W", self)
          str.force_encoding(encoding)
          str
        when /UTF_16LE/i, /UTF_16BE/i
          raise Encoding::ConverterNotFoundError
        else
          raise "Sorry, encoding #{encoding} is not supported by WriteExcel."
        end
      elsif @encoding == Encoding::ASCII
        case encoding
        when /ASCII/i, /BINARY/i, /EUCJP/i, /SJIS/i
          str = String.new(self)
          str.force_encoding(encoding)
          str
        when /UTF_8/i, /UTF_16LE/i, /UTF_16BE/i
          raise Encoding::ConverterNotFoundError
        else
          raise "Sorry, encoding #{encoding} is not supported by WriteExcel."
        end
      elsif @encoding == Encoding::BINARY
        case encoding
        when /ASCII/i, /EUCJP/i, /SJIS/i
          if self.ascii_only? || self.mbchar?('UTF8') || self.mbchar?('EUCJP') || self.mbchar?('SJIS')
            raise Encoding::UndefinedConversionError
          else
            str = String.new(self)
            str.force_encoding(encoding)
            str
          end
        when /BINARY/i
          self
        when /UTF_8/i, /UTF_16LE/i, /UTF_16BE/i
          raise Encoding::ConverterNotFoundError
        else
          raise "Sorry, encoding #{encoding} is not supported by WriteExcel."
        end
{'fraction_non_alphanumeric': 0.10251209237010454, 'fraction_numerical': 0.012326415977531596, 'mean_word_length': 1.8114035087719298, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5978520', 'n_tokens_mistral': 2086, 'n_tokens_neox': 2009, 'n_words': 510}