<commit_before># coding: utf-8

class NumericWithUnit
  class Unit
    class Config
      attr_reader :symbol, :dimension
      attr_reader :si
      
      def initialize()
        @symbol = nil
        @dimension = Hash.new(0)
        @from_si = nil
        @to_si = nil
        @si = false
      end
      
      def compile
        @dimension.delete_if{|k,v| v.zero?}
        
        @from_si ||= ->(x){x}
        @to_si ||= ->(x){x}

        self
      end
      
      def symbol=(arg)
        @symbol = arg.to_s
      end
      
      def dimension=(arg)
        raise unless arg.is_a?(Hash)
        @dimension = arg
      end
      
      def from_si=(arg)
        raise unless arg.is_a?(Proc)
        @from_si = arg
      end
      
      def from_si(&block)
        if block_given?
          @from_si = block
        else
          @from_si
        end
      end
      
      def to_si=(arg)
        raise unless arg.is_a?(Proc)
        @to_si = arg
      end
      
      def to_si(&block)
        if block_given?
          @to_si = block
        else
          @to_si
        end
      end
      
      def si=(arg)
        raise unless [TrueClass, FalseClass].any?{|klass|arg.is_a?(klass)}
        @si = arg
      end
    end
  end
end



class NumericWithUnit
  class Unit
    @@list = []
    @@prefix = {
      'Y' => 10**24, 
      'Z' => 10**21, 
      'E' => 10**18, 
      'P' => 10**15, 
      'T' => 10**12, 
      'G' => 10**9, 
      'M' => 10**6, 
      'k' => 10**3, 
      'h' => 10**2, 
      'da' => 10**1, 
      'd' => 10**-1, 
      'c' => 10**-2, 
      'm' => 10**-3, 
      'μ' => 10**-6, 
      'u' => 10**-6, 
      'n' => 10**-9, 
      'p' => 10**-12, 
      'f' => 10**-15, 
      'a' => 10**-18, 
      'z' => 10**-21, 
      'y' => 10**-24,
      'Ki' => 2**10,
      'Mi' => 2**20,
      'Gi' => 2**30,
      'Ti' => 2**40,
      'Pi' => 2**50,
      'Ei' => 2**60,
      'Zi' => 2**70,
      'Yi' => 2**80
    }
    
    # class methods

    # create new unit from derivation _(for internal use)_ .
    def self.derive
      derivation = Hash.new(0)
      yield(derivation)

      return Unit.new if derivation.empty?

      dimension = Hash.new(0)

      h = derivation.sort_by{|u,v| u.symbol}.sort_by{|u,v| v} # ←どうしよう
      
      syms_pos = h.select{|u,v| v > 0}.map{|u,v| u.symbol + ((v.abs>1) ? v.abs.to_s : '')}
      syms_neg = h.select{|u,v| v < 0}.map{|u,v| u.symbol + ((v.abs>1) ? v.abs.to_s : '')}
      symbol = syms_pos.join('.')
      symbol += '/' + (syms_neg.size==1 ? "#{syms_neg.first}" : "(#{syms_neg.join('.')})") unless syms_neg.empty?
      
      derivation.each do |u,v|
        u.dimension.each do |d,i|
          dimension[d] += i*v
        end
      end

      from_si = derivation.map{|u,v|
        prc = if v > 0
          ->(x){u.from_si(x)}
        else
          ->(x){x.quo(u.from_si(1)-u.from_si(0))} # ℃とKの変換のような場合に、変換式の切片を消すため。変換式が線形じゃないケースは想定していない
        end
        [prc, v.abs]
      }.map{|prc,v|
        ->(x){ v.times{x = prc[x]}; x }
      }.reduce{|memo, prc|
        ->(x){memo[prc[x]]}
      }
      
      to_si = derivation.map{|u,v|
        prc = if v > 0
          ->(x){u.to_si(x)}
        else
          ->(x){x.quo(u.to_si(1)-u.to_si(0))} # ℃とKの変換のような場合に、変換式の切片を消すため。変換式が線形じゃないケースは想定していない
        end
        [prc, v.abs]
      }.map{|prc,v|
        ->(x){ v.times{x = prc[x]}; x }
      }.reduce{|memo, prc|
        ->(x){memo[prc[x]]}
      }

      self.new(derivation){|config|
        config.symbol = symbol
        config.dimension = dimension
        config.from_si = from_si
        config.to_si = to_si
      }
    end
    
    # apply to_s to arg and return parsed unit.
    def self.[](arg)
      self.parse(arg.to_s)
    end
    
    # cast unit and add unit to base unit list at the same time.
    def self.[]=(key, arg)
      if arg.is_a?(Array) and arg.size == 2
        a = [key, arg.first]
        u = arg.last
      else
        a = [key]
        u = arg
      end
      @@list << (u.is_a?(self) ? u : self[u]).cast(*a)
    end
    <commit_msg>bug fix for Unit#derive<commit_after># coding: utf-8

class NumericWithUnit
  class Unit
    class Config
      attr_reader :symbol, :dimension
      attr_reader :si
      
      def initialize()
        @symbol = nil
        @dimension = Hash.new(0)
        @from_si = nil
        @to_si = nil
        @si = false
      end
      
      def compile
        @dimension.delete_if{|k,v| v.zero?}
        
        @from_si ||= ->(x){x}
        @to_si ||= ->(x){x}

        self
      end
      
      def symbol=(arg)
        @symbol = arg.to_s
      end
      
      def dimension=(arg)
        raise unless arg.is_a?(Hash)
        @dimension = arg
      end
      
      def from_si=(arg)
        raise unless arg.is_a?(Proc)
        @from_si = arg
      end
      
      def from_si(&block)
        if block_given?
          @from_si = block
        else
          @from_si
        end
      end
      
      def to_si=(arg)
        raise unless arg.is_a?(Proc)
        @to_si = arg
      end
      
      def to_si(&block)
        if block_given?
          @to_si = block
        else
          @to_si
        end
      end
      
      def si=(arg)
        raise unless [TrueClass, FalseClass].any?{|klass|arg.is_a?(klass)}
        @si = arg
      end
    end
  end
end



class NumericWithUnit
  class Unit
    @@list = []
    @@prefix = {
      'Y' => 10**24, 
      'Z' => 10**21, 
      'E' => 10**18, 
      'P' => 10**15, 
      'T' => 10**12, 
      'G' => 10**9, 
      'M' => 10**6, 
      'k' => 10**3, 
      'h' => 10**2, 
      'da' => 10**1, 
      'd' => 10**-1, 
      'c' => 10**-2, 
      'm' => 10**-3, 
      'μ' => 10**-6, 
      'u' => 10**-6, 
      'n' => 10**-9, 
      'p' => 10**-12, 
      'f' => 10**-15, 
      'a' => 10**-18, 
      'z' => 10**-21, 
      'y' => 10**-24,
      'Ki' => 2**10,
      'Mi' => 2**20,
      'Gi' => 2**30,
      'Ti' => 2**40,
      'Pi' => 2**50,
      'Ei' => 2**60,
      'Zi' => 2**70,
      'Yi' => 2**80
    }
    
    # class methods

    # create new unit from derivation _(for internal use)_ .
    def self.derive
      derivation = Hash.new(0)
      yield(derivation)
      return Unit.new if derivation.empty?
      derivation.delete_if{|k,v| k.symbol.nil?}
      
      # constructing symbol
      h = derivation.reject{|k,v| k.symbol.empty?}.sort_by{|u,v| u.symbol}.sort_by{|u,v| v}
      syms_pos = h.select{|u,v| v > 0}.map{|u,v| u.symbol + (v.abs>1 ? v.abs.to_s : '')}
      syms_neg = h.select{|u,v| v < 0}.map{|u,v| u.symbol + (v.abs>1 ? v.abs.to_s : '')}
      symbol = syms_pos.join('.')
      symbol += '/' + (syms_neg.size==1 ? "#{syms_neg.first}" : "(#{syms_neg.join('.')})") unless syms_neg.empty?
      
      # constructing dimension
      dimension = Hash.new(0)
      derivation.each do |u,v|
        u.dimension.each do |d,i|
          dimension[d] += i*v
        end
      end
      
      # constructing from_si proc
      from_si = derivation.map{|u,v|
        prc = if v > 0
          ->(x){u.from_si(x)}
        else
          ->(x){x.quo(u.from_si(1)-u.from_si(0))} # ℃とKの変換のような場合に、変換式の切片を消すため。変換式が線形じゃないケースは想定していない
        end
        [prc, v.abs]
      }.map{|prc,v|
        ->(x){ v.times{x = prc[x]}; x }
      }.reduce{|memo, prc|
        ->(x){memo[prc[x]]}
      }
      
      # constructing to_si proc
      to_si = derivation.map{|u,v|
        prc = if v > 0
          ->(x){u.to_si(x)}
        else
          ->(x){x.quo(u.to_si(1)-u.to_si(0))} # ℃とKの変換のような場合に、変換式の切片を消すため。変換式が線形じゃないケースは想定していない
        end
        [prc, v.abs]
      }.map{|prc,v|
        ->(x){ v.times{x = prc[x]}; x }
      }.reduce{|memo, prc|
        ->(x){memo[prc[x]]}
      }
      
      # deriving new unit
      self.new(derivation){|config|
        config.symbol = symbol
        config.dimension = dimension
        config.from_si = from_si
        config.to_si = to_si
      }
    end
    
    # apply to_s to arg and return parsed unit.
    def self.[](arg)
      self.parse(arg.to_s)
    end
    
    # cast unit and add unit to base unit list at the same time.
    def self.[]=(key, arg)
      if arg.is_a?(Array) and arg.size == 2
        a = [key, arg.first]
        u = arg.last
      else
        a = [key]
        u = arg
      end
      @@list << (u.is_a?(self) ? u : self[u]).cast(*a)
    end
    
{'fraction_non_alphanumeric': 0.17278927844920425, 'fraction_numerical': 0.0270431973196123, 'mean_word_length': 1.4904648390941597, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5805374', 'n_tokens_mistral': 3597, 'n_tokens_neox': 3338, 'n_words': 660}