<commit_before>#! /usr/bin/env ruby
## -*- mode: ruby -*-
## = Statistics Utility
## Author:: <NAME>
## Version:: 0.0 2010/??/?? I.Noda
##
## === History
## * [2010/??/??]: Create This File.
## * [2014/08/24]: reform
## == Usage
## * ...

#--======================================================================
#++
module Stat

  #--======================================================================
  #++
  ## Cumulate statical infomation and calculate average, variance and
  ## standard diviation.
  ## It also record history.

  class StatInfo
    #--@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
    #++
    ## minimum in given values.
    attr :min, true ;
    ## maximum in given values.
    attr :max, true ;
    ## sum of given values.
    attr :sum, true ;
    ## squared sum of given values.
    attr :qsum, true ;
    ## the number of given values.
    attr :countN, true ;
    ## history of given values.
    attr :history, true ;
    ## history maximum size.
    attr :historySize, true ;

    #--------------------------------------------------------------
    #++
    def initialize(historySize = 1)
      # historySize = 0 means infinite history

      @sum = 0.0 ;
      @qsum = 0.0 ;
      @countN = 0 ;

      @min = nil ;
      @max = nil ;

      @historySize = historySize ;
      @history = Array.new(@historySize) ;
    end

    #--------------------------------------------------------------
    #++
    def put(value)
      @min = value if (@min.nil? || value < @min) ;
      @max = value if (@max.nil? || value > @max) ;

      @sum += value ;
      @qsum += value * value ;

      if(@historySize > 0)
        @history[@countN % @history.size] = value ;
      else
        @history.push(value) ;
      end

      @countN += 1 ;
    end

    #--------------------------------------------------------------
    #++
    ## == access to history

    #------------------------------------------
    #++<commit_msg>add Exponential Moving Average.<commit_after>#! /usr/bin/env ruby
## -*- mode: ruby -*-
## = Statistics Utility
## Author:: <NAME>
## Version:: 0.0 2010/??/?? I.Noda
##
## === History
## * [2010/??/??]: Create This File.
## * [2014/08/24]: reform
## * [2014/08/29]: add exponential moving average
## == Usage
## * ...

#--======================================================================
#++
module Stat

  #--======================================================================
  #++
  ## Cumulate statical infomation and calculate average, variance and
  ## standard diviation.
  ## It also record history.

  class StatInfo
    #--@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
    #++
    ## minimum in given values.
    attr :min, true ;
    ## maximum in given values.
    attr :max, true ;
    ## sum of given values.
    attr :sum, true ;
    ## squared sum of given values.
    attr :qsum, true ;
    ## the number of given values.
    attr :countN, true ;
    ## history of given values.
    attr :history, true ;
    ## history maximum size.
    attr :historySize, true ;

    ## stepsize (alpha) for exponential moving average
    attr :alpha, true ;
    ## exponential moving average
    attr :ema, true ;

    #--------------------------------------------------------------
    #++
    def initialize(historySize = 1, alpha = 0.01)
      # historySize = 0 means infinite history

      @sum = 0.0 ;
      @qsum = 0.0 ;
      @countN = 0 ;

      @min = nil ;
      @max = nil ;

      @historySize = historySize ;
      @history = Array.new(@historySize) ;

      @alpha = alpha ;
      @ema = 0.0 ;
    end

    #--------------------------------------------------------------
    #++
    def put(value)
      @min = value if (@min.nil? || value < @min) ;
      @max = value if (@max.nil? || value > @max) ;

      @sum += value ;
      @qsum += value * value ;

      if(@historySize > 0)
        @history[@countN % @history.size] = value ;
      else
        @history.push(value) ;
      end

      @ema = ((@countN == 0) ? 
              value : 
              ((1.0-@alpha) * @ema + @alpha * value)) ;

      @countN += 1 ;
    end

    #--------------------------------------------------------------
    #++
    ## == access to history

    #------------------------------------------
    #++
{'fraction_non_alphanumeric': 0.3306921463528315, 'fraction_numerical': 0.01631321370309951, 'mean_word_length': 1.615478366849482, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4248925', 'n_tokens_mistral': 1306, 'n_tokens_neox': 1195, 'n_words': 327}