<commit_before>module Statistics
  class Summary
    extend Statistics::Quantitative

    # Generate a five number summary for a data set.
    # Includes:
    #   - minimum
    #   - 1st quartile
    #   - median
    #   - 3rd quartile
    #   - maximum
    def self.five_number_summary(data)
      {
        minimum: minimum(data),
        q1: q1(data),
        median: median(data),
        q3: q3(data),
        maximum: maximum(data)
      }

    end

    # Generate an extended summary for a data set.
    # Includes:
    #   - minimum
    #   - 1st quartile
    #   - median
    #   - 3rd quartile
    #   - maximum
    #   - range
    #   - mean
    #   - interquartile range
    #   - fences
    #   - outliers
    #   - variance
    #   - standard deviation
    #   - sum
    #   - count
    def self.extended_summary(data)
      self.five_number_summary(data).merge({
        range:    range(data),
        mean:     mean(data),
        iqr:      iqr(data),
        fences:   fences(data),
        outliers: outliers(data),
        variance: variance(data),
        std_dev:  standard_deviation(data),
        sum:      sum(data),
        count:    count(data)
      })
    end

    def self.summary(data)
      int_hash = {}
      self.five_number_summary(data).merge({
        mean:     mean(data),
        iqr:      iqr(data),
        variance: variance(data),
        std_dev:  standard_deviation(data),
        count:    count(data)
      }).each{|key,val| int_hash[key] = val.to_i}
      int_hash
    end

  end
end
<commit_msg>removing variance from summary report<commit_after>module Statistics
  class Summary
    extend Statistics::Quantitative

    # Generate a five number summary for a data set.
    # Includes:
    #   - minimum
    #   - 1st quartile
    #   - median
    #   - 3rd quartile
    #   - maximum
    def self.five_number_summary(data)
      {
        minimum: minimum(data),
        q1: q1(data),
        median: median(data),
        q3: q3(data),
        maximum: maximum(data)
      }

    end

    # Generate an extended summary for a data set.
    # Includes:
    #   - minimum
    #   - 1st quartile
    #   - median
    #   - 3rd quartile
    #   - maximum
    #   - range
    #   - mean
    #   - interquartile range
    #   - fences
    #   - outliers
    #   - variance
    #   - standard deviation
    #   - sum
    #   - count
    def self.extended_summary(data)
      self.five_number_summary(data).merge({
        range:    range(data),
        mean:     mean(data),
        iqr:      iqr(data),
        fences:   fences(data),
        outliers: outliers(data),
        variance: variance(data),
        std_dev:  standard_deviation(data),
        sum:      sum(data),
        count:    count(data)
      })
    end

    def self.summary(data)
      int_hash = {}
      self.five_number_summary(data).merge({
        mean:     mean(data),
        iqr:      iqr(data),
        std_dev:  standard_deviation(data),
        count:    count(data)
      }).each{|key,val| int_hash[key] = val.to_i}
      int_hash
    end

  end
end

{'fraction_non_alphanumeric': 0.10526315789473684, 'fraction_numerical': 0.005198180636777128, 'mean_word_length': 1.6338751069289992, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4407924', 'n_tokens_mistral': 1026, 'n_tokens_neox': 982, 'n_words': 214}