<commit_before>// Package tinystat provides the ability to compare data sets using Student's
// t-test at various levels of confidence.
package tinystat

import (
	"math"
	"sort"

	"code.google.com/p/probab/dst"
)

// A Summary is a statistical summary of a normally distributed data set.
type Summary struct {
	N        float64 // N is the number of measurements in the set.
	Min      float64 // Min is the smallest measurement.
	Max      float64 // Max is the largest measurement.
	Median   float64 // Median is the median measurement.
	Mean     float64 // Mean is the arithmetic mean of the measurements.
	Variance float64 // Variance is the sample variance of the data set.
	StdDev   float64 // StdDev is the sample standard deviation of the data set.
}

// Summarize analyzes the given data set and returns a Summary.
func Summarize(data []float64) Summary {
	min, max, mean, variance := minMaxMeanVar(data)
	median := median(data)

	return Summary{
		Min:      min,
		Max:      max,
		Mean:     mean,
		Median:   median,
		Variance: variance,
		StdDev:   math.Sqrt(variance),
		N:        float64(len(data)),
	}
}

// Difference represents the statistical difference between two samples.
type Difference struct {
	Delta  float64 // Delta is the difference between the samples' means.
	Error  float64 // Error is the margin of error at the given confidence level.
	StdDev float64 // StdDev is the standard deviation of the two samples.
}

// Significant returns true if the difference is statistically significant.
func (d Difference) Significant() bool {
	return d.Delta > d.Error
}

// Compare returns the statistical difference between the two summaries using
// Student's t-test. The confidence level must be in the range (0, 99).
func Compare(a, b Summary, confidence float64) Difference {
	// calculate the quantile for two-sided Student's t
	t := dst.StudentsTQtlFor(a.N+b.N-2, 1-((1-(confidence/100))/2))

	s := math.Sqrt(
		((a.N-1)*a.Variance + (b.N-1)*b.Variance) /
			(a.N + b.N - 2),<commit_msg>Improve the docs for Compare.<commit_after>// Package tinystat provides the ability to compare data sets using Student's
// t-test at various levels of confidence.
package tinystat

import (
	"math"
	"sort"

	"code.google.com/p/probab/dst"
)

// A Summary is a statistical summary of a normally distributed data set.
type Summary struct {
	N        float64 // N is the number of measurements in the set.
	Min      float64 // Min is the smallest measurement.
	Max      float64 // Max is the largest measurement.
	Median   float64 // Median is the median measurement.
	Mean     float64 // Mean is the arithmetic mean of the measurements.
	Variance float64 // Variance is the sample variance of the data set.
	StdDev   float64 // StdDev is the sample standard deviation of the data set.
}

// Summarize analyzes the given data set and returns a Summary.
func Summarize(data []float64) Summary {
	min, max, mean, variance := minMaxMeanVar(data)
	median := median(data)

	return Summary{
		Min:      min,
		Max:      max,
		Mean:     mean,
		Median:   median,
		Variance: variance,
		StdDev:   math.Sqrt(variance),
		N:        float64(len(data)),
	}
}

// Difference represents the statistical difference between two samples.
type Difference struct {
	Delta  float64 // Delta is the difference between the samples' means.
	Error  float64 // Error is the margin of error at the given confidence level.
	StdDev float64 // StdDev is the standard deviation of the two samples.
}

// Significant returns true if the difference is statistically significant.
func (d Difference) Significant() bool {
	return d.Delta > d.Error
}

// Compare returns the statistical difference between the two summaries using a
// two-tailed Student's t-test. The confidence level must be in the range (0,
// 100).
func Compare(a, b Summary, confidence float64) Difference {
	// calculate the quantile for two-tailed Student's t
	t := dst.StudentsTQtlFor(a.N+b.N-2, 1-((1-(confidence/100))/2))

	s := math.Sqrt(
		((a.N-1)*a.Variance + (b.N-1)*b.Variance) /
			(a.N + b.N - 2),
{'fraction_non_alphanumeric': 0.09265648102513553, 'fraction_numerical': 0.019467718087727945, 'mean_word_length': 3.2369519832985385, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2912392', 'n_tokens_mistral': 1315, 'n_tokens_neox': 1191, 'n_words': 503}