<commit_before>///
/// \author	<NAME>
///
/// \copyright Copyright 2014 <NAME> 
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
/// 
/// http://www.apache.org/licenses/LICENSE-2.0
/// 
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///

#include <celero/Archive.h>
#include <celero/PimplImpl.h>
#include <celero/Benchmark.h>
#include <celero/FileReader.h>

#include <assert.h>

#include <map>
#include <vector>
#include <algorithm>
#include <iostream>
#include <fstream>
#include <chrono>

using namespace celero;

///
/// Structure to assist with archiving data during runtime and to a file.
///
struct ArchiveEntry
{
	ArchiveEntry() :
		GroupName(),
		RunName(),
		ExperimentValue(0),
		FirstRanDate(0),
		TotalSamplesCollected(0),
		AverageBaseline(0),
		MinBaseline(0),
		MinBaseline_TimeSinceEpoch(0),
		MinStats(),
		MaxBaseline(0),
		MaxBaseline_TimeSinceEpoch(0),
		MaxStats(),
		CurrentBaseline(0),
		CurrentBaseline_TimeSinceEpoch(0),
		CurrentStats()
	{
	}

	static void WriteHeader(std::ostream& str)
	{
		str << "GroupName,RunName,ExperimentValue,FirstRanDate,TotalSamplesCollected,AverageBaseline,";
		str << "MinBaseline,MinBaselineTimeSinceEpoch,";
		str << "MinStatSize,MinStatMean,MinStatVariance,MinStatStandardDeviation,MinStatSkewness,MinStatKurtosis,";
		str << "MinStatMin,MinStatMax,";
		str << "MaxBaseline,MaxBaselineTimeSinceEpoch,";
		str << "MaxStatSize,MaxStatMean,MaxStatVariance,MaxStatStandardDeviation,MaxStatSkewness,MaxStatKurtosis,";
		str << "MaxStatMin,MaxStatMax,";
		str << "CurrentBaseline,CurrentBaselineTimeSinceEpoch,";
		str << "CurrentStatSize,CurrentStatMean,CurrentStatVariance,CurrentStatStandardDeviation,CurrentStatSkewness,CurrentStatKurtosis,";
		str << "CurrentStatMin,CurrentStatMax\n";
	}

	struct Stat
	{
		Stat() :
			Size(0),
			Mean(0),
			Variance(0),
			StandardDeviation(0),
			Skewness(0),
			Kurtosis(0),
			Min(0),
			Max(0)
		{
		}

		Stat& operator=(const celero::Statistics& s)
		{
			this->Size = s.getSize();
			this->Mean = s.getMean();
			this->Variance = s.getVariance();
			this->StandardDeviation = s.getStandardDeviation();
			this->Skewness = s.getSkewness();
			this->Kurtosis = s.getKurtosis();
			this->Min = s.getMin();
			this->Max = s.getMax();

			return *this;
		}

		uint64_t Size;
		double Mean;
		double Variance;
		double StandardDeviation;
		double Skewness;
		double Kurtosis;
		uint64_t Min;
		uint64_t Max;
	};

	std::string GroupName;
	std::string RunName;

	/// The data set size, if one was specified.
	int64_t ExperimentValue;

	uint64_t FirstRanDate;
	uint32_t TotalSamplesCollected;

	double AverageBaseline;

	double MinBaseline;
	uint64_t MinBaseline_TimeSinceEpoch;
	Stat MinStats;
			
	double MaxBaseline;
	uint64_t MaxBaseline_TimeSinceEpoch;
	Stat MaxStats;
			
	double CurrentBaseline;
	uint64_t CurrentBaseline_TimeSinceEpoch;
	Stat CurrentStats;
};

///
/// Overload operator<< to allow for easy output of result data to a human-readable text file.
///
std::ostream& operator<<(std::ostream& str, ArchiveEntry::Stat const& data)
{
	str << data.Size << ",";
	str << data.Mean << ",";
	str << data.Variance << ",";
	str << data.StandardDeviation << ",";
	str << data.Skewness << ",";
	str << data.Kurtosis << ",";
	str << data.Min << ",";
	str << data.Max << ",";
	return str;<commit_msg>Remove extra comma fromm statistics struct print function (not included in last commit<commit_after>///
/// \author	<NAME>
///
/// \copyright Copyright 2014 <NAME> 
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
/// 
/// http://www.apache.org/licenses/LICENSE-2.0
/// 
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///

#include <celero/Archive.h>
#include <celero/PimplImpl.h>
#include <celero/Benchmark.h>
#include <celero/FileReader.h>

#include <assert.h>

#include <map>
#include <vector>
#include <algorithm>
#include <iostream>
#include <fstream>
#include <chrono>

using namespace celero;

///
/// Structure to assist with archiving data during runtime and to a file.
///
struct ArchiveEntry
{
	ArchiveEntry() :
		GroupName(),
		RunName(),
		ExperimentValue(0),
		FirstRanDate(0),
		TotalSamplesCollected(0),
		AverageBaseline(0),
		MinBaseline(0),
		MinBaseline_TimeSinceEpoch(0),
		MinStats(),
		MaxBaseline(0),
		MaxBaseline_TimeSinceEpoch(0),
		MaxStats(),
		CurrentBaseline(0),
		CurrentBaseline_TimeSinceEpoch(0),
		CurrentStats()
	{
	}

	static void WriteHeader(std::ostream& str)
	{
		str << "GroupName,RunName,ExperimentValue,FirstRanDate,TotalSamplesCollected,AverageBaseline,";
		str << "MinBaseline,MinBaselineTimeSinceEpoch,";
		str << "MinStatSize,MinStatMean,MinStatVariance,MinStatStandardDeviation,MinStatSkewness,MinStatKurtosis,";
		str << "MinStatMin,MinStatMax,";
		str << "MaxBaseline,MaxBaselineTimeSinceEpoch,";
		str << "MaxStatSize,MaxStatMean,MaxStatVariance,MaxStatStandardDeviation,MaxStatSkewness,MaxStatKurtosis,";
		str << "MaxStatMin,MaxStatMax,";
		str << "CurrentBaseline,CurrentBaselineTimeSinceEpoch,";
		str << "CurrentStatSize,CurrentStatMean,CurrentStatVariance,CurrentStatStandardDeviation,CurrentStatSkewness,CurrentStatKurtosis,";
		str << "CurrentStatMin,CurrentStatMax\n";
	}

	struct Stat
	{
		Stat() :
			Size(0),
			Mean(0),
			Variance(0),
			StandardDeviation(0),
			Skewness(0),
			Kurtosis(0),
			Min(0),
			Max(0)
		{
		}

		Stat& operator=(const celero::Statistics& s)
		{
			this->Size = s.getSize();
			this->Mean = s.getMean();
			this->Variance = s.getVariance();
			this->StandardDeviation = s.getStandardDeviation();
			this->Skewness = s.getSkewness();
			this->Kurtosis = s.getKurtosis();
			this->Min = s.getMin();
			this->Max = s.getMax();

			return *this;
		}

		uint64_t Size;
		double Mean;
		double Variance;
		double StandardDeviation;
		double Skewness;
		double Kurtosis;
		uint64_t Min;
		uint64_t Max;
	};

	std::string GroupName;
	std::string RunName;

	/// The data set size, if one was specified.
	int64_t ExperimentValue;

	uint64_t FirstRanDate;
	uint32_t TotalSamplesCollected;

	double AverageBaseline;

	double MinBaseline;
	uint64_t MinBaseline_TimeSinceEpoch;
	Stat MinStats;
			
	double MaxBaseline;
	uint64_t MaxBaseline_TimeSinceEpoch;
	Stat MaxStats;
			
	double CurrentBaseline;
	uint64_t CurrentBaseline_TimeSinceEpoch;
	Stat CurrentStats;
};

///
/// Overload operator<< to allow for easy output of result data to a human-readable text file.
///
std::ostream& operator<<(std::ostream& str, ArchiveEntry::Stat const& data)
{
	str << data.Size << ",";
	str << data.Mean << ",";
	str << data.Variance << ",";
	str << data.StandardDeviation << ",";
	str << data.Skewness << ",";
	str << data.Kurtosis << ",";
	str << data.Min << ",";
	str << data.Max;
	return str;
{'fraction_non_alphanumeric': 0.13644859813084112, 'fraction_numerical': 0.011748998664886516, 'mean_word_length': 4.440087145969499, 'pattern_counts': {'":': 0, '<': 139, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4576819', 'n_tokens_mistral': 2758, 'n_tokens_neox': 2467, 'n_words': 591}