<commit_before>#include <pangolin/datalog.h>

#include <limits>
#include <fstream>
#include <iomanip>
#include <stdexcept>

#include <iostream>

namespace pangolin
{

void DataLogBlock::AddSamples(size_t num_samples, size_t dimensions, const float* data_dim_major )
{
    if(nextBlock) {
        // If next block exists, add to it instead
        nextBlock->AddSamples(num_samples, dimensions, data_dim_major);
    }else{
        if(dimensions > dim) {
            // If dimensions is too high for this block, start a new bigger one
            nextBlock = new DataLogBlock(dimensions, max_samples, start_id + samples);
        }else{
            // Try to copy samples to this block
            const size_t samples_to_copy = std::min(num_samples, SampleSpaceLeft());

            if(dimensions == dim) {
                // Copy entire block all together
                std::copy(data_dim_major, data_dim_major + samples_to_copy*dim, sample_buffer+samples*dim);
                samples += samples_to_copy;
                data_dim_major += samples_to_copy*dim;
            }else{
                // Copy sample at a time, filling with NaN's where needed.
                float* dst = sample_buffer;
                for(size_t i=0; i< samples_to_copy; ++i) {
                    std::copy(data_dim_major, data_dim_major + dimensions, dst);
                    for(size_t ii = dimensions; ii < dim; ++ii) {
                        dst[ii] = std::numeric_limits<float>::quiet_NaN();
                    }
                    dst += dimensions;
                    data_dim_major += dimensions;
                }
                samples += samples_to_copy;
            }

            // Copy remaining data to next block (this one is full)
            if(samples_to_copy < num_samples) {
                nextBlock = new DataLogBlock(dim, max_samples, start_id + Samples());
                nextBlock->AddSamples(num_samples-samples_to_copy, dimensions, data_dim_major);
            }
        }
    }
}

DataLog::DataLog(unsigned int buffer_size)
    : block_samples_alloc(buffer_size), block0(0), blockn(0), record_stats(false)
{
}

DataLog::~DataLog()
{
    Clear();
}

void DataLog::SetLabels(const std::vector<std::string> & new_labels)
{
    // Create new labels if needed
    for( unsigned int i= labels.size(); i < new_labels.size(); ++i )
        labels.push_back( std::string() );

    // Add data to existing plots
    for( unsigned int i=0; i<labels.size(); ++i )
        labels[i] = new_labels[i];
}

void DataLog::Log(unsigned int dimension, const float* vals, unsigned int samples )
{
    if(!block0) {
        // Create first block
        block0 = new DataLogBlock(dimension, block_samples_alloc, 0);
        blockn = block0;
    }

    if(record_stats) {
        while(stats.size() < dimension) {
            stats.push_back( DimensionStats() );
        }
        for(int d=0; d<dimension; ++d) {
            DimensionStats& ds = stats[d];
            for(int s=0; s<samples; ++s) {
                const float v = vals[s*dimension+d];
                ds.isMonotonic = ds.isMonotonic && (v >= ds.max);
                ds.sum += v;
                ds.sum_sq += v*v;
                ds.min = std::min(ds.min, v);
                ds.max = std::max(ds.max, v);
            }
        }<commit_msg>Fix warning.<commit_after>#include <pangolin/datalog.h>

#include <limits>
#include <fstream>
#include <iomanip>
#include <stdexcept>

#include <iostream>

namespace pangolin
{

void DataLogBlock::AddSamples(size_t num_samples, size_t dimensions, const float* data_dim_major )
{
    if(nextBlock) {
        // If next block exists, add to it instead
        nextBlock->AddSamples(num_samples, dimensions, data_dim_major);
    }else{
        if(dimensions > dim) {
            // If dimensions is too high for this block, start a new bigger one
            nextBlock = new DataLogBlock(dimensions, max_samples, start_id + samples);
        }else{
            // Try to copy samples to this block
            const size_t samples_to_copy = std::min(num_samples, SampleSpaceLeft());

            if(dimensions == dim) {
                // Copy entire block all together
                std::copy(data_dim_major, data_dim_major + samples_to_copy*dim, sample_buffer+samples*dim);
                samples += samples_to_copy;
                data_dim_major += samples_to_copy*dim;
            }else{
                // Copy sample at a time, filling with NaN's where needed.
                float* dst = sample_buffer;
                for(size_t i=0; i< samples_to_copy; ++i) {
                    std::copy(data_dim_major, data_dim_major + dimensions, dst);
                    for(size_t ii = dimensions; ii < dim; ++ii) {
                        dst[ii] = std::numeric_limits<float>::quiet_NaN();
                    }
                    dst += dimensions;
                    data_dim_major += dimensions;
                }
                samples += samples_to_copy;
            }

            // Copy remaining data to next block (this one is full)
            if(samples_to_copy < num_samples) {
                nextBlock = new DataLogBlock(dim, max_samples, start_id + Samples());
                nextBlock->AddSamples(num_samples-samples_to_copy, dimensions, data_dim_major);
            }
        }
    }
}

DataLog::DataLog(unsigned int buffer_size)
    : block_samples_alloc(buffer_size), block0(0), blockn(0), record_stats(false)
{
}

DataLog::~DataLog()
{
    Clear();
}

void DataLog::SetLabels(const std::vector<std::string> & new_labels)
{
    // Create new labels if needed
    for( unsigned int i= labels.size(); i < new_labels.size(); ++i )
        labels.push_back( std::string() );

    // Add data to existing plots
    for( unsigned int i=0; i<labels.size(); ++i )
        labels[i] = new_labels[i];
}

void DataLog::Log(unsigned int dimension, const float* vals, unsigned int samples )
{
    if(!block0) {
        // Create first block
        block0 = new DataLogBlock(dimension, block_samples_alloc, 0);
        blockn = block0;
    }

    if(record_stats) {
        while(stats.size() < dimension) {
            stats.push_back( DimensionStats() );
        }
        for(unsigned int d=0; d<dimension; ++d) {
            DimensionStats& ds = stats[d];
            for(unsigned int s=0; s<samples; ++s) {
                const float v = vals[s*dimension+d];
                ds.isMonotonic = ds.isMonotonic && (v >= ds.max);
                ds.sum += v;
                ds.sum_sq += v*v;
                ds.min = std::min(ds.min, v);
                ds.max = std::max(ds.max, v);
            }
        }
{'fraction_non_alphanumeric': 0.11510031678986272, 'fraction_numerical': 0.0033187509428269724, 'mean_word_length': 2.0260155180282977, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1252262', 'n_tokens_mistral': 2024, 'n_tokens_neox': 1948, 'n_words': 479}