<commit_before>package us.hebi.histogram.visualizer.parser;

import javafx.collections.ObservableList;
import javafx.scene.chart.XYChart.Data;
import org.HdrHistogram.*;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;

import static com.google.common.base.Preconditions.*;

/**
 * @author <NAME> < <EMAIL> >
 * @since 04 Jul 2015
 */
public class LogParser {

    public void parseLog(final ParserConfiguration config) throws IOException {
        checkNotNull(config);

        try (InputStream inputStream = new FileInputStream(config.getInputFile())) {
            HistogramLogReader logReader = new HistogramLogReader(inputStream);

            intervalData.clear();
            percentileData.clear();
            EncodableHistogram intervalHistogram = null;
            EncodableHistogram accumulatedHistogram = null;
            final List<Data<Number, Number>> rawIntervalData = new ArrayList<>(INITIAL_LIST_SIZE);

            while (true) {

                // Read interval
                intervalHistogram = logReader.nextIntervalHistogram(config.getStartTimeSec(), config.getEndTimeSec());
                if (intervalHistogram == null)
                    break;

                // Initialize accumulated once
                if (accumulatedHistogram == null) {
                    accumulatedHistogram = copyAsAccumulatorHistogram(intervalHistogram);
                }

                // Make sure histogram versions didn't change within a single log
                checkState(intervalHistogram.getClass().equals(accumulatedHistogram.getClass()),
                        "Encountered different Histogram versions in a single log");

                // Accumulate
                addToHistogram(accumulatedHistogram, intervalHistogram);

                // Add to interval data
                final double x = intervalHistogram.getEndTimeStamp() * 1E-3 - logReader.getStartTimeSec();
                final double y = intervalHistogram.getMaxValueAsDouble() / config.getOutputValueUnitRatio();
                rawIntervalData.add(new Data<>(x, y));

            }

            // Aggregate intervals for faster plotting
            storeAggregatedIntervals(rawIntervalData, config.getAggregateIntervalSamples());

            // Skip if bounds have no data, e.g., when start and end = 0
            if (accumulatedHistogram == null)
                return;
            addPercentileData(accumulatedHistogram, config);

        }

    }

    private void storeAggregatedIntervals(List<Data<Number, Number>> rawIntervals, int samples) {
        checkArgument(samples > 0, "Can't aggregate negative intervals.");

        // Resize aggregated list
        final int rawSize = rawIntervals.size();
        intervalData.clear();
        intervalData.ensureCapacity(rawSize / samples + 1);

        // Fast-track default case
        if (samples == 1) {
            intervalData.addAll(rawIntervals);
            return;
        }

        // Stride through raw data and store the maxima within a time interval
        for (int i = 0; i < rawSize; i += samples) {

            // Use lowest x value as time interval as that results in the same
            // behavior as if the histograms would have been recorded in
            // larger intervals from the beginning.
            Number x = rawIntervals.get(i).getXValue();
            Number y = rawIntervals.get(i).getYValue();

            // Search for highest y value as we are interested in the maxima
            // of each interval.
            for (int j = 1; j < samples && (i + j) < rawSize; j++) {
                Number yNext = rawIntervals.get(i + j).getYValue();
                if (yNext.doubleValue() > y.doubleValue()) {
                    y = yNext;
                }
            }

            intervalData.add(new Data<>(x, y));

        }
    }

    private void addPercentileData(EncodableHistogram accumulatedHistogram, ParserConfiguration config) {
        if (accumulatedHistogram instanceof DoubleHistogram) {
            DoublePercentileIterator percentileIterator = new DoublePercentileIterator(
                    (DoubleHistogram) accumulatedHistogram,
                    config.getPercentilesOutputTicksPerHalf());

            while (percentileIterator.hasNext()) {
                DoubleHistogramIterationValue iterationValue = percentileIterator.next();
                addPercentileDatapoint(
                        iterationValue.getPercentileLevelIteratedTo(),
                        iterationValue.getValueIteratedTo(),
                        config.getOutputValueUnitRatio());
            }
        } else {
            PercentileIterator percentileIterator = new PercentileIterator(
                    (Histogram) accumulatedHistogram,
                    config.getPercentilesOutputTicksPerHalf());

            while (percentileIterator.hasNext()) {
                HistogramIterationValue iterationValue = percentileIterator.next();<commit_msg>added helper function for storing histograms as csv files that are simpler to import by matlab<commit_after>package us.hebi.histogram.visualizer.parser;

import javafx.collections.ObservableList;
import javafx.scene.chart.XYChart.Data;
import org.HdrHistogram.*;

import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;

import static com.google.common.base.Preconditions.*;

/**
 * @author <NAME> < <EMAIL> >
 * @since 04 Jul 2015
 */
public class LogParser {

    public void parseLog(final ParserConfiguration config) throws IOException {
        checkNotNull(config);

        try (InputStream inputStream = new FileInputStream(config.getInputFile())) {
            HistogramLogReader logReader = new HistogramLogReader(inputStream);

            intervalData.clear();
            percentileData.clear();
            EncodableHistogram intervalHistogram = null;
            EncodableHistogram accumulatedHistogram = null;
            final List<Data<Number, Number>> rawIntervalData = new ArrayList<>(INITIAL_LIST_SIZE);

            while (true) {

                // Read interval
                intervalHistogram = logReader.nextIntervalHistogram(config.getStartTimeSec(), config.getEndTimeSec());
                if (intervalHistogram == null)
                    break;

                // Initialize accumulated once
                if (accumulatedHistogram == null) {
                    accumulatedHistogram = copyAsAccumulatorHistogram(intervalHistogram);
                }

                // Make sure histogram versions didn't change within a single log
                checkState(intervalHistogram.getClass().equals(accumulatedHistogram.getClass()),
                        "Encountered different Histogram versions in a single log");

                // Accumulate
                addToHistogram(accumulatedHistogram, intervalHistogram);

                // Add to interval data
                final double x = intervalHistogram.getEndTimeStamp() * 1E-3 - logReader.getStartTimeSec();
                final double y = intervalHistogram.getMaxValueAsDouble() / config.getOutputValueUnitRatio();
                rawIntervalData.add(new Data<>(x, y));

            }

            // Aggregate intervals for faster plotting
            storeAggregatedIntervals(rawIntervalData, config.getAggregateIntervalSamples());

            // Skip if bounds have no data, e.g., when start and end = 0
            if (accumulatedHistogram == null)
                return;
            addPercentileData(accumulatedHistogram, config);

        }

    }

    private void storeAggregatedIntervals(List<Data<Number, Number>> rawIntervals, int samples) {
        checkArgument(samples > 0, "Can't aggregate negative intervals.");

        // Resize aggregated list
        final int rawSize = rawIntervals.size();
        intervalData.clear();
        intervalData.ensureCapacity(rawSize / samples + 1);

        // Fast-track default case
        if (samples == 1) {
            intervalData.addAll(rawIntervals);
            return;
        }

        // Stride through raw data and store the maxima within a time interval
        for (int i = 0; i < rawSize; i += samples) {

            // Use lowest x value as time interval as that results in the same
            // behavior as if the histograms would have been recorded in
            // larger intervals from the beginning.
            Number x = rawIntervals.get(i).getXValue();
            Number y = rawIntervals.get(i).getYValue();

            // Search for highest y value as we are interested in the maxima
            // of each interval.
            for (int j = 1; j < samples && (i + j) < rawSize; j++) {
                Number yNext = rawIntervals.get(i + j).getYValue();
                if (yNext.doubleValue() > y.doubleValue()) {
                    y = yNext;
                }
            }

            intervalData.add(new Data<>(x, y));

        }
    }

    static void outputHistogramValuesAsCsv(AbstractHistogram histogram, PrintStream out) {

        out.print("valueIteratedTo,");
        out.print("countAtValueIteratedTo,");
        out.print("totalCountToThisValue,");
        out.print("totalValueToThisValue\n");

        AllValuesIterator percentileIterator = new AllValuesIterator(histogram);
        while (percentileIterator.hasNext()) {
            HistogramIterationValue next = percentileIterator.next();

            out.print(next.getValueIteratedTo());
            out.print(",");
            out.print(next.getCountAtValueIteratedTo());
            out.print(",");
            out.print(next.getTotalCountToThisValue());
            out.print(",");
            out.print(next.getTotalValueToThisValue());
            out.print(",\n");

        }

    }

    private void addPercentileData(EncodableHistogram accumulatedHistogram, ParserConfiguration config) {
        if (accumulatedHistogram instanceof DoubleHistogram) {
            DoublePercentileIterator percentileIterator = new DoublePercentileIterator(
                    (DoubleHistogram) accumulatedHistogram,
                    config.getPercentilesOutputTicksPerHalf());

            while (percentileIterator.hasNext()) {
                DoubleHistogramIterationValue iterationValue = percentileIterator.next();
                addPercentileDatapoint(
                        iterationValue.getPercentileLevelIteratedTo(),
                        iterationValue.getValueIteratedTo(),
                        config.getOutputValueUnitRatio());
            }
        } else {
            PercentileIterator percentileIterator = new PercentileIterator(
                    (Histogram) accumulatedHistogram,
                    config.getPercentilesOutputTicksPerHalf());

            while (percentileIterator.hasNext()) {
                HistogramIterationValue iterationValue = percentileIterator.next();
{'fraction_non_alphanumeric': 0.08604587904615106, 'fraction_numerical': 0.002538761447094025, 'mean_word_length': 2.175928591995393, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5451240', 'n_tokens_mistral': 2779, 'n_tokens_neox': 2589, 'n_words': 659}