<commit_before>package de.qaware.chronix.client.benchmark.resultpresenter.plot;

import de.qaware.chronix.client.benchmark.BenchmarkImport;
import de.qaware.chronix.client.benchmark.benchmarkrunner.BenchmarkRunner;
import de.qaware.chronix.client.benchmark.resultpresenter.*;
import de.qaware.chronix.database.BenchmarkDataSource;
import org.jfree.chart.ChartFactory;
import org.jfree.chart.ChartUtilities;
import org.jfree.chart.JFreeChart;
import org.jfree.chart.plot.PlotOrientation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.jfree.data.category.DefaultCategoryDataset;


import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

/**
 * Created by mcqueen666 on 17.10.16.
 */
public class StatisticsBarPlotter {

    private final Logger logger;
    private final int CHART_WIDTH = 1920;
    private final int CHART_HEIGHT = 1080;
    private final String statisticsBarPlotDirectory;
    private final List<TsdbStatistics> tsdbStatisticsList;

    public StatisticsBarPlotter(String statisticsDirectory){
        logger = LoggerFactory.getLogger(StatisticsBarPlotter.class);
        this.statisticsBarPlotDirectory = statisticsDirectory + File.separator + "bar_plots";
        this.tsdbStatisticsList = new TsdbStatisticsAnalyzer(statisticsDirectory).readTsdbStatistics();

        File dir = new File(statisticsBarPlotDirectory);
        if(!dir.exists()){
            dir.mkdirs();
        }
    }

    /**
     * Generates bar plots for calculated metrics and saves them to chronixBenchmark/statistics/bar_plots/
     *
     * @param includeQueryFunctions the QueryFunctions for which a par plot should be generated.
     */
    public void plotTsdbStatisticsForQueryFunctions(List<String> includeQueryFunctions){
        if(includeQueryFunctions != null && !includeQueryFunctions.isEmpty()) {
            logger.info("Plotting for " + String.join(",", includeQueryFunctions) + " ...");
            Map<String, List<PlotData>> plotDataPerMeasurement = new HashMap<>();
            // query time
            plotDataPerMeasurement.put("mean query time", new LinkedList<>());
            plotDataPerMeasurement.put("median query time", new LinkedList<>());
            plotDataPerMeasurement.put("total query time", new LinkedList<>());

            // cpu usage
            plotDataPerMeasurement.put("mean total cpu usage", new LinkedList<>());
            plotDataPerMeasurement.put("median total cpu usage", new LinkedList<>());
            plotDataPerMeasurement.put("maximum recorded cpu usage", new LinkedList<>());

            // disk usage
            plotDataPerMeasurement.put("mean disk usage", new LinkedList<>());
            plotDataPerMeasurement.put("median disk usage", new LinkedList<>());
            plotDataPerMeasurement.put("maximum recorded disk usage", new LinkedList<>());

            // memory usage
            plotDataPerMeasurement.put("mean total memory usage", new LinkedList<>());
            plotDataPerMeasurement.put("median total memory usage", new LinkedList<>());
            plotDataPerMeasurement.put("maximum recorded memory usage", new LinkedList<>());

            // disk write
            plotDataPerMeasurement.put("mean disk write", new LinkedList<>());
            plotDataPerMeasurement.put("median disk write", new LinkedList<>());
            plotDataPerMeasurement.put("total disk write", new LinkedList<>());

            // disk read
            plotDataPerMeasurement.put("mean disk read", new LinkedList<>());
            plotDataPerMeasurement.put("median disk read", new LinkedList<>());
            plotDataPerMeasurement.put("total disk read", new LinkedList<>());

            // network download
            plotDataPerMeasurement.put("mean network download", new LinkedList<>());
            plotDataPerMeasurement.put("median network download", new LinkedList<>());
            plotDataPerMeasurement.put("total total network download", new LinkedList<>());

            // network upload
            plotDataPerMeasurement.put("mean network upload", new LinkedList<>());
            plotDataPerMeasurement.put("median network upload", new LinkedList<>());
            plotDataPerMeasurement.put("total total network upload", new LinkedList<>());

            // letency
            plotDataPerMeasurement.put("mean latency", new LinkedList<>());
            plotDataPerMeasurement.put("median latency", new LinkedList<>());

            // build up data for plotting
            for (TsdbStatistics tsdbStatistics : tsdbStatisticsList) {
                for (QueryFunctionStatistics queryFunctionStatistics : tsdbStatistics.getQueryFunctionStatisticsList()) {
                    if (includeQueryFunctions.contains(queryFunctionStatistics.getQueryFunction())) {
                        // query time
                        plotDataPerMeasurement.get("mean query time").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanQueryTime_inMilliseconds(), "ms"));
                        plotDataPerMeasurement.get("median query time").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianQueryTime_inMilliseconds(), "ms"));
                        plotDataPerMeasurement.get("total query time").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getTotalQueryTimePerQueryFunction_inMilliseconds() / (1000), "s"));

                        // cpu usage
                        plotDataPerMeasurement.get("mean total cpu usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanTotalCpuUsagePerQuery_inPercent(), "%"));
                        plotDataPerMeasurement.get("median total cpu usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianTotalCpuUsagePerQuery_inPercent(), "%"));
                        plotDataPerMeasurement.get("maximum recorded cpu usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMaximumCpuUsageRecorded_inPercent(), "%"));

                        // disk usage
                        plotDataPerMeasurement.get("mean disk usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanDiskUsagePerQuery_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("median disk usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianDiskUsagePerQuery_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("maximum recorded disk usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMaximumDiskUsageRecorded_inBytes() / (1024*1024), "MiB"));

                        // memory usage
                        plotDataPerMeasurement.get("mean total memory usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanTotalMemoryUsage_inPercent(), "%"));
                        plotDataPerMeasurement.get("median total memory usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianTotalMemoryUsage_inPercent(), "%"));
                        plotDataPerMeasurement.get("maximum recorded memory usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMaximumMemoryUsageRecorded_inPercent(), "%"));

                        // disk write
                        plotDataPerMeasurement.get("mean disk write").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanDiskWrite_inBytes() , "Byte"));
                        plotDataPerMeasurement.get("median disk write").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianDiskWrite_inBytes() , "Byte"));
                        plotDataPerMeasurement.get("total disk write").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getTotalDiskWrite_inBytes() , "Byte"));

                        // disk write
                        plotDataPerMeasurement.get("mean disk read").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanDiskRead_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("median disk read").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianDiskRead_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("total disk read").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getTotalDiskRead_inBytes() , "Byte"));

                        // network download
                        plotDataPerMeasurement.get("mean network download").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanNetworkDownload_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("median network download").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianNetworkDownload_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("total total network download").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getTotalNetworkDownload_inBytes(), "Byte"));

                        // network upload
                        plotDataPerMeasurement.get("mean network upload").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanNetworkUpload_inBytes() , "Byte"));
                        plotDataPerMeasurement.get("median network upload").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianNetworkUpload_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("total total network upload").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getTotalNetworkUpload_inBytes(), "Byte"));

                        // latency
                        plotDataPerMeasurement.get("mean latency").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanLatency_inMilliseconds(), "ms"));
                        plotDataPerMeasurement.get("median latency").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianLatency_inMilliseconds(), "ms"));

                    }
                }
            }

            //plot the data per measurement
            BenchmarkRunner benchmarkRunner = BenchmarkRunner.getInstance();
            int normalTsQuerySize = benchmarkRunner.getBENCHMARK_TIMESERIES_METADATA_SIZE();
            int rangeQueryTsSize = benchmarkRunner.getBENCHMARK_TIMESERIES_METADATA_SIZE_QUERY_ONLY();<commit_msg>changed units for better visibility<commit_after>package de.qaware.chronix.client.benchmark.resultpresenter.plot;

import de.qaware.chronix.client.benchmark.BenchmarkImport;
import de.qaware.chronix.client.benchmark.benchmarkrunner.BenchmarkRunner;
import de.qaware.chronix.client.benchmark.resultpresenter.*;
import de.qaware.chronix.database.BenchmarkDataSource;
import org.jfree.chart.ChartFactory;
import org.jfree.chart.ChartUtilities;
import org.jfree.chart.JFreeChart;
import org.jfree.chart.plot.PlotOrientation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.jfree.data.category.DefaultCategoryDataset;


import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

/**
 * Created by mcqueen666 on 17.10.16.
 */
public class StatisticsBarPlotter {

    private final Logger logger;
    private final int CHART_WIDTH = 1920;
    private final int CHART_HEIGHT = 1080;
    private final String statisticsBarPlotDirectory;
    private final List<TsdbStatistics> tsdbStatisticsList;

    public StatisticsBarPlotter(String statisticsDirectory){
        logger = LoggerFactory.getLogger(StatisticsBarPlotter.class);
        this.statisticsBarPlotDirectory = statisticsDirectory + File.separator + "bar_plots";
        this.tsdbStatisticsList = new TsdbStatisticsAnalyzer(statisticsDirectory).readTsdbStatistics();

        File dir = new File(statisticsBarPlotDirectory);
        if(!dir.exists()){
            dir.mkdirs();
        }
    }

    /**
     * Generates bar plots for calculated metrics and saves them to chronixBenchmark/statistics/bar_plots/
     *
     * @param includeQueryFunctions the QueryFunctions for which a par plot should be generated.
     */
    public void plotTsdbStatisticsForQueryFunctions(List<String> includeQueryFunctions){
        if(includeQueryFunctions != null && !includeQueryFunctions.isEmpty()) {
            logger.info("Plotting for " + String.join(",", includeQueryFunctions) + " ...");
            Map<String, List<PlotData>> plotDataPerMeasurement = new HashMap<>();
            // query time
            plotDataPerMeasurement.put("mean query time", new LinkedList<>());
            plotDataPerMeasurement.put("median query time", new LinkedList<>());
            plotDataPerMeasurement.put("total query time", new LinkedList<>());

            // cpu usage
            plotDataPerMeasurement.put("mean total cpu usage", new LinkedList<>());
            plotDataPerMeasurement.put("median total cpu usage", new LinkedList<>());
            plotDataPerMeasurement.put("maximum recorded cpu usage", new LinkedList<>());

            // disk usage
            plotDataPerMeasurement.put("mean disk usage", new LinkedList<>());
            plotDataPerMeasurement.put("median disk usage", new LinkedList<>());
            plotDataPerMeasurement.put("maximum recorded disk usage", new LinkedList<>());

            // memory usage
            plotDataPerMeasurement.put("mean total memory usage", new LinkedList<>());
            plotDataPerMeasurement.put("median total memory usage", new LinkedList<>());
            plotDataPerMeasurement.put("maximum recorded memory usage", new LinkedList<>());

            // disk write
            plotDataPerMeasurement.put("mean disk write", new LinkedList<>());
            plotDataPerMeasurement.put("median disk write", new LinkedList<>());
            plotDataPerMeasurement.put("total disk write", new LinkedList<>());

            // disk read
            plotDataPerMeasurement.put("mean disk read", new LinkedList<>());
            plotDataPerMeasurement.put("median disk read", new LinkedList<>());
            plotDataPerMeasurement.put("total disk read", new LinkedList<>());

            // network download
            plotDataPerMeasurement.put("mean network download", new LinkedList<>());
            plotDataPerMeasurement.put("median network download", new LinkedList<>());
            plotDataPerMeasurement.put("total total network download", new LinkedList<>());

            // network upload
            plotDataPerMeasurement.put("mean network upload", new LinkedList<>());
            plotDataPerMeasurement.put("median network upload", new LinkedList<>());
            plotDataPerMeasurement.put("total total network upload", new LinkedList<>());

            // letency
            plotDataPerMeasurement.put("mean latency", new LinkedList<>());
            plotDataPerMeasurement.put("median latency", new LinkedList<>());

            // build up data for plotting
            for (TsdbStatistics tsdbStatistics : tsdbStatisticsList) {
                for (QueryFunctionStatistics queryFunctionStatistics : tsdbStatistics.getQueryFunctionStatisticsList()) {
                    if (includeQueryFunctions.contains(queryFunctionStatistics.getQueryFunction())) {
                        // query time
                        plotDataPerMeasurement.get("mean query time").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), (double)(queryFunctionStatistics.getMeanQueryTime_inMilliseconds()) / 1000.0, "s"));
                        plotDataPerMeasurement.get("median query time").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), (double)(queryFunctionStatistics.getMedianQueryTime_inMilliseconds()) / 1000.0, "s"));
                        plotDataPerMeasurement.get("total query time").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), (double)(queryFunctionStatistics.getTotalQueryTimePerQueryFunction_inMilliseconds()) / 1000.0, "s"));

                        // cpu usage
                        plotDataPerMeasurement.get("mean total cpu usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanTotalCpuUsagePerQuery_inPercent(), "%"));
                        plotDataPerMeasurement.get("median total cpu usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianTotalCpuUsagePerQuery_inPercent(), "%"));
                        plotDataPerMeasurement.get("maximum recorded cpu usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMaximumCpuUsageRecorded_inPercent(), "%"));

                        // disk usage
                        plotDataPerMeasurement.get("mean disk usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanDiskUsagePerQuery_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("median disk usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianDiskUsagePerQuery_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("maximum recorded disk usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), (double)(queryFunctionStatistics.getMaximumDiskUsageRecorded_inBytes()) / (1024.0 * 1024.0 * 1024.0), "GiB"));

                        // memory usage
                        plotDataPerMeasurement.get("mean total memory usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanTotalMemoryUsage_inPercent(), "%"));
                        plotDataPerMeasurement.get("median total memory usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianTotalMemoryUsage_inPercent(), "%"));
                        plotDataPerMeasurement.get("maximum recorded memory usage").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMaximumMemoryUsageRecorded_inPercent(), "%"));

                        // disk write
                        plotDataPerMeasurement.get("mean disk write").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanDiskWrite_inBytes() , "Byte"));
                        plotDataPerMeasurement.get("median disk write").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianDiskWrite_inBytes() , "Byte"));
                        plotDataPerMeasurement.get("total disk write").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getTotalDiskWrite_inBytes() , "Byte"));

                        // disk write
                        plotDataPerMeasurement.get("mean disk read").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanDiskRead_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("median disk read").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianDiskRead_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("total disk read").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getTotalDiskRead_inBytes() , "Byte"));

                        // network download
                        plotDataPerMeasurement.get("mean network download").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanNetworkDownload_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("median network download").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianNetworkDownload_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("total total network download").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), (double)(queryFunctionStatistics.getTotalNetworkDownload_inBytes()) / (1024.0 * 1024.0 * 1024.0), "GiB"));

                        // network upload
                        plotDataPerMeasurement.get("mean network upload").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMeanNetworkUpload_inBytes() , "Byte"));
                        plotDataPerMeasurement.get("median network upload").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), queryFunctionStatistics.getMedianNetworkUpload_inBytes(), "Byte"));
                        plotDataPerMeasurement.get("total total network upload").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), (double)(queryFunctionStatistics.getTotalNetworkUpload_inBytes()) / (1024.0 * 1024.0 * 1024.0), "GiB"));

                        // latency
                        plotDataPerMeasurement.get("mean latency").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), (double)(queryFunctionStatistics.getMeanLatency_inMilliseconds()) / 1000.0, "s"));
                        plotDataPerMeasurement.get("median latency").add(new PlotData(tsdbStatistics.getTsdbName(), queryFunctionStatistics.getQueryFunction(), (double)(queryFunctionStatistics.getMedianLatency_inMilliseconds()) / 1000.0, "s"));

                    }
                }
            }

            //plot the data per measurement
            BenchmarkRunner benchmarkRunner = BenchmarkRunner.getInstance();
            int normalTsQuerySize = benchmarkRunner.getBENCHMARK_TIMESERIES_METADATA_SIZE();
            int rangeQueryTsSize = benchmarkRunner.getBENCHMARK_TIMESERIES_METADATA_SIZE_QUERY_ONLY();
{'fraction_non_alphanumeric': 0.10664120517716953, 'fraction_numerical': 0.005092297899427116, 'mean_word_length': 3.7483376989723958, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1095944', 'n_tokens_mistral': 5962, 'n_tokens_neox': 5561, 'n_words': 1027}