<commit_before>package org.deeplearning4j.spark.stats;

import org.apache.commons.io.FilenameUtils;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.deeplearning4j.spark.api.stats.SparkTrainingStats;
import org.deeplearning4j.spark.util.SparkUtils;
import org.deeplearning4j.ui.api.Component;
import org.deeplearning4j.ui.api.LengthUnit;
import org.deeplearning4j.ui.components.chart.ChartHistogram;
import org.deeplearning4j.ui.components.chart.ChartLine;
import org.deeplearning4j.ui.components.chart.ChartTimeline;
import org.deeplearning4j.ui.components.chart.style.StyleChart;
import org.deeplearning4j.ui.components.component.ComponentDiv;
import org.deeplearning4j.ui.components.component.style.StyleDiv;
import org.deeplearning4j.ui.components.text.ComponentText;
import org.deeplearning4j.ui.components.text.style.StyleText;
import org.deeplearning4j.ui.standalone.StaticPageUtil;
import scala.Tuple3;

import java.awt.*;
import java.io.IOException;
import java.util.*;
import java.util.List;

/**
 * Utility methods for Spark training stats
 *
 * @author <NAME>
 */
public class StatsUtils {

    public static final long DEFAULT_MAX_TIMELINE_SIZE_MS = 20 * 60 * 1000;  //20 minutes

    public static void exportStats(List<EventStats> list, String outputDirectory, String filename, String delimiter, SparkContext sc) throws IOException {
        String path = FilenameUtils.concat(outputDirectory, filename);
        exportStats(list, path, delimiter, sc);
    }

    public static void exportStats(List<EventStats> list, String outputPath, String delimiter, SparkContext sc) throws IOException {
        StringBuilder sb = new StringBuilder();
        boolean first = true;
        for (EventStats e : list) {
            if (first) sb.append(e.getStringHeader(delimiter)).append("\n");
            sb.append(e.asString(delimiter)).append("\n");
            first = false;
        }
        SparkUtils.writeStringToFile(outputPath, sb.toString(), sc);
    }

    public static String getDurationAsString(List<EventStats> list, String delim) {
        StringBuilder sb = new StringBuilder();
        int num = list.size();
        int count = 0;
        for (EventStats e : list) {
            sb.append(e.getDurationMs());
            if (count++ < num - 1) sb.append(delim);
        }
        return sb.toString();
    }

    public static void exportStatsAsHtml(SparkTrainingStats sparkTrainingStats, String path, JavaSparkContext sc) throws Exception {
        exportStatsAsHtml(sparkTrainingStats, path, sc.sc());
    }

    /**
     * Generate and export a HTML representation (including charts, etc) of the Spark training statistics<br>
     * Note: exporting is done via Spark, so the path here can be a local file, HDFS, etc.
     *
     * @param sparkTrainingStats Stats to generate HTML page for
     * @param path               Path to export. May be local or HDFS
     * @param sc                 Spark context
     * @throws Exception IO errors or error generating HTML file
     */
    public static void exportStatsAsHtml(SparkTrainingStats sparkTrainingStats, String path, SparkContext sc) throws Exception {
        exportStatsAsHtml(sparkTrainingStats, DEFAULT_MAX_TIMELINE_SIZE_MS, path, sc);
    }

    /**
     * Generate and export a HTML representation (including charts, etc) of the Spark training statistics<br>
     * Note: exporting is done via Spark, so the path here can be a local file, HDFS, etc.
     *
     * @param sparkTrainingStats Stats to generate HTML page for
     * @param path               Path to export. May be local or HDFS
     * @param maxTimelineSizeMs  maximum amount of activity to show in a single timeline plot (multiple plots will be used if training exceeds this amount of time)
     * @param sc                 Spark context
     * @throws Exception IO errors or error generating HTML file
     */
    public static void exportStatsAsHtml(SparkTrainingStats sparkTrainingStats, long maxTimelineSizeMs, String path, SparkContext sc) throws Exception {
        Set<String> keySet = sparkTrainingStats.getKeySet();

        List<Component> components = new ArrayList<>();

        StyleChart styleChart = new StyleChart.Builder()
                .backgroundColor(Color.WHITE)
                .width(700, LengthUnit.Px)
                .height(400, LengthUnit.Px)
                .build();

        StyleText styleText = new StyleText.Builder()
                .color(Color.BLACK)
                .fontSize(20)
                .build();
        Component headerText = new ComponentText("Deeplearning4j - Spark Training Analysis", styleText);
        Component header = new ComponentDiv(new StyleDiv.Builder().height(40, LengthUnit.Px).width(100, LengthUnit.Percent).build(), headerText);
        components.add(header);

        Set<String> keySetInclude = new HashSet<>();
        for (String s : keySet) if (sparkTrainingStats.defaultIncludeInPlots(s)) keySetInclude.add(s);

        Collections.addAll(components, getTrainingStatsTimelineChart(sparkTrainingStats, keySetInclude, maxTimelineSizeMs));

        for (String s : keySet) {
            List<EventStats> list = new ArrayList<>(sparkTrainingStats.getValue(s));
            Collections.sort(list, new StartTimeComparator());

            double[] x = new double[list.size()];
            double[] duration = new double[list.size()];
            double minDur = Double.MAX_VALUE;
            double maxDur = -Double.MAX_VALUE;
            for (int i = 0; i < duration.length; i++) {
                x[i] = i;
                duration[i] = list.get(i).getDurationMs();
                minDur = Math.min(minDur, duration[i]);
                maxDur = Math.max(maxDur, duration[i]);
            }

            Component line = new ChartLine.Builder(s, styleChart)
                    .addSeries("Duration", x, duration)
                    .setYMin(minDur == maxDur ? minDur-1 : null)
                    .setYMax(minDur == maxDur ? minDur+1 : null)
                    .build();

            //Also build a histogram...
            Component hist = getHistogram(duration, 20, s, styleChart);

            Component[] temp;
            if (hist != null) {
                temp = new Component[]{line, hist};
            } else {
                temp = new Component[]{line};
            }

            components.add(new ComponentDiv(new StyleDiv.Builder().width(100, LengthUnit.Percent).build(), temp));


            //TODO this is really ugly
            if(list.size() > 0 && (list.get(0) instanceof ExampleCountEventStats || list.get(0) instanceof PartitionCountEventStats)){
                boolean exCount = list.get(0) instanceof ExampleCountEventStats;

                double[] y = new double[list.size()];
                double miny = Double.MAX_VALUE;
                double maxy = -Double.MAX_VALUE;
                for( int i=0; i<y.length; i++ ){
                    y[i] = (exCount ? ((ExampleCountEventStats)list.get(i)).getTotalExampleCount() : ((PartitionCountEventStats)list.get(i)).getNumPartitions());
                    miny = Math.min(miny, y[i]);
                    maxy = Math.max(maxy, y[i]);
                }

                String title = s + " / " + (exCount ? "Number of Examples" : "Number of Partitions");
                Component line2 = new ChartLine.Builder(title, styleChart)
                        .addSeries((exCount ? "Examples" : "Partitions"), x, y)
                        .setYMin(miny == maxy ? miny-1 : null)
                        .setYMax(miny == maxy ? miny+1 : null)
                        .build();


                //Also build a histogram...
                Component hist2 = getHistogram(y, 20, title, styleChart);

                Component[] temp2;
                if (hist2 != null) {
                    temp2 = new Component[]{line2, hist2};
                } else {<commit_msg>Spark histograms: only generate if min != max<commit_after>package org.deeplearning4j.spark.stats;

import org.apache.commons.io.FilenameUtils;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.deeplearning4j.spark.api.stats.SparkTrainingStats;
import org.deeplearning4j.spark.util.SparkUtils;
import org.deeplearning4j.ui.api.Component;
import org.deeplearning4j.ui.api.LengthUnit;
import org.deeplearning4j.ui.components.chart.ChartHistogram;
import org.deeplearning4j.ui.components.chart.ChartLine;
import org.deeplearning4j.ui.components.chart.ChartTimeline;
import org.deeplearning4j.ui.components.chart.style.StyleChart;
import org.deeplearning4j.ui.components.component.ComponentDiv;
import org.deeplearning4j.ui.components.component.style.StyleDiv;
import org.deeplearning4j.ui.components.text.ComponentText;
import org.deeplearning4j.ui.components.text.style.StyleText;
import org.deeplearning4j.ui.standalone.StaticPageUtil;
import scala.Tuple3;

import java.awt.*;
import java.io.IOException;
import java.util.*;
import java.util.List;

/**
 * Utility methods for Spark training stats
 *
 * @author <NAME>
 */
public class StatsUtils {

    public static final long DEFAULT_MAX_TIMELINE_SIZE_MS = 20 * 60 * 1000;  //20 minutes

    public static void exportStats(List<EventStats> list, String outputDirectory, String filename, String delimiter, SparkContext sc) throws IOException {
        String path = FilenameUtils.concat(outputDirectory, filename);
        exportStats(list, path, delimiter, sc);
    }

    public static void exportStats(List<EventStats> list, String outputPath, String delimiter, SparkContext sc) throws IOException {
        StringBuilder sb = new StringBuilder();
        boolean first = true;
        for (EventStats e : list) {
            if (first) sb.append(e.getStringHeader(delimiter)).append("\n");
            sb.append(e.asString(delimiter)).append("\n");
            first = false;
        }
        SparkUtils.writeStringToFile(outputPath, sb.toString(), sc);
    }

    public static String getDurationAsString(List<EventStats> list, String delim) {
        StringBuilder sb = new StringBuilder();
        int num = list.size();
        int count = 0;
        for (EventStats e : list) {
            sb.append(e.getDurationMs());
            if (count++ < num - 1) sb.append(delim);
        }
        return sb.toString();
    }

    public static void exportStatsAsHtml(SparkTrainingStats sparkTrainingStats, String path, JavaSparkContext sc) throws Exception {
        exportStatsAsHtml(sparkTrainingStats, path, sc.sc());
    }

    /**
     * Generate and export a HTML representation (including charts, etc) of the Spark training statistics<br>
     * Note: exporting is done via Spark, so the path here can be a local file, HDFS, etc.
     *
     * @param sparkTrainingStats Stats to generate HTML page for
     * @param path               Path to export. May be local or HDFS
     * @param sc                 Spark context
     * @throws Exception IO errors or error generating HTML file
     */
    public static void exportStatsAsHtml(SparkTrainingStats sparkTrainingStats, String path, SparkContext sc) throws Exception {
        exportStatsAsHtml(sparkTrainingStats, DEFAULT_MAX_TIMELINE_SIZE_MS, path, sc);
    }

    /**
     * Generate and export a HTML representation (including charts, etc) of the Spark training statistics<br>
     * Note: exporting is done via Spark, so the path here can be a local file, HDFS, etc.
     *
     * @param sparkTrainingStats Stats to generate HTML page for
     * @param path               Path to export. May be local or HDFS
     * @param maxTimelineSizeMs  maximum amount of activity to show in a single timeline plot (multiple plots will be used if training exceeds this amount of time)
     * @param sc                 Spark context
     * @throws Exception IO errors or error generating HTML file
     */
    public static void exportStatsAsHtml(SparkTrainingStats sparkTrainingStats, long maxTimelineSizeMs, String path, SparkContext sc) throws Exception {
        Set<String> keySet = sparkTrainingStats.getKeySet();

        List<Component> components = new ArrayList<>();

        StyleChart styleChart = new StyleChart.Builder()
                .backgroundColor(Color.WHITE)
                .width(700, LengthUnit.Px)
                .height(400, LengthUnit.Px)
                .build();

        StyleText styleText = new StyleText.Builder()
                .color(Color.BLACK)
                .fontSize(20)
                .build();
        Component headerText = new ComponentText("Deeplearning4j - Spark Training Analysis", styleText);
        Component header = new ComponentDiv(new StyleDiv.Builder().height(40, LengthUnit.Px).width(100, LengthUnit.Percent).build(), headerText);
        components.add(header);

        Set<String> keySetInclude = new HashSet<>();
        for (String s : keySet) if (sparkTrainingStats.defaultIncludeInPlots(s)) keySetInclude.add(s);

        Collections.addAll(components, getTrainingStatsTimelineChart(sparkTrainingStats, keySetInclude, maxTimelineSizeMs));

        for (String s : keySet) {
            List<EventStats> list = new ArrayList<>(sparkTrainingStats.getValue(s));
            Collections.sort(list, new StartTimeComparator());

            double[] x = new double[list.size()];
            double[] duration = new double[list.size()];
            double minDur = Double.MAX_VALUE;
            double maxDur = -Double.MAX_VALUE;
            for (int i = 0; i < duration.length; i++) {
                x[i] = i;
                duration[i] = list.get(i).getDurationMs();
                minDur = Math.min(minDur, duration[i]);
                maxDur = Math.max(maxDur, duration[i]);
            }

            Component line = new ChartLine.Builder(s, styleChart)
                    .addSeries("Duration", x, duration)
                    .setYMin(minDur == maxDur ? minDur-1 : null)
                    .setYMax(minDur == maxDur ? minDur+1 : null)
                    .build();

            //Also build a histogram...
            Component hist = null;
            if(minDur != maxDur && list.size() > 0) hist = getHistogram(duration, 20, s, styleChart);

            Component[] temp;
            if (hist != null) {
                temp = new Component[]{line, hist};
            } else {
                temp = new Component[]{line};
            }

            components.add(new ComponentDiv(new StyleDiv.Builder().width(100, LengthUnit.Percent).build(), temp));


            //TODO this is really ugly
            if(list.size() > 0 && (list.get(0) instanceof ExampleCountEventStats || list.get(0) instanceof PartitionCountEventStats)){
                boolean exCount = list.get(0) instanceof ExampleCountEventStats;

                double[] y = new double[list.size()];
                double miny = Double.MAX_VALUE;
                double maxy = -Double.MAX_VALUE;
                for( int i=0; i<y.length; i++ ){
                    y[i] = (exCount ? ((ExampleCountEventStats)list.get(i)).getTotalExampleCount() : ((PartitionCountEventStats)list.get(i)).getNumPartitions());
                    miny = Math.min(miny, y[i]);
                    maxy = Math.max(maxy, y[i]);
                }

                String title = s + " / " + (exCount ? "Number of Examples" : "Number of Partitions");
                Component line2 = new ChartLine.Builder(title, styleChart)
                        .addSeries((exCount ? "Examples" : "Partitions"), x, y)
                        .setYMin(miny == maxy ? miny-1 : null)
                        .setYMax(miny == maxy ? miny+1 : null)
                        .build();


                //Also build a histogram...
                Component hist2 = null;
                if(miny != maxy) hist2 = getHistogram(y, 20, title, styleChart);

                Component[] temp2;
                if (hist2 != null) {
                    temp2 = new Component[]{line2, hist2};
                } else {
{'fraction_non_alphanumeric': 0.10330371668126642, 'fraction_numerical': 0.008259291703166062, 'mean_word_length': 2.440904198062433, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6005229', 'n_tokens_mistral': 4512, 'n_tokens_neox': 4207, 'n_words': 1168}