<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2019, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import codecs
import csv
import logging
import os
import pickle
from io import BytesIO

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from ray.tune.logger import CSVLogger, JsonLogger, Logger
from ray.tune.result import TIME_TOTAL_S, TRAINING_ITERATION
from ray.tune.utils import flatten_dict

logger = logging.getLogger(__name__)


# Note:
# This file is introduced as a potential replacement for `ray_custom_loggers.py`.
# Therein, the code is written for tensorflow>=1.13.1; however, we've moved to using
# tensorflow 2.0 and up.
#


def record_tf_values(result, path, step, num_hist_bins=None):
    """
    Tensorboard will display under scalars, images, histograms, & distributions
    tabs.

    We manually generate the summary objects from raw data passed from tune
    via the logger.

    Currently supports:
        * Scalar (any prefix, already supported by TFLogger)
        * Seaborn plot ("seaborn_" prefix; see below for configuration details)
        * Image ("img_" prefix)
        * Histograms ("hist_" prefix)

    ```
    writer = tf.summary.create_file_writer("/some/path")
    result = {
        "my_scalor": 4,
        "hist_some_array": [1, 2, 3, 4],
        "seaborn_dict": dict(
            plot_type="lineplot",
            x=[1, 2, 3],
            y=[1, 2, 3],
        ),
        "img_some_array": np.random.rand(3, 3, 3, 3)
    }

    with writer.as_default():
        record_tf_values(result=result, path=["ray", "tune"], step=1)
    write.flush()
    """
    for attr, value in result.items():
        if value is not None:
            if attr.startswith("seaborn_"):

                # Value should be a dict which defines the plot to make. For example:
                #
                #   value = {
                #
                #      # Plot setup.
                #      plot_type: string,         # name of seaborn plotting function
                #      config: {...},             # (optional) passed to seaborn.set
                #      edit_axes_func: <callable> # (optional) edits axes e.g. set xlim
                #
                #      # Params -  to be passed to seaborn plotting method.
                #      data: pandas.DataFrame(...),
                #      x: <string>, # label of desired column of data
                #      y: <string>, # label of desired column of data , same size as x
                #      hue: <None or array like>  # same size as x and y
                #
                #   }
                #
                if not isinstance(value, dict):
                    continue

                # Get seaborn plot type and config.
                config = value.pop("config", {})
                plot_type = value.pop("plot_type", None)
                edit_axes_func = value.pop("edit_axes_func", lambda x: x)

                if not hasattr(sns, plot_type):
                    continue

                # Plot seaborn plot.
                plot_type = getattr(sns, plot_type)
                sns.set(**config)
                ax = plot_type(**value)
                edit_axes_func(ax)

                # Convert to figure to numpy array of an equivalent.
                # Save the plot to a PNG in memory.
                stream = BytesIO()
                plt.savefig(stream, format="png")
                stream.seek(0)

                # Convert PNG buffer to TF image
                image_tf = tf.image.decode_png(stream.getvalue(), channels=4)

                # Add the batch dimension
                image_tf = tf.expand_dims(image_tf, 0)

                # Save array as an image.
                name = "/".join(path + [attr])<commit_msg>Bug fix: Avoid `hasattr` check for when name is None.<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2019, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import codecs
import csv
import logging
import os
import pickle
from io import BytesIO

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from ray.tune.logger import CSVLogger, JsonLogger, Logger
from ray.tune.result import TIME_TOTAL_S, TRAINING_ITERATION
from ray.tune.utils import flatten_dict

logger = logging.getLogger(__name__)


# Note:
# This file is introduced as a potential replacement for `ray_custom_loggers.py`.
# Therein, the code is written for tensorflow>=1.13.1; however, we've moved to using
# tensorflow 2.0 and up.
#


def record_tf_values(result, path, step, num_hist_bins=None):
    """
    Tensorboard will display under scalars, images, histograms, & distributions
    tabs.

    We manually generate the summary objects from raw data passed from tune
    via the logger.

    Currently supports:
        * Scalar (any prefix, already supported by TFLogger)
        * Seaborn plot ("seaborn_" prefix; see below for configuration details)
        * Image ("img_" prefix)
        * Histograms ("hist_" prefix)

    ```
    writer = tf.summary.create_file_writer("/some/path")
    result = {
        "my_scalor": 4,
        "hist_some_array": [1, 2, 3, 4],
        "seaborn_dict": dict(
            plot_type="lineplot",
            x=[1, 2, 3],
            y=[1, 2, 3],
        ),
        "img_some_array": np.random.rand(3, 3, 3, 3)
    }

    with writer.as_default():
        record_tf_values(result=result, path=["ray", "tune"], step=1)
    write.flush()
    """
    for attr, value in result.items():
        if value is not None:
            if attr.startswith("seaborn_"):

                # Value should be a dict which defines the plot to make. For example:
                #
                #   value = {
                #
                #      # Plot setup.
                #      plot_type: string,         # name of seaborn plotting function
                #      config: {...},             # (optional) passed to seaborn.set
                #      edit_axes_func: <callable> # (optional) edits axes e.g. set xlim
                #
                #      # Params -  to be passed to seaborn plotting method.
                #      data: pandas.DataFrame(...),
                #      x: <string>, # label of desired column of data
                #      y: <string>, # label of desired column of data , same size as x
                #      hue: <None or array like>  # same size as x and y
                #
                #   }
                #
                if not isinstance(value, dict):
                    continue

                # Get seaborn plot type and config.
                config = value.pop("config", {})
                plot_type = value.pop("plot_type", None)
                edit_axes_func = value.pop("edit_axes_func", lambda x: x)

                if plot_type is None or not hasattr(sns, plot_type):
                    continue

                # Plot seaborn plot.
                plot_type = getattr(sns, plot_type)
                plt.clf()
                sns.set(**config)
                ax = plot_type(**value)
                edit_axes_func(ax)

                # Convert to figure to numpy array of an equivalent.
                # Save the plot to a PNG in memory.
                stream = BytesIO()
                plt.savefig(stream, format="png")
                stream.seek(0)

                # Convert PNG buffer to TF image
                image_tf = tf.image.decode_png(stream.getvalue(), channels=4)

                # Add the batch dimension
                image_tf = tf.expand_dims(image_tf, 0)

                # Save array as an image.
                name = "/".join(path + [attr])
{'fraction_non_alphanumeric': 0.11393746687864335, 'fraction_numerical': 0.006359300476947536, 'mean_word_length': 1.7438208781622564, 'pattern_counts': {'":': 8, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6648938', 'n_tokens_mistral': 2671, 'n_tokens_neox': 2470, 'n_words': 917}