<commit_before>"""Plots many dataset examples (storm objects)."""

import os.path
import argparse
import numpy
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as pyplot
from gewittergefahr.gg_io import storm_tracking_io as tracking_io
from gewittergefahr.gg_utils import radar_utils
from gewittergefahr.gg_utils import soundings
from gewittergefahr.gg_utils import time_conversion
from gewittergefahr.gg_utils import file_system_utils
from gewittergefahr.gg_utils import error_checking
from gewittergefahr.deep_learning import cnn
from gewittergefahr.deep_learning import testing_io
from gewittergefahr.deep_learning import input_examples
from gewittergefahr.deep_learning import model_activation
from gewittergefahr.deep_learning import deep_learning_utils as dl_utils
from gewittergefahr.deep_learning import training_validation_io as trainval_io
from gewittergefahr.plotting import plotting_utils
from gewittergefahr.plotting import radar_plotting
from gewittergefahr.plotting import sounding_plotting

# TODO(thunderhoser): This is a HACK.
DUMMY_TARGET_NAME = 'tornado_lead-time=0000-3600sec_distance=00000-10000m'

TIME_FORMAT = '%Y-%m-%d-%H%M%S'
SEPARATOR_STRING = '\n\n' + '*' * 50 + '\n\n'

SOUNDING_FIELD_NAMES = [
    soundings.U_WIND_NAME, soundings.V_WIND_NAME,
    soundings.TEMPERATURE_NAME, soundings.SPECIFIC_HUMIDITY_NAME,
    soundings.PRESSURE_NAME
]
SOUNDING_HEIGHTS_M_AGL = soundings.DEFAULT_HEIGHT_LEVELS_M_AGL

SOUNDING_FIGURE_KEY = 'sounding_figure_object'
SOUNDING_AXES_KEY = 'sounding_axes_object'
RADAR_FIGURES_KEY = 'radar_figure_objects'
RADAR_AXES_KEY = 'radar_axes_object_matrices'

IS_SOUNDING_KEY = 'is_sounding'
PMM_FLAG_KEY = 'pmm_flag'
FULL_STORM_ID_KEY = 'full_storm_id_string'
STORM_TIME_KEY = 'storm_time_unix_sec'
RADAR_FIELD_KEY = 'radar_field_name'
RADAR_HEIGHT_KEY = 'radar_height_m_agl'
LAYER_OPERATION_KEY = 'layer_operation_dict'

TITLE_FONT_SIZE = 16
FONT_SIZE_WITH_COLOUR_BARS = 16
FONT_SIZE_SANS_COLOUR_BARS = 20
FIGURE_WIDTH_INCHES = 15
FIGURE_HEIGHT_INCHES = 15
FIGURE_RESOLUTION_DPI = 300

ACTIVATION_FILE_ARG_NAME = 'input_activation_file_name'
STORM_METAFILE_ARG_NAME = 'input_storm_metafile_name'
NUM_EXAMPLES_ARG_NAME = 'num_examples'
ALLOW_WHITESPACE_ARG_NAME = 'allow_whitespace'
EXAMPLE_DIR_ARG_NAME = 'input_example_dir_name'
RADAR_FIELDS_ARG_NAME = 'radar_field_names'
RADAR_HEIGHTS_ARG_NAME = 'radar_heights_m_agl'
PLOT_SOUNDINGS_ARG_NAME = 'plot_soundings'
NUM_ROWS_ARG_NAME = 'num_radar_rows'
NUM_COLUMNS_ARG_NAME = 'num_radar_columns'
OUTPUT_DIR_ARG_NAME = 'output_dir_name'

ACTIVATION_FILE_HELP_STRING = (
    'Path to activation file (will be read by `model_activation.read_file`).  '
    'If this argument is empty, will use `{0:s}`.'
).format(STORM_METAFILE_ARG_NAME)

STORM_METAFILE_HELP_STRING = (
    'Path to Pickle file with storm IDs and times (will be read by '
    '`storm_tracking_io.read_ids_and_times`).  If this argument is empty, will '
    'use `{0:s}`.'
).format(ACTIVATION_FILE_ARG_NAME)

NUM_EXAMPLES_HELP_STRING = (
    'Number of examples (storm objects) to read from `{0:s}` or `{1:s}`.  If '
    'you want to read all examples, make this non-positive.'
).format(ACTIVATION_FILE_ARG_NAME, STORM_METAFILE_ARG_NAME)

ALLOW_WHITESPACE_HELP_STRING = (
    'Boolean flag.  If 0, will plot with no whitespace between panels or around'
    ' outside of image.')

EXAMPLE_DIR_HELP_STRING = (
    'Name of top-level directory with input examples.  Files therein will be '
    'found by `input_examples.find_example_file` and read by '
    '`input_examples.read_example_file`.')

RADAR_FIELDS_HELP_STRING = (
    '[used only if `{0:s}` is empty] List of radar fields (used as input to '
    '`input_examples.read_example_file`).  If you want to plot all radar '
    'fields, leave this argument empty.'
).format(ACTIVATION_FILE_ARG_NAME)

RADAR_HEIGHTS_HELP_STRING = (
    '[used only if `{0:s}` is empty] List of radar heights (used as input to '
    '`input_examples.read_example_file`).  If you want to plot all radar '
    'heights, leave this argument empty.'
).format(ACTIVATION_FILE_ARG_NAME)

PLOT_SOUNDINGS_HELP_STRING = (
    'Boolean flag.  If 1, will plot sounding for each example.  If 0, will not '
    'plot soundings.')

NUM_ROWS_HELP_STRING = (
    '[used only if `{0:s}` is empty] Number of rows in each storm-centered '
    'radar grid.  If you want to plot the largest grids available, leave this '
    'argument empty.'
).format(ACTIVATION_FILE_ARG_NAME)

NUM_COLUMNS_HELP_STRING = (
    '[used only if `{0:s}` is empty] Number of columns in each storm-centered '
    'radar grid.  If you want to plot the largest grids available, leave this '
    'argument empty.'
).format(ACTIVATION_FILE_ARG_NAME)

OUTPUT_DIR_HELP_STRING = (
    'Name of output directory (figures will be saved here).')

INPUT_ARG_PARSER = argparse.ArgumentParser()
INPUT_ARG_PARSER.add_argument(
    '--' + ACTIVATION_FILE_ARG_NAME, type=str, required=False, default='',
    help=ACTIVATION_FILE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + STORM_METAFILE_ARG_NAME, type=str, required=False, default='',
    help=STORM_METAFILE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + NUM_EXAMPLES_ARG_NAME, type=int, required=False, default=-1,
    help=NUM_EXAMPLES_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + ALLOW_WHITESPACE_ARG_NAME, type=int, required=False, default=1,
    help=ALLOW_WHITESPACE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + EXAMPLE_DIR_ARG_NAME, type=str, required=True,
    help=EXAMPLE_DIR_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + RADAR_FIELDS_ARG_NAME, type=str, nargs='+', required=False,
    default=[''], help=RADAR_FIELDS_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + RADAR_HEIGHTS_ARG_NAME, type=int, nargs='+', required=False,
    default=[-1], help=RADAR_HEIGHTS_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + PLOT_SOUNDINGS_ARG_NAME, type=int, required=False, default=1,
    help=PLOT_SOUNDINGS_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + NUM_ROWS_ARG_NAME, type=int, required=False, default=-1,
    help=NUM_ROWS_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + NUM_COLUMNS_ARG_NAME, type=int, required=False, default=-1,
    help=NUM_COLUMNS_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + OUTPUT_DIR_ARG_NAME, type=str, required=True,
    help=OUTPUT_DIR_HELP_STRING)


def _plot_sounding(
        list_of_predictor_matrices, model_metadata_dict, allow_whitespace,
        title_string=None):
    """Plots sounding for one example.

    :param list_of_predictor_matrices: See doc for `_plot_3d_radar_scan`.
    :param model_metadata_dict: Same.
    :param allow_whitespace: Same.
    :param title_string: Same.
    :return: figure_object: Figure handle (instance of
        `matplotlib.figure.Figure`).
    :return: axes_object: Axes handle (instance of
        `matplotlib.axes._subplots.AxesSubplot`).
    """

    training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
    sounding_field_names = training_option_dict[trainval_io.SOUNDING_FIELDS_KEY]
    sounding_heights_m_agl = training_option_dict[
        trainval_io.SOUNDING_HEIGHTS_KEY]

    sounding_matrix = numpy.expand_dims(list_of_predictor_matrices[-1], axis=0)

    # TODO(thunderhoser): Allow actual storm elevation to be used.
    metpy_dict = dl_utils.soundings_to_metpy_dictionaries(
        sounding_matrix=sounding_matrix, field_names=sounding_field_names,
        height_levels_m_agl=sounding_heights_m_agl,
        storm_elevations_m_asl=numpy.array([0.])
    )[0]

    return sounding_plotting.plot_sounding(
        sounding_dict_for_metpy=metpy_dict,
        title_string=title_string if allow_whitespace else None
    )


def _plot_3d_radar_scan(
        list_of_predictor_matrices, model_metadata_dict, allow_whitespace,
        title_string=None):
    """Plots 3-D radar scan for one example.

    J = number of panel rows in image
    K = number of panel columns in image
    F = number of radar fields

    :param list_of_predictor_matrices: List created by
        `testing_io.read_specific_examples`, except that the first axis (example
        dimension) is removed.
    :param model_metadata_dict: Dictionary returned by
        `cnn.read_model_metadata`.
    :param allow_whitespace: See documentation at top of file.
    :param title_string: Title (may be None).

    :return: figure_objects: length-F list of figure handles (instances of
        `matplotlib.figure.Figure`).
    :return: axes_object_matrices: length-F list.  Each element is a J-by-K
        numpy array of axes handles (instances of
        `matplotlib.axes._subplots.AxesSubplot`).
    """

    training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
    radar_field_names = training_option_dict[trainval_io.RADAR_FIELDS_KEY]
    radar_heights_m_agl = training_option_dict[trainval_io.RADAR_HEIGHTS_KEY]

    colour_bar_names = radar_plotting.radar_fields_and_heights_to_panel_names(
        field_names=radar_field_names, heights_m_agl=radar_heights_m_agl,
        include_units=True)
    colour_bar_names = [n.replace('\n', '') for n in colour_bar_names]

    num_radar_fields = len(radar_field_names)
    num_radar_heights = len(radar_heights_m_agl)

    num_panel_rows = int(numpy.floor(
        numpy.sqrt(num_radar_heights)
    ))
    num_panel_columns = int(numpy.ceil(
        float(num_radar_heights) / num_panel_rows
    ))

    figure_objects = [None] * num_radar_fields
    axes_object_matrices = [None] * num_radar_fields
    radar_matrix = list_of_predictor_matrices[0]

    for j in range(num_radar_fields):
        this_radar_matrix = numpy.flip(radar_matrix[..., j], axis=0)

        if not allow_whitespace:
            figure_objects[j], axes_object_matrices[j] = (
                plotting_utils.create_paneled_figure(
                    num_rows=num_panel_rows, num_columns=num_panel_columns,
                    horizontal_spacing=0., vertical_spacing=0.,
                    shared_x_axis=False, shared_y_axis=False,
                    keep_aspect_ratio=True)
            )

        figure_objects[j], axes_object_matrices[j] = (<commit_msg>Screwing with colour bars in plot_input_examples.py.<commit_after>"""Plots many dataset examples (storm objects)."""

import os.path
import argparse
import numpy
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as pyplot
from gewittergefahr.gg_io import storm_tracking_io as tracking_io
from gewittergefahr.gg_utils import radar_utils
from gewittergefahr.gg_utils import soundings
from gewittergefahr.gg_utils import time_conversion
from gewittergefahr.gg_utils import file_system_utils
from gewittergefahr.gg_utils import error_checking
from gewittergefahr.deep_learning import cnn
from gewittergefahr.deep_learning import testing_io
from gewittergefahr.deep_learning import input_examples
from gewittergefahr.deep_learning import model_activation
from gewittergefahr.deep_learning import deep_learning_utils as dl_utils
from gewittergefahr.deep_learning import training_validation_io as trainval_io
from gewittergefahr.plotting import plotting_utils
from gewittergefahr.plotting import radar_plotting
from gewittergefahr.plotting import sounding_plotting

# TODO(thunderhoser): This is a HACK.
DUMMY_TARGET_NAME = 'tornado_lead-time=0000-3600sec_distance=00000-10000m'

TIME_FORMAT = '%Y-%m-%d-%H%M%S'
SEPARATOR_STRING = '\n\n' + '*' * 50 + '\n\n'

SOUNDING_FIELD_NAMES = [
    soundings.U_WIND_NAME, soundings.V_WIND_NAME,
    soundings.TEMPERATURE_NAME, soundings.SPECIFIC_HUMIDITY_NAME,
    soundings.PRESSURE_NAME
]
SOUNDING_HEIGHTS_M_AGL = soundings.DEFAULT_HEIGHT_LEVELS_M_AGL

SOUNDING_FIGURE_KEY = 'sounding_figure_object'
SOUNDING_AXES_KEY = 'sounding_axes_object'
RADAR_FIGURES_KEY = 'radar_figure_objects'
RADAR_AXES_KEY = 'radar_axes_object_matrices'

IS_SOUNDING_KEY = 'is_sounding'
PMM_FLAG_KEY = 'pmm_flag'
FULL_STORM_ID_KEY = 'full_storm_id_string'
STORM_TIME_KEY = 'storm_time_unix_sec'
RADAR_FIELD_KEY = 'radar_field_name'
RADAR_HEIGHT_KEY = 'radar_height_m_agl'
LAYER_OPERATION_KEY = 'layer_operation_dict'

TITLE_FONT_SIZE = 16
FONT_SIZE_WITH_COLOUR_BARS = 16
FONT_SIZE_SANS_COLOUR_BARS = 20
FIGURE_WIDTH_INCHES = 15
FIGURE_HEIGHT_INCHES = 15
FIGURE_RESOLUTION_DPI = 300

ACTIVATION_FILE_ARG_NAME = 'input_activation_file_name'
STORM_METAFILE_ARG_NAME = 'input_storm_metafile_name'
NUM_EXAMPLES_ARG_NAME = 'num_examples'
ALLOW_WHITESPACE_ARG_NAME = 'allow_whitespace'
EXAMPLE_DIR_ARG_NAME = 'input_example_dir_name'
RADAR_FIELDS_ARG_NAME = 'radar_field_names'
RADAR_HEIGHTS_ARG_NAME = 'radar_heights_m_agl'
PLOT_SOUNDINGS_ARG_NAME = 'plot_soundings'
NUM_ROWS_ARG_NAME = 'num_radar_rows'
NUM_COLUMNS_ARG_NAME = 'num_radar_columns'
OUTPUT_DIR_ARG_NAME = 'output_dir_name'

ACTIVATION_FILE_HELP_STRING = (
    'Path to activation file (will be read by `model_activation.read_file`).  '
    'If this argument is empty, will use `{0:s}`.'
).format(STORM_METAFILE_ARG_NAME)

STORM_METAFILE_HELP_STRING = (
    'Path to Pickle file with storm IDs and times (will be read by '
    '`storm_tracking_io.read_ids_and_times`).  If this argument is empty, will '
    'use `{0:s}`.'
).format(ACTIVATION_FILE_ARG_NAME)

NUM_EXAMPLES_HELP_STRING = (
    'Number of examples (storm objects) to read from `{0:s}` or `{1:s}`.  If '
    'you want to read all examples, make this non-positive.'
).format(ACTIVATION_FILE_ARG_NAME, STORM_METAFILE_ARG_NAME)

ALLOW_WHITESPACE_HELP_STRING = (
    'Boolean flag.  If 0, will plot with no whitespace between panels or around'
    ' outside of image.')

EXAMPLE_DIR_HELP_STRING = (
    'Name of top-level directory with input examples.  Files therein will be '
    'found by `input_examples.find_example_file` and read by '
    '`input_examples.read_example_file`.')

RADAR_FIELDS_HELP_STRING = (
    '[used only if `{0:s}` is empty] List of radar fields (used as input to '
    '`input_examples.read_example_file`).  If you want to plot all radar '
    'fields, leave this argument empty.'
).format(ACTIVATION_FILE_ARG_NAME)

RADAR_HEIGHTS_HELP_STRING = (
    '[used only if `{0:s}` is empty] List of radar heights (used as input to '
    '`input_examples.read_example_file`).  If you want to plot all radar '
    'heights, leave this argument empty.'
).format(ACTIVATION_FILE_ARG_NAME)

PLOT_SOUNDINGS_HELP_STRING = (
    'Boolean flag.  If 1, will plot sounding for each example.  If 0, will not '
    'plot soundings.')

NUM_ROWS_HELP_STRING = (
    '[used only if `{0:s}` is empty] Number of rows in each storm-centered '
    'radar grid.  If you want to plot the largest grids available, leave this '
    'argument empty.'
).format(ACTIVATION_FILE_ARG_NAME)

NUM_COLUMNS_HELP_STRING = (
    '[used only if `{0:s}` is empty] Number of columns in each storm-centered '
    'radar grid.  If you want to plot the largest grids available, leave this '
    'argument empty.'
).format(ACTIVATION_FILE_ARG_NAME)

OUTPUT_DIR_HELP_STRING = (
    'Name of output directory (figures will be saved here).')

INPUT_ARG_PARSER = argparse.ArgumentParser()
INPUT_ARG_PARSER.add_argument(
    '--' + ACTIVATION_FILE_ARG_NAME, type=str, required=False, default='',
    help=ACTIVATION_FILE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + STORM_METAFILE_ARG_NAME, type=str, required=False, default='',
    help=STORM_METAFILE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + NUM_EXAMPLES_ARG_NAME, type=int, required=False, default=-1,
    help=NUM_EXAMPLES_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + ALLOW_WHITESPACE_ARG_NAME, type=int, required=False, default=1,
    help=ALLOW_WHITESPACE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + EXAMPLE_DIR_ARG_NAME, type=str, required=True,
    help=EXAMPLE_DIR_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + RADAR_FIELDS_ARG_NAME, type=str, nargs='+', required=False,
    default=[''], help=RADAR_FIELDS_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + RADAR_HEIGHTS_ARG_NAME, type=int, nargs='+', required=False,
    default=[-1], help=RADAR_HEIGHTS_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + PLOT_SOUNDINGS_ARG_NAME, type=int, required=False, default=1,
    help=PLOT_SOUNDINGS_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + NUM_ROWS_ARG_NAME, type=int, required=False, default=-1,
    help=NUM_ROWS_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + NUM_COLUMNS_ARG_NAME, type=int, required=False, default=-1,
    help=NUM_COLUMNS_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + OUTPUT_DIR_ARG_NAME, type=str, required=True,
    help=OUTPUT_DIR_HELP_STRING)


def _plot_sounding(
        list_of_predictor_matrices, model_metadata_dict, allow_whitespace,
        title_string=None):
    """Plots sounding for one example.

    :param list_of_predictor_matrices: See doc for `_plot_3d_radar_scan`.
    :param model_metadata_dict: Same.
    :param allow_whitespace: Same.
    :param title_string: Same.
    :return: figure_object: Figure handle (instance of
        `matplotlib.figure.Figure`).
    :return: axes_object: Axes handle (instance of
        `matplotlib.axes._subplots.AxesSubplot`).
    """

    training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
    sounding_field_names = training_option_dict[trainval_io.SOUNDING_FIELDS_KEY]
    sounding_heights_m_agl = training_option_dict[
        trainval_io.SOUNDING_HEIGHTS_KEY]

    sounding_matrix = numpy.expand_dims(list_of_predictor_matrices[-1], axis=0)

    # TODO(thunderhoser): Allow actual storm elevation to be used.
    metpy_dict = dl_utils.soundings_to_metpy_dictionaries(
        sounding_matrix=sounding_matrix, field_names=sounding_field_names,
        height_levels_m_agl=sounding_heights_m_agl,
        storm_elevations_m_asl=numpy.array([0.])
    )[0]

    return sounding_plotting.plot_sounding(
        sounding_dict_for_metpy=metpy_dict,
        title_string=title_string if allow_whitespace else None
    )


def _plot_3d_radar_scan(
        list_of_predictor_matrices, model_metadata_dict, allow_whitespace,
        title_string=None):
    """Plots 3-D radar scan for one example.

    J = number of panel rows in image
    K = number of panel columns in image
    F = number of radar fields

    :param list_of_predictor_matrices: List created by
        `testing_io.read_specific_examples`, except that the first axis (example
        dimension) is removed.
    :param model_metadata_dict: Dictionary returned by
        `cnn.read_model_metadata`.
    :param allow_whitespace: See documentation at top of file.
    :param title_string: Title (may be None).

    :return: figure_objects: length-F list of figure handles (instances of
        `matplotlib.figure.Figure`).
    :return: axes_object_matrices: length-F list.  Each element is a J-by-K
        numpy array of axes handles (instances of
        `matplotlib.axes._subplots.AxesSubplot`).
    """

    training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]
    radar_field_names = training_option_dict[trainval_io.RADAR_FIELDS_KEY]
    radar_heights_m_agl = training_option_dict[trainval_io.RADAR_HEIGHTS_KEY]

    colour_bar_names = radar_plotting.radar_fields_and_heights_to_panel_names(
        field_names=radar_field_names, heights_m_agl=radar_heights_m_agl,
        include_units=True)
    colour_bar_names = [n.replace('\n', ' ') for n in colour_bar_names]

    num_radar_fields = len(radar_field_names)
    num_radar_heights = len(radar_heights_m_agl)

    num_panel_rows = int(numpy.floor(
        numpy.sqrt(num_radar_heights)
    ))
    num_panel_columns = int(numpy.ceil(
        float(num_radar_heights) / num_panel_rows
    ))

    figure_objects = [None] * num_radar_fields
    axes_object_matrices = [None] * num_radar_fields
    radar_matrix = list_of_predictor_matrices[0]

    for j in range(num_radar_fields):
        this_radar_matrix = numpy.flip(radar_matrix[..., j], axis=0)

        if not allow_whitespace:
            figure_objects[j], axes_object_matrices[j] = (
                plotting_utils.create_paneled_figure(
                    num_rows=num_panel_rows, num_columns=num_panel_columns,
                    horizontal_spacing=0., vertical_spacing=0.,
                    shared_x_axis=False, shared_y_axis=False,
                    keep_aspect_ratio=True)
            )

        figure_objects[j], axes_object_matrices[j] = (
{'fraction_non_alphanumeric': 0.08509803921568627, 'fraction_numerical': 0.005980392156862745, 'mean_word_length': 4.741908246552209, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1878318', 'n_tokens_mistral': 7542, 'n_tokens_neox': 7316, 'n_words': 1409}