<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-
"""
Module for experiment support
"""

import argparse
import logging
import os

logger = logging.getLogger(__name__)

import matplotlib.pyplot as plt
import subprocess
import glob
import numpy as np
import copy
# import six
import pandas
import traceback
import shutil

# ----------------- my scripts --------
import io3d
import io3d.misc
# <codecell>
from . import misc
from . import volumetry_evaluation


class RunAndMakeReport:
    def __init__(self, experiment_dir, labels, sliver_reference_dir,
                 input_data_path_pattern, conf_default=None,
                 conf_list=None, show=True, use_plt=True,
                 image_basename=None, only_if_necessary=True,
                 pklz_dirs=None, experiment_name=None,
                 filename_separator='-', markers=None,
                 clean_experiment_dir=False
                 ):

        """
        Je potřeba mít připraven adresář s daty(pklz_dirs), v nichž jsou uloženy
        seedy. Dále pak soubory s konfigurací a stejným jménem jako adresář +
        přípona '.conf'.

        :param experiment_dir: base directory of experiment outputs
        :params labels: Labels for describing graphs. They are also used for
            generating dir names if pklz_dirs is not used.
        :param sliver_reference_dir: dir with sliver reference data
        :param input_data_path_pattern: Directory containing data with seeds. For
            example::

            "/home/mjirik/exp010-seeds/*-seeds.pklz"
        :param conf_default: default values for config generation.
            Example::

            conf_default = {
                'config_version': [1,0,0],
                'working_voxelsize_mm': 2.0,
                'segparams': {
                    'pairwise_alpha_per_mm2': 50,
                    'return_only_object_with_seeds': True,
                    'method': 'GC'
                }
            }
        :param experiment_name: string is used as prefix for all output files
        :param pklz_dirs: List of dirs or string with dir prefix. If the string is
            given base on labels the list is generated
            In each will be output for one experiment. There
            is also required to have file with same name as directory with
            extension.config. In this files is configuration of Lisa.
        :param image_basename: Basis for image names. It can be directory, or
            filename begining
        :param only_if_necessary: If it is True the make_all() function call
            run_experiments() and evaluation() function only if there are no
            evaluation outputs and experiment_outputs.
        :param markers: Allows to control markes for output graphs

        self.dataframe_all is storage for all outpt mesurements
        """

        self.conf_list = conf_list
        self.conf_default = conf_default
        self.labels = labels
        self.sliver_dir = sliver_reference_dir
        self.experiment_dir = experiment_dir
        self.df_all = None
        if experiment_name is None:
            aa, experiment_name = os.path.split(self.experiment_dir)
        self.experiment_name = experiment_name

        if clean_experiment_dir:
            if os.path.exists(experiment_dir):
                shutil.rmtree(experiment_dir)
            os.mkdir(experiment_dir)

        # if isinstance(pklz_dirs, six.string_types):
        if pklz_dirs is None:
            # pklz_dir is string
            pklz_dirs = [
                os.path.join(
                    experiment_dir,
                    experiment_name + filename_separator + lab.replace(' ', ''))
                for lab in labels
            ]
        self.pklz_dirs = pklz_dirs
        self.yaml_files = \
            [os.path.normpath(path) + '.yaml' for path in pklz_dirs]
        self.eval_files = \
            [os.path.normpath(path) + '_eval' for path in pklz_dirs]
        self.exp_conf_files = \
            [os.path.normpath(path) + '.config' for path in pklz_dirs]
        if image_basename is None:
            # self.image_basename, head = os.path.split(self.pklz_dirs[0])
            self.image_basename = os.path.join(
                self.experiment_dir, experiment_name + filename_separator)

        self.show = show
        self.input_data_path_pattern = input_data_path_pattern
        self.use_plt = use_plt
        self.only_if_necessary = only_if_necessary
        self.filename_separator = filename_separator
        if markers is None:
            markers = [
                "ks", "k<", "k>", "k^", "kv", "kp", 'k*', 'ok', 'kh', 'ks'
            ]
        # repeat markers to have requested length
        n = len(labels)
        markers = np.asarray(([markers] * ((n / len(markers) + 1)))
                             ).reshape(-1)[:n].tolist()
        self.markers = markers
        self.dataframe_all = None

    def make_all(self):
        """
        Run config(), run_experiments(), evaluation() and report()
        """
        self.config()
        if self.only_if_necessary:
            if self.is_evaluation_necessary():
                if self.is_run_experiments_necessary():
                    self.run_experiments()
                self.evaluation()
        else:<commit_msg>py36 make int from float after division<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-
"""
Module for experiment support
"""

import argparse
import logging
import os

logger = logging.getLogger(__name__)

import matplotlib.pyplot as plt
import subprocess
import glob
import numpy as np
import copy
# import six
import pandas
import traceback
import shutil

# ----------------- my scripts --------
import io3d
import io3d.misc
# <codecell>
from . import misc
from . import volumetry_evaluation


class RunAndMakeReport:
    def __init__(self, experiment_dir, labels, sliver_reference_dir,
                 input_data_path_pattern, conf_default=None,
                 conf_list=None, show=True, use_plt=True,
                 image_basename=None, only_if_necessary=True,
                 pklz_dirs=None, experiment_name=None,
                 filename_separator='-', markers=None,
                 clean_experiment_dir=False
                 ):

        """
        Je potřeba mít připraven adresář s daty(pklz_dirs), v nichž jsou uloženy
        seedy. Dále pak soubory s konfigurací a stejným jménem jako adresář +
        přípona '.conf'.

        :param experiment_dir: base directory of experiment outputs
        :params labels: Labels for describing graphs. They are also used for
            generating dir names if pklz_dirs is not used.
        :param sliver_reference_dir: dir with sliver reference data
        :param input_data_path_pattern: Directory containing data with seeds. For
            example::

            "/home/mjirik/exp010-seeds/*-seeds.pklz"
        :param conf_default: default values for config generation.
            Example::

            conf_default = {
                'config_version': [1,0,0],
                'working_voxelsize_mm': 2.0,
                'segparams': {
                    'pairwise_alpha_per_mm2': 50,
                    'return_only_object_with_seeds': True,
                    'method': 'GC'
                }
            }
        :param experiment_name: string is used as prefix for all output files
        :param pklz_dirs: List of dirs or string with dir prefix. If the string is
            given base on labels the list is generated
            In each will be output for one experiment. There
            is also required to have file with same name as directory with
            extension.config. In this files is configuration of Lisa.
        :param image_basename: Basis for image names. It can be directory, or
            filename begining
        :param only_if_necessary: If it is True the make_all() function call
            run_experiments() and evaluation() function only if there are no
            evaluation outputs and experiment_outputs.
        :param markers: Allows to control markes for output graphs

        self.dataframe_all is storage for all outpt mesurements
        """

        self.conf_list = conf_list
        self.conf_default = conf_default
        self.labels = labels
        self.sliver_dir = sliver_reference_dir
        self.experiment_dir = experiment_dir
        self.df_all = None
        if experiment_name is None:
            aa, experiment_name = os.path.split(self.experiment_dir)
        self.experiment_name = experiment_name

        if clean_experiment_dir:
            if os.path.exists(experiment_dir):
                shutil.rmtree(experiment_dir)
            os.mkdir(experiment_dir)

        # if isinstance(pklz_dirs, six.string_types):
        if pklz_dirs is None:
            # pklz_dir is string
            pklz_dirs = [
                os.path.join(
                    experiment_dir,
                    experiment_name + filename_separator + lab.replace(' ', ''))
                for lab in labels
            ]
        self.pklz_dirs = pklz_dirs
        self.yaml_files = \
            [os.path.normpath(path) + '.yaml' for path in pklz_dirs]
        self.eval_files = \
            [os.path.normpath(path) + '_eval' for path in pklz_dirs]
        self.exp_conf_files = \
            [os.path.normpath(path) + '.config' for path in pklz_dirs]
        if image_basename is None:
            # self.image_basename, head = os.path.split(self.pklz_dirs[0])
            self.image_basename = os.path.join(
                self.experiment_dir, experiment_name + filename_separator)

        self.show = show
        self.input_data_path_pattern = input_data_path_pattern
        self.use_plt = use_plt
        self.only_if_necessary = only_if_necessary
        self.filename_separator = filename_separator
        if markers is None:
            markers = [
                "ks", "k<", "k>", "k^", "kv", "kp", 'k*', 'ok', 'kh', 'ks'
            ]
        # repeat markers to have requested length
        n = len(labels)
        number_of_each_marker = int(np.ceil((n / len(markers) + 1)))
        markers = np.asarray(
            [markers] * number_of_each_marker
        ).reshape(-1)[:n].tolist()
        self.markers = markers
        self.dataframe_all = None

    def make_all(self):
        """
        Run config(), run_experiments(), evaluation() and report()
        """
        self.config()
        if self.only_if_necessary:
            if self.is_evaluation_necessary():
                if self.is_run_experiments_necessary():
                    self.run_experiments()
                self.evaluation()
        else:
{'fraction_non_alphanumeric': 0.0823330832708177, 'fraction_numerical': 0.0033758439609902473, 'mean_word_length': 1.9715798272499303, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '467740', 'n_tokens_mistral': 3104, 'n_tokens_neox': 3006, 'n_words': 836}