<commit_before>import os
import re
import time
import config
import plotter
import subprocess
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages

class experiment:
        """ an experiment run consists of:
                - a trial (fold index or None if cumulating results from multiple trials)
                - a model (name + command line parameters to apps/builder)
                - a trainer (name + json parameters)
                - an enhancer (name + json parameters)
                - a loss function (name)
        """
        def __init__(self, task, outdir, trials = 10):
                self.cfg = config.config()
                self.task = task
                self.dir = outdir
                self.trials = trials
                self.runs = []
                self.losses = []
                self.models = []
                self.trainers = []
                self.enhancers = []

        def log(self, *messages):
                print(time.strftime("[%Y-%m-%d %H:%M:%S]"), ' '.join(messages))

        def add_model(self, name, parameters):
                self.models.append([name, parameters])

        def add_trainer(self, name, parameters, config_name = None):
                self.trainers.append([config_name if config_name else name, parameters])

        def add_loss(self, name, parameters = "", config_name = None):
                self.losses.append([config_name if config_name else name, parameters])

        def add_enhancer(self, name, parameters = "", config_name = None):
                self.enhancers.append([config_name if config_name else name, parameters])

        def path(self, trial, mname, tname, ename, lname, extension):
                basepath = self.dir
                basepath += "trial{}".format(trial) if not (trial is None) else "result"
                basepath += "_M" + mname if mname else ""
                basepath += "_T" + tname if tname else ""
                basepath += "_E" + ename if ename else ""
                basepath += "_L" + lname if lname else ""
                basepath += extension
                return basepath

        def name(self, name_config):
                return name_config[0]

        def config(self, name_config):
                return name_config[1]

        def names(self, names_configs, name_reg = None):
                names = []
                for name_config in names_configs:
                        name = self.name(name_config)
                        if name_reg == None or re.match(name_reg, name):
                                names.append(name)
                return names

        def filter_names(self, mname_reg, tname_reg, ename_reg, lname_reg):
                mnames = self.names(self.models, mname_reg)
                tnames = self.names(self.trainers, tname_reg)
                enames = self.names(self.enhancers, ename_reg)
                lnames = self.names(self.losses, lname_reg)
                return mnames, tnames, enames, lnames

        def filter_paths(self, trial, mname_reg, tname_reg, ename_reg, lname_reg, extension):
                paths = []
                mnames, tnames, enames, lnames = self.filter_names(mname_reg, tname_reg, ename_reg, lname_reg)
                for mname, tname, ename, lname in [(u, x, y, z) for u in mnames for x in tnames for y in enames for z in lnames]:
                        paths.append(self.path(trial, mname, tname, ename, lname, extension))
                return paths

        def reg2str(self, reg):
                return reg.replace(".*", "all").replace("*", "").replace(".", "") if reg else "all"

        def train_one(self, param, lpath):
                with open(lpath, "w") as lfile:
                        self.log("running <", param, ">...")
                        subprocess.check_call((self.cfg.app_train + " " + param).split(), stdout = lfile)
                self.log("|--->training done, see <", lpath, ">")

        def plot_one(self, spath, ppath):
                plotter.plot_state_one(spath, ppath)
                self.log("|--->plotting done, see <", ppath, ">")

        def plot_many(self, spaths, ppath):
                plotter.plot_state_many(spaths, ppath)
                self.log("|--->plotting done, see <", ppath, ">")

        def plot_trial(self, spaths, ppath, names):
                plotter.plot_trial_many(spaths, ppath, names)
                self.log("|--->plotting done, see <", ppath, ">")

        def tabulate(self, cpath, lpath):
                with open(lpath, "w") as lfile:
                        subprocess.check_call((self.cfg.app_tabulate + " -i " + cpath + " -d \";\"").split(), stdout = lfile)

                with open(lpath, "r") as lfile:
                        self.log()
                        print(lfile.read())

        def summarize_by_models(self, mname_reg = ".*"):
                mname = self.reg2str(mname_reg)

        def train_trials(self, mname, mparam, tname, tparam, ename, eparam, lname, lparam):
                # train the given configuration for multiple trials
                for trial in range(self.trials):
                        os.makedirs(self.dir, exist_ok = True)
                        mpath = self.path(trial, mname, tname, ename, lname, ".model")
                        spath = self.path(trial, mname, tname, ename, lname, ".state")
                        lpath = self.path(trial, mname, tname, ename, lname, ".log")
                        ppath = self.path(trial, mname, tname, ename, lname, ".pdf")

                        param = self.task + " " + mparam + " " + tparam + " " + eparam + " " + lparam + " --model-file " + mpath
                        self.train_one(param, lpath)
                        self.plot_one(spath, ppath)
                        self.log()
<commit_msg>create json configuration files before running the experiment<commit_after>import os
import re
import json
import time
import config
import plotter
import subprocess
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages

class experiment:
        """ an experiment run consists of:
                - a trial (fold index or None if cumulating results from multiple trials)
                - a model (name + command line parameters to apps/builder)
                - a trainer (name + json parameters)
                - an enhancer (name + json parameters)
                - a loss function (name)
        """
        def __init__(self, task, outdir, trials = 10):
                self.cfg = config.config()
                self.task = task
                self.dir = outdir
                self.trials = trials
                self.runs = []
                self.losses = []
                self.models = []
                self.trainers = []
                self.enhancers = []
                os.makedirs(self.dir, exist_ok = True)
                self.save_json("config_task", task)

        def save_json(self, basename, parameters):
                with open(os.path.join(self.dir, basename + ".json"), "w") as output:
                        output.write(json.dumps(parameters, indent = 4))

        def log(self, *messages):
                print(time.strftime("[%Y-%m-%d %H:%M:%S]"), ' '.join(messages))

        def add_model(self, name, parameters):
                parameters += " --json {}".format(os.path.join(self.dir, "config_model_" + name + ".json"))
                self.models.append([name, parameters])
                subprocess.check_call((self.cfg.app_builder + " " + parameters).split(), stdout=subprocess.DEVNULL)

        def add_trainer(self, name, parameters):
                self.trainers.append([name, parameters])
                self.save_json("config_trainer_" + name, parameters)

        def add_loss(self, name, parameters = ""):
                self.losses.append([name, parameters])

        def add_enhancer(self, name, parameters):
                self.enhancers.append([name, parameters])
                self.save_json("config_enhancer_" + name, parameters)

        def path(self, trial, mname, tname, ename, lname, extension):
                basepath = self.dir
                basepath += "trial{}".format(trial) if not (trial is None) else "result"
                basepath += "_M" + mname if mname else ""
                basepath += "_T" + tname if tname else ""
                basepath += "_E" + ename if ename else ""
                basepath += "_L" + lname if lname else ""
                basepath += extension
                return basepath

        def name(self, name_config):
                return name_config[0]

        def config(self, name_config):
                return name_config[1]

        def names(self, names_configs, name_reg = None):
                names = []
                for name_config in names_configs:
                        name = self.name(name_config)
                        if name_reg == None or re.match(name_reg, name):
                                names.append(name)
                return names

        def filter_names(self, mname_reg, tname_reg, ename_reg, lname_reg):
                mnames = self.names(self.models, mname_reg)
                tnames = self.names(self.trainers, tname_reg)
                enames = self.names(self.enhancers, ename_reg)
                lnames = self.names(self.losses, lname_reg)
                return mnames, tnames, enames, lnames

        def filter_paths(self, trial, mname_reg, tname_reg, ename_reg, lname_reg, extension):
                paths = []
                mnames, tnames, enames, lnames = self.filter_names(mname_reg, tname_reg, ename_reg, lname_reg)
                for mname, tname, ename, lname in [(u, x, y, z) for u in mnames for x in tnames for y in enames for z in lnames]:
                        paths.append(self.path(trial, mname, tname, ename, lname, extension))
                return paths

        def reg2str(self, reg):
                return reg.replace(".*", "all").replace("*", "").replace(".", "") if reg else "all"

        def train_one(self, param, lpath):
                with open(lpath, "w") as lfile:
                        self.log("running <", param, ">...")
                        subprocess.check_call((self.cfg.app_train + " " + param).split(), stdout = lfile)
                self.log("|--->training done, see <", lpath, ">")

        def plot_one(self, spath, ppath):
                plotter.plot_state_one(spath, ppath)
                self.log("|--->plotting done, see <", ppath, ">")

        def plot_many(self, spaths, ppath):
                plotter.plot_state_many(spaths, ppath)
                self.log("|--->plotting done, see <", ppath, ">")

        def plot_trial(self, spaths, ppath, names):
                plotter.plot_trial_many(spaths, ppath, names)
                self.log("|--->plotting done, see <", ppath, ">")

        def tabulate(self, cpath, lpath):
                with open(lpath, "w") as lfile:
                        subprocess.check_call((self.cfg.app_tabulate + " -i " + cpath + " -d \";\"").split(), stdout = lfile)

                with open(lpath, "r") as lfile:
                        self.log()
                        print(lfile.read())

        def summarize_by_models(self, mname_reg = ".*"):
                mname = self.reg2str(mname_reg)

        def train_trials(self, mname, mparam, tname, tparam, ename, eparam, lname, lparam):
                # train the given configuration for multiple trials
                for trial in range(self.trials):
                        mpath = self.path(trial, mname, tname, ename, lname, ".model")
                        spath = self.path(trial, mname, tname, ename, lname, ".state")
                        lpath = self.path(trial, mname, tname, ename, lname, ".log")
                        ppath = self.path(trial, mname, tname, ename, lname, ".pdf")

                        param = self.task + " " + mparam + " " + tparam + " " + eparam + " " + lparam + " --model-file " + mpath
                        self.train_one(param, lpath)
                        self.plot_one(spath, ppath)
                        self.log()

{'fraction_non_alphanumeric': 0.1167745636318938, 'fraction_numerical': 0.0010653118085716627, 'mean_word_length': 1.6928508384819065, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5485602', 'n_tokens_mistral': 3638, 'n_tokens_neox': 3402, 'n_words': 900}