<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Create and train a given model."""

import os
import yaml
import logging
import sys
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                    level=logging.DEBUG,
                    stream=sys.stdout)
import natsort
# mine
import utils


def create_model(model_folder, basename, model_type, topology, training,
                 data):
    models = filter(lambda n: n.endswith(".json"), os.listdir(model_folder))
    models = filter(lambda n: n.startswith(basename), models)
    models = natsort.natsorted(models, reverse=True)
    if len(models) == 0:
        logging.info("No base model. Create it...")
        model_src = os.path.join(model_folder, "%s-0.json" % basename)
        command = "nntoolkit make %s %s > %s" % (model_type,
                                                 topology,
                                                 model_src)
        model_target = os.path.join(model_folder, "%s-1.json" % basename)
        logging.info(command)
        os.system(command)
    else:
        latest_model = models[0]
        i = int(latest_model.split("-")[-1])
        model_src = os.path.json(model_folder, "%s-%i.json" % (basename, i))
        model_target = os.path.json(model_folder,
                                    "%s-%i.json" % (basename, i+1))

    # train the model
    training = training.replace("{{training}}", data['training'])
    training = training.replace("{{validation}}", data['validating'])
    training = training.replace("{{src_model}}", model_src)
    training = training.replace("{{target_model}}", model_target)
    logging.info(training)
    os.system(training)<commit_msg>fixed error<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Create and train a given model."""

import os
import yaml
import logging
import sys
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                    level=logging.DEBUG,
                    stream=sys.stdout)
import natsort
# mine
import utils


def create_model(model_folder, basename, model_type, topology, training,
                 data):
    models = filter(lambda n: n.endswith(".json"), os.listdir(model_folder))
    models = filter(lambda n: n.startswith(basename), models)
    models = natsort.natsorted(models, reverse=True)
    if len(models) == 0:
        logging.info("No base model. Create it...")
        model_src = os.path.join(model_folder, "%s-0.json" % basename)
        command = "nntoolkit make %s %s > %s" % (model_type,
                                                 topology,
                                                 model_src)
        model_target = os.path.join(model_folder, "%s-1.json" % basename)
        logging.info(command)
        os.system(command)
    else:
        latest_model = models[0]
        i = int(latest_model.split("-")[-1].split(".")[0])
        model_src = os.path.json(model_folder, "%s-%i.json" % (basename, i))
        model_target = os.path.json(model_folder,
                                    "%s-%i.json" % (basename, i+1))

    # train the model
    training = training.replace("{{training}}", data['training'])
    training = training.replace("{{validation}}", data['validating'])
    training = training.replace("{{src_model}}", model_src)
    training = training.replace("{{target_model}}", model_target)
    logging.info(training)
    os.system(training)
{'fraction_non_alphanumeric': 0.14551804423748546, 'fraction_numerical': 0.004365541327124563, 'mean_word_length': 2.3895463510848125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '454774', 'n_tokens_mistral': 1036, 'n_tokens_neox': 1014, 'n_words': 214}