<commit_before>from os.path import abspath, dirname, join
import sys
import pickle
from time import localtime, strftime

sys.path.append(abspath(dirname(dirname(__file__))))
from utils.data_io import load_numpy_array_from_file
from utils.data_paths import DATA_DIR_PATH, MODELS_DIR_PATH, RESULTS_DIR_PATH


def calculate_rmse(true_ratings, predictions):
    return ((predictions - true_ratings) ** 2).mean()


def run(model, train_set_name, test_set_name, epochs=None, features=None):
    model.debug = True

    print('Training {modelclass} on "{train}" ratings'
          .format(modelclass=model.__class__.__name__, train=train_set_name))
    if epochs is not None:
        print('Number of epochs:', epochs)
    if features is not None:
        print('Number of features:', features)
    time_format = '%b-%d-%Hh-%Mm'
    time_stamp = strftime(time_format, localtime())
    train_file_path = join(DATA_DIR_PATH, train_set_name + '.npy')
    train_points = load_numpy_array_from_file(train_file_path)
    model.train(train_points, epochs)
    time_stamp += '_to_' + strftime(time_format, localtime())

    epochs_string = '' if epochs is None else ('_%sepochs' % epochs)
    features_string = '' if features is None else ('_%sfeatures' % features)
    template_file_name = ('svd_{train}{e}{f}_xxx_{time}'
                          .format(train=train_set_name, e=epochs_string,
                                  f=features_string, time=time_stamp))

    model_file_name = template_file_name.replace('xxx', 'model') + '.p'
    save_model(model, model_file_name)

    print('Predicting "{test}" ratings'.format(test=test_set_name))
    test_file_path = join(DATA_DIR_PATH, test_set_name + '.npy')
    test_points = load_numpy_array_from_file(test_file_path)
    predictions = model.predict(test_points)
    pred_file_name = template_file_name.replace('xxx', 'predictions') + '.dta'
    save_predictions(predictions, pred_file_name)

    true_ratings = test_points[:, 3]
    rmse = calculate_rmse(true_ratings, predictions)<commit_msg>Change run_model to set model.train_points = None after training (smaller model files)<commit_after>from os.path import abspath, dirname, join
import sys
import pickle
from time import localtime, strftime

sys.path.append(abspath(dirname(dirname(__file__))))
from utils.data_io import load_numpy_array_from_file
from utils.data_paths import DATA_DIR_PATH, MODELS_DIR_PATH, RESULTS_DIR_PATH


def calculate_rmse(true_ratings, predictions):
    return ((predictions - true_ratings) ** 2).mean()


def run(model, train_set_name, test_set_name, epochs=None, features=None):
    print('Training {modelclass} on "{train}" ratings'
          .format(modelclass=model.__class__.__name__, train=train_set_name))
    if epochs is not None:
        print('Number of epochs:', epochs)
    if features is not None:
        print('Number of features:', features)
    time_format = '%b-%d-%Hh-%Mm'
    train_file_path = join(DATA_DIR_PATH, train_set_name + '.npy')
    time_stamp = strftime(time_format, localtime())

    model.debug = True
    train_points = load_numpy_array_from_file(train_file_path)
    model.train(train_points, epochs)
    time_stamp += '_to_' + strftime(time_format, localtime())
    model.train_points = None

    epochs_string = '' if epochs is None else ('_%sepochs' % epochs)
    features_string = '' if features is None else ('_%sfeatures' % features)
    template_file_name = ('svd_{train}{e}{f}_xxx_{time}'
                          .format(train=train_set_name, e=epochs_string,
                                  f=features_string, time=time_stamp))

    model_file_name = template_file_name.replace('xxx', 'model') + '.p'
    save_model(model, model_file_name)

    print('Predicting "{test}" ratings'.format(test=test_set_name))
    test_file_path = join(DATA_DIR_PATH, test_set_name + '.npy')
    test_points = load_numpy_array_from_file(test_file_path)
    predictions = model.predict(test_points)
    pred_file_name = template_file_name.replace('xxx', 'predictions') + '.dta'
    save_predictions(predictions, pred_file_name)

    true_ratings = test_points[:, 3]
    rmse = calculate_rmse(true_ratings, predictions)
{'fraction_non_alphanumeric': 0.11071171818835371, 'fraction_numerical': 0.000958543014617781, 'mean_word_length': 4.39974126778784, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4585369', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1355, 'n_words': 279}