<commit_before>'create ege_week.makefile'
import datetime
import pdb
from pprint import pprint
import random


lines = []
lines.append('# generated by running ege_week_makefile.py at ' + str(datetime.datetime.now()))


def make_lines(date, model, training_days, hp, system):
    verbose = True
    if verbose:
        print date, model, training_days, hp, system
    target_file_prefix = '%s-%s-%03d' % (date, model, training_days)
    target_file_middle = ('-%02d' % hp if model == 'rf' else
                          '-%s-%s' % ('lin', 'lin') if model == 'lr' else  # use lin-lin as representative
                          None)
    target_file_suffix = '-0.pickle'  # use fold 0 as a representative of all the folds
    target_file = target_file_prefix + target_file_middle + target_file_suffix
    target_path = '../data/working/ege_week/%s' % target_file
    lines.append('%s += %s' % (system, target_path))
    # the rule
    # lines.append('%s: ege_week.py ../data/working/transactions-subset2.pickle' % target_path)
    lines.append('%s:' % target_path)  # no dependencies, as this makefile is run mannually
    # the recipe
    recipe_prefix = '\t~/anaconda/bin/python ege_week.py %s --model %s --td %d' % (
        date, model, training_days)
    recipe_suffix = '' if model == 'lr' else ' --hp %s' % hp
    lines.append(recipe_prefix + recipe_suffix)
    if verbose:
        pprint(lines[len(lines) - 3: len(lines)])
    lines.append(' ')


def make_hps(model, test):
    if model == 'lr':
        return (None,)
    assert model == 'rf'
    return (4,) if test else range(1, 27, 1)

# choose parameters
test = False
random.seed(123)

# define number of cores to use on each system
if test:
    systems = ('roy', 'judith', 'hp')
else:
    # run 10 jobs on my system and 6 on judiths's
    roy = 8 * ['roy']
    judith = 6 * ['judith']
    hp = 4 * ['hp']
    systems = roy + judith + hp
    random.shuffle(systems)

# define the sale_date
dates = [datetime.date(2005, 02, 15)]
if False:
    for year in (2004, 2005, 2006, 2007, 2008):
        for month in (2, 5, 8, 11):
            dates.append(datetime.date(year, month, 15))
<commit_msg>2009-02-15<commit_after>'create ege_week.makefile'
import datetime
import pdb
from pprint import pprint
import random


lines = []
lines.append('# generated by running ege_week_makefile.py at ' + str(datetime.datetime.now()))


def make_lines(date, model, training_days, hp, system):
    verbose = True
    if verbose:
        print date, model, training_days, hp, system
    target_file_prefix = '%s-%s-%03d' % (date, model, training_days)
    target_file_middle = ('-%02d' % hp if model == 'rf' else
                          '-%s-%s' % ('lin', 'lin') if model == 'lr' else  # use lin-lin as representative
                          None)
    target_file_suffix = '-0.pickle'  # use fold 0 as a representative of all the folds
    target_file = target_file_prefix + target_file_middle + target_file_suffix
    target_path = '../data/working/ege_week/%s' % target_file
    lines.append('%s += %s' % (system, target_path))
    # the rule
    # lines.append('%s: ege_week.py ../data/working/transactions-subset2.pickle' % target_path)
    lines.append('%s:' % target_path)  # no dependencies, as this makefile is run mannually
    # the recipe
    recipe_prefix = '\t~/anaconda/bin/python ege_week.py %s --model %s --td %d' % (
        date, model, training_days)
    recipe_suffix = '' if model == 'lr' else ' --hp %s' % hp
    lines.append(recipe_prefix + recipe_suffix)
    if verbose:
        pprint(lines[len(lines) - 3: len(lines)])
    lines.append(' ')


def make_hps(model, test):
    if model == 'lr':
        return (None,)
    assert model == 'rf'
    return (4,) if test else range(1, 27, 1)

# choose parameters
test = False
random.seed(123)

# define number of cores to use on each system
if test:
    systems = ('roy', 'judith', 'hp')
else:
    # run 10 jobs on my system and 6 on judiths's
    roy = 8 * ['roy']
    judith = 6 * ['judith']
    hp = 4 * ['hp']
    systems = roy + judith + hp
    random.shuffle(systems)

# define the sale_date
dates = [datetime.date(2009, 02, 15)]
if False:
    for year in (2004, 2005, 2006, 2007, 2008):
        for month in (2, 5, 8, 11):
            dates.append(datetime.date(year, month, 15))

{'fraction_non_alphanumeric': 0.13810741687979539, 'fraction_numerical': 0.028365496396186934, 'mean_word_length': 2.87917042380523, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5069739', 'n_tokens_mistral': 1572, 'n_tokens_neox': 1436, 'n_words': 438}