<commit_before>import argparse

from blocks.algorithms import GradientDescent, Adam, CompositeRule, StepClipping
from blocks.extensions import Timing, Printing, FinishAfter, ProgressBar
from blocks.extensions.monitoring import TrainingDataMonitoring
from blocks.main_loop import MainLoop
from blocks.monitoring import aggregation
from fuel.datasets.hdf5 import H5PYDataset
from fuel.schemes import SequentialScheme, ShuffledScheme
from fuel.streams import DataStream
from numpy import load

from custom_blocks import PadAndAddMasks, EarlyStopping
from network import *


parser = argparse.ArgumentParser()
parser.add_argument("-c", "--clipping", default=1.0, type=float, help="Step clipping threshold (default: 1)")
parser.add_argument("-t", "--type", default="simple", help="Type of RNN ("+NetworkType.SIMPLE_RNN+", " +
                                                           NetworkType.GRU+" or "+NetworkType.LSTM+")")
parser.add_argument("-e", "--epochs", default=0, type=int, help="Stop after this many epochs (default: 0 -- means it will" +
                                                                " run forever until you cancel manually)")
parser.add_argument("-d", "--dimensions", default="512,512,512", type=str, help="Configure number of layers and dimensions by" +
                                                                                " just enumerating dimensions (e.g. 100, 100 for" +
                                                                         " a two-layered network with dims 100)")
parser.add_argument('-f', '--file', type=str, help='Specifies the data used for training.')
parser.add_argument('-a', '--alphafile', type=str, help='Specifies the location of the alphabet file.')
parser.add_argument('-b', '--batchsize', type=int, help='Set the batch size for training, set 0 for all in one batch and one batch per epoch.')
parser.add_argument('-m', '--memory', type=int, default=1, help='Set to 0 if files should be read from disc, any other number to load into memory.')

args = parser.parse_args()

dimensions = [int(d) for d in args.dimensions.split(',')]
print('Dimensions:', dimensions)

ix2char = load(args.alphafile).item()
<commit_msg>Alphabet files are expected to be pickled.<commit_after>import argparse
import pickle

from blocks.algorithms import GradientDescent, Adam, CompositeRule, StepClipping
from blocks.extensions import Timing, Printing, FinishAfter, ProgressBar
from blocks.extensions.monitoring import TrainingDataMonitoring
from blocks.main_loop import MainLoop
from blocks.monitoring import aggregation
from fuel.datasets.hdf5 import H5PYDataset
from fuel.schemes import SequentialScheme, ShuffledScheme
from fuel.streams import DataStream

from custom_blocks import PadAndAddMasks, EarlyStopping
from network import *


parser = argparse.ArgumentParser()
parser.add_argument("-c", "--clipping", default=1.0, type=float, help="Step clipping threshold (default: 1)")
parser.add_argument("-t", "--type", default="simple", help="Type of RNN ("+NetworkType.SIMPLE_RNN+", " +
                                                           NetworkType.GRU+" or "+NetworkType.LSTM+")")
parser.add_argument("-e", "--epochs", default=0, type=int, help="Stop after this many epochs (default: 0 -- means it will" +
                                                                " run forever until you cancel manually)")
parser.add_argument("-d", "--dimensions", default="512,512,512", type=str, help="Configure number of layers and dimensions by" +
                                                                                " just enumerating dimensions (e.g. 100, 100 for" +
                                                                         " a two-layered network with dims 100)")
parser.add_argument('-f', '--file', type=str, help='Specifies the data used for training.')
parser.add_argument('-a', '--alphafile', type=str, help='Specifies the location of the alphabet file.')
parser.add_argument('-b', '--batchsize', type=int, help='Set the batch size for training, set 0 for all in one batch and one batch per epoch.')
parser.add_argument('-m', '--memory', type=int, default=1, help='Set to 0 if files should be read from disc, any other number to load into memory.')

args = parser.parse_args()

dimensions = [int(d) for d in args.dimensions.split(',')]
print('Dimensions:', dimensions)

ix2char = pickle.load(open(args.alphafile))

{'fraction_non_alphanumeric': 0.11496598639455782, 'fraction_numerical': 0.013151927437641724, 'mean_word_length': 2.9954710144927534, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1803499', 'n_tokens_mistral': 1187, 'n_tokens_neox': 1150, 'n_words': 386}