<commit_before>__author__ = 'denis'

from teafacto.data.sptensor import SparseTensor
from datetime import datetime
from matplotlib import pyplot as plt
import pickle, os, pkgutil, pandas as pd, numpy as np
from IPython import embed

from teafacto.rnnkm import RNNTFSGDSM
from teafacto.xrnnkm import RNNEKMM, RNNEOKMM, AddEKMM, MulEKMM, RNNEKMSM, AutoRNNEKMSM, RNNESMSM, RNNESMSMShort
from teafacto.optimizers import SGD, RMSProp, AdaDelta
from teafacto.rnn import GRU, LSTM, RNU


def loaddata(file):
    file = os.path.join(os.path.dirname(__file__), file)
    data = pickle.load(open(file))
    return data

def loaddic(file):
    file = os.path.join(os.path.dirname(__file__), file)
    return pickle.load(open(file))

def loadmeta(dfile):
    dfile = os.path.join(os.path.dirname(__file__), dfile)
    meta = pickle.load(open(dfile))
    return meta

def getsavepath():
    dfile = os.path.join(os.path.dirname(__file__), "../../models/%s.%s" %
                         (os.path.splitext(os.path.basename(__file__))[0], datetime.now().strftime("%Y-%m-%d=%H:%M")))
    return dfile

def run():
    # params
    dims = 100 # 100
    innerdims = dims
    negrate = 10
    numbats = 100 # 100
    epochs = 200 #20
    wreg = 0.0000001
    lr = 0.01/numbats #0.0001 # for SGD
    lr2 = 1.
    evalinter = 1
    rho = 0.95
    occ = 0.3


    ############"
    dims = 20
    innerdims = 50
    lr = 1./numbats # 8

    start = datetime.now()

    # get the data and split
    datafileprefix = "../../data/quotes/"
    tensorfile = "quotes.np.pkl"
    vocabsize = 128 # ascii characters

    data = loaddata(datafileprefix+tensorfile)

    trainX = data[:, :-1]
    labels = data[:, 1:]

    print "source data loaded in %f seconds" % (datetime.now() - start).total_seconds()

    # train model
    print "training model"
    start = datetime.now()
    model = RNNESMSMShort(dim=dims, vocabsize=vocabsize, maxiter=epochs, wreg=wreg, numbats=numbats, negrate=negrate)\
                .autosave.normalize \
            + SGD(lr=lr) \
            + GRU(dim=dims, innerdim=innerdims, wreg=wreg)
    print "model %s defined in %f" % (model.__class__.__name__, (datetime.now() - start).total_seconds())
    start = datetime.now()
    err = model.train(trainX, labels, evalinter=evalinter)
    print "model trained in %f" % (datetime.now() - start).total_seconds()
    plt.plot(err, "r")
    plt.show(block=True)

    print model.predict([[34, 65]], [67])
    print model.predict([[34, 65]], [34])

    #embed()


###################### FUNCTIONS FOR INSPECTION ##########################

def extend(instance, new_class):
    instance.__class__ = type(
                '%s_extended_with_%s' % (instance.__class__.__name__, new_class.__name__),
                (instance.__class__, new_class),
                {}
            )


def loadmodel(path):
    return RNNTFSGDSM.load(path)

<commit_msg>testing implementations with RNN LM<commit_after>__author__ = 'denis'

from teafacto.data.sptensor import SparseTensor
from datetime import datetime
from matplotlib import pyplot as plt
import pickle, os, pkgutil, pandas as pd, numpy as np
from IPython import embed

from teafacto.rnnkm import RNNTFSGDSM
from teafacto.xrnnkm import RNNEKMM, RNNEOKMM, AddEKMM, MulEKMM, RNNEKMSM, AutoRNNEKMSM, RNNESMSM, RNNESMSMShort
from teafacto.optimizers import SGD, RMSProp, AdaDelta
from teafacto.rnn import GRU, LSTM, RNU


def loaddata(file):
    file = os.path.join(os.path.dirname(__file__), file)
    data = pickle.load(open(file))
    return data

def loaddic(file):
    file = os.path.join(os.path.dirname(__file__), file)
    return pickle.load(open(file))

def loadmeta(dfile):
    dfile = os.path.join(os.path.dirname(__file__), dfile)
    meta = pickle.load(open(dfile))
    return meta

def getsavepath():
    dfile = os.path.join(os.path.dirname(__file__), "../../models/%s.%s" %
                         (os.path.splitext(os.path.basename(__file__))[0], datetime.now().strftime("%Y-%m-%d=%H:%M")))
    return dfile

def run():
    # params
    dims = 100 # 100
    innerdims = dims
    negrate = 10
    numbats = 100 # 100
    epochs = 200 #20
    wreg = 0.0000001
    lr = 0.01/numbats #0.0001 # for SGD
    lr2 = 1.
    evalinter = 1
    rho = 0.95
    occ = 0.3


    ############"
    dims = 20
    innerdims = 50
    lr = 1./numbats # 8

    start = datetime.now()

    # get the data and split
    datafileprefix = "../../data/quotes/"
    tensorfile = "quotes.np.pkl"
    vocabsize = 128 # ascii characters

    data = loaddata(datafileprefix+tensorfile)

    trainX = data[:, :-1]
    labels = data[:, 1:]

    print "source data loaded in %f seconds" % (datetime.now() - start).total_seconds()

    # train model
    print "training model"
    start = datetime.now()
    model = RNNESMSMS(dim=dims, vocabsize=vocabsize, maxiter=epochs, wreg=wreg, numbats=numbats, negrate=negrate)\
                .autosave.normalize \
            + SGD(lr=lr) \
            + GRU(dim=dims, innerdim=innerdims, wreg=wreg)
    print "model %s defined in %f" % (model.__class__.__name__, (datetime.now() - start).total_seconds())
    start = datetime.now()
    err = model.train(trainX, labels, evalinter=evalinter)
    print "model trained in %f" % (datetime.now() - start).total_seconds()
    plt.plot(err, "r")
    plt.show(block=True)

    print model.predict([[34, 65]], [67])
    print model.predict([[34, 65]], [34])

    #embed()


###################### FUNCTIONS FOR INSPECTION ##########################

def extend(instance, new_class):
    instance.__class__ = type(
                '%s_extended_with_%s' % (instance.__class__.__name__, new_class.__name__),
                (instance.__class__, new_class),
                {}
            )


def loadmodel(path):
    return RNNTFSGDSM.load(path)


{'fraction_non_alphanumeric': 0.150207182320442, 'fraction_numerical': 0.0231353591160221, 'mean_word_length': 2.959671907040328, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1430152', 'n_tokens_mistral': 2190, 'n_tokens_neox': 2031, 'n_words': 466}