<commit_before>#!/usr/bin/env python
from argparse import ArgumentParser
import numpy as np

from keras.preprocessing import sequence
from keras.optimizers import RMSprop
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten, add_shared_layer, Layer, Merge
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.utils.layer_utils import model_summary
import pickle<commit_msg>started train.py<commit_after>#!/usr/bin/env python
import numpy as np
from pybedtools import BedTool
import h5py

from keras.preprocessing import sequence
from keras.optimizers import RMSprop
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten, add_shared_layer, Layer, Merge
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.utils.layer_utils import model_summary
import pickle

# Standard library imports
import sys
import os
import errno
import argparse


class Identity(Layer):
    def __init__(self, **kwargs):
        super(Identity, self).__init__(**kwargs)

    def get_output(self, train):
        x = self.get_input(train)
        return x


def make_model():
    print 'Stacking layers'

    forward = Sequential()
    reverse = Sequential()
    model = Sequential()

    forward.add(Identity(input_shape=(1000, 4,)))
    reverse.add(Identity(input_shape=(1000, 4,)))
    inputs = [forward, reverse]

    add_shared_layer(Convolution1D(input_dim=4, input_length=1000, nb_filter=320, filter_length=26, border_mode="valid",
                                   activation="relu", subsample_length=1), inputs)

    add_shared_layer(MaxPooling1D(pool_length=975, stride=975), inputs)

    add_shared_layer(Flatten(), inputs)

    add_shared_layer(Dropout(0.2), inputs)

    add_shared_layer(Dense(input_dim=320, output_dim=1, activation="sigmoid"), inputs)

    model.add(Merge(inputs, mode='ave'))

    print 'Compiling model'
    model.compile(loss='binary_crossentropy', optimizer='rmsprop', class_mode="binary")

    model_summary(forward)

    return model


def train(train_data, valid_data, test_data, output_dir):
    model = make_model()

    print 'Running at most 30 epochs'

    checkpointer = ModelCheckpoint(filepath="train3_bestmodel.hdf5", verbose=1, save_best_only=True)
    earlystopper = EarlyStopping(monitor='val_loss', patience=30, verbose=1)

    history = model.fit([X_train_seq, X_train_seq_rc], y_train, batch_size=100, nb_epoch=30, shuffle=True,
                        show_accuracy=True, validation_data=([X_valid_seq, X_valid_seq_rc], y_valid),
                        callbacks=[checkpointer, earlystopper])

    print 'Saving final model'
    model.save_weights('train3_finalmodel.hdf5')

    print 'Saving history'
    history_file = open("train3_history.pkl", 'wb')
    pickle.dump(history.history, history_file)
    history_file.close()

    tresults = model.evaluate([X_test_seq, X_test_seq_rc], y_test, show_accuracy=True)

    print tresults


def load_data(input_dir, valid_chroms, test_chroms):
    print 'Loading data'

    f = h5py.File(input_dir + '/data.hdf5')
    windows = BedTool(input_dir + '/windows.bed.gz')
    windows_chroms = [window.chrom for window in windows]

    # Valid
    boolean_list = [chrom in valid_chroms for chrom in windows_chroms]
    valid_seq = f['x_seq'][boolean_list]
    valid_bws = f['x_bws'][boolean_list]
    valid_y = f['y'][boolean_list]
    valid_data = (valid_seq, valid_bws, valid_y)

    # Test
    boolean_list = [chrom in test_chroms for chrom in windows_chroms]
    test_seq = f['x_seq'][boolean_list]
    test_bws = f['x_bws'][boolean_list]
    test_y = f['y'][boolean_list]
    test_data = (test_seq, test_bws, test_y)

    # Train
    valid_test_chroms = valid_chroms + test_chroms
    boolean_list = [chrom not in valid_test_chroms for chrom in windows_chroms]
    train_seq = f['x_seq'][boolean_list]
    train_bws = f['x_bws'][boolean_list]
    train_y = f['y'][boolean_list]
    train_data = (train_seq, train_bws, train_y)
    f.close()

    return train_data, valid_data, test_data


def make_argument_parser():
    """
    Creates an ArgumentParser to read the options for this script from
    sys.argv
    """
    parser = argparse.ArgumentParser(
        description="Generate training, validation, and testing sets.",
        epilog='\n'.join(__doc__.strip().split('\n')[1:]).strip(),
        formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('--inputdir', '-i', type=str, required=True,
                        help='Folder containing data generated by makeData.py')
    parser.add_argument('--genome', '-g', type=str, required=True,
                        help='Genome size file.')
    parser.add_argument('--validchroms', '-v', type=str, required=False, nargs='+',
                        default=['chr11'],
                        help='Chromosome(s) to set aside for validation.')
    parser.add_argument('--testchroms', '-t', type=str, required=False, nargs='+',
                        default=['chr10'],
                        help='Chromosome(s) to set aside for testing.')
    parser.add_argument('--bigwig', '-w', type=str, required=False, nargs='*',
                        help='1D bigWig files (optional).')
    parser.add_argument('--recurrent', '-r', action='store_true',
                        help='Save data as long sequences for fully recurrent models.')
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('-o', '--outputdir', type=str,
                       help='The output directory. Causes error if the directory already exists.')
    group.add_argument('-oc', '--outputdirc', type=str,
                       help='The output directory. Will overwrite if directory already exists.')
    return parser


def main():
    """
    The main executable function
    """
    parser = make_argument_parser()
    args = parser.parse_args()

    input_dir = args.inputdir
    valid_chroms = args.validchroms
    test_chroms = args.testchroms
    bigwigs = args.bigwig
    excludes = args.exclude
    recurrent = args.recurrent

    if args.outputdir is None:
        clobber = True
        output_dir = args.outputdirc
    else:
        clobber = False
        output_dir = args.outputdir

    try:  # adapted from dreme.py by <NAME>
        os.makedirs(output_dir)
    except OSError as exc:
        if exc.errno == errno.EEXIST:
            if not clobber:
                print >> sys.stderr, ('output directory (%s) already exists '
                                      'but you specified not to clobber it') % output_dir
                sys.exit(1)
            else:
                print >> sys.stderr, ('output directory (%s) already exists '
                                      'so it will be clobbered') % output_dir

    train_data, valid_data, test_data = load_data(input_dir, valid_chroms, test_chroms)

    train(train_data, valid_data, test_data, output_dir)


if __name__ == '__main__':
    """
    See module-level docstring for a description of the script.
    """
    main()
{'fraction_non_alphanumeric': 0.10723192019950124, 'fraction_numerical': 0.009005264616237185, 'mean_word_length': 3.2340175953079178, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '270889', 'n_tokens_mistral': 2317, 'n_tokens_neox': 2198, 'n_words': 531}