<commit_before>import numpy as np
import matplotlib.pyplot as plt
from pylab import *
import os
import sys
import pickle
from keras.optimizers import SGD
from keras.callbacks import *
from keras.objectives import *
from keras.models import load_model
import keras.backend as K
#import keras.utils.visualize_util as vis_util

from models import Atrous_DenseNet
from utils.loss_function import *
from utils.metrics import *
from utils.SegDataGenerator import *
import time


def train(batch_size, nb_epoch, lr_base, lr_power, weight_decay, nb_classes, model_name, train_file_path, val_file_path,
            data_dir, label_dir, target_size=None, batchnorm_momentum=0.9, resume_training=False):
    if target_size:
        input_shape = target_size + (3,)
    else:
        input_shape = (None, None, 3)
    batch_shape = (batch_size,) + input_shape

    ###########################################################
    current_dir = os.path.dirname(os.path.realpath(__file__))
    save_path = os.path.join(current_dir, 'Models/' + model_name)
    if os.path.exists(save_path) == False:
        os.mkdir(save_path)
    ################learning rate scheduler####################
    def lr_scheduler(epoch):
        '''if lr_dict.has_key(epoch):
            lr = lr_dict[epoch]
            print 'lr: %f' % lr'''
        lr = lr_base * ((1 - float(epoch)/nb_epoch) ** lr_power)
        print('lr: %f' % lr)
        return lr
    scheduler = LearningRateScheduler(lr_scheduler)
    ######################## tfboard ###########################
    tfboard = TensorBoard(log_dir=os.path.join(save_path, 'logs'))

    ####################### make model ########################
    checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5')

    model = globals()[model_name](weight_decay=weight_decay, input_shape=input_shape, batch_momentum=batchnorm_momentum)
    sgd = SGD(lr=lr_base, momentum=0.9)
    model.compile(loss = softmax_sparse_crossentropy_ignoring_last_label, optimizer=sgd, metrics=[sparse_accuracy_ignoring_last_label])
    if resume_training:
        model.load_weights(checkpoint_path, by_name=True)
    model_path = os.path.join(save_path, "model.json")
    # save model structure
    f = open(model_path, 'w')
    model_json = model.to_json()
    f.write(model_json)
    f.close
    img_path = os.path.join(save_path, "model.png")
    #vis_util.plot(model, to_file=img_path, show_shapes=True)
    model.summary()

    #################### checkpoint saver#######################
    checkpoint = ModelCheckpoint(filepath=os.path.join(save_path, 'checkpoint_weights.hdf5'), save_weights_only=True)#.{epoch:d}

    # set data generator and train
    train_datagen = SegDataGenerator(zoom_range=[0.5, 2.0], zoom_maintain_shape=True,
                                    crop_mode='random', crop_size=target_size, #pad_size=(505, 505),
                                    rotation_range=0., shear_range=0, horizontal_flip=True,
                                    channel_shift_range=20.,
                                    fill_mode='constant', label_cval=255)
    def get_file_len(file_path):
        fp = open(file_path)
        lines = fp.readlines()
        fp.close()
        return len(lines)
    history = model.fit_generator(
                                generator=train_datagen.flow_from_directory(
                                    file_path=train_file_path, data_dir=data_dir, data_suffix='.jpg',
                                    label_dir=label_dir, label_suffix='.png', nb_classes=nb_classes,
                                    target_size=target_size, color_mode='rgb',
                                    batch_size=batch_size, shuffle=True,
                                    #save_to_dir='Images/'
                                ),
                                samples_per_epoch=get_file_len(train_file_path),
                                nb_epoch=nb_epoch,
                                callbacks=[scheduler, checkpoint],
				                nb_worker=4,
                                #validation_data=val_datagen.flow_from_directory(
                                #    file_path=val_file_path, data_dir=data_dir, data_suffix='.jpg',
                                #    label_dir=label_dir, label_suffix='.png',nb_classes=nb_classes,
                                #    target_size=target_size, color_mode='rgb',
                                #    batch_size=batch_size, shuffle=False
                                #),
                                #nb_val_samples = 64
                            )

    model.save_weights(save_path+'/model.hdf5')

if __name__ == '__main__':
    model_name = 'AtrousFCN_Resnet50_16s'
    # model_name = 'Atrous_DenseNet'
    # model_name = 'DenseNet_FCN'
    batch_size = 8
    batchnorm_momentum = 0.95
    nb_epoch = 250
    lr_base = 0.01 * (float(batch_size) / 16)
    lr_power = 0.9
    resume_training=False
    weight_decay = 0.0001/2
    nb_classes = 21
    target_size = (320, 320)
    train_file_path = os.path.expanduser('~/datasets/VOC2012/VOCdevkit/VOC2012/ImageSets/Segmentation/train.txt') #Data/VOClarge/VOC2012/ImageSets/Segmentation
    val_file_path   = os.path.expanduser('~/datasets/VOC2012/VOCdevkit/VOC2012/ImageSets/Segmentation/val64.txt')
    data_dir        = os.path.expanduser('~/datasets/VOC2012/VOCdevkit/VOC2012/JPEGImages')
    label_dir       = os.path.expanduser('~/datasets/VOC2012/VOCdevkit/VOC2012/SegmentationClass')
    config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))
    session = tf.Session(config=config)<commit_msg>train.py import all models<commit_after>import numpy as np
import matplotlib.pyplot as plt
from pylab import *
import os
import sys
import pickle
from keras.optimizers import SGD
from keras.callbacks import *
from keras.objectives import *
from keras.models import load_model
import keras.backend as K
#import keras.utils.visualize_util as vis_util

from models import *
from utils.loss_function import *
from utils.metrics import *
from utils.SegDataGenerator import *
import time


def train(batch_size, nb_epoch, lr_base, lr_power, weight_decay, nb_classes, model_name, train_file_path, val_file_path,
            data_dir, label_dir, target_size=None, batchnorm_momentum=0.9, resume_training=False):
    if target_size:
        input_shape = target_size + (3,)
    else:
        input_shape = (None, None, 3)
    batch_shape = (batch_size,) + input_shape

    ###########################################################
    current_dir = os.path.dirname(os.path.realpath(__file__))
    save_path = os.path.join(current_dir, 'Models/' + model_name)
    if os.path.exists(save_path) == False:
        os.mkdir(save_path)
    ################learning rate scheduler####################
    def lr_scheduler(epoch):
        '''if lr_dict.has_key(epoch):
            lr = lr_dict[epoch]
            print 'lr: %f' % lr'''
        lr = lr_base * ((1 - float(epoch)/nb_epoch) ** lr_power)
        print('lr: %f' % lr)
        return lr
    scheduler = LearningRateScheduler(lr_scheduler)
    ######################## tfboard ###########################
    tfboard = TensorBoard(log_dir=os.path.join(save_path, 'logs'))

    ####################### make model ########################
    checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5')

    model = globals()[model_name](weight_decay=weight_decay, input_shape=input_shape, batch_momentum=batchnorm_momentum)
    sgd = SGD(lr=lr_base, momentum=0.9)
    model.compile(loss = softmax_sparse_crossentropy_ignoring_last_label, optimizer=sgd, metrics=[sparse_accuracy_ignoring_last_label])
    if resume_training:
        model.load_weights(checkpoint_path, by_name=True)
    model_path = os.path.join(save_path, "model.json")
    # save model structure
    f = open(model_path, 'w')
    model_json = model.to_json()
    f.write(model_json)
    f.close
    img_path = os.path.join(save_path, "model.png")
    #vis_util.plot(model, to_file=img_path, show_shapes=True)
    model.summary()

    #################### checkpoint saver#######################
    checkpoint = ModelCheckpoint(filepath=os.path.join(save_path, 'checkpoint_weights.hdf5'), save_weights_only=True)#.{epoch:d}

    # set data generator and train
    train_datagen = SegDataGenerator(zoom_range=[0.5, 2.0], zoom_maintain_shape=True,
                                    crop_mode='random', crop_size=target_size, #pad_size=(505, 505),
                                    rotation_range=0., shear_range=0, horizontal_flip=True,
                                    channel_shift_range=20.,
                                    fill_mode='constant', label_cval=255)
    def get_file_len(file_path):
        fp = open(file_path)
        lines = fp.readlines()
        fp.close()
        return len(lines)
    history = model.fit_generator(
                                generator=train_datagen.flow_from_directory(
                                    file_path=train_file_path, data_dir=data_dir, data_suffix='.jpg',
                                    label_dir=label_dir, label_suffix='.png', nb_classes=nb_classes,
                                    target_size=target_size, color_mode='rgb',
                                    batch_size=batch_size, shuffle=True,
                                    #save_to_dir='Images/'
                                ),
                                samples_per_epoch=get_file_len(train_file_path),
                                nb_epoch=nb_epoch,
                                callbacks=[scheduler, checkpoint],
				                nb_worker=4,
                                #validation_data=val_datagen.flow_from_directory(
                                #    file_path=val_file_path, data_dir=data_dir, data_suffix='.jpg',
                                #    label_dir=label_dir, label_suffix='.png',nb_classes=nb_classes,
                                #    target_size=target_size, color_mode='rgb',
                                #    batch_size=batch_size, shuffle=False
                                #),
                                #nb_val_samples = 64
                            )

    model.save_weights(save_path+'/model.hdf5')

if __name__ == '__main__':
    # model_name = 'AtrousFCN_Resnet50_16s'
    # model_name = 'Atrous_DenseNet'
    model_name = 'DenseNet_FCN'
    batch_size = 8
    batchnorm_momentum = 0.95
    nb_epoch = 250
    lr_base = 0.01 * (float(batch_size) / 16)
    lr_power = 0.9
    resume_training=False
    weight_decay = 0.0001/2
    nb_classes = 21
    target_size = (320, 320)
    train_file_path = os.path.expanduser('~/datasets/VOC2012/VOCdevkit/VOC2012/ImageSets/Segmentation/train.txt') #Data/VOClarge/VOC2012/ImageSets/Segmentation
    val_file_path   = os.path.expanduser('~/datasets/VOC2012/VOCdevkit/VOC2012/ImageSets/Segmentation/val64.txt')
    data_dir        = os.path.expanduser('~/datasets/VOC2012/VOCdevkit/VOC2012/JPEGImages')
    label_dir       = os.path.expanduser('~/datasets/VOC2012/VOCdevkit/VOC2012/SegmentationClass')
    config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))
    session = tf.Session(config=config)
{'fraction_non_alphanumeric': 0.13583892617449664, 'fraction_numerical': 0.017897091722595078, 'mean_word_length': 2.5133605784344546, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '690631', 'n_tokens_mistral': 3672, 'n_tokens_neox': 3436, 'n_words': 539}