<jupyter_start><jupyter_text>Logistic<jupyter_code>import glob
import os
import itertools
import re
from collections import Counter

import numpy
import tensorflow
import keras

import deepometry.model
def collect_pathnames(directories, labels):
    """
    :param directories: List of directories to select samples from. Assumes subdirectories of each directory
                        correspond to class labels. Contents of subdirectories are NPY files containing data
                        of that label.
    :return: List of pathnames.
    """
    pathnames = []

    for directory in directories:
        subdirectories = sorted(glob.glob(os.path.join(directory, "*")))
        
        # transform the files of the same label into directory
        filelist = [glob.glob("{}/*{}*".format(subdirectory,label)) for label in labels for subdirectory in subdirectories ]
        
        subdirectory_pathnames = []
        for i in range(len(labels)):
            a = filelist[i*len(subdirectories):(i+1)*len(subdirectories)]
            subdirectory_pathnames.append( list(itertools.chain.from_iterable(a)) )        

        #nsamples = max([len(pathnames) for pathnames in subdirectory_pathnames])
        nsamples = min([len(pathnames) for pathnames in subdirectory_pathnames])

        pathnames += [list(numpy.random.permutation(pathnames)[:nsamples]) for pathnames in subdirectory_pathnames]

    pathnames = sum(pathnames, [])

    return pathnames


def load(pathnames, labels, dates):
    """
    Load training and target data.
    
    Assumes data is stored in a directory corresponding to some class label.

    :param pathnames: List of image pathnames.
    :param labels: List of class labels.
    :return: Tuple (training, target) data, as NumPy arrays.
    """
    #--- To ignore some class ---#
    #pathnames = [x for x in pathnames if (("renated" in x) and ("oid" not in x))]
    #pathnames = [x for x in pathnames if "oid" not in x]
    
    x = numpy.empty((len(pathnames),) + _shape(pathnames[0]), dtype=numpy.uint8)

    y = numpy.empty((len(pathnames),), dtype=numpy.uint8)
    
    z = numpy.empty((len(pathnames),), dtype=numpy.uint8)
    
    m = numpy.empty((len(pathnames),), dtype=numpy.uint8)

    label_to_index = {label: index for index, label in enumerate(sorted(labels))}
    
    day_to_index = {day: index for index, day in enumerate(day_of_exp)}
    
    label_to_m_index = {"Smooth Disc": 6, "Smooth Sphere": 1, "Crenated Discoid" : 4, "Crenated Disc_" : 5, "Crenated Spheroid": 3, "Crenated Spheres": 2}

    for index, pathname in enumerate(pathnames):
        if os.path.isfile(pathname) == True:

            day = re.search('parsed_data.Bag...(.*)', os.path.dirname(pathname)).group(1)
            
            label = re.search('- (.*)_Total', os.path.basename(pathname) ).group(1)
            if label == 'Crenated Disc':
                label = str(label + "_")

            x[index] = numpy.load(pathname)

            y[index] = label_to_index[label]
            
            z[index] = day_to_index[day]
            
            m[index] = label_to_m_index[label]

    print(x.shape)
    print(y.shape)
    print(z.shape)
    print(Counter(y))   
    print(Counter(m)) 
    return x, y, z, m


def _shape(pathname):
    """
    Infer the shape of the sample data from a single sample.
    
    :param pathname: Path to a sample.
    :return: Sample dimensions.
    """
    return numpy.load(pathname).shape


def get_immediate_subdirectories(a_dir):
    return [name for name in os.listdir(a_dir)
            if os.path.isdir(os.path.join(a_dir, name))]


def save_metadata_label(label,labels,day,days,file):
    with open(file, 'w') as f:
        f.write('Day\tLabel\n')
        for i in range(label.shape[0]):              
            f.write('{}\t{}\n'.format( days[day[i]] , list(sorted(labels))[label[i]]))    

            
def save_metadata_numericday(day,file):
    with open(file, 'w') as f:
        for i in range(day.shape[0]):
            f.write('{}\n'.format( day[i] ))      

            
def get_class_weights(y):
    counter = Counter(y)
    majority = max(counter.values())
    return  {cls: float(majority/count) for cls, count in counter.items()}
test_bags = ['A', 'B', 'D', 'E', 'F', 'H']
test_bags = [str(bag + str(i+1)) for bag in test_bags for i in range(3)]
labels = ["Smooth Disc", "Smooth Sphere", "Crenated Discoid", "Crenated Disc_", "Crenated Spheroid", "Crenated Spheres"]
day_of_exp = [str('D' + str(i)) for i in list(range(1,46))] 
directories = ["/parsed_data/BagA1/",
               "/parsed_data/BagA2/",
               "/parsed_data/BagA3/",
              
               "/parsed_data/BagB1/",
               "/parsed_data/BagB2/",
               "/parsed_data/BagB3/",              
               
               
               "/parsed_data/BagD1/",
               "/parsed_data/BagD2/",
               "/parsed_data/BagD3/", 
                                  
               
               "/parsed_data/BagE1/",
               "/parsed_data/BagE2/",
               "/parsed_data/BagE3/", 
               
               "/parsed_data/BagF1/",
               "/parsed_data/BagF2/",
               "/parsed_data/BagF3/",                     
               
               "/parsed_data/BagH1/",
               "/parsed_data/BagH2/",
               "/parsed_data/BagH3/"                  
              ]<jupyter_output><empty_output><jupyter_text>Sampling for training set<jupyter_code>pathnames = collect_pathnames(directories, labels)
xx, y, z, m = load(pathnames, labels, day_of_exp)
x = xx[:,:,:,0:3:2]
print("x: ", x.shape)
print("y: ", y.shape)
print(Counter(y))
print("z: ",Counter(z))
print("m: ",Counter(m))
class_weight = get_class_weights(y)<jupyter_output><empty_output><jupyter_text>Start training<jupyter_code># build session running on GPU 1
configuration = tensorflow.ConfigProto()
configuration.gpu_options.allow_growth = True
configuration.gpu_options.visible_device_list = "2"
session = tensorflow.Session(config = configuration)

# apply session
keras.backend.set_session(session)
model = deepometry.model.Model(shape=x.shape[1:], units=len(labels))

model.compile()

model.fit(
    x,
    y,
    class_weight = class_weight,
    batch_size=64,
    epochs=512,
    validation_split=0.3,
    verbose=1
)<jupyter_output><empty_output><jupyter_text>End training and save trained model<jupyter_code>model_directory = '/models/resnet'

if not os.path.exists(model_directory):
     os.makedirs(model_directory)
    
model.model.save( os.path.join(model_directory, 'model.h5') )<jupyter_output><empty_output><jupyter_text>Display loss and accuracy curve<jupyter_code>%matplotlib inline
import pandas
import pkg_resources
import matplotlib.pyplot
csv = pandas.read_csv(pkg_resources.resource_filename("deepometry", "deepometry/data/training.csv"))

_, (ax0, ax1) = matplotlib.pyplot.subplots(ncols=2, figsize=(16, 4))

ax0.plot(csv["acc"], c="r")
ax0.plot(csv["val_acc"], c="b")

ax1.plot(csv["loss"][30:], c="r")
ax1.plot(csv["val_loss"][30:], c="b");<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13472378804960541, 'fraction_numerical': 0.009582863585118376, 'mean_word_length': 2.7060052219321147, 'pattern_counts': {'":': 4, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '264121', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2377, 'n_tokens_neox': 2320, 'n_words': 506, 'path': 'STEP2_TrainResNet.ipynb', 'repo_name': 'minh-doan/Deeplearning_LabelFree_RBC', 'size': '7096'}