<commit_before>"""An example of how to use your own dataset to train a classifier that recognizes people.
"""
# MIT License
# 
# Copyright (c) 2016 <NAME>
# 
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# 
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# 
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import numpy as np
import argparse
import facenet
import os
import sys
import math
import time
import tensorflow.contrib.slim as slim

def main(args):
  
    with tf.Graph().as_default():
      
        with tf.Session() as sess:
            
            np.random.seed(seed=args.seed)
            tf.set_random_seed(args.seed)
            global_step = tf.Variable(0, trainable=False)
            
            if args.use_split_dataset:
                dataset = facenet.get_dataset(args.data_dir)
                train_set, test_set = split_dataset(dataset, args.min_nrof_images_per_class, args.nrof_train_images_per_class)
            else:
                train_set = facenet.get_dataset(args.data_dir)
                test_set = facenet.get_dataset(args.test_data_dir)

            # Check that there are at least one training image per class
            for cls in train_set:
                assert(len(cls.image_paths)>0, 'There must be at least one image for each class in the training set')            
            for cls in test_set:
                assert(len(cls.image_paths)>0, 'There must be at least one image for each class in the test set')            

            # Check that the classes are the same in the two sets
            class_names = [ cls.name for cls in train_set]
            test_class_names = [ cls.name for cls in test_set]
            assert(test_class_names==class_names, 'Classes used for training and testing must be identical')
                 
            train_images, train_labels = facenet.get_image_paths_and_labels(train_set)
            test_images, test_labels = facenet.get_image_paths_and_labels(test_set)
            nrof_classes = len(train_set)
            
            print('Number of classes: %d' % len(train_set))
            print('Number of train images: %d' % len(train_images))
            print('Number of test images: %d' % len(test_images))
            
            # Load the model
            facenet.load_model(args.model)
            
            # Get input and output tensors
            images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
            embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
            phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
        
            embedding_size = embeddings.get_shape()[1]
        
            # Run forward pass to calculate embeddings
            for phase, paths in [['train', train_images], ['test', test_images]]:
                print('Runnning forward pass on %s images' % phase)
                nrof_images = len(paths)
                nrof_batches_per_epoch = int(math.ceil(1.0*nrof_images / args.batch_size))
                emb_array = np.zeros((nrof_images, embedding_size))
                for i in range(nrof_batches_per_epoch):
                    start_index = i*args.batch_size
                    end_index = min((i+1)*args.batch_size, nrof_images)
                    paths_batch = paths[start_index:end_index]
                    images = facenet.load_data(paths_batch, False, False, args.image_size)
                    feed_dict = { images_placeholder:images, phase_train_placeholder:False }
                    emb_array[start_index:end_index,:] = sess.run(embeddings, feed_dict=feed_dict)
                
                embeddings_filename = 'embeddings_%s.npy' % phase
                print('Saving %s' % embeddings_filename)
                np.save(embeddings_filename, emb_array)
                    
            emb_array_train = np.load('embeddings_train.npy')
            emb_array_test = np.load('embeddings_test.npy')
            labels = tf.placeholder(tf.int32, [None], 'labels')
            learning_rate_placeholder = tf.placeholder(tf.float32, name='learning_rate')
            logits = slim.fully_connected(embeddings, nrof_classes, activation_fn=None, 
                    weights_initializer=tf.truncated_normal_initializer(stddev=0.1), 
                    weights_regularizer=slim.l2_regularizer(args.weight_decay),
                    scope='Logits', reuse=False)
            
            nrof_batches_per_epoch_train = int(math.ceil(1.0*len(train_labels) / args.batch_size))
            learning_rate = tf.train.exponential_decay(learning_rate_placeholder, global_step,
                args.learning_rate_decay_epochs*nrof_batches_per_epoch_train, args.learning_rate_decay_factor, staircase=True)
            tf.summary.scalar('learning_rate', learning_rate)
    
            # Calculate the average cross entropy loss across the batch
            cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
                labels=labels, logits=logits, name='cross_entropy_per_example')
            cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
            tf.add_to_collection('losses', cross_entropy_mean)
            predictions = tf.nn.softmax(logits, name='predictions')
            
            # Calculate the total losses
            regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
            total_loss = tf.add_n([cross_entropy_mean] + regularization_losses, name='total_loss')
    
            # Build a Graph that trains the model with one batch of examples and updates the model parameters
            train_op = facenet.train(total_loss, global_step, args.optimizer, 
                learning_rate, args.moving_average_decay, tf.global_variables(), False)
            
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())

            validate_every_n_epochs = 10
            for epoch in range(args.max_nrof_epochs):
                # Run training for one epoch 
                train_or_test_epoch(sess, total_loss, regularization_losses, embeddings, labels, predictions, train_op, learning_rate_placeholder, 
                    epoch, emb_array_train, np.array(train_labels), args.batch_size, 0.01, True, True)
                if epoch % validate_every_n_epochs == 0 or epoch==args.max_nrof_epochs-1:
                    # Test the classifier
                    accuracy = train_or_test_epoch(sess, total_loss, regularization_losses, embeddings, labels, predictions, None, learning_rate_placeholder, 
                        epoch, emb_array_test, np.array(test_labels), args.batch_size, 0.0, False, True)
                    print('Test accuracy: %.3f' % accuracy)
                    
            if args.output_filename:
                # Freeze and store model
                import freeze_graph
                print('Freezing and storing model')<commit_msg>Removed storing embeddings to file<commit_after>"""An example of how to use your own dataset to train a classifier that recognizes people.
"""
# MIT License
# 
# Copyright (c) 2016 <NAME>
# 
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# 
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# 
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import numpy as np
import argparse
import facenet
import os
import sys
import math
import time
import tensorflow.contrib.slim as slim

def main(args):
  
    with tf.Graph().as_default():
      
        with tf.Session() as sess:
            
            np.random.seed(seed=args.seed)
            tf.set_random_seed(args.seed)
            global_step = tf.Variable(0, trainable=False)
            
            if args.use_split_dataset:
                dataset = facenet.get_dataset(args.data_dir)
                train_set, test_set = split_dataset(dataset, args.min_nrof_images_per_class, args.nrof_train_images_per_class)
            else:
                train_set = facenet.get_dataset(args.data_dir)
                test_set = facenet.get_dataset(args.test_data_dir)

            # Check that there are at least one training image per class
            for cls in train_set:
                assert(len(cls.image_paths)>0, 'There must be at least one image for each class in the training set')            
            for cls in test_set:
                assert(len(cls.image_paths)>0, 'There must be at least one image for each class in the test set')            

            # Check that the classes are the same in the two sets
            class_names = [ cls.name for cls in train_set]
            test_class_names = [ cls.name for cls in test_set]
            assert(test_class_names==class_names, 'Classes used for training and testing must be identical')
                 
            train_images, train_labels = facenet.get_image_paths_and_labels(train_set)
            test_images, test_labels = facenet.get_image_paths_and_labels(test_set)
            nrof_classes = len(train_set)
            
            print('Number of classes: %d' % len(train_set))
            print('Number of train images: %d' % len(train_images))
            print('Number of test images: %d' % len(test_images))
            
            # Load the model
            facenet.load_model(args.model)
            
            # Get input and output tensors
            images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
            embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
            phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
            embedding_size = embeddings.get_shape()[1]
            
            # Run forward pass to calculate embeddings
            embeddings_dict = {}
            for phase, paths in [['train', train_images], ['test', test_images]]:
                print('Runnning forward pass on %s images' % phase)
                nrof_images = len(paths)
                nrof_batches_per_epoch = int(math.ceil(1.0*nrof_images / args.batch_size))
                emb_array = np.zeros((nrof_images, embedding_size))
                for i in range(nrof_batches_per_epoch):
                    start_index = i*args.batch_size
                    end_index = min((i+1)*args.batch_size, nrof_images)
                    paths_batch = paths[start_index:end_index]
                    images = facenet.load_data(paths_batch, False, False, args.image_size)
                    feed_dict = { images_placeholder:images, phase_train_placeholder:False }
                    emb_array[start_index:end_index,:] = sess.run(embeddings, feed_dict=feed_dict)
                embeddings_dict[phase] = emb_array
                    
            labels = tf.placeholder(tf.int32, [None], 'labels')
            learning_rate_placeholder = tf.placeholder(tf.float32, name='learning_rate')
            logits = slim.fully_connected(embeddings, nrof_classes, activation_fn=None, 
                    weights_initializer=tf.truncated_normal_initializer(stddev=0.1), 
                    weights_regularizer=slim.l2_regularizer(args.weight_decay),
                    scope='Logits', reuse=False)
            
            nrof_batches_per_epoch_train = int(math.ceil(1.0*len(train_labels) / args.batch_size))
            learning_rate = tf.train.exponential_decay(learning_rate_placeholder, global_step,
                args.learning_rate_decay_epochs*nrof_batches_per_epoch_train, args.learning_rate_decay_factor, staircase=True)
            tf.summary.scalar('learning_rate', learning_rate)
    
            # Calculate the average cross entropy loss across the batch
            cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
                labels=labels, logits=logits, name='cross_entropy_per_example')
            cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
            tf.add_to_collection('losses', cross_entropy_mean)
            predictions = tf.nn.softmax(logits, name='predictions')
            
            # Calculate the total losses
            regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
            total_loss = tf.add_n([cross_entropy_mean] + regularization_losses, name='total_loss')
    
            # Build a Graph that trains the model with one batch of examples and updates the model parameters
            train_op = facenet.train(total_loss, global_step, args.optimizer, 
                learning_rate, args.moving_average_decay, tf.global_variables(), False)
            
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())

            validate_every_n_epochs = 10
            for epoch in range(args.max_nrof_epochs):
                # Run training for one epoch 
                train_or_test_epoch(sess, total_loss, regularization_losses, embeddings, labels, predictions, train_op, learning_rate_placeholder, 
                    epoch, embeddings_dict['train'], np.array(train_labels), args.batch_size, 0.01, True, True)
                if epoch % validate_every_n_epochs == 0 or epoch==args.max_nrof_epochs-1:
                    # Test the classifier
                    accuracy = train_or_test_epoch(sess, total_loss, regularization_losses, embeddings, labels, predictions, None, learning_rate_placeholder, 
                        epoch, embeddings_dict['test'], np.array(test_labels), args.batch_size, 0.0, False, True)
                    print('Test accuracy: %.3f' % accuracy)
                    
            if args.output_filename:
                # Freeze and store model
                import freeze_graph
                print('Freezing and storing model')
{'fraction_non_alphanumeric': 0.0741302972802024, 'fraction_numerical': 0.004174573055028463, 'mean_word_length': 2.4964617425917734, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3712104', 'n_tokens_mistral': 4458, 'n_tokens_neox': 4298, 'n_words': 1166}