<jupyter_start><jupyter_text>GPU Version to run on MARCCThis notebook is assummed to be launched from the top level of the AutomaticLP github repo (where the notebook file is)This also assummes that you've already cloned the tensorflow object detection projecthttps://github.com/tensorflow/models (since you need to do this to add the python path to  the bashrc file which has to be done before launching jupyter labRun with the LPproject kernel **If you have not followed the instructions in install.readme quit this lab / notebook and follow thoes steps first !!!**  Tests that Tensorflow GPU is installed & this notebook can see the GPU These don't need to be run everytime but are here as a sanity / debugging check<jupyter_code># test that tensorflow can access a GPU
import tensorflow as tf
tf.test.is_built_with_cuda()
from tensorflow.python.client import device_lib
print(device_lib.list_local_devices())
import tensorflow as tf
# Make sure at least 1 GPU device is listed
print ('Default GPU Device: {}'.format(tf.test.gpu_device_name()))
# Expected Result is 0
!ml cuda/9.0
!export CUDA_VISIBLE_DEVICES=1
!echo ${CUDA_VISIBLE_DEVICES}<jupyter_output><empty_output><jupyter_text>Generating Datasets The ScriptForBashFile generates both the training and test data and TFRecords from the image files<jupyter_code>%run ScriptForBashFile<jupyter_output><empty_output><jupyter_text>Train<jupyter_code># This makes the output write to the file
# https://stackoverflow.com/questions/45200375/stdout-redirect-from-jupyter-notebook-is-landing-in-the-terminal
import sys
old_stdout = sys.stdout
sys.stdout = open('trainingLog.txt', 'w')

%cd models/research
%run object_detection/legacy/train.py --logtostderr --train_dir=../../trainingOutput/ --pipeline_config_path=../../configs/faster_rcnn_resnet50_coco.config 
%cd ../..

# Put the cell output back to normal
sys.stdout = old_stdout
%matplotlib inline
%run viewTraining.py<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code># We want to run Evaluation on the latest checkpoint 
# this code extracts the step number of the most recent checkpoint saved

import os
from os import listdir
import re
import shutil

%cd trainingOutput
checkpointList = listdir()
%cd ..

numbers = []
for filename in checkpointList:
    match = re.search("^model.ckpt-", filename)
    if not match is None:
        filename = filename.split(".")
        filename = filename[1].split("-")
        numbers.append(filename[1])

lastCheckpoint = max(numbers)
print(lastCheckpoint)
CHECKPOINT_PREFIX = "../../trainingOutput/model.ckpt-"+str(lastCheckpoint)

# check to see if the infrence_graph folder exists - if it does ask if it should be deleted
# if the directory exists the TF code won't work
if(os.path.isdir("inference_graph")):
    choice = input("The Infrence Graph Directory already exists - to continue it needs to be deleted, is that OK (y/n)")
    if 'y' in choice.lower():
        shutil.rmtree('inference_graph')
    else:
        exit()
    
%cd models/research
%run object_detection/export_inference_graph.py --input_type image_tensor --pipeline_config_path ../../configs/faster_rcnn_resnet50_coco.config --trained_checkpoint_prefix $CHECKPOINT_PREFIX --output_directory ../../inference_graph
%cd ../..

%cd models/research
%run object_detection/inference/infer_detections.py --input_tfrecord_paths=../../FromScratch/TFRecordEval.tfrecord --inference_graph=../../inference_graph/frozen_inference_graph.pb --output_tfrecord_path=../../TFExamples.tfrecord
%cd ../..

%cd tf_object_detection_cm
%run confusion_matrix.py --detections_record=../TFExamples.tfrecord --label_map=../FromScratch/data/label_map.pbtxt --output_path=confusion_matrix.csv
%cd ..<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09927749531709927, 'fraction_numerical': 0.005084292213005084, 'mean_word_length': 4.849765258215962, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '136044', 'license': 'CNRI-Python', 'n_tokens_mistral': 1171, 'n_tokens_neox': 1118, 'n_words': 350, 'path': 'GPUTraining.ipynb', 'repo_name': 'Hriste/AutomaticLP', 'size': '3737'}