<jupyter_start><jupyter_text><jupyter_code># Last updated: 2021-10-01
# check the allocated hardware
! nvidia-smi<jupyter_output><empty_output><jupyter_text>Imports<jupyter_code>import os<jupyter_output><empty_output><jupyter_text>Variables<jupyter_code># Set the absolute path to mount storage (either GCS bucket or G-Drive) on the Colab machine 
LOCAL_MOUNT_POINT = "/content/gcs"

# Set the name of the GCS bucket (optional; only needed if using GCS)
GCS_BUCKET = "my-gcs-bucket"

# If downloading a model from the TF2 Detection Model Zoo (TF Zoo: https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/tf2_detection_zoo.md), 
# specify the URL and location to save on the Colab machine:
ZOO_URL = "http://download.tensorflow.org/models/object_detection/tf2/20200711/efficientdet_d0_coco17_tpu-32.tar.gz"

LOCAL_MODEL_FOLDER = "/content/tf-models"

# TFODAPI project folder where training checkpoints will also be written and the specify the location of the TFODAPI pipeline.config
# NOTE: you may want to keep this on GCS or Drive so that training checkpoint are preserved in the event of Colab timeout/crash/etc.
PROJECT_FOLDER = "/content/gcs/my-project-folder/"

PIPELINE_CONFIG = "/content/gcs/my-project-folder/pipeline.config"<jupyter_output><empty_output><jupyter_text>Installation<jupyter_code>import os
import pathlib

# Clone the tensorflow models repository if it doesn't already exist
if "models" in pathlib.Path.cwd().parts:
  while "models" in pathlib.Path.cwd().parts:
    os.chdir('..')
elif not pathlib.Path('models').exists():
  !git clone --depth 1 https://github.com/tensorflow/models
# Install the Object Detection API
%%bash
cd models/research/
protoc object_detection/protos/*.proto --python_out=.
cp object_detection/packages/tf2/setup.py .
python -m pip install .<jupyter_output><empty_output><jupyter_text>Data Loading from GCS<jupyter_code>%%bash
echo "deb http://packages.cloud.google.com/apt gcsfuse-bionic main" > /etc/apt/sources.list.d/gcsfuse.list
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
apt -qq update
apt -qq install gcsfuse
from google.colab import auth
auth.authenticate_user()
# create mount point
%%bash -s "$LOCAL_MOUNT_POINT" "$GCS_BUCKET"
mkdir $1
gcsfuse $2 $1
# OPTIONAL: This could be a good time to copy data to the Colab machine for faster processing
%%bash
# mkdir /content/my-local-data
# cp -r /content/gcs/my-cloud-data/* /content/my-local-data<jupyter_output><empty_output><jupyter_text>Data Loading from Drive<jupyter_code>from google.colab import drive
drive.mount(LOCAL_MOUNT_POINT)
# OPTIONAL: This could be a good time to copy data to the Colab machine for faster processing
%%bash
# mkdir /content/my-local-data
# cp -rf /content/gcs/MyDrive/data-to-copy/* /content/project/<jupyter_output><empty_output><jupyter_text>Download from the Tensorflow Detection Model Zoo<jupyter_code>LOCAL_ZOO_MODEL_ZIP = os.path.join(LOCAL_MODEL_FOLDER, ZOO_URL.split('/')[-1])
%%bash -s "$LOCAL_MODEL_FOLDER" "$ZOO_URL" "$LOCAL_ZOO_MODEL_ZIP"
mkdir $1
wget -P $1 $2
tar -xvf $3 -C $1
rm $3<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>TRAIN_LOG = os.path.join(PROJECT_FOLDER, "train.nohup")
# Note that we are using a modified "_max.py" script that preserves many training checkpoints. Change the path to your local TFODAPI installation for the basic TFODAPI training script.
%%bash -s "$PROJECT_FOLDER" "$PIPELINE_CONFIG" "$TRAIN_LOG"
! nohup python /content/gcs/model_main_tf2_max.py \
--model_dir $1 \
--pipeline_config_path $2 \
--alsologtostderr > $3<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11191135734072022, 'fraction_numerical': 0.011357340720221607, 'mean_word_length': 5.518050541516246, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '632775', 'license': 'MIT', 'n_tokens_mistral': 1315, 'n_tokens_neox': 1264, 'n_words': 327, 'path': 'Basic_Training_with_TFODAPI.ipynb', 'repo_name': 'rosswin/tfodapi-tools', 'size': '3610'}