<jupyter_start><jupyter_text>import from google cloud storage<jupyter_code>import tensorflow as tf
import pandas as pd
import numpy as np
import os
! python --version<jupyter_output>Python 3.6.9<jupyter_text>Drive mounting<jupyter_code># allow colab to use gcloud storage

from google.colab import auth
auth.authenticate_user()
!mkdir gcs
# install gcs FUSE to mount files

!echo "deb http://packages.cloud.google.com/apt gcsfuse-bionic main" > /etc/apt/sources.list.d/gcsfuse.list
!curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
!apt -qq update
!apt -qq install gcsfuse
!gcsfuse --implicit-dirs [redacted] gcs
%cd gcs/cloud_mirror
# test if it works
test = pd.read_csv("test.csv")
print(test)

from IPython.display import Image
Image(filename='train/train/train/00/00b32bd5ba9cdd7c2f11e3975b3e54fa.jpg')<jupyter_output>filename  category
0      fd663cf2b6e1d7b02938c6aaae0a32d2.jpg        43
1      c7fd77508a8c355eaab0d4e10efd6b15.jpg        43
2      127f3e6d6e3491b2459812353f33a913.jpg        43
3      5ca4f2da11eda083064e6c36f37eeb81.jpg        43
4      46d681a542f2c71be017eef6aae23313.jpg        43
...                                     ...       ...
12181  5ba958eacb23cd7d1673bad4dae55784.jpg        43
12182  efbe41a1c2b666b70e337e438559808b.jpg        43
12183  79fdaa5ac5ba10dbe8004cabd8c35eb3.jpg        43
12184  ac3d136124617637a05ba66694e381ef.jpg        43
12185  7ef61d7cfbad9cfe2db4f64560e3dddd.jpg        43

[12186 rows x 2 columns]<jupyter_text>Traininghttps://www.tensorflow.org/tutorials/load_data/imageshttps://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/keras/classification.ipynbscrollTo=DLdCchMdCaWQ<jupyter_code>IMG_HEIGHT = 871
IMG_WIDTH = 871
CLASS_NAMES = ['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14',
               '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29',
               '30', '31', '32', '33', '34', '35', '36', '37', '38', '39', '40', '41']
LABEL_COLUMN = 'category'
AUTOTUNE = tf.data.experimental.AUTOTUNE

BATCH_SIZE = 32

image_count = 2683 # for 00 only

STEPS_PER_EPOCH = np.ceil(image_count/BATCH_SIZE)
list_ds = tf.data.Dataset.list_files(str ('train/train/train/00/*.jpg'))

for f in list_ds.take(5):
  print(f.numpy())
def get_label(file_path):
  # convert the path to a list of path components
  parts = tf.strings.split(file_path, os.path.sep)
  # The second to last is the class-directory
  return parts[-2] == CLASS_NAMES
def decode_img(img):
  # convert the compressed string to a 3D uint8 tensor
  img = tf.image.decode_jpeg(img, channels=3)
  # Use `convert_image_dtype` to convert to floats in the [0,1] range.
  img = tf.image.convert_image_dtype(img, tf.float32)
  # resize the image to the desired size.
  return tf.image.resize(img, [IMG_HEIGHT, IMG_WIDTH])
def process_path(file_path):
  label = get_label(file_path)
  # load the raw data from the file as a string
  img = tf.io.read_file(file_path)
  img = decode_img(img)
  return img, label
# Set `num_parallel_calls` so multiple images are loaded/processed in parallel.
labeled_ds = list_ds.map(process_path, num_parallel_calls=AUTOTUNE)

for image, label in labeled_ds.take(1):
  print("Image shape: ", image.numpy().shape)
  print("Label: ", label.numpy())<jupyter_output><empty_output><jupyter_text>Attempt to store as TFRecord?https://www.tensorflow.org/tutorials/load_data/tfrecordwalkthrough_reading_and_writing_image_data<jupyter_code>def tf_serialize_example(f0,f1):
  tf_string = tf.py_function(
    serialize_example,
    (f0,f1),  # pass these args to the above function.
    tf.string)      # the return type is `tf.string`.
  return tf.reshape(tf_string, ()) # The result is a scalar

tf_serialize_example(f0,f1,f2,f3)
filename = '00.tfrecord'
writer = tf.data.experimental.TFRecordWriter(filename)
writer.write(labeled_ds)

filenames = [labeled_ds]
raw_dataset = tf.data.TFRecordDataset(filenames)
raw_dataset<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12264150943396226, 'fraction_numerical': 0.09508440913604767, 'mean_word_length': 4.308300395256917, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '507550', 'license': 'Unlicense', 'n_tokens_mistral': 1768, 'n_tokens_neox': 1560, 'n_words': 324, 'path': '2_ProductDetection/pdt_detection_github.ipynb', 'repo_name': 'hkrsmk/beepboop-shopee-code-league-2020', 'size': '4028'}