<jupyter_start><jupyter_text>Install FiftyOne<jupyter_code>!pip install fiftyone
!pip install tensorflow torch torchvision umap-learn
!pip install ipywidgets>=7.5<jupyter_output><empty_output><jupyter_text>Count the sample number of for each label<jupyter_code>dic = dict()
dir.sort()
print(dir[0:10])
cnt = 0
for p in dir:
  with open(path+'/'+p, 'r') as f:
    ss = f.readlines()
    print(cnt)
    cnt += 1
    for s in ss:
      first = s.split()[0]
      if first in dic:
        dic[first] += 1
      else:
        dic[first] = 1
print(dic)
d = {'24': 525, '16': 99, '14': 2402, '6': 431, '23': 378, '2': 2153, '29': 237, '4': 963, '8': 33, '18': 255, '11': 144, '30': 94, '13': 175, '35': 98, '32': 618, '0': 378, '34': 266, '12': 63, '25': 98, '22': 497, '37': 741, '40': 68, '3': 142, '26': 121, '31': 48, '7': 161, '38': 486, '21': 28, '1': 96, '19': 145, '20': 37, '9': 42, '10': 148, '28': 70, '15': 38, '36': 212, '5': 114, '39': 53, '42': 89, '33': 66, '17': 26, '41': 2, '27': 207}
l = ['Apple', 'Bagel', 'Baked goods', 'Banana', 'Beer', 'Bell pepper', 'Bread', 'Broccoli', 'Burrito', 'Cabbage', 'Carrot', 'Cheese', 'Coconut', 'Egg (Food)', 'Fish', 'Garden Asparagus', 'Grapefruit', 'Guacamole', 'Juice', 'Lemon', 'Mango', 'Milk', 'Muffin', 'Mushroom', 'Orange', 'Pasta', 'Peach', 'Pear', 'Pineapple', 'Pizza', 'Pomegranate', 'Potato', 'Pumpkin', 'Radish', 'Seafood', 'Shellfish', 'Shrimp', 'Strawberry', 'Tomato', 'Turkey', 'Watermelon', 'Winter melon', 'Zucchini']

for k, item in sorted(d.items(), key=lambda x: x[1]):
  print(k, item, l[int(k)])

print(l[41])
aa = ['Apple', 'Banana','Bell pepper', 'Broccoli', 'Cabbage', 'Carrot', 'Cheese', 'Coconut', 'Cucumber', 'Egg (Food)', 'Fish', 'Grape', 'Lemon', 'Mango', 'Milk', 'Mushroom', 'Orange', 'Peach', 'Pear', 'Pineapple', 'Potato', 'Pumpkin', 'Seafood', 'Shrimp', 'Strawberry', 'Tomato', 'Watermelon', 'Winter melon']
print(len(aa))
from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Mounted at /content/drive<jupyter_text>Download Dataset from FiftyOne<jupyter_code>import fiftyone as fo
import fiftyone.zoo as foz
dataset = foz.load_zoo_dataset(
    "open-images-v6", 
    split="train", 
    label_types=["detections", "classifications"], 
    classes = ['Apple', 'Banana', 'Bell pepper', 'Broccoli', 'Cabbage', 'Carrot', 'Cheese', 'Coconut', 'Cucumber', 'Egg (Food)', 'Fish', 'Grape', 'Lemon', 'Mango', 'Milk', 'Mushroom', 'Orange', 'Peach', 'Pear', 'Pineapple', 'Potato', 'Pumpkin', 'Seafood', 'Shrimp', 'Strawberry', 'Tomato', 'Watermelon', 'Winter melon'],
    max_samples=3000,
    seed=51,
    shuffle=True,
    dataset_name="food_train",
)<jupyter_output>Downloading split 'train' to '/root/fiftyone/open-images-v6/train' if necessary
Downloading 'https://storage.googleapis.com/openimages/v6/oidv6-train-annotations-bbox.csv' to '/root/fiftyone/open-images-v6/train/labels/detections.csv'
Downloading 3000 images
 100% |████████████████| 3000/3000 [2.6m elapsed, 0s remaining, 20.3 images/s]      
Dataset info written to '/root/fiftyone/open-images-v6/info.json'
Loading 'open-images-v6' split 'train'
 100% |███████████████| 3000/3000 [37.7s elapsed, 0s remaining, 72.5 samples/s]      
Dataset 'food_train' created<jupyter_text>Check Dataset Size in Disk<jupyter_code>!du -h --max-depth=1 /root/fiftyone/open-images-v6/
dataset_val = foz.load_zoo_dataset(
    "open-images-v6", 
    split="validation", 
    label_types=["detections", "classifications"], 
    classes = ['Apple', 'Banana', 'Bell pepper', 'Broccoli', 'Cabbage', 'Carrot', 'Cheese', 'Coconut', 'Cucumber', 'Egg (Food)', 'Fish', 'Grape', 'Lemon', 'Mango', 'Milk', 'Mushroom', 'Orange', 'Peach', 'Pear', 'Pineapple', 'Potato', 'Pumpkin', 'Seafood', 'Shrimp', 'Strawberry', 'Tomato', 'Watermelon', 'Winter melon'],
    max_samples=200,
    seed=51,
    shuffle=True,
    dataset_name="food_val",
)<jupyter_output>Downloading split 'validation' to '/root/fiftyone/open-images-v6/validation' if necessary
Downloading 'https://storage.googleapis.com/openimages/2018_04/validation/validation-images-with-rotation.csv' to '/root/fiftyone/open-images-v6/validation/metadata/image_ids.csv'
Downloading 'https://storage.googleapis.com/openimages/v5/class-descriptions-boxable.csv' to '/root/fiftyone/open-images-v6/validation/metadata/classes.csv'
Downloading 'https://storage.googleapis.com/openimages/2018_04/bbox_labels_600_hierarchy.json' to '/tmp/tmpnyeh8fv2/metadata/hierarchy.json'
Downloading 'https://storage.googleapis.com/openimages/v5/validation-annotations-human-imagelabels-boxable.csv' to '/root/fiftyone/open-images-v6/validation/labels/classifications.csv'
Downloading 'https://storage.googleapis.com/openimages/v5/validation-annotations-bbox.csv' to '/root/fiftyone/open-images-v6/validation/labels/detections.csv'
Downloading 200 images
 100% |██████████████████| 200/200 [10.4s elapsed, 0s remaining, [...]<jupyter_text>Export Dataset<jupyter_code>export_dir = "/content/drive/MyDrive/fridge28"
label_field = "detections"  # for example

# The Dataset or DatasetView to export
dataset_or_view = dataset

# Export the dataset
dataset_or_view.export( 
    export_dir=export_dir,
    dataset_type=fo.types.YOLOv5Dataset,
    label_field=label_field,
    split="train"
)
export_dir = "/content/drive/MyDrive/fridge28"
label_field = "detections"  # for example

# The Dataset or DatasetView to export
dataset_or_view = dataset_val

# Export the dataset
dataset_or_view.export( 
    export_dir=export_dir,
    dataset_type=fo.types.YOLOv5Dataset,
    label_field=label_field,
    split="val"
)<jupyter_output>Directory '/content/drive/MyDrive/fridge28' already exists; export will be merged with existing files
 100% |█████████████████| 200/200 [6.7s elapsed, 0s remaining, 29.8 samples/s]<jupyter_text>Clone Yolov5 Code<jupyter_code>!git clone https://github.com/ultralytics/yolov5  # clone repo
%cd yolov5
%pip install -qr requirements.txt  # install dependencies

import torch
from IPython.display import Image, clear_output  # to display images

clear_output()
print(f"Setup complete. Using torch {torch.__version__} ({torch.cuda.get_device_properties(0).name if torch.cuda.is_available() else 'CPU'})")<jupyter_output>Setup complete. Using torch 1.9.0+cu102 (Tesla K80)<jupyter_text>Copy yaml file<jupyter_code>!cp /content/drive/MyDrive/fridge28/fridge28.yaml /content/drive/MyDrive/yolo5/data
!cp /content/drive/MyDrive/yolo5/yolov5s.pt /content/yolov5
!cp -r /content/drive/MyDrive/yolo5/runs /content/yolov5<jupyter_output><empty_output><jupyter_text>Pip Requirements<jupyter_code>%cd /content/drive/MyDrive/yolo5
%pip install -qr requirements.txt
!rm -r /content/drive/MyDrive/yolo5/runs/train/exp57<jupyter_output><empty_output><jupyter_text>Train the model<jupyter_code>!python train.py --img 640 --batch 50 --epochs 10 --data fridge43.yaml --weights /content/drive/MyDrive/yolo5/runs/train/exp56/weights/last.pt --cache
!python train.py --img 640 --batch 20 --epochs 100 --data fridge20.yaml --weights /content/drive/MyDrive/yolo5/runs/train/exp60/weights/last.pt --cache
!python train.py --img 640 --batch 20 --epochs 1500 --data fridge20.yaml --weights runs/train/exp61/weights/last.pt --cache<jupyter_output>[34m[1mtrain: [0mweights=runs/train/exp61/weights/last.pt, cfg=, data=fridge20.yaml, hyp=data/hyps/hyp.scratch.yaml, epochs=1500, batch_size=20, imgsz=640, rect=False, resume=False, nosave=False, noval=False, noautoanchor=False, evolve=None, bucket=, cache=ram, image_weights=False, device=, multi_scale=False, single_cls=False, adam=False, sync_bn=False, workers=8, project=runs/train, entity=None, name=exp, exist_ok=False, quad=False, linear_lr=False, label_smoothing=0.0, upload_dataset=False, bbox_interval=-1, save_period=-1, artifact_alias=latest, local_rank=-1, freeze=0
[34m[1mgithub: [0m⚠️ WARNING: code is out of date by 145 commits. Use 'git pull' to update or 'git clone https://github.com/ultralytics/yolov5' to download latest.
YOLOv5 🚀 v5.0-378-g79af114 torch 1.9.0+cu111 CUDA:0 (Tesla K80, 11441.1875MB)

[34m[1mhyperparameters: [0mlr0=0.01, lrf=0.2, momentum=0.937, weight_decay=0.0005, warmup_epochs=3.0, warmup_momentum=0.8, warmup_bias_lr=0.1, box=0.05, cls=0.5, cls_pw=[...]<jupyter_text>Detection<jupyter_code>!python detect.py --weights /content/drive/MyDrive/yolo5/runs/train/exp59/weights/best.pt --img 1100 --conf 0.2 --source /content/drive/MyDrive/yolo5/data/images
!du -h  /content/drive/MyDrive/yolo5
!cp -r /content/yolov5 /content/drive/MyDrive/yolo5
!cp /content/drive/MyDrive/fridge20/fridge20.yaml /content/drive/MyDrive/yolo5/data
!cp -r /content/drive/MyDrive/fridge20/labels/train /content/drive/MyDrive/fridge20/labels/val
!cp /content/drive/MyDrive/fridge20/fridge20.yaml /content/drive/MyDrive/yolo5/data
!mv /content/drive/MyDrive/yolo5/data/images/train/* /content/drive/MyDrive/yolo5/data/images
!wget https://www.kaggle.com/vermaavi/food11/download
from utils.plots import plot_results

plot_results('/content/results53.csv', '/content/')
import pandas as pd
r1 = pd.read_csv('/content/fine_tune_result.csv')
r2 = pd.read_csv('/content/results_scratch.csv')
# r1['metrics/mAP_0.5']
r1.columns.values.tolist()
col = r1['     metrics/mAP_0.5']
col2 = r2['     metrics/mAP_0.5']

c1 = r1['      train/obj_loss']
c2 = r2['      train/obj_loss']

d1 = r1['      train/box_loss']
d2 = r2['      train/box_loss']

e1 = r1['      train/cls_loss']
e2 = r2['      train/cls_loss']

# print(col.tolist())

import matplotlib.pyplot as plt
from google.colab.patches import cv2_imshow



plt.subplot(221)
plt.title("mAP@0.5")
plt.plot(col.tolist())
plt.plot(col2.tolist())

plt.subplot(222)
plt.title("train/obj_loss")
plt.plot(c1.tolist())
plt.plot(c2.tolist())

plt.subplot(223)
plt.title("train/box_loss")
plt.plot(d1.tolist())
plt.plot(d2.tolist())

plt.subplot(224)
plt.title("train/cls_loss")
plt.plot(e1.tolist())
plt.plot(e2.tolist())

plt.subplots_adjust(hspace=0.5, wspace=0.3)
plt.legend(labels=['','from scratch'],loc='best')

plt.savefig('/content/res.png')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18142033021682913, 'fraction_numerical': 0.05599761289039188, 'mean_word_length': 5.355878634639697, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 36, 'https://': 9, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '478585', 'license': 'MIT', 'n_tokens_mistral': 4371, 'n_tokens_neox': 3872, 'n_words': 770, 'path': 'SystemCode/Colab/yolov5.ipynb', 'repo_name': 'IRS-PM/IRS-PM-2021-07-05-IS03FT-Group-8-SnapYummy-Cooking-Assistant', 'size': '10054'}