<jupyter_start><jupyter_text>Food classifierDataset from ETH Zurich: [link](https://www.vision.ee.ethz.ch/datasets_extra/food-101/)<jupyter_code>from fastai import *
from fastai.vision import *
dest_path = pathlib.Path(r"D:\fastai\food")
fname = dest_path/"food-101.tar.gz"<jupyter_output><empty_output><jupyter_text>This somehow seems broken<jupyter_code># url_path = untar_data(url=URLs.FOOD, fname=fname, dest=dest_path); URLs.FOOD # http://data.vision.ee.ethz.ch/cvl/food-101.tar.gz<jupyter_output><empty_output><jupyter_text>So get the file manually, just download using your browser and move the file to where you want --> `fname`<jupyter_code># tarfile.open(fname, "r:gz").extractall(dest_path)
target_dir = dest_path/"food-101/images"; target_dir<jupyter_output><empty_output><jupyter_text>This loads all available images. Which seems to be 1k per class (there are 101 classes). This is a lot! If you want to speed this up and are okay with a smaller sample of images skip the following cell and use the next code cell instead.<jupyter_code># tfms = get_transforms()
# np.random.seed(42)
# data = ImageDataBunch.from_folder(path=target_dir, train=".", valid_pct=.2, bs=32, ds_tfms=tfms, size=224).normalize(imagenet_stats)<jupyter_output><empty_output><jupyter_text>Subsampling images to some smaller number using `n_img_per_dir`<jupyter_code>n_img_per_dir = 50 # number of images to select for each food dir
img_dirs = [] #["nachos", "lobster_roll_sandwich", "ice_cream", "hummus"]
n_img_dirs = 2 if img_dirs is None or len(img_dirs)==0 else len(img_dirs) # if None then all are used. otherwise please choose an integer

food_dirs = {_dir.name: _dir for _dir in target_dir.ls() if _dir.is_dir() if _dir.name != "models"}
if isinstance(n_img_dirs, int) and ((img_dirs is None) or len(img_dirs)==0):
    if n_img_dirs < len(food_dirs):
        img_dirs = np.random.choice(list(food_dirs.keys()), size=n_img_dirs, replace=False)
        
food_dirs = {_name: food_dirs[_name] for _name in img_dirs}
    
food_fnames = {_food: np.random.choice(["{}/{}".format(_img.parent.name, _img.name) for _img in _dir.ls()], 
                                       size=n_img_per_dir, replace=False)
               for _food, _dir in food_dirs.items()}

df = pd.DataFrame({
    "name": np.concatenate([food_fnames[_food] for _food in sorted(food_fnames)]),
    "label": np.concatenate([np.array([_food for _ in food_fnames[_food]]) for _food in sorted(food_fnames)])
})

print("# dirs", len(food_dirs), ", ".join(food_dirs.keys()))
print("# images", len(df))
df.head()
df["label"].value_counts(normalize=True)
tfms = get_transforms()
np.random.seed(42)
data = ImageDataBunch.from_df(path=target_dir, df=df, valid_pct=.2, bs=32, ds_tfms=tfms, size=224).normalize(imagenet_stats)
# data = ImageDataBunch.from_df(path=target_dir, df=df, valid_pct=.2, bs=8, ds_tfms=tfms, size=224).normalize(imagenet_stats)
data.show_batch(rows=3)
print("classes:", data.classes)
print("c", data.c)
print("num train", len(data.train_ds))
print("num valid", len(data.valid_ds))
learn = cnn_learner(data, models.resnet34, metrics=error_rate)
# learn = cnn_learner(data, models.resnet50, metrics=error_rate)
learn.fit_one_cycle(4)
learn.recorder.plot_losses()
learn.save("stage-1-34")
# learn.save("stage-1-50")
learn.export(target_dir/"export.pkl")
learn.load("stage-1-34")
learn.lr_find()
learn.recorder.plot()
learn.unfreeze()
learn.fit_one_cycle(4, max_lr=slice(1e-6, 1e-4))<jupyter_output><empty_output><jupyter_text>Inspection<jupyter_code>learn.show_results(ds_type=DatasetType.Train, rows=2)
learn.show_results(ds_type=DatasetType.Valid, rows=2)
interp = ClassificationInterpretation.from_learner(learn)
interp.plot_top_losses(6)
interp.plot_confusion_matrix()
interp.most_confused(min_val=1)<jupyter_output><empty_output><jupyter_text>Making predictions<jupyter_code>img_fname = r"strawberry_shortcake\13097.jpg" # r"guacamole\4541.jpg" r"strawberry_shortcake\13097.jpg"
img = open_image(target_dir/img_fname)
img
learn2 = load_learner(path=target_dir, file="export.pkl")
i2c = {i: c for c,i in learn2.data.c2i.items()}
pred_class, pred_idx, probs = learn2.predict(img)
topk = 2

top_idx = torch.topk(probs, topk).indices
print("pred_class", pred_class)
print("pred_idx", pred_idx)

print("\nPredictions")
for _ix in top_idx.numpy():
    print("\t{} = {:.2f} %".format(i2c[_ix], probs[_ix]*100))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14512832159890984, 'fraction_numerical': 0.02066772655007949, 'mean_word_length': 6.376884422110553, 'pattern_counts': {'":': 2, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '650503', 'license': 'Apache-2.0', 'n_tokens_mistral': 1748, 'n_tokens_neox': 1689, 'n_words': 317, 'path': 'notebooks/food_classifier.ipynb', 'repo_name': 'eschmidt42/fastai_course', 'size': '4403'}