<jupyter_start><jupyter_text>Data Exploration notebooks for Fingers datasets  Environment setup<jupyter_code>!curl https://course.fast.ai/setup/colab | bash
!pip install fastai2 --upgrade
!pip install fastcore --upgrade
!pip install nbdev --upgrade
from google.colab import drive
drive.mount('/content/drive')
from fastai2.vision.all import *
escdrive = lambda x : x.as_posix().replace(' ','\ ')
gdrive = Path('/content/drive/My Drive/fastai_v4')
config = Config()
data_path = config.d['data_path']
archive_path = config.d['archive_path']
model_path = config.d['model_path']
(data_path, archive_path,model_path)<jupyter_output><empty_output><jupyter_text>Prepare data<jupyter_code>!ls {escdrive(gdrive/'data')}
!mkdir -p {archive_path}
!cp {escdrive(gdrive/'data'/'koryakinp_fingers.zip')} {archive_path}
!ls {archive_path}/*
!unzip {archive_path}/koryakinp_fingers.zip
!ls -ald {archive_path}/*
# clean extra copies and move data to data folder
!rm -rf train
!rm -rf test
!mv fingers data/koryakinp_fingers<jupyter_output><empty_output><jupyter_text>Explore data and folder structure<jupyter_code>path = Path(data_path)/'koryakinp_fingers'
img_files = get_image_files(path)
img_files[:5]
len(img_files)
img1 = PILImage.create(img_files[0]);img1
img1.shape<jupyter_output><empty_output><jupyter_text>Build datablock<jupyter_code>pat = r'/[^/]+_([0-9RL]+).png$'
labeller = RegexLabeller(pat)
labeller(img_files[0])
def is_valid(o):
  return parent_label(o) == 'test'
is_valid(img_files[0])
is_valid(Path(str(img_files[0]).replace('train','test')))
fing_dblock = DataBlock(
    blocks=(ImageBlock, CategoryBlock),
    get_items = get_image_files,
    get_y = labeller,
    splitter = FuncSplitter(is_valid),
    item_tfms = Resize(128)
)
fing_dblock.summary(path)<jupyter_output>Setting-up type transforms pipelines
Collecting items from /root/.fastai/data/koryakinp_fingers
Found 21600 items
2 datasets of sizes 18000,3600
Setting up Pipeline: PILBase.create
Setting up Pipeline: RegexLabeller -> Categorize

Building one sample
  Pipeline: PILBase.create
    starting from
      /root/.fastai/data/koryakinp_fingers/train/66952cc5-8b4f-425b-b890-eaa5ad667a07_2L.png
    applying PILBase.create gives
      PILImage mode=RGB size=128x128
  Pipeline: RegexLabeller -> Categorize
    starting from
      /root/.fastai/data/koryakinp_fingers/train/66952cc5-8b4f-425b-b890-eaa5ad667a07_2L.png
    applying RegexLabeller gives
      2L
    applying Categorize gives
      TensorCategory(4)

Final sample: (PILImage mode=RGB size=128x128, TensorCategory(4))


Setting up after_item: Pipeline: Resize -> ToTensor
Setting up before_batch: Pipeline: 
Setting up after_batch: Pipeline: IntToFloatTensor

Building one batch
Applying item_tfms to the first sample:
  Pipeline: Resize -> ToT[...]<jupyter_text>Build Datasets<jupyter_code>fing_dsets = fing_dblock.datasets(path, verbose=True)
list(fing_dsets.vocab)<jupyter_output><empty_output><jupyter_text>Build DataFrames from Datasets<jupyter_code>valid_df = pd.DataFrame({
    'inputs':pd.Series(fing_dsets.valid.items,dtype='object'),
    'labels':pd.Series([labeller(o) for o in fing_dsets.valid.items], dtype='category'),
    'split': pd.Series([parent_label(o) for o in fing_dsets.valid.items], dtype='category'),
})
train_df = pd.DataFrame({
    'inputs':pd.Series(fing_dsets.train.items,dtype='object'),
    'labels':pd.Series([labeller(o) for o in fing_dsets.train.items], dtype='category'),
    'split': pd.Series([parent_label(o) for o in fing_dsets.train.items], dtype='category'),
})
all_df = pd.concat([valid_df, train_df]); (all_df.head(), all_df.tail())
valid_df['labels'].head()
all_df.groupby(['split','labels']).count()<jupyter_output><empty_output><jupyter_text>Build Dataloaders<jupyter_code># create new dblock with aug transforms
fing_dblock2 = fing_dblock.new(batch_tfms=aug_transforms(do_flip=False))
fing_dls = fing_dblock2.dataloaders(path)
fing_dls.show_batch(max_n=12, nrows=3)<jupyter_output><empty_output><jupyter_text>Build Model<jupyter_code>learn = cnn_learner(fing_dls, resnet34, metrics=error_rate)
learn.fine_tune(4)
learn.recorder.plot_loss()
learn.export()
learn.save('stage-1')<jupyter_output><empty_output><jupyter_text>Save Model and Clean Data to GDrive<jupyter_code>!mkdir -p {escdrive(gdrive/'models'/'koryakinp_fingers')}
!cp models/stage-1.pth {escdrive(gdrive/'models'/'koryakinp_fingers')}
!cp export.pkl {escdrive(gdrive/'models'/'koryakinp_fingers')}
dataset ='koryakinp_fingers'
(dataset, data_path,path)
!tar -czf {(Path(data_path)/(dataset + '.tgz')).as_posix()}  -C {Path(data_path).as_posix()} {path.as_posix()}  
!cp data/koryakinp_fingers.tgz {escdrive(gdrive/'data')}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13738019169329074, 'fraction_numerical': 0.020873269435569754, 'mean_word_length': 6.905723905723906, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 40, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '358479', 'license': 'MIT', 'n_tokens_mistral': 1863, 'n_tokens_neox': 1766, 'n_words': 296, 'path': 'ExploreKoryakinFingers.ipynb', 'repo_name': 'edward-h-81/fastai-butch-landingin', 'size': '4695'}