<jupyter_start><jupyter_text>Generate Datasets and Train ModelRandom selection for Train and Valid without regard for source file<jupyter_code>! ls
%reload_ext autoreload
%autoreload 2
%matplotlib inline
import config_local
from config_local import *
import numpy as np
import matplotlib.pyplot as plt
import gc

from fastai.vision import *
from fastai.metrics import error_rate

import torch
from torch import nn


import collections
import pprint
from compute_metadata import get_splits, generate_label_file, generate_lists<jupyter_output><empty_output><jupyter_text>Config<jupyter_code>np.random.seed(1234)
random.seed(1234)

binary_image_stats = ([0.5]*3, [0.5]*3)
tfms=[]
size=64
bs=64
workers=4

path = Path() / 'images'<jupyter_output><empty_output><jupyter_text>Generate Train and Valid Label Files<jupyter_code>def generate_label_file(group, image_dir='images', 
                        csv_file='labels.csv', header='fname, label'):
    if csv_file is None:
        results = []
        for v in group.values():
            for label, all_files in v.items():
                for fname in all_files:
                    results.append((os.path.join(image_dir, fname), 1 if label else 0))
        return results
    else:
        with open(os.path.join(image_dir, csv_file), 'wt') as outfile:
            if header:
                print(header, file=outfile)
            for v in group.values():
                for label, all_files in v.items():
                    for fname in all_files:
                        line = '{}, {}'.format(fname, 1 if label else 0)
                        print(line, file=outfile)
train_valid_groups_full = get_splits(image_dir='images', image_file='rp_images_index.json', 
                                     exclude=['_clipped'],
                                     thresh = 7.15)
              
generate_label_file(train_valid_groups_full[0], csv_file='labels.csv')
!ls images/*.csv
#!cat images/labels.csv
lls = ImageList.from_csv(path, 'labels.csv').split_by_rand_pct(0.2).label_from_df(cols=1).transform(tfms, size=size)
db = lls.databunch(bs=bs, num_workers=workers)
my_stats = db.batch_stats()
db = lls.databunch(bs=bs, num_workers=workers).normalize(my_stats)
db
db.batch_stats()
# train = ImageList.from_csv(path, 'train_0.csv')
# valid = ImageList.from_csv(path, 'valid_0.csv')

# lls = ItemLists(path, train, valid).label_from_df(cols=1).transform(tfms, size=size) 
# db = lls.databunch(bs=bs, num_workers=workers)#.normalize(imagenet_stats)<jupyter_output><empty_output><jupyter_text>Examine Results<jupyter_code>print('nClass: {}    classes: {}'.format(db.c, db.classes))
db
im = lls.train.get(-1)
print(len(lls.train), im.size)
im.show()<jupyter_output>490 torch.Size([599, 599])<jupyter_text>Model<jupyter_code>trial_model = nn.Sequential(
    nn.Sequential(
          nn.Conv2d(3,8,5),     # 60 × 60 × 8
          nn.ReLU(),
          nn.AvgPool2d(3, stride=2),    # 29 × 29 × 8
        
          #nn.Dropout(p=0.25),
          nn.Conv2d(8,8,5),         # 25 × 25 × 8
          nn.ReLU(),
          nn.AvgPool2d(3, stride=2),    # 12 × 12 × 8
        
          Flatten()         # 1152
    ),
    # removed model head to compute flatten size
)
trial_learn = Learner(db, trial_model, loss_func = nn.CrossEntropyLoss(), metrics=accuracy)
trial_learn.summary()
del trial_model
trial_learn.destroy()
gc.collect()
mymodel = nn.Sequential(
    nn.Sequential(
          nn.Conv2d(3,8,5),     # 60 × 60 × 8
          nn.ReLU(),
          nn.AvgPool2d(3, stride=2),    # 29 × 29 × 8
        
          #nn.Dropout(p=0.25),
          nn.Conv2d(8,8,5),         # 25 × 25 × 8
          nn.ReLU(),
          nn.AvgPool2d(3, stride=2),    # 12 × 12 × 8
        
          Flatten()         # 1152
    ),
    nn.Sequential(
#           nn.Dropout(p=0.25),
          nn.Linear(1152, 144),
          nn.ReLU(),
          nn.Dropout(p=0.8),
          nn.Linear(144, db.c) 
    )
)
learn = Learner(db, mymodel, loss_func = nn.CrossEntropyLoss(), metrics=accuracy)
learn.summary()
learn.save('initial')<jupyter_output><empty_output><jupyter_text>Train Model<jupyter_code>learn.fit_one_cycle(1, 1e-6)  # learn.fit_one_cycle(1, 0.01)
# learn.save('save-1')
learn.lr_find(start_lr=1e-8, end_lr=1)
learn.recorder.plot()
learn.load('initial')
learn.fit_one_cycle(100, 3e-3)  # learn.fit_one_cycle(1, 0.01)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1324033844042991, 'fraction_numerical': 0.03681683055110908, 'mean_word_length': 2.863957597173145, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '474121', 'license': 'MIT', 'n_tokens_mistral': 1670, 'n_tokens_neox': 1544, 'n_words': 307, 'path': 'CTG_RP_Train_Model_Shuffled.ipynb', 'repo_name': 'williamsdoug/CTG_RP', 'size': '4373'}