<jupyter_start><jupyter_text>Run 1out_dir  = 'evaluation/first_run/1'Load Modelmodel = models.densenet121(pretrained=True)classifier = nn.Sequential(OrderedDict([                          ('fc1', nn.Linear(1024, 500)),                          ('relu', nn.ReLU()),                          ('fc2', nn.Linear(500, 10)),                          ('output', nn.LogSoftmax(dim=1))                          ]))model.classifier = classifierstate_dict = torch.load('/datadrive2/amit_cvnd/deep-learning-pytorch/models.first.run/cg_e2_lr_001_loss_1_acc_71_checkpoint.pth')model.load_state_dict(state_dict)  Run 2out_dir  = 'evaluation/first_run/2_html_images'Load Modelmodel = models.densenet161(pretrained=True)classifier = nn.Sequential(OrderedDict([                          ('fc1', nn.Linear(2208, 1000)),                          ('relu', nn.ReLU()),                          ('dropout1',nn.Dropout(p=0.4)),                          ('fc2', nn.Linear(1000, 10)),                                                    ('output', nn.LogSoftmax(dim=1))                          ]))model.classifier = classifierstate_dict = torch.load('/datadrive2/amit_cvnd/deep-learning-pytorch/models.first.run/cg_no_dropout_e5_lr_0005_loss_1_acc_80_checkpoint.pth')model.load_state_dict(state_dict)<jupyter_code>## PF RUN 3
out_dir  = 'evaluation/pf_run_3/1_html_images'
#Load Model
model = models.vgg19_bn(pretrained=True)
classifier = nn.Sequential(OrderedDict([
                          ('fc1', nn.Linear(25088, 4096)),
                          ('relu1', nn.ReLU()),
                          ('dropout1',nn.Dropout(p=0.5)),
                          ('fc2', nn.Linear(4096, 1000)), 
                          ('relu2', nn.ReLU()),
                          ('dropout2',nn.Dropout(p=0.5)),
                          ('fc3', nn.Linear(1000, 5)),
                          ('output', nn.LogSoftmax(dim=1))
                          ]))
model.classifier = classifier
state_dict = torch.load('/datadrive2/amit_cvnd/transfer_learning_image_classification/models/pf_run_3/cg_e4_lr_0.0008_loss_0.242_acc_0.911_config1_checkpoint.pth')
model.load_state_dict(state_dict)
def view_classify(img, ps, version="MNIST"):
    ''' Function for viewing an image and it's predicted classes.
    '''
    ps = ps.data.numpy().squeeze()

    fig, ( ax1,ax2) = plt.subplots(figsize=(18,27), ncols=2)
    ax1.imshow(img.resize_(1, 224, 224).numpy().squeeze())
    ax1.axis('off')
   
    ax2.barh(np.arange(10), ps)
    ax2.set_aspect(0.1)
    ax2.set_yticks(np.arange(10))
    ax2.set_yticklabels(['brochure',
                            'charts',
                            'fund_manager_no_keypoints',
                            'fund_managers',
                            'junk',
                            'logos',
                            'logos_will',
                            'no_keypoints',
                            'other_logo',
                            'others'], size='small');
    ax2.set_title('Class Probability')
    ax2.set_xlim(0, 1.1)

    plt.tight_layout()
def find_classes(dir):
    classes = os.listdir(dir)
    classes.sort()
    class_to_idx = {classes[i]: i for i in range(len(classes))}
    return classes, class_to_idx
classes ,_ = find_classes(dir=data_dir + '/test')
failed_image = 0 
# Test out your network!
model.eval()
start = time.time()
# For testing Training & Evaluation data
# path = data_dir +'/test'

# For HTML Data
path = html_dir
files = glob.glob(path + '/**/*.jp*g', recursive=True)
print("Images to classify : ",len(files))
for image_path in files:
    #print(image_path)
    try:
        image = pil_loader(image_path)
    except :
        print("Image did not open : {}".format(image_path))
        failed_image +=1
        continue
    image = test_transforms(image)
    image.unsqueeze_(0) 
    # Calculate the class probabilities (softmax) for img
    with torch.no_grad():
        output = model.forward(image)
    ps = torch.exp(output)
    #print(classes[ps.data.numpy().argmax()])
    pred = classes[ps.data.numpy().argmax()]
    # Create Path to save the model
    output_path=os.path.join(out_dir,pred)
    if not os.path.exists(output_path):
        os.makedirs(output_path)    
    copy(image_path,output_path)    
    
    # Plot the image and probabilities
    #view_classify(image, ps)
    #break
end = time.time()
print("Failed Images count : {}".format(failed_image))
print("Time Taken : {}".format(end-start))<jupyter_output>Images to classify :  146124
Image did not open : /datadrive2/amit_cvnd/ComputerVisionDegree/downlaoded_images/17540.jpeg
Image did not open : /datadrive2/amit_cvnd/ComputerVisionDegree/downlaoded_images/15333.jpeg
Image did not open : /datadrive2/amit_cvnd/ComputerVisionDegree/downlaoded_images/113648.jpeg
Image did not open : /datadrive2/amit_cvnd/ComputerVisionDegree/downlaoded_images/78772.jpeg
Image did not open : /datadrive2/amit_cvnd/ComputerVisionDegree/downlaoded_images/1714.jpeg
Image did not open : /datadrive2/amit_cvnd/ComputerVisionDegree/downlaoded_images/38293.jpeg
Image did not open : /datadrive2/amit_cvnd/ComputerVisionDegree/downlaoded_images/121584.jpeg
Image did not open : /datadrive2/amit_cvnd/ComputerVisionDegree/downlaoded_images/78084.jpeg
Image did not open : /datadrive2/amit_cvnd/ComputerVisionDegree/downlaoded_images/117511.jpeg
Image did not open : /datadrive2/amit_cvnd/ComputerVisionDegree/downlaoded_images/77248.jpeg
Image did not open : /datadrive2/amit_c[...]
{'fraction_non_alphanumeric': 0.12541133455210238, 'fraction_numerical': 0.04040219378427788, 'mean_word_length': 2.564169381107492, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '345255', 'license': 'MIT', 'n_tokens_mistral': 1976, 'n_tokens_neox': 1808, 'n_words': 286, 'path': '007_transfer_learning/pf_inference.ipynb', 'repo_name': 'amitbcp/deep_learning_projects', 'size': '5470'}