<jupyter_start><jupyter_text>Calculating the Hash, Shape, Mode, Length and Ratio of each image<jupyter_code>def getImageMetaData(file_path):
    with Image.open(file_path) as img:
        img_hash = imagehash.phash(img)
        return img.size, img.mode, img_hash

def get_train_input():
    train_input = df.copy()
        
    m = train_input.path.apply(lambda x: getImageMetaData(x))
    train_input["Hash"] = [str(i[2]) for i in m]
    train_input["Shape"] = [i[0] for i in m]
    train_input["Mode"] = [str(i[1]) for i in m]
    train_input["Length"] = train_input["Shape"].apply(lambda x: x[0]*x[1])
    train_input["Ratio"] = train_input["Shape"].apply(lambda x: x[0]/x[1])
    
    
    img_counts = train_input.path.value_counts().to_dict()
    train_input["Id_Count"] = train_input.path.apply(lambda x: img_counts[x])
    return train_input

train_input = get_train_input()

train_input.head()
train_input1 = train_input[['Hash']] # Getting the Hash from the new data
train_input1['New']=1 # Creating a dummy column 1
train_input1.head()
train_input1.shape
train_input2 = train_input1.groupby('Hash').count().reset_index() # Grouping the column by Hash to aggregate at Hash level
train_input2.shape
train_input2.head()
train_input2.New.value_counts()
train_input2 = train_input2[train_input2['New']>1] # Filtering those instances where the hash is occuring multiple timeskk
train_input2.shape # Checking the shape
train_input2.head()
train_input2 = train_input2.sort_values('Hash') # Sorting the data by Hash 
train_input2.head(5) # Checking the top 5 records
train_input[train_input['Hash']=='808d0f3513f33f2d'] # FIltering only one Hash from master data
bad_ones = [] # those duplicates which have conflicting diagnosis categories, gotta remove them from training
for hash_code in train_input2.Hash.values:
    diagnosises = train_input[train_input["Hash"] == hash_code]['diagnosis'].value_counts()
    if diagnosises.shape[0] > 1:
        bad_ones.append(hash_code)
        
print('Total bad ones:', len(bad_ones))
train_input.head()
bad_train_input = train_input[train_input["Hash"].apply(lambda x: True if x in bad_ones else False)] # from Hash to original df
bad_train_input.head()

bad_train_input.index
df = pd.read_csv('../data/train.csv')
bad_indices = bad_train_input.index 
np.save('../data/bad_train_indices.npy', bad_indices)

#np.save('../data/duplicate_test_ids.npy', train_input2.Hash.values)
df.head()
bad_ones
df[df.id_code.apply(lambda x: print(x))]
df.drop(
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from IPython import display
import time

%matplotlib inline

PATH = "../input/train_images/1632c4311fc9.png"
image = mpimg.imread(PATH) # images are color images
plt.imshow(image);
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from IPython import display
import time

%matplotlib inline

PATH = "../input/train_images/a75bab2463d4.png"
image = mpimg.imread(PATH) # images are color images
plt.imshow(image);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12093023255813953, 'fraction_numerical': 0.019933554817275746, 'mean_word_length': 5.221074380165289, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '449691', 'license': 'MIT', 'n_tokens_mistral': 1061, 'n_tokens_neox': 1035, 'n_words': 248, 'path': 'notebooks/similar-duplicate-images-in-aptos-data.ipynb', 'repo_name': 'pyaf/Image-classification-pytorch', 'size': '3010'}