<jupyter_start><jupyter_text>Clean up for your vacation images<jupyter_code># Define global variables, are explained below

IMAGES = './images/' # path to the images
IMAGES_PREPROCESSED = './preprocessed/' # not used at the moment
IMAGES_PROCESSED = './processed/' # output destination
HASH_DIM = (8, 8)
import glob
import os
import shutil

import cv2
import matplotlib.pyplot as plt
%matplotlib inline
import numpy as np
import pandas as pd
import scipy.stats as sts
from sklearn.linear_model import LogisticRegression

import spring_cleaning as sc # outsourced code
# Load images and meta data

dict_files = sc.read_files(path=IMAGES, ext='jpg')

df = pd.DataFrame(dict_files, columns=['file', 'creation_date'])
df['creation_date'] = pd.to_numeric(df['creation_date'], downcast='integer')

# For ranking of images, we need some intrinsic measure for consecutiveness
# 'Creation Date' of images is a good measure
# 'File Name' as well, if pictures are taken from one and the same camera

# alternatively, sort by Filename: df.sort_values(by=['File'])
df = df.sort_values(by=['creation_date'])
df['timediff_value'] = df['creation_date'].diff(-1).abs()

# Load images
df['gray_images'] = [sc.resize_img(sc.read_img(IMAGES + cur_file, read_type='gray')) 
                                            for cur_file in df['file'].values]
df['hsv_images'] = [sc.resize_img(sc.read_img(IMAGES + cur_file, read_type='hsv')) 
                                            for cur_file in df['file'].values]
df['rgb_images'] = [sc.resize_img(sc.read_img(IMAGES + cur_file, read_type='rgb')) 
                                            for cur_file in df['file'].values]

df = df.assign(target = df['file'].str.split(n=0, pat='_', expand=True).loc[:, 0])
df['target'], _ = pd.factorize(df['target'])
df['target'] = df['target'] + 1

df[[c for c in df.columns if not c.endswith('_images')]].head()<jupyter_output><empty_output><jupyter_text>Bootstrap to augment sample size<jupyter_code>res = sc.bootstrap_data(df, n_runs=10)
res[0].columns<jupyter_output><empty_output><jupyter_text>Method 1: Estimate mean values  For Hashing<jupyter_code>summary = {}

nonsimilar_hash = sc.return_dist(res, _type='nonsimilar', 
                                 target_col='target', comp_col='hash_value')
similar_hash = sc.return_dist(res, _type='similar', 
                              target_col='target', comp_col='hash_value')

sc.plot_distributions(similar_hash, nonsimilar_hash, 
                      labels=('Hashing Score', 'Density'), title='Hash values')

print('Mean value for groups: {:.2f} +- {:.2f}'.format(
    similar_hash.mean(), similar_hash.std()))
print('Mean value for non-groups: {:.2f} +- {:.2f}\n'.format(
    nonsimilar_hash.mean(), nonsimilar_hash.std()))

summary.update({'hash_1':
        sc.performance_report(similar_hash, nonsimilar_hash, limit=56.33)})<jupyter_output><empty_output><jupyter_text>For Advanced Hashing<jupyter_code>nonsimilar_hash_adv = sc.return_dist(res, _type='nonsimilar', 
                                 target_col='target', comp_col='hash_value_adv')
similar_hash_adv = sc.return_dist(res, _type='similar', 
                              target_col='target', comp_col='hash_value_adv')

sc.plot_distributions(similar_hash_adv, nonsimilar_hash_adv, 
                      labels=('Advanced Hashing Score', 'Density'), title='Hash values')

print('Mean value for groups: {:.2f} +- {:.2f}'.format(
    similar_hash_adv.mean(), similar_hash_adv.std()))
print('Mean value for non-groups: {:.2f} +- {:.2f}\n'.format(
    nonsimilar_hash_adv.mean(), nonsimilar_hash_adv.std()))

summary.update({'hash_adv_1':
        sc.performance_report(similar_hash_adv, nonsimilar_hash_adv, limit=58.72)})<jupyter_output><empty_output><jupyter_text>For Bhattacharyya correlation<jupyter_code>nonsimilar_bhatt = sc.return_dist(res, _type='nonsimilar', 
                                 target_col='target', comp_col='bhattacharyya_corr')
similar_bhatt = sc.return_dist(res, _type='similar', 
                                  target_col='target', comp_col='bhattacharyya_corr')

sc.plot_distributions(similar_bhatt, nonsimilar_bhatt, 
                   labels=('Bhattacharyya Score', 'Density'),
                   title='Bhattacharyya correlation')

print('Mean value for groups: {:.2f} +- {:.2f}'.format(
    similar_bhatt.mean(), similar_bhatt.std()))
print('Mean value for non-groups: {:.2f} +- {:.2f}\n'.format(
    nonsimilar_bhatt.mean(), nonsimilar_bhatt.std()))

summary.update({'bhattacharyya_1': 
            sc.performance_report(similar_bhatt, nonsimilar_bhatt, limit=0.61)})<jupyter_output><empty_output><jupyter_text>For plain correlation<jupyter_code>nonsimilar_corr = sc.return_dist(res, _type='nonsimilar', 
                                 target_col='target', comp_col='correl_corr')
similar_corr = sc.return_dist(res, _type='similar', 
                                  target_col='target', comp_col='correl_corr')

sc.plot_distributions(similar_corr, nonsimilar_corr, 
                   labels=('Correlation Score', 'Density'),
                   title='Plain correlation')

print('Mean value for groups: {:.2f} +- {:.2f}'.format(
    similar_corr.mean(), similar_corr.std()))
print('Mean value for non-groups: {:.2f} +- {:.2f}\n'.format(
    nonsimilar_corr.mean(), nonsimilar_corr.std()))

summary.update({'correl_1': 
                sc.performance_report(similar_corr, nonsimilar_corr, limit=0.8)})<jupyter_output><empty_output><jupyter_text>For average hashing (cv2)<jupyter_code>nonsimilar_hash_avg = sc.return_dist(res, _type='nonsimilar', 
                                 target_col='target', comp_col='avg_hash_value_cmp')
similar_hash_avg = sc.return_dist(res, _type='similar', 
                              target_col='target', comp_col='avg_hash_value_cmp')

sc.plot_distributions(similar_hash_avg, nonsimilar_hash_avg, 
                      labels=('Hashing Score', 'Density'), title='Avg hash values')

print('Mean value for groups: {:.2f} +- {:.2f}'.format(
    similar_hash_avg.mean(), similar_hash_avg.std()))
print('Mean value for non-groups: {:.2f} +- {:.2f}\n'.format(
    nonsimilar_hash_avg.mean(), nonsimilar_hash_avg.std()))

summary.update({'avg_hash_1':
        sc.performance_report(similar_hash_avg, nonsimilar_hash_avg, limit=0.89)})<jupyter_output><empty_output><jupyter_text>For block mean hashing<jupyter_code>nonsimilar_hash_block = sc.return_dist(res, _type='nonsimilar', 
                                 target_col='target', comp_col='block_hash_value_cmp')
similar_hash_block = sc.return_dist(res, _type='similar', 
                              target_col='target', comp_col='block_hash_value_cmp')

sc.plot_distributions(similar_hash_block, nonsimilar_hash_block, 
                      labels=('Hashing Score', 'Density'), title='Block hash values')

print('Mean value for groups: {:.2f} +- {:.2f}'.format(
    similar_hash_block.mean(), similar_hash_block.std()))
print('Mean value for non-groups: {:.2f} +- {:.2f}\n'.format(
    nonsimilar_hash_block.mean(), nonsimilar_hash_block.std()))

summary.update({'block_hash_1':
        sc.performance_report(similar_hash_block, nonsimilar_hash_block, limit=0.94)})<jupyter_output><empty_output><jupyter_text>For perceptual hashing<jupyter_code>nonsimilar_hash_p = sc.return_dist(res, _type='nonsimilar', 
                                 target_col='target', comp_col='phash_value_cmp')
similar_hash_p = sc.return_dist(res, _type='similar', 
                              target_col='target', comp_col='phash_value_cmp')

sc.plot_distributions(similar_hash_p, nonsimilar_hash_p, 
                      labels=('Hashing Score', 'Density'), title='Perceptual hash values')

print('Mean value for groups: {:.2f} +- {:.2f}'.format(
    similar_hash_p.mean(), similar_hash_p.std()))
print('Mean value for non-groups: {:.2f} +- {:.2f}\n'.format(
    nonsimilar_hash_p.mean(), nonsimilar_hash_p.std()))

summary.update({'perceptual_hash_1':
        sc.performance_report(similar_hash_p, nonsimilar_hash_p, limit=0.84)})<jupyter_output><empty_output><jupyter_text>For Marr Hildreth hashing<jupyter_code>nonsimilar_hash_mh = sc.return_dist(res, _type='nonsimilar', 
                                 target_col='target', comp_col='marr_hildreth_hash_value_cmp')
similar_hash_mh = sc.return_dist(res, _type='similar', 
                              target_col='target', comp_col='marr_hildreth_hash_value_cmp')

sc.plot_distributions(similar_hash_mh, nonsimilar_hash_mh, 
                      labels=('Hashing Score', 'Density'), title='Marr Hildreth hash values')

print('Mean value for groups: {:.2f} +- {:.2f}'.format(
    similar_hash_mh.mean(), similar_hash_mh.std()))
print('Mean value for non-groups: {:.2f} +- {:.2f}\n'.format(
    nonsimilar_hash_mh.mean(), nonsimilar_hash_mh.std()))

summary.update({'marrhildreth_hash_1':
        sc.performance_report(similar_hash_mh, nonsimilar_hash_mh, limit=0.86)})<jupyter_output><empty_output><jupyter_text>For radial variance hashing<jupyter_code>nonsimilar_hash_rv = sc.return_dist(res, _type='nonsimilar', 
                                 target_col='target', comp_col='radial_variance_hash_value_cmp')
similar_hash_rv = sc.return_dist(res, _type='similar', 
                              target_col='target', comp_col='radial_variance_hash_value_cmp')

sc.plot_distributions(similar_hash_rv, nonsimilar_hash_rv, 
                      labels=('Hashing Score', 'Density'), title='Radial Variance hash values')

print('Mean value for groups: {:.2f} +- {:.2f}'.format(
    similar_hash_rv.mean(), similar_hash_rv.std()))
print('Mean value for non-groups: {:.2f} +- {:.2f}\n'.format(
    nonsimilar_hash_rv.mean(), nonsimilar_hash_rv.std()))

summary.update({'radialvariance_hash_1':
        sc.performance_report(similar_hash_rv, nonsimilar_hash_rv, limit=0.66)})<jupyter_output><empty_output><jupyter_text>Method 2: Logistic Regression fit  For hashing<jupyter_code>sc.make_logreg_fit(similar_hash, nonsimilar_hash, 
                   labels=('Hash values', 'Probabilty'), limits=(20, 65))

summary.update({'hash_2':
            sc.performance_report(similar_hash, nonsimilar_hash, limit=49.95)})<jupyter_output><empty_output><jupyter_text>For advanced hashing<jupyter_code>sc.make_logreg_fit(similar_hash_adv, nonsimilar_hash_adv, 
                   labels=('Hash values', 'Probabilty'), limits=(25, 65))

summary.update({'hash_adv_2':
            sc.performance_report(similar_hash, nonsimilar_hash, limit=53.53)})<jupyter_output><empty_output><jupyter_text>For Bhattacharyya correlation<jupyter_code>sc.make_logreg_fit(similar_bhatt, nonsimilar_bhatt, 
                   labels=('Bhattacharyya', 'Probability'), limits=(0,0.92))

summary.update({'bhattacharyya_2':
        sc.performance_report(similar_bhatt, nonsimilar_bhatt, limit=0.46)})<jupyter_output><empty_output><jupyter_text>For plain correlation<jupyter_code>sc.make_logreg_fit(similar_corr, nonsimilar_corr, 
                labels=('Correlation', 'Probability'), limits=(0,0.92))

summary.update({'correl_2':
           sc.performance_report(similar_corr, nonsimilar_corr, limit=0.47)})<jupyter_output><empty_output><jupyter_text>For average hashing (cv2)<jupyter_code>sc.make_logreg_fit(similar_hash_avg, nonsimilar_hash_avg, 
                   labels=('Normalized hash similarity', 'Probabilty'), limits=(0.3, 0.95))

summary.update({'avg_hash_2':
            sc.performance_report(similar_hash_avg, nonsimilar_hash_avg, limit=0.82)})<jupyter_output><empty_output><jupyter_text>For block mean hashing<jupyter_code>sc.make_logreg_fit(similar_hash_block, nonsimilar_hash_block, 
                   labels=('Normalized hash similarity', 'Probabilty'), limits=(0.4, 0.95))

summary.update({'block_hash_2':
            sc.performance_report(similar_hash_block, nonsimilar_hash_block, limit=0.87)})<jupyter_output><empty_output><jupyter_text>For perceptual hashing<jupyter_code>sc.make_logreg_fit(similar_hash_p, nonsimilar_hash_p, 
                   labels=('Normalized hash similarity', 'Probabilty'), limits=(0.35, 0.95))

summary.update({'p_hash_2':
            sc.performance_report(similar_hash_p, nonsimilar_hash_p, limit=0.73)})<jupyter_output><empty_output><jupyter_text>For Marr Hildreth hashing<jupyter_code>sc.make_logreg_fit(similar_hash_mh, nonsimilar_hash_mh, 
                   labels=('Normalized hash similarity', 'Probabilty'), limits=(0.35, 0.95))

summary.update({'marrhildreth_hash_2':
            sc.performance_report(similar_hash_mh, nonsimilar_hash_mh, limit=0.71)})<jupyter_output><empty_output><jupyter_text>For radial variance hashing<jupyter_code>sc.make_logreg_fit(similar_hash_rv, nonsimilar_hash_rv, 
                   labels=('Normalized hash similarity', 'Probabilty'), limits=(0.35, 0.8))

summary.update({'radialvariance_hash_2':
            sc.performance_report(similar_hash_rv, nonsimilar_hash_rv, limit=0.66)})<jupyter_output><empty_output><jupyter_text>Evaluation of methods<jupyter_code>summary = pd.DataFrame().from_dict(summary, orient='index',
                        columns=['precision', 'recall', 'f1'])
summary
#1: mean value
#2: logistic regression
summary.plot.barh(alpha=0.7, figsize=(15, 20), grid=True, legend=True, 
                  title='Overview how limits from different approaches perform',
                  colormap='tab20')

# taking the mean for the limits seems to make conservative distinctions
# the limits from logistic regression fits seem to yield better balanced
# results (f1-score)

# For our images, the winning algorithms are:
# <NAME>, Perceptual Hash and Block Mean Hash
# with limit estimates based on Logistic Regression<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13552872301500182, 'fraction_numerical': 0.013172338090010977, 'mean_word_length': 4.272376543209877, 'pattern_counts': {'":': 0, '<': 86, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '602484', 'license': 'MIT', 'n_tokens_mistral': 4870, 'n_tokens_neox': 4563, 'n_words': 725, 'path': 'Estimate limits.ipynb', 'repo_name': 'jkotula89/image_spring_cleaning', 'size': '13672'}