<jupyter_start><jupyter_text>MACHINE LEARNING PROJECT ___ PREREQUISITES To process images I'll be using OpenCV library. It is important to take a look at the [docs][1] before running the next cell  as you may want to use another OpenCV package. For this project I'll use the *'main modules package'*[1]:https://pypi.org/project/opencv-python/<jupyter_code># !pip install opencv-python
### TODO - .yaml<jupyter_output><empty_output><jupyter_text>___ IMPORTS<jupyter_code># Modules used for data handling / test
import os
import csv
import cv2
import pathlib
import time
import pickle

from utils import get_collection, show_collection, nameof, mklist


# Modules used for EDA
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np
import pandas as pd
import seaborn as sns

from matplotlib.patches import Rectangle


# Modules used for image processing
import cv2

from collections import Counter
from utils import crop_img, chi_osc, extract_img_data, get_img_rgb
from utils import resize_img, reduce_col_palette, whitespace


# Modules used for ML
from sklearn.cluster import KMeans
from utils import color_quant

from utils import data_report
### TODO ### Import a class from a module

# For a better pd.DataFrame visualization
class display(object):
    '''This class was found in 'Python Data Science Handbook' by jakevdp (<NAME>),
    which you can access though his GitHub repository
    (https://github.com/jakevdp/PythonDataScienceHandbook)'''
    
    template = '''<div style="float: left; padding: 10px;">
                  <p style='font-family:"Courier New", Courier, monospace'>{0}</p>{1}
                  </div>'''
    
    def __init__(self, *args):
        self.args = args
        
    def _repr_html_(self):
        return '\n'.join(self.template.format(a, eval(a)._repr_html_()) for a in self.args)
    
    def __repr__(self):
        return '\n\n'.join(a + '\n' + repr(eval(a)) for a in self.args)
# Modules settings
%matplotlib inline<jupyter_output><empty_output><jupyter_text>___ UPDATE UTILS<jupyter_code># This cell only needs to be executed to update utils
# if modified after been imported

%run utils<jupyter_output><empty_output><jupyter_text>___ BASIC EDA<jupyter_code>raw_data = pd.read_csv('./data/raw_museum/raw_data.csv', names=['img_ID', 'artist', 'height', 'width'])

raw_data.shape, raw_data.head()
raw_data.info()
raw_data.nunique()
# Get all paths to .jpg files
extensions = ['.jpg']

raw_museum = get_collection(path='./images/raw_museum', extensions=extensions)
big_museum = get_collection(path='./images/big_museum', extensions=extensions)
large_museum = get_collection(path='./images/large_museum', extensions=extensions)
medium_museum = get_collection(path='./images/medium_museum', extensions=extensions)
small_museum = get_collection(path='./images/small_museum', extensions=extensions)

collections = [big_museum, large_museum, medium_museum, small_museum]

print(f'{len(raw_museum)} images in raw_museum')
print(f'{len(big_museum)} images in big_museum')
print(f'{len(large_museum)} images in large_museum')
print(f'{len(medium_museum)} images in medium_museum')
print(f'{len(small_museum)} images in small_museum:')
# Get all paths to .csv files
extensions = ['.csv']

big_museum = get_collection(path='./images/big_data', extensions=extensions)
large_museum = get_collection(path='./images/large_data', extensions=extensions)
medium_museum = get_collection(path='./images/medium_data', extensions=extensions)
small_museum = get_collection(path='./images/small_data', extensions=extensions)

# Turn paths into str
big_museum = [str(i) for i in big_data]
large_museum = [str(i) for i in large_data]
medium_museum = [str(i) for i in medium_data]
small_museum = [str(i) for i in small_data]

# Build museums DataFrame
columns_names=['img_ID', 'artist', 'height', 'width', 'whitespace', 'chiaroscuro',
               'color_01', 'color_02', 'color_03', 'color_04', 'color_05', 
               'color_06', 'color_07', 'color_08', 'color_09', 'color_10']

big_museum = pd.concat((pd.read_csv(file, names=columns_names) for file in big_museum), ignore_index=True)
large_museum = pd.concat((pd.read_csv(file, names=columns_names) for file in large_museum), ignore_index=True)
medium_museum = pd.concat((pd.read_csv(file, names=columns_names) for file in medium_museum), ignore_index=True)
small_museum = pd.concat((pd.read_csv(file, names=columns_names) for file in small_museum), ignore_index=True)

museums = [big_museum, large_museum, medium_museum, small_museum]
[museum.shape for museum in museums]
[museum['img_ID'].nunique() for museum in museums]<jupyter_output><empty_output><jupyter_text>I'll keep only the common images to all museums<jupyter_code>A = big_museum['img_ID'].isin(large_museum['img_ID'])
B = big_museum['img_ID'].isin(medium_museum['img_ID'])
C = big_museum['img_ID'].isin(small_museum['img_ID'])

big_museum = big_museum[A & B & C]
A = large_museum['img_ID'].isin(big_museum['img_ID'])
B = large_museum['img_ID'].isin(medium_museum['img_ID'])
C = large_museum['img_ID'].isin(small_museum['img_ID'])

large_museum = large_museum[A & B & C]
A = medium_museum['img_ID'].isin(big_museum['img_ID'])
B = medium_museum['img_ID'].isin(large_museum['img_ID'])
C = medium_museum['img_ID'].isin(small_museum['img_ID'])

medium_museum = medium_museum[A & B & C]
A = small_museum['img_ID'].isin(big_museum['img_ID'])
B = small_museum['img_ID'].isin(large_museum['img_ID'])
C = small_museum['img_ID'].isin(medium_museum['img_ID'])

small_museum = small_museum[A & B & C]
# Update museums
museums = [big_museum, large_museum, medium_museum, small_museum]

[museum.shape for museum in museums]<jupyter_output><empty_output><jupyter_text>I'll not only drop the rows but also the images<jupyter_code># IMG_ID are not int64 type but object type, so I'll cast it
for museum in museums:
    museum['img_ID'] = museum['img_ID'].astype(str, errors='ignore')
big_museum.info()
for museum in museums:
    museum_name = nameof(museum, globals())
    del_images = 0
    
    for image in museum:
        img_ID = str(image).split('/')[-1].split('.')[0]
        
        # I can use any museum as all have the same images
        if big_museum['img_ID'][big_museum['img_ID'].str.contains(img_ID)].any():
            continue
        elif os.path.exists(image):
            os.remove(image)
            del_images += 1
        
    print(f'{del_images} images deleted from {museum_name}')
# Update museums
# Get all paths to .jpg files
extensions = ['.jpg']

raw_museum = get_collection(path='./images/raw_museum', extensions=extensions)
big_museum = get_collection(path='./images/big_museum', extensions=extensions)
large_museum = get_collection(path='./images/large_museum', extensions=extensions)
medium_museum = get_collection(path='./images/medium_museum', extensions=extensions)
small_museum = get_collection(path='./images/small_museum', extensions=extensions)

collections = [big_museum, large_museum, medium_museum, small_museum]

print(f'{len(raw_museum)} images in raw_museum')
print(f'{len(big_museum)} images in big_museum')
print(f'{len(large_museum)} images in large_museum')
print(f'{len(medium_museum)} images in medium_museum')
print(f'{len(small_museum)} images in small_museum:')<jupyter_output>5468 images in raw_collection
4276 images in large_collection
4276 images in mid_collection
4276 images in mid_sqr_collection
4276 images in low_sqr_collection:<jupyter_text>___ Let's take a closer look to the datasets<jupyter_code>l_repo = data_report(big_museum)
m_repo = data_report(large_museum)
ms_repo = data_report(medium_museum)
ls_repo = data_report(small_museum)

display('l_repo', 'm_repo', 'ms_repo', 'ls_repo')
with open('./data/big_museum/big_museum_clean.csv', "w", newline="") as datafile:
    writer = csv.writer(datafile)            
    writer.writerows(big_museum)

with open('./data/large_museum/large_museum_clean.csv', "w", newline="") as datafile:
    writer = csv.writer(datafile)            
    writer.writerows(large_museum)

with open('./data/medium_museum/medium_museum._cleancsv', "w", newline="") as datafile:
    writer = csv.writer(datafile)            
    writer.writerows(medium_museum)

with open('./data/small_museum/small_museum_clean.csv', "w", newline="") as datafile:
    writer = csv.writer(datafile)            
    writer.writerows(small_museum)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12671029149315882, 'fraction_numerical': 0.006424747174301011, 'mean_word_length': 5.562060889929742, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '98313', 'license': 'MIT', 'n_tokens_mistral': 2942, 'n_tokens_neox': 2867, 'n_words': 662, 'path': '02.- Dataset cleanup.ipynb', 'repo_name': 'mabatalla/Is-this-Picasso---Academic-ML-classificator', 'size': '8414'}