<jupyter_start><jupyter_text>Data Exploration  By <NAME>, <NAME>, <NAME>, Dwight Sablan Data Source: *Clothing Pattern Dataset* by <NAME>, <NAME>, <NAME>, <NAME>, and <NAME> https://github.com/lstearns86/clothing-pattern-datasetPattern Types:  - Solid - Checkered - Floral - Dotted - Striped - Zig Zag<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import urllib
from PIL import Image

import requests
from io import BytesIO

import glob, os, cv2
clothingDataset = pd.read_csv('googleClothingDataset.csv')
clothingDataset.head(5)
clothingDataset.describe
clothingDataset.dtypes
#display mean sizes
clothingDataset.iloc[:, 2:].mean()
#display standard deviation of sizes
clothingDataset.iloc[:, 2:].std()
#minimums
clothingDataset.iloc[:, 2:].min()
#maximums
clothingDataset.iloc[:, 2:].max()
clothingDataset.isnull().sum()<jupyter_output><empty_output><jupyter_text>Extract Images from URL 1. Store list of all URLs in to dataframe2. Iterate through dataframe:    - Extract *valid* images    - Save them to local storage    - Store the local path name to each image in a dataframe: "Images". The label of each image is a [counter]_[patter type].jpg, example: 01_floral.jpg    - Store bad URLs in a dataframe: "badURLs"<jupyter_code>dataset = clothingDataset.iloc[:, :2].values.tolist()
images = [] #df used to store valid images. Format: pattern, image
badURLs = [] #df used to store invalid images/broken urls. Format: patter, url

dir_base = os.getcwd() + '\\patterns' # Get your current directory & create a new folder 'called patterns'

for i, row in enumerate(dataset): 
    #row[0] = pattern, row[1] = url
    try:
        image_name = '0{}_{}'.format(i+1, "_" + str(row[0]) + '.jpg') # Example: 01_floral.jpg
        urllib.request.urlretrieve(row[1], image_name)
        path = os.path.join(dir_base , image_name)
        images.append([row[0], path])  # adding that locally fetched image path to pandas column

    except Exception:
        badURLs.append([row[0], row[1]])
        continue
#format the images into a dataframe
images = pd.DataFrame(images, columns = ['pattern', 'image'])
badURLs = pd.DataFrame(badURLs, columns = ['pattern', 'url'])
#storing images into a csv, so we don't have to redownload images later
images.to_csv("URL_patterns.csv")
print("Valid Image URLs:   ", len(images))
print("Invalid Image URLs: ", len(badURLs))
print("INVALID IMAGE URLS")
print('')
print("     Solid: ", len(badURLs[badURLs.pattern == 'solid']))
print("     Checkered: ", len(badURLs[badURLs.pattern == 'checkered']))
print("     Floral: ", len(badURLs[badURLs.pattern == 'floral']))
print("     Dotted: ", len(badURLs[badURLs.pattern == 'dotted']))
print("     Striped: ", len(badURLs[badURLs.pattern == 'striped']))
print("     Zig Zag: ", len(badURLs[badURLs.pattern == 'zig zag']))
print('')
print("     Total: ", len(badURLs))
print("VALID IMAGE URLS")
print('')
print("     Solid: ", len(images[images.pattern == 'solid']))
print("     Checkered: ", len(images[images.pattern == 'checkered']))
print("     Floral: ", len(images[images.pattern == 'floral']))
print("     Dotted: ", len(images[images.pattern == 'dotted']))
print("     Striped: ", len(images[images.pattern == 'striped']))
print("     Zig Zag: ", len(images[images.pattern == 'zig zag']))
print('')
print("     Total: ", len(images))
patterns = ['checkered', 'dotted', 'floral', 'solid', 'striped', 'zig zag']
badURL_totals = []
for i in patterns:
    badURL_totals.append([i, len(badURLs[badURLs.pattern == i])])
                          
goodURL_totals = []
for i in patterns:
    goodURL_totals.append([i, len(images[images.pattern == i])])
badURL_totals = pd.DataFrame(badURL_totals, columns = ['pattern', 'total'])
goodURL_totals = pd.DataFrame(goodURL_totals, columns = ['pattern', 'total'])
badURL_totals.total.to_list()
index = patterns
df = pd.DataFrame({'Valid Images': goodURL_totals.total.to_list(), 
                   'Invalid Images': badURL_totals.total.to_list()}, index=index)

ax = df.plot.bar(rot=0)
ax.set_ylim([0, 400])

plt.title('URL Images')

plt.xlabel('Pattern')
plt.ylabel('Image Count')
len(images)
#display images
size = 128, 128 #reshape to thumbnail size
for i in [0, 100, 150]:
    im = Image.open(images.image[i])
    im.thumbnail(size)
    display(im)<jupyter_output><empty_output><jupyter_text>Explore FingerCamera DatasetFolders:FingerCamera\[pattern]patterns: checkered, dotted, floral, solid, striped, zig zag<jupyter_code>path = 'FingerCamera/'
PNGimages = []
patterns = ['checkered', 'dotted', 'floral', 'solid', 'striped', 'zig zag']
pattern_counts = []
prev_count = 0
print("PNGs")
for i in patterns:
    PNGimages.extend(glob.glob(path + str(i) + '/*.png'))
    pattern_counts.append(len(PNGimages) - prev_count)
    print(i, "-", len(PNGimages) - prev_count)
    prev_count = len(PNGimages)
    
print("TOTAL -", len(PNGimages))
for i in [0, 100, 150, 250, 300, 350]:
    im = cv2.imread(PNGimages[i])
    print("Image Size: ", im.shape)
size = 128, 128 #reshape to thumbnail size
for i in [0, 100, 150, 250, 300, 350]:
    im = Image.open(PNGimages[i])
    im.thumbnail(size)
    display(im)
plt.bar(patterns, pattern_counts)
plt.title("PNG Images")
ax.set_ylim([0, 400])
plt.xlabel('Pattern')
plt.ylabel('Image Count')
index = patterns
df = pd.DataFrame({'Invalid Images': badURL_totals.total.to_list(),
                   'Valid Images': goodURL_totals.total.to_list(),
                  'PNG Images': pattern_counts}, index=index)

ax = df.plot.bar(rot=0)

plt.title('All Images')
plt.xlabel('Pattern')
plt.ylabel('Image Count')
total_valid = []
for i in range(6):
    total_valid.append(goodURL_totals.total.to_list()[i] + pattern_counts[i])
total_valid
print("ALL VALID IMAGE (valid URL + PNG)")
for i in range(6):
    print(patterns[i], "-", total_valid[i])
print("Total -", sum(total_valid))
plt.bar(patterns, total_valid)

plt.title('Total Images (Valid URL + PNG)')
plt.xlabel('Pattern')
plt.ylabel('Image Count')<jupyter_output><empty_output><jupyter_text>After images are locallying installed<jupyter_code>images = []
corrupted_images = []
for filename in glob.glob('*.jpg'): #enter local file path
    try:
        im = Image.open(filename)
        name = os.path.basename(filename)
        if name[-7:] == 'ted.jpg':
            pattern = 'dotted'
        elif name[-7:] == 'red.jpg':
            pattern = 'checkered'
        elif name[-7:] == 'lid.jpg':
            pattern = 'solid'
        elif name[-7:] == 'ral.jpg':
            pattern = 'floral'
        elif name[-7:] == 'zag.jpg':
            pattern = 'zig_zag'    
        elif name[-7:] == 'ped.jpg':
            pattern = 'striped'  
        images.append([im, pattern])
    except Exception:
        corrupted_images.append(filename)

    
print("Valid Images: ", len(images))
print("Corrupted Images: ", len(corrupted_images))<jupyter_output>Valid Images:  1229
Corrupted Images:  32
{'fraction_non_alphanumeric': 0.15937635339974016, 'fraction_numerical': 0.01458062653385304, 'mean_word_length': 4.189513108614232, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '96426', 'license': 'MIT', 'n_tokens_mistral': 2457, 'n_tokens_neox': 2359, 'n_words': 584, 'path': 'notebooks/Data Exploration.ipynb', 'repo_name': 'liemluong/Neural_Network_Art', 'size': '6989'}