<jupyter_start><jupyter_text>Step 1: associate the data from the .csv data. Strategy: 2 networks--one will be used to get probability of being by an artist, can use second to last layer as input into second network.<jupyter_code>import sys
from utils import *
%matplotlib inline
path = "data/painter-by-numbers/"

from __future__ import division,print_function

import os, json
from glob import glob
import numpy as np
np.set_printoptions(precision=4, linewidth=100)
from matplotlib import pyplot as plt
#verify directory
%pwd
artist_names = np.genfromtxt('train_info.csv', delimiter=',', dtype=None, usecols=(1)).astype(str)
artist_names = artist_names[1:] #remove the first row that contains ['filename', 'artist']
#print(artist_names[0:5])<jupyter_output><empty_output><jupyter_text>Above, I'm only using column 1 (artist ID) for the numpy array because everything is ordered, so I should just be able to iterate through the directory of painting IDs and add the corresponding artist ID to the title. The next step after getting an array of artist IDs is to iterate through the directory and add the corresponding artist ID to the title of the painting so that I can use bash to sort them into 9 subdirectories, 1 for each artist.<jupyter_code>#iterate through the files
idIndex = 0 #keep track of where we are in the artist name array
directory = 'data/painter-by-numbers/train/' #needs full directory
for filename in os.listdir(path+'train/'):
    """
    tag each file w/ artist ID--NOTE: if adding to end of name, use filename[:-4] so we don't add characters after the .jpg
    Adding the artist ID first for easier recognition with bash (check beginning + wildcard character)
    Since adding artist ID before filename we don't need to worry about the .jpg part
    """
    fullPath = os.path.join(directory, filename)
    newFileName = artist_names[idIndex] + '_' + filename
    
    finalFile = os.path.join(directory, newFileName)
    os.rename(fullPath, finalFile)
    idIndex += 1
current_directory = os.getcwd()
LESSON_HOME_DIR = current_directory
DATA_HOME_DIR = current_directory+'/data'+'/painter-by-numbers'
#Allow relative imports to directories above lesson1/
sys.path.insert(1, os.path.join(sys.path[0], '..'))
%cd $DATA_HOME_DIR
%mkdir valid
%mkdir results
%mkdir -p sample/train
%mkdir -p sample/test
%mkdir -p sample/valid
%mkdir -p sample/results
%mkdir -p test/unknown
%cd $DATA_HOME_DIR/train
g = glob('*.jpg')
shuffle = np.random.permutation(g)
#the train dataset has 79,434 images, so we will take ~20% for our validation dataset
for i in range(15800): os.rename(shuffle[i], DATA_HOME_DIR+'/valid/'+shuffle[i])
from shutil import copyfile
g = glob('*.jpg')
shuffle = np.random.permutation(g)
#now we add photos for our sample dataset
for i in range(1200): os.rename(shuffle[i], DATA_HOME_DIR+'/sample/train/'+shuffle[i])
%cd DATA_HOME_DIR/valid
g = glob('*.jpg')
shuffle = np.random.permutation(g)
#add photos to sample valid batch
for i in range(300): copyfile(shuffle[i], DATA_HOME_DIR+'/sample/valid/'+shuffle[i])<jupyter_output><empty_output><jupyter_text>The code below creates a vector of all the $unique$ artist names, iterates through them and makes appropriate directories in the train folder.<jupyter_code>directory = 'data/painter-by-numbers/'
name = 'bob/'
newDir = os.path.join(directory, name)
os.makedirs(newDir)
#TODO: perhaps make this into a function, you will need to make and move tagged files onto directories in sample/train,
# sample/valid, train, and valid

directory = 'data/painter-by-numbers/train/'

def createArtistDirs(directory):
    artistIdVector = np.unique(artist_names)
    for name in artistIdVector:
        newDirectory = os.path.join(directory, name)
        finalDir = newDirectory + '/'
        if not os.path.exists(finalDir):
            os.makedirs(finalDir)
createArtistDirs('data/painter-by-numbers/train/')
createArtistDirs('data/painter-by-numbers/valid/')
createArtistDirs('data/painter-by-numbers/sample/train/')
createArtistDirs('data/painter-by-numbers/sample/valid/')<jupyter_output><empty_output><jupyter_text>Next, we want to move the tagged files into appropriate directories that share the first $32$ characters with the filename.<jupyter_code>import shutil

directory = 'data/painter-by-numbers/train/'

def moveFilesToDir(directory):
    for filename in os.listdir(directory):
        if '.jpg' in filename:
            artistId = filename[:32]
            fullDir = os.path.join(directory,filename)

            #make the final directory for the jpg
            newDir = os.path.join(directory, artistId)
            newDir = newDir + '/'
            shutil.copy(fullDir, newDir)
moveFilesToDir('data/painter-by-numbers/train/')
moveFilesToDir('data/painter-by-numbers/valid/')
moveFilesToDir('data/painter-by-numbers/sample/train/')
moveFilesToDir('data/painter-by-numbers/sample/valid/')<jupyter_output><empty_output><jupyter_text>Next, we have to go through $each$ file we just made and put JPGs into, and chop off the first $33$ characters in the name, which are the artist ID and an underscore.<jupyter_code>for fileHead in os.listdir(directory):
    newDir = os.path.join(directory,fileHead)
    
    for filename in os.listdir(newDir):
        fullDir = os.path.join(newDir, filename)
        print(fullDir)
        newDir = os.path.join(newDir, filename[33:]) #this is the filename w/ the artistID and underscore chopped off.
        os.rename(fullDir, newDir)
artistName ='85a0c03bcbe27be6d6166b4f4833b55a'

#TODO: it appears fullDir is not removing the old filenames and is just adding new filenames on top of them--figure out how 
#to chop off that filename before moving onto the next file. 

newDir = os.path.join(directory, artistName)
for filename in os.listdir(newDir):
    fullDir = os.path.join(newDir, filename)
    print(fullDir)
    
    newDir = os.path.join(newDir, filename[33:])
    os.rename(fullDir,newDir)
import utils; reload(utils)
from utils import plots
batch_size=64
import vgg16; reload(vgg16)
from vgg16 import Vgg16
vgg = Vgg16()<jupyter_output><empty_output><jupyter_text>After declaring vgg as a Vgg16() object, we're going to do a few things: 1. Get the data in batches (train and validation)2. Fit vgg to the batches after finetuning3. save weights of vgg<jupyter_code>batches = vgg.get_batches(path+'train', batch_size=batch_size)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10217493350023471, 'fraction_numerical': 0.013143483023001095, 'mean_word_length': 4.621811785400176, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '147623', 'license': 'Apache-2.0', 'n_tokens_mistral': 2106, 'n_tokens_neox': 2005, 'n_words': 651, 'path': 'deeplearning1/nbs/Painter-By-Numbers.ipynb', 'repo_name': 'db7894/courses', 'size': '6391'}