<commit_before>from os.path import isdir,join,normpath
from os import listdir

from PIL import Image
from numpy import asfarray,dot,argmin,zeros, array, asarray, absolute
from numpy import average,sort,trace,copy
from numpy.linalg import svd,eigh,norm
from numpy import concatenate, reshape
from math import sqrt
import sys

import tkFileDialog
from patsy.util import wide_dtype_for

NUMFEATURES = 4
IMG_WIDTH = 150
IMG_HEIGHT = 220


def parseDirectory(directoryName,extension):
    '''This method returns a list of all filenames in the Directory directoryName. 
    For each file the complete absolute path is given in a normalized manner (with 
    double backslashes). Moreover only files with the specified extension are returned in 
    the list.
    '''
    if not isdir(directoryName): return
    imagefilenameslist=sorted([
        normpath(join(directoryName, fname))
        for fname in listdir(directoryName)
        if fname.lower().endswith('.'+extension)            
        ])
    return imagefilenameslist

#####################################################################################
# Implement required functions here
#
#
#

def generateListOfImgs(listOfTrainFiles):
    imgList = []
    for trainFile in listOfTrainFiles:
        imgList.append(Image.open(trainFile))
    return imgList

def convertImgListToNumpyData(imgList):
    imgArrays = []
    for img in imgList:
        imgArray = asfarray(img).reshape((1, -1))[0]
        normalizedImgArray = normalize(imgArray)
        imgArrays.append(normalizedImgArray)
    return asarray(imgArrays)

def normalize(array):
    max = array.max()
    return array / max

def calculateAverageImg(images):
    return average(images, axis=0)

def removeAverageImage(images, avgImage):
    for image in images:
        image -= avgImage
    return images

def calculateEigenfaces(faces, width, height):
    M = len(faces)
    w, v = eigh(faces.T.dot(faces))
    u = faces.dot(v)
    index = w.argsort()[::-1]
    w=w[index]
    u=u[:, index]
    return u

def transformToEigenfaceSpace(eigenfaces, face, numFeatures):
    pointToEigenspace = zeros(shape=(numFeatures))
    for i in range(numFeatures):
        pointToEigenspace[i] = dot((eigenfaces[:,i:i+1].T), face)
    return pointToEigenspace

def imageFromNormalizedArray(array, width=IMG_WIDTH, height=IMG_HEIGHT):
    return Image.fromarray(array.reshape((height, width))*250)

def mergeImage(rArray, gArray, bArray, width=IMG_WIDTH, height=IMG_HEIGHT):
    rChannel = imageFromNormalizedArray(rArray).convert('RGB').split()[0]
    gChannel = imageFromNormalizedArray(gArray).convert('RGB').split()[1]
    bChannel = imageFromNormalizedArray(bArray).convert('RGB').split()[2]
    return Image.merge('RGB', (rChannel, gChannel, bChannel))



####################################################################################
#Start of main programm

#Choose Directory which contains all training images 
TrainDir=tkFileDialog.askdirectory(title="Choose Directory of training images")
#Choose the file extension of the image files
Extension='png'

images = convertImgListToNumpyData(generateListOfImgs(parseDirectory(TrainDir, Extension)))


avgImage = calculateAverageImg(copy(images))
normedArrayOfFaces = removeAverageImage(copy(images), avgImage)

#imageFromNormalizedArray(avgImage).show()

eigenfaces = calculateEigenfaces(normedArrayOfFaces.T, len(images[0]), len(images))
transposedFaces = []
for face in normedArrayOfFaces:
    transposedFaces.append(transformToEigenfaceSpace(eigenfaces, face, NUMFEATURES))


#Image.fromarray(avgImage.reshape((220, 150))*250).show()

#Choose the image which shall be recognized
testImageDirAndFilename=tkFileDialog.askopenfilename(title="Choose Image to detect")

testImage = Image.open(testImageDirAndFilename)
testface = convertImgListToNumpyData([testImage])[0]
testface -= avgImage

testface = transformToEigenfaceSpace(eigenfaces, testface, NUMFEATURES)

distance = sys.float_info.max
closestMatchIndex = 0
for index, face in enumerate(transposedFaces):
    newdistance = norm(testface - face)
    if distance > newdistance:
        closestMatchIndex = index
        distance = newdistance

#imageFromNormalizedArray(images[closestMatchIndex]).show()
#testImage.show()

testImageArray = normalize(convertImgListToNumpyData([testImage])[0])

#mergeImage(normalize(images[closestMatchIndex]), avgImage, normalize(testImageArray)).show()
imageFromNormalizedArray(absolute(normalize(images[closestMatchIndex] - testImageArray))).show()

####################################################################################
# Implement required functionality of the main programm here

<commit_msg>refactored imports and funcitons and added distance picture <commit_after>from os.path import isdir, join, normpath
from os import listdir

from PIL import Image
from numpy import asfarray, dot, zeros, asarray, absolute
from numpy import average, copy
from numpy.linalg import eigh, norm
import sys

import tkFileDialog

NUMFEATURES = 6
IMG_WIDTH = 150
IMG_HEIGHT = 220


def parseDirectory(directoryName,extension):
    '''This method returns a list of all filenames in the Directory directoryName. 
    For each file the complete absolute path is given in a normalized manner (with 
    double backslashes). Moreover only files with the specified extension are returned in 
    the list.
    '''
    if not isdir(directoryName): return
    imagefilenameslist=sorted([
        normpath(join(directoryName, fname))
        for fname in listdir(directoryName)
        if fname.lower().endswith('.'+extension)            
        ])
    return imagefilenameslist

#####################################################################################
# Implement required functions here
#
#
#

def generateListOfImgs(listOfTrainFiles):
    imgList = []
    for trainFile in listOfTrainFiles:
        imgList.append(Image.open(trainFile))
    return imgList

def convertImgListToNumpyData(imgList):
    imgArrays = []
    for img in imgList:
        imgArray = asfarray(img).reshape((1, -1))[0]
        normalizedImgArray = normalize(imgArray)
        imgArrays.append(normalizedImgArray)
    return asarray(imgArrays)

def normalize(array):
    max = array.max()
    return array / max

def calculateAverageImg(images):
    return average(images, axis=0)

def removeAverageImage(images, avgImage):
    for image in images:
        image -= avgImage
    return images

def calculateEigenfaces(faces, width, height):
    M = len(faces)
    w, v = eigh(faces.T.dot(faces))
    u = faces.dot(v)
    index = w.argsort()[::-1]
    w=w[index]
    u=u[:, index]
    return u

def transformToEigenfaceSpace(eigenfaces, face, numFeatures):
    pointToEigenspace = zeros(shape=(numFeatures))
    for i in range(numFeatures):
        pointToEigenspace[i] = dot((eigenfaces[:,i:i+1].T), face)
    return pointToEigenspace

def imageFromNormalizedArray(array, width=IMG_WIDTH, height=IMG_HEIGHT):
    return Image.fromarray(array.reshape((height, width))*250)

def mergeImage(rArray, gArray, bArray, width=IMG_WIDTH, height=IMG_HEIGHT):
    rChannel = imageFromNormalizedArray(rArray).convert('RGB').split()[0]
    gChannel = imageFromNormalizedArray(gArray).convert('RGB').split()[1]
    bChannel = imageFromNormalizedArray(bArray).convert('RGB').split()[2]
    return Image.merge('RGB', (rChannel, gChannel, bChannel))

def calculateDistance(transposedFaces, testface, imageFileNames):
    distance = sys.float_info.max
    closestMatchIndex = 0
    for index, face in enumerate(transposedFaces):
        newdistance = norm(testface - face)
        print str(newdistance) + " image: " + imageFileNames[index]
        if distance > newdistance:
            closestMatchIndex = index
            distance = newdistance
    return (closestMatchIndex, distance)

####################################################################################
#Start of main programm

#Choose Directory which contains all training images 
TrainDir=tkFileDialog.askdirectory(title="Choose Directory of training images")
#Choose the file extension of the image files
Extension='png'

####################################################################################
# Implement required functionality of the main programm here
imageFileNames = parseDirectory(TrainDir, Extension)
images = convertImgListToNumpyData(generateListOfImgs(imageFileNames))


avgImage = calculateAverageImg(copy(images))
normedArrayOfFaces = removeAverageImage(copy(images), avgImage)

#imageFromNormalizedArray(avgImage).show()

eigenfaces = calculateEigenfaces(normedArrayOfFaces.T, len(images[0]), len(images))
transposedFaces = []
for face in normedArrayOfFaces:
    transposedFaces.append(transformToEigenfaceSpace(eigenfaces, face, NUMFEATURES))

#Image.fromarray(avgImage.reshape((220, 150))*250).show()

#Choose the image which shall be recognized
testImageDirAndFilename=tkFileDialog.askopenfilename(title="Choose Image to detect")

testImage = Image.open(testImageDirAndFilename)
testface = convertImgListToNumpyData([testImage])[0]
testface -= avgImage

testface = transformToEigenfaceSpace(eigenfaces, testface, NUMFEATURES)

closestMatchIndex, distance = calculateDistance(transposedFaces, testface, imageFileNames)
print "closest distance: " + str(distance) + " image: " + imageFileNames[closestMatchIndex]
#imageFromNormalizedArray(images[closestMatchIndex]).show()
#testImage.show()

testImageArray = normalize(convertImgListToNumpyData([testImage])[0])
closestMatchArray = normalize(images[closestMatchIndex])
diffToTestArray = absolute(normalize(closestMatchArray - testImageArray))
imageFromNormalizedArray(diffToTestArray).show()

#mergeImage(normalize(images[closestMatchIndex]), avgImage, normalize(testImageArray)).show()
mergeImage(closestMatchArray + diffToTestArray, normalize(closestMatchArray), normalize(closestMatchArray)).show()



{'fraction_non_alphanumeric': 0.14392712550607287, 'fraction_numerical': 0.006477732793522267, 'mean_word_length': 5.513513513513513, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2759304', 'n_tokens_mistral': 2934, 'n_tokens_neox': 2827, 'n_words': 674}