<commit_before>"""The aim of this script is to read the multi pie dataset """

import scipy.io
import numpy as np
import matplotlib.pyplot as plt
import glob
import cPickle as pickle
import os
import cv2
import facedetection
import fnmatch
import Image
import csv


import matplotlib.image as io
# from skimage import io
# from skimage import color
from skimage.transform import resize

from common import *

SMALL_SIZE = ((40, 30))

# TODO: make some general things with the path in order to make it work easily between
# lab machine and local
def equalizeImgGlobal(x):
  return cv2.equalizeHist(x)

def equalizeCLAHE(x):
  # Contrast Limited Adaptive Histogram Equalization
  clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(5,5))
  return clahe.apply(x)


def equalizeFromFloatGlobal(x, reshapeSize=SMALL_SIZE):
  x = x * 255
  x = np.asarray(x, dtype='uint8')
  y = x.reshape(reshapeSize)
  y =  equalizeImgGlobal(y).reshape(-1)
  return y / 255.0


def cropFromFloat(x):
  x = x * 255
  x = np.asarray(x, dtype='uint8')
  # y = x.reshape(SMALL_SIZE)
  y = facedetection.cropFace(x, rescaleForReconigtion=1)
  if y is None:
    return None
  return y.reshape(SMALL_SIZE)



def equalizeFromFloatCLAHE(x):
  x = x * 255
  x = np.asarray(x, dtype='uint8')
  y = x.reshape(SMALL_SIZE)
  y =  equalizeCLAHE(y).reshape(-1)
  return y / 255.0



def equalizePIE():
  imgs, labels = readMultiPIE()
  imgs = np.array(map(lambda x: equalizeFromFloatGlobal(x), imgs))

  return imgs, labels

def equalizeKanade(big=False):
  data, labels = readKanade(big=big, equalize=False)

  if big:
      fileName = 'equalized_kanade_big.pickle'<commit_msg>more flexible code for equalization<commit_after>"""The aim of this script is to read the multi pie dataset """

import scipy.io
import numpy as np
import matplotlib.pyplot as plt
import glob
import cPickle as pickle
import os
import cv2
import facedetection
import fnmatch
import Image
import csv


import matplotlib.image as io
# from skimage import io
# from skimage import color
from skimage.transform import resize

from common import *

SMALL_SIZE = ((40, 30))

# TODO: make some general things with the path in order to make it work easily between
# lab machine and local
def equalizeImgGlobal(x):
  return cv2.equalizeHist(x)

def equalizeCLAHE(x):
  # Contrast Limited Adaptive Histogram Equalization
  clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(5,5))
  return clahe.apply(x)


def equalizeFromFloatGlobal(x, reshapeSize=SMALL_SIZE):
  x = x * 255
  x = np.asarray(x, dtype='uint8')
  y = x.reshape(reshapeSize)
  y =  equalizeImgGlobal(y).reshape(-1)
  return y / 255.0


def cropFromFloat(x):
  x = x * 255
  x = np.asarray(x, dtype='uint8')
  # y = x.reshape(SMALL_SIZE)
  y = facedetection.cropFace(x, rescaleForReconigtion=1)
  if y is None:
    return None
  return y.reshape(SMALL_SIZE)



def equalizeFromFloatCLAHE(x, reshapeSize=SMALL_SIZE):
  x = x * 255
  x = np.asarray(x, dtype='uint8')
  y = x.reshape(reshapeSize)
  y =  equalizeCLAHE(y).reshape(-1)
  return y / 255.0



def equalizePIE():
  imgs, labels = readMultiPIE()
  imgs = np.array(map(lambda x: equalizeFromFloatGlobal(x), imgs))

  return imgs, labels

def equalizeKanade(big=False):
  data, labels = readKanade(big=big, equalize=False)

  if big:
      fileName = 'equalized_kanade_big.pickle'
{'fraction_non_alphanumeric': 0.09960996099609962, 'fraction_numerical': 0.0204020402040204, 'mean_word_length': 4.308917197452229, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2327481', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1197, 'n_words': 350}