<commit_before>"""Download the VGG face dataset from URLs given by http://www.robots.ox.ac.uk/~vgg/data/vgg_face/vgg_face_dataset.tar.gz
"""
from scipy import misc
import numpy as np
from skimage import io

import os
import socket
from urllib2 import HTTPError, URLError
from httplib import HTTPException

def main():
  socket.setdefaulttimeout(30)
  datasetDescriptor = '/home/david/datasets/download/vggface/vgg_face_dataset/files'
  textFileNames = os.listdir(datasetDescriptor)
  for textFileName in textFileNames:
    if textFileName.endswith('.txt'):
      with open(os.path.join(datasetDescriptor, textFileName), 'rt') as f:
        lines = f.readlines()
      dirName = textFileName.split('.')[0]
      classPath = os.path.join(datasetDescriptor, dirName)
      if not os.path.exists(classPath):
        os.makedirs(classPath)
      for line in lines:
        x = line.split(' ')
        fileName = x[0]
        url = x[1]
        box = np.rint(np.array(map(float, x[2:6])))  # x1,y1,x2,y2
        imagePath = os.path.join(datasetDescriptor, dirName, fileName+'.png')
        errorPath = os.path.join(datasetDescriptor, dirName, fileName+'.err')
        if not os.path.exists(imagePath) and not os.path.exists(errorPath):
          try:
            img = io.imread(url, mode='RGB')
          except (HTTPException, HTTPError, URLError, IOError, ValueError, IndexError) as e:
            errorMessage = '{}: {}'.format(url, e)
            saveErrorMessageFile(errorPath, errorMessage)
          else:
            try:
              if img.ndim == 2:
                img = toRgb(img)
              if img.ndim != 3:
                raise ValueError('Wrong number of image dimensions')
              hist = np.histogram(img, 255, density=True)
              if hist[0][0]>0.9 and hist[0][254]>0.9:
                raise ValueError('Image is mainly black or white')
              else:
                # Crop image according to dataset descriptor
                imgCropped = img[box[1]:box[3],box[0]:box[2],:]
                # Scale to 256x256
                imgResized = misc.imresize(imgCropped, (256,256))
                # Save image as .png
                misc.imsave(imagePath, imgResized)
            except ValueError as e:
              errorMessage = '{}: {}'.format(url, e)
              saveErrorMessageFile(errorPath, errorMessage)
            
def saveErrorMessageFile(fileName, errorMessage):
  print(errorMessage)<commit_msg>Caught exception related to too long file names<commit_after>"""Download the VGG face dataset from URLs given by http://www.robots.ox.ac.uk/~vgg/data/vgg_face/vgg_face_dataset.tar.gz
"""
from scipy import misc
import numpy as np
from skimage import io

import os
import socket
from urllib2 import HTTPError, URLError
from httplib import HTTPException

def main():
  socket.setdefaulttimeout(30)
  datasetDescriptor = '/home/david/datasets/download/vggface/vgg_face_dataset/files'
  textFileNames = os.listdir(datasetDescriptor)
  for textFileName in textFileNames:
    if textFileName.endswith('.txt'):
      with open(os.path.join(datasetDescriptor, textFileName), 'rt') as f:
        lines = f.readlines()
      dirName = textFileName.split('.')[0]
      classPath = os.path.join(datasetDescriptor, dirName)
      if not os.path.exists(classPath):
        os.makedirs(classPath)
      for line in lines:
        x = line.split(' ')
        fileName = x[0]
        url = x[1]
        box = np.rint(np.array(map(float, x[2:6])))  # x1,y1,x2,y2
        imagePath = os.path.join(datasetDescriptor, dirName, fileName+'.png')
        errorPath = os.path.join(datasetDescriptor, dirName, fileName+'.err')
        if not os.path.exists(imagePath) and not os.path.exists(errorPath):
          try:
            img = io.imread(url, mode='RGB')
          except (HTTPException, HTTPError, URLError, IOError, ValueError, IndexError, OSError) as e:
            errorMessage = '{}: {}'.format(url, e)
            saveErrorMessageFile(errorPath, errorMessage)
          else:
            try:
              if img.ndim == 2:
                img = toRgb(img)
              if img.ndim != 3:
                raise ValueError('Wrong number of image dimensions')
              hist = np.histogram(img, 255, density=True)
              if hist[0][0]>0.9 and hist[0][254]>0.9:
                raise ValueError('Image is mainly black or white')
              else:
                # Crop image according to dataset descriptor
                imgCropped = img[box[1]:box[3],box[0]:box[2],:]
                # Scale to 256x256
                imgResized = misc.imresize(imgCropped, (256,256))
                # Save image as .png
                misc.imsave(imagePath, imgResized)
            except ValueError as e:
              errorMessage = '{}: {}'.format(url, e)
              saveErrorMessageFile(errorPath, errorMessage)
            
def saveErrorMessageFile(fileName, errorMessage):
  print(errorMessage)
{'fraction_non_alphanumeric': 0.11313499291641368, 'fraction_numerical': 0.017405383525602103, 'mean_word_length': 2.391901166781057, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3386590', 'n_tokens_mistral': 1546, 'n_tokens_neox': 1483, 'n_words': 359}