<commit_before>'''
################################################################################
This module provides routines that read in image recognition data sets
consisting of images and their associated ground truth.  Image recognition data
sets come in different formats so there are routines for reading each format
that has been used with nupic.vision.
################################################################################
'''

from PIL import Image
from xml.dom import minidom

DEBUG = 0



'''
################################################################################
This routine reads the XML files that contain the paths to the images and the
tags which indicate what is in the image (i.e. "ground truth").
################################################################################
'''
def getImagesAndTags(filename):
  print
  print "Reading data set: ", filename
  print
  xmldoc = minidom.parse(filename)
  # Find the path to the XML file so it can be used to find the image files.
  directoryPath = filename.replace(filename.split("/")[-1],"")
  # Read the image list from the XML file and populate images and tags.
  imageList = xmldoc.getElementsByTagName('image')
  images = []
  tags = []
  for image in imageList:
    tags.append(image.attributes['tag'].value)
    filename = image.attributes['file'].value
    images.append(Image.open(directoryPath + filename))
    #imagePatches[-1].show()
  return images, tags


<commit_msg>Made some changes to eliminate an error about the number of open files when reading large data sets.<commit_after>'''
################################################################################
This module provides routines that read in image recognition data sets
consisting of images and their associated ground truth.  Image recognition data
sets come in different formats so there are routines for reading each format
that has been used with nupic.vision.
################################################################################
'''

from PIL import Image
from xml.dom import minidom

DEBUG = 0



'''
################################################################################
This routine reads the XML files that contain the paths to the images and the
tags which indicate what is in the image (i.e. "ground truth").
################################################################################
'''
def getImagesAndTags(filename):
  print
  print "Reading data set: ", filename
  print
  xmldoc = minidom.parse(filename)
  # Find the path to the XML file so it can be used to find the image files.
  directoryPath = filename.replace(filename.split("/")[-1],"")
  # Read the image list from the XML file and populate images and tags.
  imageList = xmldoc.getElementsByTagName('image')
  images = []
  tags = []
  for image in imageList:
    tags.append(image.attributes['tag'].value)
    filename = image.attributes['file'].value
    fp = open(directoryPath + filename, 'rb')
    im = Image.open(fp)
    im.load()
    images.append(im)
    fp.close()
    #imagePatches[-1].show()
  return images, tags



{'fraction_non_alphanumeric': 0.27588424437299036, 'fraction_numerical': 0.0019292604501607716, 'mean_word_length': 5.148221343873518, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7442183', 'n_tokens_mistral': 744, 'n_tokens_neox': 709, 'n_words': 300}