<commit_before># --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by <NAME>
# --------------------------------------------------------

# import datasets.face
# import os
# import datasets.imdb as imdb
# import xml.dom.minidom as minidom
# import numpy as np
# import scipy.sparse
# import scipy.io as sio
# import utils.cython_bbox
# import cPickle
# import subprocess

import os
from datasets.imdb import imdb
import datasets.ds_utils as ds_utils
import xml.etree.ElementTree as ET
import numpy as np
import scipy.sparse
import scipy.io as sio
import utils.cython_bbox
import cPickle
import subprocess
import uuid
from voc_eval import voc_eval
from fast_rcnn.config import cfg
import cv2
import PIL

class face(imdb):
    def __init__(self, image_set, split, devkit_path):
        imdb.__init__(self, 'wider')
        self._image_set = image_set         # {'train', 'test'}
        self._split = split                 # {1, 2, ..., 10}
        self._devkit_path = devkit_path     # /data2/hzjiang/Data/CS2
        # self._data_path = os.path.join(self._devkit_path, 'data')
        self._data_path = self._devkit_path;
        self._classes = ('__background__', # always index 0
                         'face')
        self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))
        self._image_ext = ['.png']
        self._image_index, self._gt_roidb = self._load_image_set_index()
        # Default to roidb handler
        self._roidb_handler = self.selective_search_roidb

        # Specific config options
        self.config = {'cleanup'  : True,
                       'use_salt' : True,
                       'top_k'    : 2000}

        assert os.path.exists(self._devkit_path), \
                'Devkit path does not exist: {}'.format(self._devkit_path)
        assert os.path.exists(self._data_path), \
                'Path does not exist: {}'.format(self._data_path)

    def image_path_at(self, i):
        """
        Return the absolute path to image i in the image sequence.
        """
        return self.image_path_from_index(self._image_index[i])

    def image_path_from_index(self, index):
        """
        Construct an image path from the image's "index" identifier.
        """
        for ext in self._image_ext:
            image_path = os.path.join(self._data_path, index)
            if os.path.exists(image_path):
                break
        assert os.path.exists(image_path), \
                'Path does not exist: {}'.format(image_path)

        return image_path

    def _load_image_set_index(self):
        """
        Load the indexes listed in this dataset's image set file.
        """
        # # Example path to image set file:
        # # self._data_path + /ImageSets/val.txt
        # # read from file
        # image_set_file = 'split%d/%s_%d_annot.txt' % (self._fold, self._image_set, self._fold)
        # # image_set_file = os.path.join(self._devkit_path, image_set_file)
        # image_set_file = os.path.join('/home/hzjiang/Code/py-faster-rcnn/CS3-splits', image_set_file)
	print self._name
        image_set_file = self._name + '_face_' + self._image_set + '_annot.txt'
        image_set_file = os.path.join(self._devkit_path, image_set_file)

        # image_set_file = 'cs3_rand_train_annot.txt'
        # image_set_file = 'wider_dets_annot_from_cs3_model.txt'
        # image_set_file = 'wider_manual_annot.txt'

        assert os.path.exists(image_set_file), \
                'Path does not exist: {}'.format(image_set_file)

        image_index = []
        gt_roidb = []
        
        with open(image_set_file) as f:
            # print len(f.lines())
            lines = f.readlines()

            idx = 0
            while idx < len(lines):
                image_name = lines[idx].split('\n')[0]
                image_name = os.path.join('WIDER_%s/images' % self._image_set, image_name)
                # print image_name
                image_ext = os.path.splitext(image_name)[1].lower()
                # print image_ext
                assert(image_ext == '.png' or image_ext == '.jpg' or image_ext == '.jpeg')

                image = PIL.Image.open(os.path.join(self._data_path, image_name))
                imw = image.size[0]<commit_msg>Update face.py<commit_after># --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by <NAME>
# --------------------------------------------------------

# import datasets.face
# import os
# import datasets.imdb as imdb
# import xml.dom.minidom as minidom
# import numpy as np
# import scipy.sparse
# import scipy.io as sio
# import utils.cython_bbox
# import cPickle
# import subprocess

import os
from datasets.imdb import imdb
import datasets.ds_utils as ds_utils
import xml.etree.ElementTree as ET
import numpy as np
import scipy.sparse
import scipy.io as sio
import utils.cython_bbox
import cPickle
import subprocess
import uuid
from voc_eval import voc_eval
from fast_rcnn.config import cfg
import cv2
import PIL

class face(imdb):
    def __init__(self, image_set, split, devkit_path):
        imdb.__init__(self, 'wider')
        self._image_set = image_set         # {'train', 'test'}
        self._split = split                 # {1, 2, ..., 10}
        self._devkit_path = devkit_path     # /data2/hzjiang/Data/CS2
        # self._data_path = os.path.join(self._devkit_path, 'data')
        self._data_path = self._devkit_path;
        self._classes = ('__background__', # always index 0
                         'face')
        self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))
        self._image_ext = ['.png']
        self._image_index, self._gt_roidb = self._load_image_set_index()
        # Default to roidb handler
        self._roidb_handler = self.selective_search_roidb

        # Specific config options
        self.config = {'cleanup'  : True,
                       'use_salt' : True,
                       'top_k'    : 2000}

        assert os.path.exists(self._devkit_path), \
                'Devkit path does not exist: {}'.format(self._devkit_path)
        assert os.path.exists(self._data_path), \
                'Path does not exist: {}'.format(self._data_path)

    def image_path_at(self, i):
        """
        Return the absolute path to image i in the image sequence.
        """
        return self.image_path_from_index(self._image_index[i])

    def image_path_from_index(self, index):
        """
        Construct an image path from the image's "index" identifier.
        """
        for ext in self._image_ext:
            image_path = os.path.join(self._data_path, index)
            if os.path.exists(image_path):
                break
        assert os.path.exists(image_path), \
                'Path does not exist: {}'.format(image_path)

        return image_path

    def _load_image_set_index(self):
        """
        Load the indexes listed in this dataset's image set file.
        """
        # # Example path to image set file:
        # # self._data_path + /ImageSets/val.txt
        # # read from file
        # image_set_file = 'split%d/%s_%d_annot.txt' % (self._fold, self._image_set, self._fold)
        # # image_set_file = os.path.join(self._devkit_path, image_set_file)
        # image_set_file = os.path.join('/home/hzjiang/Code/py-faster-rcnn/CS3-splits', image_set_file)
        image_set_file = self._name + '_face_' + self._image_set + '_annot.txt'
        image_set_file = os.path.join(self._devkit_path, image_set_file)

        # image_set_file = 'cs3_rand_train_annot.txt'
        # image_set_file = 'wider_dets_annot_from_cs3_model.txt'
        # image_set_file = 'wider_manual_annot.txt'

        assert os.path.exists(image_set_file), \
                'Path does not exist: {}'.format(image_set_file)

        image_index = []
        gt_roidb = []
        
        with open(image_set_file) as f:
            # print len(f.lines())
            lines = f.readlines()

            idx = 0
            while idx < len(lines):
                image_name = lines[idx].split('\n')[0]
                image_name = os.path.join('WIDER_%s/images' % self._image_set, image_name)
                # print image_name
                image_ext = os.path.splitext(image_name)[1].lower()
                # print image_ext
                assert(image_ext == '.png' or image_ext == '.jpg' or image_ext == '.jpeg')

                image = PIL.Image.open(os.path.join(self._data_path, image_name))
                imw = image.size[0]
{'fraction_non_alphanumeric': 0.12935895960409713, 'fraction_numerical': 0.005294049948210381, 'mean_word_length': 2.071756804524567, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3907743', 'n_tokens_mistral': 2763, 'n_tokens_neox': 2660, 'n_words': 610}