<commit_before>from extractor import BaseExtractor
import numpy as np
import sys
sys.path.append('/home/ipl/installs/caffe-rc/python/')
import caffe
import settings


class CNN_Features_CAFFE_REFERENCE(BaseExtractor):

    def __init__(self, storage, model_file=settings.MODEL_FILE, pretrained_file=settings.PRETRAINED_FILE, image_mean=settings.ILSVRC_MEAN):
        super(CNN_Features_CAFFE_REFERENCE, self).__init__(storage)
        self.STORAGE_SUB_NAME = 'cnn_feature_caffe_reference'
        self.feature_layer = 'fc7'
        self.center_crop_index = 4

        self.sub_folder = self.storage.get_sub_folder(
            self.STORAGE_SUPER_NAME, self.STORAGE_SUB_NAME)
        self.storage.ensure_dir(self.sub_folder)

        self.model_file = model_file
        self.pretrained_file = pretrained_file
        self.image_mean = image_mean

        self.net = caffe.Classifier(self.model_file,
                                    self.pretrained_file,
                                    mean=np.load(self.image_mean),
                                    channel_swap=(2, 1, 0),
                                    raw_scale=255)  # ,
                                    # image_dims=(256, 256))
        self.net.set_mode_gpu()

    def extract_all(self, data_generator, flip=False, crop=False, force=False, bbox=None):
        for t in data_generator:
            instance_name = "%s.%s" % (t['img_id'], self.FILE_NAMES_EXT)
            instance_path = self.storage.get_instance_path(
                self.STORAGE_SUPER_NAME, self.STORAGE_SUB_NAME, instance_name)
            if force or not self.storage.check_exists(instance_path):
                im = caffe.io.load_image(t['img_file'])
                if crop:
                    assert bbox is not None
                    # TODO: move to sepatate funciton
                    x, y, w, h = bbox[int(t['img_id']) - 1]
                    im = im[y:y+h, :x+w]

                if flip:
                    im = np.fliplr(im)
                self.net.predict([im])

                des = self.net.blobs[self.feature_layer].data[
                    self.center_crop_index][:, 0, 0]

                self.storage.save_instance(instance_path, des)
            else:
                des = self.storage.load_instance(instance_path)
                if len(des.shape) > 1:
                    des = des[0, :]

            yield t, des

    def extract_one(self, img_id):<commit_msg>OMG BUG! this took me 2 hours to find!<commit_after>from extractor import BaseExtractor
import numpy as np
import sys
sys.path.append('/home/ipl/installs/caffe-rc/python/')
import caffe
import settings


class CNN_Features_CAFFE_REFERENCE(BaseExtractor):

    def __init__(self, storage, model_file=settings.MODEL_FILE, pretrained_file=settings.PRETRAINED_FILE, image_mean=settings.ILSVRC_MEAN):
        super(CNN_Features_CAFFE_REFERENCE, self).__init__(storage)
        self.STORAGE_SUB_NAME = 'cnn_feature_caffe_reference'
        self.feature_layer = 'fc7'
        self.center_crop_index = 4

        self.sub_folder = self.storage.get_sub_folder(
            self.STORAGE_SUPER_NAME, self.STORAGE_SUB_NAME)
        self.storage.ensure_dir(self.sub_folder)

        self.model_file = model_file
        self.pretrained_file = pretrained_file
        self.image_mean = image_mean

        self.net = caffe.Classifier(self.model_file,
                                    self.pretrained_file,
                                    mean=np.load(self.image_mean),
                                    channel_swap=(2, 1, 0),
                                    raw_scale=255)  # ,
                                    # image_dims=(256, 256))
        self.net.set_mode_gpu()

    def extract_all(self, data_generator, flip=False, crop=False, force=False, bbox=None):
        for t in data_generator:
            instance_name = "%s.%s" % (t['img_id'], self.FILE_NAMES_EXT)
            instance_path = self.storage.get_instance_path(
                self.STORAGE_SUPER_NAME, self.STORAGE_SUB_NAME, instance_name)
            if force or not self.storage.check_exists(instance_path):
                im = caffe.io.load_image(t['img_file'])
                if crop:
                    assert bbox is not None
                    # TODO: move to sepatate funciton
                    x, y, w, h = bbox[int(t['img_id']) - 1]
                    im = im[y:y+h, x:x+w]

                if flip:
                    im = np.fliplr(im)
                self.net.predict([im])

                des = self.net.blobs[self.feature_layer].data[
                    self.center_crop_index][:, 0, 0]

                self.storage.save_instance(instance_path, des)
            else:
                des = self.storage.load_instance(instance_path)
                if len(des.shape) > 1:
                    des = des[0, :]

            yield t, des

    def extract_one(self, img_id):
{'fraction_non_alphanumeric': 0.09391588403429972, 'fraction_numerical': 0.007962433646386281, 'mean_word_length': 1.810671256454389, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '315742', 'n_tokens_mistral': 1569, 'n_tokens_neox': 1465, 'n_words': 261}