<commit_before>from __future__ import division

import keras.applications.imagenet_utils
import keras.preprocessing.image
import keras.backend

from .image import random_transform_batch, resize_image

import cv2

import os
import numpy as np
import time

from pycocotools.coco import COCO

from .anchors import anchors_for_image, anchor_targets


class CocoIterator(keras.preprocessing.image.Iterator):
    def __init__(
        self,
        data_dir,
        set_name,
        image_data_generator,
        image_min_side=600,
        image_max_side=1024,
        batch_size=1,
        shuffle=True,
        seed=None,
    ):
        self.data_dir             = data_dir
        self.set_name             = set_name
        self.coco                 = COCO(os.path.join(data_dir, 'annotations', 'instances_' + set_name + '.json'))
        self.image_ids            = self.coco.getImgIds()
        self.image_data_generator = image_data_generator
        self.image_min_side       = image_min_side
        self.image_max_side       = image_max_side

        if seed is None:
            seed = np.uint32(time.time() * 1000)

        self.load_classes()

        assert(batch_size == 1), "Currently only batch_size=1 is allowed."

        super(CocoIterator, self).__init__(len(self.image_ids), batch_size, shuffle, seed)

    def load_classes(self):
        # load class names (name -> label)
        categories = self.coco.loadCats(self.coco.getCatIds())
        self.classes = {'__background__': 0}
        for c in categories:
            self.classes[c['name']] = c['id']

        # also load the reverse (label -> name)
        self.labels = {}
        for key, value in self.classes.items():
            self.labels[value] = key

    def load_image(self, image_index):
        coco_image         = self.coco.loadImgs(self.image_ids[image_index])[0]
        path               = os.path.join(self.data_dir, self.set_name, coco_image['file_name'])
        image              = cv2.imread(path, cv2.IMREAD_COLOR)
        image, image_scale = resize_image(image, min_side=self.image_min_side, max_side=self.image_max_side)

        # set ground truth boxes
        annotations_ids = self.coco.getAnnIds(imgIds=coco_image['id'], iscrowd=False)

        # some images appear to miss annotations (like image with id 257034)
        if len(annotations_ids) == 0:
            return None<commit_msg>Use COCO structure as defined by cocoapi. https://github.com/cocodataset/cocoapi<commit_after>from __future__ import division

import keras.applications.imagenet_utils
import keras.preprocessing.image
import keras.backend

from .image import random_transform_batch, resize_image

import cv2

import os
import numpy as np
import time

from pycocotools.coco import COCO

from .anchors import anchors_for_image, anchor_targets


class CocoIterator(keras.preprocessing.image.Iterator):
    def __init__(
        self,
        data_dir,
        set_name,
        image_data_generator,
        image_min_side=600,
        image_max_side=1024,
        batch_size=1,
        shuffle=True,
        seed=None,
    ):
        self.data_dir             = data_dir
        self.set_name             = set_name
        self.coco                 = COCO(os.path.join(data_dir, 'annotations', 'instances_' + set_name + '.json'))
        self.image_ids            = self.coco.getImgIds()
        self.image_data_generator = image_data_generator
        self.image_min_side       = image_min_side
        self.image_max_side       = image_max_side

        if seed is None:
            seed = np.uint32(time.time() * 1000)

        self.load_classes()

        assert(batch_size == 1), "Currently only batch_size=1 is allowed."

        super(CocoIterator, self).__init__(len(self.image_ids), batch_size, shuffle, seed)

    def load_classes(self):
        # load class names (name -> label)
        categories = self.coco.loadCats(self.coco.getCatIds())
        self.classes = {'__background__': 0}
        for c in categories:
            self.classes[c['name']] = c['id']

        # also load the reverse (label -> name)
        self.labels = {}
        for key, value in self.classes.items():
            self.labels[value] = key

    def load_image(self, image_index):
        coco_image         = self.coco.loadImgs(self.image_ids[image_index])[0]
        path               = os.path.join(self.data_dir, 'images', self.set_name, coco_image['file_name'])
        image              = cv2.imread(path, cv2.IMREAD_COLOR)
        image, image_scale = resize_image(image, min_side=self.image_min_side, max_side=self.image_max_side)

        # set ground truth boxes
        annotations_ids = self.coco.getAnnIds(imgIds=coco_image['id'], iscrowd=False)

        # some images appear to miss annotations (like image with id 257034)
        if len(annotations_ids) == 0:
            return None
{'fraction_non_alphanumeric': 0.09402586737836173, 'fraction_numerical': 0.011496612605214535, 'mean_word_length': 2.4431095406360424, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3731093', 'n_tokens_mistral': 1559, 'n_tokens_neox': 1488, 'n_words': 321}