<commit_before>"""
An interface to the small NORB dataset. Unlike `./norb_small.py`, this reads
the original NORB file format, not the LISA lab's `.npy` version.

Currently only supports the Small NORB Dataset.

Download the dataset from
`here <http://www.cs.nyu.edu/~ylclab/data/norb-v1.0-small/>`_.

NORB dataset(s) by <NAME> and <NAME>.
"""

__authors__ = "<NAME> and <NAME>"
__copyright__ = "Copyright 2010-2014, Universite de Montreal"
__credits__ = __authors__.split(" and ")
__license__ = "3-clause BSD"
__maintainer__ = "<NAME>"
__email__ = "mkg alum mit edu (@..)"


import bz2
import gzip
import logging
import os
import warnings

import numpy
import theano


from pylearn2.datasets import dense_design_matrix
from pylearn2.datasets.cache import datasetCache
from pylearn2.space import VectorSpace, Conv2DSpace, CompositeSpace


logger = logging.getLogger(__name__)


class SmallNORB(dense_design_matrix.DenseDesignMatrix):
    """
    An interface to the small NORB dataset.

    If instantiated with default arguments, target labels are integers
    representing categories, which can be looked up using

      category_name = SmallNORB.get_category(label).

    If instantiated with multi_target=True, labels are vectors of indices
    representing:

      [ category, instance, elevation, azimuth, lighting ]

    Like with category, there are class methods that map these ints to their
    actual values, e.g:

      category = SmallNORB.get_category(label[0])
      elevation = SmallNORB.get_elevation_degrees(label[2])

    Parameters
    ----------
    which_set: str
        Must be 'train' or 'test'.
    multi_target: bool, optional
        If False, each label is an integer labeling the image catergory. If
        True, each label is a vector: [category, instance, lighting, elevation,
        azimuth]. All labels are given as integers. Use the categories,
        elevation_degrees, and azimuth_degrees arrays to map from these
        integers to actual values.
    """

    # Actual image shape may change, e.g. after being preprocessed by
    # datasets.preprocessing.Downsample
    original_image_shape = (96, 96)

    _categories = ['animal',  # four-legged animal
                   'human',  # human figure
                   'airplane',
                   'truck',
                   'car']

    @classmethod
    def get_category(cls, scalar_label):
        """
        Returns the category string corresponding to an integer category label.
        """
        return cls._categories[int(scalar_label)]

    @classmethod
    def get_elevation_degrees(cls, scalar_label):
        """
        Returns the elevation, in degrees, corresponding to an integer
        elevation label.
        """
        scalar_label = int(scalar_label)
        assert scalar_label >= 0
        assert scalar_label < 9
        return 30 + 5 * scalar_label

    @classmethod
    def get_azimuth_degrees(cls, scalar_label):
        """
        Returns the azimuth, in degrees, corresponding to an integer
        label.
        """
        scalar_label = int(scalar_label)
        assert scalar_label >= 0<commit_msg>commit before changing Norb's label remapping so that labels[:, :5] are densely populated.<commit_after>"""
An interface to the small NORB dataset. Unlike `./norb_small.py`, this reads
the original NORB file format, not the LISA lab's `.npy` version.

Currently only supports the Small NORB Dataset.

Download the dataset from
`here <http://www.cs.nyu.edu/~ylclab/data/norb-v1.0-small/>`_.

NORB dataset(s) by <NAME> and <NAME>.
"""

__authors__ = "<NAME> and <NAME>"
__copyright__ = "Copyright 2010-2014, Universite de Montreal"
__credits__ = __authors__.split(" and ")
__license__ = "3-clause BSD"
__maintainer__ = "Matthew <NAME>"
__email__ = "mkg alum mit edu (@..)"


import bz2
import gzip
import logging
import os
import warnings

import numpy
import theano


from pylearn2.datasets import dense_design_matrix
from pylearn2.datasets.cache import datasetCache
from pylearn2.space import VectorSpace, Conv2DSpace, CompositeSpace


logger = logging.getLogger(__name__)


class SmallNORB(dense_design_matrix.DenseDesignMatrix):
    """
    An interface to the small NORB dataset.

    If instantiated with default arguments, target labels are integers
    representing categories, which can be looked up using

      category_name = SmallNORB.get_category(label).

    If instantiated with multi_target=True, labels are vectors of indices
    representing:

      [ category, instance, elevation, azimuth, lighting ]

    Like with category, there are class methods that map these ints to their
    actual values, e.g:

      category = SmallNORB.get_category(label[0])
      elevation = SmallNORB.get_elevation_degrees(label[2])

    Parameters
    ----------
    which_set: str
        Must be 'train' or 'test'.
    multi_target: bool, optional
        If False, each label is an integer labeling the image catergory. If
        True, each label is a vector: [category, instance, lighting, elevation,
        azimuth]. All labels are given as integers. Use the categories,
        elevation_degrees, and azimuth_degrees arrays to map from these
        integers to actual values.
    """

    # Actual image shape may change, e.g. after being preprocessed by
    # datasets.preprocessing.Downsample
    original_image_shape = (96, 96)

    _categories = ['blank',  # no object; background & distractor only
                   'animal',  # four-legged animal
                   'human',  # human figure
                   'airplane',
                   'truck',
                   'car']

    @classmethod
    def get_category(cls, scalar_label):
        """
        Returns the category string corresponding to an integer category label.
        """
        return cls._categories[int(scalar_label)]

    @classmethod
    def get_elevation_degrees(cls, scalar_label):
        """
        Returns the elevation, in degrees, corresponding to an integer
        elevation label.
        """
        scalar_label = int(scalar_label)
        assert scalar_label >= 0
        assert scalar_label < 9
        return 30 + 5 * scalar_label

    @classmethod
    def get_azimuth_degrees(cls, scalar_label):
        """
        Returns the azimuth, in degrees, corresponding to an integer
        label.
        """
        scalar_label = int(scalar_label)
        assert scalar_label >= 0
{'fraction_non_alphanumeric': 0.08281347473487212, 'fraction_numerical': 0.008889582033686836, 'mean_word_length': 2.865581675708258, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3363423', 'n_tokens_mistral': 1901, 'n_tokens_neox': 1748, 'n_words': 616}