<commit_before>"""
Data from the Millenium dataset
"""
__authors__ = "<NAME>"
__copyright__ = "lol"
__credits__ = ["<NAME>"]
__license__ = "lol"
__maintainer__ = "LOL"
__email__ = "<EMAIL>"

import numpy as np
import cPickle as pickle
import gzip
#from theano.compat.six.moves import xrange
from pylearn2.datasets import dense_design_matrix
from pylearn2.datasets import control
from pylearn2.datasets import cache
from pylearn2.utils import serial
import numpy as np

class NANO_PARTICLE(dense_design_matrix.DenseDesignMatrix):
    """
    Running on data from a simulation run by Bobby with a small amount of particles and large number of snapshots

    Parameters
    ----------
    which_set : str
        'train', 'valid',or 'test'
    start : Where to start slicing. Will start at 0 if None
    stop : Where to stop slicing. Will not slice if None
    """

    def __init__(self, which_set, start=None, stop=None, nParticles = None):

        self.args = locals()

        if which_set not in ['train','valid', 'test']:
            raise ValueError(
            'Unrecognized which_set value "%s".' % (which_set,) +
            '". Valid values are ["train","test", "valid"].')
        if control.get_load_data():

            path = "${PYLEARN2_DATA_PATH}/nanoParticle/"
            #have to load in the whole array from the start anyway
            import h5py
            import numpy as np
            totParticles = 262144
            if nParticles is  None:
                nParticles = totParticles

            trainingFrac = .8
            validFrac = .1

            np.random.seed(0)

            idxs = np.random.choice(totParticles, nParticles)

            #if which_set == 'train':
            slice = np.s_[idxs, :]
            #elif which_set == 'valid':
            #    slice = np.s_[int(totParticles*trainingFrac):int(totParticles*trainingFrac)+nParticles, :]
            #else:
            #    assert which_set == 'test'
            #    slice = np.s_[int(totParticles*(trainingFrac+validFrac)):int(totParticles*(trainingFrac+validFrac))+nParticles, :]

            X = np.zeros((100, nParticles*6))
            y = np.zeros((100, nParticles*3))

            colors = ['b', 'g','r','y','k','m']

            def getFilename(i):
                base = path+'snapshot_'
                if i<10:
                    out= base+'00%d.hdf5'%i
                elif i<100:
                    out= base+'0%d.hdf5'%i
                else:
                    out= base+'%d.hdf5'%i
                return serial.preprocess(out)

            absMinVel, absMaxVal = 0,0
            maxCoord= 10000 #particles in a 0-10000 cube
            for i in xrange(101):
                fname = getFilename(i)
                f = h5py.File(fname, 'r')
                ids = f['PartType1']['ParticleIDs'][()]
                sorter = ids.argsort()<commit_msg>May have fixed a critical bug, testing now.<commit_after>"""
Data from the Millenium dataset
"""
__authors__ = "<NAME>"
__copyright__ = "lol"
__credits__ = ["<NAME>"]
__license__ = "lol"
__maintainer__ = "LOL"
__email__ = "<EMAIL>"

import numpy as np
import cPickle as pickle
import gzip
#from theano.compat.six.moves import xrange
from pylearn2.datasets import dense_design_matrix
from pylearn2.datasets import control
from pylearn2.datasets import cache
from pylearn2.utils import serial
import numpy as np

class NANO_PARTICLE(dense_design_matrix.DenseDesignMatrix):
    """
    Running on data from a simulation run by Bobby with a small amount of particles and large number of snapshots

    Parameters
    ----------
    which_set : str
        'train', 'valid',or 'test'
    start : Where to start slicing. Will start at 0 if None
    stop : Where to stop slicing. Will not slice if None
    """

    def __init__(self, which_set, start=None, stop=None, nParticles = None):

        self.args = locals()

        if which_set not in ['train','valid', 'test']:
            raise ValueError(
            'Unrecognized which_set value "%s".' % (which_set,) +
            '". Valid values are ["train","test", "valid"].')
        if control.get_load_data():

            path = "${PYLEARN2_DATA_PATH}/nanoParticle/"
            #have to load in the whole array from the start anyway
            import h5py
            import numpy as np
            totParticles = 262144
            if nParticles is  None:
                nParticles = totParticles

            trainingFrac = .8
            validFrac = .1

            np.random.seed(0)

            idxs = np.random.choice(totParticles, nParticles)

            #if which_set == 'train':
            slice = np.s_[idxs, :]

            #elif which_set == 'valid':
            #    slice = np.s_[int(totParticles*trainingFrac):int(totParticles*trainingFrac)+nParticles, :]
            #else:
            #    assert which_set == 'test'
            #    slice = np.s_[int(totParticles*(trainingFrac+validFrac)):int(totParticles*(trainingFrac+validFrac))+nParticles, :]

            X = np.zeros((100, nParticles*6))
            y = np.zeros((100, nParticles*3))

            colors = ['b', 'g','r','y','k','m']

            def getFilename(i):
                base = path+'snapshot_'
                if i<10:
                    out= base+'00%d.hdf5'%i
                elif i<100:
                    out= base+'0%d.hdf5'%i
                else:
                    out= base+'%d.hdf5'%i
                return serial.preprocess(out)

            absMinVel, absMaxVal = 0,0
            maxCoord= 10000 #particles in a 0-10000 cube
            for i in xrange(101):
                fname = getFilename(i)
                f = h5py.File(fname, 'r')
                ids = f['PartType1']['ParticleIDs'][()]
                sorter = ids.argsort()
{'fraction_non_alphanumeric': 0.11425594721305782, 'fraction_numerical': 0.018405973259246398, 'mean_word_length': 1.9968782518210197, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6119251', 'n_tokens_mistral': 1838, 'n_tokens_neox': 1728, 'n_words': 466}