<commit_before># Functions performing various input/output operations for the ChaLearn AutoML challenge

# Main contributor: <NAME>, August 2014
# Edits: <NAME>, October 2014

# ALL INFORMATION, SOFTWARE, DOCUMENTATION, AND DATA ARE PROVIDED "AS-IS". 
# ISABELLE GUYON, CHALEARN, AND/OR OTHER ORGANIZERS OR CODE AUTHORS DISCLAIM
# ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ANY PARTICULAR PURPOSE, AND THE
# WARRANTY OF NON-INFRIGEMENT OF ANY THIRD PARTY'S INTELLECTUAL PROPERTY RIGHTS. 
# IN NO EVENT SHALL ISABELLE GUYON AND/OR OTHER ORGANIZERS BE LIABLE FOR ANY SPECIAL, 
# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF SOFTWARE, DOCUMENTS, MATERIALS, 
# PUBLICATIONS, OR INFORMATION MADE AVAILABLE FOR THE CHALLENGE. 

import data_converter
import data_io
from data_io import vprint
import numpy as np
try:
    import cPickle as pickle
except:
    import pickle
import os
import time

import scipy.sparse

from AutoSklearn.implementations.OneHotEncoder import OneHotEncoder

import input_routines

class DataManager:
    ''' This class aims at loading and saving data easily with a cache and at generating a dictionary (self.info) in which each key is a feature (e.g. : name, format, feat_num,...).
    Methods defined here are :
    __init__ (...)
        x.__init__([(feature, value)]) -> void        
        Initialize the info dictionary with the tuples (feature, value) given as argument. It recognizes the type of value (int, string) and assign value to info[feature]. An unlimited number of tuple can be sent.
    
    getInfo (...)
        x.getInfo (filename) -> void        
        Fill the dictionary with an info file. Each line of the info file must have this format 'feature' : value
        The information is obtained from the public.info file if it exists, or inferred from the data files        

    getInfoFromFile (...)
        x.getInfoFromFile (filename) -> void        
        Fill the dictionary with an info file. Each line of the info file must have this format 'feature' : value
        
    getFormatData (...)
        x.getFormatData (filename) -> str        
        Get the format of the file ('dense', 'sparse' or 'sparse_binary') either using the 'is_sparse' feature if it exists (for example after a call of getInfoFromFile function) and then determing if it's binary or not, or determining it alone.
        
    getNbrFeatures (...)
        x.getNbrFeatures (*filenames) -> int        
        Get the number of features, using the data files given. It first checks the format of the data. If it's a matrix, the number of features is trivial. If it's a sparse file, it gets the max feature index given in every files.
        
    getTypeProblem (...)
        x.getTypeProblem (filename) -> str        
        Get the kind of problem ('binary.classification', 'multiclass.classification', 'multilabel.classification', 'regression'), using the solution file given.
    '''
    
    def __init__(self, basename, input_dir, verbose=False):
        '''Constructor'''
        self.use_pickle = False # Turn this to true to save data as pickle (inefficient)
        self.basename = basename
        if basename in input_dir:
            self.input_dir = input_dir 
        else:
            self.input_dir = input_dir + "/" + basename + "/"   
        if self.use_pickle:
            if os.path.exists ("tmp"):
                self.tmp_dir = "tmp"
            elif os.path.exists ("../tmp"):
                self.tmp_dir = "../tmp" 
            else:
                os.makedirs("tmp")
                self.tmp_dir = "tmp"

        info_file = os.path.join (self.input_dir, basename + '_public.info')
        self.info = {}
        self.getInfo (info_file)
        self.feat_type = self.loadType (os.path.join(self.input_dir, basename + '_feat.type'), verbose=verbose)
        self.data = {}

        Xtr = self.loadData(os.path.join(self.input_dir, basename + '_train.data'),
                            self.feat_type, verbose=verbose)

        # For simplicity, we convert the loaded file
        # TODO what happens with sparse matrices here?
        if 'Categorical' in self.feat_type:
            mask = np.array([bool('Categorical' == feat)
                             for feat in self.feat_type])

            encoder = OneHotEncoder(categorical_features=mask,
                                    dtype=np.float64, sparse=True)
            Xtr = encoder.fit_transform(Xtr)

        Ytr = self.loadLabel(os.path.join(self.input_dir, basename + '_train.solution'),
                             verbose=verbose)

        Xva = self.loadData(os.path.join(self.input_dir, basename + '_valid.data'),
                            self.feat_type, verbose=verbose)
        if 'Categorical' in self.feat_type:
            Xva = encoder.transform(Xva)

        Xte = self.loadData(os.path.join(self.input_dir, basename + '_test.data'),
                            self.feat_type, verbose=verbose)
        if 'Categorical' in self.feat_type:
            Xte = encoder.transform(Xte)

        self.data['X_train'] = Xtr
        self.data['Y_train'] = Ytr
        self.data['X_valid'] = Xva
        self.data['X_test'] = Xte
          
    def __repr__(self):<commit_msg>Convert data to dense format after 1HotEncoding if it was dense before<commit_after># Functions performing various input/output operations for the ChaLearn AutoML challenge

# Main contributor: <NAME>, August 2014
# Edits: <NAME>, October 2014

# ALL INFORMATION, SOFTWARE, DOCUMENTATION, AND DATA ARE PROVIDED "AS-IS". 
# ISABELLE GUYON, CHALEARN, AND/OR OTHER ORGANIZERS OR CODE AUTHORS DISCLAIM
# ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ANY PARTICULAR PURPOSE, AND THE
# WARRANTY OF NON-INFRIGEMENT OF ANY THIRD PARTY'S INTELLECTUAL PROPERTY RIGHTS. 
# IN NO EVENT SHALL ISABELLE GUYON AND/OR OTHER ORGANIZERS BE LIABLE FOR ANY SPECIAL, 
# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF SOFTWARE, DOCUMENTS, MATERIALS, 
# PUBLICATIONS, OR INFORMATION MADE AVAILABLE FOR THE CHALLENGE. 

import data_converter
import data_io
from data_io import vprint
import numpy as np
try:
    import cPickle as pickle
except:
    import pickle
import os
import time

import scipy.sparse

from AutoSklearn.implementations.OneHotEncoder import OneHotEncoder

import input_routines

class DataManager:
    ''' This class aims at loading and saving data easily with a cache and at generating a dictionary (self.info) in which each key is a feature (e.g. : name, format, feat_num,...).
    Methods defined here are :
    __init__ (...)
        x.__init__([(feature, value)]) -> void        
        Initialize the info dictionary with the tuples (feature, value) given as argument. It recognizes the type of value (int, string) and assign value to info[feature]. An unlimited number of tuple can be sent.
    
    getInfo (...)
        x.getInfo (filename) -> void        
        Fill the dictionary with an info file. Each line of the info file must have this format 'feature' : value
        The information is obtained from the public.info file if it exists, or inferred from the data files        

    getInfoFromFile (...)
        x.getInfoFromFile (filename) -> void        
        Fill the dictionary with an info file. Each line of the info file must have this format 'feature' : value
        
    getFormatData (...)
        x.getFormatData (filename) -> str        
        Get the format of the file ('dense', 'sparse' or 'sparse_binary') either using the 'is_sparse' feature if it exists (for example after a call of getInfoFromFile function) and then determing if it's binary or not, or determining it alone.
        
    getNbrFeatures (...)
        x.getNbrFeatures (*filenames) -> int        
        Get the number of features, using the data files given. It first checks the format of the data. If it's a matrix, the number of features is trivial. If it's a sparse file, it gets the max feature index given in every files.
        
    getTypeProblem (...)
        x.getTypeProblem (filename) -> str        
        Get the kind of problem ('binary.classification', 'multiclass.classification', 'multilabel.classification', 'regression'), using the solution file given.
    '''
    
    def __init__(self, basename, input_dir, verbose=False):
        '''Constructor'''
        self.use_pickle = False # Turn this to true to save data as pickle (inefficient)
        self.basename = basename
        if basename in input_dir:
            self.input_dir = input_dir 
        else:
            self.input_dir = input_dir + "/" + basename + "/"   
        if self.use_pickle:
            if os.path.exists ("tmp"):
                self.tmp_dir = "tmp"
            elif os.path.exists ("../tmp"):
                self.tmp_dir = "../tmp" 
            else:
                os.makedirs("tmp")
                self.tmp_dir = "tmp"

        info_file = os.path.join (self.input_dir, basename + '_public.info')
        self.info = {}
        self.getInfo (info_file)
        self.feat_type = self.loadType (os.path.join(self.input_dir, basename + '_feat.type'), verbose=verbose)
        self.data = {}

        sparse = True if self.info['is_sparse'] == 1 else False

        Xtr = self.loadData(os.path.join(self.input_dir, basename + '_train.data'),
                            self.feat_type, verbose=verbose)
        if 'Categorical' in self.feat_type:
            mask = np.array([bool('Categorical' == feat)
                             for feat in self.feat_type])

            encoder = OneHotEncoder(categorical_features=mask,
                                    dtype=np.float64, sparse=sparse)
            Xtr = encoder.fit_transform(Xtr)

        Ytr = self.loadLabel(os.path.join(self.input_dir, basename + '_train.solution'),
                             verbose=verbose)

        Xva = self.loadData(os.path.join(self.input_dir, basename + '_valid.data'),
                            self.feat_type, verbose=verbose)
        if 'Categorical' in self.feat_type:
            Xva = encoder.transform(Xva)

        Xte = self.loadData(os.path.join(self.input_dir, basename + '_test.data'),
                            self.feat_type, verbose=verbose)
        if 'Categorical' in self.feat_type:
            Xte = encoder.transform(Xte)

        self.data['X_train'] = Xtr
        self.data['Y_train'] = Ytr
        self.data['X_valid'] = Xva
        self.data['X_test'] = Xte
          
    def __repr__(self):
{'fraction_non_alphanumeric': 0.08724769788856851, 'fraction_numerical': 0.0020463212724397732, 'mean_word_length': 2.4263862332695987, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1531716', 'n_tokens_mistral': 3124, 'n_tokens_neox': 2956, 'n_words': 1059}