<commit_before>import numpy as np
import pandas as pd
import tables as tb

class HDFStore(object):
    def __init__(self, hdfname):
        self.pdh5 = pd.HDFStore(hdfname, mode='a', complevel=9,
                complib='blosc')
        self.h5 = self.pdh5._handle
        self._filters = tb.Filters(complevel=9, complib='blosc')
        self._files_df_columns = ('name', 'filenames')
        
        if not hasattr(self.h5.root, 'files'):
            df = pd.DataFrame(columns=self._files_df_columns).set_index('name')
            self.pdh5['files'] = df
        self.files = self.pdh5['files']

        if not hasattr(self.h5.root, 'data'):
            self.h5.create_group('/', 'data', filters=self._filters)

    def append_files(self, datafiles):
        if not isinstance(datafiles, (tuple, list)):
            datafiles = [datafiles,]
        
        names = []
        for data in datafiles:
            filename = data.filename
            name = self._name_fix(filename)
            names.append((name, filename))
            self._append(name, data)
        temp_df = pd.DataFrame(names, columns=self._files_df_columns)\
                .set_index('name')

        self.pdh5['files'] = pd.concat( (self.pdh5['files'], temp_df) )

    def _append(self, name, gcmsobj):
        group = self.h5.create_group('/data', name)

        for key, val in gcmsobj.__dict__.items():
            if isinstance(val, np.ndarray):
                self.h5.create_carray(group, key, obj=val,)<commit_msg>Added a couple of comments.<commit_after>import numpy as np
import pandas as pd
import tables as tb

class HDFStore(object):
    def __init__(self, hdfname):
        # Open the HDF file with pandas
        self.pdh5 = pd.HDFStore(hdfname, mode='a', complevel=9,
                complib='blosc')
        # Make a link to the original file as well
        self.h5 = self.pdh5._handle
        # Se up some compression filters
        self._filters = tb.Filters(complevel=9, complib='blosc')
        # These are the columns of my stored DataFrame
        self._files_df_columns = ('name', 'filenames')
        
        # Check to see if the 'files' DataFrame is already saved
        if not hasattr(self.h5.root, 'files'):
            # Create a blank DF
            df = pd.DataFrame(columns=self._files_df_columns).set_index('name')
            self.pdh5['files'] = df
        # Reference the DF
        self.files = self.pdh5['files']

        # Check if there is a data group, otherwise create it
        if not hasattr(self.h5.root, 'data'):
            self.h5.create_group('/', 'data', filters=self._filters)

    def append_files(self, datafiles):
        if not isinstance(datafiles, (tuple, list)):
            datafiles = [datafiles,]
        
        names = []
        for data in datafiles:
            filename = data.filename
            name = self._name_fix(filename)
            names.append((name, filename))
            self._append(name, data)
        temp_df = pd.DataFrame(names, columns=self._files_df_columns)\
                .set_index('name')

        self.pdh5['files'] = pd.concat( (self.pdh5['files'], temp_df) )

    def _append(self, name, gcmsobj):
        group = self.h5.create_group('/data', name)

        for key, val in gcmsobj.__dict__.items():
            if isinstance(val, np.ndarray):
                self.h5.create_carray(group, key, obj=val,)
{'fraction_non_alphanumeric': 0.12112259970457903, 'fraction_numerical': 0.00827178729689808, 'mean_word_length': 2.2278360343183983, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3062417', 'n_tokens_mistral': 1055, 'n_tokens_neox': 1031, 'n_words': 250}