<commit_before>import numpy as np
import h5py
import copy
import pandas
import os
import imp
import pickle
import scipy.interpolate
import warnings

def get_filenames(path, contains):
    cmd = 'ls ' + path
    ls = os.popen(cmd).read()
    all_filelist = ls.split('\n')
    try:
        all_filelist.remove('')
    except:
        pass
    filelist = []
    for i, filename in enumerate(all_filelist):
        if contains in filename:
            filelist.append( os.path.join(path, filename) )
    return filelist
    
def get_filename(path, contains):
    cmd = 'ls ' + path
    ls = os.popen(cmd).read()
    all_filelist = ls.split('\n')
    try:
        all_filelist.remove('')
    except:
        pass
    filelist = []
    for i, filename in enumerate(all_filelist):
        if contains in filename:
            return os.path.join(path, filename)
    return None
            
def load_bag_as_hdf5(bag, skip_messages=[]):
    output_fname = bag.split('.')[0] + '.hdf5'
    print output_fname
    if not os.path.exists(output_fname):
        mta.bag2hdf5.bag2hdf5(   bag,
                                 output_fname,
                                 max_strlen=200,
                                 skip_messages=skip_messages)    
    metadata = h5py.File(output_fname, 'r')
    return metadata

class Trajectory(object):
    def __init__(self, pd, objid):
        self.pd = pd[pd['objid']==objid]
        
        for column in self.pd.columns:
            self.__setattr__(column, self.pd[column].values)
            
    def __getitem__(self, key): # trajec attributes can be accessed just like a dictionary this way
        return self.__getattribute__(key)
        
class Dataset(object):
    def __init__(self, pd, copy=False):
        self.pd = pd
        self.keys = []
        self.__processed_trajecs__ = {}
        self.copy = copy
        
        if copy:
            self.load_keys()
            self.copy_trajectory_objects_to_dataset()
            del(self.pd)
            print 'Dataset loaded as a stand alone object - to save your dataset, use pickle:'
            print 'import pickle'
            print 'f = open(filename, "w+")'
            print 'pickle.dump(dataset, f)'
            print 'f.close()'
            
    def trajec(self, key):
        if not self.copy:
            return Trajectory(self.pd, key)
        else:
            raise ValueError('This is a copied dataset, use dictionary acccess: dataset.trajecs[key]') 
            
    def framestamp_to_timestamp(self, frame):
        t = self.pd.ix[frame]['time_epoch']
        try:
            return t.iloc[0]
        except:
            return t
            
    def timestamp_to_framestamp(self, t):
        first_time = self.pd['time_epoch'].values[0]
        first_frame = self.pd['frames'].values[0]
        last_time = self.pd['time_epoch'].values[-1]
        last_frame = self.pd['frames'].values[-1]
        func = scipy.interpolate.interp1d([first_time, last_time],[first_frame, last_frame])
        return int(func(t))
        
    def load_keys(self, keys=None):
        if keys is None:
            self.keys = np.unique(self.pd.objid).tolist()
        else:
            self.keys = keys
            
    def copy_trajectory_objects_to_dataset(self):
        self.trajecs = {}
        for key in self.keys:
            trajec = copy.copy(Trajectory(self.pd, key))
            self.trajecs.setdefault(key, trajec)
            
            
def load_data_as_pandas_dataframe_from_hdf5_file(filename, attributes=None):
    if '.pickle' in filename:
        pd = pandas.read_pickle(filename)
        return pd
    data = h5py.File(filename, 'r', swmr=True)['data']
    if attributes is None:
        attributes = {   'objid'                : 'objid',
                         'time_epoch_secs'      : 'header.stamp.secs',
                         'time_epoch_nsecs'     : 'header.stamp.nsecs',
                         'position_x'           : 'position.x',
                         'position_y'           : 'position.y',
                         'measurement_x'        : 'measurement.x',
                         'measurement_y'        : 'measurement.y',
                         'velocity_x'           : 'velocity.x',
                         'velocity_y'           : 'velocity.y',
                         'angle'                : 'angle',
                         'frames'               : 'header.frame_id',
                         'area'                 : 'size',
                         }
    index = data['header.frame_id'].flat
    d = {}
    for attribute, name in attributes.items():
        d.setdefault(attribute, data[name].flat)
    pd = pandas.DataFrame(d, index=index)
    pd = pd.drop(pd.index==[0]) # delete 0 frames (frames with no data)
    pd = calc_additional_columns(pd)
    # pd_subset = pd[pd.objid==key]
    return pd
    
def load_and_preprocess_data(hdf5_filename):
    '''
    requires that a configuration file be found in the same directory as the hdf5 file, with the same prefix
    
    returns: pandas dataframe, processed according to configuration file, and the configuration file instance
    '''
    if 'trackedobjects' not in hdf5_filename:
        print 'File is not a trackedobjects file, looking for a trackedobjects file in this directory'
        fname = get_filename(hdf5_filename, 'trackedobjects.hdf5')
        if fname is not None:
            hdf5_filename = fname
            print 'Found: ', fname
        else:
            raise ValueError('Could not find trackedobjects.hdf5 file')
                    
    pd = load_data_as_pandas_dataframe_from_hdf5_file(hdf5_filename, attributes=None)
    
    hdf5_basename = os.path.basename(hdf5_filename)
    directory = os.path.dirname(hdf5_filename)
    
    identifiercode = hdf5_basename.split('_trackedobjects')[0]
    config_filename = 'config_' + identifiercode + '.py'
    config_filename = get_filename(directory, config_filename)

    if config_filename is not None:
        Config = imp.load_source('Config', config_filename)
        config = Config.Config(directory, identifiercode)
        if config.__dict__.has_key('preprocess_data_function'):
            pd = config.__getattribute__('preprocess_data_function')(pd)
    else:
        config = None
        
    return pd, config
    
def find_instructions_related_to_objid(instructions, objid):
    for i, instruction in enumerate(instructions):
        if 'new_objid' in instruction.keys():
            if objid == instruction['new_objid']:
                print i
        if 'objids' in instruction.keys():
            if objid in instruction['objids']:
                print i<commit_msg>added function to grab config file only<commit_after>import numpy as np
import h5py
import copy
import pandas
import os
import imp
import pickle
import scipy.interpolate
import warnings

def get_filenames(path, contains):
    cmd = 'ls ' + path
    ls = os.popen(cmd).read()
    all_filelist = ls.split('\n')
    try:
        all_filelist.remove('')
    except:
        pass
    filelist = []
    for i, filename in enumerate(all_filelist):
        if contains in filename:
            filelist.append( os.path.join(path, filename) )
    return filelist
    
def get_filename(path, contains):
    cmd = 'ls ' + path
    ls = os.popen(cmd).read()
    all_filelist = ls.split('\n')
    try:
        all_filelist.remove('')
    except:
        pass
    filelist = []
    for i, filename in enumerate(all_filelist):
        if contains in filename:
            return os.path.join(path, filename)
    return None
            
def load_bag_as_hdf5(bag, skip_messages=[]):
    output_fname = bag.split('.')[0] + '.hdf5'
    print output_fname
    if not os.path.exists(output_fname):
        mta.bag2hdf5.bag2hdf5(   bag,
                                 output_fname,
                                 max_strlen=200,
                                 skip_messages=skip_messages)    
    metadata = h5py.File(output_fname, 'r')
    return metadata

class Trajectory(object):
    def __init__(self, pd, objid):
        self.pd = pd[pd['objid']==objid]
        
        for column in self.pd.columns:
            self.__setattr__(column, self.pd[column].values)
            
    def __getitem__(self, key): # trajec attributes can be accessed just like a dictionary this way
        return self.__getattribute__(key)
        
class Dataset(object):
    def __init__(self, pd, copy=False):
        self.pd = pd
        self.keys = []
        self.__processed_trajecs__ = {}
        self.copy = copy
        
        if copy:
            self.load_keys()
            self.copy_trajectory_objects_to_dataset()
            del(self.pd)
            print 'Dataset loaded as a stand alone object - to save your dataset, use pickle:'
            print 'import pickle'
            print 'f = open(filename, "w+")'
            print 'pickle.dump(dataset, f)'
            print 'f.close()'
            
    def trajec(self, key):
        if not self.copy:
            return Trajectory(self.pd, key)
        else:
            raise ValueError('This is a copied dataset, use dictionary acccess: dataset.trajecs[key]') 
            
    def framestamp_to_timestamp(self, frame):
        t = self.pd.ix[frame]['time_epoch']
        try:
            return t.iloc[0]
        except:
            return t
            
    def timestamp_to_framestamp(self, t):
        first_time = self.pd['time_epoch'].values[0]
        first_frame = self.pd['frames'].values[0]
        last_time = self.pd['time_epoch'].values[-1]
        last_frame = self.pd['frames'].values[-1]
        func = scipy.interpolate.interp1d([first_time, last_time],[first_frame, last_frame])
        return int(func(t))
        
    def load_keys(self, keys=None):
        if keys is None:
            self.keys = np.unique(self.pd.objid).tolist()
        else:
            self.keys = keys
            
    def copy_trajectory_objects_to_dataset(self):
        self.trajecs = {}
        for key in self.keys:
            trajec = copy.copy(Trajectory(self.pd, key))
            self.trajecs.setdefault(key, trajec)
            
            
def load_data_as_pandas_dataframe_from_hdf5_file(filename, attributes=None):
    if '.pickle' in filename:
        pd = pandas.read_pickle(filename)
        return pd
    data = h5py.File(filename, 'r', swmr=True)['data']
    if attributes is None:
        attributes = {   'objid'                : 'objid',
                         'time_epoch_secs'      : 'header.stamp.secs',
                         'time_epoch_nsecs'     : 'header.stamp.nsecs',
                         'position_x'           : 'position.x',
                         'position_y'           : 'position.y',
                         'measurement_x'        : 'measurement.x',
                         'measurement_y'        : 'measurement.y',
                         'velocity_x'           : 'velocity.x',
                         'velocity_y'           : 'velocity.y',
                         'angle'                : 'angle',
                         'frames'               : 'header.frame_id',
                         'area'                 : 'size',
                         }
    index = data['header.frame_id'].flat
    d = {}
    for attribute, name in attributes.items():
        d.setdefault(attribute, data[name].flat)
    pd = pandas.DataFrame(d, index=index)
    pd = pd.drop(pd.index==[0]) # delete 0 frames (frames with no data)
    pd = calc_additional_columns(pd)
    # pd_subset = pd[pd.objid==key]
    return pd
    
def load_and_preprocess_data(hdf5_filename):
    '''
    requires that a configuration file be found in the same directory as the hdf5 file, with the same prefix
    
    returns: pandas dataframe, processed according to configuration file, and the configuration file instance
    '''
    if 'trackedobjects' not in hdf5_filename:
        print 'File is not a trackedobjects file, looking for a trackedobjects file in this directory'
        fname = get_filename(hdf5_filename, 'trackedobjects.hdf5')
        if fname is not None:
            hdf5_filename = fname
            print 'Found: ', fname
        else:
            raise ValueError('Could not find trackedobjects.hdf5 file')
                    
    pd = load_data_as_pandas_dataframe_from_hdf5_file(hdf5_filename, attributes=None)
    
    hdf5_basename = os.path.basename(hdf5_filename)
    directory = os.path.dirname(hdf5_filename)
    
    identifiercode = hdf5_basename.split('_trackedobjects')[0]
    config_filename = 'config_' + identifiercode + '.py'
    config_filename = get_filename(directory, config_filename)

    if config_filename is not None:
        Config = imp.load_source('Config', config_filename)
        config = Config.Config(directory, identifiercode)
        if config.__dict__.has_key('preprocess_data_function'):
            pd = config.__getattribute__('preprocess_data_function')(pd)
    else:
        config = None
        
    return pd, config

def load_config_from_path(path):
    config_filename = get_filename(path, 'config')
    hdf5_file = os.path.basename(get_filename(path, 'trackedobjects.hdf5'))
    identifiercode = hdf5_file.split('_trackedobjects')[0]
    print 'identifiercode: ', identifiercode
    if config_filename is not None:
        Config = imp.load_source('Config', config_filename)
        config = Config.Config(path, identifiercode)
    else:
        config = None
    return config
    
def find_instructions_related_to_objid(instructions, objid):
    for i, instruction in enumerate(instructions):
        if 'new_objid' in instruction.keys():
            if objid == instruction['new_objid']:
                print i
        if 'objids' in instruction.keys():
            if objid in instruction['objids']:
                print i
{'fraction_non_alphanumeric': 0.09944951470375199, 'fraction_numerical': 0.005504852962480081, 'mean_word_length': 1.9577977720651243, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '670134', 'n_tokens_mistral': 4128, 'n_tokens_neox': 4063, 'n_words': 942}