<commit_before>import os
import uuid

### arast ###
import client


#### Single Files #####
class FileInfo(dict):
    def __init__(self, filename, shock_url=None, shock_id=None, name=None,
                 filesize=None, create_time=None, metadata=None, keep_name=False, *args):
        dict.__init__(self, *args)
        self.update({'shock_url': shock_url,
                     'shock_id' : shock_id,
                     'filesize': filesize,
                     'filename': os.path.basename(filename),
                     'local_file': filename,
                     'keep_name': keep_name,
                     'create_time': create_time,
                     'metadata': metadata})
        self.id = uuid.uuid4()

        #TODO Auto populate filesize
        
##### Set of Files ######
class FileSet(dict):
    def __init__(self, set_type, file_infos, 
                 **kwargs):
        dict.__init__(self, **kwargs)
        self.update({'type': set_type,
                     'file_infos': []})
        self.id = uuid.uuid4()
        if type(file_infos) is list:
            for f in file_infos:
                self['file_infos'].append(f)
        else:
            self['file_infos'] = [file_infos]

    @property
    def files(self):
        """ Returns file paths of all files in set"""
        return [f['local_file'] for f in self['file_infos']]

    @property
    def name(self):
        return self['name'] or None

    @property
    def type(self):<commit_msg>FileSet: @property shock_nodes<commit_after>import os
import uuid

### arast ###
import client


#### Single Files #####
class FileInfo(dict):
    def __init__(self, filename, shock_url=None, shock_id=None, name=None,
                 filesize=None, create_time=None, metadata=None, keep_name=False, *args):
        dict.__init__(self, *args)
        self.update({'shock_url': shock_url,
                     'shock_id' : shock_id,
                     'filesize': filesize,
                     'filename': os.path.basename(filename),
                     'local_file': filename,
                     'keep_name': keep_name,
                     'create_time': create_time,
                     'metadata': metadata})
        self.id = uuid.uuid4()

        #TODO Auto populate filesize
        
##### Set of Files ######
class FileSet(dict):
    def __init__(self, set_type, file_infos, 
                 **kwargs):
        dict.__init__(self, **kwargs)
        self.update({'type': set_type,
                     'file_infos': []})
        self.id = uuid.uuid4()
        if type(file_infos) is list:
            for f in file_infos:
                self['file_infos'].append(f)
        else:
            self['file_infos'] = [file_infos]

    @property
    def files(self):
        """ Returns file paths of all files in set"""
        return [f['local_file'] for f in self['file_infos']]

    @property
    def shock_nodes(self):
        return [fi['shock_id'] for fi in self['file_infos']]

    @property
    def name(self):
        return self['name'] or None

    @property
    def type(self):
{'fraction_non_alphanumeric': 0.12857142857142856, 'fraction_numerical': 0.0012987012987012987, 'mean_word_length': 1.9314938154138914, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3066765', 'n_tokens_mistral': 920, 'n_tokens_neox': 895, 'n_words': 214}