<commit_before>
''' MetaDeep worker '''
import hashlib
import ssdeep as ssd
import datetime

class MetaDeepData():
    ''' This worker computes deeper meta-data '''
    dependencies = ['sample', 'meta']

    def execute(self, input_data):
        raw_bytes = input_data['sample']['raw_bytes']
        sha1 = hashlib.sha1(raw_bytes).hexdigest()
        sha256 = hashlib.sha256(raw_bytes).hexdigest()
        ssdeep = ssd.hash(raw_bytes)
        entropy = self._entropy(raw_bytes)
        output = {name:value for name,value in locals().iteritems()
                if name not in ['self', 'input_data','raw_bytes']}
        output.update(input_data['meta'])
        return output

    def _entropy(self, s):
        # Grabbed this snippet from Rosetta Code (rosettacode.org)
        import math
        from collections import Counter
        p, lns = Counter(s), float(len(s))
        return -sum( count/lns * math.log(count/lns, 2) for count in p.values())

# Unit test: Create the class, the proper input and run the execute() method for a test
def test():
    ''' meta_deep.py: Unit test'''

    # Grab a sample
    sample = {'sample':{'raw_bytes':open('../../data/pe/bad/033d91aae8ad29ed9fbb858179271232', 'rb').read(),
              'length':0, 'filename': 'bad_033d91', 'type_tag': 'pe', 'customer':'MegaCorp', 
              'import_time':datetime.datetime.now()}}

    # Send it through meta
    import meta
    input_worker = meta.MetaData()
    _raw_output = input_worker.execute(sample)
    wrapped_output = {'meta':_raw_output}

    # Now join up the inputs<commit_msg>reorg some logic for better clarity<commit_after>
''' MetaDeep worker '''
import hashlib
import ssdeep as ssd
import datetime

class MetaDeepData():
    ''' This worker computes deeper meta-data '''
    dependencies = ['sample', 'meta']

    def __init__(self):
        ''' Initialization '''
        self.meta = {}

    def execute(self, input_data):
        raw_bytes = input_data['sample']['raw_bytes']
        self.meta['sha1'] = hashlib.sha1(raw_bytes).hexdigest()
        self.meta['sha256'] = hashlib.sha256(raw_bytes).hexdigest()
        self.meta['ssdeep'] = ssd.hash(raw_bytes)
        self.meta['entropy'] = self._entropy(raw_bytes)
        self.meta.update(input_data['meta'])
        return self.meta

    def _entropy(self, s):
        # Grabbed this snippet from Rosetta Code (rosettacode.org)
        import math
        from collections import Counter
        p, lns = Counter(s), float(len(s))
        return -sum( count/lns * math.log(count/lns, 2) for count in p.values())

# Unit test: Create the class, the proper input and run the execute() method for a test
def test():
    ''' meta_deep.py: Unit test'''

    # Grab a sample
    sample = {'sample':{'raw_bytes':open('../../data/pe/bad/033d91aae8ad29ed9fbb858179271232', 'rb').read(),
              'length':0, 'filename': 'bad_033d91', 'type_tag': 'pe', 'customer':'MegaCorp', 
              'import_time':datetime.datetime.now()}}

    # Send it through meta
    import meta
    input_worker = meta.MetaData()
    _raw_output = input_worker.execute(sample)
    wrapped_output = {'meta':_raw_output}

    # Now join up the inputs
{'fraction_non_alphanumeric': 0.13787688442211055, 'fraction_numerical': 0.022613065326633167, 'mean_word_length': 3.0728900255754477, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5187995', 'n_tokens_mistral': 1040, 'n_tokens_neox': 974, 'n_words': 254}