<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2017 Mozilla Corporation

import sys
import os
import jmespath
import yaml
from mozdef_util.utilities.toUTC import toUTC


class message(object):
    def __init__(self):
        '''
            Plugin used to fix object type discretions with cloudtrail messages
        '''
        self.registration = ['githubeventsqs']
        self.priority = 10

        with open('github_mapping.yml', 'r') as f:
            map = f.read()

        yap = yaml.load(map)
        self.eventtypes = yap.keys()
        self.yap = yap
        del(map)

    def onMessage(self, message, metadata):

        if 'tags' not in message:
            return (message, metadata)
        if 'githubeventsqs' not in message['tags']:
            return (message, metadata)

        newmessage = {}
        newmessage['details'] = {}

        newmessage['category'] = 'github'
        newmessage['tags'] = ['github', 'webhook']
        newmessage['eventsource'] = 'githubeventsqs'
        if 'event' in message['details']:
            newmessage['source'] = message['details']['event']
        else:
            newmessage['source'] = 'UNKNOWN'
        if 'request_id' in message['details']:
            newmessage['details']['request_id'] = message['details']['request_id']
        else:
            newmessage['details']['request_id'] = 'UNKNOWN'

        # iterate through top level keys - push, etc
        if newmessage['source'] in self.eventtypes:
            for key in self.yap[newmessage['source']]:
                mappedvalue = jmespath.search(self.yap[newmessage['source']][key], message)
                # JMESPath likes to silently return a None object
                if mappedvalue is not None:
                    newmessage['details'][key] = mappedvalue
            if 'commit_ts' in newmessage['details']:
                newmessage['timestamp'] = newmessage['details']['commit_ts']
                newmessage['utctimestamp'] = toUTC(newmessage['details']['commit_ts']).isoformat()
        else:
            newmessage = None

        return (newmessage, metadata)
<commit_msg>the pep check sometimes wants the empty line at the end of the file and sometimes it does not. go figure.<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2017 Mozilla Corporation

import sys
import os
import jmespath
import yaml
from mozdef_util.utilities.toUTC import toUTC


class message(object):
    def __init__(self):
        '''
            Plugin used to fix object type discretions with cloudtrail messages
        '''
        self.registration = ['githubeventsqs']
        self.priority = 10

        with open('github_mapping.yml', 'r') as f:
            map = f.read()

        yap = yaml.load(map)
        self.eventtypes = yap.keys()
        self.yap = yap
        del(map)

    def onMessage(self, message, metadata):

        if 'tags' not in message:
            return (message, metadata)
        if 'githubeventsqs' not in message['tags']:
            return (message, metadata)

        newmessage = {}
        newmessage['details'] = {}

        newmessage['category'] = 'github'
        newmessage['tags'] = ['github', 'webhook']
        newmessage['eventsource'] = 'githubeventsqs'
        if 'event' in message['details']:
            newmessage['source'] = message['details']['event']
        else:
            newmessage['source'] = 'UNKNOWN'
        if 'request_id' in message['details']:
            newmessage['details']['request_id'] = message['details']['request_id']
        else:
            newmessage['details']['request_id'] = 'UNKNOWN'

        # iterate through top level keys - push, etc
        if newmessage['source'] in self.eventtypes:
            for key in self.yap[newmessage['source']]:
                mappedvalue = jmespath.search(self.yap[newmessage['source']][key], message)
                # JMESPath likes to silently return a None object
                if mappedvalue is not None:
                    newmessage['details'][key] = mappedvalue
            if 'commit_ts' in newmessage['details']:
                newmessage['timestamp'] = newmessage['details']['commit_ts']
                newmessage['utctimestamp'] = toUTC(newmessage['details']['commit_ts']).isoformat()
        else:
            newmessage = None

        return (newmessage, metadata)
{'fraction_non_alphanumeric': 0.11782283884738527, 'fraction_numerical': 0.004268943436499467, 'mean_word_length': 2.4005805515239476, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7194850', 'n_tokens_mistral': 1298, 'n_tokens_neox': 1241, 'n_words': 376}