<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation
#
# Contributors:
# <NAME> <EMAIL>

import netaddr
import pygeoip


def isIP(ip):
    try:
        netaddr.IPNetwork(ip)
        return True
    except:
        return False


def ipLocation(ip):
    location = dict()
    try:
        gi = pygeoip.GeoIP('/home/mozdef/envs/mozdef/bot/GeoLiteCity.dat', pygeoip.MEMORY_CACHE)
        geoDict = gi.record_by_addr(str(netaddr.IPNetwork(ip)[0]))
        if geoDict is not None:
            return geoDict
        else:
            location['location'] = 'unknown'
    except ValueError as e:
        pass
    return location


class message(object):
    def __init__(self):
        '''register our criteria for being passed a message
           return a dict with fieldname:None to be sent anything with that field
           return a dict with fieldname:Value to be sent anything with that field/value
           return a string to be sent anything with any field matching that string evaluated as a regex.
           set the priority if you have a preference for order of plugins to run.
           0 goes first, 100 is assumed/default if not sent
        '''

        rdict = dict()
        rdict['details'] = dict()
        rdict['details']['sourceipaddress'] = None
        rdict['details']['destinationipaddress'] = None
        self.registration = rdict
        self.priority = 20

    def onMessage(self, message):
        if 'details' in message.keys():
            if 'sourceipaddress' in message['details'].keys():
                ipText = message['details']['sourceipaddress']
                if isIP(ipText):
                    ip = netaddr.IPNetwork(ipText)[0]
                    if (not ip.is_loopback() and not ip.is_private() and not ip.is_reserved()):
                        '''lookup geoip info'''
                        message['details']['sourceipgeolocation'] = ipLocation(ipText)
                else:
                    # invalid ip sent in the field
                    # if we send on, elastic search will error, so set it
                    # to a valid, yet meaningless value
                    message['details']['sourceipaddress'] = '0.0.0.0'

            if 'destinationipaddress' in message['details'].keys():
                ipText = message['details']['destinationipaddress']
                if isIP(ipText):
                    ip = netaddr.IPNetwork(ipText)[0]
                    if (not ip.is_loopback() and not ip.is_private() and not ip.is_reserved()):
                        '''lookup geoip info'''
                        message['details']['destinationipgeolocation'] = ipLocation(ipText)
                else:
                    # invalid ip sent in the field
                    # if we send on, elastic search will error, so set it
                    # to a valid, yet meaningless value
                    message['details']['destinationipaddress'] = '0.0.0.0'
        return message

        <commit_msg>classier geoip with perf improvement for .dat file loading<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation
#
# Contributors:
# <NAME> <EMAIL>

import netaddr
import pygeoip


def isIP(ip):
    try:
        netaddr.IPNetwork(ip)
        return True
    except:
        return False


class message(object):
    def __init__(self):
        '''register our criteria for being passed a message
           return a dict with fieldname:None to be sent anything with that field
           return a dict with fieldname:Value to be sent anything with that field/value
           return a string to be sent anything with any field matching that string evaluated as a regex.
           set the priority if you have a preference for order of plugins to run.
           0 goes first, 100 is assumed/default if not sent
        '''

        rdict = dict()
        rdict['details'] = dict()
        rdict['details']['sourceipaddress'] = None
        rdict['details']['destinationipaddress'] = None
        self.registration = rdict
        self.priority = 20
        self.geoip = pygeoip.GeoIP('/home/mozdef/envs/mozdef/bot/GeoLiteCity.dat', pygeoip.MEMORY_CACHE)

    def ipLocation(self, ip):
        location = dict()
        try:
            geoDict = self.geoip.record_by_addr(str(netaddr.IPNetwork(ip)[0]))
            if geoDict is not None:
                return geoDict
            else:
                location['location'] = 'unknown'
        except ValueError as e:
            pass
        return location

    def onMessage(self, message):
        if 'details' in message.keys():
            if 'sourceipaddress' in message['details'].keys():
                ipText = message['details']['sourceipaddress']
                if isIP(ipText):
                    ip = netaddr.IPNetwork(ipText)[0]
                    if (not ip.is_loopback() and not ip.is_private() and not ip.is_reserved()):
                        '''lookup geoip info'''
                        message['details']['sourceipgeolocation'] = self.ipLocation(ipText)
                else:
                    # invalid ip sent in the field
                    # if we send on, elastic search will error, so set it
                    # to a valid, yet meaningless value
                    message['details']['sourceipaddress'] = '0.0.0.0'

            if 'destinationipaddress' in message['details'].keys():
                ipText = message['details']['destinationipaddress']
                if isIP(ipText):
                    ip = netaddr.IPNetwork(ipText)[0]
                    if (not ip.is_loopback() and not ip.is_private() and not ip.is_reserved()):
                        '''lookup geoip info'''
                        message['details']['destinationipgeolocation'] = self.ipLocation(ipText)
                else:
                    # invalid ip sent in the field
                    # if we send on, elastic search will error, so set it
                    # to a valid, yet meaningless value
                    message['details']['destinationipaddress'] = '0.0.0.0'
        return message

        
{'fraction_non_alphanumeric': 0.09373060211049038, 'fraction_numerical': 0.007759155803848541, 'mean_word_length': 1.8875448028673836, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2805550', 'n_tokens_mistral': 1727, 'n_tokens_neox': 1615, 'n_words': 569}