<commit_before>import socket

import pygeoip

import idiokit
from abusehelper.core import events, bot
from abusehelper.contrib.experts.combiner import Expert

def is_ipv4(ip):
    try:
        socket.inet_aton(ip)
    except (ValueError, socket.error):
        return False
    return True

class GeoIPExpert(Expert):
    geoip_db = bot.Param("path to the GeoIP database")
    ip_key = bot.Param("key which has IP address as value " +
                       "(default: %default)", default="ip")

    def __init__(self, *args, **keys):
        Expert.__init__(self, *args, **keys)

        self.geoip = pygeoip.GeoIP(self.geoip_db)
        self.log.info("GeoIP initiated")

    def geomap(self, event, key):
        for ip in event.values(key, filter=is_ipv4):
            try:
                record = self.geoip.record_by_addr(ip)
            except pygeoip.GeoIPError, e:
                self.log.error("GeoIP fetch failed: %s", repr(e))
                continue

            if not record:
                continue

            augmentation = events.Event()

            cc = record.get("country_code", None)
            if cc:
                augmentation.add("geoip_cc", cc)

            latitude = record.get("latitude", None)
            if latitude:
                augmentation.add("latitude", unicode(latitude))

            longitude = record.get("longitude", None)
            if longitude:
                augmentation.add("longitude", unicode(longitude))

            if not augmentation.contains():
                continue

            augmentation.add(key, ip)
            yield augmentation

    @idiokit.stream
    def augment(self):
        while True:
            eid, event = yield idiokit.next()

            for augmentation in self.geomap(event, self.ip_key):
                yield idiokit.send(eid, augmentation)

if __name__ == "__main__":
    GeoIPExpert.from_command_line().execute()
<commit_msg>geoipexpert.py: Since many probably use the lite version of the GeoIP db, I modified the bot to comply with the license: http://geolite.maxmind.com/download/geoip/database/LICENSE.txt<commit_after>import socket

import pygeoip

import idiokit
from abusehelper.core import events, bot
from abusehelper.contrib.experts.combiner import Expert


def is_ipv4(ip):
    try:
        socket.inet_aton(ip)
    except (ValueError, socket.error):
        return False
    return True


class GeoIPExpert(Expert):
    geoip_db = bot.Param("path to the GeoIP database")
    ip_key = bot.Param("key which has IP address as value " +
                       "(default: %default)", default="ip")

    def __init__(self, *args, **keys):
        Expert.__init__(self, *args, **keys)

        self.geoip = pygeoip.GeoIP(self.geoip_db)
        self.log.info("GeoIP initiated")

    def geomap(self, event, key):
        for ip in event.values(key, filter=is_ipv4):
            try:
                record = self.geoip.record_by_addr(ip)
            except pygeoip.GeoIPError, e:
                self.log.error("GeoIP fetch failed: %s", repr(e))
                continue

            if not record:
                continue

            augmentation = events.Event()

            cc = record.get("country_code", None)
            if cc:
                augmentation.add("geoip_cc", cc)

            latitude = record.get("latitude", None)
            if latitude:
                augmentation.add("latitude", unicode(latitude))

            longitude = record.get("longitude", None)
            if longitude:
                augmentation.add("longitude", unicode(longitude))

            if not augmentation.contains():
                continue

            augmentation.add(key, ip)
            augmentation.add("geoip data source",
                "This product includes GeoLite data created by MaxMind, available from http://www.maxmind.com/.")
            yield augmentation

    @idiokit.stream
    def augment(self):
        while True:
            eid, event = yield idiokit.next()

            for augmentation in self.geomap(event, self.ip_key):
                yield idiokit.send(eid, augmentation)

if __name__ == "__main__":
    GeoIPExpert.from_command_line().execute()

{'fraction_non_alphanumeric': 0.10081223124701386, 'fraction_numerical': 0.0009555661729574773, 'mean_word_length': 2.0673992673992676, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2638442', 'n_tokens_mistral': 1253, 'n_tokens_neox': 1179, 'n_words': 301}