<commit_before>from collections import OrderedDict
from math import radians, cos, sin, asin, sqrt
from operator import itemgetter
import socket
from pygeoip import GeoIP, MEMORY_CACHE
from redis.client import StrictRedis
from config import Config
from models import Mirror
from logging import getLogger

log = getLogger(__name__)


class GeoIPLookupError(Exception):
    pass


class DistanceCalculator(object):
    _geoip4 = None
    _geoip6 = None

    def __init__(self):
        # Load the GeoIP databases into class attributes since they each need 20+ MB in memory
        if not self.__class__._geoip4:
            self.__class__._geoip4 = GeoIP(Config.GEOIP_PATH_V4, MEMORY_CACHE)
        if not self.__class__._geoip6:
            self.__class__._geoip6 = GeoIP(Config.GEOIP_PATH_V6, MEMORY_CACHE)
        self.redis = StrictRedis(Config.REDIS['HOST'], Config.REDIS['PORT'], Config.REDIS['DB'])

    @staticmethod
    def _haversine(lon1, lat1, lon2, lat2):
        """
        Calculate the great circle distance between two points
        on the earth (specified in decimal degrees)
        """
        # convert decimal degrees to radians
        lon1, lat1, lon2, lat2 = map(lambda v: radians(float(v)), [lon1, lat1, lon2, lat2])
        # haversine formula
        dlon = lon2 - lon1
        dlat = lat2 - lat1
        a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
        c = 2 * asin(sqrt(a))
        km = 6367 * c  # convert to km
        return km

    def get_mirror_distances(self, address):
        last_update = self.redis.get(Config.KEY_LAST_UPDATE)
        key = Config.KEY_MIRROR.format(address, last_update)
        distances = OrderedDict(self.redis.zrange(key, 0, -1, withscores=True))
        if not distances:
            try:
                if ":" in address:
                    record = self._geoip6.record_by_addr(address)
                else:
                    record = self._geoip4.record_by_addr(address)
            except socket.error:
                raise GeoIPLookupError()
            if not record:
                raise GeoIPLookupError()
            lat = record['latitude']
            lon = record['longitude']

            distances = OrderedDict(
                sorted(
                    (
                        (mirror.name, self._haversine(lon, lat, mirror.lon, mirror.lat))
                        for mirror in Mirror.objects.filter(age__lt=3601)
                    ),
                    key=itemgetter(1)
                )
            )
            if distances:
                self.redis.zadd(key, **distances)
                self.redis.expire(key, 60 * 10)  # 10 min
        return distances

    def get_nearest_mirror(self, address):
        try:
            distances = self.get_mirror_distances(address)
            if distances:
                return next(distances.iteritems())[0]
            return Config.FALLBACK_MIRROR
        except GeoIPLookupError:
            return Config.FALLBACK_MIRROR


if __name__ == "__main__":
    m = DistanceCalculator()
    m.get_nearest_mirror("5.9.5.200")
<commit_msg>Handle IPv6 escaped IPv4 addresses as IPv4<commit_after>from collections import OrderedDict
from math import radians, cos, sin, asin, sqrt
from operator import itemgetter
import socket
from pygeoip import GeoIP, MEMORY_CACHE
from redis.client import StrictRedis
from config import Config
from models import Mirror
from logging import getLogger

log = getLogger(__name__)


class GeoIPLookupError(Exception):
    pass


class DistanceCalculator(object):
    _geoip4 = None
    _geoip6 = None

    def __init__(self):
        # Load the GeoIP databases into class attributes since they each need 20+ MB in memory
        if not self.__class__._geoip4:
            self.__class__._geoip4 = GeoIP(Config.GEOIP_PATH_V4, MEMORY_CACHE)
        if not self.__class__._geoip6:
            self.__class__._geoip6 = GeoIP(Config.GEOIP_PATH_V6, MEMORY_CACHE)
        self.redis = StrictRedis(Config.REDIS['HOST'], Config.REDIS['PORT'], Config.REDIS['DB'])

    @staticmethod
    def _haversine(lon1, lat1, lon2, lat2):
        """
        Calculate the great circle distance between two points
        on the earth (specified in decimal degrees)
        """
        # convert decimal degrees to radians
        lon1, lat1, lon2, lat2 = map(lambda v: radians(float(v)), [lon1, lat1, lon2, lat2])
        # haversine formula
        dlon = lon2 - lon1
        dlat = lat2 - lat1
        a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
        c = 2 * asin(sqrt(a))
        km = 6367 * c  # convert to km
        return km

    def get_mirror_distances(self, address):
        last_update = self.redis.get(Config.KEY_LAST_UPDATE)
        key = Config.KEY_MIRROR.format(address, last_update)
        distances = OrderedDict(self.redis.zrange(key, 0, -1, withscores=True))
        if not distances:
            if address.startswith("::ffff:"):
                address = address.replace("::ffff:", "")
            try:
                if ":" in address:
                    record = self._geoip6.record_by_addr(address)
                else:
                    record = self._geoip4.record_by_addr(address)
            except socket.error:
                raise GeoIPLookupError()
            if not record:
                raise GeoIPLookupError()
            lat = record['latitude']
            lon = record['longitude']

            distances = OrderedDict(
                sorted(
                    (
                        (mirror.name, self._haversine(lon, lat, mirror.lon, mirror.lat))
                        for mirror in Mirror.objects.filter(age__lt=3601)
                    ),
                    key=itemgetter(1)
                )
            )
            if distances:
                self.redis.zadd(key, **distances)
                self.redis.expire(key, 60 * 10)  # 10 min
        return distances

    def get_nearest_mirror(self, address):
        try:
            distances = self.get_mirror_distances(address)
            if distances:
                return next(distances.iteritems())[0]
            return Config.FALLBACK_MIRROR
        except GeoIPLookupError:
            return Config.FALLBACK_MIRROR


if __name__ == "__main__":
    m = DistanceCalculator()
    m.get_nearest_mirror("5.9.5.200")

{'fraction_non_alphanumeric': 0.08911671924290221, 'fraction_numerical': 0.019085173501577287, 'mean_word_length': 2.000946521533365, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3979248', 'n_tokens_mistral': 1985, 'n_tokens_neox': 1858, 'n_words': 474}