<commit_before># coding: utf-8
"""
    ipcity.parser
    ~~~~~~~~~~~~~

    |4bit|255*4bit|data index|data block|
"""

import mmap
import socket
from struct import Struct

unpack_long = Struct('>L').unpack
unpack_char = Struct('B').unpack


class Database(object):
    def __init__(self, filename):
        with open(filename, 'rb') as f:
            buf = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)

        self._buf = buf

        index_count, = unpack_long(buf[:4])
        self._nonzero, = unpack_char(buf[4])
        # index offset: index length + 4 + 1020(1-255) - 1
        self._offset = index_count * 8 + 1023

    def close(self):
        self._buf.close()

    def _lookup_ipv4(self, ip):
        nip = socket.inet_aton(ip)

        # first IP number
        fip = bytearray(nip)[0]
        # 4 + (fip - 1) * 4
        fip_offset = fip * 4

        # position in the index block
        count, = unpack_long(self._buf[fip_offset:fip_offset + 4])
        pos = count * 8

        # not in record
        if fip > self._nonzero and not pos:
            return None

        offset = pos + 1024
        data_pos = None
        while offset < self._offset:
            endip = self._buf[offset:offset + 4]
            if nip <= endip:
                data_pos, = unpack_long(self._buf[offset + 4:offset + 8])
                break
            offset += 8

        if data_pos is None:
            return None

        ident = self._offset + data_pos + 1
        length, = unpack_char(self._buf[ident])
        buf = self._buf[ident+1:ident+1+length]
        return buf.decode('utf-8')

    def lookup(self, ip):
        value = self._lookup_ipv4(ip)
        # TODO
        return value<commit_msg>return IpInfo for parser<commit_after># coding: utf-8
"""
    ipcity.parser
    ~~~~~~~~~~~~~

    |4bit|255*4bit|data index|data block|
"""

import mmap
import socket
from struct import Struct
from collections import namedtuple

IpInfo = namedtuple('IpInfo', [
    'country', 'region', 'city', 'latitude', 'longitude'
])

unpack_long = Struct('>L').unpack
unpack_char = Struct('B').unpack


class Database(object):
    def __init__(self, filename):
        with open(filename, 'rb') as f:
            buf = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)

        self._buf = buf

        index_count, = unpack_long(buf[:4])
        # index offset: index length + 4 + 1020(1-255) - 1
        self._offset = index_count * 8 + 1023

    def close(self):
        self._buf.close()

    def _lookup_ipv4(self, ip):
        nip = socket.inet_aton(ip)

        # first IP number
        fip = bytearray(nip)[0]
        # 4 + (fip - 1) * 4
        fip_offset = fip * 4

        # position in the index block
        count, = unpack_long(self._buf[fip_offset:fip_offset + 4])
        pos = count * 8

        offset = pos + 1024
        data_pos = None
        while offset < self._offset:
            endip = self._buf[offset:offset + 4]
            if nip <= endip:
                data_pos, = unpack_long(self._buf[offset + 4:offset + 8])
                break
            offset += 8

        if data_pos is None:
            return None

        ident = self._offset + data_pos + 1
        length, = unpack_char(self._buf[ident])
        buf = self._buf[ident+1:ident+1+length]
        return buf.decode('utf-8')

    def lookup(self, ip):
        value = self._lookup_ipv4(ip)
        if not value:
            return None
        values = value.split()
        length = len(values)
        if length > 5:
            values = values[:5]
        elif length < 5:
            values.extend([''] * (5 - length))
        return IpInfo(*values)
{'fraction_non_alphanumeric': 0.12263892690938955, 'fraction_numerical': 0.025458527237886667, 'mean_word_length': 1.9611021069692058, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1406996', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1144, 'n_words': 293}