<commit_before>import math
import os
import struct
from itertools import chain

import numpy as np
from django.conf import settings
from django.core.cache import cache
from django.db.models.signals import m2m_changed, post_delete
from PIL import Image
from shapely import prepared
from shapely.geometry import box
from shapely.ops import unary_union

from c3nav.mapdata.models import MapUpdate
from c3nav.mapdata.utils.models import get_submodels


class MapHistory:
    # binary format (everything little-endian):
    # 1 byte (uint8): resolution
    # 2 bytes (uint16): origin x
    # 2 bytes (uint16): origin y
    # 2 bytes (uint16): origin width
    # 2 bytes (uint16): origin height
    # 2 bytes (uint16): number of updates
    # n uptates times:
    #     4 bytes (uint32): update id
    #     4 bytes (uint32): timestamp
    # width*height*2 bytes:
    #     data array (line after line) with uint16 cells
    empty_array = np.empty((0, 0), dtype=np.uint16)

    def __init__(self, resolution=settings.CACHE_RESOLUTION, x=0, y=0, updates=None, data=empty_array, filename=None):
        self.resolution = resolution
        self.x = x
        self.y = y
        self.updates = updates
        self.data = data
        self.filename = filename
        self.unfinished = False

    @classmethod
    def open(cls, filename, default_update=None):
        try:
            with open(filename, 'rb') as f:
                resolution, x, y, width, height, num_updates = struct.unpack('<BHHHHH', f.read(11))
                updates = struct.unpack('<'+'II'*num_updates, f.read(num_updates*8))
                updates = list(zip(updates[0::2], updates[1::2]))
                # noinspection PyTypeChecker
                data = np.fromstring(f.read(width*height*2), np.uint16).reshape((height, width))
                return cls(resolution, x, y, list(updates), data, filename)
        except (FileNotFoundError, struct.error):
            if default_update is None:
                default_update = MapUpdate.last_update()
            new_empty = cls(updates=[default_update])
            new_empty.save(filename)
            return new_empty

    @staticmethod
    def level_filename(level_id, mode):
        return os.path.join(settings.CACHE_ROOT, 'level_%s_%d' % (mode, level_id))

    @classmethod
    def open_level(cls, level_id, mode, default_update=None):
        return cls.open(cls.level_filename(level_id, mode), default_update)

    @classmethod
    def open_level_cached(cls, level_id, mode, cache_key=None):
        if cache_key is None:
            cache_key = MapUpdate.current_cache_key()
        cache_key = 'mapdata:map-history-%d-%s:%s' % (level_id, mode, cache_key)
        result = cache.get(cache_key, None)
        if result is None:
            result = cls.open_level(level_id, mode)
            cache.set(cache_key, result, 120)
        return result

    def save(self, filename=None):
        if filename is None:
            filename = self.filename
        with open(filename, 'wb') as f:
            self.write(f)

    def write(self, f):
        f.write(struct.pack('<BHHHHH', self.resolution, self.x, self.y, *reversed(self.data.shape),
                            len(self.updates)))
        f.write(struct.pack('<'+'II'*len(self.updates), *chain(*self.updates)))
        f.write(self.data.tobytes('C'))

    def add_new(self, geometry, data=None):
        prep = prepared.prep(geometry)
        minx, miny, maxx, maxy = geometry.bounds
        res = self.resolution
        minx = int(math.floor(minx/res))
        miny = int(math.floor(miny/res))
        maxx = int(math.ceil(maxx/res))
        maxy = int(math.ceil(maxy/res))

        direct = data is None

        if direct:
            data = self.data
            if self.resolution != settings.CACHE_RESOLUTION:
                data = None
                self.updates = self.updates[-1:]

            if not data.size:
                data = np.zeros(((maxy-miny), (maxx-minx)), dtype=np.uint16)
                self.x, self.y = minx, miny
            else:
                orig_height, orig_width = data.shape
                if minx < self.x or miny < self.y or maxx > self.x+orig_width or maxy > self.y+orig_height:
                    new_x, new_y = min(minx, self.x), min(miny, self.y)
                    new_width = max(maxx, self.x+orig_width)-new_x
                    new_height = max(maxy, self.y+orig_height)-new_y
                    new_data = np.zeros((new_height, new_width), dtype=np.uint16)
                    dx, dy = self.x-new_x, self.y-new_y
                    new_data[dy:(dy+orig_height), dx:(dx+orig_width)] = data
                    data = new_data
                    self.x, self.y = new_x, new_y
        else:
            height, width = data.shape
            minx, miny = max(minx, self.x), max(miny, self.y)
            maxx, maxy = min(maxx, self.x+width), max(maxy, self.y+height)

        new_val = len(self.updates) if direct else 1
        for iy, y in enumerate(range(miny*res, maxy*res, res), start=miny-self.y):
            for ix, x in enumerate(range(minx*res, maxx*res, res), start=minx-self.x):
                if prep.intersects(box(x, y, x+res, y+res)):
                    data[iy, ix] = new_val<commit_msg>fix typo (max instead of min) in MapHistory.add_new()<commit_after>import math
import os
import struct
from itertools import chain

import numpy as np
from django.conf import settings
from django.core.cache import cache
from django.db.models.signals import m2m_changed, post_delete
from PIL import Image
from shapely import prepared
from shapely.geometry import box
from shapely.ops import unary_union

from c3nav.mapdata.models import MapUpdate
from c3nav.mapdata.utils.models import get_submodels


class MapHistory:
    # binary format (everything little-endian):
    # 1 byte (uint8): resolution
    # 2 bytes (uint16): origin x
    # 2 bytes (uint16): origin y
    # 2 bytes (uint16): origin width
    # 2 bytes (uint16): origin height
    # 2 bytes (uint16): number of updates
    # n uptates times:
    #     4 bytes (uint32): update id
    #     4 bytes (uint32): timestamp
    # width*height*2 bytes:
    #     data array (line after line) with uint16 cells
    empty_array = np.empty((0, 0), dtype=np.uint16)

    def __init__(self, resolution=settings.CACHE_RESOLUTION, x=0, y=0, updates=None, data=empty_array, filename=None):
        self.resolution = resolution
        self.x = x
        self.y = y
        self.updates = updates
        self.data = data
        self.filename = filename
        self.unfinished = False

    @classmethod
    def open(cls, filename, default_update=None):
        try:
            with open(filename, 'rb') as f:
                resolution, x, y, width, height, num_updates = struct.unpack('<BHHHHH', f.read(11))
                updates = struct.unpack('<'+'II'*num_updates, f.read(num_updates*8))
                updates = list(zip(updates[0::2], updates[1::2]))
                # noinspection PyTypeChecker
                data = np.fromstring(f.read(width*height*2), np.uint16).reshape((height, width))
                return cls(resolution, x, y, list(updates), data, filename)
        except (FileNotFoundError, struct.error):
            if default_update is None:
                default_update = MapUpdate.last_update()
            new_empty = cls(updates=[default_update])
            new_empty.save(filename)
            return new_empty

    @staticmethod
    def level_filename(level_id, mode):
        return os.path.join(settings.CACHE_ROOT, 'level_%s_%d' % (mode, level_id))

    @classmethod
    def open_level(cls, level_id, mode, default_update=None):
        return cls.open(cls.level_filename(level_id, mode), default_update)

    @classmethod
    def open_level_cached(cls, level_id, mode, cache_key=None):
        if cache_key is None:
            cache_key = MapUpdate.current_cache_key()
        cache_key = 'mapdata:map-history-%d-%s:%s' % (level_id, mode, cache_key)
        result = cache.get(cache_key, None)
        if result is None:
            result = cls.open_level(level_id, mode)
            cache.set(cache_key, result, 120)
        return result

    def save(self, filename=None):
        if filename is None:
            filename = self.filename
        with open(filename, 'wb') as f:
            self.write(f)

    def write(self, f):
        f.write(struct.pack('<BHHHHH', self.resolution, self.x, self.y, *reversed(self.data.shape),
                            len(self.updates)))
        f.write(struct.pack('<'+'II'*len(self.updates), *chain(*self.updates)))
        f.write(self.data.tobytes('C'))

    def add_new(self, geometry, data=None):
        prep = prepared.prep(geometry)
        minx, miny, maxx, maxy = geometry.bounds
        res = self.resolution
        minx = int(math.floor(minx/res))
        miny = int(math.floor(miny/res))
        maxx = int(math.ceil(maxx/res))
        maxy = int(math.ceil(maxy/res))

        direct = data is None

        if direct:
            data = self.data
            if self.resolution != settings.CACHE_RESOLUTION:
                data = None
                self.updates = self.updates[-1:]

            if not data.size:
                data = np.zeros(((maxy-miny), (maxx-minx)), dtype=np.uint16)
                self.x, self.y = minx, miny
            else:
                orig_height, orig_width = data.shape
                if minx < self.x or miny < self.y or maxx > self.x+orig_width or maxy > self.y+orig_height:
                    new_x, new_y = min(minx, self.x), min(miny, self.y)
                    new_width = max(maxx, self.x+orig_width)-new_x
                    new_height = max(maxy, self.y+orig_height)-new_y
                    new_data = np.zeros((new_height, new_width), dtype=np.uint16)
                    dx, dy = self.x-new_x, self.y-new_y
                    new_data[dy:(dy+orig_height), dx:(dx+orig_width)] = data
                    data = new_data
                    self.x, self.y = new_x, new_y
        else:
            height, width = data.shape
            minx, miny = max(minx, self.x), max(miny, self.y)
            maxx, maxy = min(maxx, self.x+width), min(maxy, self.y+height)

        new_val = len(self.updates) if direct else 1
        for iy, y in enumerate(range(miny*res, maxy*res, res), start=miny-self.y):
            for ix, x in enumerate(range(minx*res, maxx*res, res), start=minx-self.x):
                if prep.intersects(box(x, y, x+res, y+res)):
                    data[iy, ix] = new_val
{'fraction_non_alphanumeric': 0.11466236722306525, 'fraction_numerical': 0.01024279210925645, 'mean_word_length': 2.2336706531738733, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '328939', 'n_tokens_mistral': 3445, 'n_tokens_neox': 3310, 'n_words': 812}