<commit_before>import operator
import os
import pickle
import re
import threading
from collections import deque, namedtuple
from functools import reduce

import numpy as np
from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _

from c3nav.mapdata.models import MapUpdate, Space
from c3nav.mapdata.utils.locations import CustomLocation
from c3nav.routing.router import Router


class Locator:
    filename = os.path.join(settings.CACHE_ROOT, 'locator')

    def __init__(self, stations, spaces):
        self.stations = stations
        self.spaces = spaces

    @classmethod
    def rebuild(cls, update):
        stations = LocatorStations()
        spaces = {}
        for space in Space.objects.prefetch_related('wifi_measurements'):
            new_space = LocatorSpace(
                LocatorPoint.from_measurement(measurement, stations)
                for measurement in space.wifi_measurements.all()
            )
            if new_space.points:
                spaces[space.pk] = new_space

        locator = cls(stations, spaces)
        pickle.dump(locator, open(cls.build_filename(update), 'wb'))
        return locator

    @classmethod
    def build_filename(cls, update):
        return os.path.join(settings.CACHE_ROOT, 'locator_%s.pickle' % MapUpdate.build_cache_key(*update))

    @classmethod
    def load_nocache(cls, update):
        return pickle.load(open(cls.build_filename(update), 'rb'))

    cached = None
    cache_update = None
    cache_lock = threading.Lock()

    @classmethod
    def load(cls):
        from c3nav.mapdata.models import MapUpdate
        update = MapUpdate.last_processed_update()
        if cls.cache_update != update:
            with cls.cache_lock:
                cls.cache_update = update
                cls.cached = cls.load_nocache(update)
        return cls.cached

    def locate(self, scan, permissions=None):
        router = Router.load()
        restrictions = router.get_restrictions(permissions)

        scan = LocatorPoint.clean_scan(scan, ignore_invalid_stations=True)
        scan_values = LocatorPoint.convert_scan(scan, self.stations, create=False)
        station_ids = frozenset(scan_values.keys())

        spaces = tuple((pk, space, station_ids & space.stations_set)
                       for pk, space in self.spaces.items()
                       if pk not in restrictions.spaces)
        spaces = tuple((pk, space, station_ids) for pk, space, station_ids in spaces if station_ids)
        if not spaces:
            return None
        good_spaces = tuple((pk, space, station_ids) for pk, space, station_ids in spaces if len(station_ids) >= 3)
        if not good_spaces:
            for station_id in station_ids:
                scan_values[station_id] = 0
            good_spaces = spaces

        good_spaces = sorted(good_spaces, key=lambda item: len(item[2]), reverse=True)[:10]

        best_location = None
        best_score = float('inf')

        for pk, space, station_ids in good_spaces:
            point, score = space.get_best_point(scan_values, station_ids)
            if score < best_score:
                location = CustomLocation(router.spaces[pk].level, point.x, point.y,
                                          permissions=permissions, icon='my_location')
                best_location = location
                best_score = score

        return best_location


class LocatorStations:
    def __init__(self):
        self.stations = []
        self.stations_lookup = {}

    def get(self, bssid, ssid, frequency, create=False):
        station_id = self.stations_lookup.get((bssid, ssid), None)
        if station_id is not None:
            station = self.stations[station_id]<commit_msg>check all good spaces<commit_after>import operator
import os
import pickle
import re
import threading
from collections import deque, namedtuple
from functools import reduce

import numpy as np
from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _

from c3nav.mapdata.models import MapUpdate, Space
from c3nav.mapdata.utils.locations import CustomLocation
from c3nav.routing.router import Router


class Locator:
    filename = os.path.join(settings.CACHE_ROOT, 'locator')

    def __init__(self, stations, spaces):
        self.stations = stations
        self.spaces = spaces

    @classmethod
    def rebuild(cls, update):
        stations = LocatorStations()
        spaces = {}
        for space in Space.objects.prefetch_related('wifi_measurements'):
            new_space = LocatorSpace(
                LocatorPoint.from_measurement(measurement, stations)
                for measurement in space.wifi_measurements.all()
            )
            if new_space.points:
                spaces[space.pk] = new_space

        locator = cls(stations, spaces)
        pickle.dump(locator, open(cls.build_filename(update), 'wb'))
        return locator

    @classmethod
    def build_filename(cls, update):
        return os.path.join(settings.CACHE_ROOT, 'locator_%s.pickle' % MapUpdate.build_cache_key(*update))

    @classmethod
    def load_nocache(cls, update):
        return pickle.load(open(cls.build_filename(update), 'rb'))

    cached = None
    cache_update = None
    cache_lock = threading.Lock()

    @classmethod
    def load(cls):
        from c3nav.mapdata.models import MapUpdate
        update = MapUpdate.last_processed_update()
        if cls.cache_update != update:
            with cls.cache_lock:
                cls.cache_update = update
                cls.cached = cls.load_nocache(update)
        return cls.cached

    def locate(self, scan, permissions=None):
        router = Router.load()
        restrictions = router.get_restrictions(permissions)

        scan = LocatorPoint.clean_scan(scan, ignore_invalid_stations=True)
        scan_values = LocatorPoint.convert_scan(scan, self.stations, create=False)
        station_ids = frozenset(scan_values.keys())

        spaces = tuple((pk, space, station_ids & space.stations_set)
                       for pk, space in self.spaces.items()
                       if pk not in restrictions.spaces)
        spaces = tuple((pk, space, station_ids) for pk, space, station_ids in spaces if station_ids)
        if not spaces:
            return None
        good_spaces = tuple((pk, space, station_ids) for pk, space, station_ids in spaces if len(station_ids) >= 3)
        if not good_spaces:
            for station_id in station_ids:
                scan_values[station_id] = 0
            good_spaces = spaces

        best_location = None
        best_score = float('inf')

        for pk, space, station_ids in good_spaces:
            point, score = space.get_best_point(scan_values, station_ids)
            if score < best_score:
                location = CustomLocation(router.spaces[pk].level, point.x, point.y,
                                          permissions=permissions, icon='my_location')
                best_location = location
                best_score = score

        return best_location


class LocatorStations:
    def __init__(self):
        self.stations = []
        self.stations_lookup = {}

    def get(self, bssid, ssid, frequency, create=False):
        station_id = self.stations_lookup.get((bssid, ssid), None)
        if station_id is not None:
            station = self.stations[station_id]
{'fraction_non_alphanumeric': 0.08245356140585326, 'fraction_numerical': 0.0020045436322330616, 'mean_word_length': 2.507029053420806, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4376410', 'n_tokens_mistral': 2145, 'n_tokens_neox': 2082, 'n_words': 526}