<commit_before>from time import sleep
from datetime import timedelta
from ciso8601 import parse_datetime
from django.contrib.gis.geos import Point
from django.utils import timezone
from requests.exceptions import RequestException
from django.contrib.gis.db.models import Extent
from ..import_live_vehicles import ImportLiveVehiclesCommand
from busstops.models import Operator, Service, StopPoint
from ...models import VehicleLocation, VehicleJourney


def get_latlong(item):
    position = item['position']
    return Point(position['longitude'], position['latitude'])


class Command(ImportLiveVehiclesCommand):
    operator_ids = {
        'BOWE': 'HIPK',
        'LAS': ('GAHL', 'LGEN'),
        '767STEP': ('SESX', 'GECL', 'NIBS'),
        'UNIB': 'UNOE',
        'UNO': 'UNOE',
        'RENW': 'ECWY',
        'CB': ('CBUS', 'CACB'),
        'CUBU': ('CUBU', 'RSTY'),
        'SOG': 'guernsey',
        'IOM': ('bus-vannin', 'IMHR'),
        'Rtl': ('RBUS', 'GLRB', 'KENN', 'NADS', 'THVB'),
    }
    operators = {}
    source_name = 'ZipTrip'
    url = 'https://ziptrip1.ticketer.org.uk/v1/vehiclepositions'
    ignorable_route_names = {'rr', 'rail', 'transdev', '7777', 'shop', 'pos', 'kiosk', 'rolls-royce'}

    @staticmethod
    def get_datetime(item):
        return parse_datetime(item['reported'])

    def get_extents(self):
        for operator in (
            'SBAY',
            'CBUS',
            'CUBU',
            'ECWY',
            'FALC',
            'GECL',
            'GAHL',
            'guernsey',
            'HIPK',
            'IPSW',
            'LYNX',
            'MDCL',
            'NDTR',
            'DPCE',
            'RBUS',
            'NADS',
            'SFGC',
            'SESX',
            'NIBS',
            'UNOE',
            'WHTL',
            'TRDU',
            # 'ROST',
        ):
            stops = StopPoint.objects.filter(service__operator=operator, service__current=True)
            extent = stops.aggregate(Extent('latlong'))['latlong__extent']
            if not extent:
                print(operator)
                continue
            yield {
                'maxLat': extent[3] + 0.1,
                'maxLng': extent[2] + 0.1,
                'minLat': extent[1] - 0.1,
                'minLng': extent[0] - 0.1,
            }

        # Mann
        yield {
            'maxLat': 54.5,
            'maxLng': -3.8,
            'minLat': 53.9,
            'minLng': -5.1
        }

    def get_items(self):
        fifteen_minutes_ago = timezone.now() - timedelta(minutes=15)
        for params in self.get_extents():
            try:
                response = self.session.get(self.url, params=params, timeout=5)
                items = response.json()['items']
                any_items = False
                if items:
                    for item in items:
                        if parse_datetime(item['reported']) > fifteen_minutes_ago:
                            any_items = True
                            yield item
                if not any_items:
                    print(response.url)
            except (RequestException, KeyError):
                continue
            sleep(1)

    def get_vehicle(self, item):
        operator_id, vehicle = item['vehicleCode'].split('_', 1)
        vehicle = vehicle.replace(' ', '_')

        if operator_id in self.operator_ids:
            operator_id = self.operator_ids[operator_id]

        defaults = {}
        if vehicle.isdigit():
            defaults['fleet_number'] = vehicle
        elif '-' in vehicle:
            fleet_number = vehicle.split('-')[0].replace('_', '')
            if fleet_number.isdigit():
                defaults['fleet_number'] = fleet_number

        defaults['source'] = self.source
        if type(operator_id) is tuple:
            if operator_id[0] == 'SESX' and operator_id[0] == 'CUBU':
                # vehicles may have multiple operators
                defaults['operator_id'] = operator_id[0]
                return self.vehicles.get_or_create(defaults, operator_id__in=operator_id, code=vehicle)

            # services may have multiple operators, but vehicles all have the same operator (e.g. Reading Buses)
            operator_id = operator_id[0]

        try:
            if operator_id not in self.operators:
                # cache operators by id to save a query
                self.operators[operator_id] = Operator.objects.get(id=operator_id)

            if 'fleet_number' in defaults and operator_id == 'IPSW' or operator_id == 'ROST':
                # vehicle codes differ between sources, so use fleet number
                defaults['code'] = vehicle<commit_msg>BUS-TIMES-DJANGO-3WS<commit_after>from time import sleep
from datetime import timedelta
from ciso8601 import parse_datetime
from django.contrib.gis.geos import Point
from django.utils import timezone
from requests.exceptions import RequestException
from django.contrib.gis.db.models import Extent
from ..import_live_vehicles import ImportLiveVehiclesCommand
from busstops.models import Operator, Service, StopPoint
from ...models import VehicleLocation, VehicleJourney


def get_latlong(item):
    position = item['position']
    return Point(position['longitude'], position['latitude'])


class Command(ImportLiveVehiclesCommand):
    operator_ids = {
        'BOWE': 'HIPK',
        'LAS': ('GAHL', 'LGEN'),
        '767STEP': ('SESX', 'GECL', 'NIBS'),
        'UNIB': 'UNOE',
        'UNO': 'UNOE',
        'RENW': 'ECWY',
        'CB': ('CBUS', 'CACB'),
        'CUBU': ('CUBU', 'RSTY'),
        'SOG': 'guernsey',
        'IOM': ('bus-vannin', 'IMHR'),
        'Rtl': ('RBUS', 'GLRB', 'KENN', 'NADS', 'THVB'),
    }
    operators = {}
    source_name = 'ZipTrip'
    url = 'https://ziptrip1.ticketer.org.uk/v1/vehiclepositions'
    ignorable_route_names = {'rr', 'rail', 'transdev', '7777', 'shop', 'pos', 'kiosk', 'rolls-royce'}

    @staticmethod
    def get_datetime(item):
        return parse_datetime(item['reported'])

    def get_extents(self):
        for operator in (
            'SBAY',
            'CBUS',
            'CUBU',
            'ECWY',
            'FALC',
            'GECL',
            'GAHL',
            'guernsey',
            'HIPK',
            'IPSW',
            'LYNX',
            'MDCL',
            'NDTR',
            'DPCE',
            'RBUS',
            'NADS',
            'SFGC',
            'SESX',
            'NIBS',
            'UNOE',
            'WHTL',
            'TRDU',
            # 'ROST',
        ):
            stops = StopPoint.objects.filter(service__operator=operator, service__current=True)
            extent = stops.aggregate(Extent('latlong'))['latlong__extent']
            if not extent:
                print(operator)
                continue
            yield {
                'maxLat': extent[3] + 0.1,
                'maxLng': extent[2] + 0.1,
                'minLat': extent[1] - 0.1,
                'minLng': extent[0] - 0.1,
            }

        # Mann
        yield {
            'maxLat': 54.5,
            'maxLng': -3.8,
            'minLat': 53.9,
            'minLng': -5.1
        }

    def get_items(self):
        fifteen_minutes_ago = timezone.now() - timedelta(minutes=15)
        for params in self.get_extents():
            try:
                response = self.session.get(self.url, params=params, timeout=5)
                items = response.json()['items']
                any_items = False
                if items:
                    for item in items:
                        if parse_datetime(item['reported']) > fifteen_minutes_ago:
                            any_items = True
                            yield item
                if not any_items:
                    print(response.url)
            except (RequestException, KeyError):
                continue
            sleep(1)

    def get_vehicle(self, item):
        operator_id, vehicle = item['vehicleCode'].split('_', 1)
        vehicle = vehicle.replace(' ', '_')

        if operator_id in self.operator_ids:
            operator_id = self.operator_ids[operator_id]

        defaults = {}
        if vehicle.isdigit():
            defaults['fleet_number'] = vehicle
        elif '-' in vehicle:
            fleet_number = vehicle.split('-')[0].replace('_', '')
            if fleet_number.isdigit():
                defaults['fleet_number'] = fleet_number

        defaults['source'] = self.source
        if type(operator_id) is tuple:
            if operator_id[0] == 'SESX' or operator_id[0] == 'CUBU':
                # vehicles may have multiple operators
                defaults['operator_id'] = operator_id[0]
                return self.vehicles.get_or_create(defaults, operator_id__in=operator_id, code=vehicle)

            # services may have multiple operators, but vehicles all have the same operator (e.g. Reading Buses)
            operator_id = operator_id[0]

        try:
            if operator_id not in self.operators:
                # cache operators by id to save a query
                self.operators[operator_id] = Operator.objects.get(id=operator_id)

            if 'fleet_number' in defaults and operator_id == 'IPSW' or operator_id == 'ROST':
                # vehicle codes differ between sources, so use fleet number
                defaults['code'] = vehicle
{'fraction_non_alphanumeric': 0.12320394595753807, 'fraction_numerical': 0.009757666738151404, 'mean_word_length': 1.7562056737588652, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '410461', 'n_tokens_mistral': 2846, 'n_tokens_neox': 2694, 'n_words': 621}