<commit_before>import os
from datetime import timedelta, datetime, date
from pytz.exceptions import NonExistentTimeError, AmbiguousTimeError
from django.core.management.base import BaseCommand
from django.db import transaction
from django.conf import settings
from django.utils import timezone
from timetables import txc, northern_ireland
from ...models import Region, Service, Journey, StopUsageUsage, StopPoint


ONE_DAY = timedelta(days=1)


def combine_date_time(date, time):
    combo = datetime.combine(date, time)
    try:
        return timezone.make_aware(combo)
    except NonExistentTimeError:
        return timezone.make_aware(combo + timedelta(hours=1))
    except AmbiguousTimeError:
        return timezone.make_aware(combo, is_dst=True)


def handle_transxchange(service, transxchange, day):
    if not transxchange.operating_period.contains(day):
        return
    stopusageusages = []
    transxchange.service = service
    stops = set()
    for journey_pattern in transxchange.journey_patterns.values():
        for timinglink in journey_pattern.get_timinglinks():
            stops.add(timinglink.origin.stop.atco_code)
            stops.add(timinglink.destination.stop.atco_code)
    existent_stops = StopPoint.objects.in_bulk(stops)
    for vj in transxchange.journeys:
        if not vj.should_show(day, transxchange):
            continue
        journey_stopusageusages = []
        date = day
        previous_time = None
        journey = Journey(service=service, datetime=combine_date_time(date, vj.departure_time))
        for i, cell in enumerate(vj.get_times()):
            su = cell.stopusage
            time = cell.departure_time
            if previous_time and previous_time > time:
                date += ONE_DAY
            if su.stop.atco_code in existent_stops:
                if not su.activity or su.activity.startswith('pickUp'):
                    journey_stopusageusages.append(
                        StopUsageUsage(datetime=combine_date_time(date, time),
                                       order=i, stop_id=su.stop.atco_code)
                    )
                journey.destination_id = su.stop.atco_code
            previous_time = time
        if journey.destination_id:
            journey.save()
            for suu in journey_stopusageusages:
                suu.journey = journey
            stopusageusages += journey_stopusageusages
    StopUsageUsage.objects.bulk_create(stopusageusages)


def handle_ni_grouping(service, grouping, day):
    for journey in grouping['Journeys']:
        if not northern_ireland.should_show(journey, day) or not journey['StopUsages']:
            continue
        stopusageusages = []<commit_msg>'remove last StopUsageUsage of the journey if necessary' (for Manchester etc)<commit_after>import os
from datetime import timedelta, datetime, date
from pytz.exceptions import NonExistentTimeError, AmbiguousTimeError
from django.core.management.base import BaseCommand
from django.db import transaction
from django.conf import settings
from django.utils import timezone
from timetables import txc, northern_ireland
from ...models import Region, Service, Journey, StopUsageUsage, StopPoint


ONE_DAY = timedelta(days=1)


def combine_date_time(date, time):
    combo = datetime.combine(date, time)
    try:
        return timezone.make_aware(combo)
    except NonExistentTimeError:
        return timezone.make_aware(combo + timedelta(hours=1))
    except AmbiguousTimeError:
        return timezone.make_aware(combo, is_dst=True)


def handle_transxchange(service, transxchange, day):
    if not transxchange.operating_period.contains(day):
        return
    stopusageusages = []
    transxchange.service = service
    stops = set()
    for journey_pattern in transxchange.journey_patterns.values():
        for timinglink in journey_pattern.get_timinglinks():
            stops.add(timinglink.origin.stop.atco_code)
            stops.add(timinglink.destination.stop.atco_code)
    existent_stops = StopPoint.objects.in_bulk(stops)
    for vj in transxchange.journeys:
        if not vj.should_show(day, transxchange):
            continue
        journey_stopusageusages = []
        date = day
        previous_time = None
        journey = Journey(service=service, datetime=combine_date_time(date, vj.departure_time))
        for i, cell in enumerate(vj.get_times()):
            su = cell.stopusage
            time = cell.departure_time
            if previous_time and previous_time > time:
                date += ONE_DAY
            if su.stop.atco_code in existent_stops:
                if not su.activity or su.activity.startswith('pickUp'):
                    journey_stopusageusages.append(
                        StopUsageUsage(datetime=combine_date_time(date, time),
                                       order=i, stop_id=su.stop.atco_code)
                    )
                journey.destination_id = su.stop.atco_code
            previous_time = time
        # remove last StopUsageUsage of the journey if necessary
        if journey.destination_id:
            if i == journey_stopusageusages[-1].order:
                assert not su.activity
                journey_stopusageusages = journey_stopusageusages[:-1]
            journey.save()
            for suu in journey_stopusageusages:
                suu.journey = journey
            stopusageusages += journey_stopusageusages
    StopUsageUsage.objects.bulk_create(stopusageusages)


def handle_ni_grouping(service, grouping, day):
    for journey in grouping['Journeys']:
        if not northern_ireland.should_show(journey, day) or not journey['StopUsages']:
            continue
        stopusageusages = []
{'fraction_non_alphanumeric': 0.07348804500703235, 'fraction_numerical': 0.0010548523206751054, 'mean_word_length': 2.689364461738003, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3308727', 'n_tokens_mistral': 1635, 'n_tokens_neox': 1604, 'n_words': 364}