<commit_before>"""Represent TransXChange concepts, and generate a matrix timetable from
TransXChange documents
"""
from __future__ import print_function
import os
import re
import pickle as pickle
import xml.etree.cElementTree as ET
import calendar
from datetime import date, datetime
from django.utils.dateparse import parse_duration
from django.utils.text import slugify
from django.core.cache import cache
from titlecase import titlecase

NS = {
    'txc': 'http://www.transxchange.org.uk/'
}
# A safe date far from any daylight savings changes or leap seconds
DUMMY_DATE = date(2016, 4, 5)
DESCRIPTION_REGEX = re.compile(r'.+,([^ ].+)$')
WEEKDAYS = {day: i for i, day in enumerate(calendar.day_name)}


def time_between(end, start):
    """Return the timedelta between two times (by converting them to datetimes)."""
    return datetime.combine(DUMMY_DATE, end) - datetime.combine(DUMMY_DATE, start)


def add_time(time, delta):
    """Add a timededelta the delta between two times (by naively converting them to datetimes)."""
    return (datetime.combine(DUMMY_DATE, time) + delta).time()


def sanitize_description_part(part):
    """Given an oddly formatted part like 'Bus Station bay 5,Blyth',
    returns a shorter, more normal version like 'Blyth'
    """
    sanitized_part = DESCRIPTION_REGEX.match(part.strip())
    return sanitized_part.group(1) if sanitized_part is not None else part


class Stop(object):
    """A TransXChange StopPoint."""
    stop = None
    locality = None

    def __init__(self, element):
        self.atco_code = element.find('txc:StopPointRef', NS).text
        self.common_name = element.find('txc:CommonName', NS).text
        locality_element = element.find('txc:LocalityName', NS)
        if locality_element is not None:
            self.locality = locality_element.text

    def __str__(self):
        if not self.locality or self.locality in self.common_name:
            return self.common_name
        return '%s %s' % (self.locality, self.common_name)

    def is_at(self, text):
        """Whether a given slugified tring roughly matches either
        this stop's locality's name, or this stop's name
        (e.g. 'kings-lynn' matches 'kings-lynn-bus-station' and vice versa).
        """
        name = slugify(self.stop.locality.name if self.stop else self.locality)
        if name and name in text or text in name:
            return True
        name = slugify(self.common_name)
        return text in name or name in text


class Row(object):
    """A row in a grouping in a timetable.
    Each row is associated with a Stop, and a list of times.
    """
    def __init__(self, part):
        self.part = part
        part.row = self
        self.times = []
        self.sequencenumbers = {}


class Cell(object):
    """Represents a special cell in a timetable, spanning multiple rows and columns,
    with some text like 'then every 5 minutes until'.
    """
    def __init__(self, colspan, rowspan, duration):
        self.colspan = colspan
        self.rowspan = rowspan
        self.duration = duration

    def __str__(self):
        if self.duration.seconds == 3600:
            return 'then hourly until'
        if self.duration.seconds % 3600 == 0:
            return 'then every %d hours until' % (self.duration.seconds / 3600)
        return 'then every %d minutes until' % (self.duration.seconds / 60)


class Grouping(object):
    """Probably either 'outbound' or 'inbound'.
    (Could perhaps be extended to group by weekends, bank holidays in the future).
    """
    def __init__(self, direction, service_description_parts):
        self.direction = direction
        self.service_description_parts = service_description_parts
        self.column_heads = []
        self.column_feet = []
        self.journeypatterns = []
        self.journeys = []
        self.rows = {}

    def has_minor_stops(self):
        for row in self.rows:
            if row.part.timingstatus == 'OTH':
                return True
        return False

    def starts_at(self, locality_name):
        return self.rows[0].part.stop.is_at(locality_name)

    def ends_at(self, locality_name):
        return self.rows[-1].part.stop.is_at(locality_name)

    def do_heads_and_feet(self):
        self.rows = list(self.rows.values())

        if not self.journeys:
            return

        if self.rows and self.rows[0].part.sequencenumber is None:
            for journeypattern in self.journeypatterns:
                self.rows.sort(key=lambda r, j=journeypattern: r.sequencenumbers.get(j.id, 0))

        prev_journey = None
        head_span = 0
        in_a_row = 0
        prev_difference = None
        difference = None
        foot_span = 0
        for i, journey in enumerate(self.journeys):
            if prev_journey:
                if not journey.operating_profile and not prev_journey.operating_profile:<commit_msg>Fix order, closes #13<commit_after>"""Represent TransXChange concepts, and generate a matrix timetable from
TransXChange documents
"""
from __future__ import print_function
import os
import re
import pickle as pickle
import xml.etree.cElementTree as ET
import calendar
from functools import total_ordering
from datetime import date, datetime
from django.utils.dateparse import parse_duration
from django.utils.text import slugify
from django.core.cache import cache
from titlecase import titlecase

NS = {
    'txc': 'http://www.transxchange.org.uk/'
}
# A safe date far from any daylight savings changes or leap seconds
DUMMY_DATE = date(2016, 4, 5)
DESCRIPTION_REGEX = re.compile(r'.+,([^ ].+)$')
WEEKDAYS = {day: i for i, day in enumerate(calendar.day_name)}


def time_between(end, start):
    """Return the timedelta between two times (by converting them to datetimes)."""
    return datetime.combine(DUMMY_DATE, end) - datetime.combine(DUMMY_DATE, start)


def add_time(time, delta):
    """Add a timededelta the delta between two times (by naively converting them to datetimes)."""
    return (datetime.combine(DUMMY_DATE, time) + delta).time()


def sanitize_description_part(part):
    """Given an oddly formatted part like 'Bus Station bay 5,Blyth',
    returns a shorter, more normal version like 'Blyth'
    """
    sanitized_part = DESCRIPTION_REGEX.match(part.strip())
    return sanitized_part.group(1) if sanitized_part is not None else part


class Stop(object):
    """A TransXChange StopPoint."""
    stop = None
    locality = None

    def __init__(self, element):
        self.atco_code = element.find('txc:StopPointRef', NS).text
        self.common_name = element.find('txc:CommonName', NS).text
        locality_element = element.find('txc:LocalityName', NS)
        if locality_element is not None:
            self.locality = locality_element.text

    def __str__(self):
        if not self.locality or self.locality in self.common_name:
            return self.common_name
        return '%s %s' % (self.locality, self.common_name)

    def is_at(self, text):
        """Whether a given slugified tring roughly matches either
        this stop's locality's name, or this stop's name
        (e.g. 'kings-lynn' matches 'kings-lynn-bus-station' and vice versa).
        """
        name = slugify(self.stop.locality.name if self.stop else self.locality)
        if name and name in text or text in name:
            return True
        name = slugify(self.common_name)
        return text in name or name in text


@total_ordering
class Row(object):
    """A row in a grouping in a timetable.
    Each row is associated with a Stop, and a list of times.
    """
    def __init__(self, part):
        self.part = part
        part.row = self
        self.times = []
        self.sequencenumbers = {}

    def __le__(self, other):
        for key in self.sequencenumbers:
            if key in other.sequencenumbers:
                return self.sequencenumbers[key] < other.sequencenumbers[key]
        return True

    def __eq__(self, other):
        return self is other


class Cell(object):
    """Represents a special cell in a timetable, spanning multiple rows and columns,
    with some text like 'then every 5 minutes until'.
    """
    def __init__(self, colspan, rowspan, duration):
        self.colspan = colspan
        self.rowspan = rowspan
        self.duration = duration

    def __str__(self):
        if self.duration.seconds == 3600:
            return 'then hourly until'
        if self.duration.seconds % 3600 == 0:
            return 'then every %d hours until' % (self.duration.seconds / 3600)
        return 'then every %d minutes until' % (self.duration.seconds / 60)


class Grouping(object):
    """Probably either 'outbound' or 'inbound'.
    (Could perhaps be extended to group by weekends, bank holidays in the future).
    """
    def __init__(self, direction, service_description_parts):
        self.direction = direction
        self.service_description_parts = service_description_parts
        self.column_heads = []
        self.column_feet = []
        self.journeypatterns = []
        self.journeys = []
        self.rows = {}

    def has_minor_stops(self):
        for row in self.rows:
            if row.part.timingstatus == 'OTH':
                return True
        return False

    def starts_at(self, locality_name):
        return self.rows[0].part.stop.is_at(locality_name)

    def ends_at(self, locality_name):
        return self.rows[-1].part.stop.is_at(locality_name)

    def do_heads_and_feet(self):
        self.rows = list(self.rows.values())

        if not self.journeys:
            return

        if self.rows and self.rows[0].part.sequencenumber is None:
            self.rows.sort()

        prev_journey = None
        head_span = 0
        in_a_row = 0
        prev_difference = None
        difference = None
        foot_span = 0
        for i, journey in enumerate(self.journeys):
            if prev_journey:
                if not journey.operating_profile and not prev_journey.operating_profile:
{'fraction_non_alphanumeric': 0.09413178449829762, 'fraction_numerical': 0.006308832365311436, 'mean_word_length': 2.7502816372512204, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1393660', 'n_tokens_mistral': 3083, 'n_tokens_neox': 2880, 'n_words': 914}