<commit_before>import random
import sys
from collections import namedtuple, defaultdict
from datetime import datetime, date, time, timedelta
from logging import getLogger
from operator import itemgetter
from StringIO import StringIO
from urllib2 import urlopen
from zipfile import ZipFile

from django.http import Http404
from django.db import transaction, reset_queries

from molly.apps.places import EntityCache
from molly.apps.places.models import (Entity, Route, StopOnRoute, Source,
                                      Journey, ScheduledStop)
from molly.apps.places.providers import BaseMapsProvider, NaptanMapsProvider
from molly.conf.settings import batch
from molly.utils.i18n import set_name_in_language

logger = getLogger(__name__)

weekbool = namedtuple('weekbool', 'mon tue wed thu fri sat sun')

class AtcoCifTimetableProvider(BaseMapsProvider):
    """
    This provider loads ATCO-CIF dumps and parses them to extract timetable
    data. This only handles a subset of the ATCO-CIF standard, in particular,
    it deals with the TfGM released data (the only released data atm)
    """
    
    def __init__(self, url):
        """
        URL to an ATCO-CIF zip to be used
        """
        self._url = url
        self._cache = EntityCache()
        self._entity_type = NaptanMapsProvider(None)._get_entity_types()['BCT']
    
    @batch('%d 10 * * wed' % random.randint(0, 59))
    def import_data(self, metadata, output):
        
        deleted_routes = set(Route.objects.filter(external_ref__startswith=self._url).values_list('external_ref'))
        archive = ZipFile(StringIO(urlopen(self._url).read()))
        for file in archive.namelist():
            output.write(file)<commit_msg>Update providers following internal changes to Naptan provider<commit_after>import random
import sys
from collections import namedtuple, defaultdict
from datetime import datetime, date, time, timedelta
from logging import getLogger
from operator import itemgetter
from StringIO import StringIO
from urllib2 import urlopen
from zipfile import ZipFile

from django.http import Http404
from django.db import transaction, reset_queries

from molly.apps.places import EntityCache
from molly.apps.places.models import (Entity, Route, StopOnRoute, Source,
                                      Journey, ScheduledStop)
from molly.apps.places.providers import BaseMapsProvider, NaptanMapsProvider
from molly.conf.settings import batch
from molly.utils.i18n import set_name_in_language

logger = getLogger(__name__)

weekbool = namedtuple('weekbool', 'mon tue wed thu fri sat sun')

class AtcoCifTimetableProvider(BaseMapsProvider):
    """
    This provider loads ATCO-CIF dumps and parses them to extract timetable
    data. This only handles a subset of the ATCO-CIF standard, in particular,
    it deals with the TfGM released data (the only released data atm)
    """
    
    def __init__(self, url):
        """
        URL to an ATCO-CIF zip to be used
        """
        self._url = url
        self._cache = EntityCache()
        self._entity_type = NaptanMapsProvider(None)._get_entity_types()['BCT'][0]
    
    @batch('%d 10 * * wed' % random.randint(0, 59))
    def import_data(self, metadata, output):
        
        deleted_routes = set(Route.objects.filter(external_ref__startswith=self._url).values_list('external_ref'))
        archive = ZipFile(StringIO(urlopen(self._url).read()))
        for file in archive.namelist():
            output.write(file)
{'fraction_non_alphanumeric': 0.078205865439908, 'fraction_numerical': 0.006612995974698103, 'mean_word_length': 3.5776315789473685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4026728', 'n_tokens_mistral': 1016, 'n_tokens_neox': 987, 'n_words': 314}