<commit_before>"""
Add the relevant live source to stop points if departure boards are available from each that source

Usage:

    ./manage.py import_live_stops
"""

import requests
import json
from bs4 import BeautifulSoup
from time import sleep
from django.core.management.base import BaseCommand
from busstops.models import StopPoint, LiveSource


kent = LiveSource.objects.get_or_create(name='Kent')[0]
yorkshire = LiveSource.objects.get_or_create(name='Y')[0]
travelwest = LiveSource.objects.get_or_create(name='west')[0]
ayrshire = LiveSource.objects.get_or_create(name='ayr')[0]
buckinghamshire = LiveSource.objects.get_or_create(name='buck')[0]
cambridgeshire = LiveSource.objects.get_or_create(name='camb')[0]
aberdeen = LiveSource.objects.get_or_create(name='aber')[0]
cardiff = LiveSource.objects.get_or_create(name='card')[0]
swindon = LiveSource.objects.get_or_create(name='swin')[0]
metrobus = LiveSource.objects.get_or_create(name='metr')[0]



class Command(BaseCommand):

    @staticmethod
    def maybe_add_acislive_source(stop, live_source, prefix):
        url = 'http://%s.acislive.com/pip/stop_simulator.asp' % prefix
        request = requests.get(
            url,
            {'naptan': stop.naptan_code}
        )
        print request.url
        soup = BeautifulSoup(request.text, 'html.parser')
        if soup.title and soup.title.text != 'Sorry':
            stop.live_sources.add(live_source)
            print soup.title
        else:
            print soup.title
        print '\n'
        sleep(1)

    @staticmethod
    def maybe_add_acisconnect_source(stop, live_source, prefix):
        print stop.get_absolute_url()
        url = 'http://%s.acisconnect.com/Text/WebDisplay.aspx' % prefix
        request = requests.get(
            url,
            {'stopRef': stop.pk}
        )
        soup = BeautifulSoup(request.text, 'html.parser')
        text = soup.find(id='UpdatePanel1').text
        if 'System unavailable' not in text:
            stop.live_sources.add(live_source)
        else:
            print text
        sleep(1)

    @staticmethod
    def get_clustered_stops(subdomain):
        url = 'http://%s.acisconnect.com/ConnectService.svc/GetClusteredStops' % subdomain
        params = {
            'topLeft': {'lon': -100, 'lat': 100, 'CLASS_NAME': 'OpenLayers.LonLat'},
            'bottomRight': {'lon': 100, 'lat': -100, 'CLASS_NAME': 'OpenLayers.LonLat'},
            'zoomLevel': 5
        }
        response = requests.post(url, json=params)
        json_string = response.json().get('d').replace('MapStopResponse=', '')
        parsed_json = json.loads(json_string)
        return parsed_json['Stops'] if 'Stops' in parsed_json else parsed_json['AllFoundStops']

    @staticmethod
    def get_stops_for_cluster(subdomain, cluster_id):
        url = 'http://%s.acisconnect.com/ConnectService.svc/GetStopsForCluster' % subdomain
        params = {
            'clusterID': cluster_id
        }
        response = requests.post(url, json=params)
        json_string = response.json().get('d').replace('MapStopResponse=', '')
        parsed_json = json.loads(json_string)
        print response.text
        sleep(1)
        return parsed_json['Stops']

    def handle(self, *args, **options):
        for subdomain, livesource in (
            ('cambridgeshire', cambridgeshire),
            ('buckinghamshire', buckinghamshire),
            ('ayrshire', ayrshire),
            ('travelwest', travelwest),
            ('aberdeen', aberdeen),
            ('cardiff', cardiff),
            ('swindon', swindon),
            ('metrobus', metrobus)
        ):
            print subdomain
            stop_ids = []
            for cluster in self.get_clustered_stops(subdomain):
                if cluster['ClusterCount'] == 1:
                    stop_ids.append(cluster['StopRef'])
                else:
                    stop_ids.extend((stop['StopRef'] for stop in self.get_stops_for_cluster(subdomain, cluster['ClusterId'])))
            print stop_ids
            stoppoints = StopPoint.objects.filter(pk__in=stop_ids)
            livesource.stoppoint_set.add(*stoppoints)
<commit_msg>Fix Ayrshire live stop import<commit_after>"""
Add the relevant live source to stop points if departure boards are available from each that source

Usage:

    ./manage.py import_live_stops
"""

import requests
import json
from bs4 import BeautifulSoup
from time import sleep
from django.core.management.base import BaseCommand
from busstops.models import StopPoint, LiveSource


kent = LiveSource.objects.get_or_create(name='Kent')[0]
yorkshire = LiveSource.objects.get_or_create(name='Y')[0]
travelwest = LiveSource.objects.get_or_create(name='west')[0]
ayrshire = LiveSource.objects.get_or_create(name='ayr')[0]
buckinghamshire = LiveSource.objects.get_or_create(name='buck')[0]
cambridgeshire = LiveSource.objects.get_or_create(name='camb')[0]
aberdeen = LiveSource.objects.get_or_create(name='aber')[0]
cardiff = LiveSource.objects.get_or_create(name='card')[0]
swindon = LiveSource.objects.get_or_create(name='swin')[0]
metrobus = LiveSource.objects.get_or_create(name='metr')[0]



class Command(BaseCommand):

    @staticmethod
    def maybe_add_acislive_source(stop, live_source, prefix):
        url = 'http://%s.acislive.com/pip/stop_simulator.asp' % prefix
        request = requests.get(
            url,
            {'naptan': stop.naptan_code}
        )
        print request.url
        soup = BeautifulSoup(request.text, 'html.parser')
        if soup.title and soup.title.text != 'Sorry':
            stop.live_sources.add(live_source)
            print soup.title
        else:
            print soup.title
        print '\n'
        sleep(1)

    @staticmethod
    def maybe_add_acisconnect_source(stop, live_source, prefix):
        print stop.get_absolute_url()
        url = 'http://%s.acisconnect.com/Text/WebDisplay.aspx' % prefix
        request = requests.get(
            url,
            {'stopRef': stop.pk}
        )
        soup = BeautifulSoup(request.text, 'html.parser')
        text = soup.find(id='UpdatePanel1').text
        if 'System unavailable' not in text:
            stop.live_sources.add(live_source)
        else:
            print text
        sleep(1)

    @staticmethod
    def get_clustered_stops(subdomain):
        url = 'http://%s.acisconnect.com/ConnectService.svc/GetClusteredStops' % subdomain
        params = {
            'topLeft': {'lon': -100, 'lat': 100, 'CLASS_NAME': 'OpenLayers.LonLat'},
            'bottomRight': {'lon': 100, 'lat': -100, 'CLASS_NAME': 'OpenLayers.LonLat'},
            'zoomLevel': 5
        }
        response = requests.post(url, json=params)
        json_string = response.json().get('d').replace('MapStopResponse=', '')
        parsed_json = json.loads(json_string)
        return parsed_json['Stops'] if 'Stops' in parsed_json else parsed_json['AllFoundStops']

    @staticmethod
    def get_stops_for_cluster(subdomain, cluster_id):
        url = 'http://%s.acisconnect.com/ConnectService.svc/GetStopsForCluster' % subdomain
        params = {
            'clusterID': cluster_id
        }
        response = requests.post(url, json=params)
        json_string = response.json().get('d').replace('MapStopResponse=', '')
        parsed_json = json.loads(json_string)
        print response.text
        sleep(1)
        return parsed_json['Stops'] if 'Stops' in parsed_json else parsed_json['AllFoundStops']

    def handle(self, *args, **options):
        for subdomain, livesource in (
            ('cambridgeshire', cambridgeshire),
            ('buckinghamshire', buckinghamshire),
            ('ayrshire', ayrshire),
            ('travelwest', travelwest),
            ('aberdeen', aberdeen),
            ('cardiff', cardiff),
            ('swindon', swindon),
            ('metrobus', metrobus)
        ):
            print subdomain
            stop_ids = []
            for cluster in self.get_clustered_stops(subdomain):
                if cluster['ClusterCount'] == 1:
                    stop_ids.append(cluster['StopRef'])
                else:
                    stop_ids.extend((stop['StopRef'] for stop in self.get_stops_for_cluster(subdomain, cluster['ClusterId'])))
            print stop_ids
            stoppoints = StopPoint.objects.filter(pk__in=stop_ids)
            livesource.stoppoint_set.add(*stoppoints)

{'fraction_non_alphanumeric': 0.12255136168179646, 'fraction_numerical': 0.00692785475394171, 'mean_word_length': 2.72630173564753, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4434356', 'n_tokens_mistral': 2616, 'n_tokens_neox': 2549, 'n_words': 478}