<commit_before>import requests
import pytz
import re
from bs4 import BeautifulSoup
from datetime import datetime, date
from django.conf import settings
from django.utils.text import slugify


DESTINATION_REGEX = re.compile(r'.+\((.+)\)')


def get_tfl_departures(stop, services):
    timezone = pytz.timezone('Europe/London')
    req = requests.get('http://api.tfl.gov.uk/StopPoint/%s/arrivals' % stop.pk)
    return ({
        'time': timezone.fromutc(datetime.strptime(item.get('expectedArrival'), '%Y-%m-%dT%H:%M:%SZ')),
        'service': services.get(item.get('lineName')) or item.get('lineName'),
        'destination': item.get('destinationName'),
    } for item in req.json()) if req.status_code == 200 else ()


def get_acislive_departures(prefix, stop, services):
    req = requests.get('http://%s.acislive.com/pip/stop_simulator_table.asp' % prefix, {
        'naptan': stop.naptan_code
    })
    if req.status_code != 200:
        return ()
    soup = BeautifulSoup(req.text, 'html.parser')
    cells = [cell.text.strip() for cell in soup.find_all('td')]
    rows = (cells[i * 4 - 4:i * 4] for i in range(1, (len(cells)/4) + 1))
    return ({
        'time': row[2],
        'service': services.get(row[0]) or row[0],
        'destination': row[1]
    } for row in rows)


def get_acisconnect_departures(prefix, stop, services):
    req = requests.get('http://%s.acisconnect.com/Text/WebDisplay.aspx' % prefix, {
        'stopRef': stop.pk
    })
    if req.status_code != 200:
        return ()
    soup = BeautifulSoup(req.text, 'html.parser')
    rows = (row.findAll('td') for row in soup.find(id='GridViewRTI').findAll('tr')[1:])
    return ({
        'time': row[4].text.replace('1 Mins', '1 min').replace('Mins', 'mins'),
        'service': services.get(row[0].text) or row[0].text,
        'destination': row[2].text
    } for row in rows)


def transportapi_row(item, services):
    if item['best_departure_estimate'] is None:
        return
    if 'date' in item:
        departure_time = datetime.strptime(item['date'] + ' ' + item['best_departure_estimate'], '%Y-%m-%d %H:%M')
    else:
        departure_time = datetime.strptime(item['best_departure_estimate'], '%H:%M')
    destination = item.get('direction')
    destination_matches = DESTINATION_REGEX.match(destination)<commit_msg>Hush error for acisconnect stops without departures<commit_after>import requests
import pytz
import re
from bs4 import BeautifulSoup
from datetime import datetime, date
from django.conf import settings
from django.utils.text import slugify


DESTINATION_REGEX = re.compile(r'.+\((.+)\)')


def get_tfl_departures(stop, services):
    timezone = pytz.timezone('Europe/London')
    req = requests.get('http://api.tfl.gov.uk/StopPoint/%s/arrivals' % stop.pk)
    return ({
        'time': timezone.fromutc(datetime.strptime(item.get('expectedArrival'), '%Y-%m-%dT%H:%M:%SZ')),
        'service': services.get(item.get('lineName')) or item.get('lineName'),
        'destination': item.get('destinationName'),
    } for item in req.json()) if req.status_code == 200 else ()


def get_acislive_departures(prefix, stop, services):
    req = requests.get('http://%s.acislive.com/pip/stop_simulator_table.asp' % prefix, {
        'naptan': stop.naptan_code
    })
    if req.status_code != 200:
        return ()
    soup = BeautifulSoup(req.text, 'html.parser')
    cells = [cell.text.strip() for cell in soup.find_all('td')]
    rows = (cells[i * 4 - 4:i * 4] for i in range(1, (len(cells)/4) + 1))
    return ({
        'time': row[2],
        'service': services.get(row[0]) or row[0],
        'destination': row[1]
    } for row in rows)


def get_acisconnect_departures(prefix, stop, services):
    req = requests.get('http://%s.acisconnect.com/Text/WebDisplay.aspx' % prefix, {
        'stopRef': stop.pk
    })
    if req.status_code != 200:
        return ()
    soup = BeautifulSoup(req.text, 'html.parser')
    table = soup.find(id='GridViewRTI')
    if table is None:
        return ()
    rows = (row.findAll('td') for row in table.findAll('tr')[1:])
    return ({
        'time': row[4].text.replace('1 Mins', '1 min').replace('Mins', 'mins'),
        'service': services.get(row[0].text) or row[0].text,
        'destination': row[2].text
    } for row in rows)


def transportapi_row(item, services):
    if item['best_departure_estimate'] is None:
        return
    if 'date' in item:
        departure_time = datetime.strptime(item['date'] + ' ' + item['best_departure_estimate'], '%Y-%m-%d %H:%M')
    else:
        departure_time = datetime.strptime(item['best_departure_estimate'], '%H:%M')
    destination = item.get('direction')
    destination_matches = DESTINATION_REGEX.match(destination)
{'fraction_non_alphanumeric': 0.1662425784563189, 'fraction_numerical': 0.011450381679389313, 'mean_word_length': 3.4668560606060606, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5279395', 'n_tokens_mistral': 1662, 'n_tokens_neox': 1622, 'n_words': 334}