<commit_before>from datetime import datetime, timedelta

from utils import (
    get_problems_dict,
    remove_tfl_specifics,
    send_tweet,
    set_problems_dict,
)

from sources import tflapi, trackernet
from updown import sources as scs
from outputs import website, alexa

import settings, os, json


# Some "constants" and globals
station_list = None
twitter = None

SOURCE_RELIABILITY_ORDER = ['trackernet', 'tflapi']

DEFAULT_EXCUSE = 'There are step free access issues at this station.'


def set_problem_for_station(station, problem):

    problems = get_problems_dict()

    problems[station] = problem

    set_problems_dict(problems)

    return True


# Update the problems, delete old ones etc
def update_problems(problems):
    problems_to_remove = []

    for problem in problems.keys():
        if problem and problem[0:1] != '_':

            tweet = None

            if problems[problem].get('new-problem', False) and problem:
                # Longest station name is <NAME> for Maritime Greenwich at 34 chars. This leaves 106
                for source in SOURCE_RELIABILITY_ORDER:
                    if problems[problem].get(source) and problems[problem][source]['text']:
                        tweet = problem + ': ' + problems[problem][source]['text']
                        break

                if not tweet or len(tweet) > 140:
                    if problems[problem]['information']:
                        tweet = 'New information on step free access at ' + problem
                    else:
                        tweet = 'Step free access issues reported at ' + problem

                send_tweet(tweet)

            problems[problem]['new-problem'] = False

            if problems[problem]['resolved']:
                # If it's resolved, see if it's old enough to delete
                if problems[problem]['resolved'] + timedelta(hours=12) < datetime.now():
                    problems_to_remove.append(problem)
                    break

                if problems[problem]['resolved'] > datetime.now() - timedelta(minutes=1):
                    # Work out how long it took them to resolve
                    start_time = problems[problem]['time']
                    end_time = problems[problem]['resolved']

                    problems[problem]['time-to-resolve'] = int((end_time - start_time).seconds)

                    # Longest station name is Cutty Sark for Maritime Greenwich at 34 chars. This leaves 106
                    tweet = 'Step free access has been restored at ' + problem
                    send_tweet(tweet)

    for problem in list(set(problems_to_remove)):
        if problem in problems:<commit_msg>Try and kill nulls forever (again)<commit_after>from datetime import datetime, timedelta

from utils import (
    get_problems_dict,
    remove_tfl_specifics,
    send_tweet,
    set_problems_dict,
)

from sources import tflapi, trackernet
from updown import sources as scs
from outputs import website, alexa

import settings, os, json


# Some "constants" and globals
station_list = None
twitter = None

SOURCE_RELIABILITY_ORDER = ['trackernet', 'tflapi']

DEFAULT_EXCUSE = 'There are step free access issues at this station.'


def set_problem_for_station(station, problem):

    problems = get_problems_dict()

    problems[station] = problem

    set_problems_dict(problems)

    return True


# Update the problems, delete old ones etc
def update_problems(problems):
    problems_to_remove = []

    for problem in problems.keys():
        if problem and problem[0:1] != '_':

            tweet = None

            if problems[problem].get('new-problem', False) and problem:
                # Longest station name is Cutty Sark for Maritime Greenwich at 34 chars. This leaves 106
                for source in SOURCE_RELIABILITY_ORDER:
                    if problems[problem].get(source) and problems[problem][source]['text']:
                        tweet = problem + ': ' + problems[problem][source]['text']
                        break

                if not tweet or len(tweet) > 140:
                    if problems[problem]['information']:
                        tweet = 'New information on step free access at ' + problem
                    elif problem != 'null':
                        tweet = 'Step free access issues reported at ' + problem

                send_tweet(tweet)

            problems[problem]['new-problem'] = False

            if problems[problem]['resolved']:
                # If it's resolved, see if it's old enough to delete
                if problems[problem]['resolved'] + timedelta(hours=12) < datetime.now():
                    problems_to_remove.append(problem)
                    break

                if problems[problem]['resolved'] > datetime.now() - timedelta(minutes=1):
                    # Work out how long it took them to resolve
                    start_time = problems[problem]['time']
                    end_time = problems[problem]['resolved']

                    problems[problem]['time-to-resolve'] = int((end_time - start_time).seconds)

                    # Longest station name is <NAME> for Maritime Greenwich at 34 chars. This leaves 106
                    if problem != 'null':
                        tweet = 'Step free access has been restored at ' + problem
                    send_tweet(tweet)

    for problem in list(set(problems_to_remove)):
        if problem in problems:
{'fraction_non_alphanumeric': 0.08478940592238367, 'fraction_numerical': 0.0066212985102078355, 'mean_word_length': 2.1163323782234955, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '780651', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1357, 'n_words': 437}