<commit_before>import argparse

import requests
import xmltodict

# http://www.nationalrail.co.uk/100296.aspx
# https://lite.realtime.nationalrail.co.uk/OpenLDBWS/
# http://zetcode.com/db/sqlitepythontutorial/

# create table departures (crs TEXT, platform TEXT, std TEXT, etd TEXT, origin TEXT, destination TEXT);
from utils.database import insert_into_db

xml_payload = """<?xml version="1.0"?>
<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns1="http://thalesgroup.com/RTTI/2014-02-20/ldb/" xmlns:ns2="http://thalesgroup.com/RTTI/2010-11-01/ldb/commontypes">
  <SOAP-ENV:Header>
    <ns2:AccessToken>
      <ns2:TokenValue>{KEY}</ns2:TokenValue>
    </ns2:AccessToken>
  </SOAP-ENV:Header>
  <SOAP-ENV:Body>
    <ns1:GetDepartureBoardRequest>
      <ns1:numRows>100</ns1:numRows>
      <ns1:crs>{CRS}</ns1:crs>
    </ns1:GetDepartureBoardRequest>
  </SOAP-ENV:Body>
</SOAP-ENV:Envelope>
"""


def fetch_trains(url, key, crs):
    headers = {'content-type': 'text/xml'}
    payload = xml_payload.replace("{KEY}", key).replace("{CRS}", crs)
    response = requests.post(url, data=payload, headers=headers)

    data = xmltodict.parse(response.content)
    services = data["soap:Envelope"]["soap:Body"]["GetDepartureBoardResponse"]["GetStationBoardResult"]["lt2:trainServices"]["lt2:service"]

    for service in services:
        yield {
            "crs": crs,
            "origin": service["lt2:origin"]["lt2:location"]["lt2:locationName"],
            "destination": service["lt2:destination"]["lt2:location"]["lt2:locationName"],
            "std": service.get("lt2:std"),
            "etd": service.get("lt2:etd"),
            "platform": service.get("lt2:platform", "-"),
        }


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='National Rail Data Collector')
    parser.add_argument('--key', help='API Key', required=True)
    parser.add_argument('--url', help='API URL', default="http://lite.realtime.nationalrail.co.uk/OpenLDBWS/ldb6.asmx")
    parser.add_argument('--crs', help='CRS Station Code (default is Thatcham)', default="THA")
    parser.add_argument('--db', help='SQLite DB Name', default="../data/trains.db")
    args = parser.parse_args()

    departures = fetch_trains(args.url, args.key, args.crs)
    insert_into_db(args.db, "departures", departures)<commit_msg>Changed to the newest version of the SOAP API<commit_after>import argparse
from time import sleep

import requests
import xmltodict

# http://www.nationalrail.co.uk/100296.aspx
# https://lite.realtime.nationalrail.co.uk/OpenLDBWS/
# http://zetcode.com/db/sqlitepythontutorial/

# create table departures (crs TEXT, platform TEXT, std TEXT, etd TEXT, origin TEXT, destination TEXT);
from utils.database import insert_into_db

xml_payload = """<?xml version="1.0"?>
<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns1="http://thalesgroup.com/RTTI/2016-02-16/ldb/" xmlns:ns2="http://thalesgroup.com/RTTI/2013-11-28/Token/types">
  <SOAP-ENV:Header>
    <ns2:AccessToken>
      <ns2:TokenValue>{KEY}</ns2:TokenValue>
    </ns2:AccessToken>
  </SOAP-ENV:Header>
  <SOAP-ENV:Body>
    <ns1:GetDepBoardWithDetailsRequest>
      <ns1:numRows>100</ns1:numRows>
      <ns1:crs>{CRS}</ns1:crs>
      <ns1:timeWindow>100</ns1:timeWindow>
    </ns1:GetDepBoardWithDetailsRequest>
  </SOAP-ENV:Body>
</SOAP-ENV:Envelope>
"""


def fetch_trains(url, key, crs):
    headers = {'content-type': 'text/xml'}
    payload = xml_payload.replace("{KEY}", key).replace("{CRS}", crs)
    response = requests.post(url, data=payload, headers=headers)

    data = xmltodict.parse(response.content)
    services = data["soap:Envelope"]["soap:Body"]["GetDepBoardWithDetailsResponse"]["GetStationBoardResult"]["lt5:trainServices"]["lt5:service"]

    for service in services:
        yield {
            "crs": crs,
            "origin": service["lt5:origin"]["lt4:location"]["lt4:locationName"],
            "destination": service["lt5:destination"]["lt4:location"]["lt4:locationName"],
            "std": service.get("lt4:std"),
            "etd": service.get("lt4:etd"),
            "platform": service.get("lt4:platform", "-"),
        }


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='National Rail Data Collector')
    parser.add_argument('--key', help='API Key', required=True)
    parser.add_argument('--url', help='API URL', default="http://lite.realtime.nationalrail.co.uk/OpenLDBWS/ldb9.asmx")
    parser.add_argument('--crs', help='CRS Station Code (default is Thatcham)', default="THA")
    parser.add_argument('--db', help='SQLite DB Name', default="../data/trains.db")
    args = parser.parse_args()

    while True:
        departures = fetch_trains(args.url, args.key, args.crs)
        insert_into_db(args.db, "departures", departures)
        sleep(10)
{'fraction_non_alphanumeric': 0.17520049352251696, 'fraction_numerical': 0.022414147645486326, 'mean_word_length': 4.252699784017278, 'pattern_counts': {'":': 14, '<': 39, '<?xml version=': 2, '>': 39, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4864432', 'n_tokens_mistral': 1727, 'n_tokens_neox': 1697, 'n_words': 257}