<jupyter_start><jupyter_text>Donwload and extract GTFS data PLEASE DO NOT RUN THIS ON THE HOSTED AZURE NOTEBOOK!<jupyter_code>import requests
import time
import os
import zipfile
def etl_data(sources):
    for i in sources:
        directory = os.path.join(os.path.abspath(os.path.curdir), f'data/downloads/{i[0]}')
        if not os.path.exists(directory):
            os.makedirs(directory)
        time.sleep(0.25)  # be polite
        try:
            print(f'Downloading {i[1]}')
            req = requests.get(i[1], stream=True)
        except Exception as exc:
            print(f'Error attempting to download {i[1]}\nStatus code: {req.status_code}')
        filepath = os.path.join(directory, f'{i[0]}.zip')
        with open(filepath, 'wb') as fp:
            for chunk in req.iter_content(chunk_size=1024):
                if chunk:
                    fp.write(chunk)
        print(f"Extracting {filepath}")
        try:
            zf = zipfile.ZipFile(filepath)
            zf.extractall(os.path.dirname(os.path.abspath(filepath)))
        except zipfile.BadZipFile:
            print(f'Unable to extract {filepath}: Bad zip file')
sources = [
    ('sfmta', 'https://gtfs.sfmta.com/transitdata/google_transit.zip'),
    ('bart', 'http://www.bart.gov/dev/schedules/google_transit.zip'),
    ('caltrain', 'http://www.caltrain.com/Assets/GTFS/caltrain/CT-GTFS.zip'),
    ('ac-transit', 'http://www.actransit.org/wp-content/uploads/GTFSFall18.zip'),
    ('capital-corridor', 'http://www.capitolcorridor.org/googletransit/google_transit.zip'),
    ('palo-alto-shuttle', 'http://mjcaction.com/MJC_GTFS_Public/cityofpaloalto_google_transit.zip'),
    ('commuteorg-san-mateo', 'http://www.commute.org/files/gtfs/Masterzip.zip'),
    ('cccta', 'http://cccta.org/GTFS/google_transit.zip'),
    ('emery-go-round', 'http://data.trilliumtransit.com/gtfs/emerygoround-ca-us/emerygoround-ca-us.zip'),
    ('fast-transit', 'http://data.trilliumtransit.com/gtfs/fairfield-ca-us/fairfield-ca-us.zip'),
    ('golden-gate-transit', 'https://realtime.goldengate.org/gtfsstatic/GTFSTransitData.zip'),
    ('wheels-bus', 'http://webwatch.lavta.org/TMGTFSRealTimeWebService/GTFS-Static/google_transit.zip'),
    ('marin-transit', 'http://www.marintransit.org/data/google_transit.zip'),
    ('mission-bay-tma', 'http://data.trilliumtransit.com/gtfs/missionbaytma-ca-us/missionbaytma-ca-us.zip'),
    ('mvgo-mountain-view', 'http://data.trilliumtransit.com/gtfs/mountainview-ca-us/mountainview-ca-us.zip'),
    ('petaluma-transit', 'http://data.trilliumtransit.com/gtfs/petalumatransit-petaluma-ca-us/petalumatransit-petaluma-ca-us.zip'),
    ('rio-vista-delta-breeze', 'http://data.trilliumtransit.com/gtfs/riovista-ca-us/riovista-ca-us.zip'),
    ('samtrans', 'http://www.samtrans.com/Assets/GTFS/samtrans/ST-GTFS.zip'),
    ('santa-clara-vta', 'http://www.vta.org/sfc/servlet.shepherd/document/download/069A0000001NUea'),
    ('soltrans', 'http://data.trilliumtransit.com/gtfs/soltrans-ca-us/soltrans-ca-us.zip'),
    ('sonoma-county-transit', 'http://data.trilliumtransit.com/gtfs/sonomacounty-ca-us/sonomacounty-ca-us.zip'),
    ('stanford-marguerite-shuttle', 'https://transportation-forms.stanford.edu/google/google_transit.zip'),
    ('tideline-water-taxi', 'http://trilliumtransit.com/transit_feeds/tidelinewatertaxi-ca-us/tidelinewatertaxi-ca-us.zip'),
    ('tri-delta-transit', 'http://192.168.127.12/rtt/public/utility/gtfs.aspx'),
    ('westcat', 'http://data.trilliumtransit.com/gtfs/westcat-ca-us/westcat-ca-us.zip')
]
etl_data(sources)<jupyter_output>Downloading https://gtfs.sfmta.com/transitdata/google_transit.zip
Extracting /Users/cmollet/projects/github.com/apa-dev/DataJam-NPC19/GTFS/data/downloads/sfmta/sfmta.zip
Downloading http://www.bart.gov/dev/schedules/google_transit.zip
Extracting /Users/cmollet/projects/github.com/apa-dev/DataJam-NPC19/GTFS/data/downloads/bart/bart.zip
Downloading http://www.caltrain.com/Assets/GTFS/caltrain/CT-GTFS.zip
Extracting /Users/cmollet/projects/github.com/apa-dev/DataJam-NPC19/GTFS/data/downloads/caltrain/caltrain.zip
Downloading http://www.actransit.org/wp-content/uploads/GTFSFall18.zip
Extracting /Users/cmollet/projects/github.com/apa-dev/DataJam-NPC19/GTFS/data/downloads/ac-transit/ac-transit.zip
Downloading http://www.capitolcorridor.org/googletransit/google_transit.zip
Extracting /Users/cmollet/projects/github.com/apa-dev/DataJam-NPC19/GTFS/data/downloads/capital-corridor/capital-corridor.zip
Downloading http://mjcaction.com/MJC_GTFS_Public/cityofpaloalto_google_transit.zip
Extracting /Us[...]
{'fraction_non_alphanumeric': 0.17043211230533012, 'fraction_numerical': 0.010309278350515464, 'mean_word_length': 3.9727371864776444, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 4, 'lorem ipsum': 0, 'www.': 12, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '319149', 'license': 'Apache-2.0', 'n_tokens_mistral': 1795, 'n_tokens_neox': 1728, 'n_words': 160, 'path': 'GTFS/etl_data.ipynb', 'repo_name': 'apa-dev/DataJam-NPC19', 'size': '4559'}