<jupyter_start><jupyter_text>Download Citi Bike Data Primary Author<NAME> Description:This notebook downloads all Citi Bike data from October 2013 - October 2021 and saves them as CSV files with consistent names.  Outputs:```data/citibike/date_prefixes.npydata/citibike/csv/*-citibike-tripdata.csvdata/citibike/zip/*-citibike-tripdata*zip```<jupyter_code>import numpy as np
import os
import pandas as pd
import time
import urllib
import zipfile

import warnings
warnings.filterwarnings('ignore')
DATA_DIR = '../data'
def peek(df):
    display(df.iloc[0:3, :])
    print(len(df))
YEAR_STRINGS = [str(y) for y in range(2014, 2021)]
MONTH_STRINGS = [f'{m:02d}' for m in range(1, 13)]

# Capture October 2013 - October 2021 to make year-over-year comparisons which
# include 2021 easier.
MONTH_2013_STRINGS = [f'{m:02d}' for m in range(10, 13)]
MONTH_2021_STRINGS = [f'{m:02d}' for m in range(1, 11)]

DATE_STRINGS = (
    [f'2013{m}' for m in MONTH_2013_STRINGS] +
    [f'{y}{m}' for y in YEAR_STRINGS for m in MONTH_STRINGS] +
    [f'2021{m}' for m in MONTH_2021_STRINGS]
)
print(DATE_STRINGS)

# Mark the date strings for which Citi Bike data is a .csv.zip  file.
CSV_ZIP_YEARS = set([str(y) for y in range(2017, 2022)])
CSV_ZIP_YEAR_DATE_STRINGS = [d for d in DATE_STRINGS if d[0:4] in CSV_ZIP_YEARS]
np.save(f'{DATA_DIR}/citibike/date_prefixes.npy', DATE_STRINGS)
for date in DATE_STRINGS:
    ext = 'csv.zip' if date in CSV_ZIP_YEAR_DATE_STRINGS else 'zip'
    url = f'https://s3.amazonaws.com/tripdata/{date}-citibike-tripdata.{ext}'
    filename_zip = f'{DATA_DIR}/citibike/zip/{date}-citibike-tripdata.{ext}'

    print(f'Downloading: {url}')
    time_start = time.time()

    try:
        urllib.request.urlretrieve(url, filename_zip)
        time_end = time.time()
        print(f'Wrote file: {filename_zip} in {(time_end - time_start):0.2f}s')
    except urllib.error.HTTPError as error:
        print(f'HTTP error code {error.code} retrieving: {url}')

    with zipfile.ZipFile(filename_zip, 'r') as zip_file:
        zip_file.extractall(f'{DATA_DIR}/citibike/csv/')
# Some CSV files aren't named the same. Standardize the names.
RENAME_DATE_STRINGS = [
    '2013-10',
    '2013-11',
    '2013-12',
    '2014-01',
    '2014-02',
    '2014-03',
    '2014-04',
    '2014-05',
    '2014-06',
    '2014-07',
    '2014-08',
]
for date in RENAME_DATE_STRINGS:
    origin = f'{DATA_DIR}/citibike/csv/{date} - Citi Bike trip data.csv'
    destination = (f'{DATA_DIR}/citibike/csv/{date.replace("-", "")}' +
                   '-citibike-tripdata.csv')
    os.rename(origin, destination)
for date in DATE_STRINGS:
    filename_csv = f'{DATA_DIR}/citibike/csv/{date}-citibike-tripdata.csv'
    df = pd.read_csv(filename_csv)
    print(filename_csv)
    peek(df)<jupyter_output>../data/citibike/csv/201310-citibike-tripdata.csv
{'fraction_non_alphanumeric': 0.14088495575221238, 'fraction_numerical': 0.05486725663716814, 'mean_word_length': 3.9405594405594404, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '469294', 'license': 'MIT', 'n_tokens_mistral': 1216, 'n_tokens_neox': 1080, 'n_words': 240, 'path': 'notebooks/cc0001_download_citibike_data.ipynb', 'repo_name': 'cpcarey/nyu-cusp-ads-bike', 'size': '2830'}