<jupyter_start><jupyter_text>1. Luther – Scraping  Imports & Versions<jupyter_code>import numpy as np
import pandas as pd
import requests
from bs4 import BeautifulSoup
import pickle
from datetime import datetime, timedelta
import time
import re
import sys

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import os
chromedriver = "/Applications/chromedriver"
os.environ["webdriver.chrome.driver"] = chromedriver
list_of_imports = [('Numpy', np), 
                   ('Pandas', pd), 
#                   ('Beautiful Soup', BeautifulSoup), 
                   ('Selenium', webdriver)
                  ]

for mod in list_of_imports:
    print(f"{mod[0]}: {mod[1].__version__}")

print("Python:", sys.version)<jupyter_output>Numpy: 1.12.1
Pandas: 0.20.1
Selenium: 3.6.0
Python: 3.6.1 |Anaconda 4.4.0 (x86_64)| (default, May 11 2017, 13:04:09) 
[GCC 4.2.1 Compatible Apple LLVM 6.0 (clang-600.0.57)]<jupyter_text>Necessary Functions and Variables<jupyter_code>def make_flight_url(date_str, flight_num, route_num):
    """
    Takes date string, flight number, and route number, 
    returns URL to search for a particular flight/route/day. 
    ---
    IN: 
    date_str, YYYYMMDD (str)
    flight_num (str)
    route_num (str)
    
    OUT:
    search URL (str)
    """

    # base URL constructors
    base_url1 = 'http://flightaware.com/live/flight/'
    base_url2 = '/history/'
    base_url3 = '/KLGA/KORD'
    
    # merge vars with URL bases
    search_url = (base_url1 + flight_num + base_url2 + 
                  date_str + '/' + route_num + base_url3)

    return search_url
def scrape_flight_soup(soup, flight_num, search_date):
    """
    Scrapes pertinient information off single flight page, returns record
    for that flight (one record), returns None if no record for that day.
    ---
    IN: 
    soup, BS4 object of webpage
    flight_num, flight number as searched on FlightAware (str)
    search_date, date to search (datetime obj)
    
    OUT: four flight arrival times (list)
    """
    
    date_str = datetime.strftime(search_date, "%Y%m%d")
    
    # is there a flight that day?
    names = []
    for meta in soup.find_all('meta'):
        names.append(meta.get('name'))
    if not 'airline' in names:
        return 'No Flight'
    
    # was the flight canceled?
    if 'cancelled' in soup.find(class_="flightPageSummary").text.lower():
        return 'Canceled'
    
    # if flight arrived
    try:
        details = soup.find(class_="flightPageDetails")
        details_sub = details.find(attrs={"data-template": "live/flight/details"})
        spans = list(details_sub.find_all('span'))
        arrival_times = []
        fptd_divs = details_sub.find_all(class_="flightPageTimeData")

        # pulls from the four relevant indices of fptd_divs
        for i in [9,11,12,14]:
            time_str = fptd_divs[i].text.strip().split(' ')[0]
            arrival_times.append(time_str)

        arr_conv = map(lambda x: datetime.strptime(x, "%I:%M%p").time(), arrival_times)
        arrival_times = list(map(lambda x: datetime.combine(search_date, x), arr_conv))
        return arrival_times

    except Exception as e:
        print(f"*** {flight_num}, {date_str}: ERROR: {e}")
        return None
def scrape_fn(days, start_date_str, flight_num, route_num, df=None):
    """
    Make a version that takes either days/fn/rn or a list of URLs?
    
    Goes through a series of steps to gather data for a given flight 
    number and route over a given length of time. Appends each record
    to a dataframe (provided or generated). Also saves df for each 
    round as a .pkl.
    ---
    IN:
    days, number of days to scrape, starting yesterday (int)
    start_date_str, date from which to search backwards, YYYYMMDD (str)
    flight_num, flight number as searched on FlightAware (str)
    route_num, route number as searched on FlightAware (str)
    df, pandas dataframe
    
    OUT: 
    pandas dataframe, list of timeout URLS (list)
    """
    
    # makes df if none passed
    if df is None:
        df = pd.DataFrame(columns=['airline',
                                   'f_num',
                                   'origin',
                                   'dest',
                                   'date',
                                   'land_act',
                                   'arr_act',
                                   'land_sch',
                                   'arr_sch'])

    # starts Selenium and sets timeout preferences
    driver = webdriver.Chrome(chromedriver)
    driver.set_page_load_timeout(20)
    driver.set_script_timeout(20)
    
    start_date = datetime.strptime(start_date_str, "%Y%m%d").date()
    # today = datetime.now().date()
    no_flight_count = 0 
    miss_list = []
    
    # loop to search each date
    for d in range(days):
        time.sleep(np.random.uniform(1.0,2.0))
        search_date = start_date - timedelta(days=d)
        date_str = datetime.strftime(search_date, "%Y%m%d")
        record_a = ['American', flight_num, 'LGA', 'ORD', search_date]
        flight_url = make_flight_url(date_str, flight_num, route_num)
        
        try:
            driver.get(flight_url)
            flight_soup = BeautifulSoup(driver.page_source, 'html.parser')
            record_b = scrape_flight_soup(flight_soup, flight_num, search_date)
        except Exception as e:
            print(f"*** {flight_num}, {date_str}: ERROR: {e}")
            miss_list.append(flight_url)
            record_b = None
        
        if record_b == None:
            continue 
        elif record_b == 'Canceled':
            no_flight_count = 0
            print(f"{flight_num}, {date_str}: canceled")        
        elif record_b == 'No Flight':
            no_flight_count += 1
            print(f"{flight_num}, {date_str}: no flight")
            if no_flight_count == 7:
                print(f"{flight_num}: 7 consecutive days of no flights as of {date_str}!")
                break
        else:
            no_flight_count = 0
            record = record_a + record_b
            print(f"{flight_num}, {date_str}: flight data recorded")
            df.loc[len(df)] = record    
    
    # pickle the current round
    timestamp = datetime.strftime(datetime.now(), "%m%d_%H%M%S")
    picklepath = f'../data/{flight_num}_{timestamp}.pkl'
    with open(picklepath, 'wb') as picklefile:
        pickle.dump(df, picklefile)
        
    # save missed URLs as text file
    filepath = f'../data/{flight_num}_missed_{timestamp}.txt'
    with open(filepath, 'w') as f:
        for url in miss_list:
            f.write(url+'\n')
    
    driver.close()
    
    return df, miss_list
def multiple_flights(days, start_date_str, flight_list):
    """
    Finds all flights in a list of flight number/route number tuples
    over however many days provided and returns data in a concatenated
    dataframe.
    ---
    IN: 
    days, number of days to search (int)
    start_date_str, date from which to search BACKWARDS, YYYYMMDD (str)
    flight_list, list of flight number/route numbers (string tuples in list)
    
    OUT: 
    dataframe with all flight info (pandas df)
    """
    
    master_miss_list = []
    flight_df = pd.DataFrame(columns=['airline',
                                   'f_num',
                                   'origin',
                                   'dest',
                                   'date',
                                   'land_act',
                                   'arr_act',
                                   'land_sch',
                                   'arr_sch'])
    
    for fn, rn in flight_list:
        print(f"\n\n<<<<< {fn} >>>>>")
        flight_df, miss_list = scrape_fn(days, start_date_str, fn, rn, df=flight_df)
        master_miss_list += miss_list

    return flight_df, master_miss_list
def cleanup(url_list):
    """
    Takes a list of URLs that were missed the first time and returns
    successful querys as a df of the same format as the other
    functions, as well as a miss list.
    
    THIS IS A QUICK PATCH!!! Too much copy-paste. Trying to modify
    scrape_fn() to accept either a list of URLS or a (days/fn/rn) 
    tuple instead, because they are mostly similar.
    ---
    IN: 
    url_list, list of url strings to search
    
    OUT: 
    pandas dataframe, list of timeout URLS (list)
    """
    
    df = pd.DataFrame(columns=['airline',
                                   'f_num',
                                   'origin',
                                   'dest',
                                   'date',
                                   'land_act',
                                   'arr_act',
                                   'land_sch',
                                   'arr_sch'])

    # starts Selenium and sets timeout preferences
    driver = webdriver.Chrome(chromedriver)
    driver.set_page_load_timeout(60)
    driver.set_script_timeout(20)
    
    no_flight_count = 0 
    miss_list = []
    
    for flight_url in url_list:
        p = re.compile(r'(AAL\d{3,4}).*(\d{8})')
        flight_num, date_str = p.search(flight_url).group(1,2)
        search_date = datetime.strptime(date_str, "%Y%m%d").date()
        record_a = ['American', flight_num, 'LGA', 'ORD', search_date]
        
        try:
            driver.get(flight_url)
            flight_soup = BeautifulSoup(driver.page_source, 'html.parser')
            record_b = scrape_flight_soup(flight_soup, flight_num, search_date)
        except Exception as e:
            print(f"*** {flight_num}, {date_str}: ERROR: {e}")
            miss_list.append(flight_url)
            record_b = None
        
        if record_b == None:
            continue 
        elif record_b == 'Canceled':
            no_flight_count = 0
            print(f"{flight_num}, {date_str}: canceled")        
        elif record_b == 'No Flight':
            no_flight_count += 1
            print(f"{flight_num}, {date_str}: no flight")
            if no_flight_count == 7:
                print(f"{flight_num}: 7 consecutive days of no flights as of {date_str}!")
                break
        else:
            no_flight_count = 0
            record = record_a + record_b
            print(f"{flight_num}, {date_str}: flight data recorded")
            df.loc[len(df)] = record    
    
    # pickle the current round
    timestamp = datetime.strftime(datetime.now(), "%m%d_%H%M%S")
    picklepath = f'../data/cleanup_{timestamp}.pkl'
    with open(picklepath, 'wb') as picklefile:
        pickle.dump(df, picklefile)
    
    # save missed URLs as text file
    filepath = f'../data/cleanup_missed_{timestamp}.txt'
    with open(filepath, 'w') as f:
        for url in miss_list:
            f.write(url+'\n')    
    
    driver.close()
    
    return df, miss_list
# All the LGA-ORD flights:

flight_urls = [
"http://flightaware.com/live/flight/AAL321/history/20171003/0130Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL371/history/20171003/0030Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL359/history/20171002/2330Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL361/history/20171002/2230Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL383/history/20171002/2200Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL345/history/20171002/2130Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL343/history/20171002/2030Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL358/history/20171002/1930Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL187/history/20171002/1830Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL332/history/20171002/1730Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL390/history/20171002/1630Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL337/history/20171002/1530Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL328/history/20171002/1430Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL373/history/20171002/1330Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL1619/history/20171002/1230Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL289/history/20171002/1130Z/KLGA/KORD",
"http://flightaware.com/live/flight/AAL304/history/20171002/1030Z/KLGA/KORD"
]
# if I have time, I'll write a function to get this by scraping.
# but for now, the 'manual' way...

flights = []
for url in flight_urls:
#     fn_p = re.compile(r'AAL\d{3,4}')
#     rn_p = re.compile(r'\d{4}Z')
#     fn = fn_p.search(url).group()
#     rn = rn_p.search(url).group()
    p = re.compile(r'(AAL\d{3,4}).*(\d{4}Z)')
    fn, rn = p.search(url).group(1,2)
    flights.append((fn,rn))<jupyter_output><empty_output><jupyter_text>Let the Scrapes Begin<jupyter_code>start = '20171002'
AAL_r1, missed_r1 = multiple_flights(365, start, flights[0:5])
AAL_r2, missed_r2 = multiple_flights(365, start, flights[5:9])
AAL_r3, missed_r3 = multiple_flights(365, start, flights[9:14])
AAL_r4, missed_r4 = multiple_flights(365, start, flights[14:])
AAL = pd.concat([AAL_r1, AAL_r2, AAL_r3, AAL_r4], ignore_index=True)
AAL_missed = missed_r1 + missed_r2 + missed_r3 + missed_r4
len(AAL_missed)<jupyter_output><empty_output><jupyter_text>Now That I Have It All...<jupyter_code>picklepath = f'../data/AAL_pass1.pkl'
with open(picklepath, 'wb') as picklefile:
    pickle.dump(AAL, picklefile)

filepath = f'../data/AAL_missed1.txt'
with open(filepath, 'w') as f:
    for url in AAL_missed:
        f.write(url+'\n')<jupyter_output><empty_output><jupyter_text>And the Cleanup<jupyter_code>AAL_stragglers1, missed_again = cleanup(AAL_missed)
AAL_stragglers2, missed_again2 = cleanup(missed_again)
AAL_stragglers3, foo = cleanup(missed_again2)
AAL = pd.concat([AAL, AAL_stragglers1, AAL_stragglers2, AAL_stragglers3], ignore_index=True)<jupyter_output><empty_output><jupyter_text>Most Importantly... Pickled!<jupyter_code>picklepath = f'../data/AAL_complete.pkl'
with open(picklepath, 'wb') as picklefile:
    pickle.dump(AAL, picklefile)
AAL.describe()<jupyter_output><empty_output><jupyter_text>Grabbing the last week to add to the test set<jupyter_code>AAL_recent, missed = multiple_flights(2, '20171004', flights)
AAL_strags, missed_again = cleanup(missed)
AAL_strags2, missed_again = cleanup(missed_again)
AAL_recent = pd.concat([AAL_recent, AAL_strags, AAL_strags2], ignore_index=True)
picklepath = f'../data/AAL_recent.pkl'
with open(picklepath, 'wb') as picklefile:
    pickle.dump(AAL_recent, picklefile)
AAL_recent.shape<jupyter_output><empty_output><jupyter_text>Merging with Round 1 Set<jupyter_code>with open('../data/AAL_complete.pkl', 'rb') as pf: 
    AAL_round1 = pickle.load(pf)
AAL_complete = pd.concat([AAL_round1, AAL_recent])
AAL_complete.shape
picklepath = f'../data/AAL_complete.pkl'
with open(picklepath, 'wb') as picklefile:
    pickle.dump(AAL_complete, picklefile)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11823025561475686, 'fraction_numerical': 0.02866392392257368, 'mean_word_length': 2.497994810096721, 'pattern_counts': {'":': 1, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '364858', 'license': 'MIT', 'n_tokens_mistral': 5384, 'n_tokens_neox': 4939, 'n_words': 1083, 'path': '2-delays-expected/code/1-Luther-Scraping.ipynb', 'repo_name': 'davidluther/Metis-Projects', 'size': '14827'}