<jupyter_start><jupyter_text>On-Time Flight Arrival Prediction  Abstract:Objective of this notebook is to predict the probability for flight delay.  This notebook is divided in 4 parts.    1. Data Cleaning    2. ML Model    3. Model Accuracy    4. Prediction  Data source: https://topcs.blob.core.windows.net/public/FlightData.csv  Dataset Loading:<jupyter_code># Use this line to Load the dataset from the server
#!curl https://topcs.blob.core.windows.net/public/FlightData.csv -o flightdata.csv
import pandas as pd

%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns

sns.set()
df=pd.read_csv("flightdata.csv")
df.head(5)<jupyter_output><empty_output><jupyter_text>Part 1: Data Cleaning<jupyter_code>df.shape
df.isnull().values.any()
df.isnull().sum()
## Droping the column
df = df.drop('Unnamed: 25', axis=1)
df.isnull().sum()
df = df[["MONTH", "DAY_OF_MONTH", "DAY_OF_WEEK", "ORIGIN", "DEST", "CRS_DEP_TIME", "ARR_DEL15"]]
df.isnull().sum()
df[df.isnull().values.any(axis=1)].head()
df = df.fillna({'ARR_DEL15': 1})
df.iloc[177:185]
import math

for index, row in df.iterrows():
    df.loc[index, 'CRS_DEP_TIME'] = math.floor(row['CRS_DEP_TIME'] / 100)
df.head()
df = pd.get_dummies(df, columns=['ORIGIN', 'DEST'])
df.head()<jupyter_output><empty_output><jupyter_text>Part 2: ML Model<jupyter_code>from sklearn.model_selection import train_test_split
train_x, test_x, train_y, test_y = train_test_split(df.drop('ARR_DEL15', axis=1), df['ARR_DEL15'], test_size=0.2, random_state=42)
from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier(random_state=13)
model.fit(train_x, train_y)
predicted = model.predict(test_x)
model.score(test_x, test_y)<jupyter_output><empty_output><jupyter_text>Part 3: Accuracy<jupyter_code>from sklearn.metrics import roc_auc_score
probabilities = model.predict_proba(test_x)
roc_auc_score(test_y, probabilities[:, 1])
from sklearn.metrics import confusion_matrix
confusion_matrix(test_y, predicted)
from sklearn.metrics import precision_score

train_predictions = model.predict(train_x)
precision_score(train_y, train_predictions)
from sklearn.metrics import recall_score

recall_score(train_y, train_predictions)
from sklearn.metrics import roc_curve

fpr, tpr, _ = roc_curve(test_y, probabilities[:, 1])
plt.plot(fpr, tpr)
plt.plot([0, 1], [0, 1], color='grey', lw=1, linestyle='--')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')<jupyter_output><empty_output><jupyter_text>Part 4: Prediction<jupyter_code>def predict_delay(departure_date_time, origin, destination):
    from datetime import datetime

    try:
        departure_date_time_parsed = datetime.strptime(departure_date_time, '%d/%m/%Y %H:%M:%S')
    except ValueError as e:
        return 'Error parsing date/time - {}'.format(e)

    month = departure_date_time_parsed.month
    day = departure_date_time_parsed.day
    day_of_week = departure_date_time_parsed.isoweekday()
    hour = departure_date_time_parsed.hour

    origin = origin.upper()
    destination = destination.upper()

    input = [{'MONTH': month,
              'DAY': day,
              'DAY_OF_WEEK': day_of_week,
              'CRS_DEP_TIME': hour,
              'ORIGIN_ATL': 1 if origin == 'ATL' else 0,
              'ORIGIN_DTW': 1 if origin == 'DTW' else 0,
              'ORIGIN_JFK': 1 if origin == 'JFK' else 0,
              'ORIGIN_MSP': 1 if origin == 'MSP' else 0,
              'ORIGIN_SEA': 1 if origin == 'SEA' else 0,
              'DEST_ATL': 1 if destination == 'ATL' else 0,
              'DEST_DTW': 1 if destination == 'DTW' else 0,
              'DEST_JFK': 1 if destination == 'JFK' else 0,
              'DEST_MSP': 1 if destination == 'MSP' else 0,
              'DEST_SEA': 1 if destination == 'SEA' else 0 }]

    return model.predict_proba(pd.DataFrame(input))[0][0]
predict_delay('1/10/2018 21:45:00', 'JFK', 'ATL') #test_x<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13137755102040816, 'fraction_numerical': 0.02066326530612245, 'mean_word_length': 4.476256983240224, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '533125', 'license': 'MIT', 'n_tokens_mistral': 1506, 'n_tokens_neox': 1412, 'n_words': 316, 'path': 'On-Time Flight Arrivals.ipynb', 'repo_name': 'pranav0904/Flight-Analysis', 'size': '3920'}