<jupyter_start><jupyter_text>The goal of this project is to challenge is<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
# Python ≥3.5 is required
import sys
assert sys.version_info >= (3, 5)

# Scikit-Learn ≥0.20 is required
import sklearn
assert sklearn.__version__ >= "0.20"

# Common imports
import numpy as np
import os

# to make this notebook's output stable across runs
np.random.seed(42)

# To plot pretty figures
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)

# Where to save the figures
PROJECT_ROOT_DIR = "./finalproject"
CHAPTER_ID = "NYCTaxi"
IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID)
os.makedirs(IMAGES_PATH, exist_ok=True)

def save_fig(fig_id, tight_layout=True, fig_extension="png", resolution=300):
    path = os.path.join(IMAGES_PATH, fig_id + "." + fig_extension)
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format=fig_extension, dpi=resolution)

# Ignore useless warnings (see SciPy issue #5998)
import warnings
warnings.filterwarnings(action="ignore", message="^internal gelsd")
print('Directories created and several libraries successfully imported')
# List of libraries

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import json
from pandas.io.json import json_normalize
import seaborn as sns
import matplotlib.pyplot as plt
from plotly.offline import init_notebook_mode, iplot
import plotly.graph_objs as go
from plotly import tools
import plotly.figure_factory as ff
import numpy as np
init_notebook_mode(connected = True)
import datetime
import missingno as msno
print('Additional libraries successfully loaded')
import os
os.environ['KAGGLE_USERNAME'] = "hl533njit"
os.environ['KAGGLE_KEY'] = "02a6cedbb0e0800d3413432e287f2257"
!kaggle competitions download -c nyc-taxi-trip-duration
# api copied from kaggle
print('Kaggle Datasets Successfully Downloaded')
# extract training zipfile
import zipfile
zip = zipfile.ZipFile('train.zip')
zip.extractall()
print('Training Dataset Successfully UnZipped to train.csv')
#read first five rows of training dataset
import pandas as pd
df_train = pd.read_csv('train.csv')
df_train.head()
#return information about the dataframe including the data types of each column and memory usage
# of the entire data
df_train.info()
# extract Test zipfile
import zipfile
zip = zipfile.ZipFile('test.zip')
zip.extractall()
print('Test Dataset Successfully UnZipped to test.csv')
#read first five rows of test dataset
import pandas as pd
df_test = pd.read_csv('test.csv')
df_test.head()
# return information about the dataframe including the data types of each column and memory usage
# of the test dataset
df_test.info()
# Descriptive statistics that summarize the central tendency, dispersion and shape of the 
# training dataset’s distribution, excluding NaN values
df_train.describe()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08809135399673736, 'fraction_numerical': 0.01631321370309951, 'mean_word_length': 5.828507795100223, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '351086', 'license': 'MIT', 'n_tokens_mistral': 1006, 'n_tokens_neox': 951, 'n_words': 322, 'path': 'CS634Final_NYCTaxi.ipynb', 'repo_name': 'hl533/CS634FinalProject', 'size': '3065'}