<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import scipy.stats 
import pandas as pd
import statsmodels.api as sm
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression 
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline<jupyter_output>/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.
  import pandas.util.testing as tm<jupyter_text>1. Read the csv file and show columns<jupyter_code>from google.colab import drive
drive.mount('/content/drive/')
%cd '/content/drive/My Drive/Colab Notebooks/db'
!ls
df = pd.read_csv('cruise_ship_info.csv')
df.columns<jupyter_output><empty_output><jupyter_text>2. Basic statistic analysis<jupyter_code>pd.isnull(df).sum()
df[df.duplicated()]
df[["Age", "Tonnage", "passengers", "length", "cabins", "passenger_density", "crew"]].describe()
# CREW ANALYSIS -- NORMAL DISTRIBUTION

crew_min = pd.DataFrame.min(df["crew"])
crew_max = pd.DataFrame.max(df["crew"])

crew_mean = pd.DataFrame.mean(df["crew"])
crew_std = pd.DataFrame.std(df["crew"])

crew_x = np.linspace(crew_min, crew_max)
crew_y = scipy.stats.norm.pdf(crew_x,crew_mean,crew_std)

plt.plot(crew_x, crew_y, color = "black")

## Sigma 1

pt1 = crew_mean + crew_std
plt.plot([pt1 ,pt1 ],[0.0,scipy.stats.norm.pdf(pt1 ,crew_mean, crew_std)], color='black')

pt2 = crew_mean - crew_std
plt.plot([pt2 ,pt2 ],[0.0,scipy.stats.norm.pdf(pt2 ,crew_mean, crew_std)], color='black')

ptx = np.linspace(pt1, pt2, 10)
pty = scipy.stats.norm.pdf(ptx, crew_mean, crew_std)

plt.fill_between(ptx, pty, color='#0b559f', alpha=1.0)

## Simga +2

pt1 = crew_mean + crew_std
plt.plot([pt1 ,pt1 ],[0.0,scipy.stats.norm.pdf(pt1, crew_mean, crew_std)], color='black')

pt2 = crew_mean + 2.0 * crew_std
plt.plot([pt2 ,pt2 ],[0.0,scipy.stats.norm.pdf(pt2, crew_mean, crew_std)], color='black')

ptx = np.linspace(pt1, pt2, 10)
pty = scipy.stats.norm.pdf(ptx, crew_mean, crew_std)

plt.fill_between(ptx, pty, color='#2b7bba', alpha=1.0)

## Sigma -2 

pt1 = crew_mean - crew_std
plt.plot([pt1 ,pt1 ],[0.0,scipy.stats.norm.pdf(pt1, crew_mean, crew_std)], color='black')

pt2 = crew_mean - 2.0 * crew_std
plt.plot([pt2 ,pt2 ],[0.0,scipy.stats.norm.pdf(pt2, crew_mean, crew_std)], color='black')

ptx = np.linspace(pt1, pt2, 10)
pty = scipy.stats.norm.pdf(ptx,crew_mean,crew_std)

plt.fill_between(ptx, pty, color='#2b7bba', alpha=1.0)

## Sigma +3

pt1 = crew_mean + 2.0 * crew_std
plt.plot([pt1 ,pt1 ],[0.0,scipy.stats.norm.pdf(pt1, crew_mean, crew_std)], color='black')

pt2 = crew_mean + 3.0 * crew_std
plt.plot([pt2 ,pt2 ],[0.0,scipy.stats.norm.pdf(pt2, crew_mean, crew_std)], color='black')

ptx = np.linspace(pt1, pt2, 10)
pty = scipy.stats.norm.pdf(ptx,crew_mean,crew_std)

plt.fill_between(ptx, pty, color='#539ecd', alpha=1.0)

## Sigma -3

pt1 = crew_mean - 2.0 * crew_std
plt.plot([pt1 ,pt1 ],[0.0,scipy.stats.norm.pdf(pt1, crew_mean, crew_std)], color='black')

pt2 = crew_mean - 3.0 * crew_std
plt.plot([pt2 ,pt2 ],[0.0,scipy.stats.norm.pdf(pt2, crew_mean, crew_std)], color='black')

ptx = np.linspace(pt1, pt2, 10)
pty = scipy.stats.norm.pdf(ptx,crew_mean,crew_std)

plt.fill_between(ptx, pty, color='#539ecd', alpha=1.0)

## Sigma +4

pt1 = crew_mean + 3.0 * crew_std
plt.plot([pt1 ,pt1 ],[0.0,scipy.stats.norm.pdf(pt1, crew_mean, crew_std)], color='black')

pt2 = crew_mean + 10.0 * crew_std
plt.plot([pt2 ,pt2 ],[0.0,scipy.stats.norm.pdf(pt2, crew_mean, crew_std)], color='black')

ptx = np.linspace(pt1, pt2, 10)
pty = scipy.stats.norm.pdf(ptx,crew_mean,crew_std)

plt.fill_between(ptx, pty, color='#89bedc', alpha=1.0)

## Plotting

plt.grid()

plt.xlim(crew_min, crew_max)
plt.ylim(0,0.12)

plt.title("Normal Distribution of Crew Members")

plt.xlabel("Crew Members")
plt.ylabel("Normal Distribution")

plt.show
# CREW ANALYSIS -- BOXPLOT

crew_boxplot = df.boxplot(column=['crew'])<jupyter_output><empty_output><jupyter_text>**Dispersion Analysis**<jupyter_code>crew_age = sns.regplot(x=df["Age"], y=df["crew"])
crew_tonnage = sns.regplot(x=df["Tonnage"], y=df["crew"])

crew_pass = sns.regplot(x=df["passengers"], y=df["crew"])
crew_length = sns.regplot(x=df["length"], y=df["crew"])
crew_cabins = sns.regplot(x=df["cabins"], y=df["crew"])
crew_denpass = sns.regplot(x=df["passenger_density"], y=df["crew"])<jupyter_output><empty_output><jupyter_text>**Correlation Analysis**<jupyter_code>corr = df.corr()

sns.heatmap(corr, annot = True, yticklabels=corr.columns, xticklabels=corr.columns)<jupyter_output><empty_output><jupyter_text>For our Exploratory Data Analysis, we start assesing the most important variable for the recommendation algorithm: crew members. A remarkable fact is that the standard deviation is considerable: 44.92%. The latter means that our data extends through a large range around the median value. Likewise, the bloxpot graph allows to see the existance of few outliers which might have an impact on the position of our mean value.The most outstanding discovery of the EDA process is the correlation between variables. We start from assigning crew members as a dependant value in the y axis and the independent value in the x axis for the remaining values. We're, thus, assuming that the number of crew members depends on the condition of the ship -- e.g. the oldest or larger the ship is, a more significant crew is needed to attend the ship. There's a strong correlation in 5 of the 7 independent variables analyzed, which are going to be part of the model to be trained.   3 and 4. Column Selection and Deleting For our project, we're going to make an algorithm which recommends a number of crew members based on relatable variables such as cabins, tonnage, passengers capacity and length. Age and passenger density are being deleted because they doesn't provide further information value given the results of the scatter plots and heatmap during the EDA process.We'll also delete de Ship_name and Cruise_line column, because they're cathegorical values that doesn't provide further numeric value based on ordinal relationships or a binary system.<jupyter_code>working_df = df.drop(["Ship_name", "Cruise_line", "Age", "passenger_density"], axis=1)
working_df<jupyter_output><empty_output><jupyter_text>5. One-Hot Encoding<jupyter_code>ct = ColumnTransformer([("Cruise_line", OneHotEncoder(), [1])], remainder = 'passthrough')
ohe_df = ct.fit_transform(df)
pd.DataFrame(ohe_df)<jupyter_output><empty_output><jupyter_text>6. Clusters for Training and Testing<jupyter_code># Outliers cleaning with interquartile range

Q1 = working_df.quantile(0.25)
Q3 = working_df.quantile(0.75)
IQR = Q3 - Q1
IQR
print(working_df < (Q1 - 1.5 * IQR)) or (working_df > (Q3 + 1.5 * IQR))
clean_df = working_df[~((working_df < (Q1 - 1.5 * IQR)) | (working_df > (Q3 + 1.5 * IQR))).any(axis=1)]
clean_df
# Obtaining an Aleatory Sample (60% of total clean data)

sample_df = clean_df.sample(frac=0.6)
sample_df
y = sample_df["crew"]
X = sample_df[["Tonnage", "passengers", "length", "cabins"]].iloc[:,0:4].values

print(X.shape)
print(y.shape)
X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.25, random_state=0)
print(X_test.shape)
print(X_train.shape)
print(y_test.shape)
print(y_train.shape)<jupyter_output>(23, 4)
(67, 4)
(23,)
(67,)<jupyter_text>7. Model Training<jupyter_code>reg = LinearRegression().fit(X_train, y_train)
y_train_predict = reg.predict(X_train)
y_test_predict = reg.predict(X_test)
plt.scatter(y_train_predict,  y_train_predict - y_train,
            c='blue')       
plt.scatter(y_test_predict,  y_test_predict - y_test,
            c='red')
plt.xlabel('Predicted values')
plt.ylabel('Residual')
plt.hlines(y=0, xmin=-10, xmax=50, color='black', lw=2)
plt.xlim([-10, 50])<jupyter_output><empty_output><jupyter_text>8. Hyperparameters and Conclusions<jupyter_code>rmse = mean_squared_error(y_train, y_train_predict)
r2 = r2_score(y_train, y_train_predict)

print(rmse)
print(r2)<jupyter_output>0.6600178607455991
0.9356032340986901
{'fraction_non_alphanumeric': 0.14087564829333013, 'fraction_numerical': 0.03160053069593535, 'mean_word_length': 6.003378378378378, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '325707', 'license': 'MIT', 'n_tokens_mistral': 3378, 'n_tokens_neox': 3201, 'n_words': 734, 'path': 'Basic_Exercise_DS_Python2.ipynb', 'repo_name': 'XavierCarrera/Platzi-Master-DS-Exercises', 'size': '8291'}