<jupyter_start><jupyter_text>Data Prep and Linear Regression for Car Prediction  Import Libraries<jupyter_code>import pandas as pd               # pandas is a dataframe library
import matplotlib.pyplot as plt   # matplotlib.pyplot plots data
import numpy as np                # numpy provides N dim object support
import pickle                     # Used to save our model

# do ploting inline instead of in a seperate window
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Load and review our data<jupyter_code>df = pd.read_csv("./Car price data.csv")   # Load the data<jupyter_output><empty_output><jupyter_text>Check its shape and look at some sample data<jupyter_code>df.shape
df.head(8)<jupyter_output><empty_output><jupyter_text>Check for any null values<jupyter_code>df.isnull().values.any()<jupyter_output><empty_output><jupyter_text>Convert any string values to numeric (our algorithms dont like non numeric)Hint - can we use any better way to convert values here?<jupyter_code>def handle_non_numerical_data(df):
    columns = df.columns.values
    for column in columns:
        text_digit_vals = {}
        def convert_to_int(val):
            return text_digit_vals[val]

        if df[column].dtype != np.int64 and df[column].dtype != np.float64:
            column_contents = df[column].values.tolist()
            unique_elements = set(column_contents)
            x = 0
            for unique in unique_elements:
                if unique not in text_digit_vals:
                    text_digit_vals[unique] = x
                    x+=1

            df[column] = list(map(convert_to_int, df[column]))

    return df
df2 = handle_non_numerical_data(df)
print(df2.columns)
print(df2.shape)
df2.head(8)<jupyter_output>Index(['symboling', 'make', 'fuel-type', 'aspiration', 'num-of-doors',
       'body-style', 'drive-wheels', 'engine-location', 'wheel-base', 'length',
       'width', 'height', 'curb-weight', 'engine-type', 'num-of-cylinders',
       'engine-size', 'fuel-system', 'bore', 'stroke', 'compression-ratio',
       'horsepower', 'peak-rpm', 'city-mpg', 'highway-mpg', 'price'],
      dtype='object')
(193, 25)<jupyter_text>Splitting the data into training and test sets70% for training and 30% for testing<jupyter_code>from sklearn.cross_validation import train_test_split

feature_col_names = ['symboling', 'make', 'fuel-type', 'aspiration', 'num-of-doors',
       'body-style', 'drive-wheels', 'engine-location', 'wheel-base', 'length',
       'width', 'height', 'curb-weight', 'engine-type', 'num-of-cylinders',
       'engine-size', 'fuel-system', 'bore', 'stroke', 'compression-ratio',
       'horsepower', 'peak-rpm', 'city-mpg', 'highway-mpg']
predicted_class_names = ['price']

X = df2[feature_col_names].values
y = df2[predicted_class_names].values
split_test_size = 0.30                 # test_size is 0.30 which is 30%

X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=split_test_size,random_state=7)
print("{0:0.2f}% in training set".format((len(X_train)/len(df2.index))*100))
print("{0:0.2f}% in test set".format((len(X_test)/len(df2.index))*100))<jupyter_output>69.95% in training set
30.05% in test set<jupyter_text>Lets have a look at how the data is split and how its correlated<jupyter_code>print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)<jupyter_output>(135, 24)
(135, 1)
(58, 24)
(58, 1)<jupyter_text>Correlation now<jupyter_code>def plot_corr(df, size=11):
    corr = df.corr()
    fig, ax = plt.subplots(figsize=(size,size))
    ax.matshow(corr)
    plt.xticks(range(len(corr.columns)),corr.columns)
    plt.yticks(range(len(corr.columns)),corr.columns)
plot_corr(df2)<jupyter_output><empty_output><jupyter_text>Lets fire up the Learning Algorithm (Linear regression)<jupyter_code>from sklearn.linear_model import LinearRegression

lin_model =LinearRegression()
lin_model.fit(X_train,y_train.ravel())

coeff = lin_model.score(X_test,y_test)            # Lets output the coefficient of determination (Accuracy)
print(coeff)

print(y_test[0:10].ravel())
lin_model.predict(X_test)[0:10]<jupyter_output>0.8728338598563763
[ 7957 11199  5195  9279  6692  9549 10345 22470 12170 18399]<jupyter_text>Plot a graph to compare Actual prices and Predicted prices<jupyter_code>plt.scatter(y_test,lin_model.predict(X_test))
plt.xlabel("Prices: $Y_i$")
plt.ylabel("Predicted Prices: $\hat{Y}_i$")
plt.title("Prices vs Predicted Prices: $Y_i$ vs $\hat{Y}_i$")<jupyter_output><empty_output><jupyter_text>Save our predictive model to disk<jupyter_code># save the model to disk
filename = '../car_prediction_model.pkl'
pickle.dump(lin_model, open(filename, 'wb'))<jupyter_output><empty_output><jupyter_text>Time to Pickle our X_Test and Y_Test Array so we can use it in our deploy section<jupyter_code>filename = '../X_test.pkl'
pickle.dump(X_test,open(filename,'wb'))

filename = '../y_test.pkl'
pickle.dump(y_test,open(filename,'wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13245967741935483, 'fraction_numerical': 0.02842741935483871, 'mean_word_length': 4.755220417633411, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '434751', 'license': 'MIT', 'n_tokens_mistral': 1873, 'n_tokens_neox': 1731, 'n_words': 372, 'path': 'Azure-ml Model Creation and Deploy/01. Model Creation/Creating our Car Prediction Model.ipynb', 'repo_name': 'craftsmanship2018/azure-ml', 'size': '4960'}