<jupyter_start><jupyter_text>Import the various libraries required to perform the exploratory data analysis and visualization of various feature valuesThe entire idea is to start with the data exploration look at which variables have an impact on the overall choice between self drive and autonomous vehicle acceptance. To get to that point we start with the text responses and encode the categorical data using a numeric scale starting from 0. The label column specifies the choice of a responder where self drive is considered 0 and autonomous vehicle acceptance is considered 1.<jupyter_code># this is a property which enforces the plots to be embedded inline
%matplotlib inline
# this is a cosmetic property controling the resoliution of the plot
%config InlineBackend.figure_format = 'retina'
import numpy as np # numpy library is for matrix math operations
import pandas as pd # pandas library is to access and manipulate the data using the dataframe
import matplotlib.pyplot as plt # matplotlib is to plot the data and draw visuals
import seaborn as sns # seaborn is a library for better visualisation
data_path = './text-survey-report-clean.csv' # data_path variable is used to define the path of the input data file

rawData = pd.read_csv(data_path) # rawdata is the dataframe we use to load the data
rawData.shape # shape will allow us to see the dimensions of the data
rawData.describe() # describe would show the basic stats of the data in each column
rawData.dtypes # dtypes would show the datatype of each column used for the subsequent encoding
rawData = rawData.drop('automation_level', axis=1) # dropping the automation level column as it is relatively 
# insignificant and users have made multiple entries and one hot encoding of this feature is irrelevant

# Replacing the labels of self drive and autonomous drive as 0 and 1 respectively
rawData['driving_preference'].replace({"Self-drive": 0, "Be driven by an autonomous car": 1}, inplace=True)

rawData.head() # going through the top records
# percentage of respondants accepting autonomous vehicle adoption by gender
rawData[["gender", "driving_preference"]].groupby(['gender'], as_index=False).mean().sort_values(by='driving_preference', ascending=False)
# percentage of respondants accepting autonomous vehicle adoption by age group
rawData[["age_group", "driving_preference"]].groupby(['age_group'], as_index=False).mean().sort_values(by='driving_preference', ascending=False)
# percentage of respondants accepting autonomous vehicle adoption by driving experience
rawData[["distance_driven", "driving_preference"]].groupby(['distance_driven'], as_index=False).mean().sort_values(by='driving_preference', ascending=False)
# percentage of respondants accepting autonomous vehicle adoption by legal driver status
# (the respondent has a drivers license)
rawData[["legal_driver", "driving_preference"]].groupby(['legal_driver'], as_index=False).mean().sort_values(by='driving_preference', ascending=False)
# Below is a violinplot - density plot representing the safety perception for different age groups
# The plot is shaded by gender
sns.violinplot('age_group', # Horizontal axis
               'safety_perception', # Vertical axis
               data=rawData, # Data source
               scale="count",
               hue="gender",
               split=True)

# Set title
plt.title('Safety perception by Age')

# Set x-axis label
plt.xlabel('Age Group')

# Set y-axis label
plt.ylabel('Safety Perception')
# Below is a violinplot - density plot representing the safety perception for different age groups
# The plot is shaded by label
sns.violinplot('age_group', # Horizontal axis
               'safety_perception', # Vertical axis
               data=rawData, # Data source
               scale="count",
               hue="driving_preference",
               split=True)

# Set title
plt.title('Safety perception by Age')

# Set x-axis label
plt.xlabel('Age Group')

# Set y-axis label
plt.ylabel('Safety Perception')
sns.pointplot(x="age_group", y="driving_preference", hue="gender", data=rawData,
              markers=["^", "o"], linestyles=["-", "--"])

# Set title
plt.title('Autonomous Vehicle Perception')

# Set x-axis label
plt.xlabel('Age Group')

# Set y-axis label
plt.ylabel('Readiness for Auto Drive')
# The following lines of code encodes categorical variables and maps them to numeric values for ease of calculation
# Replacing the likelyhood of consideration of smart cars in the next 5 and 10 years
rawData['likelyhood_partial_five'].replace({"Extremely likely": 4, "Somewhat likely": 3, "Neither likely nor unlikely": 2, "Somewhat unlikely": 1, "Extremely unlikely": 0}, inplace=True)
rawData['likelyhood_auto_ten'].replace({"Extremely likely": 4, "Somewhat likely": 3, "Neither likely nor unlikely": 2, "Somewhat unlikely": 1, "Extremely unlikely": 0}, inplace=True)

# Replacing the willingness to pay according to the order of magnitude
rawData['willingness_to_pay_extra'].replace({"$0-2500": 0, "$2500-5000": 1, "$5000-7500": 2, "$7500 & above": 3}, inplace=True)

# Replacing the opinion on self driving cars
rawData['opinion_on_autonomous_vehicles'].replace({"Fully-self driving cars will not be safe": 0, "I would trust a fully self-driving car to drive for me": 1, "Traveling in a fully self-driving car will be a positive experience": 2, "A fully self-driving car will free up my time and let me multitask": 3}, inplace=True)

# Replacing the level of concern
rawData['level_of_concern'].replace({"Very concerned": 2, "Moderately concerned": 1, "Slightly concerned": 0}, inplace=True)

# Replacing the perceived utility
rawData['perceived_usefulness'].replace({"Definitely yes": 4, "Probably yes": 3, "Might or might not": 2, "Probably not": 1, "Definitely not": 0}, inplace=True)

# Replacing the age group with number in descending order
rawData['age_group'].replace({">60 years": 0, "50-60 years": 1, "40-50 years": 2, "30-40 years": 3, "20-30 years": 4, "<20 years": 5}, inplace=True)

# Encoding the legal_driver status no - 0 and yes 1
rawData['legal_driver'].replace({"No": 0, "Yes": 1}, inplace=True)

# Encoding the distance driven from low to high
rawData['distance_driven'].replace({">20000 kilometers": 3, "10000-20000 kilometers": 2, "5000-10000 kilometers": 1, "0-5000 kilometers": 0}, inplace=True)

# Encoding the gender male - 0 and female 1
rawData['gender'].replace({"Male": 0, "Female": 1}, inplace=True)
cleanData = rawData.dropna()
cleanData.head()<jupyter_output><empty_output><jupyter_text>Further data mungingNow that we have converted the categorical variables into numbers which can meaningfully represent an individuals driving choice we need to work with the feature columns, the perceived value columns and the trust factor columns to normalize the ranks into fractions or percentages so that we can easily find the properties which increses the perceived value of the product and drives the customer decisionRank to percentile conversion and normalization:(100 / (highest rank - 1) * (highest rank - current rank)) / sum of all ranks (100 / (highest rank - 1) * (highest rank - current rank))<jupyter_code>cleanData.head(), cleanData.shape
# Apply the above convertRank2Percent function and convert the feature columns,
# the perceived value columns and the trust factor columns 
# Define a function to convert a rank into a percentile and then we normalize by adding all the percentile
# values and multiplying the total with each percentile value to get a percentage fraction which adds up to 1
# then we multiply by 100 and return the value ** experiment with removing the 100 multiplier
def convertRank2Percent(rank, maxRank):
    total = 0.0
    percentile = (100 / (maxRank - 1)) * (maxRank - rank)
    for i in range(1, maxRank):
        total += (100/(maxRank-1))*(maxRank-i)
    return (percentile / total) * 100
convertRank2Percent(2, 8) # test of the convert function
cols = list(cleanData.columns)

def getGroup(cols, groupName):
    queryCols = [f for f in cols if not f.find(groupName) ]
    return queryCols

groupNames = ['benefit_', 'new_feature_', 'trust_']
queryColsListByGroup  = [getGroup(cols, queryTerm) for queryTerm in groupNames] 
queryColsListByGroup

def custom_apply_function(data, column, maxRank):
    return np.vectorize(convertRank2Percent, otypes=[np.float]) (data[[column]], maxRank)

def findMaxRank(queryColsList):
    maxRankInGroup = cleanData.filter(queryColsList)[:1].max(axis=1).values.max()
    return maxRankInGroup

def applyMagic(cleanData):
    for queryColsList in queryColsListByGroup:
        maxRank = findMaxRank(queryColsList)
        print('maxRank for group is: ' ,maxRank) # print the rank for each group ...
        for queryCol in queryColsList:
            cleanData[[queryCol]] =  custom_apply_function(cleanData, queryCol, int(maxRank))
    return

# Apply Magic... Pass the DataFrame to which we want to apply the function
applyMagic(cleanData)
  
cleanData.shape
cleanData.head()
# Below is a violinplot - density plot representing the safety perception for different age groups
# The plot is shaded by gender
sns.violinplot('age_group', # Horizontal axis
               'safety_perception', # Vertical axis
               data=cleanData, # Data source
               scale="count",
               hue="gender",
               split=True)

# Set title
plt.title('Safety perception by Age')

# Set x-axis label
plt.xlabel('Age Group')

# Set y-axis label
plt.ylabel('Safety Perception')
# Below is a violinplot - density plot representing the safety perception for different age groups
# The plot is shaded by label
sns.violinplot('age_group', # Horizontal axis
               'safety_perception', # Vertical axis
               data=rawData, # Data source
               scale="count",
               hue="driving_preference",
               split=True)

# Set title
plt.title('Safety perception by Age')

# Set x-axis label
plt.xlabel('Age Group')

# Set y-axis label
plt.ylabel('Safety Perception')
# Below is a violinplot - density plot representing the safety perception for different age groups
# The plot is shaded by label
sns.violinplot('gender', # Horizontal axis
               'safety_perception', # Vertical axis
               data=rawData, # Data source
               scale="count",
               hue="driving_preference",
               split=True)

# Set title
plt.title('Safety perception by Gender')

# Set x-axis label
plt.xlabel('gender')

# Set y-axis label
plt.ylabel('Safety Perception')

# Set title
plt.title('Autonomous Vehicle Safety Perception')

# Set x-axis label
plt.xlabel('Gender')

# Set y-axis label
plt.ylabel('Perception')
sns.pointplot(x="age_group", y="driving_preference", hue="gender", data=cleanData,
              palette={0: "g", 1: "m"},
              markers=["^", "o"], linestyles=["-", "--"]);

# Set title
plt.title('Autonomous Vehicle Perception')

# Set x-axis label
plt.xlabel('Age Group')

# Set y-axis label
plt.ylabel('Readiness for Auto Drive')
# Compute the correlation matrix
corr = cleanData.corr()

# Generate a mask for the upper triangle
mask = np.zeros_like(corr, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True

# Set up the matplotlib figure
f, ax = plt.subplots(figsize=(25, 10))

# Generate a custom diverging colormap
cmap = sns.diverging_palette(220, 10, as_cmap=True)

# Draw the heatmap with the mask and correct aspect ratio
sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})<jupyter_output><empty_output><jupyter_text>Inferences from the above correlation plotThe above correlation plot clearly suggests that the label attribute or an individuals choice of an autonomous vehicle is positively correlated with perceived value of utility, safety perception, goverment regulations, safety rating etc.  Model Prediction ...<jupyter_code>from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import make_scorer, accuracy_score
cleanData.head()
X_all = cleanData.drop(['driving_preference'], axis=1)
y_all = cleanData['driving_preference']

X_train, X_test, y_train, y_test = train_test_split(X_all, y_all, test_size=0.5, random_state=23)
randomForest = RandomForestClassifier()
logisticRegression = LogisticRegression()
randomForest.fit(X_train, y_train)
logisticRegression.fit(X_train, y_train)
acc_scorer = make_scorer(accuracy_score)<jupyter_output><empty_output><jupyter_text>Predictions<jupyter_code>predictionsRf = randomForest.predict(X_test)
print(accuracy_score(y_test, predictionsRf))
predictionsLr = logisticRegression.predict(X_test)
print(accuracy_score(y_test, predictionsLr))<jupyter_output>0.7<jupyter_text>Coefficient & Intercept<jupyter_code>logisticRegression.coef_
logisticRegression.intercept_<jupyter_output><empty_output><jupyter_text>Cross Validation<jupyter_code>from sklearn.cross_validation import KFold
def run_kfold(clf):
    kf = KFold(199, n_folds=10)
    outcomes = []
    fold = 0
    for train_index, test_index in kf:
        fold += 1
        X_train, X_test = X_all.values[train_index], X_all.values[test_index]
        y_train, y_test = y_all.values[train_index], y_all.values[test_index]
        clf.fit(X_train, y_train)
        predictions = clf.predict(X_test)
        accuracy = accuracy_score(y_test, predictions)
        outcomes.append(accuracy)
        print("Fold {0} accuracy: {1}".format(fold, accuracy))     
    mean_outcome = np.mean(outcomes)
    print("Mean Accuracy: {0}".format(mean_outcome))
run_kfold(randomForest)
run_kfold(logisticRegression)<jupyter_output>Fold 1 accuracy: 0.8
Fold 2 accuracy: 0.95
Fold 3 accuracy: 0.85
Fold 4 accuracy: 0.75
Fold 5 accuracy: 0.6
Fold 6 accuracy: 0.75
Fold 7 accuracy: 0.5
Fold 8 accuracy: 0.9
Fold 9 accuracy: 0.85
Fold 10 accuracy: 1.0
Mean Accuracy: 0.7949999999999999
{'fraction_non_alphanumeric': 0.09703098106712564, 'fraction_numerical': 0.01800057372346529, 'mean_word_length': 4.682559087204564, 'pattern_counts': {'":': 43, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '653805', 'license': 'MIT', 'n_tokens_mistral': 4370, 'n_tokens_neox': 3991, 'n_words': 1430, 'path': 'market-research-exploration-autonomous-vehicle.ipynb', 'repo_name': 'debadyutirc/market-research-exploration-autonomous-vehicle', 'size': '13944'}