<jupyter_start><jupyter_text>All location entries in unleabled dataset is marked with ? intentionally to show unlabled data.<jupyter_code>#Information about the data variables in labled dataset
BLE_RSSI.info()
#Information about the data variables in Unlabled dataset
BLE_RSSI_UL.info()
#description of labled data including count, mean, std,min,max,median, 1st and 2nd quartiles.
BLE_RSSI.describe()
#description of Unlabled data including count, mean, std,min,max,median, 1st and 2nd quartiles.
BLE_RSSI_UL.describe()
locations = BLE_RSSI['location'][:] #The output variable, location, isolated from labled dataframe
dates = BLE_RSSI['date'][:]##The object type input variable, date, isolated from labled dataframe
features = BLE_RSSI.drop(['date','location'], axis = 1)#64int iBeacon RSSI valued variables for data processing
%matplotlib inline
import matplotlib.pyplot as plt 
features.hist(figsize=(20,15))# Plotting all the beacons values 
#x axis = RSSI value
#y axis = number of data points at a perticular RSSI value
plt.savefig('Leabeled_beacons_HIstograms.png')
plt.show()
#A quick look at the output variable, location, categories
locations.head(10)
locations.hist(figsize=(50,50))# Plotting all the beacons values 
#x axis = Location
#y axis = number of data points at a perticular location
plt.savefig('Leabeled_location_histogram.png')
plt.show()
from sklearn.preprocessing import OneHotEncoder
#Number the categories from 0 to 104
locations_encoded, locations_categories = pd.factorize(locations)
#One hot encoding
encoder = OneHotEncoder()
locations_1hot = encoder.fit_transform(locations_encoded.reshape(-1,1))
locations_categories 
#The following histogram shows the number of points at each category in location variable
plt.hist(locations_encoded)
locations_1hot.toarray()

#Correlation among the features
features.corr()
from pandas.plotting import scatter_matrix
#Scatter plot matrix to observe the similarities and diffeences between features visually
scatter_matrix(features, figsize=(12, 8))
plt.savefig('scatter_matrix.png')
plt.show()
from sklearn.model_selection import train_test_split
X = features
y = locations
#Deviding data into train, validate and test
#X_train1, X_test, y_train1, y_test indicates only train and test data groups
X_train1, X_test, y_train1, y_test =  train_test_split(X, y, test_size = 0.18, random_state =1017)
#X_test, y_test,X_train, X_val, y_train,y_val indicates train, validation and test dat groups
X_train, X_val, y_train,y_val = train_test_split(X_train1, y_train1, test_size=0.18, random_state=6296)
y_train1.hist(figsize=(50,50))# Plotting all the beacons values 
#x axis = Location
#y axis = number of data pointsat a perticular location
plt.savefig('Leabeled_location_histogram_trainset.png')
plt.show()
y_test.hist(figsize=(50,50))# Plotting all the beacons values 
#x axis = Location
#y axis = number of data pointsat a perticular location
plt.savefig('Leabeled_location_histogram_testset.png')
plt.show()
#The lengths of test, validation and test datasets
print('Length of X_train1, y_train1:',len(X_train1))
print('Length of X_train, y_train  :',len(X_train))
print('Length of X_val, y_val      :',len(X_val))
print('Length of X_test,y_test     :',len(X_test))
#Logistic Regression Model using default settings
from sklearn.linear_model import LogisticRegression

model_lr = LogisticRegression()
model_lr.fit(X_train, y_train)
from sklearn.model_selection import cross_val_predict
from sklearn.model_selection import cross_val_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report

#Performance analysis of default Logistic Regression Model
print('Logistic regression model score (Default): ',model_lr.score(X_test, y_test))
print(classification_report(y_test, model_lr.predict(X_test), digits=3))
import numpy as np
#The following for loop finds the best value of C
score = [];
for x in range (1 , 20):
    model_lr.set_params(C =x)
    model_lr.fit(X_train, y_train)
    score.append(model_lr.score(X_val, y_val))
    
x = np.arange(1,20,1);
y1 = score
plt.plot(x, y1)
plt.xlabel('C')
plt.ylabel('Score')
#Visulization of the coefficints of the model
model_lr = LogisticRegression(multi_class='ovr', C = 5,penalty='l2')
model_lr.fit(X_train, y_train)
model_lr.coef_
#Logistic regression model using K-fold CV
import warnings
warnings.filterwarnings("ignore")
from sklearn.model_selection import cross_val_score
score = []
ary = []
for x in range (2 , 27):
    ary = cross_val_score(model_lr, X_train1, y_train1, cv=x)
    score.append(max(ary[:]))
    
x = np.arange(2,27,1);
y_1 = score
plt.plot(x,y_1)
plt.xlabel('K-fold')
plt.ylabel('Score')
#from the graph above, CV = 26 seems a good fit
cross_val_score(model_lr, X_train1, y_train1, cv=26)
#Performance analysis of Logistic regression using K-fold cross validation
predicted = cross_val_predict(model_lr, X_test, y_test)
print('Logistic regression model score (k-fold cross validation): ',accuracy_score(y_test, predicted))
print(classification_report(y_test, predicted, digits=3))
#Fine-tuning of the logistic regression model using grid search CV
from sklearn.model_selection import GridSearchCV

# Tuning parameters: Penalty & the inverse of regularization strength (C)
parameters = {'penalty':('l1', 'l2'), 'C':[5,6,7,8,9,10]}
model_lr2 = LogisticRegression()
grid_search_lr = GridSearchCV(model_lr2, parameters,cv=5)
grid_search_lr.fit(X_train1,y_train1)
grid_search_lr.best_params_ #shows the best params
#Performance analysis Logistic Regression using GridSearchCV 
print('Logistic regression model score (GridSearchCV): ',grid_search_lr.score(X_test, y_test))
print(classification_report(y_test, grid_search_lr.predict(X_test), digits=3))
#Feature selection and fine tuning using PCA, selectKbest and grid search.
#Pipelining: chaining a PCA , a selectKbest and a logistic regression
#This code block took 26.5min to finish!

from sklearn.model_selection import KFold
from sklearn.pipeline import Pipeline
from sklearn.pipeline import FeatureUnion
from sklearn.decomposition import PCA
from sklearn.feature_selection import SelectKBest

#PCA
#The following statement is same for all three model: 
#Logistic Rregression, Decision Tree, Ramdom Forest
pca = PCA(n_components=5) 

#SelectKbest
#The following statement is same for all three model: 
#Logistic Rregression, Decision Tree, Ramdom Forest
selection = SelectKBest(k=5)

#Combination of PCA and SelectKbest features
#The following statement is same for all three model: 
#Logistic Rregression, Decision Tree, Ramdom Forest
combined_features = FeatureUnion([("pca", pca), ("univ_select", selection)])

#Logistic reression classifier
lr3 = LogisticRegression()

#pipelining
pipeline = Pipeline([("features", combined_features), ("lr3", lr3)])

#The total parameters of PCA, selectKbest and logistic regression
#Tunig parameters
    #PCA: n_components
    #SelectKbest: select_k
    #Logistic regression: C & penalty
param_grid = dict(features__pca__n_components= [5,10],
                  features__univ_select__k=[5,10],
                  lr3__C=[5,7],lr3__penalty=['l1','l2'])

grid_search = GridSearchCV(pipeline, param_grid=param_grid, cv = 3)
grid_search.fit(X_train1, y_train1)# Fitting the training data
print(grid_search.best_params_)# Best logistic regression params
print(grid_search.best_estimator_)#Best parameters f logistic regression
#Performance analysis of Logistic Regression using PCA, SlectKbest and GridSearchCV
prediction = grid_search.predict(X_test)
print('Logistic regression model score (Pipeline): ',accuracy_score(y_test, prediction))
print(classification_report(y_test, prediction, digits=3))
#Decision Tree default model
from sklearn.tree import DecisionTreeClassifier
model_dt = DecisionTreeClassifier()
model_dt.fit(X_train, y_train)
#Performance analysis of default Decision tree model
print('Decision tree model score (Default): ', model_dt.score(X_test, y_test))
print(classification_report(y_test, model_dt.predict(X_test), digits=3))
#Decision tree model using K-fold CV
score = []
ary = []
for x in range (2 , 27):
    ary = cross_val_score(model_dt, X_train1, y_train1, cv=x)
    score.append(max(ary[:]))
    
x = np.arange(2,27,1);
y_1 = score
plt.plot(x,y_1)
plt.xlabel('K-fold')
plt.ylabel('Score')
#K-Fold Cross Validation of Decision tree model
kfold = KFold(n_splits=24, random_state=7)
score_arr = cross_val_score(model_dt, X, y, cv=kfold)

#Performance Analysis of K-Fold Cross Validation of Decision tree model
print('Decision tree model mean score (K-fold cross validation): ',score_arr.mean())
predicted = cross_val_predict(model_dt, X_test, y_test)
print(classification_report(y_test, predicted, digits=3))
#Feature selection and fine tuning using PCA, selectKbest and grid search.
#Pipelining: chaining a PCA , a selectKbest and a decision tree
#This code block took 24.1sec to finish!

#Tuning parameters: criterion & max_depth
parameters = {'criterion':('gini', 'entropy'), 'max_depth':np.arange(1,50)}
dTree = GridSearchCV(DecisionTreeClassifier(), parameters, cv =24)
dTree.fit(X_train1,y_train1)
dTree.best_params_#Best params
#Performance analysis Decision Tree using GridSearchCV 
print('Decision tree model score (GridSearchCV): ',dTree.score(X_test,y_test))
print(classification_report(y_test, dTree.predict(X_test), digits=3))
#Feature selection and fine tuning using PCA, selectKbest and grid search.
#Pipelining: chaining a PCA , a selectKbest and a decision tree
#This code block took 4.5min to finish!

#Decision tree
tree =DecisionTreeClassifier();

#pipelinig
pipeline = Pipeline([("features", combined_features), ("tree", tree)])

#Tunig parameters
#PCA: n_components
#SelectKbest: select_k
#Decision Tree: criterion & max_depth
param_grid = dict(features__pca__n_components= [1,5,8,12],
                  features__univ_select__k=[1,5,8,12],
                  tree__criterion=['gini', 'entropy'], tree__max_depth=np.arange(1,30))

tree_search = GridSearchCV(pipeline, param_grid=param_grid, cv = 5)
tree_search.fit(X_train1, y_train1)
print(tree_search.best_params_)
#Performance analysis of Decision tree using PCA, SelectKbest and GridSearchCV
prediction = tree_search.predict(X_test)
print('Decision tree model score (Pipeline): ', tree_search.score(X_test,y_test))
print(classification_report(y_test, prediction, digits=3))
#Default model of Random Forest
from sklearn.ensemble import RandomForestClassifier
model_rf = RandomForestClassifier()
model_rf.fit(X_train, y_train)
#Performance analysis of default random forest model
print('Random Forest model score (Default):',model_rf.score(X_test, y_test))
print(classification_report(y_test, model_rf.predict(X_test), digits=3))
#Random forest model using K-fold CV
score = []
ary = []
for x in range (2 , 27):
    ary = cross_val_score(model_rf, X_train1, y_train1, cv=x)
    score.append(max(ary[:]))
    
x = np.arange(2,27,1);
y_1 = score
plt.plot(x,y_1)
plt.xlabel('K-fold')
plt.ylabel('Score')
#K-Fold Cross Validation of Random Forest model
kfold = KFold(n_splits=22, random_state=7)
score_arr = cross_val_score(model_rf, X, y, cv=kfold)

#Performance Analysis of K-Fold Cross Validation of Decision tree model
print('Random forest model mean score (K-fold cross validation): ',score_arr.mean())
predicted = cross_val_predict(model_rf, X_test, y_test)
print(classification_report(y_test, predicted, digits=3))
#Fine tuning of the logistic regression model using grid search CV
#This code block took 1.8min to finish!

#Tuning parameters: max_features & n_estimators
parameters = {'max_features':['log2', 'auto',5,8,1,3,0.1,0.7], 'n_estimators':[10,15,20,30,40]}
rf = RandomForestClassifier(bootstrap = True,n_jobs = -1)
grid_search_rf = GridSearchCV(rf, parameters,cv=8)
grid_search_rf.fit(X_train1,y_train1)
grid_search_rf.best_params_#The best params
#Performance analysis Random Forest using GridSearchCV 
print('Random forest model score (GridSearchCV): ',grid_search_rf.score(X_test,y_test))
print(classification_report(y_test, grid_search_rf.predict(X_test), digits=3))
#Feature selection and fine tuning using PCA, selectKbest and grid search.
#Pipelining: chaining a PCA , a selectKbest and a random forest
#This code block took 13.6min to finish!

#Random Forest Classifier
rftree =RandomForestClassifier(bootstrap = True,n_jobs = -1);

#pipelining
pipeline = Pipeline([("features", combined_features), ("rftree", rftree)])

#Tunig parameters
#PCA: n_components
#SelectKbest: select_k
#Random Forest: max_features & n_estimators
param_grid = dict(features__pca__n_components= [3,5,8,12],
                  features__univ_select__k=[3,5,8,12],
                  rftree__max_features=['log2','auto',5,3,0.1], rftree__n_estimators =[20,30,40,50,60] )

rftree_search = GridSearchCV(pipeline, param_grid=param_grid, cv = 5)
rftree_search.fit(X_train1, y_train1)
print(rftree_search.best_params_)
##Performance analysis of Random forest using PCA, SlectKbest and GridSearchCV
print('Random forest model score (Pipeline): ', rftree_search.score(X_test,y_test))
print(classification_report(y_test, rftree_search.predict(X_test), digits=3))
#Voting classifier
#voting classifier collects the best models and 
#outputs the result based on the best outcome of the best models 
#This code block took approx. 15min to finish!


from sklearn.ensemble import VotingClassifier
estimators = [('rf1',rftree_search),('dt',dTree),('rf2',grid_search_rf)]
voting_clf = VotingClassifier(estimators, voting='soft')
voting_clf.fit(X_train1,y_train1)
#Voting Classifier vs The best models
#This code block took approx. 17min to finish!

#The following commented for loop is the loop version 
#of voting classifer vs the best models comparison
#taken from the hands on machine learning book

#for models in (rftree_search, dTree, grid_search_rf, voting_clf):
#    models.fit(X_train1,y_train1)
#    y_pred = models.predict(X_test)
#    print(models.__class__.__name__,accuracy_score(y_test,y_pred))


#Comparison between the best classifiers
#tree_search.fit(X_train1,y_train1)
y_pred = tree_search.predict(X_test)
print('Decision Tree Grid Search CV with PCA:',accuracy_score(y_test,y_pred))
#dTree.fit(X_train1,y_train1)
y_pred = dTree.predict(X_test)
print('Decision Tree Grid Search CV without PCA::',accuracy_score(y_test,y_pred))
#grid_search_rf.fit(X_train1,y_train1)
y_pred = grid_search_rf.predict(X_test)
print('Random Forest Tree Grid Search without PCA:',accuracy_score(y_test,y_pred))
#voting_clf.fit(X_train1,y_train1)
y_pred = voting_clf.predict(X_test)
print('Voting Classifier:',accuracy_score(y_test,y_pred))<jupyter_output>Decision Tree Grid Search CV with PCA: 0.328125
Decision Tree Grid Search CV without PCA:: 0.3203125
Random Forest Tree Grid Search without PCA: 0.33203125
Voting Classifier: 0.3125
{'fraction_non_alphanumeric': 0.10100873332881999, 'fraction_numerical': 0.020106966352988965, 'mean_word_length': 6.774736842105263, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '191649', 'license': 'MIT', 'n_tokens_mistral': 5120, 'n_tokens_neox': 4841, 'n_words': 1373, 'path': 'EE259_Final_Project.ipynb', 'repo_name': 'jbp261/OPTIMAL-CLASSIFICATION-MODEL-OF-BLE-RSSI-DATASET', 'size': '14771'}