<jupyter_start><jupyter_text>Original source code: https://machinelearningmastery.com/tune-learning-rate-for-gradient-boosting-with-xgboost-in-python/<jupyter_code>from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import StratifiedKFold
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot
model = xgboost.XGBClassifier(nthread=16 ,objective='binary:logistic')
learning_rate = [0.2, 0.3, 0.5, 0.7, 0.9]
n_estimators = [1000,5000,6000, 7000, 10000]
param_grid = dict(learning_rate=learning_rate, n_estimators = n_estimators)
kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=7)
grid_search = GridSearchCV(model, param_grid, scoring="neg_log_loss", n_jobs=-1, cv=kfold)
grid_result = grid_search.fit(trainX, trainY)
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
	print("%f (%f) with: %r" % (mean, stdev, param))
# plot
pyplot.errorbar(learning_rate, means, yerr=stds)
pyplot.title("XGBoost learning_rate vs Log Loss")
pyplot.xlabel('learning_rate')
pyplot.ylabel('Log Loss')
pyplot.show()
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
import pickle
#kfold = StratifiedKFold(n_splits = 10, random_state=7)
xgb = xgboost.XGBClassifier(max_depth=500, n_estimators=1000, nthread=8 , objective='binary:logistic', learning_rate = 0.2 )
#results = cross_val_score(xgb, trainX, trainY, cv=kfold)
#print(results)
xgb.fit(trainX,trainY)
result = xgb.predict(testX)
acc = accuracy_score(testY, result)
cm = confusion_matrix(testY, result)
cr = classification_report(testY,result, target_names=['Building','Other'])
print 'Overall accuracy = {}\n'.format(acc)
#print 'Slum accuracy = {}\n'.format(cm[0,0]/np.sum(cm[0,:]))
print 'Confusion Matrix \n {}\n'.format(cm)
print 'Classification Report \n {}\n'.format(cr)

fname = 'LS-xgboost-2-Cl-model.sav'
pickle.dump(xgb, open(fname, 'wb'))<jupyter_output><empty_output><jupyter_text>Original source code: http://scikit-learn.org/stable/auto_examples/model_selection/plot_randomized_search.html<jupyter_code>from sklearn.ensemble import RandomForestClassifier

# Utility function to report best scores
def report(results, n_top=3):
    for i in range(1, n_top + 1):
        candidates = np.flatnonzero(results['rank_test_score'] == i)
        for candidate in candidates:
            print("Model with rank: {0}".format(i))
            print("Mean validation score: {0:.3f} (std: {1:.3f})".format(
                  results['mean_test_score'][candidate],
                  results['std_test_score'][candidate]))
            print("Parameters: {0}".format(results['params'][candidate]))
            print("")


#kfold = KFold(n_splits = 10, random_state=100)
rf = RandomForestClassifier(n_estimators = 500)
#results = cross_val_score(rf,trainX, trainY, cv=kfold)
#print results.mean(), results.std()
param_grid = {"max_depth": [3, None],
              "min_samples_split": [2, 3],
              "min_samples_leaf": [1, 3],
              "n_estimators": [500, 1000],
              "bootstrap": [True, False],
              "criterion": ["gini", "entropy"]}
grid_search = GridSearchCV(rf, param_grid = param_grid, scoring="neg_log_loss", n_jobs=-1, cv=kfold)
grid_result = grid_search.fit(trainX, trainY)
report(grid_result.cv_results_)


rf = RandomForestClassifier(n_estimators = 500, bootstrap = False, min_samples_leaf = 1, min_samples_split = 2, max_depth = None)
rf.fit(trainX,trainY)
result = rf.predict(testX)
acc = accuracy_score(testY, result)
cm = confusion_matrix(testY, result)
cr = classification_report(testY,result, target_names=['Building','Other'])
print 'Overall accuracy = {}\n'.format(acc)
#print 'Slum accuracy = {}\n'.format(cm[0,0]/np.sum(cm[0,:]))
print 'Confusion Matrix \n {}\n'.format(cm)
print 'Classification Report \n {}\n'.format(cr)
fname = 'LS-rf-2-Cl-model.sav'
pickle.dump(rf, open(fname, 'wb'))

from sklearn.naive_bayes import GaussianNB

nb = GaussianNB()
nb.fit(trainX,trainY)
result = nb.predict(testX)
acc = accuracy_score(testY, result)
cm = confusion_matrix(testY, result)
cr = classification_report(testY,result, target_names=['Building','Other'])
print 'Overall accuracy = {}\n'.format(acc)
print 'Confusion Matrix \n {}\n'.format(cm)
print 'Classification Report \n {}\n'.format(cr)

fname = 'LS-gnb-2-Cl-model.sav'
pickle.dump(nb, open(fname, 'wb'))
from sklearn.tree import DecisionTreeClassifier

dt = DecisionTreeClassifier(random_state=100)
dt.fit(trainX,trainY)
result = dt.predict(testX)
acc = accuracy_score(testY, result)
cm = confusion_matrix(testY, result)
cr = classification_report(testY,result, target_names=['Building','Other'])
print 'Overall accuracy = {}\n'.format(acc)
#print 'Slum accuracy = {}\n'.format(cm[0,0]/np.sum(cm[0,:]))
print 'Confusion Matrix \n {}\n'.format(cm)
print 'Classification Report \n {}\n'.format(cr)

fname = 'LS-dt-2-Cl-model.sav'
pickle.dump(dt, open(fname, 'wb'))
from sklearn.neighbors import KNeighborsClassifier
cv = StratifiedKFold(n_splits = 10, random_state =7 )
knn = KNeighborsClassifier()
n_neighbors = list(np.arange(3,11,1))
#print n_neighs
params_grid = dict(n_neighbors= n_neighbors)
knn_grid_search = GridSearchCV(estimator = knn, n_jobs = -1, param_grid = params_grid)
knn_grid_result = knn_grid_search.fit(trainX, trainY)
report(knn_grid_result.cv_results_)

knn = KNeighborsClassifier(n_neighbors = 10)
knn.fit(trainX, trainY)
result= knn.predict(testX)
acc = accuracy_score(testY, result)
cm = confusion_matrix(testY, result)
cr = classification_report(testY,result, target_names=['Building','Other'])
print 'Overall accuracy = {}\n'.format(acc)
#print 'Slum accuracy = {}\n'.format(cm[0,0]/np.sum(cm[0,:]))
print 'Confusion Matrix \n {}\n'.format(cm)
print 'Classification Report \n {}\n'.format(cr)

fname = 'LS-knn-2-Cl-model.sav'
pickle.dump(knn, open(fname, 'wb'))

from sklearn.svm import SVC
cv = StratifiedKFold(n_splits = 10, random_state =7 )
Cs = np.logspace(-6, -1, 10)
svmclf = SVC( kernel = 'poly')
svm_grid_search = GridSearchCV(estimator = svmclf, param_grid = dict(C=Cs), n_jobs = -1)
svm_grid_result = svm_grid_search.fit(trainX, trainY)
report(svm_grid_result.cv_results_)
svmclf = SVC(C=0.00016681005372000591)
svmclf.fit(trainX, trainY)
result = svmclf.predict(testX)
acc = accuracy_score(testY, result)
cm = confusion_matrix(testY, result)
cr = classification_report(testY,result, target_names=['Buildings','Other'])
print 'Overall accuracy = {}\n'.format(acc)
#print 'Slum accuracy = {}\n'.format(cm[0,0]/np.sum(cm[0,:]))
print 'Confusion Matrix \n {}\n'.format(cm)
print 'Classification Report \n {}\n'.format(cr)
from sklearn.neural_network import MLPClassifier

mlp = MLPClassifier()
params_grid={
'learning_rate': ["constant", "invscaling", "adaptive"],
'hidden_layer_sizes': [(100,100,100,100), (100,100,100,100,100), (100,100,100,100,100,100,100,100)],
'alpha': [0.0001, 0.00001, 0.01],
'activation': ["logistic", "relu", "tanh"]
}

mlp_grid_search = GridSearchCV(estimator=mlp,param_grid=params_grid,n_jobs=-1,cv=kfold)
mlp_grid_result = mlp_grid_search.fit(trainX, trainY)
report(mlp_grid_result.cv_results_)

mlp = MLPClassifier(activation='logistic', learning_rate = 'invscaling',hidden_layer_sizes=(100, 100, 100, 100), alpha = 0.00001)
mlp.fit(trainX, trainY)
result = mlp.predict(testX)
acc = accuracy_score(testY, result)
cm = confusion_matrix(testY, result)
cr = classification_report(testY,result, target_names=['Building','Other'])
print 'Overall accuracy = {}\n'.format(acc)
#print 'Slum accuracy = {}\n'.format(cm[0,0]/np.sum(cm[0,:]))
print 'Confusion Matrix \n {}\n'.format(cm)
print 'Classification Report \n {}\n'.format(cr)

fname = 'LS-mlp-2-Cl-model.sav'
pickle.dump(mlp, open(fname, 'wb'))
from sklearn.ensemble import AdaBoostClassifier
adb = AdaBoostClassifier()
params_grid = dict(n_estimators=[50,100,500,1000,5000], learning_rate=[0.01, 0.007, 0.0001, 0.1, 0.0007])
adb_grid_search = GridSearchCV(estimator=adb, param_grid = params_grid, cv=kfold)
adb_search_result = adb_grid_search.fit(trainX, trainY)
report(adb_search_result.cv_results_)
adb = AdaBoostClassifier(n_estimators=500, learning_rate=0.1)
adb.fit(trainX, trainY)
result = adb.predict(testX)
acc = accuracy_score(testY, result)
cm = confusion_matrix(testY, result)
cr = classification_report(testY,result, target_names=['Building','Other'])
print 'Overall accuracy = {}\n'.format(acc)
#print 'Slum accuracy = {}\n'.format(cm[0,0]/np.sum(cm[0,:]))
print 'Confusion Matrix \n {}\n'.format(cm)
print 'Classification Report \n {}\n'.format(cr)

fname = 'LS-adaboost-2-Cl-model.sav'
pickle.dump(adb, open(fname, 'wb'))<jupyter_output><empty_output><jupyter_text>Implement majority voting using all the classifiers from above (with the properly tuned parameters)<jupyter_code>from sklearn.ensemble import VotingClassifier

estimators = [('xgb',xgb), ('rf',rf),('nb',nb), ('dt',dt), ('knn',knn), ('mlp',mlp), ('adb',adb)]
vclf = VotingClassifier(estimators = estimators, n_jobs = -1)
vclf.fit(trainX, trainY)
result = vclf.predict(testX)
acc = accuracy_score(testY, result)
cm = confusion_matrix(testY, result)
cr = classification_report(testY,result, target_names=['Building','Other'])
print 'Overall accuracy = {}\n'.format(acc)
#print 'Slum accuracy = {}\n'.format(cm[0,0]/np.sum(cm[0,:]))
print 'Confusion Matrix \n {}\n'.format(cm)
print 'Classification Report \n {}\n'.format(cr)

fname = 'LS-majority-2-Cl-model.sav'
pickle.dump(adb, open(fname, 'wb'))
# Now predict for the entire image
from osgeo import gdal
import ogr
import matplotlib.pyplot as plt
import numpy as np
import fiona

driver = gdal.GetDriverByName('GTiff')
#rasterFileName = "/home/gadiraju/data/bl-slums/raw-img/MUL_mosaic_415.tif" #path to raster
rasterFileName1 = "/scratch/slums/bl-slums/raw-img/landsat7/PS-feb-2002-extract.tif"
dataset1 = gdal.Open(rasterFileName1)
rasterFileName2 = "/scratch/slums/bl-slums/raw-img/landsat7/ETM-feb-2002-extract-NDBI.tif"
dataset2 = gdal.Open(rasterFileName2)
rasterFileName3 = "/scratch/slums/bl-slums/raw-img/landsat7/simple-feb-2002-extract.tif"
dataset3 = gdal.Open(rasterFileName3)
rasterFileName4 = "/scratch/slums/bl-slums/raw-img/landsat7/adv-feb-2002-extract.tif"
dataset4 = gdal.Open(rasterFileName4)
rasterFileName5 = "/scratch/slums/bl-slums/raw-img/landsat7/higher-feb-2002-extract.tif"
dataset5 = gdal.Open(rasterFileName5)


bands1=[]
bands2=[]
bands3=[]
bands4=[]
bands5=[]
data_all_bands = []
cols = dataset1.RasterXSize
rows = dataset1.RasterYSize

transform = dataset1.GetGeoTransform()


img_data = np.zeros((rows, cols,36)) - 1
print img_data.shape

for i in range(6):
    band = dataset1.GetRasterBand(i+1)
    data = band.ReadAsArray(0,0,cols,rows).astype(np.float)
    img_data[:,:,i] = data
    # data_all_bands.append(band.ReadAsArray(0,0,cols,rows).astype(np.float))
band = dataset2.GetRasterBand(1)
data = band.ReadAsArray(0,0,cols,rows).astype(np.float)
img_data[:,:,6] = data
for i in range(8):#simple
    band = dataset3.GetRasterBand(i+1)
    data = band.ReadAsArray(0,0,cols,rows).astype(np.float)
    img_data[:,:,7+i] = data
for i in range(10):#adv
    band = dataset4.GetRasterBand(i+1)
    data = band.ReadAsArray(0,0,cols,rows).astype(np.float)
    img_data[:,:,15+i] = data
for i in range(11):#higher
    band = dataset5.GetRasterBand(i+1) 
    data = band.ReadAsArray(0,0,cols,rows).astype(np.float)
    img_data[:,:,25+i] = data

print img_data.shape

import osr

def write_image(imgrows, imgcols, imgbands, imggeotrans,edges, opath):
    driver = gdal.GetDriverByName('GTiff')
    outRaster = driver.Create(opath, imgrows, imgcols, imgbands, gdal.GDT_Float32)
    outRaster.SetGeoTransform(imggeotrans)
    outband = outRaster.GetRasterBand(1)
    outband.WriteArray(edges)
    outRasterSRS = osr.SpatialReference()
    outRasterSRS.ImportFromEPSG(32643)
    outRaster.SetProjection(outRasterSRS.ExportToWkt())
    outband.FlushCache()


model = rf
result_raster = np.zeros((rows, cols)) - 1
for i in range(rows):
    curr_row = img_data[i,:,:]
    result_raster[i,:] = model.predict(curr_row)
    if i%1000 == 0 and i > 0:
        print 'Finished {} rows'.format(i)
write_image(cols, rows, 1,transform, result_raster, '/scratch/slums/bl-slums/clf-img/LS-RF-Feb-2002.tif')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1571028411364546, 'fraction_numerical': 0.03137254901960784, 'mean_word_length': 6.059887005649718, 'pattern_counts': {'":': 6, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '370326', 'license': 'MIT', 'n_tokens_mistral': 5004, 'n_tokens_neox': 4706, 'n_words': 862, 'path': 'pixel-based/multi-class/landsat-clf/Pixel Based - All Models - Training 2 Class.ipynb', 'repo_name': 'NCSU-STAC/Slum-Classification', 'size': '12495'}