<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
!pip install rasterio
import rasterio
!pip install geopandas
import geopandas as gpd
import numpy as np
from google.colab import drive
drive.mount('/content/gdrive')
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.model_selection import GridSearchCV
lokus=['jabodetabek','mebidangro','gerbangkertasusila','kedungsepur','maminasata','bandungraya']
from sklearn import preprocessing
C_range = [0.1,1,10,100]
param_grid = dict( C=C_range)
string_grid=[]
for lokasi in lokus:
  print(lokasi)
  dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_'+lokasi+'.tif')
  dataset_bands=pd.DataFrame()
  for i in dataset.indexes:
    temp=dataset.read(i)
    temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
    dataset_bands=temp.join(dataset_bands)
  dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
  dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
  dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
  dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
  dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]
  svclassifier = SVC(kernel='rbf')
  minmax_=preprocessing.MinMaxScaler()
  X=minmax_.fit_transform(dataset[['rad_VIIRS','SAVI_landsat7','NDVI_landsat7','MNDWI_landsat7','NDBI_landsat7','LSE_der_landsat7']])
  y=dataset[['U_class']]
  cv = StratifiedShuffleSplit(n_splits=10, test_size=0.2, random_state=42)
  grid = GridSearchCV(SVC(), param_grid=param_grid, cv=cv,verbose=1)
  grid.fit(X, y)
  str_r="The best parameters are %s %s with a score of %0.2f"% (grid.best_params_,lokasi, grid.best_score_)
  string_grid.append(str_r)
from sklearn.ensemble import RandomForestClassifier
param_grid = { 
    'n_estimators': [100, 200, 500],
    'criterion' :['entropy'],
    'max_depth':[4,5,6]
}
string_grid3=[]
for lokasi in lokus:
  print(lokasi)
  dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_'+lokasi+'.tif')
  dataset_bands=pd.DataFrame()
  for i in dataset.indexes:
    temp=dataset.read(i)
    temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
    dataset_bands=temp.join(dataset_bands)
  dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
  dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
  dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
  dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
  dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]
  rfclassifier = RandomForestClassifier(random_state=123)
  minmax_=preprocessing.MinMaxScaler()
  X=minmax_.fit_transform(dataset[['rad_VIIRS','SAVI_landsat7','NDVI_landsat7','MNDWI_landsat7','NDBI_landsat7','LSE_der_landsat7']])
  cv = StratifiedShuffleSplit(n_splits=10, test_size=0.2, random_state=42)
  grid = GridSearchCV(rfclassifier, param_grid=param_grid, cv=cv,verbose=1)
  grid.fit(X, y)
  str_r="The best parameters are %s %s with a score of %0.2f"% (grid.best_params_,lokasi, grid.best_score_)
  string_grid3.append(str_r)
string_grid
string_grid3
!pip install xgboost
import xgboost as xgb
from sklearn.ensemble import RandomForestClassifier
parameters = {'nthread':[4], #when use hyperthread, xgboost may become slower
              'objective':['binary:logistic'],
              'learning_rate': [0.07], #so called `eta` value
              'max_depth': [4,5,6],
              'min_child_weight': [11],
              'silent': [1],
              'subsample': [0.8],
              'colsample_bytree': [0.7],
              'n_estimators': [100,200,500], #number of trees, change it to 1000 for better results
              'seed': [123]}
string_grid4=[]
for lokasi in lokus:
  print(lokasi)
  dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_'+lokasi+'.tif')
  dataset_bands=pd.DataFrame()
  for i in dataset.indexes:
    temp=dataset.read(i)
    temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
    dataset_bands=temp.join(dataset_bands)
  dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
  dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
  dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
  dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
  dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]
  rfclassifier = xgb.XGBClassifier()
  minmax_=preprocessing.MinMaxScaler()
  X=minmax_.fit_transform(dataset[['rad_VIIRS','SAVI_landsat7','NDVI_landsat7','MNDWI_landsat7','NDBI_landsat7','LSE_der_landsat7']])
  y=dataset[['U_class']]
  cv = StratifiedShuffleSplit(n_splits=10, test_size=0.2, random_state=42)
  grid = GridSearchCV(rfclassifier, param_grid=parameters, cv=cv)
  grid.fit(X, y)
  str_r="The best parameters are %s %s with a score of %0.2f"% (grid.best_params_,lokasi, grid.best_score_)
  string_grid4.append(str_r)
string_grid
string_grid3
string_grid4
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_jabodetabek.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
svclassifier = SVC(kernel='rbf',C=100,random_state=123)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
svclassifier.fit(X_train, y_train)
y_pred = svclassifier.predict(X_test)
from sklearn.metrics import classification_report, confusion_matrix
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,n_estimators=500,criterion='entropy')
rfclassifier.fit(X_train,y_train)
y_pred=rfclassifier.predict(X_test)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, learning_rate=0.07, max_depth= 4, 
                             min_child_weight=11, n_estimators= 500, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
y_pred=xgclassifier.predict(X_test)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
from sklearn.model_selection import KFold,cross_val_predict,cross_val_score,cross_validate
X=dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']]
y=dataset[['U_class']]
svc_acc_jabodetabek=np.mean(cross_val_score(svclassifier, X, y, cv=10,scoring='roc_auc'))
rbf_acc_jabodetabek=np.mean(cross_val_score(rfclassifier, X, y, cv=10,scoring='roc_auc'))
xgb_acc_jabodetabek=np.mean(cross_val_score(xgclassifier, X, y, cv=10,scoring='roc_auc'))
## Gerbang kertasusila
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_gerbangkertasusila.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)

svclassifier = SVC(kernel='rbf',C=100,random_state=123)
svclassifier.fit(X_train, y_train)
y_pred = svclassifier.predict(X_test)
from sklearn.metrics import classification_report, confusion_matrix
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,
                                      n_estimators=200,criterion='entropy')
rfclassifier.fit(X_train,y_train)
y_pred=rfclassifier.predict(X_test)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, 
                                 learning_rate=0.05, max_depth= 5, 
                             min_child_weight=11, n_estimators= 500, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
y_pred=xgclassifier.predict(X_test)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
from sklearn.model_selection import KFold,cross_val_predict,cross_val_score,cross_validate
X=dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']]
y=dataset[['U_class']]
svc_acc_gerbang_=np.mean(cross_val_score(svclassifier, X, y, cv=10,scoring='roc_auc'))
rbf_acc_gerbang_=np.mean(cross_val_score(rfclassifier, X, y, cv=10,scoring='roc_auc'))
xgb_acc_gerbang_=np.mean(cross_val_score(xgclassifier, X, y, cv=10,scoring='roc_auc'))

## Mebidangro
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_mebidangro.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)

svclassifier = SVC(kernel='rbf',C=100,random_state=123)
svclassifier.fit(X_train, y_train)
y_pred = svclassifier.predict(X_test)
from sklearn.metrics import classification_report, confusion_matrix
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,n_estimators=500,criterion='entropy')
rfclassifier.fit(X_train,y_train)
y_pred=rfclassifier.predict(X_test)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, 
                                 learning_rate=0.05, max_depth= 4, 
                             min_child_weight=11, n_estimators= 500, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
y_pred=xgclassifier.predict(X_test)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
from sklearn.model_selection import KFold,cross_val_predict,cross_val_score,cross_validate
X=dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']]
y=dataset[['U_class']]
svc_acc_mebidangro_=np.mean(cross_val_score(svclassifier, X, y, cv=10,scoring='roc_auc'))
rbf_acc_mebidangro_=np.mean(cross_val_score(rfclassifier, X, y, cv=10,scoring='roc_auc'))
xgb_acc_mebidangro_=np.mean(cross_val_score(xgclassifier, X, y, cv=10,scoring='roc_auc'))

## Kedungsepur
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_kedungsepur.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)

svclassifier = SVC(kernel='rbf',C=100,random_state=123)
svclassifier.fit(X_train, y_train)
y_pred = svclassifier.predict(X_test)
from sklearn.metrics import classification_report, confusion_matrix
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,n_estimators=500,criterion='entropy')
rfclassifier.fit(X_train,y_train)
y_pred=rfclassifier.predict(X_test)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, 
                                 learning_rate=0.05, max_depth= 4, 
                             min_child_weight=11, n_estimators= 500, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
y_pred=xgclassifier.predict(X_test)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
from sklearn.model_selection import KFold,cross_val_predict,cross_val_score,cross_validate
X=dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']]
y=dataset[['U_class']]
svc_acc_kedungsepur_=np.mean(cross_val_score(svclassifier, X, y, cv=10,scoring='roc_auc'))
rbf_acc_kedungsepur_=np.mean(cross_val_score(rfclassifier, X, y, cv=10,scoring='roc_auc'))
xgb_acc_kedungsepur_=np.mean(cross_val_score(xgclassifier, X, y, cv=10,scoring='roc_auc'))

## Maminsasat
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_maminasata.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)

svclassifier = SVC(kernel='rbf',C=100,random_state=123)
svclassifier.fit(X_train, y_train)
y_pred = svclassifier.predict(X_test)
from sklearn.metrics import classification_report, confusion_matrix
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,n_estimators=500,criterion='entropy')
rfclassifier.fit(X_train,y_train)
y_pred=rfclassifier.predict(X_test)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, 
                                 learning_rate=0.05, max_depth= 4, 
                             min_child_weight=11, n_estimators= 500, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
y_pred=xgclassifier.predict(X_test)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
from sklearn.model_selection import KFold,cross_val_predict,cross_val_score,cross_validate
X=dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']]
y=dataset[['U_class']]
svc_acc_maminasata_=np.mean(cross_val_score(svclassifier, X, y, cv=10,scoring='roc_auc'))
rbf_acc_maminasata_=np.mean(cross_val_score(rfclassifier, X, y, cv=10,scoring='roc_auc'))
xgb_acc_maminasata_=np.mean(cross_val_score(xgclassifier, X, y, cv=10,scoring='roc_auc'))

## Bandung
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_bandungraya.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)

svclassifier = SVC(kernel='rbf',C=100,random_state=123)
svclassifier.fit(X_train, y_train)
y_pred = svclassifier.predict(X_test)
from sklearn.metrics import classification_report, confusion_matrix
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,n_estimators=500,criterion='entropy')
rfclassifier.fit(X_train,y_train)
y_pred=rfclassifier.predict(X_test)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, 
                                 learning_rate=0.05, max_depth= 4, 
                             min_child_weight=11, n_estimators= 500, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
y_pred=xgclassifier.predict(X_test)
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
from sklearn.model_selection import KFold,cross_val_predict,cross_val_score,cross_validate
X=dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']]
y=dataset[['U_class']]
svc_acc_bandungraya_=np.mean(cross_val_score(svclassifier, X, y, cv=10,scoring='roc_auc'))
rbf_acc_bandungraya_=np.mean(cross_val_score(rfclassifier, X, y, cv=10,scoring='roc_auc'))
xgb_acc_bandungraya_=np.mean(cross_val_score(xgclassifier, X, y, cv=10,scoring='roc_auc'))

print(xgb_acc_jabodetabek)
print(xgb_acc_gerbang_)
print(xgb_acc_mebidangro_)
print(xgb_acc_kedungsepur_)
print(xgb_acc_maminasata_)
print(xgb_acc_bandungraya_)
print(svc_acc_jabodetabek)
print(svc_acc_gerbang_)
print(svc_acc_mebidangro_)
print(svc_acc_kedungsepur_)
print(svc_acc_maminasata_)
print(svc_acc_bandungraya_)
print(rbf_acc_jabodetabek)
print(rbf_acc_gerbang_)
print(rbf_acc_mebidangro_)
print(rbf_acc_kedungsepur_)
print(rbf_acc_maminasata_)
print(rbf_acc_bandungraya_)
!pip install statsmodels
from statsmodels.stats.contingency_tables import mcnemar
#mcnemar jabodetabek
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_jabodetabek.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
svclassifier = SVC(kernel='rbf',C=100,random_state=123)
svclassifier.fit(X_train, y_train)
y_pred_svc = svclassifier.predict(X_test)
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,n_estimators=500,criterion='entropy')
rfclassifier.fit(X_train,y_train)
y_pred_rf=rfclassifier.predict(X_test)
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, learning_rate=0.07, max_depth= 4, 
                             min_child_weight=11, n_estimators= 500, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
y_pred_xg=xgclassifier.predict(X_test)
d={'test':y_test.U_class.to_numpy(),'svc':y_pred_svc,'rf':y_pred_rf,'xg':y_pred_xg}
df=pd.DataFrame(d)
#svc_rf:
y_y=df.loc[(df.svc==df.test)&(df.rf==df.test)].shape[0]
y_n=df.loc[(df.svc!=df.test)&(df.rf==df.test)].shape[0]
n_y=df.loc[(df.svc==df.test)&(df.rf!=df.test)].shape[0]
n_n=df.loc[(df.svc!=df.test)&(df.rf!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_svc_rf=mcnemar(tab, exact=False, correction=True)
#svc_xg:
y_y=df.loc[(df.svc==df.test)&(df.xg==df.test)].shape[0]
y_n=df.loc[(df.svc!=df.test)&(df.xg==df.test)].shape[0]
n_y=df.loc[(df.svc==df.test)&(df.xg!=df.test)].shape[0]
n_n=df.loc[(df.svc!=df.test)&(df.xg!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_svc_xg=mcnemar(tab, exact=False, correction=True)
#xg_rf:
y_y=df.loc[(df.xg==df.test)&(df.rf==df.test)].shape[0]
y_n=df.loc[(df.xg!=df.test)&(df.rf==df.test)].shape[0]
n_y=df.loc[(df.xg==df.test)&(df.rf!=df.test)].shape[0]
n_n=df.loc[(df.xg!=df.test)&(df.rf!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_xg_rf=mcnemar(tab, exact=False, correction=True)
mc_nemar_jabodetabek=[res_svc_rf.pvalue,res_svc_xg.pvalue,res_xg_rf.pvalue]
#mcnemar mebidangro
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_mebidangro.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
svclassifier = SVC(kernel='rbf',C=100,random_state=123)
svclassifier.fit(X_train, y_train)
y_pred_svc = svclassifier.predict(X_test)
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,n_estimators=200,criterion='entropy')
rfclassifier.fit(X_train,y_train)
y_pred_rf=rfclassifier.predict(X_test)
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, learning_rate=0.07, max_depth= 4, 
                             min_child_weight=11, n_estimators= 200, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
y_pred_xg=xgclassifier.predict(X_test)
d={'test':y_test.U_class.to_numpy(),'svc':y_pred_svc,'rf':y_pred_rf,'xg':y_pred_xg}
df=pd.DataFrame(d)
#svc_rf:
y_y=df.loc[(df.svc==df.test)&(df.rf==df.test)].shape[0]
y_n=df.loc[(df.svc!=df.test)&(df.rf==df.test)].shape[0]
n_y=df.loc[(df.svc==df.test)&(df.rf!=df.test)].shape[0]
n_n=df.loc[(df.svc!=df.test)&(df.rf!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_svc_rf=mcnemar(tab, exact=False, correction=True)
#svc_xg:
y_y=df.loc[(df.svc==df.test)&(df.xg==df.test)].shape[0]
y_n=df.loc[(df.svc!=df.test)&(df.xg==df.test)].shape[0]
n_y=df.loc[(df.svc==df.test)&(df.xg!=df.test)].shape[0]
n_n=df.loc[(df.svc!=df.test)&(df.xg!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_svc_xg=mcnemar(tab, exact=False, correction=True)
#xg_rf:
y_y=df.loc[(df.xg==df.test)&(df.rf==df.test)].shape[0]
y_n=df.loc[(df.xg!=df.test)&(df.rf==df.test)].shape[0]
n_y=df.loc[(df.xg==df.test)&(df.rf!=df.test)].shape[0]
n_n=df.loc[(df.xg!=df.test)&(df.rf!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_xg_rf=mcnemar(tab, exact=False, correction=True)
mc_nemar_mebidangro=[res_svc_rf.pvalue,res_svc_xg.pvalue,res_xg_rf.pvalue]
#mcnemar gerbangkeratsusilo
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_gerbangkertasusila.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
svclassifier = SVC(kernel='rbf',C=100,random_state=123)
svclassifier.fit(X_train, y_train)
y_pred_svc = svclassifier.predict(X_test)
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,n_estimators=200,criterion='entropy')
rfclassifier.fit(X_train,y_train)
y_pred_rf=rfclassifier.predict(X_test)
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, learning_rate=0.07, max_depth= 4, 
                             min_child_weight=11, n_estimators= 100, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
y_pred_xg=xgclassifier.predict(X_test)
d={'test':y_test.U_class.to_numpy(),'svc':y_pred_svc,'rf':y_pred_rf,'xg':y_pred_xg}
df=pd.DataFrame(d)
#svc_rf:
y_y=df.loc[(df.svc==df.test)&(df.rf==df.test)].shape[0]
y_n=df.loc[(df.svc!=df.test)&(df.rf==df.test)].shape[0]
n_y=df.loc[(df.svc==df.test)&(df.rf!=df.test)].shape[0]
n_n=df.loc[(df.svc!=df.test)&(df.rf!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_svc_rf=mcnemar(tab, exact=False, correction=True)
#svc_xg:
y_y=df.loc[(df.svc==df.test)&(df.xg==df.test)].shape[0]
y_n=df.loc[(df.svc!=df.test)&(df.xg==df.test)].shape[0]
n_y=df.loc[(df.svc==df.test)&(df.xg!=df.test)].shape[0]
n_n=df.loc[(df.svc!=df.test)&(df.xg!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_svc_xg=mcnemar(tab, exact=False, correction=True)
#xg_rf:
y_y=df.loc[(df.xg==df.test)&(df.rf==df.test)].shape[0]
y_n=df.loc[(df.xg!=df.test)&(df.rf==df.test)].shape[0]
n_y=df.loc[(df.xg==df.test)&(df.rf!=df.test)].shape[0]
n_n=df.loc[(df.xg!=df.test)&(df.rf!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_xg_rf=mcnemar(tab, exact=False, correction=True)
mc_nemar_gerbangkertasusila=[res_svc_rf.pvalue,res_svc_xg.pvalue,res_xg_rf.pvalue]
#mcnemar kedungsepur
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_kedungsepur.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
svclassifier = SVC(kernel='rbf',C=100,random_state=123)
svclassifier.fit(X_train, y_train)
y_pred_svc = svclassifier.predict(X_test)
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,n_estimators=100,criterion='entropy')
rfclassifier.fit(X_train,y_train)
y_pred_rf=rfclassifier.predict(X_test)
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, learning_rate=0.07, max_depth= 4, 
                             min_child_weight=11, n_estimators= 500, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
y_pred_xg=xgclassifier.predict(X_test)
d={'test':y_test.U_class.to_numpy(),'svc':y_pred_svc,'rf':y_pred_rf,'xg':y_pred_xg}
df=pd.DataFrame(d)
#svc_rf:
y_y=df.loc[(df.svc==df.test)&(df.rf==df.test)].shape[0]
y_n=df.loc[(df.svc!=df.test)&(df.rf==df.test)].shape[0]
n_y=df.loc[(df.svc==df.test)&(df.rf!=df.test)].shape[0]
n_n=df.loc[(df.svc!=df.test)&(df.rf!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_svc_rf=mcnemar(tab, exact=False, correction=True)
#svc_xg:
y_y=df.loc[(df.svc==df.test)&(df.xg==df.test)].shape[0]
y_n=df.loc[(df.svc!=df.test)&(df.xg==df.test)].shape[0]
n_y=df.loc[(df.svc==df.test)&(df.xg!=df.test)].shape[0]
n_n=df.loc[(df.svc!=df.test)&(df.xg!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_svc_xg=mcnemar(tab, exact=False, correction=True)
#xg_rf:
y_y=df.loc[(df.xg==df.test)&(df.rf==df.test)].shape[0]
y_n=df.loc[(df.xg!=df.test)&(df.rf==df.test)].shape[0]
n_y=df.loc[(df.xg==df.test)&(df.rf!=df.test)].shape[0]
n_n=df.loc[(df.xg!=df.test)&(df.rf!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_xg_rf=mcnemar(tab, exact=False, correction=True)
mc_nemar_kedungsepur=[res_svc_rf.pvalue,res_svc_xg.pvalue,res_xg_rf.pvalue]
#mcnemar maminasata
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_maminasata.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
svclassifier = SVC(kernel='rbf',C=100,random_state=123)
svclassifier.fit(X_train, y_train)
y_pred_svc = svclassifier.predict(X_test)
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,n_estimators=500,criterion='entropy')
rfclassifier.fit(X_train,y_train)
y_pred_rf=rfclassifier.predict(X_test)
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, learning_rate=0.07, max_depth= 4, 
                             min_child_weight=11, n_estimators= 500, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
y_pred_xg=xgclassifier.predict(X_test)
d={'test':y_test.U_class.to_numpy(),'svc':y_pred_svc,'rf':y_pred_rf,'xg':y_pred_xg}
df=pd.DataFrame(d)
#svc_rf:
y_y=df.loc[(df.svc==df.test)&(df.rf==df.test)].shape[0]
y_n=df.loc[(df.svc!=df.test)&(df.rf==df.test)].shape[0]
n_y=df.loc[(df.svc==df.test)&(df.rf!=df.test)].shape[0]
n_n=df.loc[(df.svc!=df.test)&(df.rf!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_svc_rf=mcnemar(tab, exact=False, correction=True)
#svc_xg:
y_y=df.loc[(df.svc==df.test)&(df.xg==df.test)].shape[0]
y_n=df.loc[(df.svc!=df.test)&(df.xg==df.test)].shape[0]
n_y=df.loc[(df.svc==df.test)&(df.xg!=df.test)].shape[0]
n_n=df.loc[(df.svc!=df.test)&(df.xg!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_svc_xg=mcnemar(tab, exact=False, correction=True)
#xg_rf:
y_y=df.loc[(df.xg==df.test)&(df.rf==df.test)].shape[0]
y_n=df.loc[(df.xg!=df.test)&(df.rf==df.test)].shape[0]
n_y=df.loc[(df.xg==df.test)&(df.rf!=df.test)].shape[0]
n_n=df.loc[(df.xg!=df.test)&(df.rf!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_xg_rf=mcnemar(tab, exact=False, correction=True)
mc_nemar_maminasata=[res_svc_rf.pvalue,res_svc_xg.pvalue,res_xg_rf.pvalue]
#mcnemar bandungraya
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_bandungraya.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
svclassifier = SVC(kernel='rbf',C=100,random_state=123)
svclassifier.fit(X_train, y_train)
y_pred_svc = svclassifier.predict(X_test)
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,n_estimators=500,criterion='entropy')
rfclassifier.fit(X_train,y_train)
y_pred_rf=rfclassifier.predict(X_test)
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, learning_rate=0.07, max_depth= 4, 
                             min_child_weight=11, n_estimators= 100, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
y_pred_xg=xgclassifier.predict(X_test)
d={'test':y_test.U_class.to_numpy(),'svc':y_pred_svc,'rf':y_pred_rf,'xg':y_pred_xg}
df=pd.DataFrame(d)
#svc_rf:
y_y=df.loc[(df.svc==df.test)&(df.rf==df.test)].shape[0]
y_n=df.loc[(df.svc!=df.test)&(df.rf==df.test)].shape[0]
n_y=df.loc[(df.svc==df.test)&(df.rf!=df.test)].shape[0]
n_n=df.loc[(df.svc!=df.test)&(df.rf!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_svc_rf=mcnemar(tab, exact=False, correction=True)
#svc_xg:
y_y=df.loc[(df.svc==df.test)&(df.xg==df.test)].shape[0]
y_n=df.loc[(df.svc!=df.test)&(df.xg==df.test)].shape[0]
n_y=df.loc[(df.svc==df.test)&(df.xg!=df.test)].shape[0]
n_n=df.loc[(df.svc!=df.test)&(df.xg!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_svc_xg=mcnemar(tab, exact=False, correction=True)
#xg_rf:
y_y=df.loc[(df.xg==df.test)&(df.rf==df.test)].shape[0]
y_n=df.loc[(df.xg!=df.test)&(df.rf==df.test)].shape[0]
n_y=df.loc[(df.xg==df.test)&(df.rf!=df.test)].shape[0]
n_n=df.loc[(df.xg!=df.test)&(df.rf!=df.test)].shape[0]
tab=[[y_y,y_n],[n_y,n_n]]
res_xg_rf=mcnemar(tab, exact=False, correction=True)
mc_nemar_bandungraya=[res_svc_rf.pvalue,res_svc_xg.pvalue,res_xg_rf.pvalue]
lokus=['jabodetabek','mebidangro','gerbangkertasusila','kedungsepur','maminasata','bandungraya']
mc_nemar_jabodetabek
mc_nemar_mebidangro
mc_nemar_gerbangkertasusila
mc_nemar_kedungsepur
mc_nemar_maminasata
mc_nemar_bandungraya
sorted_idx = np.argsort(xgclassifier.feature_importances_)[::-1]
for index in sorted_idx:
    print([['rad_VIIRS','SAVI','MNDWI','NDVI','NDBI','LSE'][index], xgclassifier.feature_importances_[index]]) 
#feature importance jabodetabek
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_jabodetabek.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, learning_rate=0.07, max_depth= 4, 
                             min_child_weight=11, n_estimators= 500, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
sorted_idx = np.argsort(xgclassifier.feature_importances_)[::-1]
for index in sorted_idx:
    print([['rad_VIIRS','SAVI','MNDWI','NDVI','NDBI','LSE'][index], xgclassifier.feature_importances_[index]]) 
#feature importance gerbangkertasusila
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_gerbangkertasusila.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, learning_rate=0.07, max_depth= 6, 
                             min_child_weight=11, n_estimators= 100, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
sorted_idx = np.argsort(xgclassifier.feature_importances_)[::-1]
for index in sorted_idx:
    print([['rad_VIIRS','SAVI','MNDWI','NDVI','NDBI','LSE'][index], xgclassifier.feature_importances_[index]]) 
#feature importance mebidangro
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_mebidangro.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, learning_rate=0.07, max_depth= 4, 
                             min_child_weight=11, n_estimators= 200, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
sorted_idx = np.argsort(xgclassifier.feature_importances_)[::-1]
for index in sorted_idx:
    print([['rad_VIIRS','SAVI','MNDWI','NDVI','NDBI','LSE'][index], xgclassifier.feature_importances_[index]]) 
#feature importance kedungsepur
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_kedungsepur.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, learning_rate=0.07, max_depth= 4, 
                             min_child_weight=11, n_estimators= 500, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
sorted_idx = np.argsort(xgclassifier.feature_importances_)[::-1]
for index in sorted_idx:
    print([['rad_VIIRS','SAVI','MNDWI','NDVI','NDBI','LSE'][index], xgclassifier.feature_importances_[index]]) 
#feature importance mc_nemar_maminasata
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_maminasata.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
xgclassifier = xgb.XGBClassifier(random_state=123,colsample_bytree= 0.7, learning_rate=0.07, max_depth= 4, 
                             min_child_weight=11, n_estimators= 500, nthread= 4, objective= 'binary:logistic', 
                             seed= 123, silent=1, subsample= 0.8)
xgclassifier.fit(X_train,y_train)
sorted_idx = np.argsort(xgclassifier.feature_importances_)[::-1]
for index in sorted_idx:
    print([['rad_VIIRS','SAVI','MNDWI','NDVI','NDBI','LSE'][index], xgclassifier.feature_importances_[index]]) 
#feature importance bandung
from sklearn.metrics import classification_report, confusion_matrix
dataset = rasterio.open('/content/gdrive/My Drive/Urban_monitoring/Urban_/compiled_GHSL_30_train_2014_bandungraya.tif')
dataset_bands=pd.DataFrame()
for i in dataset.indexes:
  temp=dataset.read(i)
  temp=pd.DataFrame(data=np.array(temp).flatten()).rename(columns={0:i})
  dataset_bands=temp.join(dataset_bands)
dataset_bands.rename(columns={1:'BU_class',2:'NDVI_landsat7',3:'NDBI_landsat7',4:'MNDWI_landsat7',
                            5:'SAVI_landsat7',6:'LSE_landsat7',7:'rad_VIIRS'},inplace=True)
dataset_bands['U_class']=dataset_bands.BU_class.apply(lambda y: 1 if y>=3 else 0)
dataset_bands['LSE_der_landsat7']=dataset_bands.NDVI_landsat7.apply(lambda y: 0.995 if y < -0.185 else (
    0.970 if y< 0.157 else (1.0098+0.047*np.log(y) if y<0.727 else 0.990)))
dataset=dataset_bands.query('U_class==0').sample(10000).append(dataset_bands.query('U_class==1').sample(10000))
minmax_=preprocessing.MinMaxScaler()
dataset=dataset.reset_index()[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDBI_landsat7','NDVI_landsat7','LSE_der_landsat7','U_class']]  
X_train, X_test, y_train, y_test = train_test_split(dataset[['rad_VIIRS','SAVI_landsat7','MNDWI_landsat7','NDVI_landsat7','NDBI_landsat7','LSE_der_landsat7']],
                                                    dataset[['U_class']], test_size = 0.20)
X_train=minmax_.fit_transform(X_train)
X_test=minmax_.fit_transform(X_test)
rfclassifier = RandomForestClassifier(random_state=123,max_depth=6,n_estimators=200,criterion='entropy')
rfclassifier.fit(X_train,y_train)
sorted_idx = np.argsort(rfclassifier.feature_importances_)[::-1]
for index in sorted_idx:
    print([['rad_VIIRS','SAVI','MNDWI','NDVI','NDBI','LSE'][index], rfclassifier.feature_importances_[index]])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16131147540983606, 'fraction_numerical': 0.04366858661940629, 'mean_word_length': 6.664493344199946, 'pattern_counts': {'":': 0, '<': 68, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '234669', 'license': 'MIT', 'n_tokens_mistral': 24138, 'n_tokens_neox': 22720, 'n_words': 2302, 'path': '01_ICST2020/02_UrbanClassification_model_checking.ipynb', 'repo_name': 'achmfirmansyah/sweet_project', 'size': '56425'}