<jupyter_start><jupyter_text>Preparação<jupyter_code>from google.colab import drive

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from IPython import display

from sklearn.preprocessing import MinMaxScaler
from keras.utils import to_categorical
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split

import tensorflow
from tensorflow.python.keras.layers import Dense
from tensorflow.python.keras import Sequential

from time import time
drive.mount('/content/drive/')
cd 'drive/My Drive/RSDD/util-conservado-4imgs'<jupyter_output>/content/drive/.shortcut-targets-by-id/1fnUO42Y1JIVhpQTvYUVLczSGjkzFv64h/RSDD/util-conservado-4imgs<jupyter_text>Carregamento dos dados<jupyter_code>points = pd.read_csv("points.csv")
points.describe()
bands = pd.read_csv("bands.csv")
#bands.columns = ['PointId', 'Year', "Month", "Day", "Class", "Index", "Latitude", "Longitude", "Band1", "Band2", "Band3", "Band4", "Band5", "Band6", "Band7", "Band8", "Band8a", "Band9", "Band10", "Band11", "Band12"]
bands.describe()<jupyter_output><empty_output><jupyter_text>Cálculo dos índices<jupyter_code># NDVI = (band8 - band4) / (band8 + band4)
#bands['ndvi'] = bands.apply(lambda row: (row.Band8 - row.Band4) / (row.Band8 + row.Band4), axis=1)

# SAVI = (band8 - band4) / (band8 + band4 + 0.5) * 1.5
#bands['savi'] = bands.apply(lambda row: (row.Band8 - row.Band4) / (row.Band8 + row.Band4 + 0.5) * 1.5, axis=1)

# "NDVI-GREEN": "$B03 * ($B08 - $B04) / ($B08 + $B04)"
bands['ndvi-green'] = bands.apply(lambda row: row.Band3 * (row.Band8 - row.Band4 ) / (row.Band8 + row.Band4), axis=1)

# "PSSRa": "$B08 / $B04"
bands['pssra'] = bands.apply(lambda row: row.Band8 / row.Band4, axis=1)

# "PSSRc": "$B08 / $B02",
bands['pssrc'] = bands.apply(lambda row: row.Band8 / row.Band2, axis=1)

# "NDWI": "($B03 - $B08) / ($B03 + $B08)"
bands['ndwi'] = bands.apply(lambda row: (row.Band3 - row.Band8) / (row.Band3 + row.Band8), axis=1)

# "MNDWI": "($B03 - $B11) / ($B03 + $B11)"
bands['mndwi'] = bands.apply(lambda row: (row.Band3 - row.Band11) / (row.Band3 + row.Band11), axis=1)

# "MSI":  "$B11 / $B08"
bands['msi'] = bands.apply(lambda row: row.Band11 / row.Band8, axis=1)


bands.describe()<jupyter_output><empty_output><jupyter_text>Análise dos dados<jupyter_code># Análise
dadosParaAnalise = bands.groupby(['PointId', 'Month', 'Day'])["PointId", 'Month', 'Day', "Band1", "Band2", "Band3", "Band4", "Band5", "Band6", "Band7", "Band8", "Band8a", "Band9", "Band10", "Band11", "Band12", "ndvi", "savi", "ndvi-green", "pssra", "pssrc", "ndwi", "mndwi", "msi"].mean()
dadosParaAnalise
numeroDePontos = 1600
pD = 0
pC = 0
i = 0

ndvisDesmatado = np.empty([3, 800])
ndvisConservado = np.empty([3, 800])

while (i < 1600*4):

    # Obter bandas e PointID
    row1 = dadosParaAnalise.iloc[i]
    row2 = dadosParaAnalise.iloc[i+1]
    row3 = dadosParaAnalise.iloc[i+2]
    pointId = row1['PointId']

    # Consultar POINTS e obter classe
    pointRow = points.loc[points['PointId'] == pointId]
    classe = pointRow['Class'].iloc[0]

    if classe == 2:
      ndvisDesmatado[0, pD] = row1["ndvi"]
      ndvisDesmatado[1, pD] = row2["ndvi"]
      ndvisDesmatado[2, pD] = row3["ndvi"]
      pD = pD + 1
    else:
      ndvisConservado[0, pC] = row1["ndvi"]
      ndvisConservado[1, pC] = row2["ndvi"]
      ndvisConservado[2, pC] = row3["ndvi"]
      pC = pC + 1

    i = i + 4

plt.hist(ndvisConservado[0], bins = np.arange(0, 0.8, 0.05)) 
plt.title("histogram conservado 1ª imagem") 
plt.show()

plt.hist(ndvisDesmatado[0], bins = np.arange(0, 0.8, 0.05)) 
plt.title("histogram desmatado 1ª imagem") 
plt.show()

plt.hist(ndvisConservado[1], bins = np.arange(0, 0.8, 0.05)) 
plt.title("histogram conservado 2ª imagem") 
plt.show()

plt.hist(ndvisDesmatado[1], bins = np.arange(0, 0.8, 0.05)) 
plt.title("histogram desmatado 2ª imagem") 
plt.show()

plt.hist(ndvisConservado[2], bins = np.arange(0, 0.8, 0.05)) 
plt.title("histogram conservado 3ª imagem") 
plt.show()

plt.hist(ndvisDesmatado[2], bins = np.arange(0, 0.8, 0.05)) 
plt.title("histogram desmatado 3ª imagem") 
plt.show()<jupyter_output><empty_output><jupyter_text>Preparação do conjunto de dados<jupyter_code># Agrupamento dos dados de um mesmo ponto
dados = bands.groupby(['PointId', 'Month', 'Day'])["PointId", "Band1", "Band2", "Band3", "Band4", "Band5", "Band6", "Band7", "Band8", "Band8a", "Band9", "Band10", "Band11", "Band12", "ndvi", "savi", "ndvi-green", "pssra", "pssrc", "ndwi", "mndwi", "msi"].mean()
dados
# Separação do conjunto de dados

# Combinações
## 1 - 3 imagens de entrada, todos os atributos
## 2 - 3 imagens de entrada, apenas as bandas
## 3 - 3 imagens de entrada, apenas os índices
## 4 - 2 imagens de entrada, todos os atributos
## 5 - 2 imagens de entrada, apenas as bandas
## 6 - 2 imagens de entrada, apenas os índices

def getXandY (combinacao):

  numeroDePontos = 1600
  bandas = 13
  indices = 8

  if combinacao == 1:
    numeroEntradas = 3 * (bandas + indices)
  elif combinacao == 2:
    numeroEntradas = 3 * (bandas)
  elif combinacao == 3:
    numeroEntradas = 3 * (indices)
  elif combinacao == 4:
    numeroEntradas = 2 * (bandas + indices)
  elif combinacao == 5:
    numeroEntradas = 2 * (bandas)
  elif combinacao == 6:
    numeroEntradas = 2 * (indices)
  else:
    raise Exception("Código de combinação inválido")


  x = np.empty([numeroDePontos, numeroEntradas])
  y = np.empty([numeroDePontos, 2])

  idx = 0
  pontos = 0

  # Passar por cada linha de DADOS
  while (pontos < numeroDePontos):

    # Obter bandas e PointID
    row1 = dados.iloc[idx]
    row2 = dados.iloc[idx+1]
    row3 = dados.iloc[idx+2]
    pointId = row1['PointId']

    # Consultar POINTS e obter classe
    pointRow = points.loc[points['PointId'] == pointId]
    classe = pointRow['Class'].iloc[0]
    
    # Definir X baseado na combinacao
    if combinacao == 1:
      x[pontos] = [row1["Band1"], row1["Band2"], row1["Band3"], row1["Band4"], row1["Band5"], row1["Band6"], row1["Band7"], row1["Band8"], row1["Band8a"], row1["Band9"], row1["Band10"], row1["Band11"], row1["Band12"], row1["ndvi"], row1["savi"], row1["ndvi-green"], row1["pssra"], row1["pssrc"], row1["ndwi"], row1["mndwi"], row1["msi"],
                  row2["Band1"], row2["Band2"], row2["Band3"], row2["Band4"], row2["Band5"], row2["Band6"], row2["Band7"], row2["Band8"], row2["Band8a"], row2["Band9"], row2["Band10"], row2["Band11"], row2["Band12"], row2["ndvi"], row2["savi"], row2["ndvi-green"], row2["pssra"], row2["pssrc"], row2["ndwi"], row2["mndwi"], row2["msi"],
                  row3["Band1"], row3["Band2"], row3["Band3"], row3["Band4"], row3["Band5"], row3["Band6"], row3["Band7"], row3["Band8"], row3["Band8a"], row3["Band9"], row3["Band10"], row3["Band11"], row3["Band12"], row3["ndvi"], row3["savi"], row3["ndvi-green"], row3["pssra"], row3["pssrc"], row3["ndwi"], row3["mndwi"], row3["msi"] ]
    elif combinacao == 2:
      x[pontos] = [row1["Band1"], row1["Band2"], row1["Band3"], row1["Band4"], row1["Band5"], row1["Band6"], row1["Band7"], row1["Band8"], row1["Band8a"], row1["Band9"], row1["Band10"], row1["Band11"], row1["Band12"],
                  row2["Band1"], row2["Band2"], row2["Band3"], row2["Band4"], row2["Band5"], row2["Band6"], row2["Band7"], row2["Band8"], row2["Band8a"], row2["Band9"], row2["Band10"], row2["Band11"], row2["Band12"],
                  row3["Band1"], row3["Band2"], row3["Band3"], row3["Band4"], row3["Band5"], row3["Band6"], row3["Band7"], row3["Band8"], row3["Band8a"], row3["Band9"], row3["Band10"], row3["Band11"], row3["Band12"] ]
    elif combinacao == 3:
      x[pontos] = [row1["ndvi"], row1["savi"], row1["ndvi-green"], row1["pssra"], row1["pssrc"], row1["ndwi"], row1["mndwi"], row1["msi"],
                  row2["ndvi"], row2["savi"], row2["ndvi-green"], row2["pssra"], row2["pssrc"], row2["ndwi"], row2["mndwi"], row2["msi"],
                  row3["ndvi"], row3["savi"], row3["ndvi-green"], row3["pssra"], row3["pssrc"], row3["ndwi"], row3["mndwi"], row3["msi"] ]
    elif combinacao == 4:
      x[pontos] = [row1["Band1"], row1["Band2"], row1["Band3"], row1["Band4"], row1["Band5"], row1["Band6"], row1["Band7"], row1["Band8"], row1["Band8a"], row1["Band9"], row1["Band10"], row1["Band11"], row1["Band12"], row1["ndvi"], row1["savi"], row1["ndvi-green"], row1["pssra"], row1["pssrc"], row1["ndwi"], row1["mndwi"], row1["msi"],
                  row2["Band1"], row2["Band2"], row2["Band3"], row2["Band4"], row2["Band5"], row2["Band6"], row2["Band7"], row2["Band8"], row2["Band8a"], row2["Band9"], row2["Band10"], row2["Band11"], row2["Band12"], row2["ndvi"], row2["savi"], row2["ndvi-green"], row2["pssra"], row2["pssrc"], row2["ndwi"], row2["mndwi"], row2["msi"] ]
    elif combinacao == 5:
      x[pontos] = [row1["Band1"], row1["Band2"], row1["Band3"], row1["Band4"], row1["Band5"], row1["Band6"], row1["Band7"], row1["Band8"], row1["Band8a"], row1["Band9"], row1["Band10"], row1["Band11"], row1["Band12"],
                  row2["Band1"], row2["Band2"], row2["Band3"], row2["Band4"], row2["Band5"], row2["Band6"], row2["Band7"], row2["Band8"], row2["Band8a"], row2["Band9"], row2["Band10"], row2["Band11"], row2["Band12"] ]
    elif combinacao == 6:
      x[pontos] = [row1["ndvi"], row1["savi"], row1["ndvi-green"], row1["pssra"], row1["pssrc"], row1["ndwi"], row1["mndwi"], row1["msi"],
                  row2["ndvi"], row2["savi"], row2["ndvi-green"], row2["pssra"], row2["pssrc"], row2["ndwi"], row2["mndwi"], row2["msi"] ]
    else:
      raise Exception("Código de combinação inválido")
    

    # Definir Y
    if classe == 0:
      # conservado
      y[pontos] = [0, 1]
    else:
      # desmatado
      y[pontos] = [1, 0]


    idx = idx + 4
    pontos = pontos + 1

  return x, y<jupyter_output><empty_output><jupyter_text>Separação entre treinamento e teste<jupyter_code># Separação entre treinamento e teste
def getTreinamentoAndTeste (combinacao):
  x, y = getXandY(combinacao)

  # Normalização
  scX = MinMaxScaler()
  x = scX.fit_transform(x)

  # Separação
  treinoX, testeX, treinoY, testeY = train_test_split(x, y, test_size=0.3, random_state=27, shuffle=True)

  return treinoX, testeX, treinoY, testeY<jupyter_output><empty_output><jupyter_text>Rede neural  Modelo<jupyter_code>def createAnnModel(numeroEntradas, activations, optimizer):
  units = numeroEntradas*2+1

  model = Sequential()
  model.add(Dense(units, input_dim=numeroEntradas, kernel_initializer='normal', activation=activations[0]))
  model.add(Dense(2, activation='softmax'))
  model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
  
  return model<jupyter_output><empty_output><jupyter_text>Treinamento<jupyter_code>def treinamento (treinoX, treinoY):

  numeroEntradas = treinoX.shape[1]

  tempoInicio = time()

  model = createAnnModel(numeroEntradas, ['relu','sigmoid'], 'Adam')
  history = model.fit(treinoX, treinoY, validation_split=0.25, epochs=200, shuffle=True)

  tempo = time() - tempoInicio

  return model, history, tempo<jupyter_output><empty_output><jupyter_text>Análise dos resultados<jupyter_code>def plotRestult (history):
  plt.plot(history.history['accuracy'])
  plt.plot(history.history['val_accuracy'])

  plt.title('model accuracy')
  plt.ylabel('accuracy')
  plt.xlabel('epoch')
  plt.legend(['train', 'test'], loc='upper left')
  plt.show()
def ConfusionMatrix (model, testeX, testeY):
  predY = model.predict(testeX)
  matrix = confusion_matrix(testeY.argmax(axis=1), predY.argmax(axis=1))

  print("Matriz confusão:")
  print(matrix)

  tn = matrix[0,0]
  fp = matrix[0,1]
  fn = matrix[1,0]
  tp = matrix[1,1]

  return tn, fp, fn, tp<jupyter_output><empty_output><jupyter_text>Execução<jupyter_code>numCombinacoes = 6
results = np.empty([numCombinacoes, 8])

for i in range(numCombinacoes):
  combinacao = i + 1
  
  treinoX, testeX, treinoY, testeY = getTreinamentoAndTeste(combinacao)
  model, history, tempo = treinamento(treinoX, treinoY)

  plotRestult(history)

  tn, fp, fn, tp = ConfusionMatrix(model, testeX, testeY)
  
  precision = tp / (tp+fp)
  accuracy = (tp+tn) / (tp+tn+fp+fn)

  results[i] = [combinacao, tn, fp, fn, tp, precision, accuracy, tempo]
print ("#"
    + "\t" + "TN"
    + "\t" + "FP"
    + "\t" + "FN"
    + "\t" + "TP"
    + "\t" + "Precision(%)"
    + "\t" + "Accuracy (%)"
    + "\t" + "Tempo (s)" )

for i in range(numCombinacoes):
  combinacao, tn, fp, fn, tp, precision, accuracy, tempo = results[i]
  print (str(combinacao)
      + "\t" + str(tn)
      + "\t" + str(fp)
      + "\t" + str(fn)
      + "\t" + str(tp)
      + "\t" + str(round(precision*100,2))
      + "\t" + str(round(accuracy*100,2))
      + "\t" + str(round(tempo, 2)) )<jupyter_output>#	TN	FP	FN	TP	Precision(%)	Accuracy (%)	Tempo (s)
1.0	236.0	15.0	12.0	217.0	93.53	94.38	13.65
2.0	238.0	13.0	19.0	210.0	94.17	93.33	12.52
3.0	241.0	10.0	24.0	205.0	95.35	92.92	12.48
4.0	225.0	26.0	12.0	217.0	89.3	92.08	13.59
5.0	231.0	20.0	20.0	209.0	91.27	91.67	12.7
6.0	228.0	23.0	12.0	217.0	90.42	92.71	12.06
{'fraction_non_alphanumeric': 0.20410358871897333, 'fraction_numerical': 0.066548835779605, 'mean_word_length': 4.678010471204188, 'pattern_counts': {'":': 6, '<': 41, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '656977', 'license': 'MIT', 'n_tokens_mistral': 5687, 'n_tokens_neox': 5361, 'n_words': 1065, 'path': 'notebooks/ann.ipynb', 'repo_name': 'demusis/srdesmatamentos', 'size': '13013'}