<jupyter_start><jupyter_text>Installation des librairies<jupyter_code>!pip install -q -q -q tifffile
!pip install -q -q -q tqdm
import os
#quelque-chose
os.chdir('/tf/work/challenge-donnees-sat')<jupyter_output><empty_output><jupyter_text>Lecture des librairies et framework<jupyter_code>%load_ext autoreload
%autoreload 2

import s3fs
import shutil

from pathlib import Path
import random

import numpy as np
from tifffile import TiffFile
import matplotlib.pyplot as plt
import colorsys
import PIL
from PIL import Image
from tqdm import tqdm
import matplotlib.patches as mpatches
import time

from framework.dataset import LandCoverData as LCD<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>Téléchargement + lecture des images de la BDD<jupyter_code>fs = s3fs.S3FileSystem(client_kwargs={'endpoint_url': 'https://minio.lab.sspcloud.fr'})
fs.get('projet-challengedata-ens/data/additional_files_earthcube_emu4zqr.zip', 'additional_files_earthcube_emu4zqr.zip')
shutil.unpack_archive('additional_files_earthcube_emu4zqr.zip')<jupyter_output><empty_output><jupyter_text>Trainning et Test<jupyter_code>DATA_FOLDER_STR = 'dataset'
DATA_FOLDER = Path(DATA_FOLDER_STR).expanduser()
# path to the unzipped dataset: contains directories train/ and test/
DATASET_FOLDER = DATA_FOLDER

# get all train images and masks
train_images_paths = sorted(list(DATASET_FOLDER.glob('train/images/*.tif')))
train_masks_paths = sorted(list(DATASET_FOLDER.glob('train/masks/*.tif')))
# get all test images
test_images_paths = sorted(list(DATASET_FOLDER.glob('test/images/*.tif')))<jupyter_output><empty_output><jupyter_text>Visualize data points<jupyter_code>def test(image, mask):
    """Calcule des pixels
    Args : 
        image (numpy.array[uint8])
        mask (numpy.array[uint8]): the mask in 8-bit
        
    """
    colors_mask = np.asarray(LCD.CLASSES_COLORPALETTE.items())
    
    colimg= np.array(image)
    colmask = np.array(mask)
    colmask_test = np.array(mask)
    count = 0
    
    
    a = 4
    no_data = np.empty((0,a))
    clouds = np.empty((0,a))
    artificial = np.empty((0,a))
    cultivated = np.empty((0,a))
    broadleaf = np.empty((0,a))
    coniferous = np.empty((0,a))
    herbaceous = np.empty((0,a))
    natural = np.empty((0,a))
    snow = np.empty((0,a))
    water = np.empty((0,a))
    axisconca = 0
    
    for i in range (colmask.shape[0]):
        for j in range (colmask.shape[1]):            
            if colmask_test[i][j] == 0 :
                no_data = np.concatenate((no_data,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [0,0,0]
            if colmask_test[i][j] == 1 :
                clouds  = np.concatenate((clouds,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [255,25,236]
            if colmask_test[i][j] == 2 :
                artificial  = np.concatenate((artificial,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [215,25,28]
            if colmask_test[i][j] == 3 :
                cultivated  = np.concatenate((cultivated,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [211,154,92]
            if colmask_test[i][j] == 4 :
                test = colimg[i][j]
                broadleaf  = np.concatenate((broadleaf,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [33,115,55]
            if colmask_test[i][j] == 5 :
                coniferous  = np.concatenate((coniferous,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [21,75,35]
            if colmask_test[i][j] == 6 :
                herbaceous  = np.concatenate((herbaceous,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [118,209,93]
            if colmask_test[i][j] == 7 :
                natural  = np.concatenate((natural,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [130,130,130]
            if colmask_test[i][j] == 8 :
                snow  = np.concatenate((snow,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [255,255,255]
            if colmask_test[i][j] == 9 :
                water  = np.concatenate((water,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [43,61,255]
                
        count = count +1
    return colmask_test, no_data, clouds, artificial, cultivated, broadleaf, coniferous, herbaceous, natural, snow, water


#def moy_pix(no_data, clouds, artificial, cultivated, broadleaf, coniferous, herbaceous, natural, snow, water):


def test2(colimage):

    
    image= np.array(colimage)
    
    for i in range (image.shape[0]):
        for j in range (image.shape[1]):            
            if ((no_data_inf[0]<image[i][j][0]<no_data_inf[0])and(no_data_inf[1]<image[i][j][1]<no_data_inf[1])and(no_data_inf[2]<image[i][j][2]<no_data_inf[2])and(no_data_inf[3]<image[i][j][3]<no_data_inf[3])) :
                test_no_data +=1
            if ((clouds_inf[0]<image[i][j][0]<clouds_inf[0])and(clouds_inf[1]<image[i][j][1]<clouds_inf[1])and(clouds_inf[2]<image[i][j][2]<clouds_inf[2])and(clouds_inf[3]<image[i][j][3]<clouds_inf[3])) :
                test_clouds +=1
                #colmask_test[i][j] == [255,25,236]
            if ((artificial_inf[0]<image[i][j][0]<artificial_inf[0])and(artificial_inf[1]<image[i][j][1]<artificial_inf[1])and(artificial_inf[2]<image[i][j][2]<artificial_inf[2])and(artificial_inf[3]<image[i][j][3]<artificial_inf[3])) :
                test_artificial +=1
                #colmask_test[i][j] == [215,25,28]
            if ((cultivated_inf[0]<image[i][j][0]<cultivated_inf[0])and(cultivated_inf[1]<image[i][j][1]<cultivated_inf[1])and(cultivated_inf[2]<image[i][j][2]<cultivated_inf[2])and(cultivated_inf[3]<image[i][j][3]<cultivated_inf[3])) :
                test_cultivated  += 1
                #colmask_test[i][j] == [211,154,92]
            if ((broadleaf_inf[0]<image[i][j][0]<broadleaf_inf[0])and(broadleaf_inf[1]<image[i][j][1]<broadleaf_inf[1])and(broadleaf_inf[2]<image[i][j][2]<broadleaf_inf[2])and(broadleaf_inf[3]<image[i][j][3]<broadleaf_inf[3])) :
                
                test_broadleaf  += 1
                #colmask_test[i][j] == [33,115,55]
            if ((coniferous_inf[0]<image[i][j][0]<coniferous_inf[0])and(coniferous_inf[1]<image[i][j][1]<coniferous_inf[1])and(coniferous_inf[2]<image[i][j][2]<coniferous_inf[2])and(coniferous_inf[3]<image[i][j][3]<coniferous_inf[3])) :
                test_coniferous  += 1
                #colmask_test[i][j] == [21,75,35]
            if ((herbaceous_inf[0]<image[i][j][0]<herbaceous_inf[0])and(herbaceous_inf[1]<image[i][j][1]<herbaceous_inf[1])and(herbaceous_inf[2]<image[i][j][2]<herbaceous_inf[2])and(herbaceous[3]<image[i][j][3]<herbaceous_inf[3])) :
                test_herbaceous  += 1
                #colmask_test[i][j] == [118,209,93]
            if ((natural_inf[0]<image[i][j][0]<natural_inf[0])and(natural_inf[1]<image[i][j][1]<natural_inf[1])and(natural_inf[2]<image[i][j][2]<natural_inf[2])and(natural_inf[3]<image[i][j][3]<natural_inf[3])) :
                test_natural  += 1
                #colmask_test[i][j] == [130,130,130]
            if ((snow_inf[0]<image[i][j][0]<snow_inf[0])and(snow_inf[1]<image[i][j][1]<snow_inf[1])and(snow_inf[2]<image[i][j][2]<snow_inf[2])and(snow_inf[3]<image[i][j][3]<snow_inf[3])) :
                test_snow  += 1
                #colmask_test[i][j] == [255,255,255]
            if ((no_data_inf[0]<image[i][j][0]<no_data_inf[0])and(no_data_inf[1]<image[i][j][1]<no_data_inf[1])and(no_data_inf[2]<image[i][j][2]<no_data_inf[2])and(no_data_inf[3]<image[i][j][3]<no_data_inf[3])) :
                test_water  += 1
                #colmask_test[i][j] == [43,61,255]
    
    return test_no_data, test_clouds, test_artificial, test_cultivated,  test_broadleaf, test_coniferous, test_herbaceous, test_natural, test_snow, test_water

def intervalle(no_data, clouds, artificial, cultivated,  broadleaf, coniferous, herbaceous, natural, snow, water, no_data_in, clouds_in, artificial_in, cultivated_in,  broadleaf_in, coniferous_in, herbaceous_in, natural_in, snow_in, water_in):
    no_data_inf = no_data-no_data_in
    no_data_sup = no_data+no_data_in
    clouds_inf = clouds-clouds_in
    clouds_sup = clouds+clouds_in
    artificial_inf = artificial-artificial_in
    artificial_sup = artificial+artificial_in
    cultivated_inf = cultivated-cultivated_in
    cultivated_sup = cultivated+cultivated_in
    broadleaf_inf = broadleaf-broadleaf_in
    broadleaf_sup = broadleaf+broadleaf_in
    coniferous_inf = coniferous-coniferous_in
    coniferous_sup = coniferous+coniferous_in
    herbaceous_inf = herbaceous-herbaceous_in
    herbaceous_sup = herbaceous+herbaceous_in
    natural_inf = natural-natural_in
    natural_sup = natural+natural_in
    snow_inf = snow-snow_in
    snow_sup = snow+snow_in
    water_inf = water-water_in
    water_sup = water+water_in
    
    return(no_data_inf, no_data_sup, clouds_inf, clouds_sup, artificial_inf, artificial_sup, cultivated_inf, cultivated_sup,
           broadleaf_inf, broadleaf_sup, coniferous_inf, coniferous_sup, herbaceous_inf, herbaceous_sup,
           natural_inf, natural_sup, snow_inf, snow_sup, water_inf, water_sup)
display_max = 2200
display_min = 0
a = 4
no_data = np.empty((0,a))
clouds = np.empty((0,a))
artificial = np.empty((0,a))
cultivated = np.empty((0,a))
broadleaf = np.empty((0,a))
coniferous = np.empty((0,a))
herbaceous = np.empty((0,a))
natural = np.empty((0,a))
snow = np.empty((0,a))
water = np.empty((0,a))
axisconca = 0
no_data_moy_tot = np.empty((0,a))
clouds_moy_tot = np.empty((0,a))
artificial_moy_tot = np.empty((0,a))
cultivated_moy_tot = np.empty((0,a))
broadleaf_moy_tot = np.empty((0,a))
coniferous_moy_tot = np.empty((0,a))
herbaceous_moy_tot = np.empty((0,a))
natural_moy_tot = np.empty((0,a))
snow_moy_tot = np.empty((0,a))
water_moy_tot = np.empty((0,a))

for k, idx in enumerate(random.sample(range(len(train_images_paths)), 10)):
    axisconca = 0
    image_path = train_images_paths[idx]
    mask_path = train_masks_paths[idx]
    assert image_path.name == mask_path.name
    
    #fig, axs = plt.subplots(1, 3, figsize=(20, 20))
    
    with TiffFile(image_path) as tif:
        image = tif.asarray()  
    if image.dtype == np.uint16:
        iscale = display_max - display_min
        scale = 255 / iscale
        byte_im = (image) * scale
        byte_im = (byte_im.clip(0, 255) + 0.5).astype(np.uint8)
    colimg = byte_im
    #axs[0].axis("off")
    #im = axs[0].imshow(image)

    with TiffFile(mask_path) as tif :
        colmask = tif.asarray()
        colmask_test = colmask
    
    for i in range (colmask.shape[0]):
        for j in range (colmask.shape[1]):            
            if colmask_test[i][j] == 0 :
                no_data = np.concatenate((no_data,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [0,0,0]
            if colmask_test[i][j] == 1 :
                clouds  = np.concatenate((clouds,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [255,25,236]
            if colmask_test[i][j] == 2 :
                artificial  = np.concatenate((artificial,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [215,25,28]
            if colmask_test[i][j] == 3 :
                cultivated  = np.concatenate((cultivated,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [211,154,92]
            if colmask_test[i][j] == 4 :
                test = colimg[i][j]
                broadleaf  = np.concatenate((broadleaf,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [33,115,55]
            if colmask_test[i][j] == 5 :
                coniferous  = np.concatenate((coniferous,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [21,75,35]
            if colmask_test[i][j] == 6 :
                herbaceous  = np.concatenate((herbaceous,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [118,209,93]
            if colmask_test[i][j] == 7 :
                natural  = np.concatenate((natural,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [130,130,130]
            if colmask_test[i][j] == 8 :
                snow  = np.concatenate((snow,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [255,255,255]
            if colmask_test[i][j] == 9 :
                water  = np.concatenate((water,[colimg[i][j]]), axis = axisconca)
                #colmask_test[i][j] == [43,61,255]
    
    classes_colorpalette = {c: color/255. for (c, color) in LCD.CLASSES_COLORPALETTE.items()}
    #show_mask = np.empty((*mask.shape, 3))
    #show_test = np.empty((*colmask_test.shape, 3))
    classes = LCD.CLASSES
    #for c, color in classes_colorpalette.items():
     #   show_mask[mask == c, :] = color
      #  show_test[colmask_test == c, : ] = color
    #axs[1].axis("off")
    #axs[2].axis("off")
    #im = axs[1].imshow(show_mask)
    handles = []
    #for c, color in classes_colorpalette.items():
     #   handles.append(mpatches.Patch(color=color, label=classes[c]))
      #  axs[1].legend(handles=handles)
       # axs[2].legend(handles=handles)
    
    
    
    #axs[0].set_title(f'Image: {image_path.name}')
    #axs[1].set_title('Mask')
    #im = axs[2].imshow(show_test)
    #print("0",no_data_moy)
    #print("1",clouds_moy)
    #print("2", artificial_moy)
    #print("3", cultivated_moy)
    #print("4", broadleaf_moy)
    #print("5", coniferous_moy)
    #print("6", herbaceous_moy)
    #print("7", natural_moy)
    #print("8", snow_moy)
    #print("9", water_moy)
    print(" ")
no_data_moy = np.mean(no_data,axis=0)
clouds_moy = np.mean(clouds, axis=0)
artificial_moy = np.mean(artificial, axis=0)
cultivated_moy = np.mean(cultivated, axis=0)
broadleaf_moy = np.mean(broadleaf, axis=0)
coniferous_moy = np.mean(coniferous, axis=0)
herbaceous_moy = np.mean(herbaceous, axis=0)
natural_moy = np.mean(natural, axis=0)
snow_moy = np.mean(snow, axis=0)
water_moy = np.mean(water, axis=0)

#print(water_tot_moy)
print(artificial_moy)
print("nodata :        "  , no_data_moy)
print("clouds :        "  ,clouds_moy)
print("artificial :    "  ,artificial_moy)
print("cultivated :    ", cultivated_moy)
print("broadleaf :     ", broadleaf_moy)
print("coniferous :    ", coniferous_moy)
print("herbaceous :    ", herbaceous_moy)
print("natural :       ", natural_moy)
print("snow :          ", snow_moy)
print("water :         ", water_moy)
print(water)
E_no_data = np.nanstd(no_data, axis = 0)
E_clouds = np.nanstd(clouds, axis = 0)
E_artificial = np.nanstd(artificial, axis = 0)
E_cultivated = np.nanstd(cultivated, axis = 0)
E_broadleaf = np.nanstd(broadleaf, axis = 0)
E_coniferous = np.nanstd(coniferous, axis = 0)
E_herbaceous = np.nanstd(herbaceous, axis = 0)
E_natural = np.nanstd(natural, axis = 0)
E_snow = np.nanstd(snow, axis = 0)
E_water = np.nanstd(water, axis = 0)
print("nodata :        "  ,E_no_data)
print("clouds :        "  ,E_clouds)
print("artificial :    "  ,E_artificial)
print("cultivated :    " , E_cultivated)
print("broadleaf :     " , E_broadleaf)
print("coniferous :    " , E_coniferous)
print("herbaceous :    " , E_herbaceous)
print("natural :       " , E_natural)
print("snow :          " , E_snow)
print("water :         " , E_water)<jupyter_output><empty_output><jupyter_text>train<jupyter_code>def couleurtrain(no_data, clouds, artificial, cultivated,  broadleaf, coniferous, herbaceous, natural, snow, water):
    
    #Calcul des moyennes
    no_data_moy = np.nanmean(no_data, axis=0)
    clouds_moy = np.nanmean(clouds, axis=0)
    cultivated_moy = np.nanmean(cultivated, axis=0)
    artificial_moy = np.nanmean(artificial, axis=0)
    broadleaf_moy = np.nanmean(broadleaf, axis=0)
    coniferous_moy = np.nanmean(coniferous, axis=0)
    herbaceous_moy = np.nanmean(herbaceous, axis=0)
    natural_moy = np.nanmean(natural, axis=0)
    snow_moy = np.nanmean(snow, axis=0)
    water_moy = np.nanmean(water, axis=0)
    
    #calcul des écart-types
    E_no_data = np.nanstd(no_data, axis=0)
    E_clouds = np.nanstd(clouds, axis=0)
    E_cultivated = np.nanstd(cultivated, axis=0)
    E_artificial = np.nanstd(artificial, axis=0)
    E_broadleaf = np.nanstd(broadleaf, axis=0)
    E_coniferous = np.nanstd(coniferous, axis=0)
    E_herbaceous = np.nanstd(herbaceous, axis=0)
    E_natural = np.nanstd(natural, axis=0)
    E_snow = np.nanstd(snow, axis=0)
    E_water = np.nanstd(water, axis=0)
    
    #variables centrées
    if(np.all(E_no_data)):
        no_data_train = (no_data-no_data_moy)/E_no_data
    else :
        no_data_train = [0,0,0,0]
    if(np.all(E_clouds)):
        clouds_train = (clouds-clouds_moy)/E_clouds
    else :
        clouds_train = [0,0,0,0]
    if(np.all(E_cultivated)):
        cultivated_train = (cultivated-cultivated_moy)/E_cultivated
    else :
        cultivated_train = [0,0,0,0]
    if(np.all(E_artificial)):
        artificial_train = (artificial-artificial_moy)/E_artificial
    else :
        artificial_train = [0,0,0,0]
    if(np.all(E_broadleaf)):
        broadleaf_train = (broadleaf-broadleaf_moy)/E_broadleaf
    else :
        broadleaf_train = [0,0,0,0]
    if(np.all(E_coniferous)):
        coniferous_train = (coniferous-coniferous_moy)/E_coniferous
    else :
        coniferous_train = [0,0,0,0]
    if(np.all(E_herbaceous)):
        herbaceous_train = (herbaceous-herbaceous_moy)/E_herbaceous
    else :
        herbaceous_train = [0,0,0,0]
    if(np.all(E_natural)):
        natural_train = (natural-natural_moy)/E_natural
    else :
        natural_train = [0,0,0,0]
    if(np.all(E_snow)):
        snow_train = (snow-snow_moy)/E_snow
    else :
        snow_train = [0,0,0,0]
    if(np.all(E_water)):
        water_train = (water-water_moy)/E_water
    else :
        water_train = [0,0,0,0]
        
    no_data_t = np.nanmean(no_data_train, axis=0)
    clouds_t = np.nanmean(clouds_train, axis=0)
    cultivated_t = np.nanmean(cultivated_train, axis=0)
    artificial_t = np.nanmean(artificial_train, axis=0)
    broadleaf_t = np.nanmean(broadleaf_train, axis=0)
    coniferous_t = np.nanmean(coniferous_train, axis=0)
    herbaceous_t = np.nanmean(herbaceous_train, axis=0)
    natural_t = np.nanmean(natural_train, axis=0)
    snow_t = np.nanmean(snow_train, axis=0)
    water_t = np.nanmean(water_train, axis=0)
    return no_data_train, clouds_train, artificial_train, cultivated_train,  broadleaf_train, coniferous_train, herbaceous_train, natural_train, snow_train, water_train

def pixelimage (conca, colimg, colmask):
    
    mask_class = np.empty((0))
    for i in range(colimg.shape[0]):
        for j in range(colimg.shape[1]):
            conca = np.concatenate((conca, [colimg[i][j]]), axis = 0)
            mask_class = np.concatenate((mask_class, [colmask[i][j]]), axis = 0)
            
    conca = np.c_[conca, mask_class]        
    return conca
print(colimage)
print(colimg)
print(colmask)
display_max = 2200
display_min = 0
a = 4
b=0
colimg2=np.empty((0,a))
couleur = np.empty((0,5))
axisconca = 0
start = time.perf_counter()

for k, idx in enumerate(random.sample(range(len(train_images_paths)), 10)):
    axisconca = 0
    image_path = train_images_paths[idx]
    mask_path = train_masks_paths[idx]
    assert image_path.name == mask_path.name
    
    #fig, axs = plt.subplots(1, 2, figsize=(20, 20))
    
    with TiffFile(image_path) as tif:
        image = tif.asarray()  
    if image.dtype == np.uint16:
        iscale = display_max - display_min
        scale = 255 / iscale
        byte_im = (image) * scale
        byte_im = (byte_im.clip(0, 255) + 0.5).astype(np.uint8)
    image = byte_im
    colimg = byte_im
    with TiffFile(mask_path) as tif :
        colmask = tif.asarray()
        colmask_test = colmask
    
    colimage = pixelimage(colimg2, colimg, colmask)
    
    couleur = np.concatenate((couleur, colimage), axis = 0)
    
    #axs[0].axis("off")
    #im = axs[0].imshow(image)
    #classes_colorpalette = {c: color/255. for (c, color) in LCD.CLASSES_COLORPALETTE.items()}
    #show_mask = np.empty((*colmask.shape, 3))
    #classes = LCD.CLASSES
    #for c, color in classes_colorpalette.items():
     #   show_mask[colmask == c, :] = color
        
    #axs[1].axis("off")
    #im = axs[1].imshow(show_mask)
    #handles = []
    #for c, color in classes_colorpalette.items():
     #   handles.append(mpatches.Patch(color=color, label=classes[c]))
      #  axs[1].legend(handles=handles)

    #colimage = pixelimage(colimg2, colimg, colmask)
    #colimage


end = time.perf_counter()

print("temps en seconde : ", end - start)

print(couleur.shape)
start = time.perf_counter()
a = 4
axisconca = 0
no_data = np.empty((0,a))
clouds = np.empty((0,a))
artificial = np.empty((0,a))
cultivated = np.empty((0,a))
broadleaf = np.empty((0,a))
coniferous = np.empty((0,a))
herbaceous = np.empty((0,a))
natural = np.empty((0,a))
snow = np.empty((0,a))
water = np.empty((0,a))


couleur_moy = np.mean(couleur, axis =0)
E_couleur = np.nanstd(couleur, axis = 0)
    
couleur_train = (couleur-couleur_moy)/E_couleur

for i in range (couleur.shape[0]):
    if couleur[i][4] == 0 :
        no_data = np.concatenate((no_data,[couleur_train[i, 0:4]]), axis = axisconca)
        #colmask_test[i][j] == [0,0,0]
    if couleur[i][4] == 1 :
        clouds  = np.concatenate((clouds,[couleur_train[i, 0:4]]), axis = axisconca)
        #colmask_test[i][j] == [255,25,236]
    if couleur[i][4] == 2 :
        artificial  = np.concatenate((artificial,[couleur_train[i, 0:4]]), axis = axisconca)
        #colmask_test[i][j] == [215,25,28]
    if couleur[i][4] == 3 :
        cultivated  = np.concatenate((cultivated,[couleur_train[i, 0:4]]), axis = axisconca)
        #colmask_test[i][j] == [211,154,92]
    if couleur[i][4] == 4 :
        broadleaf  = np.concatenate((broadleaf,[couleur_train[i, 0:4]]), axis = axisconca)
        #colmask_test[i][j] == [33,115,55]
    if couleur[i][4] == 5 :
        coniferous  = np.concatenate((coniferous,[couleur_train[i, 0:4]]), axis = axisconca)
        #colmask_test[i][j] == [21,75,35]
    if couleur[i][4] == 6 :
        herbaceous  = np.concatenate((herbaceous,[couleur_train[i, 0:4]]), axis = axisconca)
        #colmask_test[i][j] == [118,209,93]
    if couleur[i][4] == 7 :
        natural  = np.concatenate((natural,[couleur_train[i, 0:4]]), axis = axisconca)
        #colmask_test[i][j] == [130,130,130]
    if couleur[i][4] == 8 :
        snow  = np.concatenate((snow,[couleur_train[i, 0:4]]), axis = axisconca)
        #colmask_test[i][j] == [255,255,255]
    if couleur[i][4] == 9 :
        water  = np.concatenate((water,[couleur_train[i, 0:4]]), axis = axisconca)
        #colmask_test[i][j] == [43,61,255]
    

#train_no_data, train_clouds, train_artificial, train_cultivated, train_broadleaf, train_coniferous, train_herbaceous, train_natural, train_snow, train_water = couleurtrain(no_data, clouds, artificial, cultivated,  broadleaf, coniferous, herbaceous, natural, snow, water)
    
#print(water_tot_moy)
end = time.perf_counter()

print("temps en seconde : ", end - start)
print("no_data    :",np.mean(no_data, axis=0))
print("clouds     :",np.mean(clouds, axis=0))
print("artificial :",np.mean(artificial, axis=0))
print("cultivated :",np.mean(cultivated, axis=0))
print("broadleaf  :",np.mean(broadleaf, axis=0))
print("coniferous :",np.mean(coniferous, axis=0))
print("herbaceous :",np.mean(herbaceous, axis=0))
print("natural    :",np.mean(natural, axis=0))
print("snow       :",np.mean(snow, axis=0))
print("water      :",np.mean(water, axis=0))

no_data_moy = (sum(np.mean(no_data, axis=0)))
clouds_moy = (sum(np.mean(clouds, axis=0)))
artificial_moy = (sum(np.mean(artificial, axis=0)))
cultivated_moy = (sum(np.mean(cultivated, axis=0)))
broadleaf_moy = (sum(np.mean(broadleaf, axis=0)))
coniferous_moy = (sum(np.mean(coniferous, axis=0)))
herbaceous_moy = (sum(np.mean(herbaceous, axis=0)))
natural_moy = (sum(np.mean(natural, axis=0)))
snow_moy = (sum(np.mean(snow, axis=0)))
water_moy = (sum(np.mean(water, axis=0)))
train = np.transpose(([[0,1,2,3,4,5,6,7,8,9],[no_data_moy,clouds_moy, artificial_moy, cultivated_moy, broadleaf_moy, coniferous_moy, herbaceous_moy, natural_moy, snow_moy, water_moy]]))
train
print("no_data    :",sum((np.mean(no_data, axis=0))))
print("clouds     :",sum((np.mean(clouds, axis=0))))
print("artificial :",sum((np.mean(artificial, axis=0))))
print("cultivated :",sum((np.mean(cultivated, axis=0))))
print("broadleaf  :",sum((np.mean(broadleaf, axis=0))))
print("coniferous :",sum((np.mean(coniferous, axis=0))))
print("herbaceous :",sum((np.mean(herbaceous, axis=0))))
print("natural    :",sum((np.mean(natural, axis=0))))
print("snow       :",sum((np.mean(snow, axis=0))))
print("water      :",sum((np.mean(water, axis=0))))
start = time.perf_counter()
a=0
rien_tot = np.empty((0,a))
count =0
display_max = 2200
display_min = 0
for k, idx in enumerate(random.sample(range(len(train_images_paths)), 1)):
    axisconca = 0
    image_path = train_images_paths[idx]
    mask_path = train_masks_paths[idx]
    assert image_path.name == mask_path.name
    
    fig, axs = plt.subplots(1, 2, figsize=(20, 20))
    
    with TiffFile(image_path) as tif:
        image = tif.asarray()  
    if image.dtype == np.uint16:
        iscale = display_max - display_min
        scale = 255 / iscale
        byte_im = (image) * scale
        byte_im = (byte_im.clip(0, 255) + 0.5).astype(np.uint8)
    image = byte_im
    colimg = byte_im
    with TiffFile(mask_path) as tif :
        colmask = tif.asarray()
        colmask_test = colmask
    
    colimage = pixelimage(colimg2, colimg, colmask)
    
    couleur = np.concatenate((couleur, colimage), axis = 0)
    Resul = pixel(train, test_sum)
    axs[0].axis("off")
    im = axs[0].imshow(image)
    classes_colorpalette = {c: color/255. for (c, color) in LCD.CLASSES_COLORPALETTE.items()}
    show_mask = np.empty((*colmask.shape, 3))
    classes = LCD.CLASSES
    for c, color in classes_colorpalette.items():
        show_mask[colmask == c, :] = color
        
    axs[1].axis("off")
    im = axs[1].imshow(show_mask)
    handles = []
    for c, color in classes_colorpalette.items():
        handles.append(mpatches.Patch(color=color, label=classes[c]))
        axs[1].legend(handles=handles)

end = time.perf_counter()

print("temps en seconde : ", end - start)
no_data_test = np.count_nonzero(Resul == 0)
clouds_test = np.count_nonzero(Resul == 1)
artificial_test = np.count_nonzero(Resul ==2)
cultivated_test = np.count_nonzero(Resul == 3)
broadleaf_test = np.count_nonzero(Resul == 4)
coniferous_test = np.count_nonzero(Resul == 5)
herbaceous_test = np.count_nonzero(Resul == 6)
natural_test = np.count_nonzero(Resul == 7)
snow_test = np.count_nonzero(Resul == 8)
water_test = np.count_nonzero(Resul == 9)

print("no_data_test", no_data_test)
print("clouds_test     :", clouds_test)
print("artificial_test :",artificial_test)
print("cultivated_test :",cultivated_test)
print("broadleaf_test  :",broadleaf_test)
print("coniferous_test :",coniferous_test)
print("herbaceous_test :",herbaceous_test)
print("natural_test    :",natural_test)
print("snow_test       :",snow_test)
print("water_test      :",water_test)

for i in range(water.shape[0]) : 
    print(water[i])
    

print(np.mean(water))
print(np.mean(water, axis =0))
print(np.sum(np.mean(water, axis =0)))
no_data_colmask = np.count_nonzero(colmask == 0)
clouds_colmask = np.count_nonzero(colmask == 1)
artificial_colmask = np.count_nonzero(colmask ==2)
cultivated_colmask = np.count_nonzero(colmask == 3)
broadleaf_colmask = np.count_nonzero(colmask == 4)
coniferous_colmask = np.count_nonzero(colmask == 5)
herbaceous_colmask = np.count_nonzero(colmask == 6)
natural_colmask = np.count_nonzero(colmask == 7)
snow_colmask = np.count_nonzero(colmask == 8)
water_colmask = np.count_nonzero(colmask == 9)

print("no_data_test", no_data_colmask)
print("clouds_test     :", clouds_colmask)
print("artificial_test :",artificial_colmask)
print("cultivated_test :",cultivated_colmask)
print("broadleaf_test  :",broadleaf_colmask)
print("coniferous_test :",coniferous_colmask)
print("herbaceous_test :",herbaceous_colmask)
print("natural_test    :",natural_colmask)
print("snow_test       :",snow_colmask)
print("water_test      :",water_colmask)<jupyter_output>no_data_test 0
clouds_test     : 0
artificial_test : 6
cultivated_test : 312
broadleaf_test  : 15208
coniferous_test : 45334
herbaceous_test : 4676
natural_test    : 0
snow_test       : 0
water_test      : 0<jupyter_text>Test<jupyter_code>def couleurtest(image):
    colimg= np.array(image)
    a=4
    RGB = np.empty((0,a))
    
    #pixel = colimg.shape[0]*colimg.shape[1]
    for i in range(colimg.shape[0]):
        for j in range(colimg.shape[1]):
            RGB = np.concatenate((RGB,[colimg[i][j]]), axis = 0)
            
    RGB_moy = np.nanmean(RGB,axis=0)
    E_RGB = np.nanstd(RGB, axis = 0)
    
    RGB_test = (RGB-RGB_moy)/E_RGB
    return RGB_test
def pixel(train, test):
    pixel = []
    for i in range(test.shape[0]):
        train_ = train
        b = (train_[:,1]-test[i])
        c= abs(b)
        pixel_= np.nanargmin(c)
        pixel = np.concatenate((pixel, [pixel_]))
    return pixel
    Resul = pixel(train, test_sum)
print(test_sum[358])
b = train
a = (b[:,1]-test_sum[358])
c = abs(a)
print(c)
print(np.nanargmin(c))
start = time.perf_counter()
a=0
rien_tot = np.empty((0,a))
count =0
display_max = 2200
display_min = 0
for k, idx in enumerate(random.sample(range(len(test_images_paths)), 1)):
    axisconca = 0
    image_path = test_images_paths[idx]
       
    fig, axs = plt.subplots(1, 1, figsize=(20, 20))
    
    with TiffFile(image_path) as tif:
        image = tif.asarray()  
    if image.dtype == np.uint16:
        iscale = display_max - display_min
        scale = 255 / iscale
        byte_im = (image) * scale
        byte_im = (byte_im.clip(0, 255) + 0.5).astype(np.uint8)
    image = byte_im
    test= couleurtest(image)
    test_sum = np.sum(test, axis=1)
    Resul = pixel(train, test_sum)
    axs.axis("off")
    im = axs.imshow(image)
    
end = time.perf_counter()

print("temps en seconde : ", end - start)
no_data_test = np.count_nonzero(Resul == 0)
clouds_test = np.count_nonzero(Resul == 1)
artificial_test = np.count_nonzero(Resul ==2)
cultivated_test = np.count_nonzero(Resul == 3)
broadleaf_test = np.count_nonzero(Resul == 4)
coniferous_test = np.count_nonzero(Resul == 5)
herbaceous_test = np.count_nonzero(Resul == 6)
natural_test = np.count_nonzero(Resul == 7)
snow_test = np.count_nonzero(Resul == 8)
water_test = np.count_nonzero(Resul == 9)

print("no_data_test", no_data_test)
print("clouds_test     :", clouds_test)
print("artificial_test :",artificial_test)
print("cultivated_test :",cultivated_test)
print("broadleaf_test  :",broadleaf_test)
print("coniferous_test :",coniferous_test)
print("herbaceous_test :",herbaceous_test)
print("natural_test    :",natural_test)
print("snow_test       :",snow_test)
print("water_test      :",water_test)
print(list(zip(*np.where(Resul ==1))))
test_sum[5]
#no_data, clouds, artificial, cultivated, broadleaf, coniferous, herbaceous, natural, snow, water = intervalle(no_data_moy, clouds_moy,artificial_moy, cultivated_moy, broadleaf_moy, coniferous_moy,herbaceous_moy, natural_moy,snow_moy, water_moy,E_no_data, E_clouds, E_artificial, E_cultivated,  E_broadleaf, E_coniferous, E_herbaceous, E_natural, E_snow, E_water)
rien_tot = np.empty((0,a))
count =0
display_max = 2200
display_min = 0
for k, idx in enumerate(random.sample(range(len(test_images_paths)), 10)):
    axisconca = 0
    image_path = test_images_paths[idx]
       
    #fig, axs = plt.subplots(1, 3, figsize=(20, 20))
    
    with TiffFile(image_path) as tif:
        image = tif.asarray()  
    if image.dtype == np.uint16:
        iscale = display_max - display_min
        scale = 255 / iscale
        byte_im = (image) * scale
        byte_im = (byte_im.clip(0, 255) + 0.5).astype(np.uint8)
    colimage = byte_im
    #axs[0].axis("off")
    #im = axs[0].imshow(image)
        
    image= np.array(colimage)
    count+=1
    print(count)
    for i in range (image.shape[0]):
        for j in range (image.shape[1]):            
            if ((no_data_inf[0]<image[i][j][0]<no_data_sup[0])and(no_data_inf[1]<image[i][j][1]<no_data_sup[1])and(no_data_inf[2]<image[i][j][2]<no_data_sup[2])and(no_data_inf[3]<image[i][j][3]<no_data_sup[3])) :
                test_no_data =test_no_data+1
            if ((clouds_inf[0]<image[i][j][0]<clouds_sup[0])and(clouds_inf[1]<image[i][j][1]<clouds_sup[1])and(clouds_inf[2]<image[i][j][2]<clouds_sup[2])and(clouds_inf[3]<image[i][j][3]<clouds_sup[3])) :
                test_clouds = test_clouds+1
                #colmask_test[i][j] == [255,25,236]
            if ((artificial_inf[0]<image[i][j][0]<artificial_sup[0])and(artificial_inf[1]<image[i][j][1]<artificial_sup[1])and(artificial_inf[2]<image[i][j][2]<artificial_sup[2])and(artificial_inf[3]<image[i][j][3]<artificial_sup[3])) :
                test_artificial =test_artificial +1
                #colmask_test[i][j] == [215,25,28]
            if ((cultivated_inf[0]<image[i][j][0]<cultivated_sup[0])and(cultivated_inf[1]<image[i][j][1]<cultivated_sup[1])and(cultivated_inf[2]<image[i][j][2]<cultivated_sup[2])and(cultivated_inf[3]<image[i][j][3]<cultivated_sup[3])) :
                test_cultivated  = test_cultivated +1
                #colmask_test[i][j] == [211,154,92]
            if ((broadleaf_inf[0]<image[i][j][0]<broadleaf_sup[0])and(broadleaf_inf[1]<image[i][j][1]<broadleaf_sup[1])and(broadleaf_inf[2]<image[i][j][2]<broadleaf_sup[2])and(broadleaf_inf[3]<image[i][j][3]<broadleaf_sup[3])) :    
                test_broadleaf  = test_broadleaf +1
                #colmask_test[i][j] == [33,115,55]
            if ((coniferous_inf[0]<image[i][j][0]<coniferous_sup[0])and(coniferous_inf[1]<image[i][j][1]<coniferous_sup[1])and(coniferous_inf[2]<image[i][j][2]<coniferous_sup[2])and(coniferous_inf[3]<image[i][j][3]<coniferous_sup[3])) :
                test_coniferous  = test_coniferous+1
                #colmask_test[i][j] == [21,75,35]
            if ((herbaceous_inf[0]<image[i][j][0]<herbaceous_sup[0])and(herbaceous_inf[1]<image[i][j][1]<herbaceous_sup[1])and(herbaceous_inf[2]<image[i][j][2]<herbaceous_sup[2])and(herbaceous_inf[3]<image[i][j][3]<herbaceous_sup[3])) :
                test_herbaceous  = test_herbaceous+1
                #colmask_test[i][j] == [118,209,93]
            if ((natural_inf[0]<image[i][j][0]<natural_sup[0])and(natural_inf[1]<image[i][j][1]<natural_sup[1])and(natural_inf[2]<image[i][j][2]<natural_sup[2])and(natural_inf[3]<image[i][j][3]<natural_sup[3])) :
                test_natural  = test_natural+1
                #colmask_test[i][j] == [130,130,130]
            if ((snow_inf[0]<image[i][j][0]<snow_sup[0])and(snow_inf[1]<image[i][j][1]<snow_sup[1])and(snow_inf[2]<image[i][j][2]<snow_sup[2])and(snow_inf[3]<image[i][j][3]<snow_sup[3])) :
                test_snow  = test_snow+1
                #colmask_test[i][j] == [255,255,255]
            if ((water_inf[0]<image[i][j][0]<water_sup[0])and(water_inf[1]<image[i][j][1]<water_sup[1])and(water_inf[2]<image[i][j][2]<water_sup[2])and(water_inf[3]<image[i][j][3]<water_sup[3])) :
                test_water  = test_water+1
                #colmask_test[i][j] == [43,61,255]
            #else : 
             #   rien_tot  = np.concatenate((rien_tot,[image[i][j]]), axis = axisconca)
              #  rien = rien+1

print("0 : " ,test_no_data)
print("1 : " ,test_clouds)
print("2 : " ,test_artificial)
print("3 : " ,test_cultivated)
print("4 : ", test_broadleaf)
print("5 : ", test_coniferous)
print("6 : ", test_herbaceous)
print("7 : ", test_natural)
print("8 : ", test_snow)
print("9 : ", test_water)

print(test_no_data+test_clouds+test_artificial+test_cultivated+test_broadleaf+test_coniferous+test_herbaceous+test_natural+test_snow+test_water)

print(rien_tot)
print(coniferous_inf[0],"  ", coniferous_sup[0]), print(coniferous_inf[1],"  ", coniferous_sup[1])
print(coniferous_inf[2],"  ", coniferous_sup[2])
print(coniferous_inf[3],"  ", coniferous_sup[3])
print(image[55][:])
def show_image(image, display_min=50, display_max=400, ax=None):
    """Show an image.
    Args:
        image (numpy.array[uint16]): the image. If the image is 16-bit, apply bytescaling to convert to 8-bit
    """
    if image.dtype == np.uint16:
        iscale = display_max - display_min
        scale = 255 / iscale
        byte_im = (image) * scale
        byte_im = (byte_im.clip(0, 255) + 0.5).astype(np.uint8)
        image = byte_im
    # show image
    if ax is None:
        fig, ax = plt.subplots(figsize=(10, 10))
    ax.axis("off")
    im = ax.imshow(image)
    
    return im

def show_mask(mask, classes_colorpalette, classes=None, add_legend=True, ax=None):
    """Show a a semantic segmentation mask.
    Args:
       mask (numpy.array[uint8]): the mask in 8-bit
       classes_colorpalette (dict[int, tuple]): dict mapping class index to an RGB color in [0, 1]
       classes (list[str], optional): list of class labels
       add_legend
    """
    show_mask = np.empty((*mask.shape, 3))
    for c, color in classes_colorpalette.items():
        show_mask[mask == c, :] = color
    if ax is None:
        fig, ax = plt.subplots(figsize=(10, 10))
    ax.axis("off")
    im = ax.imshow(show_mask)
    if add_legend:
        # show legend mapping pixel colors to class names
        import matplotlib.patches as mpatches
        handles = []
        for c, color in classes_colorpalette.items():
            handles.append(mpatches.Patch(color=color, label=classes[c]))
        ax.legend(handles=handles)
    return im<jupyter_output><empty_output><jupyter_text>Test d'un réseau de neuronnes avec une seule couche.<jupyter_code># Nous avons des images en 256*256 pixels, soit 65 536 pixels par images avec 10 classes différentes, soit 655 360 paramètres pour la première couche du réseau de neuronnes.
# Pour ce faire, nous allons créer 10 filtres unicolores pour chaque classes. A savoir, nos images comportent 4 couleurs : Rouge, vert, bleu mais aussi spectroscopie dans l'infrarouge proche SPIR.
# Dans ce premier cas, on ne prendra pas en compte le SPIR mais uniquement le spectre du visible. 
# P : matrice de pixels 246*256 soit 1*65 536
# W : matrices filtres de tailles 65 536 * 10
# S : matrices de 1*10 allant de 0 à 1 (pourcentage de classe) => sum(S) = 1
# 
# De ce fait, lors de la phase de test, l'inconnue dans notre opération est W, notre matric de filtre.
# Pour trouver la valeur de la matrixe W, on tentera de miimiser notre erreur E:
# E = sum((P.W-S)²)                   => pour les dimension des matrices :  sum (((1 * 65 536 . 65 536 * 10) - 1*10)²)
# Si E = 0, on aura trouver le filtre le plus proche possible pour passer de notre image satellite au mask que nous appliquons. 
# Pour nous approcher de E = 0, nous allons donc devoir faire varier notre filtre W. Pour ce faire, on utilisera le gradient de W. Soit 
# W(i+1 ) = W(i) - f'(W(i)) * fac     avec fac proche de 0 pour le faire varier petit à petit. 
# Le gradient de W est  : Grad = t(P)*(2*(P.W-S))    avec t(P) la transposé de la matrice P
# les 10 classes en RGB et hexa
# No-data => 0;0;0      FFFFFF
# Clouds => 255;25;23   FF19EC
# Artificial surfaces and constructions => 215;25;28   D7191C
# Cultivated ares => 211;154;92   D39A5C
# broadleaf tree cover => 33;115;55   217337
# Coniferous tree vover => 21;75;35   154B23
# Herbaceous vegetation => 118;209;93   76D15D
# Natural material surfaces => 130;130;130   828282
# Permanent snow-covered surfaces => 255;255;255   FFFFFF
# Water bodies => 43;61;255   2B3DFF
https://github.com/Hvass-Labs/TensorFlow-Tutorials/blob/master/01_Simple_Linear_Model.ipynb
a = 4
    no_data = np.empty((0,a))
    clouds = np.empty((0,a))
    artificial = np.empty((0,a))
    cultivated = np.empty((0,a))
    broadleaf = np.empty((0,a))
    coniferous = np.empty((0,a))
    herbaceous = np.empty((0,a))
    natural = np.empty((0,a))
    snow = np.empty((0,a))
    water = np.empty((0,a))
    axisconca = 0
    
    
    
no_data_tot_moy = np.nanmean(no_data_moy_tot,axis=0)
clouds_tot_moy = np.nanmean(clouds_moy_tot, axis=0)
artificial_tot_moy = np.nanmean(artificial_moy_tot, axis=0)
cultivated_tot_moy = np.nanmean(cultivated_moy_tot, axis=0)
broadleaf_tot_moy = np.nanmean(broadleaf_moy_tot, axis=0)
coniferous_tot_moy = np.nanmean(coniferous_moy_tot, axis=0)
herbaceous_tot_moy = np.nanmean(herbaceous_moy_tot, axis=0)
natural_tot_moy = np.nanmean(natural_moy_tot, axis=0)
snow_tot_moy = np.nanmean(snow_moy_tot, axis=0)
water_tot_moy = np.nanmean(water_moy_tot, axis=0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1569711950270429, 'fraction_numerical': 0.034335919336286436, 'mean_word_length': 3.760251630941286, 'pattern_counts': {'":': 0, '<': 191, '<?xml version=': 0, '>': 43, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '656122', 'license': 'MIT', 'n_tokens_mistral': 16849, 'n_tokens_neox': 15794, 'n_words': 2579, 'path': 'Quentin/Bourbon pointu.ipynb', 'repo_name': 'Dragusto/challenge-donnees-sat', 'size': '40861'}