<jupyter_start><jupyter_text>Tile-wise pixel statistics<jupyter_code>#%matplotlib inline
import numpy as np
from pathlib import Path
import os
import rasterio as rio
import rasterio.plot as rioplot
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.pyplot as plt
import numpy as np

from skimage import data, img_as_float
from skimage import exposure
from PIL import Image
import pandas as pd
DATADIR = "../../data/train"
RAW_MAPS = DATADIR + "/map"     #'../data/raw/images'
def getPixels(file):
    with rio.open(file) as inds:
         
        ima = inds.read()

        vals, counts = np.unique(ima, return_counts=True)

        dic = (dict(zip(vals, counts)))
        if 127 not in dic.keys():
            dic[127] = 0
        if 255 not in dic.keys():
            dic[255] = 0
    return  dic
stats = []
for file in Path(RAW_MAPS).iterdir():
    if file.name.endswith(('.tif', '.tiff')):
        dic = getPixels(file)
        stats.append([file.name, dic[0], dic[127], dic[255]])
df = pd.DataFrame.from_dict(stats)
df.columns = ['name', 'numPixel_noRoad', 'numPixel_pavedRoad', 'numPixel_unpavedRoad']
col_list= list(df)
col_list.remove('name')
col_list
df['sumPixels'] = df[col_list].sum(axis=1)
df['relative_noRoad']    = df['numPixel_noRoad']/df['sumPixels']
df['relative_pavedRoad'] = df['numPixel_pavedRoad']/df['sumPixels']
df['relative_unpavedRoad'] = df['numPixel_unpavedRoad']/df['sumPixels']
#df.to_csv('numOfPixPerClassPerTile.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13951935914552738, 'fraction_numerical': 0.014686248331108143, 'mean_word_length': 4.081355932203389, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '246805', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 568, 'n_tokens_neox': 539, 'n_words': 126, 'path': 'notebooks/visualize_imagery/Tile-wise_pixel_statstics_LISA-Copy1.ipynb', 'repo_name': 'BerlinRDT/roaddetection', 'size': '1498'}