<jupyter_start><jupyter_text>Information Points Extractor Responsabilities of this file *  This file is responsable for extracting information from rasters through the occurence point and save to a Numpy Arrays  Who is Running?<jupyter_code>project_root = "/content/drive/MyDrive/TFC_MatheusSasso"
from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).<jupyter_text>Variable Parameters<jupyter_code># Collect for all species or only for the taxonkey one
collect_all = False #@param {type:"boolean"}

# Geneal parameters used on this notebook
taxonKey=2874484 #@param {type:"integer"}<jupyter_output><empty_output><jupyter_text>Fixed Paramets<jupyter_code># Reference Data Paths
occurrences_root = project_root + "/Data/GBIF_Ocurrences"
base_csv_files_path = project_root + "/Data/Standarized_Brazil_Data/TXT_Aux_Files"


# Required parametes to instantiate Raster_Information_Collector
coorection_limit = 10 
raster_information_collector_output_dir = project_root +  "/Data/Rasters_As_Numpy_Arrays"<jupyter_output><empty_output><jupyter_text>Package Downloads<jupyter_code>! pip install geopandas --quiet
! pip install rasterio --quiet<jupyter_output><empty_output><jupyter_text>Imports<jupyter_code>import os
import gc
import rasterio
import numpy as np
from matplotlib import rcParams
rcParams['font.family'] = 'sans-serif'
rcParams['font.sans-serif'] = ['Arial']
import matplotlib.pyplot as plt
from matplotlib import pyplot
from typing import List,Tuple
from sklearn.utils import Bunch

import geopandas as gpd
from rasterio.plot import show
from rasterio.windows import Window<jupyter_output><empty_output><jupyter_text>Getting Specie Name<jupyter_code>!wget https://raw.githubusercontent.com/climate-and-health-datasci-Unicamp/permapy/main/utils/species_taxon_id_dict.py
from species_taxon_id_dict import *
species_name =  species_taxon_id_dict[taxonKey]
species_name<jupyter_output>--2020-12-23 18:39:42--  https://raw.githubusercontent.com/climate-and-health-datasci-Unicamp/permapy/main/utils/species_taxon_id_dict.py
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 192.168.127.12, 192.168.127.12, 172.16.31.10, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|192.168.127.12|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 2147 (2.1K) [text/plain]
Saving to: ‘species_taxon_id_dict.py.2’

species_taxon_id_di 100%[===================>]   2.10K  --.-KB/s    in 0s      

2020-12-23 18:39:42 (37.6 MB/s) - ‘species_taxon_id_dict.py.2’ saved [2147/2147]<jupyter_text>Retrieving aux Classes<jupyter_code>!wget https://raw.githubusercontent.com/climate-and-health-datasci-Unicamp/permapy/main/utils/utils.py
!wget https://raw.githubusercontent.com/climate-and-health-datasci-Unicamp/permapy/main/utils/raster_utils.py
from utils import Utils
from raster_utils import Raster_Utils

raster_utils = Raster_Utils()
utils_methods = Utils()<jupyter_output><empty_output><jupyter_text>Creating Information Collector Output Folder<jupyter_code>utils_methods.create_folder_structure(raster_information_collector_output_dir)<jupyter_output><empty_output><jupyter_text>Raster_Information_Collector<jupyter_code># from typing import List,Tuple
# import gc
# from sklearn.utils import Bunch
# import geopandas as gpd
# import rasterio
# import numpy as np
# from rasterio.plot import show
# from rasterio.windows import Window
# from osgeo import gdal
# from matplotlib import pyplot

class Raster_Information_Collector:
  """
  This class is reponsable for extracting data from rasters on GBIF occurrence locations

  Attributes
  ----------
  raster_base_configs : str
      Directory to save coverages
  coorection_limit : int
      Limit of iterations to correct no information points
  raster_standards : object
      Raster standards object
  """

  def __init__(self, output_dir:str,raster_utils,utils_methods,coorection_limit:int=10):
      """
      Parameters
      ----------
      raster_base_configs : str
          Directory to save coverages
      coorection_limit : int
          Limit of iterations to correct no information points
      raster_utils : object
          Raster standards object
      """

      self.output_dir = output_dir
      self.raster_utils = raster_utils
      self.coorection_limit = coorection_limit
      self.utils_methods = utils_methods


  def _fill_peristent_no_data_values_with_median_value(self,raster_occurrences_array):
      """ For grids that still with empty value after the board points treatment, this function fill it with the mean value"""

      median_value = np.median(raster_occurrences_array[[raster_occurrences_array!=self.raster_utils.no_data_val]])
      for i,elem in enumerate(raster_occurrences_array):
        if elem == self.raster_utils.no_data_val:
          raster_occurrences_array[i] = median_value
      
      return raster_occurrences_array

  def save_coverges_to_numpy(self,specie_dir:str,species_name:str,root_raster_files_list:List[str]):
    """ Save all extracted to a numpy array"""

    data = gpd.read_file(specie_dir)
    coordinates = np.array((np.array(data['LATITUDE']),np.array(data['LONGITUDE']))).T
        
    # determine coverage values for each of the training & testing points
    Long = coordinates[:,1]
    Lat = coordinates[:,0]

    all_env_values_list = []
    for i,fp in enumerate(root_raster_files_list):
        
        # Exctraction occurences from rasters. As each raster file can have a 
        # different resolution, ix and iy are calculated in every step.
        raster_array,_,xgrid,ygrid,_,_ = self.raster_utils.get_raster_infos(fp)
        ix = np.searchsorted(xgrid,Long)
        iy = np.searchsorted(ygrid,Lat)
        raster_occurrences_array = raster_array[-iy, ix].T
        
        #treating cases where points that should be inside country are outside
        del raster_array

        #tretaing cases that still with no data values
        raster_occurrences_array= self._fill_peristent_no_data_values_with_median_value(raster_occurrences_array)
        
        #selecting the env value on the occurrence position
        all_env_values_list.append(raster_occurrences_array)

        del raster_occurrences_array
        del ix
        del iy
        gc.collect()
        

    coverage= np.stack([value for value in all_env_values_list]).T
    del all_env_values_list
    gc.collect() 

    self.utils_methods.save_nparray_to_folder(coverage,self.output_dir,species_name)
    
    del coverage
    gc.collect()<jupyter_output><empty_output><jupyter_text>List rasters locations<jupyter_code>list_raster_files = open(f'{base_csv_files_path}/list_raster_files.txt', 'r').read().splitlines()
list_names_raster = open(f'{base_csv_files_path}/list_names_raster.txt', 'r').read().splitlines()<jupyter_output><empty_output><jupyter_text>Performing Collection Creating collector instance<jupyter_code>raster_collector = Raster_Information_Collector(output_dir=raster_information_collector_output_dir,raster_utils=raster_utils,utils_methods=utils_methods,coorection_limit=coorection_limit)<jupyter_output><empty_output><jupyter_text>Performing collections for each plant<jupyter_code>if not collect_all:
  specie_shp_path = os.path.join(occurrences_root,species_name,species_name+".shp")
  raster_collector.save_coverges_to_numpy(specie_dir=specie_shp_path,
                                          species_name=species_name,
                                          root_raster_files_list=list_raster_files)<jupyter_output><empty_output><jupyter_text>Checking the response format<jupyter_code>if not collect_all:
  saved_numpy_array = utils_methods.retrieve_data_from_np_array(raster_information_collector_output_dir+'/'+species_name + '.npy')
  saved_numpy_array.shape,saved_numpy_array<jupyter_output><empty_output><jupyter_text>Executing pipeline step for all studied species<jupyter_code>if collect_all:
  for tax_id, species_name in species_taxon_id_dict.items():
      specie_shp_path = os.path.join(occurrences_root,species_name,species_name+".shp")
      raster_collector.save_coverges_to_numpy(specie_dir=specie_shp_path,
                                          species_name=species_name,
                                          root_raster_files_list=list_raster_files)<jupyter_output>Reading raster bio1_annual_mean_temperature.tif
{'fraction_non_alphanumeric': 0.08664387598726865, 'fraction_numerical': 0.013674407638807026, 'mean_word_length': 4.783231083844581, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 55, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '446992', 'license': 'MIT', 'n_tokens_mistral': 2916, 'n_tokens_neox': 2742, 'n_words': 521, 'path': 'notebooks/TCC/2_Information_Points_Extractor.ipynb', 'repo_name': 'math-sasso/masters_research', 'size': '8484'}