<jupyter_start><jupyter_text>Identification of interesting VIIRS/MODIS filesThis notebook process a general bash file that retrieve VIIRS and MODIS images to return a new bash file with the images that mathches ICESat-2 data in a given temporal window.Prerequisites:- bash script file with the name of all the VIIRS/MODIS files in a given specific region (see https://search.earthdata.nasa.gov/search?m=-41.467241310846475!75.05859375!4!1!0!0%2C2)Imput:- Maximum temporal tolerance between ICESat-2 ann VIIRS/MOIDS retrievals- Spatial extent: this should be a subset of the spatial extent used to retrieve the scipt with the names of all the VIIRS/MODIS filesOutput:- Modified bash file with the names of the VIIRS/MODIS files that temporaly match with retrievals from ICESat-2In order to download the files, make the script an executable by running the line 'chmod 777 download.sh' from the command line. After that is complete, the file can be executed by typing './download.sh'.<jupyter_code>import pandas as pd
from icepyx import query as ipd

from utils_cloud import viirs_date, modis_date
# Spatial extend for the ICESat-2 search 
# Observation: in case that the VIIRS file retrieve images in a different spatial extent, then we are no going to have matches, 
# independently of the window. 
#spatial_extent = [-50, 70, -49, 71]
spatial_extent = [-50, 68, -45, 73]

# Temporal window in minutes
minutes = 20
hr = minutes / 60 
# Maximim number of files we are interested in retrieve 
max_filtered = 2#1000000000000


# lenght of line if bash file with VIIRS imagen name
viirs_length_bash = 146
# lenght of line if bash file with MODIS imagen name
modis_aqua_length_bash = 121
modis_terra_length_bash = 121


# path with bash retrieval files
#path_retrieve = "data/VIIRS_bash/"
path_retrieve = "data/Earthdata_scripts/"<jupyter_output><empty_output><jupyter_text>Select your satellite data<jupyter_code>#satellite = "VIIRS"
satellite = "MODIS-Aqua"
#satellite = "MODIS-Terra"

if satellite == "VIIRS":
    
    # name of the original bash file to retrieve the data
    exe_old = "VIIRS-Greenland-download.sh"
    # name of the output bash file 
    exe_new = "VIIRS-Greenland-download-filtered.sh"
    lenght_bash = viirs_length_bash
    
if satellite == "MODIS-Aqua":
    
    exe_old = "MODIS-Aqua-Greenland-download.sh"
    exe_new = "MODIS-Aqua-Greenland-download-filtered.sh"    
    lenght_bash = modis_aqua_length_bash
    
if satellite == "MODIS-Terra":
    
    exe_old = "MODIS-Terra-Greenland-download.sh"
    exe_new = "MODIS-Terra-Greenland-download-filtered.sh"    
    lenght_bash = modis_terra_length_bash
file = open(path_retrieve + exe_old, 'r')
new_file = open(path_retrieve + exe_new, 'x')

#viirs_names = []

counter = 0
counter_filtered = 0

for line in file:
    
    #print(line)
    
    if ("https://ladsweb.modaps.eosdis.nasa.gov" in line) and (len(line) == lenght_bash):
                
        if counter_filtered < max_filtered:

            if satellite == "VIIRS":
                Vfile = line[:-1].split("/")[-1]
                f_t = viirs_date(Vfile)

            if satellite == "MODIS-Aqua" or satellite == "MODIS-Terra":
                Mafile = line[:-1].split("/")[-1]
                #print(Mafile)
                f_t = modis_date(Mafile)
                #print(f_t)
                
            # Temporal search window for 
            start = f_t - pd.DateOffset(hours=hr)
            end   = f_t + pd.DateOffset(hours=hr)

            start_date_str = start.strftime('%Y-%m-%d')
            end_date_str   = end.strftime('%Y-%m-%d')
            start_time_str = start.strftime('%H:%M:%S')
            end_time_str   = end.strftime('%H:%M:%S')

            try:

                region_a = ipd.Query("ATL06", spatial_extent, [start_date_str, end_date_str], start_time_str, end_time_str)  # This was different before
                avail_granules = region_a.avail_granules(ids=True)

                #viirs_names.append(Vfile)
                #new_file.write(Vfile + "\n") # add \n
                assert len(avail_granules) > 0
                new_file.write(line)
                counter_filtered += 1

            except AssertionError:
                None
                
            counter += 1
        
        if counter % 50 == 0:
            print(">>> There are a total of", counter_filtered, "files founded out of", counter, "file names.")
            
        #break
        
    else:
        
        new_file.write(line)
        
    
new_file.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09340659340659341, 'fraction_numerical': 0.02131868131868132, 'mean_word_length': 2.6760904684975766, 'pattern_counts': {'":': 5, '<': 10, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '220818', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 1589, 'n_tokens_neox': 1428, 'n_words': 414, 'path': 'Climate_bash.ipynb', 'repo_name': 'alicecima/CloudMask', 'size': '4550'}