<commit_before>'''
This script creates maps of model outputs at roughly 10km resolution (0.1x0.1 degrees), where each output pixel
represents the total value in the pixel (not the density) (hence, the aggregated results).
This is currently only set up for gross emissions from biomass+soil and net flux from biomass+soil.
It iterates through all the model outputs that are supplied.
First, it rewindows the model output, pixel area tile, and tcd tile into 400x400 (0.1x0.1 degree) windows, instead of the native
40000x1 pixel windows.
Then it calculates the per pixel value for each model output pixel and sums those values within each 0.1x0.1 degree
aggregated pixel.
It converts cumulative carbon gain to CO2 gain per year, converts cumulative CO2 flux to CO2 flux per year, and
converts cumulative gross CO2 emissions to gross CO2 emissions per year.
The user has to supply a tcd threshold for which forest pixels to include in the results.
sample command: python mp_aggregate_results_to_10_km.py -tcd 30 -t no_shifting_ag -sagg s3://gfw2-data/climate/carbon_model/10km_output_aggregation/biomass_soil/standard/20191107/net_flux_t_CO2e_per_year_biomass_soil_10km_tcd30_modelv1_1_2_std_2019_11_07.tif
'''


import multiprocessing
import aggregate_results_to_10_km
import subprocess
from functools import partial
import argparse
import os
import glob
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

def main():

    # Files to download for this script. 'true'/'false' says whether the input directory and pattern should be
    # changed for a sensitivity analysis. This does not need to change based on what run is being done;
    # this assignment should be true for all sensitivity analyses and the standard model.
    download_dict = {
             cn.gross_emis_all_gases_all_drivers_biomass_soil_dir: [cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil],
             cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types],
             cn.net_flux_dir: [cn.pattern_net_flux]
             }

    # Sole argument for the script: the tree cover density threshold (pixels below this will not be aggregated)
    parser = argparse.ArgumentParser(description='Create maps of model output at approximately 10 km resolution')
    parser.add_argument('--tcd-threshold', '-tcd', required=True,
                        help='Tree cover density threshold above which pixels will be included in the aggregation.')
    parser.add_argument('--model-type', '-t', required=True,
                        help='{}'.format(cn.model_type_arg_help))
    parser.add_argument('--std-net-flux-aggreg', '-sagg', required=False,
                        help='The s3 standard model net flux aggregated tif, for comparison with the sensitivity analysis map')
    args = parser.parse_args()
    thresh = args.tcd_threshold
    thresh = int(thresh)
    sensit_type = args.model_type
    std_net_flux = args.std_net_flux_aggreg

    # Checks whether the canopy cover argument is valid
    if thresh < 0 or thresh > 99:
        raise Exception('Invalid tcd. Please provide an integer between 0 and 99.')

    # Checks whether the sensitivity analysis argument is valid
    uu.check_sensit_type(sensit_type)

    # tile_id_list = ['00N_070W'] # test tiles
    tile_id_list = 'all'

    # # Pixel area tiles-- necessary for calculating sum of pixels for any set of tiles
    # uu.s3_flexible_download(cn.pixel_area_dir, cn.pattern_pixel_area, '.', sensit_type, tile_id_list)
    # # tree cover density tiles-- necessary for filtering sums by tcd
    # uu.s3_flexible_download(cn.tcd_dir, cn.pattern_tcd, '.', sensit_type, tile_id_list)

    print "Model outputs to process are:", download_dict

    # List of output directories. Modified later for sensitivity analysis.
    # Output pattern is determined later.
    output_dir_list = [cn.output_aggreg_dir]

    # If the model run isn't the standard one, the output directory is changed
    if sensit_type != 'std':
        print "Changing output directory and file name pattern based on sensitivity analysis"
        output_dir_list = uu.alter_dirs(sensit_type, output_dir_list)


    # Iterates through the types of tiles to be processed
    for dir, download_pattern in download_dict.items():

        download_pattern_name = download_pattern[0]

        # Downloads the model output tiles to be processed
        uu.s3_flexible_download(dir, download_pattern_name, '.', sensit_type, tile_id_list)

        # Renames the tiles according to the sensitivity analysis before creating dummy tiles.
        # The renaming function requires a whole tile name, so this passes a dummy time name that is then stripped a few
        # lines later.
        tile_id = '30N_090W'     # a dummy tile id (but it has to be a real tile id). It is removed later.
        output_pattern = uu.sensit_tile_rename(sensit_type, tile_id, download_pattern_name)
        pattern = output_pattern[9:-4]

        print pattern


        # Lists the tiles of the particular type that is being iterates through.
        # Excludes all intermediate files
        tile_list = uu.tile_list_spot_machine(".", "{}.tif".format(pattern))
        print tile_list
        # from https://stackoverflow.com/questions/12666897/removing-an-item-from-list-matching-a-substring
        tile_list = [i for i in tile_list if not ('hanson_2013' in i)]
        tile_list = [i for i in tile_list if not ('rewindow' in i)]
        tile_list = [i for i in tile_list if not ('10km' in i)]

        # tile_list = ['00N_070W_cumul_gain_AGCO2_BGCO2_t_ha_all_forest_types_2001_15_biomass_swap.tif']  # test tiles

        print tile_list
        print "There are {} tiles to process".format(str(len(tile_list))) + "\n"
        print "Processing:", dir, "; ", pattern

        # Converts the 10x10 degree Hansen tiles that are in windows of 40000x1 pixels to windows of 400x400 pixels,<commit_msg>Creating aggregated rasters for legal amazon loss sensitivity analysis.<commit_after>'''
This script creates maps of model outputs at roughly 10km resolution (0.1x0.1 degrees), where each output pixel
represents the total value in the pixel (not the density) (hence, the aggregated results).
This is currently only set up for gross emissions from biomass+soil and net flux from biomass+soil.
It iterates through all the model outputs that are supplied.
First, it rewindows the model output, pixel area tile, and tcd tile into 400x400 (0.1x0.1 degree) windows, instead of the native
40000x1 pixel windows.
Then it calculates the per pixel value for each model output pixel and sums those values within each 0.1x0.1 degree
aggregated pixel.
It converts cumulative carbon gain to CO2 gain per year, converts cumulative CO2 flux to CO2 flux per year, and
converts cumulative gross CO2 emissions to gross CO2 emissions per year.
The user has to supply a tcd threshold for which forest pixels to include in the results.
sample command: python mp_aggregate_results_to_10_km.py -tcd 30 -t no_shifting_ag -sagg s3://gfw2-data/climate/carbon_model/10km_output_aggregation/biomass_soil/standard/20191107/net_flux_t_CO2e_per_year_biomass_soil_10km_tcd30_modelv1_1_2_std_2019_11_07.tif
'''


import multiprocessing
import aggregate_results_to_10_km
import subprocess
from functools import partial
import argparse
import os
import glob
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

def main():

    # Files to download for this script. 'true'/'false' says whether the input directory and pattern should be
    # changed for a sensitivity analysis. This does not need to change based on what run is being done;
    # this assignment should be true for all sensitivity analyses and the standard model.
    download_dict = {
             cn.gross_emis_all_gases_all_drivers_biomass_soil_dir: [cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil],
             cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types],
             cn.net_flux_dir: [cn.pattern_net_flux]
             }

    # Sole argument for the script: the tree cover density threshold (pixels below this will not be aggregated)
    parser = argparse.ArgumentParser(description='Create maps of model output at approximately 10 km resolution')
    parser.add_argument('--tcd-threshold', '-tcd', required=True,
                        help='Tree cover density threshold above which pixels will be included in the aggregation.')
    parser.add_argument('--model-type', '-t', required=True,
                        help='{}'.format(cn.model_type_arg_help))
    parser.add_argument('--std-net-flux-aggreg', '-sagg', required=False,
                        help='The s3 standard model net flux aggregated tif, for comparison with the sensitivity analysis map')
    args = parser.parse_args()
    thresh = args.tcd_threshold
    thresh = int(thresh)
    sensit_type = args.model_type
    std_net_flux = args.std_net_flux_aggreg

    # Checks whether the canopy cover argument is valid
    if thresh < 0 or thresh > 99:
        raise Exception('Invalid tcd. Please provide an integer between 0 and 99.')

    # Checks whether the sensitivity analysis argument is valid
    uu.check_sensit_type(sensit_type)

    # tile_id_list = ['00N_070W'] # test tiles
    tile_id_list = 'all'

    # # Pixel area tiles-- necessary for calculating sum of pixels for any set of tiles
    # uu.s3_flexible_download(cn.pixel_area_dir, cn.pattern_pixel_area, '.', sensit_type, tile_id_list)
    # # tree cover density tiles-- necessary for filtering sums by tcd
    # uu.s3_flexible_download(cn.tcd_dir, cn.pattern_tcd, '.', sensit_type, tile_id_list)

    print "Model outputs to process are:", download_dict

    # List of output directories. Modified later for sensitivity analysis.
    # Output pattern is determined later.
    output_dir_list = [cn.output_aggreg_dir]

    # If the model run isn't the standard one, the output directory is changed
    if sensit_type != 'std':
        print "Changing output directory and file name pattern based on sensitivity analysis"
        output_dir_list = uu.alter_dirs(sensit_type, output_dir_list)


    # Iterates through the types of tiles to be processed
    for dir, download_pattern in download_dict.items():

        download_pattern_name = download_pattern[0]

        print download_pattern_name

        # Downloads the model output tiles to be processed
        uu.s3_flexible_download(dir, download_pattern_name, '.', sensit_type, tile_id_list)

        # Renames the tiles according to the sensitivity analysis before creating dummy tiles.
        # The renaming function requires a whole tile name, so this passes a dummy time name that is then stripped a few
        # lines later.
        tile_id = '30N_090W'     # a dummy tile id (but it has to be a real tile id). It is removed later.
        output_pattern = uu.sensit_tile_rename(sensit_type, tile_id, download_pattern_name)
        pattern = output_pattern[9:-4]

        print pattern

        # Lists the tiles of the particular type that is being iterates through.
        # Excludes all intermediate files
        tile_list = uu.tile_list_spot_machine(".", "{}.tif".format(pattern))
        # from https://stackoverflow.com/questions/12666897/removing-an-item-from-list-matching-a-substring
        tile_list = [i for i in tile_list if not ('hanson_2013' in i)]
        tile_list = [i for i in tile_list if not ('rewindow' in i)]
        tile_list = [i for i in tile_list if not ('10km' in i)]

        # tile_list = ['00N_070W_cumul_gain_AGCO2_BGCO2_t_ha_all_forest_types_2001_15_biomass_swap.tif']  # test tiles

        print tile_list
        print "There are {} tiles to process".format(str(len(tile_list))) + "\n"
        print "Processing:", dir, "; ", pattern

        # Converts the 10x10 degree Hansen tiles that are in windows of 40000x1 pixels to windows of 400x400 pixels,
{'fraction_non_alphanumeric': 0.06843248663101605, 'fraction_numerical': 0.02289438502673797, 'mean_word_length': 3.6034615384615383, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '239008', 'n_tokens_mistral': 3745, 'n_tokens_neox': 3416, 'n_words': 1263}