<commit_before>### Creates tiles in which each natural non-mangrove forest pixel is the number of years that trees are believed to have been growing there between 2001 and 2015.
### It is based on the annual Hansen loss data and the 2000-2012 Hansen gain data (as well as the 2000 tree cover density data).
### First it calculates rasters of gain years for natural forest pixels that had loss only, gain only, neither loss nor gain, and both loss and gain.
### The gain years for each of these conditions are calculated according to rules that are found in the function called by the multiprocessor command.
### Then it combines those four rasters into a single gain year raster for each tile.
### This is one of the natural forest inputs for the carbon gain model.
### If different input rasters for loss (e.g., 2001-2017) and gain (e.g., 2000-2018) are used, the constants in create_gain_year_count_natrl_forest.py must be changed.

import multiprocessing
import utilities
import gain_year_count_natrl_forest


# biomass_tile_list = utilities.tile_list(utilities.biomass_dir)
# biomass_tile_list = ["00N_000E", "00N_050W", "00N_060W", "00N_010E", "00N_020E", "00N_030E", "00N_040E", "10N_000E", "10N_010E", "10N_010W", "10N_020E", "10N_020W"] # test tiles
biomass_tile_list = ['10N_080W'] # test tile
print biomass_tile_list

# # For downloading all tiles in the folders
# utilities.s3_folder_download('{}'.format(utilities.loss_dir), '.')
# utilities.s3_folder_download('{}'.format(utilities.gain_dir), '.')
# utilities.s3_folder_download('{}'.format(utilities.tcd_dir), '.')
# utilities.s3_folder_download('{}'.format(utilities.mangrove_biomass_dir), '.')

# For copying individual tiles to s3 for testing
for tile in biomass_tile_list:

    utilities.s3_file_download('{0}{1}.tif'.format(utilities.loss_dir, tile), '.')
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(utilities.gain_dir, utilities.pattern_gain, tile), '.')
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(utilities.tcd_dir, utilities.pattern_tcd, tile), '.')
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(utilities.mangrove_biomass_dir, utilities.pattern_mangrove_biomass, tile), '.')

# count = multiprocessing.cpu_count()
# pool = multiprocessing.Pool(count/10)
# pool.map(gain_year_count_natrl_forest.create_gain_year_count, biomass_tile_list)

# For single processor use
for tile in biomass_tile_list:

    gain_year_count_natrl_forest.create_gain_year_count(tile)<commit_msg>Redoing gain year count for non-mangrove natural forests. Need to mask mangrove pixels (exclude mangrove pixels). Testing the masking.<commit_after>### Creates tiles in which each natural non-mangrove forest pixel is the number of years that trees are believed to have been growing there between 2001 and 2015.
### It is based on the annual Hansen loss data and the 2000-2012 Hansen gain data (as well as the 2000 tree cover density data).
### First it calculates rasters of gain years for natural forest pixels that had loss only, gain only, neither loss nor gain, and both loss and gain.
### The gain years for each of these conditions are calculated according to rules that are found in the function called by the multiprocessor command.
### Then it combines those four rasters into a single gain year raster for each tile.
### This is one of the natural forest inputs for the carbon gain model.
### If different input rasters for loss (e.g., 2001-2017) and gain (e.g., 2000-2018) are used, the constants in create_gain_year_count_natrl_forest.py must be changed.

import multiprocessing
import utilities
import gain_year_count_natrl_forest


# biomass_tile_list = utilities.tile_list(utilities.biomass_dir)
# biomass_tile_list = ["00N_000E", "00N_050W", "00N_060W", "00N_010E", "00N_020E", "00N_030E", "00N_040E", "10N_000E", "10N_010E", "10N_010W", "10N_020E", "10N_020W"] # test tiles
biomass_tile_list = ['10N_080W'] # test tile
print biomass_tile_list

# # For downloading all tiles in the folders
# utilities.s3_folder_download('{}'.format(utilities.loss_dir), '.')
# utilities.s3_folder_download('{}'.format(utilities.gain_dir), '.')
# utilities.s3_folder_download('{}'.format(utilities.tcd_dir), '.')
# utilities.s3_folder_download('{}'.format(utilities.mangrove_biomass_dir), '.')

# # For copying individual tiles to s3 for testing
# for tile in biomass_tile_list:
#
#     utilities.s3_file_download('{0}{1}.tif'.format(utilities.loss_dir, tile), '.')
#     utilities.s3_file_download('{0}{1}_{2}.tif'.format(utilities.gain_dir, utilities.pattern_gain, tile), '.')
#     utilities.s3_file_download('{0}{1}_{2}.tif'.format(utilities.tcd_dir, utilities.pattern_tcd, tile), '.')
#     utilities.s3_file_download('{0}{1}_{2}.tif'.format(utilities.mangrove_biomass_dir, utilities.pattern_mangrove_biomass, tile), '.')

# count = multiprocessing.cpu_count()
# pool = multiprocessing.Pool(count/10)
# pool.map(gain_year_count_natrl_forest.create_gain_year_count, biomass_tile_list)

# For single processor use
for tile in biomass_tile_list:

    gain_year_count_natrl_forest.create_gain_year_count(tile)
{'fraction_non_alphanumeric': 0.11349331235247836, 'fraction_numerical': 0.04838709677419355, 'mean_word_length': 5.899592944369064, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3219621', 'n_tokens_mistral': 1861, 'n_tokens_neox': 1626, 'n_words': 475}