<commit_before>### This script calculates the cumulative above and belowground carbon gain in mangrove forest pixels from 2001-2015.
### It multiplies the annual biomass gain rate by the number of years of gain by the biomass-to-carbon conversion.

import multiprocessing
import utilities
import cumulative_gain_mangrove

### Need to update and install some packages on spot machine before running
### sudo pip install rasterio --upgrade

# mangrove_biomass_tile_list = utilities.tile_list(utilities.mangrove_biomass_dir)
# biomass_tile_list = ['20S_110E', '30S_110E'] # test tiles
mangrove_biomass_tile_list = ['10S_080W'] # test tiles
print mangrove_biomass_tile_list

# # For downloading all tiles in the input folders
# download_list = [utilities.annual_gain_AGB_mangrove_dir, utilities.annual_gain_BGB_mangrove_dir, utilities.gain_year_count_mangrove_dir]
#
# for input in download_list:
#     utilities.s3_folder_download('{}'.format(input), '.')

# For copying individual tiles to spot machine for testing
for tile in mangrove_biomass_tile_list:

    utilities.s3_file_download('{0}{1}_{2}.tif'.format(utilities.annual_gain_AGB_mangrove_dir, utilities.pattern_annual_gain_AGB_mangrove, tile), '.')           # annual AGB gain rate tiles
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(utilities.annual_gain_BGB_mangrove_dir, utilities.pattern_annual_gain_BGB_mangrove, tile), '.')  # annual AGB gain rate tiles
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(utilities.gain_year_count_mangrove_dir, utilities.pattern_gain_year_count_mangrove, tile), '.')        # number of years with gain tiles

count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count / 4)
pool.map(cumulative_gain_mangrove.cumulative_gain, mangrove_biomass_tile_list)

# # For single processor use
# for tile in mangrove_biomass_tile_list:
#
#     cumulative_gain_mangrove.cumulative_gain(tile)
<commit_msg>Working on cumulative gain for mangroves now.<commit_after>### This script calculates the cumulative above and belowground carbon gain in mangrove forest pixels from 2001-2015.
### It multiplies the annual biomass gain rate by the number of years of gain by the biomass-to-carbon conversion.

import multiprocessing
import utilities
import cumulative_gain_mangrove

### Need to update and install some packages on spot machine before running
### sudo pip install rasterio --upgrade

# mangrove_biomass_tile_list = utilities.tile_list(utilities.mangrove_biomass_dir)
# biomass_tile_list = ['20S_110E', '30S_110E'] # test tiles
mangrove_biomass_tile_list = ['10S_080W'] # test tiles
print mangrove_biomass_tile_list

# # For downloading all tiles in the input folders
# download_list = [utilities.annual_gain_AGB_mangrove_dir, utilities.annual_gain_BGB_mangrove_dir, utilities.gain_year_count_mangrove_dir]
#
# for input in download_list:
#     utilities.s3_folder_download('{}'.format(input), '.')

# For copying individual tiles to spot machine for testing
for tile in mangrove_biomass_tile_list:

    utilities.s3_file_download('{0}{1}_{2}.tif'.format(utilities.annual_gain_AGB_mangrove_dir, utilities.pattern_annual_gain_AGB_mangrove, tile), '.')           # annual AGB gain rate tiles
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(utilities.annual_gain_BGB_mangrove_dir, utilities.pattern_annual_gain_BGB_mangrove, tile), '.')  # annual AGB gain rate tiles
    utilities.s3_file_download('{0}{1}_{2}.tif'.format(utilities.gain_year_count_mangrove_dir, utilities.pattern_gain_year_count_mangrove, tile), '.')        # number of years with gain tiles

# count = multiprocessing.cpu_count()
# pool = multiprocessing.Pool(count / 4)
# pool.map(cumulative_gain_mangrove.cumulative_gain, mangrove_biomass_tile_list)

# For single processor use
for tile in mangrove_biomass_tile_list:

    cumulative_gain_mangrove.cumulative_gain(tile)

{'fraction_non_alphanumeric': 0.08824292758889177, 'fraction_numerical': 0.019205813651699974, 'mean_word_length': 6.299242424242424, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '815845', 'n_tokens_mistral': 1397, 'n_tokens_neox': 1242, 'n_words': 290}