<commit_before>###

import multiprocessing
import non_mangrove_non_planted_WHRC_biomass_2000
import pandas as pd
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

biomass_tile_list = uu.tile_list(cn.WHRC_biomass_2000_unmasked_dir)
# biomass_tile_list = ['80N_020E', '00N_000E', '00N_020E', '00N_110E'] # test tiles: no mangrove or planted forest, mangrove only, planted forest only, mangrove and planted forest
# biomass_tile_list = ['80N_020E', '00N_020E', '00N_000E', '00N_110E'] # test tiles: no mangrove or planted forest, planted forest only, mangrove only, mangrove and planted forest
# biomass_tile_list = ['00N_000E']
print biomass_tile_list
print "There are {} tiles to process".format(str(len(biomass_tile_list)))

# For downloading all tiles in the input folders.
# Mangrove biomass and full-extent planted forests are used to mask out mangrove and planted forests from the natural forests.
download_list = [cn.mangrove_biomass_2000_dir, cn.annual_gain_AGC_planted_forest_unmasked_dir, cn.WHRC_biomass_2000_unmasked_dir]

# for input in download_list:
#     uu.s3_folder_download(input, '.')

# # For copying individual tiles to spot machine for testing
# for tile in biomass_tile_list:
#
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.mangrove_biomass_2000_dir, tile, cn.pattern_mangrove_biomass_2000), '.')  # mangrove aboveground biomass
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.annual_gain_AGC_planted_forest_unmasked_dir, tile, cn.pattern_annual_gain_AGC_planted_forest_unmasked), '.')  # planted forest extent (not masked by mangroves)
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.WHRC_biomass_2000_unmasked_dir, tile, cn.pattern_WHRC_biomass_2000_unmasked), '.')   # WHRC biomass 2000 not masked by anything

# For multiprocessing
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count/3)
pool.map(non_mangrove_non_planted_WHRC_biomass_2000.mask_biomass, biomass_tile_list)
pool.close()
pool.join()
<commit_msg>Creating biomass 2000 tiles with mangroves and planted forests masked out.<commit_after>###

import multiprocessing
import non_mangrove_non_planted_WHRC_biomass_2000
import pandas as pd
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

biomass_tile_list = uu.tile_list(cn.WHRC_biomass_2000_unmasked_dir)
# biomass_tile_list = ['80N_020E', '00N_000E', '00N_020E', '00N_110E'] # test tiles: no mangrove or planted forest, mangrove only, planted forest only, mangrove and planted forest
# biomass_tile_list = ['80N_020E', '00N_020E', '00N_000E', '00N_110E'] # test tiles: no mangrove or planted forest, planted forest only, mangrove only, mangrove and planted forest
# biomass_tile_list = ['00N_000E']
print biomass_tile_list
print "There are {} tiles to process".format(str(len(biomass_tile_list)))

# For downloading all tiles in the input folders.
# Mangrove biomass and full-extent planted forests are used to mask out mangrove and planted forests from the natural forests.
download_list = [cn.mangrove_biomass_2000_dir, cn.annual_gain_AGC_planted_forest_unmasked_dir, cn.WHRC_biomass_2000_unmasked_dir]

# for input in download_list:
#     uu.s3_folder_download(input, '.')

# # For copying individual tiles to spot machine for testing
# for tile in biomass_tile_list:
#
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.mangrove_biomass_2000_dir, tile, cn.pattern_mangrove_biomass_2000), '.')  # mangrove aboveground biomass
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.annual_gain_AGC_planted_forest_unmasked_dir, tile, cn.pattern_annual_gain_AGC_planted_forest_unmasked), '.')  # planted forest extent (not masked by mangroves)
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.WHRC_biomass_2000_unmasked_dir, tile, cn.pattern_WHRC_biomass_2000_unmasked), '.')   # WHRC biomass 2000 not masked by anything

# For multiprocessing
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count/2)
pool.map(non_mangrove_non_planted_WHRC_biomass_2000.mask_biomass, biomass_tile_list)
pool.close()
pool.join()

{'fraction_non_alphanumeric': 0.09943806498900562, 'fraction_numerical': 0.049352553139506476, 'mean_word_length': 6.753787878787879, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '134838', 'n_tokens_mistral': 1700, 'n_tokens_neox': 1442, 'n_words': 328}