<commit_before>from cdo import Cdo
from tempfile import mkstemp
import os

from flyingpigeon.utils import drs_filename, get_variable, calc_grouping, sort_by_filename
from flyingpigeon import config

import logging
logger = logging.getLogger("PYWPS")

DIR_MASKS = config.masks_dir()
DIR_SHP = config.shapefiles_dir()


def countries():
    """
    :return: a list of all country codes.
    """
    countries = _COUNTRIES_.keys()
    # countries = ['DEU', 'FRA', 'GBR', 'ESP', 'ITA']
    countries.sort()
    return countries


def countries_longname():
    """
    :return: the long name of all countries.
    """
    longname = ''
    for country in countries():
        longname = longname + "%s : %s \n" % (country, _COUNTRIES_[country]['longname'])
    return longname


def masking(resource, sftlf, threshold=50, land_area=True, prefix=None):
    """
    Set land/sea areas to nan.

    :param resource: string path to netCDF resource
    :param sftlf: land_area fraction netCDF file
    :param threshold: Percentage of land area
    :param land_area: If True (default), sea areas will set to nan
    :param prefix:  prefix for filename. If prefix is not set, a filename will be created

    :returns str: path to netCDF file
    """
    cdo = Cdo()
    #######################
    # TODO check sftlf unit
    #######################
    th = threshold / 100.0

    ###################
    # generate the mask
    if land_area is True:
        mask = cdo.gtc(th, input=sftlf, output='mask.nc')
    else:
        # TODO: fix the operator stc
        mask = cdo.ltc(th, input=sftlf, output='mask.nc')

    # generate output filename
    if prefix is not None:
        nc_masked = prefix + '.nc'
    else:
        _, nc_masked = mkstemp(dir='.', suffix='.nc')

    #################
    # divide by mask

    _ = cdo.div(input=[resource, mask], output=nc_masked)

    return nc_masked
<commit_msg>update comment<commit_after>from cdo import Cdo
from tempfile import mkstemp
import os

from flyingpigeon.utils import drs_filename, get_variable, calc_grouping, sort_by_filename
from flyingpigeon import config

import logging
logger = logging.getLogger("PYWPS")

DIR_MASKS = config.masks_dir()
DIR_SHP = config.shapefiles_dir()


def countries():
    """
    :return: a list of all country codes.
    """
    countries = _COUNTRIES_.keys()
    # countries = ['DEU', 'FRA', 'GBR', 'ESP', 'ITA']
    countries.sort()
    return countries


def countries_longname():
    """
    :return: the long name of all countries.
    """
    longname = ''
    for country in countries():
        longname = longname + "%s : %s \n" % (country, _COUNTRIES_[country]['longname'])
    return longname


def masking(resource, sftlf, threshold=50, land_area=True, prefix=None):
    """
    Set land/sea areas to nan.

    :param resource: string path to netCDF resource
    :param sftlf: land_area fraction netCDF file
    :param threshold: Percentage of land area
    :param land_area: If True (default), sea areas will set to nan
    :param prefix:  prefix for filename. If prefix is not set, a filename will be created

    :returns str: path to netCDF file
    """
    cdo = Cdo()
    #######################
    # TODO check sftlf unit
    #######################
    th = threshold / 100.0

    ###################
    # generate the mask
    if land_area is True:
        mask = cdo.gtc(th, input=sftlf, output='mask.nc')
    else:
        # TODO: check the operator ltc/stc
        mask = cdo.ltc(th, input=sftlf, output='mask.nc')

    # generate output filename
    if prefix is not None:
        nc_masked = prefix + '.nc'
    else:
        _, nc_masked = mkstemp(dir='.', suffix='.nc')

    #################
    # divide by mask

    _ = cdo.div(input=[resource, mask], output=nc_masked)

    return nc_masked

{'fraction_non_alphanumeric': 0.1459374178280305, 'fraction_numerical': 0.003155403628714173, 'mean_word_length': 3.0859291084854994, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7183388', 'n_tokens_mistral': 1241, 'n_tokens_neox': 1190, 'n_words': 375}