<commit_before>import os
import re
import six
from collections import OrderedDict
import numpy
import math
from netCDF4 import Dataset, default_fillvals


NUMBER_REGEXP = re.compile('\d+')


def get_fill_value(dtype):
    if isinstance(dtype, six.string_types):
        dtype = numpy.dtype(dtype)
    return default_fillvals[dtype.str[1:]]


def get_fill_value_for_variable(variable):
    if hasattr(variable, 'fill_value'):
        return variable.fill_value
    elif hasattr(variable, '_FillValue'):
        return variable._FillValue
    elif hasattr(variable, 'missing_value'):
        return variable.missing_value
    return get_fill_value(variable.dtype)


def get_dtype_string(variable):
    dtype = str(variable.dtype)
    if "'" in dtype:
        dtype = dtype.split("'")[1]

    return dtype


def copy_dimension(source_dataset, target_dataset, name, overwrite=True, allow_unlimited=False):
    """
    Copies a dimension to a netCDF file, deleting it if it already exists (unless overwrite is false, in which case
    this raises an exception).
    Copies the size if the dimension is not unlimited.

    :param source_dataset: the source netCDF dataset
    :param target_dataset: the target netCDF to copy into.  Must be in edit / write mode.
    :param name: the name of the dimension
    :param overwrite: if true, overwrite the dimension if found in target.  May cause bad side effects if other variables
    :param allow_unlimited: if true, allow unlimited dimensions to remain unlimited
    depend on that dimension in the target.

    :return new dimension
    """

    if name in target_dataset.dimensions:
        if overwrite:
            del target_dataset.dimensions[name]
        else:
            raise Exception("Target dimension already exists, and overwrite is false")
    source_dimension = source_dataset.dimensions[name]
    if allow_unlimited and source_dimension.isunlimited():
        return target_dataset.createDimension(name, None)
    else:
        return target_dataset.createDimension(name, len(source_dimension))


def copy_variable(source_dataset, target_dataset, name, overwrite=True, **kwargs):
    """
    Copies a variable to a netCDF file, deleting it if it already exists (unless overwrite is false).
    Copies the required dimensions first, if they don't already exist.
    Copies the variable's attributes across.
    Raises exception if dimensions already exist and do not match the size required by this variable.

    :param source_dataset: the source netCDF dataset
    :param target_dataset: the target netCDF to copy into.  Must be in edit / write mode.
    :param name: the name of the variable
    :param overwrite: if true, overwrite the variable if it exists in the target
    """

    if name in target_dataset.variables:
        if overwrite:
            del target_dataset.variables[name]
        else:
            raise Exception("Target variable already exists, and overwrite is false")

    source_variable = source_dataset.variables[name]
    for dimension_name in source_variable.dimensions:
        if dimension_name in target_dataset.dimensions:
            source_dimension = source_dataset.dimensions[dimension_name]
            target_dimension = target_dataset.dimensions[dimension_name]
            if not (len(target_dimension) == len(source_dimension) or
                        target_dimension.isunlimited() == source_dimension.isunlimited()):
                raise Exception("Dimension already exists in target, but has different size")
        else:
            copy_dimension(source_dataset, target_dataset, dimension_name)
        if (dimension_name in source_dataset.variables and not dimension_name in target_dataset.variables
            and dimension_name != name):
            copy_variable(source_dataset, target_dataset, dimension_name)

    if 'fill_value' not in kwargs and source_variable.dtype != numpy.dtype('str'):
        kwargs['fill_value'] = get_fill_value_for_variable(source_variable)

    target_variable = target_dataset.createVariable(name, source_variable.dtype, source_variable.dimensions, **kwargs)
    target_variable[:] = source_variable[:]
    for attribute_name in source_variable.ncattrs():
        if not attribute_name in target_variable.ncattrs():
            target_variable.setncattr(attribute_name, source_variable.getncattr(attribute_name))


def copy_attributes(source, target, attribute_names, overwrite=True):
    """
    Copies attributes from source object to target object, overwriting if already exists.

    :param source: the source object (dataset, variable, etc)
    :param target: the target object
    :param attribute_names: tuple / list of attribute names to copy across
    """

    for attribute_name in attribute_names:
        # Skip protected attributes that must be set in other ways
        if attribute_name in ('_FillValue'):
            continue

        if hasattr(target, attribute_name) and not overwrite:
            raise Exception("Attribute already exists in target, but overwrite is false")
        setattr(target, attribute_name, getattr(source, attribute_name))<commit_msg>Added output variable name to copy_variable<commit_after>import os
import re
import six
from collections import OrderedDict
import numpy
import math
from netCDF4 import Dataset, default_fillvals


NUMBER_REGEXP = re.compile('\d+')


def get_fill_value(dtype):
    if isinstance(dtype, six.string_types):
        dtype = numpy.dtype(dtype)
    return default_fillvals[dtype.str[1:]]


def get_fill_value_for_variable(variable):
    if hasattr(variable, 'fill_value'):
        return variable.fill_value
    elif hasattr(variable, '_FillValue'):
        return variable._FillValue
    elif hasattr(variable, 'missing_value'):
        return variable.missing_value
    return get_fill_value(variable.dtype)


def get_dtype_string(variable):
    dtype = str(variable.dtype)
    if "'" in dtype:
        dtype = dtype.split("'")[1]

    return dtype


def copy_dimension(source_dataset, target_dataset, name, overwrite=True, allow_unlimited=False):
    """
    Copies a dimension to a netCDF file, deleting it if it already exists (unless overwrite is false, in which case
    this raises an exception).
    Copies the size if the dimension is not unlimited.

    :param source_dataset: the source netCDF dataset
    :param target_dataset: the target netCDF to copy into.  Must be in edit / write mode.
    :param name: the name of the dimension
    :param overwrite: if true, overwrite the dimension if found in target.  May cause bad side effects if other variables
    :param allow_unlimited: if true, allow unlimited dimensions to remain unlimited
    depend on that dimension in the target.

    :return new dimension
    """

    if name in target_dataset.dimensions:
        if overwrite:
            del target_dataset.dimensions[name]
        else:
            raise Exception("Target dimension already exists, and overwrite is false")
    source_dimension = source_dataset.dimensions[name]
    if allow_unlimited and source_dimension.isunlimited():
        return target_dataset.createDimension(name, None)
    else:
        return target_dataset.createDimension(name, len(source_dimension))


def copy_variable(source_dataset, target_dataset, name, out_name=None, overwrite=True, **kwargs):
    """
    Copies a variable to a netCDF file, deleting it if it already exists (unless overwrite is false).
    Copies the required dimensions first, if they don't already exist.
    Copies the variable's attributes across.
    Raises exception if dimensions already exist and do not match the size required by this variable.

    :param source_dataset: the source netCDF dataset
    :param target_dataset: the target netCDF to copy into.  Must be in edit / write mode.
    :param name: the name of the variable
    :param out_name: the output name of the variable
    :param overwrite: if true, overwrite the variable if it exists in the target
    """

    if not out_name:
        out_name = name

    if out_name in target_dataset.variables:
        if overwrite:
            del target_dataset.variables[out_name]
        else:
            raise Exception("Target variable already exists, and overwrite is false")

    source_variable = source_dataset.variables[name]
    for dimension_name in source_variable.dimensions:
        if dimension_name in target_dataset.dimensions:
            source_dimension = source_dataset.dimensions[dimension_name]
            target_dimension = target_dataset.dimensions[dimension_name]
            if not (len(target_dimension) == len(source_dimension) or
                        target_dimension.isunlimited() == source_dimension.isunlimited()):
                raise Exception("Dimension already exists in target, but has different size")
        else:
            copy_dimension(source_dataset, target_dataset, dimension_name)
        if (dimension_name in source_dataset.variables and not dimension_name in target_dataset.variables
            and dimension_name != name):
            copy_variable(source_dataset, target_dataset, dimension_name)

    if 'fill_value' not in kwargs and source_variable.dtype != numpy.dtype('str'):
        kwargs['fill_value'] = get_fill_value_for_variable(source_variable)

    target_variable = target_dataset.createVariable(out_name, source_variable.dtype, source_variable.dimensions, **kwargs)
    target_variable[:] = source_variable[:]
    for attribute_name in source_variable.ncattrs():
        if not attribute_name in target_variable.ncattrs():
            target_variable.setncattr(attribute_name, source_variable.getncattr(attribute_name))


def copy_attributes(source, target, attribute_names, overwrite=True):
    """
    Copies attributes from source object to target object, overwriting if already exists.

    :param source: the source object (dataset, variable, etc)
    :param target: the target object
    :param attribute_names: tuple / list of attribute names to copy across
    """

    for attribute_name in attribute_names:
        # Skip protected attributes that must be set in other ways
        if attribute_name in ('_FillValue'):
            continue

        if hasattr(target, attribute_name) and not overwrite:
            raise Exception("Attribute already exists in target, but overwrite is false")
        setattr(target, attribute_name, getattr(source, attribute_name))
{'fraction_non_alphanumeric': 0.06797235023041474, 'fraction_numerical': 0.000576036866359447, 'mean_word_length': 3.5568678915135608, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1750080', 'n_tokens_mistral': 2750, 'n_tokens_neox': 2680, 'n_words': 948}