<commit_before>try: # Python 2
    from cStringIO import StringIO as BytesIO
except ImportError: # Python 3
    from io import BytesIO
import numpy as np
import warnings

import xray
from xray.backends.common import AbstractWritableDataStore
from xray.conventions import (is_valid_nc3_name, coerce_nc3_dtype,
                              encode_cf_variable)
from xray.utils import Frozen
from xray.pycompat import iteritems, basestring, unicode_type

def _encode_unicode_data(data):
    # Scipy's NetCDF 3 does not support unicode types
    if data.dtype.kind == 'U':
        return np.core.defchararray.encode(data, 'utf-8')
    return data

def _decode_string_data(data):
    # Convert all strings to unicode when loading
    if data.dtype.kind == 'S':
        return np.core.defchararray.decode(data, 'utf-8', 'replace')
    return data

class ScipyDataStore(AbstractWritableDataStore):
    """Store for reading and writing data via scipy.io.netcdf.

    This store has the advantage of being able to be initialized with a
    StringIO object, allow for serialization without writing to disk.

    It only supports the NetCDF3 file-format.
    """
    def __init__(self, filename_or_obj, mode='r', mmap=None, version=1):
        import scipy
        if mode != 'r' and scipy.__version__ < '0.13':
            warnings.warn('scipy %s detected; '
                          'the minimal recommended version is 0.13. '
                          'Older version of this library do not reliably '
                          'read and write files.'
                          % scipy.__version__, ImportWarning)

        import scipy.io
        # if filename is a NetCDF3 bytestring we store it in a StringIO
        if (isinstance(filename_or_obj, basestring)
            and filename_or_obj.startswith('CDF')):
            # TODO: this check has the unfortunate side-effect that
            # paths to files cannot start with 'CDF'.
            filename_or_obj = BytesIO(filename_or_obj)
        self.ds = scipy.io.netcdf.netcdf_file(
            filename_or_obj, mode=mode, mmap=mmap, version=version)

    def open_store_variable(self, var):
        return xray.Variable(var.dimensions, _decode_string_data(var.data), var._attributes)

    @property
    def attrs(self):
        return Frozen(self.ds._attributes)

    @property
    def dimensions(self):
        return Frozen(self.ds.dimensions)

    def set_dimension(self, name, length):
        if name in self.dimensions:
            raise ValueError('%s does not support modifying dimensions'
                             % type(self).__name__)
        self.ds.createDimension(name, length)

    def _validate_attr_key(self, key):
        if not is_valid_nc3_name(key):
            raise ValueError("Not a valid attribute name")

    def _cast_attr_value(self, value):
        if isinstance(value, basestring):
            if not isinstance(value, unicode_type):
                value = value.decode('utf-8')<commit_msg>Decode attribute values when loading with scipy<commit_after>from collections import OrderedDict
try: # Python 2
    from cStringIO import StringIO as BytesIO
except ImportError: # Python 3
    from io import BytesIO
import numpy as np
import warnings

import xray
from xray.backends.common import AbstractWritableDataStore
from xray.conventions import (is_valid_nc3_name, coerce_nc3_dtype,
                              encode_cf_variable)
from xray.utils import Frozen
from xray.pycompat import iteritems, basestring, unicode_type

def _encode_unicode_data(data):
    # Scipy's NetCDF 3 does not support unicode types
    if data.dtype.kind == 'U':
        return np.core.defchararray.encode(data, 'utf-8')
    return data

def _decode_string_data(data):
    # Convert all strings to unicode when loading
    if data.dtype.kind == 'S':
        return np.core.defchararray.decode(data, 'utf-8', 'replace')
    return data

def _decode_string(s):
    if isinstance(s, bytes):
        return s.decode('utf-8', 'replace')
    return s

class ScipyDataStore(AbstractWritableDataStore):
    """Store for reading and writing data via scipy.io.netcdf.

    This store has the advantage of being able to be initialized with a
    StringIO object, allow for serialization without writing to disk.

    It only supports the NetCDF3 file-format.
    """
    def __init__(self, filename_or_obj, mode='r', mmap=None, version=1):
        import scipy
        if mode != 'r' and scipy.__version__ < '0.13':
            warnings.warn('scipy %s detected; '
                          'the minimal recommended version is 0.13. '
                          'Older version of this library do not reliably '
                          'read and write files.'
                          % scipy.__version__, ImportWarning)

        import scipy.io
        # if filename is a NetCDF3 bytestring we store it in a StringIO
        if (isinstance(filename_or_obj, basestring)
            and filename_or_obj.startswith('CDF')):
            # TODO: this check has the unfortunate side-effect that
            # paths to files cannot start with 'CDF'.
            filename_or_obj = BytesIO(filename_or_obj)
        self.ds = scipy.io.netcdf.netcdf_file(
            filename_or_obj, mode=mode, mmap=mmap, version=version)

    def open_store_variable(self, var):
        return xray.Variable(var.dimensions, _decode_string_data(var.data), \
            OrderedDict((k,_decode_string(v)) for (k,v) in iteritems(var._attributes)))

    @property
    def attrs(self):
        return Frozen(OrderedDict((k, _decode_string(v)) \
                for (k,v) in iteritems(self.ds._attributes)))

    @property
    def dimensions(self):
        return Frozen(self.ds.dimensions)

    def set_dimension(self, name, length):
        if name in self.dimensions:
            raise ValueError('%s does not support modifying dimensions'
                             % type(self).__name__)
        self.ds.createDimension(name, length)

    def _validate_attr_key(self, key):
        if not is_valid_nc3_name(key):
            raise ValueError("Not a valid attribute name")

    def _cast_attr_value(self, value):
        if isinstance(value, basestring):
            if not isinstance(value, unicode_type):
                value = value.decode('utf-8')
{'fraction_non_alphanumeric': 0.08186016881669056, 'fraction_numerical': 0.00589265806657111, 'mean_word_length': 2.5824301197946378, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1646651', 'n_tokens_mistral': 1849, 'n_tokens_neox': 1811, 'n_words': 544}