<commit_before>#!python
# coding=utf-8

import os
import tempfile
import operator
from glob import glob

import pytz
import pyncml
import netCDF4
import numpy as np

import logging
logger = logging.getLogger("pyaxiom")
logger.addHandler(logging.NullHandler())

try:
    from nco import Nco
except ImportError:
    logger.warning("NCO not found.  The NCO python bindings are required to use 'Collection.combine'.")


class DotDict(object):
    def __init__(self, *args, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)

    def __repr__(self):
        import pprint
        return pprint.pformat(vars(self), indent=2)


class Collection(object):

    @classmethod
    def from_ncml_file(cls, ncml_path, apply_to_members=None):
        try:
            with open(ncml_path) as f:
                return cls(pyncml.scan(f.read(), apply_to_members=apply_to_members))
        except BaseException:
            logger.exception("Could not load Collection from NcML.  Please check the NcML.")

    @classmethod
    def from_directory(cls, directory, suffix=".nc", subdirs=True, dimName='time', apply_to_members=None):

        if not os.path.isdir(directory):
            logger.error("Directory {0} does not exists or I do not have the correct permissions to access".format(directory))

        # Create NcML pointing to the directory
        ncml = """<?xml version="1.0" encoding="UTF-8"?>
                    <netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2">
                        <aggregation dimName="{0}" type="joinExisting">
                            <scan location="{1}" suffix="{2}" subdirs="{3}" />
                        </aggregation>
                    </netcdf>
               """.format(dimName, directory, suffix, subdirs)
        try:
            return cls(pyncml.scan(ncml, apply_to_members=apply_to_members))
        except BaseException:
            logger.exception("Could not load Collection from Directory.")

    @classmethod
    def from_glob(cls, glob_string, timevar_name='time', ncml=None):
        dataset_name      = None
        dataset_starting  = None
        dataset_ending    = None
        dataset_variables = []
        dataset_members   = []

        files = glob(glob_string)
        logger.info("Processing aggregation containing {!s} files".format(len(files)))
        for i, filepath in enumerate(files):
            logger.info("Processing member ({0}/{1}) - {2} ".format(i+1, len(files), filepath))
            nc = None
            try:
                if ncml is not None:
                    # Apply NcML
                    tmp_f, tmp_fp = tempfile.mkstemp(prefix="nc")
                    os.close(tmp_f)
                    nc = pyncml.apply(filepath, ncml, output_file=tmp_fp)
                else:
                    nc = netCDF4.Dataset(filepath)

                if dataset_name is None:
                    dataset_name = getattr(nc, 'name', getattr(nc, 'title', None))

                timevar = nc.variables.get(timevar_name)
                if timevar is None:
                    logger.error("Time variable '{0}' was not found in file '{1}'. Skipping.".format(timevar_name, filepath))
                    continue

                # Start/Stop of NetCDF file
                starting  = netCDF4.num2date(np.min(timevar[:]), units=timevar.units)
                ending    = netCDF4.num2date(np.max(timevar[:]), units=timevar.units)
                variables = filter(None, [ nc.variables[v].standard_name if hasattr(nc.variables[v], 'standard_name') else None for v in nc.variables.keys() ])

                dataset_variables = list(set(dataset_variables + variables))

                if starting.tzinfo is None:
                    starting = starting.replace(tzinfo=pytz.utc)
                if ending.tzinfo is None:
                    ending = ending.replace(tzinfo=pytz.utc)
                if dataset_starting is None or starting < dataset_starting:
                    dataset_starting = starting
                if dataset_ending is None or ending > dataset_ending:
                    dataset_ending = ending

                member = pyncml.DotDict(path=filepath, standard_names=variables, starting=starting, ending=ending)
                dataset_members.append(member)
            except BaseException:
                logger.exception("Something went wrong with {0}".format(filepath))
                continue
            finally:
                nc.close()
                try:
                    os.remove(tmp_fp)
                except (OSError, UnboundLocalError):
                    pass

        dataset_members = sorted(dataset_members, key=operator.attrgetter('starting'))
        return cls(pyncml.DotDict(name=dataset_name,
                                  timevar_name=timevar_name,
                                  starting=dataset_starting,
                                  ending=dataset_ending,
                                  standard_names=dataset_variables,
                                  members=dataset_members))

    @classmethod
    def combine(self, members, output_file, dimension=None, start_index=None, stop_index=None, stride=None):
        """ Combine many files into a single file on disk.  Defaults to using the 'time' dimension. """
        nco = None
        try:
            nco = Nco()
        except BaseException:
            raise ImportError("NCO not found.  The NCO python bindings are required to use 'Collection.combine'.")

        if len(members) > 0 and hasattr(members[0], 'path'):
            # A member DotDoct was passed in, we only need the paths
            members = [ m.path for m in members ]

        options  = ['-4']
        options += ['-L', '3']
        if dimension is not None:
            if start_index is None:
                start_index = 0
            if stop_index is None:
                stop_index = ''
            if stride is None:
                stride = 1
            options += ['-d', '{0},{1},{2},{3}'.format(dimension, start_index, stop_index, stride)]
        nco.ncrcat(input=members, output=output_file, options=options)

    def __init__(self, aggregation):
        self.aggregation = aggregation

    def bins(self, delta, starting, hard_start=None, hard_end=None):
        ending = starting + delta

        windows = []

        member_length = len(self.aggregation.members)
        last_member = self.aggregation.members[-1]
        index = 0<commit_msg>Don't append to the history attribute when combining files<commit_after>#!python
# coding=utf-8

import os
import tempfile
import operator
from glob import glob

import pytz
import pyncml
import netCDF4
import numpy as np

import logging
logger = logging.getLogger("pyaxiom")
logger.addHandler(logging.NullHandler())

try:
    from nco import Nco
except ImportError:
    logger.warning("NCO not found.  The NCO python bindings are required to use 'Collection.combine'.")


class DotDict(object):
    def __init__(self, *args, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)

    def __repr__(self):
        import pprint
        return pprint.pformat(vars(self), indent=2)


class Collection(object):

    @classmethod
    def from_ncml_file(cls, ncml_path, apply_to_members=None):
        try:
            with open(ncml_path) as f:
                return cls(pyncml.scan(f.read(), apply_to_members=apply_to_members))
        except BaseException:
            logger.exception("Could not load Collection from NcML.  Please check the NcML.")

    @classmethod
    def from_directory(cls, directory, suffix=".nc", subdirs=True, dimName='time', apply_to_members=None):

        if not os.path.isdir(directory):
            logger.error("Directory {0} does not exists or I do not have the correct permissions to access".format(directory))

        # Create NcML pointing to the directory
        ncml = """<?xml version="1.0" encoding="UTF-8"?>
                    <netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2">
                        <aggregation dimName="{0}" type="joinExisting">
                            <scan location="{1}" suffix="{2}" subdirs="{3}" />
                        </aggregation>
                    </netcdf>
               """.format(dimName, directory, suffix, subdirs)
        try:
            return cls(pyncml.scan(ncml, apply_to_members=apply_to_members))
        except BaseException:
            logger.exception("Could not load Collection from Directory.")

    @classmethod
    def from_glob(cls, glob_string, timevar_name='time', ncml=None):
        dataset_name      = None
        dataset_starting  = None
        dataset_ending    = None
        dataset_variables = []
        dataset_members   = []

        files = glob(glob_string)
        logger.info("Processing aggregation containing {!s} files".format(len(files)))
        for i, filepath in enumerate(files):
            logger.info("Processing member ({0}/{1}) - {2} ".format(i+1, len(files), filepath))
            nc = None
            try:
                if ncml is not None:
                    # Apply NcML
                    tmp_f, tmp_fp = tempfile.mkstemp(prefix="nc")
                    os.close(tmp_f)
                    nc = pyncml.apply(filepath, ncml, output_file=tmp_fp)
                else:
                    nc = netCDF4.Dataset(filepath)

                if dataset_name is None:
                    dataset_name = getattr(nc, 'name', getattr(nc, 'title', None))

                timevar = nc.variables.get(timevar_name)
                if timevar is None:
                    logger.error("Time variable '{0}' was not found in file '{1}'. Skipping.".format(timevar_name, filepath))
                    continue

                # Start/Stop of NetCDF file
                starting  = netCDF4.num2date(np.min(timevar[:]), units=timevar.units)
                ending    = netCDF4.num2date(np.max(timevar[:]), units=timevar.units)
                variables = filter(None, [ nc.variables[v].standard_name if hasattr(nc.variables[v], 'standard_name') else None for v in nc.variables.keys() ])

                dataset_variables = list(set(dataset_variables + variables))

                if starting.tzinfo is None:
                    starting = starting.replace(tzinfo=pytz.utc)
                if ending.tzinfo is None:
                    ending = ending.replace(tzinfo=pytz.utc)
                if dataset_starting is None or starting < dataset_starting:
                    dataset_starting = starting
                if dataset_ending is None or ending > dataset_ending:
                    dataset_ending = ending

                member = pyncml.DotDict(path=filepath, standard_names=variables, starting=starting, ending=ending)
                dataset_members.append(member)
            except BaseException:
                logger.exception("Something went wrong with {0}".format(filepath))
                continue
            finally:
                nc.close()
                try:
                    os.remove(tmp_fp)
                except (OSError, UnboundLocalError):
                    pass

        dataset_members = sorted(dataset_members, key=operator.attrgetter('starting'))
        return cls(pyncml.DotDict(name=dataset_name,
                                  timevar_name=timevar_name,
                                  starting=dataset_starting,
                                  ending=dataset_ending,
                                  standard_names=dataset_variables,
                                  members=dataset_members))

    @classmethod
    def combine(self, members, output_file, dimension=None, start_index=None, stop_index=None, stride=None):
        """ Combine many files into a single file on disk.  Defaults to using the 'time' dimension. """
        nco = None
        try:
            nco = Nco()
        except BaseException:
            raise ImportError("NCO not found.  The NCO python bindings are required to use 'Collection.combine'.")

        if len(members) > 0 and hasattr(members[0], 'path'):
            # A member DotDoct was passed in, we only need the paths
            members = [ m.path for m in members ]

        options  = ['-4']  # NetCDF4
        options += ['-L', '3']  # Level 3 compression
        options += ['-h']  # Don't append to the history global attribute
        if dimension is not None:
            if start_index is None:
                start_index = 0
            if stop_index is None:
                stop_index = ''
            if stride is None:
                stride = 1
            options += ['-d', '{0},{1},{2},{3}'.format(dimension, start_index, stop_index, stride)]
        nco.ncrcat(input=members, output=output_file, options=options)

    def __init__(self, aggregation):
        self.aggregation = aggregation

    def bins(self, delta, starting, hard_start=None, hard_end=None):
        ending = starting + delta

        windows = []

        member_length = len(self.aggregation.members)
        last_member = self.aggregation.members[-1]
        index = 0
{'fraction_non_alphanumeric': 0.09793303333079094, 'fraction_numerical': 0.005796659293722828, 'mean_word_length': 1.9518235029266096, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 2, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '650714', 'n_tokens_mistral': 3706, 'n_tokens_neox': 3500, 'n_words': 920}