<commit_before># Copyright (c) 2013-2017 University Corporation for Atmospheric Research/Unidata.
# Distributed under the terms of the MIT License.
# SPDX-License-Identifier: MIT
"""
Code to support reading and parsing catalog files from a THREDDS Data Server (TDS).

They help identifying the latest dataset and finding proper URLs to access the data.
"""

from collections import OrderedDict
from datetime import datetime
import logging
import re
import xml.etree.ElementTree as ET
try:
    from urlparse import urljoin, urlparse
except ImportError:
    # Python 3
    from urllib.parse import urljoin, urlparse

from .http_util import create_http_session, urlopen
from .metadata import TDSCatalogMetadata

log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())  # Python 2.7 needs a handler set
log.setLevel(logging.ERROR)


class IndexableMapping(OrderedDict):
    """Extend ``OrderedDict`` to allow index-based access to values."""

    def __getitem__(self, item):
        """Return an item either by index or name."""
        try:
            item + ''  # Raises if item not a string
            return super(IndexableMapping, self).__getitem__(item)
        except TypeError:
            return list(self.values())[item]


class DatasetCollection(IndexableMapping):
    """Extend ``IndexableMapping`` to allow datetime-based filter queries."""

    default_regex = re.compile(r'(?P<year>\d{4})(?P<month>[01]\d)(?P<day>[012]\d)_'
                               r'(?P<hour>[012]\d)(?P<minute>[0-5]\d)')

    def _get_datasets_with_times(self, regex):
        # Set the default regex if we don't have one
        if regex is None:
            regex = self.default_regex
        else:
            regex = re.compile(regex)

        # Loop over the collection looking for keys that match our regex
        found_date = False
        for ds in self:
            match = regex.search(ds)

            # If we find one, make a datetime and yield it along with the value
            if match:
                found_date = True
                date_parts = match.groupdict()
                dt = datetime(int(date_parts.get('year', 0)), int(date_parts.get('month', 0)),
                              int(date_parts.get('day', 0)), int(date_parts.get('hour', 0)),
                              int(date_parts.get('minute', 0)),
                              int(date_parts.get('second', 0)),
                              int(date_parts.get('microsecond', 0)))
                yield dt, self[ds]<commit_msg>Change line 44 to include 0123 for day instead of 012. Change email<commit_after># Copyright (c) 2013-2017 University Corporation for Atmospheric Research/Unidata.
# Distributed under the terms of the MIT License.
# SPDX-License-Identifier: MIT
"""
Code to support reading and parsing catalog files from a THREDDS Data Server (TDS).

They help identifying the latest dataset and finding proper URLs to access the data.
"""

from collections import OrderedDict
from datetime import datetime
import logging
import re
import xml.etree.ElementTree as ET
try:
    from urlparse import urljoin, urlparse
except ImportError:
    # Python 3
    from urllib.parse import urljoin, urlparse

from .http_util import create_http_session, urlopen
from .metadata import TDSCatalogMetadata

log = logging.getLogger(__name__)
log.addHandler(logging.StreamHandler())  # Python 2.7 needs a handler set
log.setLevel(logging.ERROR)


class IndexableMapping(OrderedDict):
    """Extend ``OrderedDict`` to allow index-based access to values."""

    def __getitem__(self, item):
        """Return an item either by index or name."""
        try:
            item + ''  # Raises if item not a string
            return super(IndexableMapping, self).__getitem__(item)
        except TypeError:
            return list(self.values())[item]


class DatasetCollection(IndexableMapping):
    """Extend ``IndexableMapping`` to allow datetime-based filter queries."""

    default_regex = re.compile(r'(?P<year>\d{4})(?P<month>[01]\d)(?P<day>[0123]\d)_'
                               r'(?P<hour>[012]\d)(?P<minute>[0-5]\d)')

    def _get_datasets_with_times(self, regex):
        # Set the default regex if we don't have one
        if regex is None:
            regex = self.default_regex
        else:
            regex = re.compile(regex)

        # Loop over the collection looking for keys that match our regex
        found_date = False
        for ds in self:
            match = regex.search(ds)

            # If we find one, make a datetime and yield it along with the value
            if match:
                found_date = True
                date_parts = match.groupdict()
                dt = datetime(int(date_parts.get('year', 0)), int(date_parts.get('month', 0)),
                              int(date_parts.get('day', 0)), int(date_parts.get('hour', 0)),
                              int(date_parts.get('minute', 0)),
                              int(date_parts.get('second', 0)),
                              int(date_parts.get('microsecond', 0)))
                yield dt, self[ds]
{'fraction_non_alphanumeric': 0.10135267594589296, 'fraction_numerical': 0.013330719466771222, 'mean_word_length': 2.523480662983425, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4287612', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1370, 'n_words': 457}