<commit_before>"""Base class for ResourceSync capabilities with large lists of resources.

Handles possibly large lists by including support for both sitemaps and
sitemapindexes. Extends ListBase to add the support for sitemapindexes.
"""

import collections
import math
import os
from datetime import datetime
import re
import sys
import itertools

from .hashes import Hashes
from .list_base import ListBase
from .mapper import Mapper, MapperError
from .resource import Resource
from .sitemap import Sitemap
from .url_authority import UrlAuthority
from .url_or_file_open import url_or_file_open


class ListBaseWithIndex(ListBase):
    """Class that add handling of sitemapindexes to ListBase.

    Splitting of a list into multiple sitemaps with a sitemapindex is currently
    handled based solely on the number of entries in the list. The configurable
    self.max_sitemap_entries controls the number of entries that will be written
    in a single sitemap or a component sitemap that has a sitemapindex. Support
    for sitemapindexes can be disabled by setting allow_multifile False.

    resources - an iterable of resources

    count - add optional explicit setting of the number of items in
        resources which is useful when this is an iterator/generator.
        Is used instead of trying len(resources)

    md - metadata information for the list (<rs:md>)

    ln - link information for the list (<rs:ln>)

    allow_multifile - set False to disable support for indexes. Defaults to True

    mapper - Mapper instance used to map between file names and URIs so that
        the correct URIs can be written into a sitemapindex which correspond
        to those that the component sitemap files will be exposed as
    """

    def __init__(self, resources=None, count=None, md=None, ln=None, uri=None,
                 capability_name='unknown', allow_multifile=None, mapper=None,
                 spec_version='1.1', add_lastmod=False, resources_class=None):
        """Initialize ListBaseWithIndex."""
        self.resources_class = list if resources_class is None else resources_class
        if (resources is None):
            resources = self.resources_class()
        super(ListBaseWithIndex, self).__init__(
            resources=resources,
            count=count,
            md=md,
            ln=ln,
            uri=uri,
            capability_name=capability_name,
            spec_version=spec_version,
            add_lastmod=add_lastmod)
        # specific to lists with indexes
        self.max_sitemap_entries = 50000
        self.mapper = mapper
        self.allow_multifile = (
            True if (
                allow_multifile is None) else allow_multifile)
        self.check_url_authority = False
        self.content_length = 0
        self.num_files = 0            # Number of files read
        self.bytes_read = 0           # Aggregate of content_length values

    # INPUT

    def read(self, uri=None, resources=None, index_only=False):
        """Read sitemap from a URI including handling sitemapindexes.

        If index_only is True then individual sitemaps references in a sitemapindex
        will not be read. This will result in no resources being returned and is
        useful only to read the metadata and links listed in the sitemapindex.

        Includes the subtlety that if the input URI is a local file and is a
        sitemapindex which contains URIs for the individual sitemaps, then these
        are mapped to the filesystem also.
        """
        try:
            fh = url_or_file_open(uri)
            self.num_files += 1
        except IOError as e:
            raise IOError(
                "Failed to load sitemap/sitemapindex from %s (%s)" %
                (uri, str(e)))
        # Get the Content-Length if we can (works fine for local files)
        try:
            self.content_length = int(fh.info()['Content-Length'])
            self.bytes_read += self.content_length
            self.logger.debug(
                "Read %d bytes from %s" %
                (self.content_length, uri))
        except KeyError:
            # If we don't get a length then c'est la vie
            self.logger.debug("Read ????? bytes from %s" % (uri))
            pass
        self.logger.info("Read sitemap/sitemapindex from %s" % (uri))
        s = self.new_sitemap()
        s.parse_xml(fh=fh, resources=self, capability=self.capability_name)
        # what did we read? sitemap or sitemapindex?
        if (s.parsed_index):
            # sitemapindex
            if (not self.allow_multifile):
                raise ListBaseIndexError(
                    "Got sitemapindex from %s but support for sitemapindex disabled" %
                    (uri))
            self.logger.info(
                "Parsed as sitemapindex, %d sitemaps" %
                (len(
                    self.resources)))
            sitemapindex_is_file = self.is_file_uri(uri)
            if (index_only):
                # don't read the component sitemaps
                self.sitemapindex = True
                return
            # now loop over all entries to read each sitemap and add to
            # resources
            sitemaps = self.resources
            self.resources = self.resources_class()
            self.logger.info("Now reading %d sitemaps" % len(sitemaps.uris()))
            for sitemap_uri in sorted(sitemaps.uris()):
                self.read_component_sitemap(
                    uri, sitemap_uri, s, sitemapindex_is_file)
        else:
            # sitemap
            self.logger.info("Parsed as sitemap, %d resources" %
                             (len(self.resources)))

    def read_component_sitemap(
            self, sitemapindex_uri, sitemap_uri, sitemap, sitemapindex_is_file):
        """Read a component sitemap of a Resource List with index.

        Each component must be a sitemap with the
        """
        if (sitemapindex_is_file):
            if (not self.is_file_uri(sitemap_uri)):
                # Attempt to map URI to local file
                remote_uri = sitemap_uri
                sitemap_uri = self.mapper.src_to_dst(remote_uri)
                self.logger.info(
                    "Mapped %s to local file %s" %
                    (remote_uri, sitemap_uri))
            else:
                # The individual sitemaps should be at a URL (scheme/server/path)
                # that the sitemapindex URL can speak authoritatively about
                if (self.check_url_authority
                        and not UrlAuthority(sitemapindex_uri).has_authority_over(sitemap_uri)):
                    raise ListBaseIndexError(
                        "The sitemapindex (%s) refers to sitemap at a location it does not have authority over (%s)" %
                        (sitemapindex_uri, sitemap_uri))
        try:
            fh = url_or_file_open(sitemap_uri)
            self.num_files += 1
        except IOError as e:
            raise ListBaseIndexError(
                "Failed to load sitemap from %s listed in sitemap index %s (%s)" %
                (sitemap_uri, sitemapindex_uri, str(e)))
        # Get the Content-Length if we can (works fine for local files)
        try:
            self.content_length = int(fh.info()['Content-Length'])
            self.bytes_read += self.content_length
        except KeyError:
            # If we don't get a length then c'est la vie
            pass
        self.logger.info(
            "Reading sitemap from %s (%d bytes)" %
            (sitemap_uri, self.content_length))
        component = sitemap.parse_xml(fh=fh, sitemapindex=False)
        # Copy resources into self, check any metadata
        for r in component:
            self.resources.add(r)
        # FIXME - if rel="up" check it goes to correct place
        # FIXME - check capability

    # OUTPUT

    def requires_multifile(self):
        """Return False or the number of component sitemaps required.

        In the case that no len() is available for self.resources then
        then self.count must be set beforehand to avoid an exception.
        """
        if (self.max_sitemap_entries is None
                or len(self) <= self.max_sitemap_entries):
            return(False)
        return(int(math.ceil(len(self) / float(self.max_sitemap_entries))))

    def as_xml(self, allow_multifile=False, basename="/tmp/sitemap.xml"):
        """Return XML serialization of this list.

        If this list can be serialized as a single sitemap then the
        superclass method is used.<commit_msg>Handle missing Content-Length<commit_after>"""Base class for ResourceSync capabilities with large lists of resources.

Handles possibly large lists by including support for both sitemaps and
sitemapindexes. Extends ListBase to add the support for sitemapindexes.
"""

import collections
import math
import os
from datetime import datetime
import re
import sys
import itertools

from .hashes import Hashes
from .list_base import ListBase
from .mapper import Mapper, MapperError
from .resource import Resource
from .sitemap import Sitemap
from .url_authority import UrlAuthority
from .url_or_file_open import url_or_file_open


class ListBaseWithIndex(ListBase):
    """Class that add handling of sitemapindexes to ListBase.

    Splitting of a list into multiple sitemaps with a sitemapindex is currently
    handled based solely on the number of entries in the list. The configurable
    self.max_sitemap_entries controls the number of entries that will be written
    in a single sitemap or a component sitemap that has a sitemapindex. Support
    for sitemapindexes can be disabled by setting allow_multifile False.

    resources - an iterable of resources

    count - add optional explicit setting of the number of items in
        resources which is useful when this is an iterator/generator.
        Is used instead of trying len(resources)

    md - metadata information for the list (<rs:md>)

    ln - link information for the list (<rs:ln>)

    allow_multifile - set False to disable support for indexes. Defaults to True

    mapper - Mapper instance used to map between file names and URIs so that
        the correct URIs can be written into a sitemapindex which correspond
        to those that the component sitemap files will be exposed as
    """

    def __init__(self, resources=None, count=None, md=None, ln=None, uri=None,
                 capability_name='unknown', allow_multifile=None, mapper=None,
                 spec_version='1.1', add_lastmod=False, resources_class=None):
        """Initialize ListBaseWithIndex."""
        self.resources_class = list if resources_class is None else resources_class
        if (resources is None):
            resources = self.resources_class()
        super(ListBaseWithIndex, self).__init__(
            resources=resources,
            count=count,
            md=md,
            ln=ln,
            uri=uri,
            capability_name=capability_name,
            spec_version=spec_version,
            add_lastmod=add_lastmod)
        # specific to lists with indexes
        self.max_sitemap_entries = 50000
        self.mapper = mapper
        self.allow_multifile = (
            True if (
                allow_multifile is None) else allow_multifile)
        self.check_url_authority = False
        self.content_length = 0
        self.num_files = 0            # Number of files read
        self.bytes_read = 0           # Aggregate of content_length values

    # INPUT

    def read(self, uri=None, resources=None, index_only=False):
        """Read sitemap from a URI including handling sitemapindexes.

        If index_only is True then individual sitemaps references in a sitemapindex
        will not be read. This will result in no resources being returned and is
        useful only to read the metadata and links listed in the sitemapindex.

        Includes the subtlety that if the input URI is a local file and is a
        sitemapindex which contains URIs for the individual sitemaps, then these
        are mapped to the filesystem also.
        """
        try:
            fh = url_or_file_open(uri)
            self.num_files += 1
        except IOError as e:
            raise IOError(
                "Failed to load sitemap/sitemapindex from %s (%s)" %
                (uri, str(e)))
        # Get the Content-Length if we can (works fine for local files)
        try:
            self.content_length = int(fh.info()['Content-Length'])
            self.bytes_read += self.content_length
            self.logger.debug(
                "Read %d bytes from %s" %
                (self.content_length, uri))
        except (KeyError, TypeError):
            # If we don't get a length then c'est la vie
            self.logger.debug("Read ????? bytes from %s" % (uri))
            pass
        self.logger.info("Read sitemap/sitemapindex from %s" % (uri))
        s = self.new_sitemap()
        s.parse_xml(fh=fh, resources=self, capability=self.capability_name)
        # what did we read? sitemap or sitemapindex?
        if (s.parsed_index):
            # sitemapindex
            if (not self.allow_multifile):
                raise ListBaseIndexError(
                    "Got sitemapindex from %s but support for sitemapindex disabled" %
                    (uri))
            self.logger.info(
                "Parsed as sitemapindex, %d sitemaps" %
                (len(
                    self.resources)))
            sitemapindex_is_file = self.is_file_uri(uri)
            if (index_only):
                # don't read the component sitemaps
                self.sitemapindex = True
                return
            # now loop over all entries to read each sitemap and add to
            # resources
            sitemaps = self.resources
            self.resources = self.resources_class()
            self.logger.info("Now reading %d sitemaps" % len(sitemaps.uris()))
            for sitemap_uri in sorted(sitemaps.uris()):
                self.read_component_sitemap(
                    uri, sitemap_uri, s, sitemapindex_is_file)
        else:
            # sitemap
            self.logger.info("Parsed as sitemap, %d resources" %
                             (len(self.resources)))

    def read_component_sitemap(
            self, sitemapindex_uri, sitemap_uri, sitemap, sitemapindex_is_file):
        """Read a component sitemap of a Resource List with index.

        Each component must be a sitemap with the
        """
        if (sitemapindex_is_file):
            if (not self.is_file_uri(sitemap_uri)):
                # Attempt to map URI to local file
                remote_uri = sitemap_uri
                sitemap_uri = self.mapper.src_to_dst(remote_uri)
                self.logger.info(
                    "Mapped %s to local file %s" %
                    (remote_uri, sitemap_uri))
            else:
                # The individual sitemaps should be at a URL (scheme/server/path)
                # that the sitemapindex URL can speak authoritatively about
                if (self.check_url_authority
                        and not UrlAuthority(sitemapindex_uri).has_authority_over(sitemap_uri)):
                    raise ListBaseIndexError(
                        "The sitemapindex (%s) refers to sitemap at a location it does not have authority over (%s)" %
                        (sitemapindex_uri, sitemap_uri))
        try:
            fh = url_or_file_open(sitemap_uri)
            self.num_files += 1
        except IOError as e:
            raise ListBaseIndexError(
                "Failed to load sitemap from %s listed in sitemap index %s (%s)" %
                (sitemap_uri, sitemapindex_uri, str(e)))
        # Get the Content-Length if we can (works fine for local files)
        try:
            self.content_length = int(fh.info()['Content-Length'])
            self.bytes_read += self.content_length
        except (KeyError, TypeError):
            # If we don't get a length then c'est la vie
            pass
        self.logger.info(
            "Reading sitemap from %s (%d bytes)" %
            (sitemap_uri, self.content_length))
        component = sitemap.parse_xml(fh=fh, sitemapindex=False)
        # Copy resources into self, check any metadata
        for r in component:
            self.resources.add(r)
        # FIXME - if rel="up" check it goes to correct place
        # FIXME - check capability

    # OUTPUT

    def requires_multifile(self):
        """Return False or the number of component sitemaps required.

        In the case that no len() is available for self.resources then
        then self.count must be set beforehand to avoid an exception.
        """
        if (self.max_sitemap_entries is None
                or len(self) <= self.max_sitemap_entries):
            return(False)
        return(int(math.ceil(len(self) / float(self.max_sitemap_entries))))

    def as_xml(self, allow_multifile=False, basename="/tmp/sitemap.xml"):
        """Return XML serialization of this list.

        If this list can be serialized as a single sitemap then the
        superclass method is used.
{'fraction_non_alphanumeric': 0.06735599953265568, 'fraction_numerical': 0.0014020329477742728, 'mean_word_length': 2.0945408532176426, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6515355', 'n_tokens_mistral': 4770, 'n_tokens_neox': 4609, 'n_words': 1595}