<commit_before>"""Read and write ResourceSync documents as sitemaps"""

import re
import os
import sys
import logging
from xml.etree.ElementTree import ElementTree, Element, parse, tostring
import StringIO

from resource import Resource
from resource_container import ResourceContainer
from mapper import Mapper, MapperError

SITEMAP_NS = 'http://www.sitemaps.org/schemas/sitemap/0.9'
RS_NS = 'http://www.openarchives.org/rs/terms/'

class SitemapIndexError(Exception):
    """Exception on attempt to read a sitemapindex instead of sitemap 
    or vice-versa

    Provides both a message and a place to store the etree so that
    the parse tree may be reused as a sitemapingex.
    """

    def __init__(self, message=None, etree=None):
        self.message = message
        self.etree = etree

    def __repr__(self):
        return(self.message)

class SitemapParseError(Exception):
    pass

class SitemapDupeError(Exception):
    pass

class Sitemap(object):
    """Read and write sitemaps

    Implemented as a separate class that uses ResourceContainer 
    (ResourceList or ChangeList) and Resource classes as data objects. 
    Reads and write sitemaps, including multiple file sitemaps.

    This class does not automatically handle the reading or writing
    of a sitemapindex and multiple sitemap documents. Instead it will
    take a default and throw and exception if the other case is found
    so that the calling code can handle it.
    """

    def __init__(self, pretty_xml=False, allow_multifile=True, mapper=None):
        self.logger = logging.getLogger('sitemap')
        self.pretty_xml=pretty_xml
        self.allow_multifile=allow_multifile
        self.mapper=mapper
        self.max_sitemap_entries=50000
        # Classes used when parsing
        self.resource_class=Resource
        # Information recorded for logging
        self.resources_created=0    # Set during parsing sitemap
        self.parsed_index=None      # Set True for sitemapindex, False for sitemap


    ##### Write the XML for a sitemap or sitemapindex #####

    def resources_as_xml(self, resources, num_resources=None, sitemapindex=False, fh=None):
        """Return XML for a set of resources in sitemap format
        
        Arguments:
        - resources - either an iterable or iterator of Resource objects.
        - num_resources - if not None then only that number will be written
        - sitemapindex - set True to write sitemapindex instead of sitemap
        - fh - write to filehandle fh instead of returning string

        """
        # element names depending on sitemapindex or not
        root_element = ( 'sitemapindex' if (sitemapindex) else 'urlset' )
        item_element = ( 'sitemap' if (sitemapindex) else 'url')
        # namespaces and other settings
        namespaces = { 'xmlns': SITEMAP_NS, 'xmlns:rs': RS_NS }
        root = Element(root_element, namespaces)
        if (self.pretty_xml):
            root.text="\n"
        # <rs:md>
        if (hasattr(resources,'md')):
            self.add_md_to_etree(root,resources.md)
        # <rs:ln>
        if (hasattr(resources,'ln')):
            for ln in resources.ln:
                self.add_ln_to_etree(root,ln)
        # <url> entries from either an iterable or an iterator
        for r in resources:
            e=self.resource_etree_element(r, element_name=item_element)
            root.append(e)
            if (num_resources is not None):
                num_resources-=1
                if (num_resources==0):
                    break
        # have tree, now serialize
        tree = ElementTree(root);
        xml_buf=None
        if (fh is None):
            xml_buf=StringIO.StringIO()
            fh=xml_buf
        if (sys.version_info < (2,7)):
            tree.write(fh,encoding='UTF-8')
        else:
            tree.write(fh,encoding='UTF-8',xml_declaration=True,method='xml')
        if (xml_buf is not None):
            return(xml_buf.getvalue())
<commit_msg>Remove unused code to limit number of resource written in resources_as_xml()<commit_after>"""Read and write ResourceSync documents as sitemaps"""

import re
import os
import sys
import logging
from xml.etree.ElementTree import ElementTree, Element, parse, tostring
import StringIO

from resource import Resource
from resource_container import ResourceContainer
from mapper import Mapper, MapperError

SITEMAP_NS = 'http://www.sitemaps.org/schemas/sitemap/0.9'
RS_NS = 'http://www.openarchives.org/rs/terms/'

class SitemapIndexError(Exception):
    """Exception on attempt to read a sitemapindex instead of sitemap 
    or vice-versa

    Provides both a message and a place to store the etree so that
    the parse tree may be reused as a sitemapingex.
    """

    def __init__(self, message=None, etree=None):
        self.message = message
        self.etree = etree

    def __repr__(self):
        return(self.message)

class SitemapParseError(Exception):
    pass

class SitemapDupeError(Exception):
    pass

class Sitemap(object):
    """Read and write sitemaps

    Implemented as a separate class that uses ResourceContainer 
    (ResourceList or ChangeList) and Resource classes as data objects. 
    Reads and write sitemaps, including multiple file sitemaps.

    This class does not automatically handle the reading or writing
    of a sitemapindex and multiple sitemap documents. Instead it will
    take a default and throw and exception if the other case is found
    so that the calling code can handle it.
    """

    def __init__(self, pretty_xml=False, allow_multifile=True, mapper=None):
        self.logger = logging.getLogger('sitemap')
        self.pretty_xml=pretty_xml
        self.allow_multifile=allow_multifile
        self.mapper=mapper
        self.max_sitemap_entries=50000
        # Classes used when parsing
        self.resource_class=Resource
        # Information recorded for logging
        self.resources_created=0    # Set during parsing sitemap
        self.parsed_index=None      # Set True for sitemapindex, False for sitemap


    ##### Write the XML for a sitemap or sitemapindex #####

    def resources_as_xml(self, resources, sitemapindex=False, fh=None):
        """Return XML for a set of resources in sitemap format
        
        Arguments:
        - resources - either an iterable or iterator of Resource objects.
        - sitemapindex - set True to write sitemapindex instead of sitemap
        - fh - write to filehandle fh instead of returning string

        """
        # element names depending on sitemapindex or not
        root_element = ( 'sitemapindex' if (sitemapindex) else 'urlset' )
        item_element = ( 'sitemap' if (sitemapindex) else 'url')
        # namespaces and other settings
        namespaces = { 'xmlns': SITEMAP_NS, 'xmlns:rs': RS_NS }
        root = Element(root_element, namespaces)
        if (self.pretty_xml):
            root.text="\n"
        # <rs:md>
        if (hasattr(resources,'md')):
            self.add_md_to_etree(root,resources.md)
        # <rs:ln>
        if (hasattr(resources,'ln')):
            for ln in resources.ln:
                self.add_ln_to_etree(root,ln)
        # <url> entries from either an iterable or an iterator
        for r in resources:
            e=self.resource_etree_element(r, element_name=item_element)
            root.append(e)
        # have tree, now serialize
        tree = ElementTree(root);
        xml_buf=None
        if (fh is None):
            xml_buf=StringIO.StringIO()
            fh=xml_buf
        if (sys.version_info < (2,7)):
            tree.write(fh,encoding='UTF-8')
        else:
            tree.write(fh,encoding='UTF-8',xml_declaration=True,method='xml')
        if (xml_buf is not None):
            return(xml_buf.getvalue())

{'fraction_non_alphanumeric': 0.07958789439793947, 'fraction_numerical': 0.00334835801674179, 'mean_word_length': 2.6788252013263856, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 31}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3695372', 'n_tokens_mistral': 2288, 'n_tokens_neox': 2222, 'n_words': 705}