<commit_before>#!/usr/bin/env python

# getisbn.py - script to get ISBN prefix data
#
# Copyright (C) 2010, 2011, 2014 <NAME>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA

"""This script downloads XML data from the International ISBN Agency
website and provides a compact form of all group prefixes, and registrant
ranges for those prefixes suitable for the numdb module. This data is needed
to correctly split ISBNs into an EAN.UCC prefix, a group prefix, a registrant,
an item number and a check-digit."""

import xml.sax
import urllib


# The place where the current version of RangeMessage.xml can be downloaded.
download_url = 'https://www.isbn-international.org/export_rangemessage.xml'


def _wrap(text):
    """Generator that returns lines of text that are no longer than
    max_len."""
    while text:
        i = len(text)
        if i > 73:
            i = text.rindex(',', 20, 73)
        yield text[:i]
        text = text[i + 1:]


class RangeHandler(xml.sax.ContentHandler):

    def __init__(self):
        self._gather = None
        self._prefix = None
        self._agency = None
        self._range = None
        self._length = None
        self._ranges = []
        self._last = None
        self._topranges = {}

    def startElement(self, name, attrs):
        if name in ('MessageSerialNumber', 'MessageDate', 'Prefix',
                    'Agency', 'Range', 'Length'):
            self._gather = ''

    def characters(self, content):
        if self._gather is not None:
            self._gather += content

    def endElement(self, name):
        if name == 'MessageSerialNumber':
            print '# file serial %s' % self._gather.strip()
        elif name == 'MessageDate':
            print '# file date %s' % self._gather.strip()
        elif name == 'Prefix':
            self._prefix = self._gather.strip()
        elif name == 'Agency':
            self._agency = self._gather.strip()
        elif name == 'Range':
            self._range = self._gather.strip()
        elif name == 'Length':
            self._length = int(self._gather.strip())
        elif name == 'Rule' and self._length:
            self._ranges.append(tuple(x[:self._length]
                                      for x in self._range.split('-')))
        elif name == 'Rules':
            if '-' in self._prefix:
                p, a = self._prefix.split('-')
                if p != self._last:
                    print p
                    self._last = p
                    for line in _wrap(','.join(r[0] + '-' + r[1]
                                               for r in self._topranges[p])):
                        print ' %s' % line
                print (' %s agency="%s"' % (a, self._agency)).encode('utf-8')
                for line in _wrap(','.join(r[0] + '-' + r[1]
                                           for r in self._ranges)):
                    print '  %s' % line
            else:
                self._topranges[self._prefix] = self._ranges
            self._ranges = []
        self._gather = None


if __name__ == '__main__':
    print '# generated from RangeMessage.xml, downloaded from'
    print '# %s' % download_url
    parser = xml.sax.make_parser()
    parser.setContentHandler(RangeHandler())
    parser.parse(urllib.urlopen(download_url))
    #parser.parse('RangeMessage.xml')<commit_msg>Use ElementTree for simpler XML parsing<commit_after>#!/usr/bin/env python

# getisbn.py - script to get ISBN prefix data
#
# Copyright (C) 2010, 2011, 2014 <NAME>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA

"""This script downloads XML data from the International ISBN Agency
website and provides a compact form of all group prefixes, and registrant
ranges for those prefixes suitable for the numdb module. This data is needed
to correctly split ISBNs into an EAN.UCC prefix, a group prefix, a registrant,
an item number and a check-digit."""

from xml.etree import ElementTree
import urllib


# the location of the ISBN Ranges XML file
download_url = 'https://www.isbn-international.org/export_rangemessage.xml'


def ranges(group):
    for rule in group.find('Rules').findall('Rule'):
        length = int(rule.find('Length').text.strip())
        if length:
            yield '-'.join(
                x[:length]
                for x in rule.find('Range').text.strip().split('-'))


def wrap(text):
    while text:
        i = len(text)
        if i > 73:
            i = text.rindex(',', 20, 73)
        yield text[:i]
        text = text[i + 1:]


def get(f=None):
    if f is None:
        yield '# generated from RangeMessage.xml, downloaded from'
        yield '# %s' % download_url
        f = urllib.urlopen(download_url)
    else:
        yield '# generated from %r' % f

    # parse XML document
    msg = ElementTree.parse(f).getroot()

    # dump data from document
    yield '# file serial %s' % msg.find('MessageSerialNumber').text.strip()
    yield '# file date %s' % msg.find('MessageDate').text.strip()

    top_groups = dict(
        (x.find('Prefix').text.strip(), x)
        for x in msg.find('EAN.UCCPrefixes').findall('EAN.UCC'))

    prevtop = None
    for group in msg.find('RegistrationGroups').findall('Group'):
        top, prefix = group.find('Prefix').text.strip().split('-')
        agency = group.find('Agency').text.strip()
        if top != prevtop:
            yield top
            for line in wrap(','.join(ranges(top_groups[top]))):
                yield ' %s' % line
            prevtop = top
        yield ' %s agency="%s"' % (prefix, agency)
        for line in wrap(','.join(ranges(group))):
            yield '  %s' % line


if __name__ == '__main__':
    # get('RangeMessage.xml')
    for row in get():
        print row.encode('utf-8')
{'fraction_non_alphanumeric': 0.1102495418017764, 'fraction_numerical': 0.009868884816015791, 'mean_word_length': 2.3716730038022815, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '589454', 'n_tokens_mistral': 2054, 'n_tokens_neox': 1958, 'n_words': 728}