<commit_before>#!/usr/bin/env python3

# update/gs1_ai.py - script to get GS1 application identifiers
#
# Copyright (C) 2019 <NAME>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA

"""This script downloads GS1 application identifiers from the GS1 web site."""

import datetime
import json
import re

import lxml.html
import requests


# the location of the GS1 application identifiers
download_url = 'https://www.gs1.org/standards/barcodes/application-identifiers'


def fetch_ais():
    """Download application identifiers frm the GS1 website."""
    response = requests.get(download_url)
    document = lxml.html.document_fromstring(response.content)
    element = document.findall('.//script[@type="application/ld+json"]')[1]
    for entry in json.loads(element.text)['@graph']:
        yield (
            entry['skos:prefLabel'].strip(),             # AI
            entry['gs1meta:formatAIvalue'].strip()[3:],  # format
            entry['gs1meta:requiresFNC1'],               # require FNC1
            [x['@value'] for x in entry['schema:name'] if x['@language'] == 'en'][0].strip(),
            [x['@value'] for x in entry['schema:description'] if x['@language'] == 'en'][0].strip())


def group_ai_ranges():
    """Combine downloaded application identifiers into ranges."""
    first = None
    prev = (None, ) * 5
    for value in sorted(fetch_ais()):
        if value[1:] != prev[1:]:
            if first:
                yield first, *prev
            first = value[0]
        prev = value
    yield first, *prev


if __name__ == '__main__':
    print('# generated from %s' % download_url)<commit_msg>Fix Python 3.6 issue in update script<commit_after>#!/usr/bin/env python3

# update/gs1_ai.py - script to get GS1 application identifiers
#
# Copyright (C) 2019 <NAME>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA

"""This script downloads GS1 application identifiers from the GS1 web site."""

import datetime
import json
import re

import lxml.html
import requests


# the location of the GS1 application identifiers
download_url = 'https://www.gs1.org/standards/barcodes/application-identifiers'


def fetch_ais():
    """Download application identifiers frm the GS1 website."""
    response = requests.get(download_url)
    document = lxml.html.document_fromstring(response.content)
    element = document.findall('.//script[@type="application/ld+json"]')[1]
    for entry in json.loads(element.text)['@graph']:
        yield (
            entry['skos:prefLabel'].strip(),             # AI
            entry['gs1meta:formatAIvalue'].strip()[3:],  # format
            entry['gs1meta:requiresFNC1'],               # require FNC1
            [x['@value'] for x in entry['schema:name'] if x['@language'] == 'en'][0].strip(),
            [x['@value'] for x in entry['schema:description'] if x['@language'] == 'en'][0].strip())


def group_ai_ranges():
    """Combine downloaded application identifiers into ranges."""
    first = None
    prev = (None, ) * 5
    for value in sorted(fetch_ais()):
        if value[1:] != prev[1:]:
            if first:
                yield (first, *prev)
            first = value[0]
        prev = value
    yield (first, *prev)


if __name__ == '__main__':
    print('# generated from %s' % download_url)
{'fraction_non_alphanumeric': 0.11358719169190827, 'fraction_numerical': 0.016443098225876245, 'mean_word_length': 3.344924812030075, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '759154', 'n_tokens_mistral': 1332, 'n_tokens_neox': 1265, 'n_words': 489}