<commit_before>from __future__ import print_function

import argparse
import os
import yaml
import sys

try:
    # py2
    from urllib import urlretrieve
    import urllib
except ImportError:
    # py3
    from urllib.request import urlretrieve
    import urllib.request

from lxml import etree


def main():
    parser = argparse.ArgumentParser(usage='Downloads GML files from a set of WFS service in a pseudo-paginated way using bounding boxes and combine them again to one file. The WFS services are specified in settings.py.')
    parser.add_argument('config', help='config file')
    parser.add_argument('--no-download', help='skip the download', action='store_true')
    parser.add_argument('--no-combine', help='skip the combine', action='store_true')

    args = parser.parse_args()

    with open(args.config) as f:
        config = yaml.load(f.read())

    if not args.no_download:
        download_files(config)

    if not args.no_combine:
        combine_files(config)


def download_files(config):

    if (sys.version_info > (3, 0)):
        # Python 3 code in this block
        opener = urllib.request.build_opener()
        opener.addheaders = [('User-agent', 'wfs-downloader/0.1')]
        urllib.request.install_opener(opener)
    else:
        # Python 2 code in this block
        urllib.URLopener.version = "wfs-downloader/0.1"

    west_range = list(arange(config['bbox']['west'], config['bbox']['east'], config['size']))
    south_range = list(arange(config['bbox']['south'], config['bbox']['north'], config['size']))

    for west in west_range:<commit_msg>Print help text by default (#4)<commit_after>from __future__ import print_function

import argparse
import os
import yaml
import sys

try:
    # py2
    from urllib import urlretrieve
    import urllib
except ImportError:
    # py3
    from urllib.request import urlretrieve
    import urllib.request

from lxml import etree


class Parser(argparse.ArgumentParser):

    def error(self, message):
        self.print_help()
        sys.exit(2)


def main():
    parser = Parser(usage='Downloads GML files from a set of WFS service in a pseudo-paginated '
                          'way using bounding boxes and combine them again to one file. The WFS '
                          'services are specified in settings.py.')
    parser.add_argument('config', help='config file', default=None)
    parser.add_argument('--no-download', help='skip the download', action='store_true')
    parser.add_argument('--no-combine', help='skip the combine', action='store_true')

    args = parser.parse_args()

    with open(args.config) as f:
        config = yaml.load(f.read())

    if not args.no_download:
        download_files(config)

    if not args.no_combine:
        combine_files(config)


def download_files(config):

    if (sys.version_info > (3, 0)):
        # Python 3 code in this block
        opener = urllib.request.build_opener()
        opener.addheaders = [('User-agent', 'wfs-downloader/0.1')]
        urllib.request.install_opener(opener)
    else:
        # Python 2 code in this block
        urllib.URLopener.version = "wfs-downloader/0.1"

    west_range = list(arange(config['bbox']['west'], config['bbox']['east'], config['size']))
    south_range = list(arange(config['bbox']['south'], config['bbox']['north'], config['size']))

    for west in west_range:
{'fraction_non_alphanumeric': 0.118558236520703, 'fraction_numerical': 0.0065534703604408695, 'mean_word_length': 3.288633461047254, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4121363', 'n_tokens_mistral': 1059, 'n_tokens_neox': 1023, 'n_words': 280}