<commit_before>import subprocess

def add_args(args, names, item):
    for name in names:
        value = name % item
        if value:
            args.append(value)

def make_args(item, default_user_agent, wpull_exe):
    # -----------------------------------------------------------------------
    # BASE ARGUMENTS
    # -----------------------------------------------------------------------
    user_agent = item.get('user_agent') or default_user_agent

    args = [wpull_exe,
        '-U', user_agent,
        '--header', 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        '--quiet',
        '-o', '%(item_dir)s/wpull.log' % item,
        '--database', '%(item_dir)s/wpull.db' % item,
        '--save-cookies', '%(cookie_jar)s' % item,
        '--no-check-certificate',
        '--delete-after',
        '--no-robots',
        '--page-requisites',
        '--no-parent',
        '--inet4-only',
        '--timeout', '20',
        '--tries', '3',
        '--waitretry', '5',
        '--warc-file', '%(item_dir)s/%(warc_file_base)s' % item,
        '--warc-max-size', '10737418240',
        '--warc-header', 'operator: Archive Team',
        '--warc-header', 'downloaded-by: ArchiveBot',
        '--warc-header', 'archivebot-job-ident: %(ident)s' % item,
        '--python-script', 'wpull_hooks.py'
    ]

    # -----------------------------------------------------------------------
    # !ao < FILE
    # -----------------------------------------------------------------------
    if 'source_url_file' in item:
        add_args(args, ['-i', '%(source_url_file)s'], item)
    else:
        add_args(args, ['%(url)s'], item)

    # -----------------------------------------------------------------------
    # RECURSIVE FETCH / HOST-SPANNING
    # -----------------------------------------------------------------------
    if item.get('recursive'):
        add_args(args, ['--recursive', '--level', '%(depth)s'], item)
        args.append('--span-hosts-allow')
        args.append('page-requisites')
    else:
        args.append('--span-hosts-allow')
        args.append('page-requisites')

    args.append('--span-hosts-allow')

    if item.get('recursive') and not item.get('no_offsite_links'):
        args.append('page-requisites,linked-pages')
    else:
        args.append('page-requisites')

    # -----------------------------------------------------------------------
    # PHANTOMJS CONFIGURATION
    # -----------------------------------------------------------------------
    if item.get('grabber') == 'phantomjs':
        item.log_output('Telling wpull to use PhantomJS.')

        phantomjs_args = [
            '--phantomjs',
            '--phantomjs-scroll', item.get('phantomjs_scroll'),
            '--phantomjs-wait', item.get('phantomjs_wait')
        ]<commit_msg>Remove duplicate span-hosts-allow argument.  #90.<commit_after>import subprocess

def add_args(args, names, item):
    for name in names:
        value = name % item
        if value:
            args.append(value)

def make_args(item, default_user_agent, wpull_exe):
    # -----------------------------------------------------------------------
    # BASE ARGUMENTS
    # -----------------------------------------------------------------------
    user_agent = item.get('user_agent') or default_user_agent

    args = [wpull_exe,
        '-U', user_agent,
        '--header', 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        '--quiet',
        '-o', '%(item_dir)s/wpull.log' % item,
        '--database', '%(item_dir)s/wpull.db' % item,
        '--save-cookies', '%(cookie_jar)s' % item,
        '--no-check-certificate',
        '--delete-after',
        '--no-robots',
        '--page-requisites',
        '--no-parent',
        '--inet4-only',
        '--timeout', '20',
        '--tries', '3',
        '--waitretry', '5',
        '--warc-file', '%(item_dir)s/%(warc_file_base)s' % item,
        '--warc-max-size', '10737418240',
        '--warc-header', 'operator: Archive Team',
        '--warc-header', 'downloaded-by: ArchiveBot',
        '--warc-header', 'archivebot-job-ident: %(ident)s' % item,
        '--python-script', 'wpull_hooks.py'
    ]

    # -----------------------------------------------------------------------
    # !ao < FILE
    # -----------------------------------------------------------------------
    if 'source_url_file' in item:
        add_args(args, ['-i', '%(source_url_file)s'], item)
    else:
        add_args(args, ['%(url)s'], item)

    # -----------------------------------------------------------------------
    # RECURSIVE FETCH / HOST-SPANNING
    # -----------------------------------------------------------------------
    if item.get('recursive'):
        add_args(args, ['--recursive', '--level', '%(depth)s'], item)

    args.append('--span-hosts-allow')

    if item.get('recursive') and not item.get('no_offsite_links'):
        args.append('page-requisites,linked-pages')
    else:
        args.append('page-requisites')

    # -----------------------------------------------------------------------
    # PHANTOMJS CONFIGURATION
    # -----------------------------------------------------------------------
    if item.get('grabber') == 'phantomjs':
        item.log_output('Telling wpull to use PhantomJS.')

        phantomjs_args = [
            '--phantomjs',
            '--phantomjs-scroll', item.get('phantomjs_scroll'),
            '--phantomjs-wait', item.get('phantomjs_wait')
        ]
{'fraction_non_alphanumeric': 0.3554588320638375, 'fraction_numerical': 0.007616974972796518, 'mean_word_length': 1.1893608574831283, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1866319', 'n_tokens_mistral': 1638, 'n_tokens_neox': 1558, 'n_words': 253}