<commit_before>import signal
import asyncio
import urllib.parse
import urllib.request
import json

from .core import Miner


def search(query=None, params=None, mine_ids=None, info_only=None, **kwargs):
    query = '(*:*)' if not query else query
    params = params if params else {}
    mine_ids = True if mine_ids else False
    info_only = True if info_only else info_only

    if not kwargs.get('loop'):
        loop = asyncio.get_event_loop()
    else:
        loop = kwargs['loop']
    miner = Miner(loop, **kwargs)

    if info_only:
        url = miner.make_url('/advancedsearch.php?')
        params = urllib.parse.urlencode(miner.get_search_params(query, params))
        resp = urllib.request.urlopen(url + params)
        j = json.loads(resp.read().decode('utf-8'))
        search_info = j['responseHeader']
        search_info['numFound'] = j.get('response', {}).get('numFound', 0)
        return search_info

    try:
        loop.add_signal_handler(signal.SIGINT, loop.stop)
    except RuntimeError:
        pass

    loop.run_until_complete(miner.search(query, params=params, mine_ids=mine_ids))


def mine_items(identifiers, params=None, callback=None, **kwargs):
    """Concurrently retrieve metadata from Archive.org items.

    Args:
        identifiers: An iterable yielding Archive.org item identifiers.
        response_callback: A callback function to call on each
            aiohttp.client.ClientResponse object successfully returned.
        max_tasks: The maximum number of tasks to run concurrently.
        max_retries: The maximum number of times to retry a given request.
        secure: A boolean indicating if a secure connection should be used or not.
        hosts: A list of hosts to cycle through randomly for each request.
    """
    if not kwargs.get('loop'):
        loop = asyncio.get_event_loop()
    else:
        loop = kwargs['loop']
    miner = Miner(loop, **kwargs)
    try:
        loop.add_signal_handler(signal.SIGINT, loop.stop)
    except RuntimeError:
        pass
    loop.run_until_complete(miner.mine_items(identifiers, callback))<commit_msg>Ignore RuntimeError exceptions.<commit_after>import signal
import asyncio
import urllib.parse
import urllib.request
import json

from .core import Miner


def search(query=None, params=None, mine_ids=None, info_only=None, **kwargs):
    query = '(*:*)' if not query else query
    params = params if params else {}
    mine_ids = True if mine_ids else False
    info_only = True if info_only else info_only

    if not kwargs.get('loop'):
        loop = asyncio.get_event_loop()
    else:
        loop = kwargs['loop']
    miner = Miner(loop, **kwargs)

    if info_only:
        url = miner.make_url('/advancedsearch.php?')
        params = urllib.parse.urlencode(miner.get_search_params(query, params))
        resp = urllib.request.urlopen(url + params)
        j = json.loads(resp.read().decode('utf-8'))
        search_info = j['responseHeader']
        search_info['numFound'] = j.get('response', {}).get('numFound', 0)
        return search_info

    try:
        loop.add_signal_handler(signal.SIGINT, loop.stop)
        loop.run_until_complete(miner.search(query, params=params, mine_ids=mine_ids))
    except RuntimeError:
        pass


def mine_items(identifiers, params=None, callback=None, **kwargs):
    """Concurrently retrieve metadata from Archive.org items.

    Args:
        identifiers: An iterable yielding Archive.org item identifiers.
        response_callback: A callback function to call on each
            aiohttp.client.ClientResponse object successfully returned.
        max_tasks: The maximum number of tasks to run concurrently.
        max_retries: The maximum number of times to retry a given request.
        secure: A boolean indicating if a secure connection should be used or not.
        hosts: A list of hosts to cycle through randomly for each request.
    """
    if not kwargs.get('loop'):
        loop = asyncio.get_event_loop()
    else:
        loop = kwargs['loop']
    miner = Miner(loop, **kwargs)
    try:
        loop.add_signal_handler(signal.SIGINT, loop.stop)
        loop.run_until_complete(miner.mine_items(identifiers, callback))
    except RuntimeError:
        pass
{'fraction_non_alphanumeric': 0.09723207948899928, 'fraction_numerical': 0.0009462976105985332, 'mean_word_length': 3.108843537414966, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '451119', 'n_tokens_mistral': 1239, 'n_tokens_neox': 1223, 'n_words': 357}