<commit_before>import argparse
import logging
import logging_common
import mail.reader as mail
import re
import twits.reader as twits

from data_extraction import doc_handler, email_handler, link_handler
from functools import partial
from multiprocessing.pool import ThreadPool
from rss.reader import parse_feed_by_name, sources as rss_sources
from util.write import write_data

file_name_getter = re.compile(r'(.+?)\.py')

logging_common.setup()
log = logging.getLogger('app')


def get_cli_args(scrappers=None):
    if scrappers is None:
        scrappers = get_scrappers()

    arg_parser = argparse.ArgumentParser()
    arg_parser.add_argument('--unicode-json', action='store_true')

    action_parsers = arg_parser.add_subparsers(dest='action', help='Actions')

    run_parser = action_parsers.add_parser('run', help='Run scrappers')
    run_parser.add_argument('names', nargs='+', choices=scrappers['all'])
    run_parser.add_argument('--no-replace-redirects', action='store_true',
        help='Disable redirects replacing inside email documents')

    action_parsers.add_parser('list', help='List scrappers')

    args = arg_parser.parse_args()
    return args


def parse_host_arg(value):
    matches = re.match(r'^((?:[\w-]+)|(?:\d+\.\d+\.\d+\.\d+))(?::(\d+))?$', value)
    if not matches:
        raise argparse.ArgumentTypeError('Wrong host value: %s' % value)

    return {
        'host': matches.group(1),
        'port': int(matches.group(2) or 0),
    }


def get_scrappers():
    scrappers = {'rss': rss_sources.get_source_names()}

    scrappers['all'] = ['all'] + sorted(['twitter', 'mail'] + scrappers['rss'])
    return scrappers


def run_scrappers(args, scrappers):
    log.measure_start('run_scrappers')
    log.info('Start scrappers run')

    names_set = set(args.names)
    if 'all' in names_set:
        rss_handlers = scrappers['rss']
    else:
        rss_handlers = set(scrappers['rss']).intersection(names_set)

    pool = ThreadPool()
    rss_result = pool.map_async(partial(_run_rss_handler, args), rss_handlers)

    twitter_result = None
    if 'twitter' in names_set or 'all' in names_set:
        twitter_result = pool.apply_async(partial(_run_twitter_handler, args))

    mail_result = None
    if 'mail' in names_set or 'all' in names_set:
        mail_result = pool.apply_async(partial(_run_mail_handler, args))

    rss_data = rss_result.get()
    log.info('Got RSS docs')

    twitter_data = twitter_result.get() if twitter_result else []<commit_msg>Disable Twitter in all<commit_after>import argparse
import logging
import logging_common
import mail.reader as mail
import re
import twits.reader as twits

from data_extraction import doc_handler, email_handler, link_handler
from functools import partial
from multiprocessing.pool import ThreadPool
from rss.reader import parse_feed_by_name, sources as rss_sources
from util.write import write_data

file_name_getter = re.compile(r'(.+?)\.py')

logging_common.setup()
log = logging.getLogger('app')


def get_cli_args(scrappers=None):
    if scrappers is None:
        scrappers = get_scrappers()

    arg_parser = argparse.ArgumentParser()
    arg_parser.add_argument('--unicode-json', action='store_true')

    action_parsers = arg_parser.add_subparsers(dest='action', help='Actions')

    run_parser = action_parsers.add_parser('run', help='Run scrappers')
    run_parser.add_argument('names', nargs='+', choices=scrappers['all'])
    run_parser.add_argument('--no-replace-redirects', action='store_true',
        help='Disable redirects replacing inside email documents')

    action_parsers.add_parser('list', help='List scrappers')

    args = arg_parser.parse_args()
    return args


def parse_host_arg(value):
    matches = re.match(r'^((?:[\w-]+)|(?:\d+\.\d+\.\d+\.\d+))(?::(\d+))?$', value)
    if not matches:
        raise argparse.ArgumentTypeError('Wrong host value: %s' % value)

    return {
        'host': matches.group(1),
        'port': int(matches.group(2) or 0),
    }


def get_scrappers():
    scrappers = {'rss': rss_sources.get_source_names()}

    scrappers['all'] = ['all'] + sorted(['twitter', 'mail'] + scrappers['rss'])
    return scrappers


def run_scrappers(args, scrappers):
    log.measure_start('run_scrappers')
    log.info('Start scrappers run')

    names_set = set(args.names)
    if 'all' in names_set:
        rss_handlers = scrappers['rss']
    else:
        rss_handlers = set(scrappers['rss']).intersection(names_set)

    pool = ThreadPool()
    rss_result = pool.map_async(partial(_run_rss_handler, args), rss_handlers)

    twitter_result = None
    if 'twitter' in names_set:
        twitter_result = pool.apply_async(partial(_run_twitter_handler, args))

    mail_result = None
    if 'mail' in names_set or 'all' in names_set:
        mail_result = pool.apply_async(partial(_run_mail_handler, args))

    rss_data = rss_result.get()
    log.info('Got RSS docs')

    twitter_data = twitter_result.get() if twitter_result else []
{'fraction_non_alphanumeric': 0.13140446051838456, 'fraction_numerical': 0.0012055455093429777, 'mean_word_length': 4.2289915966386555, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5680527', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1601, 'n_words': 351}