<commit_before># -*- coding: utf-8 -*-

import sys
from pprint import pprint
import click

from dlstats import constants
from dlstats.fetchers import FETCHERS, FETCHERS_DATASETS
from dlstats import client

opt_fetcher = click.option('--fetcher', '-f', 
              required=True, type=click.Choice(FETCHERS.keys()), 
              help='Fetcher choice')

opt_dataset = click.option('--dataset', '-d', 
              required=False, 
              help='Run selected dataset only')

@click.group()
def cli():
    """Fetchers commands."""
    pass

@cli.command('list')
def cmd_list():
    """Show fetchers list"""
    print("----------------------------------------------------")    
    for key in FETCHERS.keys():                                
        print(key)
    print("----------------------------------------------------")

@cli.command('datasets')
@opt_fetcher
def cmd_dataset_list(fetcher):
    """Show datasets list"""

    datasets = FETCHERS_DATASETS.get(fetcher, None)
    ctx = client.Context()
    if not datasets:
        #TODO: translation
        ctx.log_error("Les datasets de ce fetcher ne sont pas définis.")
        return
        
    print("----------------------------------------------------")    
    for key in datasets.keys():                                
        print(key)
    print("----------------------------------------------------")

@cli.command('run', context_settings=client.DLSTATS_SETTINGS)
@client.opt_verbose
@client.opt_silent
@client.opt_debug
@client.opt_logger
@client.opt_logger_conf
@client.opt_mongo_url
@client.opt_es_url
@opt_fetcher<commit_msg>extends datasets list command<commit_after># -*- coding: utf-8 -*-

import sys
from pprint import pprint
import click

from dlstats import constants
from dlstats.fetchers import FETCHERS, FETCHERS_DATASETS
from dlstats import client

opt_fetcher = click.option('--fetcher', '-f', 
              required=True, type=click.Choice(FETCHERS.keys()), 
              help='Fetcher choice')

opt_dataset = click.option('--dataset', '-d', 
              required=False, 
              help='Run selected dataset only')

@click.group()
def cli():
    """Fetchers commands."""
    pass

@cli.command('list')
def cmd_list():
    """Show fetchers list"""
    print("----------------------------------------------------")    
    for key in FETCHERS.keys():                                
        print(key)
    print("----------------------------------------------------")

@cli.command('datasets')
@client.opt_verbose
@client.opt_debug
@client.opt_logger
@client.opt_logger_conf
@client.opt_mongo_url
@opt_fetcher
def cmd_dataset_list(fetcher, **kwargs):
    """Show datasets list"""

    ctx = client.Context(**kwargs)

    f = FETCHERS[fetcher](db=ctx.mongo_database(), es_client=ctx.es_client())

    if ctx.verbose:
        func_name = 'datasets_long_list'
    else:
        func_name = 'datasets_list'

    have_func = hasattr(f, func_name)

    if not have_func:
        ctx.log_error("not implemented %s() method in fetcher" % func_name)
        return

    datasets = getattr(f, func_name)()
    if not datasets:
        ctx.log_error("Not datasets for this fetcher")
        return
        
    if ctx.verbose:
        for key, name in datasets:
            print(key, name)
    else:
        for key in datasets:
            print(key)

@cli.command('run', context_settings=client.DLSTATS_SETTINGS)
@client.opt_verbose
@client.opt_silent
@client.opt_debug
@client.opt_logger
@client.opt_logger_conf
@client.opt_mongo_url
@client.opt_es_url
@opt_fetcher
{'fraction_non_alphanumeric': 0.19820073095305032, 'fraction_numerical': 0.000562271577171774, 'mean_word_length': 2.043627031650984, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7369013', 'n_tokens_mistral': 1037, 'n_tokens_neox': 1032, 'n_words': 238}