<commit_before>import argparse
import logging

from rsk_mind.dataset import Statistics

logging.basicConfig()
logger = logging.getLogger('rsk-mind')
logger.setLevel(logging.DEBUG)


def get_analytics(setting):
    logger.info('Reading datasset')
    DATASOURCE = setting.DATASOURCE
    datasource = DATASOURCE['IN']['class'](*DATASOURCE['IN']['params'])
    dataset = datasource.read()
    logger.info('Finish reading datasset')

    logger.info('Analysis on datasset')
    analytics = Statistics(dataset)
    logger.debug(analytics.statistics)
    logger.info('Finish analysis on datasset')

    if setting.ANALYSIS['persist']:
        logger.debug('Saving analysis on disk')
        logger.debug('Complete saving analysis on disk')
    else:
        logger.debug('Print on console analysis output')


def update_engine(setting):
    pass


def transformation(setting):
    DATASOURCE = setting.DATASOURCE

    datasource = DATASOURCE['IN']['class'](*DATASOURCE['IN']['params'])
    dataset = datasource.read()

    dataset.setTransformer(setting.TRANSFORMER())
    dataset.applyTransformations()

    datasource = DATASOURCE['OUT']['class'](*DATASOURCE['OUT']['params'])
    datasource.write(dataset)


def execute_from_command_line(argv, setting):
    argv = argv[1:]

    parser = argparse.ArgumentParser(description='')
    parser.add_argument('command', help='Command to execute!')
    params = parser.parse_args(argv)
<commit_msg>Save analysis on file<commit_after>import argparse, logging, json

from rsk_mind.dataset import Statistics

logging.basicConfig()
logger = logging.getLogger('rsk-mind')
logger.setLevel(logging.DEBUG)


def get_analytics(setting):

    logger.info('Reading dataset')
    DATASOURCE = setting.DATASOURCE
    datasource = DATASOURCE['IN']['class'](*DATASOURCE['IN']['params'])
    dataset = datasource.read()
    logger.info('Finish reading dataset')

    logger.info('Analysis on dataset')
    analytics = Statistics(dataset)
    logger.debug(analytics.statistics)
    logger.info('Finish analysis on dataset')

    if setting.ANALYSIS['persist']:
        logger.debug('Saving analysis on disk')
        with open(setting.ANALYSIS['out'], 'w') as out:
            json.dump(analytics.statistics, out, indent=4, sort_keys=True)
        logger.debug('Complete saving analysis on disk')
    else:
        logger.debug('Print on console analysis output')


def update_engine(setting):
    pass


def transformation(setting):
    DATASOURCE = setting.DATASOURCE

    datasource = DATASOURCE['IN']['class'](*DATASOURCE['IN']['params'])
    dataset = datasource.read()

    dataset.setTransformer(setting.TRANSFORMER())
    dataset.applyTransformations()

    datasource = DATASOURCE['OUT']['class'](*DATASOURCE['OUT']['params'])
    datasource.write(dataset)


def execute_from_command_line(argv, setting):
    argv = argv[1:]

    parser = argparse.ArgumentParser(description='')
    parser.add_argument('command', help='Command to execute!')
    params = parser.parse_args(argv)

{'fraction_non_alphanumeric': 0.132890365448505, 'fraction_numerical': 0.0009966777408637873, 'mean_word_length': 4.565619223659889, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '631452', 'n_tokens_mistral': 929, 'n_tokens_neox': 864, 'n_words': 174}