<commit_before>import click
import logging
import pandas as pd

import mia

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("mia")

LOG_LEVELS = {
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING
}


@click.group()
@click.version_option(version='0.7.0', prog_name='mia')
@click.option('--log-level', default='info',
              type=click.Choice(['debug', 'info', 'warning']),
              help="Level of logging to report")
def cli(log_level):
    logger.setLevel(LOG_LEVELS[log_level])


###############################################################################
# MIA Reduction
###############################################################################


@cli.command()
@click.argument('image-directory', type=click.Path())
@click.argument('masks-directory', type=click.Path())
@click.option('--output-file', '-o', default=None,
              help="Name of csv file to write the results of the reduction to")
@click.option('--BIRADS-file', '-b', default=None,
              help="Name of csv file containing BI-RADS classes from dataset")
@click.option('--num-processes', default=2,
              help="Num of processes to use for the reduction.")
def reduction(image_directory, masks_directory, output_file, birads_file,
              num_processes):
    mia.reduction.run_reduction(image_directory, masks_directory, output_file,
                                birads_file, num_processes)


@cli.command()
@click.argument('csv-file', type=click.Path())
@click.argument('output-file', type=click.Path())
def feature_statistics(csv_file, output_file):
    mia.reduction.feature_statistics(csv_file, output_file)


@cli.command()
@click.argument('image-file', type=click.Path())
@click.argument('mask-file', required=False, type=click.Path())
def detect_blobs(image_file, mask_file):
    img, msk = mia.utils.preprocess_image(image_file, mask_file)
    blob_df = mia.features.blobs.detect_blobs(img, msk)
    mia.plotting.plot_blobs(img, blob_df[['x', 'y', 'radius']].as_matrix())


@cli.command()
@click.argument('image-file', type=click.Path())
@click.argument('mask-file', required=False, type=click.Path())
def detect_linear(image_file, mask_file):
    img, msk = mia.utils.preprocess_image(image_file, mask_file)
    _, line_image = mia.features.linear_structure.detect_linear(img, msk)
    mia.plotting.plot_linear_structure(img, line_image)


###############################################################################
# MIA Analysis
###############################################################################


@cli.group()
def analysis():
    pass


@analysis.command()
@click.argument('csv-file', type=click.Path())
@click.argument('columns', nargs=-1)
@click.option('--output-file', '-o', default=None,
              help="Name of output file to store results of analysis in")
def tSNE(csv_file, columns, output_file):
    data_frame = pd.DataFrame.from_csv(csv_file)
    data_frame = data_frame[list(columns)]
    fit_output = mia.analysis.tSNE(data_frame)
    fit_output.to_csv(output_file)<commit_msg>Add command for raw image reduction.<commit_after>import click
import logging
import pandas as pd

import mia

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("mia")

LOG_LEVELS = {
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING
}


@click.group()
@click.version_option(version='0.7.0', prog_name='mia')
@click.option('--log-level', default='info',
              type=click.Choice(['debug', 'info', 'warning']),
              help="Level of logging to report")
def cli(log_level):
    logger.setLevel(LOG_LEVELS[log_level])


###############################################################################
# MIA Reduction
###############################################################################


@cli.command()
@click.argument('image-directory', type=click.Path())
@click.argument('masks-directory', type=click.Path())
@click.option('--output-file', '-o', default=None,
              help="Name of csv file to write the results of the reduction to")
@click.option('--BIRADS-file', '-b', default=None,
              help="Name of csv file containing BI-RADS classes from dataset")
@click.option('--num-processes', default=2,
              help="Num of processes to use for the reduction.")
def reduction(image_directory, masks_directory, output_file, birads_file,
              num_processes):
    mia.reduction.run_reduction(image_directory, masks_directory, output_file,
                                birads_file, num_processes)


@cli.command()
@click.argument('csv-file', type=click.Path())
@click.argument('output-file', type=click.Path())
def feature_statistics(csv_file, output_file):
    mia.reduction.feature_statistics(csv_file, output_file)


@cli.command()
@click.argument('image-file', type=click.Path())
@click.argument('mask-file', required=False, type=click.Path())
def detect_blobs(image_file, mask_file):
    img, msk = mia.utils.preprocess_image(image_file, mask_file)
    blob_df = mia.features.blobs.detect_blobs(img, msk)
    mia.plotting.plot_blobs(img, blob_df[['x', 'y', 'radius']].as_matrix())


@cli.command()
@click.argument('image-file', type=click.Path())
@click.argument('mask-file', required=False, type=click.Path())
def detect_linear(image_file, mask_file):
    img, msk = mia.utils.preprocess_image(image_file, mask_file)
    _, line_image = mia.features.linear_structure.detect_linear(img, msk)
    mia.plotting.plot_linear_structure(img, line_image)


###############################################################################
# MIA Analysis
###############################################################################


@cli.group()
def analysis():
    pass


@analysis.command()
@click.argument('image-directory', type=click.Path())
@click.argument('masks-directory', type=click.Path())
def raw_reduction(image_directory, masks_directory):
    import numpy as np
    from skimage import transform
    import re
    import os.path
    from convolve_tools import deformable_covolution
    rname = re.compile(r"p(\d{3}-\d{3}-\d{5})-([a-z]{2})\.png")
    feature_matrix = []

    kernel = mia.features.blobs._log_kernel(8.0)
    for img_path, msk_path in mia.io_tools.iterate_directory(image_directory,
                                                             masks_directory):
        print img_path
        name = os.path.basename(img_path)
        img, msk = mia.utils.preprocess_image(img_path, msk_path)
        # flip images so they follow the same orientation
        if 'r' in re.match(rname, name).group(2):
            img = np.fliplr(img)
            msk = np.fliplr(msk)
        img = transform.resize(img, (3328, 2560))
        msk = transform.resize(msk, (3328, 2560))
        # img = transform.pyramid_reduce(img, 4)
        img = transform.pyramid_reduce(img, np.sqrt(2)*7)
        msk = transform.pyramid_reduce(msk, np.sqrt(2)*7)
        msk[msk < 1] = 0
        msk[msk == 1] = 1
        img = img * msk

        img = -deformable_covolution(img, msk, kernel)
        img = img.flatten()

        feature_matrix.append(img)

    feature_matrix = np.vstack(feature_matrix)
    np.save('log_images.npy', feature_matrix)


@analysis.command()
@click.argument('csv-file', type=click.Path())
@click.argument('columns', nargs=-1)
@click.option('--output-file', '-o', default=None,
              help="Name of output file to store results of analysis in")
def tSNE(csv_file, columns, output_file):
    data_frame = pd.DataFrame.from_csv(csv_file)
    data_frame = data_frame[list(columns)]
    fit_output = mia.analysis.tSNE(data_frame)
    fit_output.to_csv(output_file)
{'fraction_non_alphanumeric': 0.215892833918585, 'fraction_numerical': 0.005462348809988295, 'mean_word_length': 4.310773480662983, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2588136', 'n_tokens_mistral': 2438, 'n_tokens_neox': 2342, 'n_words': 449}