<commit_before>#!/usr/bin/python3
# coding: utf-8

__author__ = "<NAME>"
__license__ = "MIT"
__copyright__ = "2015, ESRF"
__date__ = "20/04/2020"

import argparse
import os
import logging
import freesas
from freesas.cormap import gof
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("cormap")
import numpy
from itertools import combinations
from collections import namedtuple
datum = namedtuple("datum", ["index", "filename", "data"])

import platform
operatingSystem = platform.system()
if operatingSystem == "Windows":
    import glob


def parse():
    """ Parse input and return list of files.
    :return: list of input files
    """
    usage = "cormap.py FILES [OPTIONS]"
    description = "Measure pair-wise dimilarity of spectra "
    epilog = """cormap.py is an open-source implementation of
    the cormap algorithm in datcmp (from ATSAS).
    It does not scale the data and assume they are already scaled 
    """
    version = "autorg.py version %s from %s" % (freesas.version, freesas.date)
    parser = argparse.ArgumentParser(usage=usage, description=description, epilog=epilog)
    parser.add_argument("file", metavar="FILE", nargs='+', help="dat files to compare")
    parser.add_argument("-v", "--verbose", default=False, help="switch to verbose mode", action='store_true')
    parser.add_argument("-V", "--version", action='version', version=version)
    args = parser.parse_args()
    if args.verbose:
        logging.root.setLevel(logging.DEBUG)
    files = [i for i in args.file if os.path.exists(i)]
    if operatingSystem == "Windows" and files == []:
        files = glob.glob(args.file[0])
        files.sort()
    input_len = len(files)
    logger.debug("%s input files" % input_len)
    return files


def compare(lstfiles):
    res = ["Pair-wise Correlation Map",
           ""
           "                                C       Pr(>C)"]
    data = []
    for i, f in enumerate(lstfiles):
        try:
            ary = numpy.loadtxt(f)
        except ValueError as e:
            print(e)
        if ary.ndim > 1 and ary.shape[1] > 1:
            ary = ary[:, 1]
        d = datum(i + 1, f, ary)
        data.append(d)
    for a, b in combinations(data, 2):
        r = gof(a.data, b.data)
        res.append("%6i vs. %6i          %6i     %8.6f" % (a.index, b.index, r.c, r.P))
    res.append("")
    for a in data:
        res.append("%6i         %8f + %8f * %s" % (a.index, 0.0, 1.0, a.filename))
    res.append("")
    print(os.linesep.join(res))
    return res
<commit_msg>Implented extended data loader in cormap app...<commit_after>#!/usr/bin/python3
# coding: utf-8

__author__ = "<NAME>"
__license__ = "MIT"
__copyright__ = "2015, ESRF"
__date__ = "20/04/2020"

import argparse
import os
import logging
import freesas
from freesas.cormap import gof
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("cormap")
import numpy
from itertools import combinations
from collections import namedtuple
from freesas.sasio import load_scattering_data
datum = namedtuple("datum", ["index", "filename", "data"])

import platform
operatingSystem = platform.system()
if operatingSystem == "Windows":
    import glob


def parse():
    """ Parse input and return list of files.
    :return: list of input files
    """
    usage = "cormap.py FILES [OPTIONS]"
    description = "Measure pair-wise dimilarity of spectra "
    epilog = """cormap.py is an open-source implementation of
    the cormap algorithm in datcmp (from ATSAS).
    It does not scale the data and assume they are already scaled
    """
    version = "autorg.py version %s from %s" % (freesas.version, freesas.date)
    parser = argparse.ArgumentParser(usage=usage, description=description, epilog=epilog)
    parser.add_argument("file", metavar="FILE", nargs='+', help="dat files to compare")
    parser.add_argument("-v", "--verbose", default=False, help="switch to verbose mode", action='store_true')
    parser.add_argument("-V", "--version", action='version', version=version)
    args = parser.parse_args()
    if args.verbose:
        logging.root.setLevel(logging.DEBUG)
    files = [i for i in args.file if os.path.exists(i)]
    if operatingSystem == "Windows" and files == []:
        files = glob.glob(args.file[0])
        files.sort()
    input_len = len(files)
    logger.debug("%s input files" % input_len)
    return files


def compare(lstfiles):
    res = ["Pair-wise Correlation Map",
           ""
           "                                C       Pr(>C)"]
    data = []
    for i, f in enumerate(lstfiles):
        try:
            ary = load_scattering_data(f)
        except ValueError as e:
            print(e)
        if ary.ndim > 1 and ary.shape[1] > 1:
            ary = ary[:, 1]
        d = datum(i + 1, f, ary)
        data.append(d)
    for a, b in combinations(data, 2):
        r = gof(a.data, b.data)
        res.append("%6i vs. %6i          %6i     %8.6f" % (a.index, b.index, r.c, r.P))
    res.append("")
    for a in data:
        res.append("%6i         %8f + %8f * %s" % (a.index, 0.0, 1.0, a.filename))
    res.append("")
    print(os.linesep.join(res))
    return res

{'fraction_non_alphanumeric': 0.1355767360435713, 'fraction_numerical': 0.012837969266679634, 'mean_word_length': 2.761521580102414, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1318257', 'n_tokens_mistral': 1714, 'n_tokens_neox': 1667, 'n_words': 476}