<commit_before>#!/usr/bin/env python
'''
compare
-------
Contains code necessary to evaluate the objective function,

.. math:: \chi^2 = w^2 (x_r - x_c)^2

where :math:`w` is a weight, :math:`x_r` is the reference data point's value,
and :math:`x_c` is the calculated or force field's value for the data point.

'''
from __future__ import print_function
from __future__ import absolute_import
from collections import defaultdict
import sys
import argparse
from argparse import RawTextHelpFormatter
import logging
import logging.config
import numpy as np

import calculate
import constants as co
import datatypes

logger = logging.getLogger(__name__)

def main(args):
    parser = return_compare_parser()
    opts = parser.parse_args(args)
    r_data = calculate.main(opts.reference.split())
    c_data = calculate.main(opts.calculate.split())
    r_dict = data_by_type(r_data)
    c_dict = data_by_type(c_data)
    r_dict, c_dict = trim_data(r_dict, c_dict)
    score = compare_data(r_dict, c_dict, output=opts.output, doprint=opts.print)
#    score = compare_data(r_data, c_data)
    # Pretty readouts. Maybe opts.output could have 3 values:
    # True, False or None
    # Then I wouldn't need 2 if statements here.
#    if opts.output or opts.print:
#        pretty_data_comp(r_data, c_data, output=opts.output, doprint=opts.print)
    logger.log(1, '>>> score: {}'.format(score))

def trim_data(dict1,dict2):
    """
    Within a data type of these dictionaries, data that is not present in both
    lists.
    """
    for typ in dict1:
        if typ == 't':
            to_remove = []
            # Sometimes calculations are compressed into one macromodel command
            # file. This results in the 'pre' and 'opt' structures in the same
            # file. Geometeric data from each of these would correspond to r
            # and c data for 'pre' and 'opt' structures,respectively. This would
            # mean that when trying to compare two data poitns from r and c they
            # would not have the exact same lable since they are from different
            # structure indicies in the the file. So instaed of just trying to
            # see if the labels are the same, we should try and pull out the
            # filename and the atoms that comprise that data point. This is what
            # is done with the regex below.
            for d1 in dict1[typ]:
                #if not any(x.lbl == d1.lbl for x in dict2[typ]):
                if not any(co.RE_T_LBL.split(x.lbl)[1] == co.RE_T_LBL.split(d1.lbl)[1] and 
                           co.RE_T_LBL.split(x.lbl)[2] == co.RE_T_LBL.split(d1.lbl)[2] 
                           for x in dict2[typ]):
                    to_remove.append(d1)
            for d2 in dict2[typ]:
                #if not any(x.lbl == d2.lbl for x in dict1[typ]):
                if not any(co.RE_T_LBL.split(x.lbl)[1] == co.RE_T_LBL.split(d2.lbl)[1] and 
                           co.RE_T_LBL.split(x.lbl)[2] == co.RE_T_LBL.split(d2.lbl)[2] 
                           for x in dict1[typ]):
                    to_remove.append(d2)
            for datum in to_remove:
                if datum in dict1[typ] and datum in dict2[typ]:
                    raise AssertionError("The data point that is flagged to be \
                    removed is present in both data sets.")
                # We may want to keep track of the data that is removed. 
                if datum in dict1[typ]:
                    dict1[typ].remove(datum) 
                if datum in dict2[typ]:
                    dict2[typ].remove(datum) 
            if to_remove:
                logger.log(20, '>>> Removed Data: {}'.format(len(to_remove)))
        dict1[typ] = np.array(dict1[typ], dtype=datatypes.Datum)
        dict2[typ] = np.array(dict2[typ], dtype=datatypes.Datum)
    return dict1, dict2

def data_by_type(data_iterable):
    """
    Takes a iterable of data and creates a dictionary of data types and sets
    up an array.
    """

    data_by_typ = {}
    for datum in data_iterable:
        if datum.typ not in data_by_typ:
            data_by_typ[datum.typ] = []
        data_by_typ[datum.typ].append(datum)
    # Parts of the code rely on the data to be in an array form and so we must
    # make the dictionary an array. This doesn't make sense to have here since
    # we may have to remove data in trim_data().
    #for typ in data_by_typ:
    #    data_by_typ[typ] = np.array(data_by_typ[typ], dtype=datatypes.Datum)
    return data_by_typ

def compare_data(r_dict, c_dict, output=None, doprint=False):
    """
    This function was formerly called pretty_data_comp().
    Now only one function is needed to calculate the score, with the options to
    print and direct to an output available.
    """

    strings = []
    strings.append('--' + ' Label '.ljust(30, '-') +
                   '--' + ' Weight '.center(7, '-') +
                   '--' + ' R. Value '.center(11, '-') +
                   '--' + ' C. Value '.center(11, '-') +
                   '--' + ' Score '.center(11, '-') +
                   '--' + ' Row ' + '--')
    score_typ = defaultdict(float)
    num_typ = defaultdict(int)
    score_tot = 0.
    total_num = 0
    # Do we want the datatypes always reported in the same order? This allows
    # the same order of data type to be printed the same everytime.
    data_types = []
    for typ in r_dict:
        data_types.append(typ)
    data_types.sort()
    total_num_energy = 0
    for typ in data_types:
        if typ in ['e','eo','ea','eao']:
            total_num_energy += len(r_dict[typ])
    for typ in data_types:
        total_num += int(len(r_dict[typ]))
        if typ in ['e','eo','ea','eao']:
            correlate_energies(r_dict[typ],c_dict[typ])
        import_weights(r_dict[typ])
        for r,c in zip(r_dict[typ],c_dict[typ]):
            if c.typ == 't':
                diff = abs(r.val - c.val)
                if diff > 180.:
                    diff = 360. - diff
            else:
                diff = r.val - c.val
            #score = (r.wht**2 * diff**2)
            if typ in ['e', 'eo', 'ea', 'eao']:
                score = (r.wht**2 * diff**2)/total_num_energy
            else:
                score = (r.wht**2 * diff**2)/len(r_dict[typ])
            score_tot += score
            score_typ[c.typ] += score
            num_typ[c.typ] += 1
            if c.typ == 'eig':
                if c.idx_1 == c.idx_2:
                    if r.val < 1100:
                        score_typ[c.typ + '-d-low'] += score
                        num_typ[c.typ + '-d-low'] += 1
                    else:
                        score_typ[c.typ + '-d-high'] += score
                        num_typ[c.typ + '-d-high'] += 1
                else:
                    score_typ[c.typ + '-o'] += score
                    num_typ[c.typ + '-o'] += 1
            strings.append('  {:<30}  {:>7.2f}  {:>11.4f}  {:>11.4f}  {:>11.4f}  '\
                       '{!s:>5} '.format(
                        c.lbl, r.wht, r.val, c.val, score, c.ff_row))
    strings.append('-' * 89)
    strings.append('{:<20} {:20.4f}'.format('Total score:', score_tot))
    strings.append('{:<30} {:10d}'.format('Total Num. data points:', total_num))
    for k, v in num_typ.items():
        strings.append('{:<30} {:10d}'.format(k + ':', v))
    strings.append('-' * 89)
    for k, v in score_typ.items():
        strings.append('{:<20} {:20.4f}'.format(k + ':', v))
    if output:
        with open(output, 'w') as f:
            for line in strings:
                f.write('{}\n'.format(line))
    if doprint:
        for line in strings:
            print(line)
    return score_tot<commit_msg>small fix<commit_after>#!/usr/bin/env python
'''
compare
-------
Contains code necessary to evaluate the objective function,

.. math:: \chi^2 = w^2 (x_r - x_c)^2

where :math:`w` is a weight, :math:`x_r` is the reference data point's value,
and :math:`x_c` is the calculated or force field's value for the data point.

'''
from __future__ import print_function
from __future__ import absolute_import
from collections import defaultdict
import sys
import argparse
from argparse import RawTextHelpFormatter
import logging
import logging.config
import numpy as np

import calculate
import constants as co
import datatypes

logger = logging.getLogger(__name__)

def main(args):
    parser = return_compare_parser()
    opts = parser.parse_args(args)
    r_data = calculate.main(opts.reference.split())
    c_data = calculate.main(opts.calculate.split())
    r_dict = data_by_type(r_data)
    c_dict = data_by_type(c_data)
    r_dict, c_dict = trim_data(r_dict, c_dict)
    score = compare_data(r_dict, c_dict, output=opts.output, doprint=opts.print)
#    score = compare_data(r_data, c_data)
    # Pretty readouts. Maybe opts.output could have 3 values:
    # True, False or None
    # Then I wouldn't need 2 if statements here.
#    if opts.output or opts.print:
#        pretty_data_comp(r_data, c_data, output=opts.output, doprint=opts.print)
    logger.log(1, '>>> score: {}'.format(score))

def trim_data(dict1,dict2):
    """
    Within a data type of these dictionaries, data that is not present in both
    lists.
    """
    for typ in dict1:
        if typ == 't':
            to_remove = []
            # Sometimes calculations are compressed into one macromodel command
            # file. This results in the 'pre' and 'opt' structures in the same
            # file. Geometeric data from each of these would correspond to r
            # and c data for 'pre' and 'opt' structures,respectively. This would
            # mean that when trying to compare two data poitns from r and c they
            # would not have the exact same lable since they are from different
            # structure indicies in the the file. So instaed of just trying to
            # see if the labels are the same, we should try and pull out the
            # filename and the atoms that comprise that data point. This is what
            # is done with the regex below.
            for d1 in dict1[typ]:
                #if not any(x.lbl == d1.lbl for x in dict2[typ]):
                if not any(co.RE_T_LBL.split(x.lbl)[1] == co.RE_T_LBL.split(d1.lbl)[1] and 
                           co.RE_T_LBL.split(x.lbl)[2] == co.RE_T_LBL.split(d1.lbl)[2] 
                           for x in dict2[typ]):
                    to_remove.append(d1)
            for d2 in dict2[typ]:
                #if not any(x.lbl == d2.lbl for x in dict1[typ]):
                if not any(co.RE_T_LBL.split(x.lbl)[1] == co.RE_T_LBL.split(d2.lbl)[1] and 
                           co.RE_T_LBL.split(x.lbl)[2] == co.RE_T_LBL.split(d2.lbl)[2] 
                           for x in dict1[typ]):
                    to_remove.append(d2)
            for datum in to_remove:
                if datum in dict1[typ] and datum in dict2[typ]:
                    raise AssertionError("The data point that is flagged to be \
                    removed is present in both data sets.")
                # We may want to keep track of the data that is removed. 
                if datum in dict1[typ]:
                    dict1[typ].remove(datum) 
                if datum in dict2[typ]:
                    dict2[typ].remove(datum) 
            if to_remove:
                logger.log(20, '>>> Removed Data: {}'.format(len(to_remove)))
        dict1[typ] = np.array(dict1[typ], dtype=datatypes.Datum)
        dict2[typ] = np.array(dict2[typ], dtype=datatypes.Datum)
    return dict1, dict2

def data_by_type(data_iterable):
    """
    Takes a iterable of data and creates a dictionary of data types and sets
    up an array.
    """

    data_by_typ = {}
    for datum in data_iterable:
        if datum.typ not in data_by_typ:
            data_by_typ[datum.typ] = []
        data_by_typ[datum.typ].append(datum)
    # Parts of the code rely on the data to be in an array form and so we must
    # make the dictionary an array. This doesn't make sense to have here since
    # we may have to remove data in trim_data().
    #for typ in data_by_typ:
    #    data_by_typ[typ] = np.array(data_by_typ[typ], dtype=datatypes.Datum)
    return data_by_typ

def compare_data(r_dict, c_dict, output=None, doprint=False):
    """
    This function was formerly called pretty_data_comp().
    Now only one function is needed to calculate the score, with the options to
    print and direct to an output available.
    """

    strings = []
    strings.append('--' + ' Label '.ljust(30, '-') +
                   '--' + ' Weight '.center(7, '-') +
                   '--' + ' R. Value '.center(11, '-') +
                   '--' + ' C. Value '.center(11, '-') +
                   '--' + ' Score '.center(11, '-') +
                   '--' + ' Row ' + '--')
    score_typ = defaultdict(float)
    num_typ = defaultdict(int)
    score_tot = 0.
    total_num = 0
    # Do we want the datatypes always reported in the same order? This allows
    # the same order of data type to be printed the same everytime.
    data_types = []
    for typ in r_dict:
        data_types.append(typ)
    data_types.sort()
    total_num_energy = 0
    for typ in data_types:
        if typ in ['e','eo','ea','eao']:
            total_num_energy += len(r_dict[typ])
    for typ in data_types:
        total_num += int(len(r_dict[typ]))
        if typ in ['e','eo','ea','eao']:
            correlate_energies(r_dict[typ],c_dict[typ])
        import_weights(r_dict[typ])
        for r,c in zip(r_dict[typ],c_dict[typ]):
            if c.typ == 't':
                diff = abs(r.val - c.val)
                if diff > 180.:
                    diff = 360. - diff
            else:
                diff = r.val - c.val
            #score = (r.wht**2 * diff**2)
            if typ in ['e', 'eo', 'ea', 'eao']:
                score = (r.wht**2 * diff**2)/total_num_energy
            else:
                score = (r.wht**2 * diff**2)/len(r_dict[typ])
            score_tot += score
            score_typ[c.typ] += score
            num_typ[c.typ] += 1
            if c.typ == 'eig':
                if c.idx_1 == c.idx_2:
                    if r.val < 1100:
                        score_typ[c.typ + '-d-low'] += score
                        num_typ[c.typ + '-d-low'] += 1
                    else:
                        score_typ[c.typ + '-d-high'] += score
                        num_typ[c.typ + '-d-high'] += 1
                else:
                    score_typ[c.typ + '-o'] += score
                    num_typ[c.typ + '-o'] += 1
#            print(c.lbl, r.wht, r.val, c.val, score, c.ff_row)
            if c.ff_row is None:
                strings.append('  {:<30}  {:>7.2f}  {:>11.4f}  {:>11.4f}  {:>11.4f}  '.format(
                         c.lbl, r.wht, r.val, c.val, score))
            else:
                strings.append('  {:<30}  {:>7.2f}  {:>11.4f}  {:>11.4f}  {:>11.4f}  '\
                       '{!:>5} '.format(
                        c.lbl, r.wht, r.val, c.val, score, c.ff_row))
#            print(strings)
    strings.append('-' * 89)
    strings.append('{:<20} {:20.4f}'.format('Total score:', score_tot))
    strings.append('{:<30} {:10d}'.format('Total Num. data points:', total_num))
    for k, v in num_typ.items():
        strings.append('{:<30} {:10d}'.format(k + ':', v))
    strings.append('-' * 89)
    for k, v in score_typ.items():
        strings.append('{:<20} {:20.4f}'.format(k + ':', v))
    if output:
        with open(output, 'w') as f:
            for line in strings:
                f.write('{}\n'.format(line))
    if doprint:
        for line in strings:
            print(line)
    return score_tot
{'fraction_non_alphanumeric': 0.1261819575773064, 'fraction_numerical': 0.015780730897009966, 'mean_word_length': 1.8885403210924525, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5851965', 'n_tokens_mistral': 5207, 'n_tokens_neox': 5031, 'n_words': 1448}