<commit_before>#!/usr/bin/python
'''
Short script to evaluate the objective function.
'''
from collections import defaultdict
import argparse
import itertools
import logging
import sys
import yaml

from calculate import run_calculate
from datatypes import Datum, datum_sort_key
import constants as cons

def parse(args):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        '--calculate', '-c', type=str,
        metavar = '" commands for calculate.py"',
        help=('These commands produce the calculated data. Leave one space '
              'after the 1st quotation mark enclosing the arguments.'))
    parser.add_argument(
        '--reference', '-r', type=str,
        metavar='" commands for calculate.py"',
        help=('These commands produce the reference data. Leave one space '
              'after the 1st quotation mark enclosing the arguments.'))
    parser.add_argument(
        '--output', '-o', type=str, metavar='filename.txt',
        help='Write data to file.')
    parser.add_argument(
        '--print', '-p', action='store_true', dest='doprint',
        help='Print data.')
    opts = parser.parse_args(args)
    return opts

def convert_energies(data_cal, data_ref):
    energies_ref = [x for x in data_ref if x.dtype in ('energy', 'energy2')]
    energies_cal = [x for x in data_cal if x.dtype in ('energy', 'energy2')]
    groups_ref = defaultdict(list)
    for datum in energies_ref:
        groups_ref[datum.group].append(datum)
    groups_cal = defaultdict(list)
    for datum in energies_cal:
        groups_cal[datum.group].append(datum)
    for gnum_ref, gnum_cal in itertools.izip(sorted(groups_ref), sorted(groups_cal)):
        group_energies_ref = groups_ref[gnum_ref]
        group_energies_cal = groups_cal[gnum_cal]
        value, index = min((datum.value, index) for index, datum in enumerate(group_energies_ref))
        minimum_cal = group_energies_cal[index].value
        for datum in group_energies_cal:
            datum.value -= minimum_cal

def import_steps(params):
    for param in params:
        param.step = cons.steps[param.ptype]

# def import_steps(params, yamlfile='steps.yaml', **kwargs):
#     '''
#     Grabs step sizes for parameters from a yaml file. Can also take
#     arguments to override the dictionary.
#     '''
#     with open(yamlfile, 'r') as f:
#         steps = yaml.load(f)
#     for key,value in kwargs.iteritems():
#         steps[key] = value
#     for param in params:
#         param.step = steps[param.ptype]

def import_weights(data, yamlfile='weights.yaml', **kwargs):
    with open(yamlfile, 'r') as f:
        weights = yaml.load(f)
    for key, value in kwargs.iteritems():
        weights[key] = value
    for datum in data:
        if datum.dtype == 'eig':
            if datum.i == 0 and datum.j == 0:
                datum.weight = weights['eig_i']
            elif datum.i == datum.j:
                datum.weight = weights['eig_d']
            else:
                datum.weight = weights['eig_o']
        elif datum.dtype == 'hess':<commit_msg>Added missing weight import for dtype eigz.<commit_after>#!/usr/bin/python
'''
Short script to evaluate the objective function.
'''
from collections import defaultdict
import argparse
import itertools
import logging
import sys
import yaml

from calculate import run_calculate
from datatypes import Datum, datum_sort_key
import constants as cons

def parse(args):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        '--calculate', '-c', type=str,
        metavar = '" commands for calculate.py"',
        help=('These commands produce the calculated data. Leave one space '
              'after the 1st quotation mark enclosing the arguments.'))
    parser.add_argument(
        '--reference', '-r', type=str,
        metavar='" commands for calculate.py"',
        help=('These commands produce the reference data. Leave one space '
              'after the 1st quotation mark enclosing the arguments.'))
    parser.add_argument(
        '--output', '-o', type=str, metavar='filename.txt',
        help='Write data to file.')
    parser.add_argument(
        '--print', '-p', action='store_true', dest='doprint',
        help='Print data.')
    opts = parser.parse_args(args)
    return opts

def convert_energies(data_cal, data_ref):
    energies_ref = [x for x in data_ref if x.dtype in ('energy', 'energy2')]
    energies_cal = [x for x in data_cal if x.dtype in ('energy', 'energy2')]
    groups_ref = defaultdict(list)
    for datum in energies_ref:
        groups_ref[datum.group].append(datum)
    groups_cal = defaultdict(list)
    for datum in energies_cal:
        groups_cal[datum.group].append(datum)
    for gnum_ref, gnum_cal in itertools.izip(sorted(groups_ref), sorted(groups_cal)):
        group_energies_ref = groups_ref[gnum_ref]
        group_energies_cal = groups_cal[gnum_cal]
        value, index = min((datum.value, index) for index, datum in enumerate(group_energies_ref))
        minimum_cal = group_energies_cal[index].value
        for datum in group_energies_cal:
            datum.value -= minimum_cal

def import_steps(params):
    for param in params:
        param.step = cons.steps[param.ptype]

# def import_steps(params, yamlfile='steps.yaml', **kwargs):
#     '''
#     Grabs step sizes for parameters from a yaml file. Can also take
#     arguments to override the dictionary.
#     '''
#     with open(yamlfile, 'r') as f:
#         steps = yaml.load(f)
#     for key,value in kwargs.iteritems():
#         steps[key] = value
#     for param in params:
#         param.step = steps[param.ptype]

def import_weights(data, yamlfile='weights.yaml', **kwargs):
    with open(yamlfile, 'r') as f:
        weights = yaml.load(f)
    for key, value in kwargs.iteritems():
        weights[key] = value
    for datum in data:
        if datum.dtype == 'eig' or datum.dtype == 'eigz':
            if datum.i == 0 and datum.j == 0:
                datum.weight = weights['eig_i']
            elif datum.i == datum.j:
                datum.weight = weights['eig_d']
            else:
                datum.weight = weights['eig_o']
        elif datum.dtype == 'hess':
{'fraction_non_alphanumeric': 0.11463532442440831, 'fraction_numerical': 0.0019320560296248591, 'mean_word_length': 2.8322023442319555, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4568489', 'n_tokens_mistral': 1940, 'n_tokens_neox': 1926, 'n_words': 542}