<commit_before>#!/usr/bin/python
'''average_beta.py [options] file1 file2 ... fileN

Average over different DMQMC calculations contained in the output files, file1 file2 ... fileN, at each inverse temperature (beta) contained in the output files.'''

import optparse
import re
import scipy
import scipy.stats
import scipy.interpolate
import sys
from math import sqrt
from math import ceil
from math import floor
from math import exp

try:
    import matplotlib.pyplot as pyplot
    USE_MATPLOTLIB = True
except ImportError:
    USE_MATPLOTLIB = False

# data columns
BETA_COL = 0
SHIFT_COL = 1
TR_HRHO_COL = 4
TR_H2RHO_COL = 5
TR_RHO_COL = 2
H2_is_present = False
H_is_present = False
S_is_present = False
C_is_present = False
M2_is_present = False

TR_SRHO_INDEX = -1
TR_CRHO_INDEX = -1
TR_M2RHO_INDEX = -1
TR_HRHO_INDEX = -1
TR_H2RHO_INDEX = -1
PARTICLES_COL = 6


class Stats:
    def __init__(self, mean, se):
        self.mean = mean
        self.se = se

class Data:
    def __init__(self, shift, Tr_rho, numerators, init_lists=True):
        if init_lists:
            self.shift = [shift]
            self.Tr_rho = [Tr_rho]
            self.numerators = [numerators]
        else:
            self.shift = shift
            self.Tr_rho = Tr_rho
            self.numerators = numerators
    def append(self, shift, Tr_rho, numerators):
        self.shift.append(shift)
        self.Tr_rho.append(Tr_rho)
        self.numerators.append(numerators)
    def calculate_stats(self):
        s = Data(0,0,0,False)
        for attr in ('shift', 'Tr_rho'):
            mean = scipy.mean(getattr(self, attr))
            se = scipy.stats.sem(getattr(self, attr))
            setattr(s, attr, Stats(mean, se))
        mean = scipy.mean(self.numerators,0) 
        se = scipy.stats.sem(self.numerators,0)
        setattr(s, 'numerators', Stats(mean,se))

        return s
    def calculate_covs(self):
        covs = []
        for i in range(0,len(self.numerators[0])):
            covariance_matrix = scipy.cov(scipy.vstack((scipy.array(self.numerators)[:,i],self.Tr_rho)))
            covs.append(covariance_matrix[0][1]/len(self.numerators))
        return covs

def get_estimator_headings(data_files):
    global H_is_present
    global H2_is_present
    global S_is_present
    global C_is_present
    global M2_is_present
    global TR_HRHO_INDEX
    global TR_H2RHO_INDEX
    global TR_SRHO_INDEX
    global TR_CRHO_INDEX
    global TR_M2RHO_INDEX
    global PARTICLES_COL
# Get collumn headings from input file
    start_regex = '^ # iterations'
    data_file = data_files[0]
    in_file = open(data_file)
    estimator_headings = []
    estimator_col_no = []
    while in_file:
       line = in_file.next()
       if re.search(start_regex, line): 
           headings = line.split()
           index = 0
           for k in range(0,len(headings)):
               if headings[k] == '\\sum\\rho_{ij}H_{ji}':
                   estimator_col_no.append(k-TR_RHO_COL)
                   estimator_headings.append('Tr[Hp]/Tr[p]')
                   H_is_present = True
                   TR_HRHO_INDEX = index
                   index = index + 1
               elif headings[k] == '\\sum\\rho_{ij}S_{ji}':
                   estimator_col_no.append(k-TR_RHO_COL)
                   estimator_headings.append('Tr[Sp]/Tr[p]')
                   S_is_present = True
                   TR_SRHO_INDEX = index
                   index = index + 1
               elif headings[k] == '\\sum\\rho_{ij}C_{ji}':
                   estimator_col_no.append(k-TR_RHO_COL)
                   estimator_headings.append('Tr[Cp]/Tr[p]')
                   C_is_present = True
                   TR_CRHO_INDEX = index
                   index = index + 1 
               elif headings[k] == '\\sum\\rho_{ij}M2{ji}':
                   estimator_col_no.append(k-TR_RHO_COL)
                   estimator_headings.append('Tr[Mp]/Tr[p]')
                   M2_is_present = True
                   TR_M2RHO_INDEX = index
                   index = index + 1
               elif headings[k] == '\\sum\\rho_{ij}H2{ji}':
                   estimator_col_no.append(k-TR_RHO_COL)
                   estimator_headings.append('Tr[H2p]/Tr[p]')
                   H2_is_present = True
                   TR_H2RHO_INDEX = index
                   index = index + 1
               else:
                   print '# Warning: Column name, '+headings[k]+', not recognised...'
           PARTICLES_COL = TR_RHO_COL + index + 1
           break<commit_msg>Remove unnecessary warning and unused variables in average_beta.py.<commit_after>#!/usr/bin/python
'''average_beta.py [options] file1 file2 ... fileN

Average over different DMQMC calculations contained in the output files, file1 file2 ... fileN, at each inverse temperature (beta) contained in the output files.'''

import optparse
import re
import scipy
import scipy.stats
import scipy.interpolate
import sys
from math import sqrt
from math import ceil
from math import floor
from math import exp

try:
    import matplotlib.pyplot as pyplot
    USE_MATPLOTLIB = True
except ImportError:
    USE_MATPLOTLIB = False

# data columns
BETA_COL = 0
SHIFT_COL = 1
TR_RHO_COL = 2
H2_is_present = False
H_is_present = False
S_is_present = False
C_is_present = False
M2_is_present = False

TR_SRHO_INDEX = -1
TR_CRHO_INDEX = -1
TR_M2RHO_INDEX = -1
TR_HRHO_INDEX = -1
TR_H2RHO_INDEX = -1
PARTICLES_COL = 6


class Stats:
    def __init__(self, mean, se):
        self.mean = mean
        self.se = se

class Data:
    def __init__(self, shift, Tr_rho, numerators, init_lists=True):
        if init_lists:
            self.shift = [shift]
            self.Tr_rho = [Tr_rho]
            self.numerators = [numerators]
        else:
            self.shift = shift
            self.Tr_rho = Tr_rho
            self.numerators = numerators
    def append(self, shift, Tr_rho, numerators):
        self.shift.append(shift)
        self.Tr_rho.append(Tr_rho)
        self.numerators.append(numerators)
    def calculate_stats(self):
        s = Data(0,0,0,False)
        for attr in ('shift', 'Tr_rho'):
            mean = scipy.mean(getattr(self, attr))
            se = scipy.stats.sem(getattr(self, attr))
            setattr(s, attr, Stats(mean, se))
        mean = scipy.mean(self.numerators,0) 
        se = scipy.stats.sem(self.numerators,0)
        setattr(s, 'numerators', Stats(mean,se))

        return s
    def calculate_covs(self):
        covs = []
        for i in range(0,len(self.numerators[0])):
            covariance_matrix = scipy.cov(scipy.vstack((scipy.array(self.numerators)[:,i],self.Tr_rho)))
            covs.append(covariance_matrix[0][1]/len(self.numerators))
        return covs

def get_estimator_headings(data_files):
    global H_is_present
    global H2_is_present
    global S_is_present
    global C_is_present
    global M2_is_present
    global TR_HRHO_INDEX
    global TR_H2RHO_INDEX
    global TR_SRHO_INDEX
    global TR_CRHO_INDEX
    global TR_M2RHO_INDEX
    global PARTICLES_COL
# Get collumn headings from input file
    start_regex = '^ # iterations'
    data_file = data_files[0]
    in_file = open(data_file)
    estimator_headings = []
    estimator_col_no = []
    while in_file:
       line = in_file.next()
       if re.search(start_regex, line): 
           headings = line.split()
           index = 0
           for k in range(0,len(headings)):
               if headings[k] == '\\sum\\rho_{ij}H_{ji}':
                   estimator_col_no.append(k-TR_RHO_COL)
                   estimator_headings.append('Tr[Hp]/Tr[p]')
                   H_is_present = True
                   TR_HRHO_INDEX = index
                   index = index + 1
               elif headings[k] == '\\sum\\rho_{ij}S_{ji}':
                   estimator_col_no.append(k-TR_RHO_COL)
                   estimator_headings.append('Tr[Sp]/Tr[p]')
                   S_is_present = True
                   TR_SRHO_INDEX = index
                   index = index + 1
               elif headings[k] == '\\sum\\rho_{ij}C_{ji}':
                   estimator_col_no.append(k-TR_RHO_COL)
                   estimator_headings.append('Tr[Cp]/Tr[p]')
                   C_is_present = True
                   TR_CRHO_INDEX = index
                   index = index + 1 
               elif headings[k] == '\\sum\\rho_{ij}M2{ji}':
                   estimator_col_no.append(k-TR_RHO_COL)
                   estimator_headings.append('Tr[Mp]/Tr[p]')
                   M2_is_present = True
                   TR_M2RHO_INDEX = index
                   index = index + 1
               elif headings[k] == '\\sum\\rho_{ij}H2{ji}':
                   estimator_col_no.append(k-TR_RHO_COL)
                   estimator_headings.append('Tr[H2p]/Tr[p]')
                   H2_is_present = True
                   TR_H2RHO_INDEX = index
                   index = index + 1
           PARTICLES_COL = TR_RHO_COL + index + 1
           break
{'fraction_non_alphanumeric': 0.09787709497206704, 'fraction_numerical': 0.010614525139664804, 'mean_word_length': 2.007728494623656, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2946104', 'n_tokens_mistral': 3020, 'n_tokens_neox': 2938, 'n_words': 612}