<commit_before>#!/usr/bin/env python
import sys

class MulticastGroupStatRecord(object):
    def __init__(self, group_index, num_receivers, num_flows, max_link_mbps, avg_link_mbps, traffic_conc, link_mbps_std_dev, flow_tracker_response_time):
        self.group_index = group_index
        self.num_receivers = num_receivers
        self.num_flows = num_flows
        self.max_link_mbps = max_link_mbps
        self.avg_link_mbps = avg_link_mbps
        self.traffic_conc = traffic_conc
        self.link_mbps_std_dev = link_mbps_std_dev
        self.flow_tracker_response_time = flow_tracker_response_time

def mean_confidence_interval(data, confidence=0.95):
	import scipy.stats
	from numpy import mean, array, sqrt 
	a = 1.0 * array(data)
	n = len(a)
	m, se = mean(a), scipy.stats.sem(a)
	# calls the inverse CDF of the Student's t distribution
	h = se * scipy.stats.t._ppf((1+confidence)/2., n-1)
	return m-h, m+h
    

def read_log_set(filepath_prefix, num_logs, output_filepath):
    group_records = []  # list of lists -> group_records[group_index] = list of MulticastGroupStatRecords
    num_groups_list = []
    
    for log_index in range(0, num_logs):
        filepath = filepath_prefix + str(log_index) + '.log'
        log_file = open(filepath, 'r')
        num_groups = 0
        for line in log_file:
            if 'Group:' in line:
                #print line,
                num_groups += 1
                split_line = line.split(' ')
                group_index = int(split_line[0][len('Group:'):])
                num_receivers = int(split_line[1][len('NumReceivers:'):])
                num_flows = int(split_line[2][len('TotalNumFlows:'):])
                flow_tracker_response_time = float(split_line[3][len('ResponseTime:'):])
                max_link_mbps = float(split_line[4][len('MaxLinkUsageMbps:'):])
                avg_link_mbps = float(split_line[5][len('AvgLinkUsageMbps:'):])
                traffic_conc = float(split_line[6][len('TrafficConcentration:'):])
                link_mbps_std_dev = float(split_line[7][len('LinkUsageStdDev:'):])
                group_record = MulticastGroupStatRecord(group_index, num_receivers, num_flows, max_link_mbps, avg_link_mbps, traffic_conc, link_mbps_std_dev, flow_tracker_response_time)
                
                if group_index < len(group_records):
                    group_records[group_index].append(group_record)
                else:
                    group_records.append([group_record])
        num_groups_list.append(num_groups)
        log_file.close()
        print 'Processed log: ' + str(filepath)
        
    return group_records, num_groups_list


def print_group_record_statistics(group_records, num_groups_list):
    avg_num_groups_supported = float(sum(num_groups_list)) / len(num_groups_list)
    ci_upper, ci_lower = mean_confidence_interval(num_groups_list)
    print 'Average # Groups Supported: ' + str(avg_num_groups_supported) + '\t[' + str(ci_lower) + ', ' + str(ci_upper) + ']'
    print 'num_groups = ' + str(avg_num_groups_supported) + ';'
    print 'num_groups_ci = ' + str(abs(ci_upper - ci_lower) / 2) + ';'
    print ' '
    
    traffic_conc_avgs = []
    traffic_conc_cis = []
    link_std_dev_avgs = []
    link_std_dev_cis = []
    link_avg_mbps_avgs = []
    link_avg_mbps_cis = []
    response_time_avgs = []
    respone_time_cis = []
    num_flows_avgs = []
    num_flows_cis = []
    
    for group_index in range(0, len(group_records)):
        if len(group_records[group_index]) < 2:
            # Ignore entries that do not produce valid confidence intervals
            continue
            
        print 'Group #' + str(group_index) + ' stats:'
        print '# Trials:\t\t' + str(len(group_records[group_index]))
        
        num_receivers_list = [float(r.num_receivers) for r in group_records[group_index]]
        avg = sum(num_receivers_list) / len(num_receivers_list)
        ci_upper, ci_lower = mean_confidence_interval(num_receivers_list)
        print 'NumReceivers:\t\t' + str(avg) + '\t[' + str(ci_lower) + ', ' + str(ci_upper) + ']'
        
        num_flows_list = [float(r.num_flows) for r in group_records[group_index]]
        avg = sum(num_flows_list) / len(num_flows_list)
        num_flows_avgs.append(avg)
        ci_upper, ci_lower = mean_confidence_interval(num_flows_list)
        num_flows_cis.append(abs(ci_upper - ci_lower) / 2)
        print 'TotalNumFlows:\t\t' + str(avg) + '\t[' + str(ci_lower) + ', ' + str(ci_upper) + ']'
        
        response_time_list = [float(r.flow_tracker_response_time) for r in group_records[group_index]]
        avg = sum(response_time_list) / len(response_time_list)
        response_time_avgs.append(avg)
        ci_upper, ci_lower = mean_confidence_interval(response_time_list)
        num_flows_cis.append(abs(ci_upper - ci_lower) / 2)
        print 'ResponseTime:\t\t' + str(avg) + '\t[' + str(ci_lower) + ', ' + str(ci_upper) + ']'
        
        max_link_mbps_list = [float(r.max_link_mbps) for r in group_records[group_index]]
        avg = sum(max_link_mbps_list) / len(max_link_mbps_list)
        ci_upper, ci_lower = mean_confidence_interval(max_link_mbps_list)
        print 'MaxLinkUsageMbps:\t' + str(avg) + '\t[' + str(ci_lower) + ', ' + str(ci_upper) + ']'
        
        avg_link_mbps_list = [float(r.avg_link_mbps) for r in group_records[group_index]]
        avg = sum(avg_link_mbps_list) / len(avg_link_mbps_list)<commit_msg>Further fixes for log analyzing script to match new output format.<commit_after>#!/usr/bin/env python
import sys

class MulticastGroupStatRecord(object):
    def __init__(self, group_index, num_receivers, num_flows, max_link_mbps, avg_link_mbps, traffic_conc, link_mbps_std_dev, flow_tracker_response_time):
        self.group_index = group_index
        self.num_receivers = num_receivers
        self.num_flows = num_flows
        self.max_link_mbps = max_link_mbps
        self.avg_link_mbps = avg_link_mbps
        self.traffic_conc = traffic_conc
        self.link_mbps_std_dev = link_mbps_std_dev
        self.flow_tracker_response_time = flow_tracker_response_time

def mean_confidence_interval(data, confidence=0.95):
	import scipy.stats
	from numpy import mean, array, sqrt 
	a = 1.0 * array(data)
	n = len(a)
	m, se = mean(a), scipy.stats.sem(a)
	# calls the inverse CDF of the Student's t distribution
	h = se * scipy.stats.t._ppf((1+confidence)/2., n-1)
	return m-h, m+h
    

def read_log_set(filepath_prefix, num_logs, output_filepath):
    group_records = []  # list of lists -> group_records[group_index] = list of MulticastGroupStatRecords
    num_groups_list = []
    
    for log_index in range(0, num_logs):
        filepath = filepath_prefix + str(log_index) + '.log'
        log_file = open(filepath, 'r')
        num_groups = 0
        for line in log_file:
            if 'Group:' in line:
                #print line,
                num_groups += 1
                split_line = line.split(' ')
                group_index = int(split_line[0][len('Group:'):])
                num_receivers = int(split_line[1][len('NumReceivers:'):])
                num_flows = int(split_line[2][len('TotalNumFlows:'):])
                flow_tracker_response_time = float(split_line[3][len('ResponseTime:'):])
                max_link_mbps = float(split_line[4][len('MaxLinkUsageMbps:'):])
                avg_link_mbps = float(split_line[5][len('AvgLinkUsageMbps:'):])
                traffic_conc = float(split_line[6][len('TrafficConcentration:'):])
                link_mbps_std_dev = float(split_line[7][len('LinkUsageStdDev:'):])
                group_record = MulticastGroupStatRecord(group_index, num_receivers, num_flows, max_link_mbps, avg_link_mbps, traffic_conc, link_mbps_std_dev, flow_tracker_response_time)
                
                if group_index < len(group_records):
                    group_records[group_index].append(group_record)
                else:
                    group_records.append([group_record])
        num_groups_list.append(num_groups)
        log_file.close()
        print 'Processed log: ' + str(filepath)
        
    return group_records, num_groups_list


def print_group_record_statistics(group_records, num_groups_list):
    avg_num_groups_supported = float(sum(num_groups_list)) / len(num_groups_list)
    ci_upper, ci_lower = mean_confidence_interval(num_groups_list)
    print 'Average # Groups Supported: ' + str(avg_num_groups_supported) + '\t[' + str(ci_lower) + ', ' + str(ci_upper) + ']'
    print 'num_groups = ' + str(avg_num_groups_supported) + ';'
    print 'num_groups_ci = ' + str(abs(ci_upper - ci_lower) / 2) + ';'
    print ' '
    
    traffic_conc_avgs = []
    traffic_conc_cis = []
    link_std_dev_avgs = []
    link_std_dev_cis = []
    link_avg_mbps_avgs = []
    link_avg_mbps_cis = []
    response_time_avgs = []
    response_time_cis = []
    num_flows_avgs = []
    num_flows_cis = []
    
    for group_index in range(0, len(group_records)):
        if len(group_records[group_index]) < 2:
            # Ignore entries that do not produce valid confidence intervals
            continue
            
        print 'Group #' + str(group_index) + ' stats:'
        print '# Trials:\t\t' + str(len(group_records[group_index]))
        
        num_receivers_list = [float(r.num_receivers) for r in group_records[group_index]]
        avg = sum(num_receivers_list) / len(num_receivers_list)
        ci_upper, ci_lower = mean_confidence_interval(num_receivers_list)
        print 'NumReceivers:\t\t' + str(avg) + '\t[' + str(ci_lower) + ', ' + str(ci_upper) + ']'
        
        num_flows_list = [float(r.num_flows) for r in group_records[group_index]]
        avg = sum(num_flows_list) / len(num_flows_list)
        num_flows_avgs.append(avg)
        ci_upper, ci_lower = mean_confidence_interval(num_flows_list)
        num_flows_cis.append(abs(ci_upper - ci_lower) / 2)
        print 'TotalNumFlows:\t\t' + str(avg) + '\t[' + str(ci_lower) + ', ' + str(ci_upper) + ']'
        
        response_time_list = [float(r.flow_tracker_response_time) for r in group_records[group_index]]
        avg = sum(response_time_list) / len(response_time_list)
        response_time_avgs.append(avg)
        ci_upper, ci_lower = mean_confidence_interval(response_time_list)
        response_time_cis.append(abs(ci_upper - ci_lower) / 2)
        print 'ResponseTime:\t\t' + str(avg) + '\t[' + str(ci_lower) + ', ' + str(ci_upper) + ']'
        
        max_link_mbps_list = [float(r.max_link_mbps) for r in group_records[group_index]]
        avg = sum(max_link_mbps_list) / len(max_link_mbps_list)
        ci_upper, ci_lower = mean_confidence_interval(max_link_mbps_list)
        print 'MaxLinkUsageMbps:\t' + str(avg) + '\t[' + str(ci_lower) + ', ' + str(ci_upper) + ']'
        
        avg_link_mbps_list = [float(r.avg_link_mbps) for r in group_records[group_index]]
        avg = sum(avg_link_mbps_list) / len(avg_link_mbps_list)
{'fraction_non_alphanumeric': 0.11375661375661375, 'fraction_numerical': 0.004378762999452655, 'mean_word_length': 3.0513673318551366, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7231784', 'n_tokens_mistral': 3913, 'n_tokens_neox': 3785, 'n_words': 607}