<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
analyze.py

Created by <NAME> on 2012-10-23.
Copyright (c) 2012 University of Strathclyde. All rights reserved.
"""
import argparse
import csv
from itertools import cycle
import matplotlib.pyplot as plt
import numpy as np
import os
import os.path
import scipy.stats as stats


### Parse command line arguments
parser = argparse.ArgumentParser(description="DM simulation -- Analysis script")
parser.add_argument('input_dir', help='directory with simulation results')
parser.add_argument('--confidence', dest='confidence', default=0.99,
                    type=float, help='confidence value (default: 0.99)')
args = parser.parse_args()
input_dir = args.input_dir
confidence = args.confidence

### Merge results from files
# Get files (as strings)
extension = ".out"
file_names = set([f[:f.find(extension)] for _, _, files in os.walk(input_dir) for f in files if f.endswith(extension)])
file_paths = [os.path.join(root, f) for root, _, files in os.walk(input_dir) for f in files if f.endswith(extension)]
# Initial processing of data (includes warm-up period)
ref_column = 'sr_number'
for name in file_names:
  # Read data from files
  data_in = []
  for fp in file_paths:
    if name in fp:
      with open(fp, 'rt') as f:
        reader = csv.DictReader(f)
        dct = {}
        for row in reader:
          for key in row:
            val = float(row[key]) if key != ref_column else int(row[key])
            dct.setdefault(key, []).append(val)
        data_in.append(dct)
  # Reduce...
  # Get number of replications (equal to number of dictionaries in data_in)
  repetitions = len(data_in)
  # Compute mean
  zipped = zip(*[dct[key] for dct in data_in for key in dct.keys() if key != ref_column])
  means = list(map(lambda x: sum(x)/repetitions, zipped))
  # Compute standard deviation
  zipped = zip(*[dct[key] for dct in data_in for key in dct.keys() if key != ref_column])
  sds = [np.sqrt(sum(map(lambda x: (x-mean)**2, tup)) / (repetitions - 1)) for (tup, mean) in zip(zipped, means)]
  # Compute standard error for the mean
  ses = list(map(lambda x: x/np.sqrt(repetitions), sds))
  # Compute confidence intervals for the mean
  cis = list(map(lambda x: x * stats.t.ppf(0.5 + confidence/2, repetitions-1), ses))
  # Save to a file
  # Create save dir if doesn't exist
  save_dir = input_dir + '/transient'
  if not os.path.exists(save_dir):
    os.makedirs(save_dir)
  with open(save_dir + '/' + name + extension, 'w', newline='', encoding='utf-8') as f:
    writer = csv.writer(f, delimiter=',')
    zip_input = [data_in[0][ref_column], means, sds, ses, cis]
    out_headers = [ref_column, 'mean', 'sd', 'se', 'ci']
    writer.writerow(out_headers)
    for tup in zip(*zip_input):
      writer.writerow(tup)
<commit_msg>Add option for transient/steady-state analysis.<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
analyze.py

Created by <NAME> on 2012-10-23.
Copyright (c) 2012 University of Strathclyde. All rights reserved.
"""
import argparse
import csv
import numpy as np
import os
import os.path
import scipy.stats as stats
import sys


### Parse command line arguments
parser = argparse.ArgumentParser(description="DM simulation -- Statistical analysis script")
parser.add_argument('input_dir', help='directory with simulation results')
parser.add_argument('mode', help='transient or steady-state')
parser.add_argument('--confidence', dest='confidence', default=0.99,
                    type=float, help='confidence value (default: 0.99)')
args = parser.parse_args()
input_dir = args.input_dir
mode = args.mode
confidence = args.confidence

### Common params
# Output dir names
transient_dir = 'transient'
ss_dir = 'steady-state'
# File names and paths
extension = ".out"
file_names = set([f[:f.find(extension)] for root, _, files in os.walk(input_dir) for f in files \
                  if f.endswith(extension) and transient_dir not in root and ss_dir not in root])
file_paths = [os.path.join(root, f) for root, _, files in os.walk(input_dir) for f in files \
              if f.endswith(extension) and transient_dir not in root and ss_dir not in root]
# Reference column
ref_column = 'sr_number'
# Ask for warm-up period index (if mode is steady-state)
if mode.lower() == 'steady-state':
  save_dir = input_dir + '/' + ss_dir
  warmup = int(input('Warm-up period index: '))
elif mode.lower() == 'transient':
  save_dir = input_dir + '/' + transient_dir
  warmup = 0
else:
  sys.exit('Unknown mode specified.')

### Merge results from files
for name in file_names:
  # Read data from files
  data_in = []
  for fp in file_paths:
    if name in fp:
      with open(fp, 'rt') as f:
        reader = csv.DictReader(f)
        dct = {}
        for row in reader:
          # Exclude data with index lower than specified warm-up period
          if int(row[ref_column]) > warmup:
            for key in row:
              val = float(row[key]) if key != ref_column else int(row[key])
              dct.setdefault(key, []).append(val)
        data_in.append(dct)
  # Get number of replications (equal to number of dictionaries in data_in)
  repetitions = len(data_in)
  # Map and reduce...
  # Compute mean
  zipped = zip(*[dct[key] for dct in data_in for key in dct.keys() if key != ref_column])
  means = list(map(lambda x: sum(x)/repetitions, zipped))
  # Compute standard deviation
  zipped = zip(*[dct[key] for dct in data_in for key in dct.keys() if key != ref_column])
  sds = [np.sqrt(sum(map(lambda x: (x-mean)**2, tup)) / (repetitions - 1)) for (tup, mean) in zip(zipped, means)]
  # Compute standard error for the mean
  ses = list(map(lambda x: x/np.sqrt(repetitions), sds))
  # Compute confidence intervals for the mean
  cis = list(map(lambda x: x * stats.t.ppf(0.5 + confidence/2, repetitions-1), ses))
  # Save to a file
  # Create save dir if doesn't exist already
  if not os.path.exists(save_dir):
    os.makedirs(save_dir)
  with open(save_dir + '/' + name + extension, 'w', newline='', encoding='utf-8') as f:
    writer = csv.writer(f, delimiter=',')
    zip_input = [data_in[0][ref_column], means, sds, ses, cis]
    out_headers = [ref_column, 'mean', 'sd', 'se', 'ci']
    writer.writerow(out_headers)
    for tup in zip(*zip_input):
      writer.writerow(tup)

{'fraction_non_alphanumeric': 0.1291660022324988, 'fraction_numerical': 0.008770531015786957, 'mean_word_length': 3.63905325443787, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1712425', 'n_tokens_mistral': 2143, 'n_tokens_neox': 2064, 'n_words': 661}