<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Collections.Concurrent;
using System.Diagnostics;

namespace Ricochet {
    static class TimingHelper {
        static ConcurrentDictionary<string, CircularArrayList<double>> stats = new ConcurrentDictionary<string, CircularArrayList<double>>();

        public static void Add(string s, double v) {
            if (!stats.ContainsKey(s)) {
                stats[s] = new CircularArrayList<double>(200000, 200000);
            }
            stats[s].Add(v);
        }
        public static void Add(string s, Stopwatch sw) {
            Add(s, sw.Elapsed.TotalMilliseconds);
        }

        public static CircularArrayList<double> Get(string s) {
            CircularArrayList<double> ret = null;
            if (!stats.TryGetValue(s, out ret)) {
                ret = new CircularArrayList<double>(1, 1);
            }
            return ret;
        }

        public static Dictionary<string, BasicStats> Summary() {
            var ret = new Dictionary<string, BasicStats>();
            foreach (var item in stats) {
                ret.Add(item.Key, new BasicStats(item.Value.ToArray()));
            }
            return ret;
        }
        // public 
        // static CircularArrayList<double> workQueueTimes = new CircularArrayList<double>(20000, 20000);

        internal static void Reset() {
            stats = new ConcurrentDictionary<string, CircularArrayList<double>>();
        }
    }
}<commit_msg>fixed bug with timing helper<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Collections.Concurrent;
using System.Diagnostics;

namespace Ricochet {
    static class TimingHelper {
        static ConcurrentDictionary<string, CircularArrayList<double>> stats = new ConcurrentDictionary<string, CircularArrayList<double>>();

        public static void Add(string s, double v) {
            var mystats = stats;
            if (!mystats.ContainsKey(s)) {
                mystats[s] = new CircularArrayList<double>(200000, 200000);
            }
            mystats[s].Add(v);
        }
        public static void Add(string s, Stopwatch sw) {
            Add(s, sw.Elapsed.TotalMilliseconds);
        }

        public static CircularArrayList<double> Get(string s) {
            CircularArrayList<double> ret = null;
            if (!stats.TryGetValue(s, out ret)) {
                ret = new CircularArrayList<double>(1, 1);
            }
            return ret;
        }

        public static Dictionary<string, BasicStats> Summary() {
            var ret = new Dictionary<string, BasicStats>();
            foreach (var item in stats) {
                ret.Add(item.Key, new BasicStats(item.Value.ToArray()));
            }
            return ret;
        }
        // public 
        // static CircularArrayList<double> workQueueTimes = new CircularArrayList<double>(20000, 20000);

        internal static void Reset() {
            stats = new ConcurrentDictionary<string, CircularArrayList<double>>();
        }
    }
}
{'fraction_non_alphanumeric': 0.10156739811912226, 'fraction_numerical': 0.015047021943573668, 'mean_word_length': 2.2930856553147576, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5388244', 'n_tokens_mistral': 867, 'n_tokens_neox': 820, 'n_words': 211}