<commit_before>﻿using System;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Text;

namespace AGS.Engine
{
    public static class Measure
    {
        private static ConcurrentDictionary<string, Stopwatch> _measurements = new ConcurrentDictionary<string, Stopwatch>();
        private static int _numIndents;
        private static string _lastKey;

        public static void Start(string key)
        {
            var watch = new Stopwatch();
            _measurements[key] = watch;
            if (_lastKey != null) Debug.WriteLine("");
            StringBuilder sb = new StringBuilder();
            indent(sb);
            sb.Append($"{key}: ");
            Debug.Write(sb.ToString());
            _lastKey = key;
            watch.Restart();
            _numIndents++;
        }

        public static void End(string key)
        {
            if (!_measurements.TryRemove(key, out var watch)) throw new Exception($"Missing watch for {key}");
            var time = watch.Elapsed;
            watch.Stop();
            var lastKey = _lastKey;
            _lastKey = null;
            _numIndents--;
            if (key == lastKey)
            {
                Debug.WriteLine(time);
                return;
            }
            StringBuilder sb = new StringBuilder();
            indent(sb);
            sb.Append($"{time} ({key})");
            Debug.WriteLine(sb.ToString());
        }

        private static void indent(StringBuilder sb)
        {
            for (int i = 0; i < _numIndents; i++) sb.Append(' ');
        }
    }
}
<commit_msg>Debug- tagging for measurement tool<commit_after>﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;

namespace AGS.Engine
{
    public static class Measure
    {
        private static ConcurrentDictionary<string, Stopwatch> _measurements = new ConcurrentDictionary<string, Stopwatch>();
        private static int _numIndents;
        private static string _lastKey;
        private static ConcurrentDictionary<string, ConcurrentDictionary<string, Stopwatch>> _tags = new ConcurrentDictionary<string, ConcurrentDictionary<string, Stopwatch>>();

        public static void Start(string key)
        {
            var watch = new Stopwatch();
            _measurements[key] = watch;
            if (_lastKey != null) Debug.WriteLine("");
            StringBuilder sb = new StringBuilder();
            indent(sb);
            sb.Append($"{key}: ");
            Debug.Write(sb.ToString());
            _lastKey = key;
            watch.Restart();
            _numIndents++;
        }

        public static void End(string key)
        {
            if (!_measurements.TryRemove(key, out var watch)) throw new Exception($"Missing watch for {key}");
            var time = watch.Elapsed;
            watch.Stop();
            var lastKey = _lastKey;
            _lastKey = null;
            _numIndents--;
            if (key == lastKey)
            {
                Debug.WriteLine(time);
                return;
            }
            StringBuilder sb = new StringBuilder();
            indent(sb);
            sb.Append($"{time} ({key})");
            Debug.WriteLine(sb.ToString());
        }

        public static void StartTag(string key)
        {
            _tags[key] = new ConcurrentDictionary<string, Stopwatch>();
            Measure.Start(key);
        }

        public static void EndTag(string key)
        {
            Measure.End(key);
            if (!_tags.TryRemove(key, out var stopwatches))
            {
                throw new Exception($"Missing key for tag measurement: {key}");
            }
            var sum = stopwatches.Values.Sum(w => w.ElapsedTicks);
            var average = (float)sum / stopwatches.Count;
            var totalTime = TimeSpan.FromTicks(sum);
            var averageTime = TimeSpan.FromTicks((long)average);
            Debug.WriteLine($"Tag: {key}, Instances: {stopwatches.Count}, Total: {totalTime}, Average: {averageTime}");
        }

        public static string StartTagInstance(string key)
        {
            if (!_tags.TryGetValue(key, out var values))
                return null;
            var watch = new Stopwatch();
            var id = Guid.NewGuid().ToString();
            values[id] = watch;
            watch.Restart();
            return id;
        }

        public static void EndTagInstance(string key, string id)
        {
            if (id == null) return;
            _tags[key][id].Stop();
        }

        private static void indent(StringBuilder sb)
        {
            for (int i = 0; i < _numIndents; i++) sb.Append(' ');
        }
    }
}
{'fraction_non_alphanumeric': 0.10815378115758344, 'fraction_numerical': 0.0004224757076468103, 'mean_word_length': 1.801775147928994, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4620647', 'n_tokens_mistral': 1228, 'n_tokens_neox': 1206, 'n_words': 321}