<commit_before>﻿using System;
using System.Diagnostics;
using System.IO;
using System.Text;

namespace Garage
{
    public class Record
    {
        public DateTime Start { get; set; }
        public DateTime End { get; set; }
        public int Id { get; set; }
        public TimeSpan Duration => End - Start;
    }

    class Program
    {
        static void Main(string[] args)
        {
            AppDomain.MonitoringIsEnabled = true;
            var sp = Stopwatch.StartNew();

            const string path = @"data.txt";
            CreateSummaryOptimized(path);

            Console.WriteLine($"Took: {sp.ElapsedMilliseconds:#,#} ms and allocated {AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize / 1024:#,#} kb with peak working set of {Process.GetCurrentProcess().PeakWorkingSet64 / 1024:#,#} kb");
        }

        const int DICT_BLOCK_SIZE = 1000;
        const int recordsize = 50;

        private static void CreateSummaryOptimized(string path)
        {
            var dict = new int[9999999 + 1 / DICT_BLOCK_SIZE][];
            var file = new FileInfo(path);

            var record = new byte[recordsize];
            using (var records = file.OpenRead())
            {
                for (int i = 0; i < file.Length; i += recordsize)
                {
                    records.Read(record, 0, recordsize);
                    ProcessLine(record, dict);
                }
            }
            using (var output = File.CreateText("summary.txt"))
            {
                for (int i = 0; i < dict.Length; i++)
                {
                    if (dict[i] != null)
                    {
                        for (int j = 0; j < dict[i].Length; j++)
                        {
                            if (dict[i][j] > 0)
                                output.WriteLine($"{i * DICT_BLOCK_SIZE + j:D10} {TimeSpan.FromSeconds(dict[i][j]):c}");
                        }
                    }
                }
            }
        }

        private static void ProcessLine(byte[] record, int[][] dict)
        {
            int durationSeconds;
            if (!BufferEqual(record, startIndex1: 0, startIndex2: 20, length: 10)) // Same day check
            {
                var parts = ToString(record).Split(' ');
                var start = DateTime.Parse(parts[0]);
                var end = DateTime.Parse(parts[1]);
                durationSeconds = (int)(end - start).TotalSeconds;
            }
            else
            {
                var start = ParseTime(record, index: 11);
                var end = ParseTime(record, index: 31);
                durationSeconds = end - start;
            }
            var id = ParseInt(record, startIndex: 40, length: 8);
            //Interlocked.Add(ref dict[id], durationSeconds);
            if (dict[id / DICT_BLOCK_SIZE] == null) dict[id / DICT_BLOCK_SIZE] = new int[DICT_BLOCK_SIZE];
            dict[id / DICT_BLOCK_SIZE][id % DICT_BLOCK_SIZE] += durationSeconds;
        }

        /// <summary>
        /// returns the number of seconds since midnight
        /// </summary>
        /// <param name="record"></param>
        /// <param name="index"></param>
        /// <returns></returns><commit_msg>Took: 1.497 ms and allocated 145.922 kb with peak working set of 69.028 kb<commit_after>﻿using System;
using System.Diagnostics;
using System.IO;
using System.Text;

namespace Garage
{
    public class Record
    {
        public DateTime Start { get; set; }
        public DateTime End { get; set; }
        public int Id { get; set; }
        public TimeSpan Duration => End - Start;
    }

    class Program
    {
        static void Main(string[] args)
        {
            AppDomain.MonitoringIsEnabled = true;
            var sp = Stopwatch.StartNew();

            const string path = @"data.txt";
            CreateSummaryOptimized(path);

            Console.WriteLine($"Took: {sp.ElapsedMilliseconds:#,#} ms and allocated {AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize / 1024:#,#} kb with peak working set of {Process.GetCurrentProcess().PeakWorkingSet64 / 1024:#,#} kb");
        }

        const int DICT_BLOCK_SIZE = 1000;
        const int recordsize = 50;

        private static void CreateSummaryOptimized(string path)
        {
            var dict = new int[9999999 + 1 / DICT_BLOCK_SIZE][];
            var file = new FileInfo(path);

            var record = new byte[recordsize];
            using (var records = file.OpenRead())
            {
                for (int i = 0; i < file.Length; i += recordsize)
                {
                    records.Read(record, 0, recordsize);
                    ProcessLine(record, dict);
                }
            }
            using (var output = File.CreateText("summary.txt"))
            {
                for (int i = 0; i < dict.Length; i++)
                {
                    if (dict[i] != null)
                    {
                        for (int j = 0; j < dict[i].Length; j++)
                        {
                            if (dict[i][j] > 0)
                                output.WriteLine($"{i * DICT_BLOCK_SIZE + j:D10} {TimeSpan.FromSeconds(dict[i][j]):c}");
                        }
                    }
                }
            }
        }

        private static void ProcessLine(byte[] record, int[][] dict)
        {
            int durationSeconds;
            if (!BufferEqual(record, startIndex1: 0, startIndex2: 20, length: 10)) // Same day check
            {
                var start = ParseDateTime(record, 0);
                var end = ParseDateTime(record, 20);
                durationSeconds = (int)(end - start).TotalSeconds;
            }
            else
            {
                var start = ParseTime(record, index: 11);
                var end = ParseTime(record, index: 31);
                durationSeconds = end - start;
            }
            var id = ParseInt(record, startIndex: 40, length: 8);
            //Interlocked.Add(ref dict[id], durationSeconds);
            if (dict[id / DICT_BLOCK_SIZE] == null) dict[id / DICT_BLOCK_SIZE] = new int[DICT_BLOCK_SIZE];
            dict[id / DICT_BLOCK_SIZE][id % DICT_BLOCK_SIZE] += durationSeconds;
        }

        private static DateTime ParseDateTime(byte[] record, int offset)
        {
            return new DateTime(kind: DateTimeKind.Utc,
                year: ParseInt(record, offset, 4),
                month: ParseInt(record, offset + 5, 2),
                day: ParseInt(record, offset + 8, 2),
                hour: ParseInt(record, offset + 11, 2),
                minute: ParseInt(record, offset + 14, 2),
                second: ParseInt(record, offset + 17, 2)
                );
        }

        /// <summary>
        /// returns the number of seconds since midnight
        /// </summary>
        /// <param name="record"></param>
        /// <param name="index"></param>
        /// <returns></returns>
{'fraction_non_alphanumeric': 0.11363636363636363, 'fraction_numerical': 0.017836593785960874, 'mean_word_length': 1.489437880415324, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4405457', 'n_tokens_mistral': 2001, 'n_tokens_neox': 1803, 'n_words': 489}