<commit_before>﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Win32.SafeHandles;

namespace Optimized.Crc.File.Calculator
{
    public class AstarothCrcCalculator
    {
        // Using this native call avoid the need for the 4KB allocations on every file!
        [DllImport("Kernel32.dll", SetLastError = true)]
        public static extern bool ReadFile(SafeFileHandle handle, byte[] bytes, int numBytesToRead, out int numBytesRead, IntPtr mustBeZero);

        static void Main(string[] args)
        {
            BufferPool.GetBuffer(10000);
            // queues for the various actors
            var crcComputationWork = new BlockingCollection<CrcComputationWork>();
            var readDirectoryWork = new Queue<ReadDirectoryWork>();
            var readFileWork = new BlockingCollection<ReadFileWork>();

            SpinCrcComputations(crcComputationWork);

            SpinFileReaders(readFileWork, crcComputationWork);

			// Start the actors machinery by registering the initial directories
			foreach (var path in args)
			{
				readDirectoryWork.Enqueue(new ReadDirectoryWork
				{
					DirectoryName = path
				});
			}

            SpinDirectoryReader(readDirectoryWork, readFileWork);

          
            Console.WriteLine("Running computations...");
            Console.ReadLine();
        }

		private static void SpinDirectoryReader(Queue<ReadDirectoryWork> readDirectoryWork, BlockingCollection<ReadFileWork> readFileWork)<commit_msg>Explain that the collection is only single threaded, so we'll avoid trying to use is concurrently<commit_after>﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Win32.SafeHandles;

namespace Optimized.Crc.File.Calculator
{
    public class AstarothCrcCalculator
    {
        // Using this native call avoid the need for the 4KB allocations on every file!
        [DllImport("Kernel32.dll", SetLastError = true)]
        public static extern bool ReadFile(SafeFileHandle handle, byte[] bytes, int numBytesToRead, out int numBytesRead, IntPtr mustBeZero);

        static void Main(string[] args)
        {
            BufferPool.GetBuffer(10000);
			// single threaded collection, we only use a single thread here
			var readDirectoryWork = new Queue<ReadDirectoryWork>();
			// queues for the various actors
			var crcComputationWork = new BlockingCollection<CrcComputationWork>();
            var readFileWork = new BlockingCollection<ReadFileWork>();

            SpinCrcComputations(crcComputationWork);

            SpinFileReaders(readFileWork, crcComputationWork);

			// Start the actors machinery by registering the initial directories
			foreach (var path in args)
			{
				readDirectoryWork.Enqueue(new ReadDirectoryWork
				{
					DirectoryName = path
				});
			}

            SpinDirectoryReader(readDirectoryWork, readFileWork);

          
            Console.WriteLine("Running computations...");
            Console.ReadLine();
        }

		private static void SpinDirectoryReader(Queue<ReadDirectoryWork> readDirectoryWork, BlockingCollection<ReadFileWork> readFileWork)
{'fraction_non_alphanumeric': 0.07488610478359908, 'fraction_numerical': 0.0056947608200455585, 'mean_word_length': 3.5329032258064514, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2398849', 'n_tokens_mistral': 982, 'n_tokens_neox': 879, 'n_words': 248}