<commit_before>﻿using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Text;

namespace %%%PROJECT_ID%%%
{
	internal static class ResourceReader
	{
        public static string GetResourceManifest()
        {
            return ReadTextFile("Resources/ResourceMapping.txt") ?? "";
        }

		public static string ReadTextFile(string path)
		{
			IList<byte> bytes = ReadBytes(path);
            if (bytes == null) return null;
            bool hasBom = bytes.Count >= 3 && bytes[0] == 239 && bytes[1] == 187 && bytes[2] == 191;
			StringBuilder output = new StringBuilder(bytes.Count); 
			output.Append(bytes.Skip<byte>(hasBom ? 3 : 0).Select<byte, char>(b => (char)b).ToArray<char>());
			return output.ToString();
		}

		public static string ReadByteCodeFile()
		{
			return ReadTextFile("Resources/ByteCode.txt");
		}
		
        public static System.Drawing.Bitmap ReadImageFile(string path)
        {
            IList<byte> data = ReadBytes(path);
            if (data == null)
            {
                return null;
            }
            return (Bitmap)Bitmap.FromStream(new MemoryStream(data.ToArray()));
        }

		// C# does not like it when you access resources from different threads.
        private static object resMutex = new object();

        public static System.IO.Stream GetResourceStream(string path)
        {
            lock (resMutex)
            {
                System.Reflection.Assembly assembly = typeof(ResourceReader).Assembly;
                path = assembly.GetName().Name + "." + path.Replace('\\', '.').Replace('/', '.').TrimStart('.');
                return assembly.GetManifestResourceStream(path);
            }
        }

		private const int BUFFER_LENGTH = 1000;
		private static readonly byte[] BUFFER = new byte[BUFFER_LENGTH];

		public static List<byte> ReadBytes(string path)
		{
			System.IO.Stream stream = GetResourceStream(path);
			if (stream == null)
			{
				return null;
			}
			StringBuilder sb = new StringBuilder();
			List<byte> output = new List<byte>();

			int bytesRead = 1;
			while (bytesRead > 0)
			{
				bytesRead = stream.Read(BUFFER, 0, BUFFER_LENGTH);
				if (bytesRead == BUFFER_LENGTH)
				{
					output.AddRange(BUFFER);
				}
				else
				{
					for (int i = 0; i < bytesRead; ++i)
					{
						output.Add(BUFFER[i]);
					}
					bytesRead = 0;
				}
			}
			return output;
		}
	}
}<commit_msg>Make the ResourceReader mutex in C# projects a little stronger. Was still running into threading issues when finally tested on a large project.<commit_after>﻿using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Text;

namespace %%%PROJECT_ID%%%
{
	internal static class ResourceReader
	{
        public static string GetResourceManifest()
        {
            return ReadTextFile("Resources/ResourceMapping.txt") ?? "";
        }

		public static string ReadTextFile(string path)
		{
			IList<byte> bytes = ReadBytes(path);
            if (bytes == null) return null;
            bool hasBom = bytes.Count >= 3 && bytes[0] == 239 && bytes[1] == 187 && bytes[2] == 191;
			StringBuilder output = new StringBuilder(bytes.Count); 
			output.Append(bytes.Skip<byte>(hasBom ? 3 : 0).Select<byte, char>(b => (char)b).ToArray<char>());
			return output.ToString();
		}

		public static string ReadByteCodeFile()
		{
			return ReadTextFile("Resources/ByteCode.txt");
		}
		
        public static System.Drawing.Bitmap ReadImageFile(string path)
        {
            IList<byte> data = ReadBytes(path);
            if (data == null)
            {
                return null;
            }
            return (Bitmap)Bitmap.FromStream(new MemoryStream(data.ToArray()));
        }

		// C# does not like it when you access resources from different threads.
        private static object resMutex = new object();

        public static System.IO.Stream GetResourceStream(string path)
        {
            System.Reflection.Assembly assembly = typeof(ResourceReader).Assembly;
            path = assembly.GetName().Name + "." + path.Replace('\\', '.').Replace('/', '.').TrimStart('.');
            return assembly.GetManifestResourceStream(path);
        }

		private const int BUFFER_LENGTH = 1000;
		private static readonly byte[] BUFFER = new byte[BUFFER_LENGTH];

		public static List<byte> ReadBytes(string path)
		{
            lock (resMutex)
            {
				System.IO.Stream stream = GetResourceStream(path);
				if (stream == null)
				{
					return null;
				}
				StringBuilder sb = new StringBuilder();
				List<byte> output = new List<byte>();

				int bytesRead = 1;
				while (bytesRead > 0)
				{
					bytesRead = stream.Read(BUFFER, 0, BUFFER_LENGTH);
					if (bytesRead == BUFFER_LENGTH)
					{
						output.AddRange(BUFFER);
					}
					else
					{
						for (int i = 0; i < bytesRead; ++i)
						{
							output.Add(BUFFER[i]);
						}
						bytesRead = 0;
					}
				}
				return output;
			}
		}
	}
}
{'fraction_non_alphanumeric': 0.1226, 'fraction_numerical': 0.0096, 'mean_word_length': 2.4371134020618554, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6259557', 'n_tokens_mistral': 1694, 'n_tokens_neox': 1422, 'n_words': 368}