<commit_before>﻿using NBitcoin.DataEncoders;
using NBitcoin.Protocol;
using Org.BouncyCastle.Crypto.Digests;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Numerics;
using System.Reflection;
using System.Runtime.Serialization;
using System.Security;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace NBitcoin
{
	public static class Extensions
	{
		public static byte[] ReadBytes(this Stream stream, int count)
		{
			byte[] result = new byte[count];
			stream.Read(result, 0, count);
			return result;
		}
		public static IEnumerable<T> Resize<T>(this List<T> list, int count)
		{
			if(list.Count == count)
				return new T[0];

			List<T> removed = new List<T>();

			for(int i = list.Count - 1 ; i + 1 > count ; i--)
			{
				removed.Add(list[i]);
				list.RemoveAt(i);
			}

			while(list.Count < count)
			{
				list.Add(default(T));
			}
			return removed;
		}
		public static IEnumerable<List<T>> Partition<T>(this IEnumerable<T> source, int max)
		{
			return Partition<T>(source, () => max);
		}
		public static IEnumerable<List<T>> Partition<T>(this IEnumerable<T> source, Func<int> max)
		{
			var partitionSize = max();
			List<T> toReturn = new List<T>(partitionSize);
			foreach(var item in source)
			{
				toReturn.Add(item);
				if(toReturn.Count == partitionSize)
				{
					yield return toReturn;
					toReturn = new List<T>(partitionSize);
				}
			}
			if(toReturn.Any())
			{
				yield return toReturn;
			}
		}



		public static int ReadEx(this Stream stream, byte[] buffer, int offset, int count, CancellationToken cancellation = default(CancellationToken))
		{
			int readen = 0;
			while(readen < count)
			{
				int thisRead = 0;
				if(stream is NetworkStream) //Big performance problem with begin read for other stream than NetworkStream
				{
					var ar = stream.BeginRead(buffer, offset + readen, count - readen, null, null);
					if(!ar.CompletedSynchronously)
					{
						WaitHandle.WaitAny(new WaitHandle[] { ar.AsyncWaitHandle, cancellation.WaitHandle }, -1);
					}
					cancellation.ThrowIfCancellationRequested();
					thisRead = stream.EndRead(ar);
					if(thisRead == 0 && (stream is Message.CustomNetworkStream) && ((Message.CustomNetworkStream)stream).Connected)
					{
						return -1;
					}
				}
				else
				{
					cancellation.ThrowIfCancellationRequested();
					thisRead = stream.Read(buffer, offset + readen, count - readen);
				}
				if(thisRead == -1)
					return -1;
				if(thisRead == 0 && (stream is FileStream || stream is MemoryStream))
				{
					if(stream.Length == stream.Position)
						return -1;
				}
				readen += thisRead;
			}
			return readen;
		}
		public static void AddOrReplace<TKey, TValue>(this Dictionary<TKey, TValue> dico, TKey key, TValue value)
		{
			if(dico.ContainsKey(key))
			{
				dico.Remove(key);
				dico.Add(key, value);
			}
			else
			{
				dico.Add(key, value);
			}
		}
		public static bool TryAdd<TKey, TValue>(this Dictionary<TKey, TValue> dictionary, TKey key, TValue value)
		{
			if(!dictionary.ContainsKey(key))
			{
				dictionary.Add(key, value);
				return true;
			}
			return false;
		}<commit_msg>generalize dictionary helper<commit_after>﻿using NBitcoin.DataEncoders;
using NBitcoin.Protocol;
using Org.BouncyCastle.Crypto.Digests;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Numerics;
using System.Reflection;
using System.Runtime.Serialization;
using System.Security;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace NBitcoin
{
	public static class Extensions
	{
		public static byte[] ReadBytes(this Stream stream, int count)
		{
			byte[] result = new byte[count];
			stream.Read(result, 0, count);
			return result;
		}
		public static IEnumerable<T> Resize<T>(this List<T> list, int count)
		{
			if(list.Count == count)
				return new T[0];

			List<T> removed = new List<T>();

			for(int i = list.Count - 1 ; i + 1 > count ; i--)
			{
				removed.Add(list[i]);
				list.RemoveAt(i);
			}

			while(list.Count < count)
			{
				list.Add(default(T));
			}
			return removed;
		}
		public static IEnumerable<List<T>> Partition<T>(this IEnumerable<T> source, int max)
		{
			return Partition<T>(source, () => max);
		}
		public static IEnumerable<List<T>> Partition<T>(this IEnumerable<T> source, Func<int> max)
		{
			var partitionSize = max();
			List<T> toReturn = new List<T>(partitionSize);
			foreach(var item in source)
			{
				toReturn.Add(item);
				if(toReturn.Count == partitionSize)
				{
					yield return toReturn;
					toReturn = new List<T>(partitionSize);
				}
			}
			if(toReturn.Any())
			{
				yield return toReturn;
			}
		}



		public static int ReadEx(this Stream stream, byte[] buffer, int offset, int count, CancellationToken cancellation = default(CancellationToken))
		{
			int readen = 0;
			while(readen < count)
			{
				int thisRead = 0;
				if(stream is NetworkStream) //Big performance problem with begin read for other stream than NetworkStream
				{
					var ar = stream.BeginRead(buffer, offset + readen, count - readen, null, null);
					if(!ar.CompletedSynchronously)
					{
						WaitHandle.WaitAny(new WaitHandle[] { ar.AsyncWaitHandle, cancellation.WaitHandle }, -1);
					}
					cancellation.ThrowIfCancellationRequested();
					thisRead = stream.EndRead(ar);
					if(thisRead == 0 && (stream is Message.CustomNetworkStream) && ((Message.CustomNetworkStream)stream).Connected)
					{
						return -1;
					}
				}
				else
				{
					cancellation.ThrowIfCancellationRequested();
					thisRead = stream.Read(buffer, offset + readen, count - readen);
				}
				if(thisRead == -1)
					return -1;
				if(thisRead == 0 && (stream is FileStream || stream is MemoryStream))
				{
					if(stream.Length == stream.Position)
						return -1;
				}
				readen += thisRead;
			}
			return readen;
		}
		public static void AddOrReplace<TKey, TValue>(this IDictionary<TKey, TValue> dico, TKey key, TValue value)
		{
			if(dico.ContainsKey(key))
			{
				dico.Remove(key);
				dico.Add(key, value);
			}
			else
			{
				dico.Add(key, value);
			}
		}
		public static bool TryAdd<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key, TValue value)
		{
			if(!dictionary.ContainsKey(key))
			{
				dictionary.Add(key, value);
				return true;
			}
			return false;
		}
{'fraction_non_alphanumeric': 0.12312811980033278, 'fraction_numerical': 0.003932839207381637, 'mean_word_length': 3.307491856677524, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '869085', 'n_tokens_mistral': 2680, 'n_tokens_neox': 2172, 'n_words': 528}