<commit_before>﻿using System;
using System.IO;
using System.Diagnostics;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System.Runtime.CompilerServices;

namespace DemoInfo.BitStreamImpl
{
	public unsafe class UnsafeBitStream : IBitStream
	{
		private const int SLED = 4;
		private const int BUFSIZE = 2048 + SLED;

		private int Offset;
		private Stream Underlying;
		private GCHandle HBuffer;
		private byte* PBuffer;
		private byte[] Buffer = new byte[BUFSIZE];

		private int BitsInBuffer;

		private readonly Stack<long> ChunkTargets = new Stack<long>();
		private long LazyGlobalPosition = 0;
		private long ActualGlobalPosition { get { return LazyGlobalPosition + Offset; } }

		public void Initialize(Stream underlying)
		{
			HBuffer = GCHandle.Alloc(Buffer, GCHandleType.Pinned);
			PBuffer = (byte*)HBuffer.AddrOfPinnedObject().ToPointer();

			this.Underlying = underlying;
			RefillBuffer();

			Offset = SLED * 8;
		}

		void IDisposable.Dispose()
		{
			PBuffer = (byte*)IntPtr.Zero.ToPointer(); // null it out
			HBuffer.Free();
			Buffer = null;
		}

		[MethodImpl(MethodImplOptions.AggressiveInlining)]
		private bool TryAdvance(int howMuch)
		{
			/*
			 * So the problem is: Apparently mono can't inline the old Advance() because
			 * that would mess up the call stack: Advance->RefillBuffer->Stream.Read
			 * which could then throw. Advance's stack frame would be missing.
			 *
			 * Because of that, the call to RefillBuffer has to be inlined manually:
			 * if (TryAdvance(howMuch)) RefillBuffer();
			 *
			 * Ugly, but it works.
			 */
			return (Offset += howMuch) >= BitsInBuffer;
		}

		private void RefillBuffer()
		{
			do {<commit_msg>UnsafeBitStream will never leak memory again.<commit_after>﻿using System;
using System.IO;
using System.Diagnostics;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System.Runtime.CompilerServices;

namespace DemoInfo.BitStreamImpl
{
	public unsafe class UnsafeBitStream : IBitStream
	{
		private const int SLED = 4;
		private const int BUFSIZE = 2048 + SLED;

		private int Offset;
		private Stream Underlying;
		private GCHandle HBuffer;
		private byte* PBuffer;
		private byte[] Buffer = new byte[BUFSIZE];

		private int BitsInBuffer;

		private readonly Stack<long> ChunkTargets = new Stack<long>();
		private long LazyGlobalPosition = 0;
		private long ActualGlobalPosition { get { return LazyGlobalPosition + Offset; } }

		public void Initialize(Stream underlying)
		{
			HBuffer = GCHandle.Alloc(Buffer, GCHandleType.Pinned);
			PBuffer = (byte*)HBuffer.AddrOfPinnedObject().ToPointer();

			this.Underlying = underlying;
			RefillBuffer();

			Offset = SLED * 8;
		}

		~UnsafeBitStream()
		{
			Dispose();
		}

		void IDisposable.Dispose()
		{
			Dispose();
			GC.SuppressFinalize(this);
		}

		private void Dispose()
		{
			var nullptr = (byte*)IntPtr.Zero.ToPointer();
			if (PBuffer != nullptr) {
				// GCHandle docs state that Free() must only be called once.
				// So we use PBuffer to ensure that.
				PBuffer = nullptr;
				HBuffer.Free();
			}
		}

		[MethodImpl(MethodImplOptions.AggressiveInlining)]
		private bool TryAdvance(int howMuch)
		{
			/*
			 * So the problem is: Apparently mono can't inline the old Advance() because
			 * that would mess up the call stack: Advance->RefillBuffer->Stream.Read
			 * which could then throw. Advance's stack frame would be missing.
			 *
			 * Because of that, the call to RefillBuffer has to be inlined manually:
			 * if (TryAdvance(howMuch)) RefillBuffer();
			 *
			 * Ugly, but it works.
			 */
			return (Offset += howMuch) >= BitsInBuffer;
		}

		private void RefillBuffer()
		{
			do {
{'fraction_non_alphanumeric': 0.09758742206560043, 'fraction_numerical': 0.003795066413662239, 'mean_word_length': 3.6946564885496183, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6185093', 'n_tokens_mistral': 1335, 'n_tokens_neox': 1106, 'n_words': 338}