<commit_before>﻿using System;
using System.IO;
using SpssLib.FileParser.Records;

namespace SpssLib.FileParser
{
	/// <summary>
	/// Provides the functionality to write the compressed values to the underlying writer
	/// </summary>
	public class CompressedRecordWriter : IRecordWriter
	{
		/// <summary>
		/// Underlying stream writer
		/// </summary>
		private readonly BinaryWriter _writer;
		/// <summary>
		/// The compression bias (compressed numbers will be stored as value + bias) 
		/// </summary>
		private readonly double _bias;
		/// <summary>
		/// The system missing value
		/// </summary>
		private readonly double _sysMiss;

		/// <summary>
		/// Next index in the block that is to be written (from 0 to 7)
		/// </summary>
		private int _blockIndex = 0;
		/// <summary>
		/// Next index to be written on the uncompressed data buffer. This should be incremented in steps of 8
		/// </summary>
		private int _uncompressedIndex = 0;
		/// <summary>
		/// The buffer to accumulate the uncompressed data. The uncompressed data item is written in a 8 byte block, 
		/// and the it can have a max of 8 items (the size of the compressed block)
		/// </summary>
		private byte[] _uncompressedBuffer = new byte[Constants.BlockByteSize * Constants.BlockByteSize];
		
		private const byte Padding = 0;
		// 1 to 251 are the compressed values
		private const byte EndOfFile = 252;
		private const byte UncompressedValue = 253;
		private const byte SpaceCharsBlock = 254;
		private const byte SysmissCode = 255;

		/// <summary>
		/// Creates a compressed recodr writer
		/// </summary>
		/// <param name="writer">The underlying binary writer with the stream</param>
		/// <param name="bias">The compression bias</param>
		/// <param name="sysMiss">The system missing value</param>
		public CompressedRecordWriter(BinaryWriter writer, double bias, double sysMiss)
		{
			_writer = writer;
			_bias = bias;
			_sysMiss = sysMiss;
		}

		/// <summary>
		/// Writes a sysmiss value on the stream.
		/// This is to be used for null values mostly
		/// </summary>
		public void WriteSysMiss()
		{
			WriteCompressedCode(SysmissCode);
		}

		/// <summary><commit_msg>Fixed writing compressed sysmiss bug<commit_after>﻿using System;
using System.IO;
using SpssLib.FileParser.Records;

namespace SpssLib.FileParser
{
	/// <summary>
	/// Provides the functionality to write the compressed values to the underlying writer
	/// </summary>
	public class CompressedRecordWriter : IRecordWriter
	{
		/// <summary>
		/// Underlying stream writer
		/// </summary>
		private readonly BinaryWriter _writer;
		/// <summary>
		/// The compression bias (compressed numbers will be stored as value + bias) 
		/// </summary>
		private readonly double _bias;
		/// <summary>
		/// The system missing value
		/// </summary>
		private readonly double _sysMiss;

		/// <summary>
		/// Next index in the block that is to be written (from 0 to 7)
		/// </summary>
		private int _blockIndex = 0;
		/// <summary>
		/// Next index to be written on the uncompressed data buffer. This should be incremented in steps of 8
		/// </summary>
		private int _uncompressedIndex = 0;
		/// <summary>
		/// The buffer to accumulate the uncompressed data. The uncompressed data item is written in a 8 byte block, 
		/// and the it can have a max of 8 items (the size of the compressed block)
		/// </summary>
		private byte[] _uncompressedBuffer = new byte[Constants.BlockByteSize * Constants.BlockByteSize];
		
		private const byte Padding = 0;
		// 1 to 251 are the compressed values
		private const byte EndOfFile = 252;
		private const byte UncompressedValue = 253;
		private const byte SpaceCharsBlock = 254;
		private const byte SysmissCode = 255;

		/// <summary>
		/// Creates a compressed recodr writer
		/// </summary>
		/// <param name="writer">The underlying binary writer with the stream</param>
		/// <param name="bias">The compression bias</param>
		/// <param name="sysMiss">The system missing value</param>
		public CompressedRecordWriter(BinaryWriter writer, double bias, double sysMiss)
		{
			_writer = writer;
			_bias = bias;
			_sysMiss = sysMiss;
		}

		/// <summary>
		/// Writes a sysmiss value on the stream.
		/// This is to be used for null values mostly
		/// </summary>
		public void WriteSysMiss()
		{
			WriteCompressedCode(SysmissCode);
			CheckBlock();
		}

		/// <summary>
{'fraction_non_alphanumeric': 0.11154374713696748, 'fraction_numerical': 0.010994044892349977, 'mean_word_length': 3.024884792626728, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2548317', 'n_tokens_mistral': 1427, 'n_tokens_neox': 1223, 'n_words': 484}