<commit_before>using System;
using System.Linq;
using Xunit;

namespace CnDream.Core.Test
{
    public class PackerTests
    {
        [Fact]
        public void SmallPayload()
        {
            var packer = new DataPacker(new IdentityTransformer());
            var output = new byte[32];
            var input = new byte[5];

            new Random().NextBytes(input);

            var bytesWritten = packer.PackData(1, 2, new ArraySegment<byte>(input), new ArraySegment<byte>(output));

            VerifyOutput(input, output, bytesWritten);
        }

        [Fact]
        public void LargePayload()
        {
            var packer = new DataPacker(new IdentityTransformer());
            var output = new byte[160];
            var input = new byte[128];

            new Random().NextBytes(input);

            var bytesWritten = packer.PackData(1, 2, new ArraySegment<byte>(input), new ArraySegment<byte>(output));

            VerifyOutput(input, output, bytesWritten);
        }

        private void VerifyOutput( byte[] input, byte[] output, int bytesWritten )
        {
            Assert.True(bytesWritten > input.Length);
            Assert.True(bytesWritten > 0 && bytesWritten <= output.Length);

            var pos = ParseHeader(output, bytesWritten);
            Assert.True(pos > 0 && pos < bytesWritten - 1);

            var pairId = DataUnpacker.ReadInt(output, ref pos);
            Assert.Equal(1, pairId);

            var serialId = DataUnpacker.ReadInt(output, ref pos);
            Assert.Equal(2, serialId);

            var payloadSize = DataUnpacker.ReadInt(output, ref pos);
            Assert.Equal(input.Length, payloadSize);

            Assert.True(Enumerable.SequenceEqual(output.Skip(pos).Take(input.Length), input));<commit_msg>condition<commit_after>using System;
using System.Linq;
using Xunit;

namespace CnDream.Core.Test
{
    public class PackerTests
    {
        [Fact]
        public void SmallPayload()
        {
            var packer = new DataPacker(new IdentityTransformer());
            var output = new byte[32];
            var input = new byte[5];

            new Random().NextBytes(input);

            var bytesWritten = packer.PackData(1, 2, new ArraySegment<byte>(input), new ArraySegment<byte>(output));

            VerifyOutput(input, output, bytesWritten);
        }

        [Fact]
        public void LargePayload()
        {
            var packer = new DataPacker(new IdentityTransformer());
            var output = new byte[160];
            var input = new byte[128];

            new Random().NextBytes(input);

            var bytesWritten = packer.PackData(1, 2, new ArraySegment<byte>(input), new ArraySegment<byte>(output));

            VerifyOutput(input, output, bytesWritten);
        }

        private void VerifyOutput( byte[] input, byte[] output, int bytesWritten )
        {
            Assert.True(bytesWritten > input.Length);
            Assert.True(bytesWritten <= output.Length);
            Assert.True(bytesWritten % (new IdentityTransformer().OutputBlockSize) == 0);

            var pos = ParseHeader(output, bytesWritten);
            Assert.True(pos > 0 && pos < bytesWritten - 1);

            var pairId = DataUnpacker.ReadInt(output, ref pos);
            Assert.Equal(1, pairId);

            var serialId = DataUnpacker.ReadInt(output, ref pos);
            Assert.Equal(2, serialId);

            var payloadSize = DataUnpacker.ReadInt(output, ref pos);
            Assert.Equal(input.Length, payloadSize);

            Assert.True(Enumerable.SequenceEqual(output.Skip(pos).Take(input.Length), input));
{'fraction_non_alphanumeric': 0.10923672566371681, 'fraction_numerical': 0.00995575221238938, 'mean_word_length': 2.2497753818508537, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7459683', 'n_tokens_mistral': 1006, 'n_tokens_neox': 947, 'n_words': 237}