<commit_before>/*
 * (C) YANDEX LLC, 2014
 *
 * The Source Code called "YoctoDB" available at
 * https://bitbucket.org/yandex/yoctodb is subject to the terms of the
 * Mozilla Public License, v. 2.0 (hereinafter - MPL).
 *
 * A copy of the MPL is available at http://mozilla.org/MPL/2.0/.
 */

package com.yandex.yoctodb.v1.mutable.segment;

import com.yandex.yoctodb.util.buf.Buffer;
import org.junit.Assert;
import org.junit.Test;
import com.yandex.yoctodb.util.OutputStreamWritable;
import com.yandex.yoctodb.v1.V1DatabaseFormat;

import java.io.ByteArrayOutputStream;
import java.io.IOException;

import static com.yandex.yoctodb.v1.mutable.segment.Utils.calculateDigest;

/**
 * @author svyatoslav
 *         Date: 08.11.13
 */
public class V1PayloadSegmentBinaryTest {

    @Test
    public void writingPayloadTest() throws IOException {
        final V1PayloadSegment v1PayloadSegment = new V1PayloadSegment();
        for (int i = 0; i < 15; i++) {
            final String payload = "payload" + i;
            v1PayloadSegment.addDocument(i, payload.getBytes());
        }

        final ByteArrayOutputStream os = new ByteArrayOutputStream();
        final OutputStreamWritable outputStreamWritable = v1PayloadSegment.buildWritable();
        outputStreamWritable.writeTo(os);
        Assert.assertEquals(os.size(), outputStreamWritable.getSizeInBytes() + 12);

        final Buffer byteBuffer = Buffer.from(os.toByteArray());

        final long fullSizeInBytes = byteBuffer.getLong();
        Assert.assertEquals(fullSizeInBytes, outputStreamWritable.getSizeInBytes());

        final int payloadCode = byteBuffer.getInt();
        Assert.assertEquals(V1DatabaseFormat.SegmentType.PAYLOAD.getCode(), payloadCode);

        final byte[] digest = calculateDigest(byteBuffer, V1DatabaseFormat.MESSAGE_DIGEST_ALGORITHM);

        final long elementsSizeInBytes = byteBuffer.getLong();
        Assert.assertEquals(193, elementsSizeInBytes);

        final int elementsCount = byteBuffer.getInt();
        Assert.assertEquals(15, elementsCount);

        final int[] offsets = new int[elementsCount + 1];

        for (int i = 0; i <= elementsCount; i++) {
            offsets[i] = byteBuffer.getInt();
        }
        Assert.assertArrayEquals(new int[]{0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 89, 98, 107, 116, 125}, offsets);

        for (int i = 0; i < elementsCount; i++) {
            final byte[] currentElementBytes = new byte[offsets[i + 1] - offsets[i]];
            byteBuffer.get(currentElementBytes);
            Assert.assertEquals(("payload" + i), new String(currentElementBytes));
        }

        final long digestSize = byteBuffer.getLong();
        Assert.assertEquals(V1DatabaseFormat.DIGEST_SIZE_IN_BYTES, digestSize);
        for (int i = 0; i < digestSize; i++) {
            byte actualDigestByte = byteBuffer.get();
            Assert.assertEquals(digest[i], actualDigestByte);
        }

        //check that input has not remaining
        Assert.assertFalse(byteBuffer.hasRemaining());
    }
}<commit_msg>Fixed unit test after #6<commit_after>/*
 * (C) YANDEX LLC, 2014
 *
 * The Source Code called "YoctoDB" available at
 * https://bitbucket.org/yandex/yoctodb is subject to the terms of the
 * Mozilla Public License, v. 2.0 (hereinafter - MPL).
 *
 * A copy of the MPL is available at http://mozilla.org/MPL/2.0/.
 */

package com.yandex.yoctodb.v1.mutable.segment;

import com.yandex.yoctodb.util.buf.Buffer;
import org.junit.Assert;
import org.junit.Test;
import com.yandex.yoctodb.util.OutputStreamWritable;
import com.yandex.yoctodb.v1.V1DatabaseFormat;

import java.io.ByteArrayOutputStream;
import java.io.IOException;

import static com.yandex.yoctodb.v1.mutable.segment.Utils.calculateDigest;

/**
 * @author svyatoslav
 *         Date: 08.11.13
 */
public class V1PayloadSegmentBinaryTest {

    @Test
    public void writingPayloadTest() throws IOException {
        final V1PayloadSegment v1PayloadSegment = new V1PayloadSegment();
        for (int i = 0; i < 15; i++) {
            final String payload = "payload" + i;
            v1PayloadSegment.addDocument(i, payload.getBytes());
        }

        final ByteArrayOutputStream os = new ByteArrayOutputStream();
        final OutputStreamWritable outputStreamWritable = v1PayloadSegment.buildWritable();
        outputStreamWritable.writeTo(os);
        Assert.assertEquals(os.size(), outputStreamWritable.getSizeInBytes() + 12);

        final Buffer byteBuffer = Buffer.from(os.toByteArray());

        final long fullSizeInBytes = byteBuffer.getLong();
        Assert.assertEquals(fullSizeInBytes, outputStreamWritable.getSizeInBytes());

        final int payloadCode = byteBuffer.getInt();
        Assert.assertEquals(V1DatabaseFormat.SegmentType.PAYLOAD.getCode(), payloadCode);

        final byte[] digest = calculateDigest(byteBuffer, V1DatabaseFormat.MESSAGE_DIGEST_ALGORITHM);

        final long elementsSizeInBytes = byteBuffer.getLong();
        Assert.assertEquals(257, elementsSizeInBytes);

        final int elementsCount = byteBuffer.getInt();
        Assert.assertEquals(15, elementsCount);

        final long[] offsets = new long[elementsCount + 1];

        for (int i = 0; i <= elementsCount; i++) {
            offsets[i] = byteBuffer.getLong();
        }
        Assert.assertArrayEquals(new long[]{0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 89, 98, 107, 116, 125}, offsets);

        for (int i = 0; i < elementsCount; i++) {
            final byte[] currentElementBytes = new byte[(int) (offsets[i + 1] - offsets[i])];
            byteBuffer.get(currentElementBytes);
            Assert.assertEquals(("payload" + i), new String(currentElementBytes));
        }

        final long digestSize = byteBuffer.getLong();
        Assert.assertEquals(V1DatabaseFormat.DIGEST_SIZE_IN_BYTES, digestSize);
        for (int i = 0; i < digestSize; i++) {
            byte actualDigestByte = byteBuffer.get();
            Assert.assertEquals(digest[i], actualDigestByte);
        }

        //check that input has not remaining
        Assert.assertFalse(byteBuffer.hasRemaining());
    }
}
{'fraction_non_alphanumeric': 0.1140193031244888, 'fraction_numerical': 0.024701455913626696, 'mean_word_length': 3.159183673469388, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1468566', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1770, 'n_words': 425}