<commit_before>package ru.r2cloud.jradio.lrpt;

import java.util.ArrayList;
import java.util.List;

public class VCDU {
	
	public static final int SIZE = 892;
	public static final int VITERBI_SIZE = (SIZE / 4 + 32 ) * 4;
	public static final int VITERBI_TAIL_SIZE = (VITERBI_SIZE + 1) * 16;

	private int version;
	private VCDUId id;
	private int counter;
	private byte signalling;
	private IN_SDU insertZone;
	private M_PDU mPdu;
	private List<Packet> packets = new ArrayList<>();
	private Packet partial;
	private byte[] data;

	public void readExternal(VCDU previous, byte[] data) {
		this.data = data;
		version = (data[0] & 0xFF) >> 6;
		id = new VCDUId();
		id.setSpacecraftId(data[0] & 0b0011_1111 + (byte) (data[1] >> 6));
		id.setVirtualChannelId(data[1] & 0b0011_1111);
		counter = (data[2] & 0xFF) << 16 | (data[3] & 0xFF) << 8 | (data[4] & 0xFF) << 0;
		signalling = data[5];
		insertZone = new IN_SDU();
		insertZone.setEncryption(data[6] == 0xFF);
		insertZone.setKeyNumber(data[7]);
		mPdu = new M_PDU();
		mPdu.setSpareBits((byte) (data[8] >> 3));
		mPdu.setHeaderFirstPointer(((data[8] & 0b0000_0111) << 8) | (data[9] & 0xFF));
		Packet previousPartial = null;
		if (previous != null && previous.counter + 1 == counter) {
			previousPartial = previous.partial;
		}
		if (mPdu.getHeaderFirstPointer() != 0b111_1111_1111) {
			if (mPdu.getHeaderFirstPointer() != 0 && previousPartial != null) {
				byte[] newUserData = new byte[previousPartial.getUserData().length + mPdu.getHeaderFirstPointer()];
				System.arraycopy(previousPartial.getUserData(), 0, newUserData, 0, previousPartial.getUserData().length);
				System.arraycopy(data, 10, newUserData, previousPartial.getUserData().length, mPdu.getHeaderFirstPointer());
				int userDataIndex = 0;
				// primary header was not read
				if (previousPartial.getVersion() == -1) {
					readPrimaryHeader(newUserData, 0, previousPartial);
					userDataIndex += 6;
				}
				// primary header was read, but secondary header was not.
				if (previousPartial.isSecondaryHeader() && previousPartial.getNumberOfDays() == -1) {
					readSecondaryHeader(newUserData, userDataIndex, previousPartial);
					userDataIndex += 8;
				}

				if (userDataIndex != 0) {
					byte[] userDataWithoutHeaders = new byte[newUserData.length - userDataIndex];
					System.arraycopy(newUserData, userDataIndex, userDataWithoutHeaders, 0, userDataWithoutHeaders.length);
					previousPartial.setUserData(userDataWithoutHeaders);
				} else {
					previousPartial.setUserData(newUserData);
				}

				packets.add(previousPartial);
			}
			int index = 10 + mPdu.getHeaderFirstPointer();
			// 6 is for minimum header size
			while (data.length >= index + 6) {
				Packet packet = new Packet();
				readPrimaryHeader(data, index, packet);
				// for +1 see the length field description
				int userDataLength = packet.getLength() + 1;
				int userDataIndex;
				if (packet.isSecondaryHeader() && data.length >= index + 6 + 8) {
					readSecondaryHeader(data, index + 6, packet);
					userDataLength -= 8;
					// 9 + 1
					userDataIndex = 14;
				} else {
					// 5 + 1
					userDataIndex = 6;
				}
<commit_msg>validate partial packet length<commit_after>package ru.r2cloud.jradio.lrpt;

import java.util.ArrayList;
import java.util.List;

public class VCDU {
	
	public static final int SIZE = 892;
	public static final int VITERBI_SIZE = (SIZE / 4 + 32 ) * 4;
	public static final int VITERBI_TAIL_SIZE = (VITERBI_SIZE + 1) * 16;

	private int version;
	private VCDUId id;
	private int counter;
	private byte signalling;
	private IN_SDU insertZone;
	private M_PDU mPdu;
	private List<Packet> packets = new ArrayList<>();
	private Packet partial;
	private byte[] data;

	public void readExternal(VCDU previous, byte[] data) {
		this.data = data;
		version = (data[0] & 0xFF) >> 6;
		id = new VCDUId();
		id.setSpacecraftId(data[0] & 0b0011_1111 + (byte) (data[1] >> 6));
		id.setVirtualChannelId(data[1] & 0b0011_1111);
		counter = (data[2] & 0xFF) << 16 | (data[3] & 0xFF) << 8 | (data[4] & 0xFF) << 0;
		signalling = data[5];
		insertZone = new IN_SDU();
		insertZone.setEncryption(data[6] == 0xFF);
		insertZone.setKeyNumber(data[7]);
		mPdu = new M_PDU();
		mPdu.setSpareBits((byte) (data[8] >> 3));
		mPdu.setHeaderFirstPointer(((data[8] & 0b0000_0111) << 8) | (data[9] & 0xFF));
		Packet previousPartial = null;
		if (previous != null && previous.counter + 1 == counter) {
			previousPartial = previous.partial;
		}
		if (mPdu.getHeaderFirstPointer() != 0b111_1111_1111) {
			if (mPdu.getHeaderFirstPointer() != 0 && previousPartial != null) {
				byte[] newUserData = new byte[previousPartial.getUserData().length + mPdu.getHeaderFirstPointer()];
				System.arraycopy(previousPartial.getUserData(), 0, newUserData, 0, previousPartial.getUserData().length);
				System.arraycopy(data, 10, newUserData, previousPartial.getUserData().length, mPdu.getHeaderFirstPointer());
				int userDataIndex = 0;
				// primary header was not read
				if (previousPartial.getVersion() == -1) {
					readPrimaryHeader(newUserData, 0, previousPartial);
					userDataIndex += 6;
				}
				// primary header was read, but secondary header was not.
				if (previousPartial.isSecondaryHeader() && previousPartial.getNumberOfDays() == -1) {
					readSecondaryHeader(newUserData, userDataIndex, previousPartial);
					userDataIndex += 8;
				}

				if (userDataIndex != 0) {
					byte[] userDataWithoutHeaders = new byte[newUserData.length - userDataIndex];
					System.arraycopy(newUserData, userDataIndex, userDataWithoutHeaders, 0, userDataWithoutHeaders.length);
					previousPartial.setUserData(userDataWithoutHeaders);
				} else {
					previousPartial.setUserData(newUserData);
				}
				
				//sometimes user data cannot be recovered even if VCDU is next
				int expectedLength = previousPartial.getLength() + 1;
				if (previousPartial.isSecondaryHeader()) {
					expectedLength -= 8;
				}
				if (previousPartial.getUserData().length == expectedLength) {
					packets.add(previousPartial);
				}

			}
			int index = 10 + mPdu.getHeaderFirstPointer();
			// 6 is for minimum header size
			while (data.length >= index + 6) {
				Packet packet = new Packet();
				readPrimaryHeader(data, index, packet);
				// for +1 see the length field description
				int userDataLength = packet.getLength() + 1;
				int userDataIndex;
				if (packet.isSecondaryHeader() && data.length >= index + 6 + 8) {
					readSecondaryHeader(data, index + 6, packet);
					userDataLength -= 8;
					// 9 + 1
					userDataIndex = 14;
				} else {
					// 5 + 1
					userDataIndex = 6;
				}

{'fraction_non_alphanumeric': 0.12884644535394876, 'fraction_numerical': 0.03304532363195392, 'mean_word_length': 3.830161054172767, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1781442', 'n_tokens_mistral': 2647, 'n_tokens_neox': 2123, 'n_words': 530}