<commit_before>package org.voovan.network.messagesplitter;

import java.math.BigInteger;
import java.nio.ByteBuffer;

import org.voovan.network.IoSession;
import org.voovan.network.MessageSplitter;
import org.voovan.tools.TString;

/**
 * Http 消息分割类
 * 
 * @author helyho
 *
 * Voovan Framework.
 * WebSite: https://github.com/helyho/Voovan
 * Licence: Apache v2 License
 */
public class HttpMessageSplitter implements MessageSplitter {

	private static final String	BODY_TAG	= "\r\n\r\n";

	@Override
	public boolean canSplite(IoSession session, byte[] buffer) {

		if (isHttpFrame(buffer)) {
			return true;
		} else if (isWebSocketFrame(ByteBuffer.wrap(buffer))) {
			return true;
		}

		return false;
	}

	public static boolean isHttpFrame(byte[] buffer) {
		String bufferString = new String(buffer);
		String[] boundaryLines = TString.searchByRegex(bufferString, "boundary=[^ \\r\\n]+");
		String[] contentLengthLines = TString.searchByRegex(bufferString, "Content-Length: \\d+");

		// 包含\r\n\r\n,这个时候报文有可能加载完成
		if (bufferString.contains(BODY_TAG)) {
			// 1.包含 content Length 的则通过获取 contentLenght 来计算报文的总长度,长度相等时,返回成功
			if (contentLengthLines.length == 1) {
				int contentLength = Integer.valueOf(contentLengthLines[0].split(" ")[1].trim());
				int totalLength = bufferString.indexOf(BODY_TAG) + 4 + contentLength;
				if (buffer.length >= totalLength) {
					return true;
				}
			}
		
			// 2. 如果是 HTTP 请求报文
			// POST方法的multipart/form-data类型,且没有指定ContentLength,则需要使用--boundary--的结尾形式来判断
			if (bufferString.contains("multipart/form-data") 
					&& boundaryLines.length == 1 
					&& bufferString.trim().endsWith("--" + boundaryLines[0] + "--")) {
				return true;
			}
			
			// 3. 如果是 HTTP 响应报文 chunk
			// 则trim 后判断最后一个字符是否是 0
			if (bufferString.contains("chunked") 
					&& bufferString.trim().endsWith("\r\n0")) {
				return true;
			}
			
			// 4 HEAD,CONNECT,DELETE,GET,TRACE,OPTIONS等请求,没有报文内容
			if (bufferString.startsWith("GET") 
					|| bufferString.startsWith("TRACE") 
					|| bufferString.startsWith("OPTIONS")
					|| bufferString.startsWith("HEAD") 
					|| bufferString.startsWith("DELETE") 
					|| bufferString.startsWith("CONNECT")) {
				return true;
			}
		}

		return false;
	}

	public static boolean isWebSocketFrame(ByteBuffer buffer) {
		// 接受数据的大小
		int maxpacketsize = buffer.remaining();
		// 期望数据包的实际大小
		int expectPackagesize = 2;
		if (maxpacketsize < expectPackagesize) {
			return false;
		}
		byte finByte = buffer.get();
		boolean fin = finByte >> 8 != 0;
		byte rsv = (byte) ((finByte & ~(byte) 128) >> 4);
		if (rsv != 0) {
			return false;
		}
		byte maskByte = buffer.get();
		boolean mask = (maskByte & -128) != 0;
		int payloadlength = (byte) (maskByte & ~(byte) 128);
		int optcode = (byte) (finByte & 15);

		if (!fin) {
			if (optcode == 9 || optcode == 10 || optcode == 8) {
				return false;
			}
		}

		if (payloadlength >= 0 && payloadlength <= 125) {
		} else {
			if (optcode == 9 || optcode == 10 || optcode == 8) {
				return false;
			}
			if (payloadlength == 126) {
				expectPackagesize += 2;
				byte[] sizebytes = new byte[3];
				sizebytes[1] = buffer.get();
				sizebytes[2] = buffer.get();
				payloadlength = new BigInteger(sizebytes).intValue();
			} else {
				expectPackagesize += 8;
				byte[] bytes = new byte[8];
				for (int i = 0; i < 8; i++) {
					bytes[i] = buffer.get();
				}
				long length = new BigInteger(bytes).longValue();
				if (length <= Integer.MAX_VALUE) {
					payloadlength = (int) length;
				}
			}
		}

		expectPackagesize += (mask ? 4 : 0);
		expectPackagesize += payloadlength;

		// 如果实际接受的数据小于数据包的大小则报错
		if (maxpacketsize < expectPackagesize) {
			return false;
		} else {
			return true;
		}
	}

}
<commit_msg>Http 消息截断器 的 boundary 判断报文结束判断的 bug<commit_after>package org.voovan.network.messagesplitter;

import java.math.BigInteger;
import java.nio.ByteBuffer;

import org.voovan.network.IoSession;
import org.voovan.network.MessageSplitter;
import org.voovan.tools.TString;

/**
 * Http 消息分割类
 * 
 * @author helyho
 *
 * Voovan Framework.
 * WebSite: https://github.com/helyho/Voovan
 * Licence: Apache v2 License
 */
public class HttpMessageSplitter implements MessageSplitter {

	private static final String	BODY_TAG	= "\r\n\r\n";

	@Override
	public boolean canSplite(IoSession session, byte[] buffer) {

		if (isHttpFrame(buffer)) {
			return true;
		} else if (isWebSocketFrame(ByteBuffer.wrap(buffer))) {
			return true;
		}

		return false;
	}

	public static boolean isHttpFrame(byte[] buffer) {
		String bufferString = new String(buffer);
		String[] boundaryLines = TString.searchByRegex(bufferString, "boundary=[^ \\r\\n]+");
		String[] contentLengthLines = TString.searchByRegex(bufferString, "Content-Length: \\d+");

		// 包含\r\n\r\n,这个时候报文有可能加载完成
		if (bufferString.contains(BODY_TAG)) {
			// 1.包含 content Length 的则通过获取 contentLenght 来计算报文的总长度,长度相等时,返回成功
			if (contentLengthLines.length == 1) {
				int contentLength = Integer.valueOf(contentLengthLines[0].split(" ")[1].trim());
				int totalLength = bufferString.indexOf(BODY_TAG) + 4 + contentLength;
				if (buffer.length >= totalLength) {
					return true;
				}
			}
		
			// 2. 如果是 HTTP 请求报文
			// POST方法的multipart/form-data类型,且没有指定ContentLength,则需要使用--boundary--的结尾形式来判断
			if (bufferString.contains("multipart/form-data") 
					&& boundaryLines.length == 1 
					&& bufferString.trim().endsWith("--" + boundaryLines[0].replace("boundary=", "") + "--")) {
				return true;
			}
			
			// 3. 如果是 HTTP 响应报文 chunk
			// 则trim 后判断最后一个字符是否是 0
			if (bufferString.contains("chunked") 
					&& bufferString.trim().endsWith("\r\n0")) {
				return true;
			}
			
			// 4 HEAD,CONNECT,DELETE,GET,TRACE,OPTIONS等请求,没有报文内容
			if (bufferString.startsWith("GET") 
					|| bufferString.startsWith("TRACE") 
					|| bufferString.startsWith("OPTIONS")
					|| bufferString.startsWith("HEAD") 
					|| bufferString.startsWith("DELETE") 
					|| bufferString.startsWith("CONNECT")) {
				return true;
			}
		}

		return false;
	}

	public static boolean isWebSocketFrame(ByteBuffer buffer) {
		// 接受数据的大小
		int maxpacketsize = buffer.remaining();
		// 期望数据包的实际大小
		int expectPackagesize = 2;
		if (maxpacketsize < expectPackagesize) {
			return false;
		}
		byte finByte = buffer.get();
		boolean fin = finByte >> 8 != 0;
		byte rsv = (byte) ((finByte & ~(byte) 128) >> 4);
		if (rsv != 0) {
			return false;
		}
		byte maskByte = buffer.get();
		boolean mask = (maskByte & -128) != 0;
		int payloadlength = (byte) (maskByte & ~(byte) 128);
		int optcode = (byte) (finByte & 15);

		if (!fin) {
			if (optcode == 9 || optcode == 10 || optcode == 8) {
				return false;
			}
		}

		if (payloadlength >= 0 && payloadlength <= 125) {
		} else {
			if (optcode == 9 || optcode == 10 || optcode == 8) {
				return false;
			}
			if (payloadlength == 126) {
				expectPackagesize += 2;
				byte[] sizebytes = new byte[3];
				sizebytes[1] = buffer.get();
				sizebytes[2] = buffer.get();
				payloadlength = new BigInteger(sizebytes).intValue();
			} else {
				expectPackagesize += 8;
				byte[] bytes = new byte[8];
				for (int i = 0; i < 8; i++) {
					bytes[i] = buffer.get();
				}
				long length = new BigInteger(bytes).longValue();
				if (length <= Integer.MAX_VALUE) {
					payloadlength = (int) length;
				}
			}
		}

		expectPackagesize += (mask ? 4 : 0);
		expectPackagesize += payloadlength;

		// 如果实际接受的数据小于数据包的大小则报错
		if (maxpacketsize < expectPackagesize) {
			return false;
		} else {
			return true;
		}
	}

}

{'fraction_non_alphanumeric': 0.14773788869611637, 'fraction_numerical': 0.014680368343787536, 'mean_word_length': 3.2725199543899657, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '984302', 'n_tokens_mistral': 3244, 'n_tokens_neox': 2795, 'n_words': 559}