<commit_before>#ifndef	BUFFER_H
#define	BUFFER_H

#include <string.h> /* memmove(3), memcpy(3), etc.  */

#include <deque>

struct iovec;

#define	BUFFER_SEGMENT_SIZE		(2048)
#define	BUFFER_SEGMENT_CACHE_LIMIT	(512)	/* 1MB of data.  */

typedef	uint16_t	buffer_segment_size_t;

/*
 * A BufferSegment is a contiguous chunk of data which may be at most a fixed
 * size of BUFFER_SEGMENT_SIZE.  The data in a BufferSegment may begin at a
 * non-zero offset within the BufferSegment and may end prematurely.  This
 * allows for skip()/trim() semantics.
 *
 * BufferSegments are reference counted and mutable operations copy before
 * doing a write.  You must provide your own locking or use only a single
 * thread.  One normally does not use BufferSegments directly unless one is
 * importing or exporting data in a performance-critical path.  Normal usage
 * uses a Buffer.
 */
class BufferSegment {
	uint8_t data_[BUFFER_SEGMENT_SIZE];<commit_msg>Use a normal word size for segment size, which improves performance a shocking amount.<commit_after>#ifndef	BUFFER_H
#define	BUFFER_H

#include <string.h> /* memmove(3), memcpy(3), etc.  */

#include <deque>

struct iovec;

#define	BUFFER_SEGMENT_SIZE		(2048)
#define	BUFFER_SEGMENT_CACHE_LIMIT	(512)	/* 1MB of data.  */

typedef	unsigned buffer_segment_size_t;

/*
 * A BufferSegment is a contiguous chunk of data which may be at most a fixed
 * size of BUFFER_SEGMENT_SIZE.  The data in a BufferSegment may begin at a
 * non-zero offset within the BufferSegment and may end prematurely.  This
 * allows for skip()/trim() semantics.
 *
 * BufferSegments are reference counted and mutable operations copy before
 * doing a write.  You must provide your own locking or use only a single
 * thread.  One normally does not use BufferSegments directly unless one is
 * importing or exporting data in a performance-critical path.  Normal usage
 * uses a Buffer.
 */
class BufferSegment {
	uint8_t data_[BUFFER_SEGMENT_SIZE];
{'fraction_non_alphanumeric': 0.07022900763358779, 'fraction_numerical': 0.012213740458015267, 'mean_word_length': 4.2991913746630726, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1148850', 'n_tokens_mistral': 626, 'n_tokens_neox': 596, 'n_words': 246}