<commit_before>#include <common/buffer.h>
#include <common/endian.h>

#include <xcodec/xcodec.h>
#include <xcodec/xcodec_cache.h>
#include <xcodec/xcodec_encoder.h>
#include <xcodec/xcodec_hash.h>

typedef	std::pair<unsigned, uint64_t> offset_hash_pair_t;

struct xcodec_special_p {
	bool operator() (uint8_t ch) const
	{
		return (XCODEC_CHAR_SPECIAL(ch));
	}
};

XCodecEncoder::XCodecEncoder(XCodec *codec)
: log_("/xcodec/encoder"),
  cache_(codec->cache_),
  window_()
{ }

XCodecEncoder::~XCodecEncoder()
{ }

/*
 * This takes a view of a data stream and turns it into a series of references
 * to other data, declarations of data to be referenced, and data that needs
 * escaped.
 */
void
XCodecEncoder::encode(Buffer *output, Buffer *input)
{
	if (input->length() < XCODEC_SEGMENT_LENGTH) {
		input->escape(XCODEC_ESCAPE_CHAR, xcodec_special_p());
		output->append(input);
		input->clear();
		return;
	}

	XCodecHash<XCODEC_SEGMENT_LENGTH> xcodec_hash;
	offset_hash_pair_t candidate;
	bool have_candidate;
	Buffer outq;
	unsigned o = 0;

	have_candidate = false;

	while (!input->empty()) {
		BufferSegment *seg;
		input->moveout(&seg);

		outq.append(seg);

		const uint8_t *p;
		for (p = seg->data(); p < seg->end(); p++) {
			xcodec_hash.roll(*p);
			if (++o < XCODEC_SEGMENT_LENGTH)
				continue;

			unsigned start = o - XCODEC_SEGMENT_LENGTH;
			uint64_t hash = xcodec_hash.mix();

			/*
			 * If there is a pending candidate hash that wouldn't
			 * overlap with tis one, declare it now.
			 */
			if (have_candidate && candidate.first + XCODEC_SEGMENT_LENGTH <= start) {
				BufferSegment *nseg;
				encode_declaration(output, &outq, candidate.first, candidate.second, &nseg);

				o -= candidate.first + XCODEC_SEGMENT_LENGTH;
				start = o - XCODEC_SEGMENT_LENGTH;

				have_candidate = false;

				/*
				 * If, on top of that, the just-declared hash is
				 * the same as the current hash, consider referencing
				 * it immediately.
				 */
				if (hash == candidate.second) {
					if (!encode_reference(output, &outq, start, hash, nseg)) {
						nseg->unref();
						DEBUG(log_) << "Collision in adjacent-declare pass.";
						continue;
					}
					nseg->unref();

					o = 0;

					DEBUG(log_) << "Hit in adjacent-declare pass.";
					continue;
				}
				nseg->unref();
			}

			/*
			 * Now attempt to encode this hash as a reference if it
			 * has been defined before.
			 */
			BufferSegment *oseg = cache_->lookup(hash);
			if (oseg != NULL) {
				/*
				 * This segment already exists.  If it's
				 * identical to this chunk of data, then that's
				 * positively fantastic.
				 */
				if (encode_reference(output, &outq, start, hash, oseg)) {
					oseg->unref();
					o = 0;
					have_candidate = false;
					continue;
				}
				oseg->unref();
				DEBUG(log_) << "Collision in first pass.";
				continue;
			}

			/*
			 * Not defined before, it's a candidate for declaration if
			 * we don't already have one.
			 */
			if (have_candidate) {
				ASSERT(candidate.first + XCODEC_SEGMENT_LENGTH > start);
				continue;
			}
			candidate.first = start;
			candidate.second = hash;
			have_candidate = true;
		}

		seg->unref();
	}

	/*
	 * There's a hash we can declare, do it.
	 */
	if (have_candidate) {
		ASSERT(!outq.empty());
		encode_declaration(output, &outq, candidate.first, candidate.second, NULL);
		have_candidate = false;
	}

	if (!outq.empty()) {
		Buffer suffix(outq);
		outq.clear();

		suffix.escape(XCODEC_ESCAPE_CHAR, xcodec_special_p());

		output->append(suffix);
		outq.clear();
	}

	ASSERT(!have_candidate);
	ASSERT(outq.empty());
	ASSERT(input->empty());
}

void
XCodecEncoder::encode_declaration(Buffer *output, Buffer *input, unsigned offset, uint64_t hash, BufferSegment **segp)
{
	if (offset != 0) {
		Buffer prefix;
		input->moveout(&prefix, 0, offset);<commit_msg>Comment the XCodecEncoder::encode() routine better.<commit_after>#include <common/buffer.h>
#include <common/endian.h>

#include <xcodec/xcodec.h>
#include <xcodec/xcodec_cache.h>
#include <xcodec/xcodec_encoder.h>
#include <xcodec/xcodec_hash.h>

typedef	std::pair<unsigned, uint64_t> offset_hash_pair_t;

struct xcodec_special_p {
	bool operator() (uint8_t ch) const
	{
		return (XCODEC_CHAR_SPECIAL(ch));
	}
};

XCodecEncoder::XCodecEncoder(XCodec *codec)
: log_("/xcodec/encoder"),
  cache_(codec->cache_),
  window_()
{ }

XCodecEncoder::~XCodecEncoder()
{ }

/*
 * This takes a view of a data stream and turns it into a series of references
 * to other data, declarations of data to be referenced, and data that needs
 * escaped.
 */
void
XCodecEncoder::encode(Buffer *output, Buffer *input)
{
	if (input->length() < XCODEC_SEGMENT_LENGTH) {
		input->escape(XCODEC_ESCAPE_CHAR, xcodec_special_p());
		output->append(input);
		input->clear();
		return;
	}

	XCodecHash<XCODEC_SEGMENT_LENGTH> xcodec_hash;
	offset_hash_pair_t candidate;
	bool have_candidate;
	Buffer outq;
	unsigned o = 0;

	have_candidate = false;

	/*
	 * While there is input.
	 */
	while (!input->empty()) {
		/*
		 * Take the first BufferSegment out of the input Buffer.
		 */
		BufferSegment *seg;
		input->moveout(&seg);

		/*
		 * And add it to a temporary Buffer where input is queued.
		 */
		outq.append(seg);

		/*
		 * And for every byte in this BufferSegment.
		 */
		const uint8_t *p;
		for (p = seg->data(); p < seg->end(); p++) {
			/*
			 * Add it to the rolling hash.
			 */
			xcodec_hash.roll(*p);

			/*
			 * If the rolling hash does not cover an entire
			 * segment yet, just keep adding to it.
			 */
			if (++o < XCODEC_SEGMENT_LENGTH)
				continue;

			/*
			 * And then mix the hash's internal state into a
			 * uint64_t that we can use to refer to that data
			 * and to look up possible past occurances of that
			 * data in the XCodecCache.
			 */
			unsigned start = o - XCODEC_SEGMENT_LENGTH;
			uint64_t hash = xcodec_hash.mix();

			/*
			 * If there is a pending candidate hash that wouldn't
			 * overlap with the data that the rolling hash presently
			 * covers, declare it now.
			 */
			if (have_candidate && candidate.first + XCODEC_SEGMENT_LENGTH <= start) {
				BufferSegment *nseg;
				encode_declaration(output, &outq, candidate.first, candidate.second, &nseg);

				o -= candidate.first + XCODEC_SEGMENT_LENGTH;
				start = o - XCODEC_SEGMENT_LENGTH;

				have_candidate = false;

				/*
				 * If, on top of that, the just-declared hash is
				 * the same as the current hash, consider referencing
				 * it immediately.
				 */
				if (hash == candidate.second) {
					/*
					 * If it's a hash collision, though, nevermind.
					 * Skip trying to use this hash as a reference,
					 * too, and go on to the next one.
					 */
					if (!encode_reference(output, &outq, start, hash, nseg)) {
						nseg->unref();
						DEBUG(log_) << "Collision in adjacent-declare pass.";
						continue;
					}
					nseg->unref();

					/*
					 * But if there's no collection, then we can move
					 * on to looking for the *next* hash/data to declare
					 * or reference.
					 */
					o = 0;

					DEBUG(log_) << "Hit in adjacent-declare pass.";
					continue;
				}
				nseg->unref();
			}

			/*
			 * Now attempt to encode this hash as a reference if it
			 * has been defined before.
			 */
			BufferSegment *oseg = cache_->lookup(hash);
			if (oseg != NULL) {
				/*
				 * This segment already exists.  If it's
				 * identical to this chunk of data, then that's
				 * positively fantastic.
				 */
				if (encode_reference(output, &outq, start, hash, oseg)) {
					oseg->unref();

					o = 0;

					/*
					 * We have output any data before this hash
					 * in escaped form, so any candidate hash
					 * before it is invalid now.
					 */
					have_candidate = false;
					continue;
				}

				/*
				 * This hash isn't usable because it collides
				 * with another, so keep looking for something
				 * viable.
				 */
				oseg->unref();
				DEBUG(log_) << "Collision in first pass.";
				continue;
			}

			/*
			 * Not defined before, it's a candidate for declaration if
			 * we don't already have one.
			 */
			if (have_candidate) {
				/*
				 * We already have a hash that occurs earlier,
				 * isn't a collision and includes data that's
				 * covered by this hash, so don't remember it
				 * and keep going.
				 */
				ASSERT(candidate.first + XCODEC_SEGMENT_LENGTH > start);
				continue;
			}

			/*
			 * The hash at this offset doesn't collide with any
			 * other and is the first viable hash we've seen so far
			 * in the stream, so remember it so that if we don't
			 * find something to reference we can declare this one
			 * for future use.
			 */
			candidate.first = start;
			candidate.second = hash;
			have_candidate = true;
		}

		seg->unref();
	}

	/*
	 * Done processing input.  If there's still data in the outq Buffer,
	 * then we need to declare any candidate hashes and escape any data
	 * after them.
	 */

	/*
	 * There's a hash we can declare, do it.
	 */
	if (have_candidate) {
		ASSERT(!outq.empty());
		encode_declaration(output, &outq, candidate.first, candidate.second, NULL);
		have_candidate = false;
	}

	/*
	 * There's data after that hash or no candidate hash, so
	 * just escape it.
	 */
	if (!outq.empty()) {
		Buffer suffix(outq);
		outq.clear();

		suffix.escape(XCODEC_ESCAPE_CHAR, xcodec_special_p());

		output->append(suffix);
		outq.clear();
	}

	ASSERT(!have_candidate);
	ASSERT(outq.empty());
	ASSERT(input->empty());
}

void
XCodecEncoder::encode_declaration(Buffer *output, Buffer *input, unsigned offset, uint64_t hash, BufferSegment **segp)
{
	if (offset != 0) {
		Buffer prefix;
		input->moveout(&prefix, 0, offset);
{'fraction_non_alphanumeric': 0.12258466652815292, 'fraction_numerical': 0.002908788697278205, 'mean_word_length': 2.9600987248046073, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3678354', 'n_tokens_mistral': 3859, 'n_tokens_neox': 3119, 'n_words': 934}