<commit_before>package se.fnord.katydid.internal;

import se.fnord.katydid.ComparisonStatus;
import se.fnord.katydid.DataTester;

import java.nio.ByteBuffer;
import java.util.Objects;

public abstract class AbstractDataTester implements DataTester {
	private final String name;
	public AbstractDataTester(String name) {
		this.name = name;
	}

	protected abstract String formatValue(Object o);

	protected abstract int sizeOf(int itemIndex);

	protected abstract int itemCount();

	public String formatName(TestingContext context, int index) {
		return context.name();
	}

	protected ComparisonStatus compareToLevel0(TestingContext context) {
		throw new UnsupportedOperationException();
	}

	protected ComparisonStatus doCompareTo(int pass, TestingContext context) {
		if (pass == 0)
			return compareToLevel0(context);
		else
			return skip(context);

	}

	@Override
	public final ComparisonStatus compareTo(int pass, TestingContext context) {
		context.down(name);
		int start = context.buffer().position();
		try {
			return doCompareTo(pass, context);
		}
		finally {
			context.buffer().position(start + length());
			context.up();
		}
	}


	@Override
	public int passCount() {
		return 1;
	}

	protected ComparisonStatus skip(TestingContext context) {
		ByteBuffer bb = context.buffer();
		if (!checkHasRemaining(context, length()))
			return ComparisonStatus.ERROR;
		bb.position(bb.position() + length());
		return ComparisonStatus.EQUAL;
	}

	public int length() {
		return localPosition(itemCount());
	}

	private int localPosition(int itemIndex) {
		int localPosition = 0;
		for (int i = 0; i < itemIndex; i++) {
			localPosition += sizeOf(i);
		}
		return localPosition;
	}

	private String formatLocation(TestingContext context, int itemIndex) {
		int localPos = localPosition(itemIndex);
		int globalPos = context.globalPosition(localPos);

		return String.format("<local: %04x, global: %04x>", localPos, globalPos);
	}
<commit_msg>Cache the length of the AbstractDataTester since it's used frequently and always traverses the entire sub-tree<commit_after>package se.fnord.katydid.internal;

import se.fnord.katydid.ComparisonStatus;
import se.fnord.katydid.DataTester;

import java.nio.ByteBuffer;
import java.util.Objects;

public abstract class AbstractDataTester implements DataTester {
	private final String name;
	private int length = -1;
	public AbstractDataTester(String name) {
		this.name = name;
	}

	protected abstract String formatValue(Object o);

	protected abstract int sizeOf(int itemIndex);

	protected abstract int itemCount();

	public String formatName(TestingContext context, int index) {
		return context.name();
	}

	protected ComparisonStatus compareToLevel0(TestingContext context) {
		throw new UnsupportedOperationException();
	}

	protected ComparisonStatus doCompareTo(int pass, TestingContext context) {
		if (pass == 0)
			return compareToLevel0(context);
		else
			return skip(context);

	}

	@Override
	public final ComparisonStatus compareTo(int pass, TestingContext context) {
		context.down(name);
		int start = context.buffer().position();
		try {
			return doCompareTo(pass, context);
		}
		finally {
			context.buffer().position(start + length());
			context.up();
		}
	}


	@Override
	public int passCount() {
		return 1;
	}

	protected ComparisonStatus skip(TestingContext context) {
		ByteBuffer bb = context.buffer();
		if (!checkHasRemaining(context, length()))
			return ComparisonStatus.ERROR;
		bb.position(bb.position() + length());
		return ComparisonStatus.EQUAL;
	}

	public int length() {
		if (length == -1)
			length = localPosition(itemCount());
		return length;
	}

	private int localPosition(int itemIndex) {
		int localPosition = 0;
		for (int i = 0; i < itemIndex; i++) {
			localPosition += sizeOf(i);
		}
		return localPosition;
	}

	private String formatLocation(TestingContext context, int itemIndex) {
		int localPos = localPosition(itemIndex);
		int globalPos = context.globalPosition(localPos);

		return String.format("<local: %04x, global: %04x>", localPos, globalPos);
	}

{'fraction_non_alphanumeric': 0.10500369731328568, 'fraction_numerical': 0.005422726152329307, 'mean_word_length': 4.788873038516405, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7058844', 'n_tokens_mistral': 1319, 'n_tokens_neox': 1178, 'n_words': 328}