<commit_before>package com.projectvalis.reverse_rabin;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import junit.framework.Assert;

import org.jbehave.core.annotations.BeforeScenario;
import org.jbehave.core.annotations.BeforeStories;
import org.jbehave.core.annotations.BeforeStory;
import org.jbehave.core.annotations.Given;
import org.jbehave.core.annotations.Named;
import org.jbehave.core.annotations.Then;
import org.jbehave.core.annotations.When;
import org.jbehave.core.steps.Steps;
import org.rabinfingerprint.polynomial.Polynomial;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.projectvalis.util.rabin.RabinFingerprintLong_SmooshMod;


/**
 * BDD step file for the 'can we reverse a rabin hash' experiment
 * @author snerd
 *
 */
public class ReverseRabinSteps extends Steps {
	static Logger LOGGER = LoggerFactory.getLogger(ReverseRabinSteps.class);
		
	private byte[] generatedByteARR;
	private Polynomial rabinPolynomial;
	private RabinFingerprintLong_SmooshMod fingerprinter;
	
	private List<Long> xordFingerprintAL;
	private List<Long> appendedXordFingerprintAL;
	
//	private long[][] allXorPossibilities;
	private List<Long> allXorPossibilitiesAL;
	
	@BeforeScenario
	public void setup() {
		rabinPolynomial = Polynomial.createIrreducible(53); 
		fingerprinter = new RabinFingerprintLong_SmooshMod(rabinPolynomial);
		
		LOGGER.info(
				"GENERATED POLYNOMIAL IS: " + rabinPolynomial.toHexString());	
	}
	
	
	
	
	
	
	
	@Given("an array of $numBytesI bytes")
	public void createByteArray(@Named("numBytesI") int numBytesI) {
		Random randy = new Random();
		generatedByteARR = new byte[numBytesI];
		
		for (int i = 0; i < numBytesI; i ++) {
			generatedByteARR[i] = (byte)randy.nextInt(256);
		}
		
		String generatedBytesS = getHexString(generatedByteARR);
		LOGGER.info("GENERATED BYTES ARE: " + generatedBytesS.toUpperCase());
	}
	
	
	
	
	
	
	
	@Then("a data structure is created containing all 10,384 possible "
			+ "sets of input")
	public void generateAllSets() {
		allXorPossibilitiesAL = new ArrayList<Long>();
		long fingerprintL = fingerprinter.getFingerprintLong();
		List<Long> polynomialAL = fingerprinter.getPushTableAsList();
		
		for (int i = 0; i < polynomialAL.size(); i ++) {
			long polynomial_L = polynomialAL.get(i);
//LOGGER.info("polynomial and index are: " + String.format("%X", polynomial_L) + " " + i);		
			List<Long> rolledBackFingerprintAL = 
					rollbackFingerprintFirst(polynomial_L, fingerprintL, i);
			
			long tailByteL = rolledBackFingerprintAL.get(1);
//LOGGER.info("rollback result was: " + String.format("%X", rolledBackFingerprintAL.get(0)) 
//		+ " " + String.format("%X", rolledBackFingerprintAL.get(1)) );

		
			allXorPossibilitiesAL.addAll(
					rollbackFingerprintSecond(rolledBackFingerprintAL));
		}
		
		
		
		
//		allXorPossibilities = new long[512][8];
//		
//		long fingerprintL = fingerprinter.getFingerprintLong();
//		List<Long> polynomialAL = fingerprinter.getPushTableAsList();
//		
//		for (int i = 0; i < /*polynomialAL.size()*/10; i ++) {
//			long polynomial_L = polynomialAL.get(i);
//			
//			allXorPossibilities[i][0] = 
//					rollbackFingerprintFirst(polynomial_L, fingerprintL, i);
//LOGGER.info("rollback fingerprint and result was: " + String.format("%X", fingerprintL) + 
//		" " + String.format("%X", allXorPossibilities[i][0]));
//
//LOGGER.info("xord against: " + String.format("%X", polynomial_L));
//
//			for (int k = 1; k < 8; k++) {
//				allXorPossibilities[i][k] = 
//						allXorPossibilities[i][0] ^ polynomialAL.get(k);
//LOGGER.info("second xor at index: " + i + "/" + k + " is: " + 
//						String.format("%X", allXorPossibilities[i][k]) + 
//						" pushtable val was: " + String.format("%X", polynomialAL.get(k)));
//			}
//			
//		}
				
	}
	
	
	
	
	
	
	
	
	
	
	@When("the byte array is fingerprinted")
	public void fingerPrintByteArray() {
		fingerprinter.pushBytes(generatedByteARR);
	}
	


	
	
	
	
	
	
	
	
	
	@Then("the correct eight bits is appended to the head of the fingerprint "
			+ "such that\n the first [n] bits in the result are equal to the "
			+ "push table index used to \n generate the result.")
	public void checkAppendedBits() {
		
		for (int i = 0; i < appendedXordFingerprintAL.size(); i ++) {
			
//			String indexBinaryS = Integer.toBinaryString(i);
//			
//			while (indexBinaryS.length() < 8) { 
//				indexBinaryS = "0" + indexBinaryS; }
			
			Long currentFingerprintL = appendedXordFingerprintAL.get(i);

			
			int currentFingerprintHeadI = 
					(int)((currentFingerprintL >> 45) & 0x1FF);
//LOGGER.info("current fingerprint: " + Long.toBinaryString(currentFingerprintL));	
//LOGGER.info("current fingerprint head 8bits are: " + currentFingerprintHeadI);
			Assert.assertTrue("BITS AT HEAD OF APPENDED LIST NOT CORRECT! " + 
					"EXPECTED: " + i + " GOT: " + currentFingerprintHeadI,
					i == currentFingerprintHeadI);
			
			
//			String appendedHeadBits_S = 
//					Long.toBinaryString(currentFingerprintL);
//			
//			appendedHeadBits_S = appendedHeadBits_S.substring(0, 8);
//			LOGGER.info(appendedHeadBits_S + " " + appendedHeadBits_S.length());
//			Assert.assertTrue(appendedHeadBits_S.length() == 8);
//			LOGGER.info(indexBinaryS + " " + appendedHeadBits_S);
//			Assert.assertTrue(indexBinaryS.contentEquals(appendedHeadBits_S));
		}
		
	}
	
	
	
	
	
	
	
	@When("the fingerprinted array is xor'd against every entry "
			+ "in the push table")
	public void xorFingerprintAll() {
		long fingerprintL = fingerprinter.getFingerprintLong();
		List<Long> polynomialAL = fingerprinter.getPushTableAsList();
		
//		xordFingerprintAL = polynomialAL.parallelStream()
//												 .map(x -> x ^ fingerprintL)
//												 .map(x -> removeTailByte(fingerprintL))
//												 .collect(Collectors.toList());
		
		xordFingerprintAL = new ArrayList<Long>();
		appendedXordFingerprintAL = new ArrayList<Long>();
		
		for (int i = 0; i < polynomialAL.size(); i ++) {
			long xordFingerprintL = polynomialAL.get(i) ^ fingerprintL;
//LOGGER.info("fingerprint and xord fingerprint are: " + fingerprintL + " " + xordFingerprintL);
			xordFingerprintL = removeTailByte(xordFingerprintL);
//LOGGER.info("de-tailed xord fingerprint is: " + xordFingerprintL);
			xordFingerprintAL.add(xordFingerprintL);
			
			long appendedXordFingerprintL = appendByteToHead(i, xordFingerprintAL.get(i));
			appendedXordFingerprintAL.add(appendedXordFingerprintL);
		}
		

//		appendedXordFingerprintAL = 
//				IntStream.range(0, xordFingerprintAL.size())
//					//.filter(i -> Long.toBinaryString(xordFingerprintAL.get(i)).length() < 56)
//					.mapToObj(i -> appendByteToHead(i, xordFingerprintAL.get(i)))
//					.collect(Collectors.toList());
			
	}
	

	

	
	
	@Then("the original 8 bytes is retrieved from the fingerprint.")
	public void retrieveBytes() {	
		int matchCountI = 0;
		boolean matchFoundB = false;
		long fingerprintL = fingerprinter.getFingerprintLong();
LOGGER.info("fingerprintL was: " + fingerprintL);
		generateAllSets();

		for (int i = 0; i < allXorPossibilitiesAL.size(); i ++) {
			fingerprinter.reset();
			long answerCandidateL = allXorPossibilitiesAL.get(i);
			
LOGGER.info("answerCandidateL is: " + answerCandidateL);	
			
			byte[] answerCandidateByteARR = 
					ByteBuffer.allocate(Long.SIZE / Byte.SIZE).
						putLong(answerCandidateL).array();		

LOGGER.info("answerCandidateL is in hex: " + getHexString(answerCandidateByteARR).toUpperCase());	
			fingerprinter.pushBytes(answerCandidateByteARR);
LOGGER.info("current and original fingerprint are: " + fingerprinter.getFingerprintLong() + " " + fingerprintL);
			if (fingerprinter.getFingerprintLong() == fingerprintL) {
//				matchFoundB = true;
				matchCountI++;
				
//				Assert.assertFalse(
//						"SECOND MATCH FOUND WHEN THERE SHOULD "
//						+ "BE ONLY ONE!", matchFoundB);
//				
//				matchFoundB = true;
LOGGER.info("**************************************************");
fingerprinter.reset();
fingerprinter.pushBytes(generatedByteARR);
LOGGER.info(fingerprinter.getFingerprintLong() + " ");

fingerprinter.reset();
fingerprinter.pushBytes(answerCandidateByteARR);
LOGGER.info(fingerprinter.getFingerprintLong() + "\n");

LOGGER.info("original and answer candidate byte arrays are: " + 
		Arrays.toString(generatedByteARR) + " " + Arrays.toString(answerCandidateByteARR));
LOGGER.info("**************************************************");


				if (!matchFoundB) {
					boolean tempMatchB = true;
					
					for (int k = 0; k < answerCandidateByteARR.length; k ++) {
						if (answerCandidateByteARR[k] != generatedByteARR[k]) {
							tempMatchB = false;
							break;
						}
					}
					
					matchFoundB = tempMatchB;
					
				}

//				for (int moo = 0;
//						moo < answerCandidateByteARR.length; moo ++) {
//			
//					Assert.assertTrue(
//						"FINGERPRINTS MATCH FOR NON-MATCHING BYTE ARRAYS!", 
//							answerCandidateByteARR[moo] == 
//								generatedByteARR[moo]);
//				}
				
			}				
			
		}
		
		LOGGER.info("match count is: " + matchCountI);
		
		Assert.assertTrue(
			"NO MATCHING BYTE ARRAY FOUND FOR GIVEN FINGERPRINT!", matchFoundB);
		
	}
	
	
	
	/**
	 * simple utility plucked from the interwebs to convert a byte array to 
	 * a hex string (yes I am that lazy)
	 * 
	 * yoinked from: http://www.rgagnon.com/javadetails/java-0596.html
	 * 
	 * 
	 * @param b
	 * @return
	 * @throws Exception
	 */
	public String getHexString(byte[] b) {
		  String result = "";
		  for (int i=0; i < b.length; i++) {
		    result +=
		          Integer.toString( ( b[i] & 0xff ) + 0x100, 16).substring( 1 );
		  }
		  return result;
	}
	
	
	
	public long removeTailByte(long xordFingerprint) {
		return xordFingerprint >> 8;
	}
	
	
	public long getTailByte(long xordFingerprint) {
		return 0x00000000000000FF & xordFingerprint;
	}
	
	
	
	
	/**
	 * 
	 * @param listIndex
	 * @param xordFingerprint
	 * @return
	 */
	public long appendByteToHead(int listIndex, long xordFingerprint) {

		// head is the bits that are indexed @ > 45 - which means they were
		// part of the value that generated the index which selected the
		// member of the push table against which this fingerprint was 
		// xor'd
		int xordFingerprintHeadLengthI = Math.abs(
				Long.toBinaryString(xordFingerprint).length() - 45);	
		
		// figure out the current value of the existing bits in head, and what
		// the delta is between what's already there and what needs to be there. 
		// then setup to update the head value accordingly.
		int xordFingerprintHeadI = (int)(xordFingerprint >> 45);
		int diffIndexAndXordHeadI = listIndex - xordFingerprintHeadI;
		xordFingerprintHeadI += diffIndexAndXordHeadI;
		
		String headBinaryS = Integer.toBinaryString(xordFingerprintHeadI);
		//while (headBinaryS.length() < 8) { headBinaryS = "0" + headBinaryS; }
		
		String headlessXordFingerprintS = 
				Long.toBinaryString(xordFingerprint).substring(xordFingerprintHeadLengthI);
		
		String appendedXordFingerprintS = headBinaryS + headlessXordFingerprintS;
		
		return Long.parseLong(appendedXordFingerprintS, 2);
			
	}
	
	
	/**
	 * returns list{rolled back fingerprint, tail byte appended to rolled 
	 * back fingerprint}
	 * 
	 * @param polynomial
	 * @param fingerprint
	 * @param xorIndex
	 * @return
	 */
	public List<Long> rollbackFingerprintFirst(long polynomial, long fingerprint, int xorIndex) {
		long xordFingerprintL = polynomial ^ fingerprint;
//LOGGER.info("xordFingerprintL was: " + xordFingerprintL);
		long tailByteL = getTailByte(xordFingerprintL);
//LOGGER.info("tailByteL was: " + tailByteL);
		xordFingerprintL = removeTailByte(xordFingerprintL);			
		long appendedXordFingerprintL = appendByteToHead(xorIndex, xordFingerprintL);
//LOGGER.info("returning " + ((appendedXordFingerprintL << 8) | (tailByteL & 0xFF)));
		//return (appendedXordFingerprintL << 8) | (tailByteL & 0xFF);
		
		List<Long> returnAL = new ArrayList<Long>();
		returnAL.add(appendedXordFingerprintL);
		returnAL.add(tailByteL);
		returnAL.add((long)xorIndex);
		return returnAL;
	}
	

	

	public List<Long> rollbackFingerprintSecond(List<Long> resultFromFirstRollbackAL) {
		long fingerprintL = resultFromFirstRollbackAL.get(0);
		long tailByteL = resultFromFirstRollbackAL.get(1);
		
		List<Long> returnAL = new ArrayList<Long>();

		
		//
		// this yields ~511 results, the correct answer usually not among them
		// 
		//
// 
		// doesn't work
//		String xorIndexFirstS = 
//				Long.toBinaryString(resultFromFirstRollbackAL.get(2));
//		
//		while (xorIndexFirstS.length() < 3) {
//			xorIndexFirstS = "0" + xorIndexFirstS;
//		}
//		
//		int xorIndexSecondI = Integer.parseInt(xorIndexFirstS.substring(0, 3), 2);
		
		
		// neither does this
//		long firstXorIndexValL = resultFromFirstRollbackAL.get(2);
//		String firstXorIndexValS = Long.toString(firstXorIndexValL);
//		
//		int xorIndexSecondI = Integer.parseInt(firstXorIndexValS) >> 5;
//		List<Long> polynomialAL = fingerprinter.getPushTableAsList();
//		long polynomialL = polynomialAL.get(xorIndexSecondI);
//		long xordFingerprintL = polynomialL ^ fingerprintL;	
//		long validCandidateL = 
//				(xordFingerprintL << 8) | (tailByteL & 0xFF);
//		returnAL.add(validCandidateL);
//		
		
		
		
			//
		    // this yields 2048 results -- the correct on is among them
			//	
		
		
		for (int i = 0; i < 8; i ++) {
			List<Long> polynomialAL = fingerprinter.getPushTableAsList();
			long polynomialL = polynomialAL.get(i);
//LOGGER.info("second rollback on poly against fingerprint: " +
//			String.format("%X", polynomialL) + " " + String.format("%X", fingerprintL));
			long xordFingerprintL = polynomialL ^ fingerprintL;	
		
			
//LOGGER.info("rolled back fingerprint + tailbyte is: " + String.format("%X", xordFingerprintL));	
//LOGGER.info(">> 53 against fingerprint + tailbyte: " + String.format("%X", xordFingerprintL) + 
//		" is: " + (int)(xordFingerprintL >> 53));
			
			if (i == (int)(xordFingerprintL >> 53)) { 
				long validCandidateL = 
						(xordFingerprintL << 8) | (tailByteL & 0xFF);
				returnAL.add(validCandidateL);
			}
					
		}
		
		return returnAL;		
	}
	
	
	
}







<commit_msg>cleaned up the code a bit<commit_after>package com.projectvalis.reverse_rabin;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;

import junit.framework.Assert;

import org.jbehave.core.annotations.BeforeScenario;
import org.jbehave.core.annotations.Given;
import org.jbehave.core.annotations.Named;
import org.jbehave.core.annotations.Then;
import org.jbehave.core.annotations.When;
import org.jbehave.core.steps.Steps;
import org.rabinfingerprint.polynomial.Polynomial;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.projectvalis.util.rabin.RabinFingerprintLong_SmooshMod;

/**
 * BDD step file for the 'can we reverse a rabin hash' experiment
 * 
 * @author snerd
 *
 */
public class ReverseRabinSteps extends Steps {
	static Logger LOGGER = LoggerFactory.getLogger(ReverseRabinSteps.class);

	private byte[] generatedByteARR;

	private Polynomial rabinPolynomial;
	private RabinFingerprintLong_SmooshMod fingerprinter;

	private List<Long> xordFingerprintAL;
	private List<Long> appendedXordFingerprintAL;
	private List<Long> allXorPossibilitiesAL;

	
	
	@BeforeScenario
	public void setup() {
		rabinPolynomial = Polynomial.createIrreducible(53);
		fingerprinter = new RabinFingerprintLong_SmooshMod(rabinPolynomial);

		LOGGER.info("GENERATED POLYNOMIAL IS: " + rabinPolynomial.toHexString());
	}
	
	

	@Given("an array of $numBytesI bytes")
	public void createByteArray(@Named("numBytesI") int numBytesI) {
		Random randy = new Random();
		generatedByteARR = new byte[numBytesI];

		for (int i = 0; i < numBytesI; i++) {
			generatedByteARR[i] = (byte) randy.nextInt(256);
		}

		String generatedBytesS = getHexString(generatedByteARR);
		LOGGER.info("GENERATED BYTES ARE: " + generatedBytesS.toUpperCase());
	}

	
	
	@Then("a data structure is created containing all 10,384 possible "
			+ "sets of input")
	public void generateAllSets() {
		allXorPossibilitiesAL = new ArrayList<Long>();
		long fingerprintL = fingerprinter.getFingerprintLong();
		List<Long> polynomialAL = fingerprinter.getPushTableAsList();

		for (int i = 0; i < polynomialAL.size(); i++) {
			long polynomial_L = polynomialAL.get(i);

			List<Long> rolledBackFingerprintAL = rollbackFingerprintFirst(
					polynomial_L, fingerprintL, i);

			// long tailByteL = rolledBackFingerprintAL.get(1);

			allXorPossibilitiesAL
					.addAll(rollbackFingerprintSecond(rolledBackFingerprintAL));
		}

	}

	@When("the byte array is fingerprinted")
	public void fingerPrintByteArray() {
		fingerprinter.pushBytes(generatedByteARR);
	}
	
	

	@Then("the correct eight bits is appended to the head of the fingerprint "
			+ "such that\n the first [n] bits in the result are equal to the "
			+ "push table index used to \n generate the result.")
	public void checkAppendedBits() {

		for (int i = 0; i < appendedXordFingerprintAL.size(); i++) {

			Long currentFingerprintL = appendedXordFingerprintAL.get(i);

			int currentFingerprintHeadI = 
					(int) ((currentFingerprintL >> 45) & 0x1FF);

			Assert.assertTrue("BITS AT HEAD OF APPENDED LIST NOT CORRECT! "
					+ "EXPECTED: " + i + " GOT: " + currentFingerprintHeadI,
					i == currentFingerprintHeadI);

		}

	}

	@When("the fingerprinted array is xor'd against every entry "
			+ "in the push table")
	public void xorFingerprintAll() {
		long fingerprintL = fingerprinter.getFingerprintLong();
		List<Long> polynomialAL = fingerprinter.getPushTableAsList();
		xordFingerprintAL = new ArrayList<Long>();
		appendedXordFingerprintAL = new ArrayList<Long>();

		for (int i = 0; i < polynomialAL.size(); i++) {
			long xordFingerprintL = polynomialAL.get(i) ^ fingerprintL;
			xordFingerprintL = removeTailByte(xordFingerprintL);
			xordFingerprintAL.add(xordFingerprintL);
			
			long appendedXordFingerprintL = appendByteToHead(
					i, xordFingerprintAL.get(i));
			
			appendedXordFingerprintAL.add(appendedXordFingerprintL);
		}

	}

	
	@Then("the original 8 bytes is retrieved from the fingerprint.")
	public void retrieveBytes() {
		int matchCountI = 0;
		boolean matchFoundB = false;	
		int actualMatchIndexI = -1;
		long fingerprintL = fingerprinter.getFingerprintLong();
		LOGGER.debug("fingerprintL was: " + fingerprintL);
		
		generateAllSets();

		for (int i = 0; i < allXorPossibilitiesAL.size(); i++) {
			fingerprinter.reset();
			long answerCandidateL = allXorPossibilitiesAL.get(i);

			LOGGER.trace("answerCandidateL is: " + answerCandidateL);

			byte[] answerCandidateByteARR = ByteBuffer
					.allocate(Long.SIZE / Byte.SIZE).putLong(answerCandidateL)
					.array();

			LOGGER.trace("answerCandidateL is in hex: "
					+ getHexString(answerCandidateByteARR).toUpperCase());
			
			fingerprinter.pushBytes(answerCandidateByteARR);
			
			LOGGER.trace("current and original fingerprint are: "
					+ fingerprinter.getFingerprintLong() + " " + fingerprintL);
			
			if (fingerprinter.getFingerprintLong() == fingerprintL) {
				matchCountI++;

//				LOGGER.info("************************************************");
//				fingerprinter.reset();
//				fingerprinter.pushBytes(generatedByteARR);
//				LOGGER.info(fingerprinter.getFingerprintLong() + " ");
//
//				fingerprinter.reset();
//				fingerprinter.pushBytes(answerCandidateByteARR);
//				LOGGER.info(fingerprinter.getFingerprintLong() + "\n");
//
//				LOGGER.info("original and answer candidate byte arrays are: "
//						+ Arrays.toString(generatedByteARR) + " "
//						+ Arrays.toString(answerCandidateByteARR));
//				LOGGER.info("************************************************");

				if (!matchFoundB) {
					boolean tempMatchB = true;

					for (int k = 0; k < answerCandidateByteARR.length; k++) {
						if (answerCandidateByteARR[k] != generatedByteARR[k]) {
							tempMatchB = false;
							break;
						}
					}

					matchFoundB = tempMatchB;
					if (matchFoundB) {
						actualMatchIndexI = i;
					}
				}

			}

		}

		LOGGER.info("match count is: " + matchCountI);
		LOGGER.info("actual match index is: " + actualMatchIndexI);

		Assert.assertTrue(
				"NO MATCHING BYTE ARRAY FOUND FOR GIVEN FINGERPRINT!",
				matchFoundB);

	}

	/**
	 * simple utility plucked from the interwebs to convert a byte array to a
	 * hex string (yes I am that lazy)
	 * 
	 * yoinked from: http://www.rgagnon.com/javadetails/java-0596.html
	 * 
	 * 
	 * @param b
	 * @return
	 * @throws Exception
	 */
	public String getHexString(byte[] b) {
		String result = "";
		for (int i = 0; i < b.length; i++) {
			result += Integer.toString((b[i] & 0xff) + 0x100, 16).substring(1);
		}
		return result;
	}

	
	/**
	 * removes the tail byte of a fingerprint by shifting right 8bits
	 * @param xordFingerprint
	 * @return
	 */
	public long removeTailByte(long xordFingerprint) {
		return xordFingerprint >> 8;
	}

	
	
	/**
	 * returns only the tail byte of a fingerprint
	 * @param xordFingerprint
	 * @return
	 */
	public long getTailByte(long xordFingerprint) {
		return 0x00000000000000FF & xordFingerprint;
	}

	
	
	/**
	 * shifts the fingerprint 45 bits to the right, then ensures that what
	 * remains is updated to equal a given byte. 
	 * 
	 * @param listIndex
	 * @param xordFingerprint
	 * @return
	 */
	public long appendByteToHead(int listIndex, long xordFingerprint) {

		// head is the bits that are indexed @ > 45 - which means they were
		// part of the value that generated the index which selected the
		// member of the push table against which this fingerprint was
		// xor'd
		int xordFingerprintHeadLengthI = Math.abs(Long.toBinaryString(
				xordFingerprint).length() - 45);

		// figure out the current value of the existing bits in head, and what
		// the delta is between what's already there and what needs to be there.
		// then setup to update the head value accordingly.
		int xordFingerprintHeadI = (int) (xordFingerprint >> 45);
		int diffIndexAndXordHeadI = listIndex - xordFingerprintHeadI;
		xordFingerprintHeadI += diffIndexAndXordHeadI;

		String headBinaryS = Integer.toBinaryString(xordFingerprintHeadI);

		String headlessXordFingerprintS = Long.toBinaryString(xordFingerprint)
				.substring(xordFingerprintHeadLengthI);

		String appendedXordFingerprintS = headBinaryS
				+ headlessXordFingerprintS;

		return Long.parseLong(appendedXordFingerprintS, 2);

	}

	/**
	 * assumes what's being passed is eight bytes fingerprinted. what's 
	 * returned is, given a polynomial and an XOR table index, what the 
	 * previous state of the fingerprint was (ie - when the first seven / eight
	 * bytes were pushed).
	 * 
	 * returns 
	 * 
	 * @param polynomial
	 * @param fingerprint
	 * @param xorIndex
	 * @return
	 * 		LIST{rolled back fingerprint, tail byte appended to rolled back
	 * fingerprint prior to XOR (the eighth byte pushed)}
	 */
	public List<Long> rollbackFingerprintFirst(long polynomial,
			long fingerprint, int xorIndex) {

		long xordFingerprintL = polynomial ^ fingerprint;
		long tailByteL = getTailByte(xordFingerprintL);
		xordFingerprintL = removeTailByte(xordFingerprintL);

		long appendedXordFingerprintL = appendByteToHead(xorIndex,
				xordFingerprintL);

		List<Long> returnAL = new ArrayList<Long>();
		returnAL.add(appendedXordFingerprintL);
		returnAL.add(tailByteL);
		returnAL.add((long) xorIndex);
		return returnAL;
	}

	
	/**
	 * assumes what's being passed is the result of a fingerprinted stream of
	 * eight bytes being rolled back by rollbackFingerprintFirst. because only
	 * eight bytes were pushed, what this method tries to figure out is what
	 * the fingerprint looked like prior to the seventh byte being pushed. given
	 * that only three bits could've been used to determine the XOR index value,
	 * what's returned is a set of candidates where, after XOR against all 
	 * possible XOR values with index values representable in three bits, the
	 * first three bits are equal to the index of the XOR value used.
	 * 
	 * @param resultFromFirstRollbackAL
	 * @return
	 */
	public List<Long> rollbackFingerprintSecond(
			List<Long> resultFromFirstRollbackAL) {

		long fingerprintL = resultFromFirstRollbackAL.get(0);
		long tailByteL = resultFromFirstRollbackAL.get(1);
		List<Long> returnAL = new ArrayList<Long>();

		for (int i = 0; i < 8; i++) {
			List<Long> polynomialAL = fingerprinter.getPushTableAsList();
			long polynomialL = polynomialAL.get(i);
			long xordFingerprintL = polynomialL ^ fingerprintL;

			if (i == (int) (xordFingerprintL >> 53)) {
				
				long validCandidateL = 
						(xordFingerprintL << 8)| (tailByteL & 0xFF);
				
				returnAL.add(validCandidateL);
			}

		}

		return returnAL;
	}

}
{'fraction_non_alphanumeric': 0.12450735944663396, 'fraction_numerical': 0.0077616021877262124, 'mean_word_length': 3.823860329776916, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3228541', 'n_tokens_mistral': 9588, 'n_tokens_neox': 8118, 'n_words': 1830}