<commit_before>#include "PrefixTable.h"
#include "NGM.h"
#include "CS.h"

//#include "SequenceLocation.h"

#include "Timing.h"

#include <stdexcept>

#undef module_name
#define module_name "PFTABLE"

#include <cmath>
#include <algorithm>
#include <sstream>
#include <stdio.h>

static uint const refTabCookie = 0x1701D;

int lastSeqTotal = 0;

int CompactPrefixTable::maxPrefixFreq = 1000;

ulong CompactPrefixTable::lastPrefix;
int CompactPrefixTable::lastBin;
int CompactPrefixTable::lastPos;

uint CompactPrefixTable::skipCount;
uint CompactPrefixTable::skipBuild;

static const unsigned char ReverseTable16[] = { 0x00, 0x04, 0x08, 0x0C, 0x01, 0x05, 0x09, 0x0D, 0x02, 0x06, 0x0A, 0x0E, 0x03, 0x07, 0x0B,
		0x0F };

//static const unsigned char ReverseTable256[] = { 0, 64, 128, 192, 16, 80, 144,
//		208, 32, 96, 160, 224, 48, 112, 176, 240, 4, 68, 132, 196, 20, 84, 148,
//		212, 36, 100, 164, 228, 52, 116, 180, 244, 8, 72, 136, 200, 24, 88, 152,
//		216, 40, 104, 168, 232, 56, 120, 184, 248, 12, 76, 140, 204, 28, 92,
//		156, 220, 44, 108, 172, 236, 60, 124, 188, 252, 1, 65, 129, 193, 17, 81,
//		145, 209, 33, 97, 161, 225, 49, 113, 177, 241, 5, 69, 133, 197, 21, 85,
//		149, 213, 37, 101, 165, 229, 53, 117, 181, 245, 9, 73, 137, 201, 25, 89,
//		153, 217, 41, 105, 169, 233, 57, 121, 185, 249, 13, 77, 141, 205, 29,
//		93, 157, 221, 45, 109, 173, 237, 61, 125, 189, 253, 2, 66, 130, 194, 18,
//		82, 146, 210, 34, 98, 162, 226, 50, 114, 178, 242, 6, 70, 134, 198, 22,
//		86, 150, 214, 38, 102, 166, 230, 54, 118, 182, 246, 10, 74, 138, 202,
//		26, 90, 154, 218, 42, 106, 170, 234, 58, 122, 186, 250, 14, 78, 142,
//		206, 30, 94, 158, 222, 46, 110, 174, 238, 62, 126, 190, 254, 3, 67, 131,
//		195, 19, 83, 147, 211, 35, 99, 163, 227, 51, 115, 179, 243, 7, 71, 135,
//		199, 23, 87, 151, 215, 39, 103, 167, 231, 55, 119, 183, 247, 11, 75,
//		139, 203, 27, 91, 155, 219, 43, 107, 171, 235, 59, 123, 187, 251, 15,
//		79, 143, 207, 31, 95, 159, 223, 47, 111, 175, 239, 63, 127, 191, 255 };

//Works only for 4 byte
inline ulong revComp(ulong prefix) {
	static const int shift = 32 - CS::prefixBits;

	//Compute complement
	ulong compPrefix = (prefix ^ 0xAAAAAAAA) & CS::prefixMask;
	//Reverse
	compPrefix = compPrefix << shift;
	ulong compRevPrefix = (ReverseTable16[compPrefix & 0x0f] << 28) | (ReverseTable16[(compPrefix >> 4) & 0x0f] << 24)
			| (ReverseTable16[(compPrefix >> 8) & 0x0f] << 20) | (ReverseTable16[(compPrefix >> 12) & 0x0f] << 16)
			| (ReverseTable16[(compPrefix >> 16) & 0x0f] << 12) | (ReverseTable16[(compPrefix >> 20) & 0x0f] << 8)
			| (ReverseTable16[(compPrefix >> 24) & 0x0f] << 4) | (ReverseTable16[(compPrefix >> 28) & 0x0f]);

//	ulong compRevPrefix = (ReverseTable256[compPrefix & 0xff] << 24)
//			| (ReverseTable256[(compPrefix >> 8) & 0xff] << 16)
//			| (ReverseTable256[(compPrefix >> 16) & 0xff] << 8)
//			| (ReverseTable256[(compPrefix >> 24) & 0xff]);

	return compRevPrefix;
}

int * CompactPrefixTable::CountKmerFreq(uint length) {

	Log.Message("Number of k-mers: %d", length);
	int * freq = new int[length];
	memset(freq, 0, length);

	//for (int i = 0; i < SequenceProvider.GetRefCount(); ++i) {
	int i = 0;
	lastPrefix = 111111;
	lastBin = -1;
	m_CurGenSeq = i;

	if (!NGM.DualStrand() || !(m_CurGenSeq % 2)) {
		uint offset = 0;
		uint len = SequenceProvider.GetConcatRefLen();
		char * seq = new char[len + 2];
		SequenceProvider.DecodeRefSequence(seq, m_CurGenSeq, 0, len);

		CS::PrefixIteration(seq, len, &CompactPrefixTable::CountKmer, 0, 0, freq, m_RefSkip, offset);
		delete[] seq;
		seq = 0;
	}
	//}
	return freq;<commit_msg>Set spacer char for concatinating references back to 'N' to avoid interferance with PrefixIteration.<commit_after>#include "PrefixTable.h"
#include "NGM.h"
#include "CS.h"

//#include "SequenceLocation.h"

#include "Timing.h"

#include <stdexcept>

#undef module_name
#define module_name "PFTABLE"

#include <cmath>
#include <algorithm>
#include <sstream>
#include <stdio.h>

static uint const refTabCookie = 0x1701D;

int lastSeqTotal = 0;

int CompactPrefixTable::maxPrefixFreq = 1000;

ulong CompactPrefixTable::lastPrefix;
int CompactPrefixTable::lastBin;
int CompactPrefixTable::lastPos;

uint CompactPrefixTable::skipCount;
uint CompactPrefixTable::skipBuild;

static const unsigned char ReverseTable16[] = { 0x00, 0x04, 0x08, 0x0C, 0x01, 0x05, 0x09, 0x0D, 0x02, 0x06, 0x0A, 0x0E, 0x03, 0x07, 0x0B,
		0x0F };

//static const unsigned char ReverseTable256[] = { 0, 64, 128, 192, 16, 80, 144,
//		208, 32, 96, 160, 224, 48, 112, 176, 240, 4, 68, 132, 196, 20, 84, 148,
//		212, 36, 100, 164, 228, 52, 116, 180, 244, 8, 72, 136, 200, 24, 88, 152,
//		216, 40, 104, 168, 232, 56, 120, 184, 248, 12, 76, 140, 204, 28, 92,
//		156, 220, 44, 108, 172, 236, 60, 124, 188, 252, 1, 65, 129, 193, 17, 81,
//		145, 209, 33, 97, 161, 225, 49, 113, 177, 241, 5, 69, 133, 197, 21, 85,
//		149, 213, 37, 101, 165, 229, 53, 117, 181, 245, 9, 73, 137, 201, 25, 89,
//		153, 217, 41, 105, 169, 233, 57, 121, 185, 249, 13, 77, 141, 205, 29,
//		93, 157, 221, 45, 109, 173, 237, 61, 125, 189, 253, 2, 66, 130, 194, 18,
//		82, 146, 210, 34, 98, 162, 226, 50, 114, 178, 242, 6, 70, 134, 198, 22,
//		86, 150, 214, 38, 102, 166, 230, 54, 118, 182, 246, 10, 74, 138, 202,
//		26, 90, 154, 218, 42, 106, 170, 234, 58, 122, 186, 250, 14, 78, 142,
//		206, 30, 94, 158, 222, 46, 110, 174, 238, 62, 126, 190, 254, 3, 67, 131,
//		195, 19, 83, 147, 211, 35, 99, 163, 227, 51, 115, 179, 243, 7, 71, 135,
//		199, 23, 87, 151, 215, 39, 103, 167, 231, 55, 119, 183, 247, 11, 75,
//		139, 203, 27, 91, 155, 219, 43, 107, 171, 235, 59, 123, 187, 251, 15,
//		79, 143, 207, 31, 95, 159, 223, 47, 111, 175, 239, 63, 127, 191, 255 };

//Works only for 4 byte
inline ulong revComp(ulong prefix) {
	static const int shift = 32 - CS::prefixBits;

	//Compute complement
	ulong compPrefix = (prefix ^ 0xAAAAAAAA) & CS::prefixMask;
	//Reverse
	compPrefix = compPrefix << shift;
	ulong compRevPrefix = (ReverseTable16[compPrefix & 0x0f] << 28) | (ReverseTable16[(compPrefix >> 4) & 0x0f] << 24)
			| (ReverseTable16[(compPrefix >> 8) & 0x0f] << 20) | (ReverseTable16[(compPrefix >> 12) & 0x0f] << 16)
			| (ReverseTable16[(compPrefix >> 16) & 0x0f] << 12) | (ReverseTable16[(compPrefix >> 20) & 0x0f] << 8)
			| (ReverseTable16[(compPrefix >> 24) & 0x0f] << 4) | (ReverseTable16[(compPrefix >> 28) & 0x0f]);

//	ulong compRevPrefix = (ReverseTable256[compPrefix & 0xff] << 24)
//			| (ReverseTable256[(compPrefix >> 8) & 0xff] << 16)
//			| (ReverseTable256[(compPrefix >> 16) & 0xff] << 8)
//			| (ReverseTable256[(compPrefix >> 24) & 0xff]);

	return compRevPrefix;
}

int * CompactPrefixTable::CountKmerFreq(uint length) {

	Log.Message("Number of k-mers: %d", length);
	int * freq = new int[length];
	memset(freq, 0, length);

	//for (int i = 0; i < SequenceProvider.GetRefCount(); ++i) {
	int i = 0;
	lastPrefix = 111111;
	lastBin = -1;
	m_CurGenSeq = i;

	if (!NGM.DualStrand() || !(m_CurGenSeq % 2)) {

		int refCount = SequenceProvider.GetRefCount();

		for (int i = 0; i < refCount; ++i) {
			uint offset = SequenceProvider.GetRefStart(i);

			uint offset = 0;
			uint len = SequenceProvider.GetConcatRefLen();
			char * seq = new char[len + 2];
			SequenceProvider.DecodeRefSequence(seq, m_CurGenSeq, 0, len);

			CS::PrefixIteration(seq, len, &CompactPrefixTable::CountKmer, 0, 0, freq, m_RefSkip, offset);
			delete[] seq;
			seq = 0;
		}
		//int j = 0;
		//while (j < refCount && m_Location >= SequenceProvider.GetRefStart(j)) {
		//	j += (NGM.DualStrand()) ? 2 : 1;
		//}
	}
	//}
	return freq;
{'fraction_non_alphanumeric': 0.18204653622421998, 'fraction_numerical': 0.2168164992067689, 'mean_word_length': 3.89010989010989, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6051041', 'n_tokens_mistral': 4710, 'n_tokens_neox': 3084, 'n_words': 973}