<commit_before>/*
 * jaccard.cpp
 *
 *  Created on: Apr 24, 2015
 *      Author: nek3d
 */

#include "jaccard.h"

Jaccard::Jaccard(ContextJaccard *context)
: IntersectFile(context),
 _queryUnion(0),
 _dbUnion(0),
 _intersectionVal(0),
 _unionVal(0),
 _numIntersections(0)
{

}

bool Jaccard::findNext(RecordKeyVector &hits) {
	if (nextSortedFind(hits)) {
		checkSplits(hits);
		_intersectionVal += getTotalIntersection(hits);
		return true;
	}
	return false;
}

void Jaccard::cleanupHits(RecordKeyVector &hits)
{
//	upCastFRM(_queryFRM)->deleteMergedRecord(hits);
}


bool Jaccard::finalizeCalculations() {
	_sweep->closeOut(true);
	_queryUnion = _sweep->getQueryTotalRecordLength();
	_dbUnion = _sweep->getDatabaseTotalRecordLength();
	_unionVal = _queryUnion + _dbUnion;
	return true;
}

void  Jaccard::giveFinalReport(RecordOutputMgr *outputMgr) {
	// header
	outputMgr->checkForHeader();

	cout << "intersection\tunion-intersection\tjaccard\tn_intersections" << endl;

	unsigned long adjustedUnion = _unionVal - _intersectionVal;

	cout << _intersectionVal << "\t" << adjustedUnion << "\t" <<
			(float) _intersectionVal / (float)adjustedUnion << "\t" << _numIntersections << endl;
}

unsigned long Jaccard::getTotalIntersection(RecordKeyVector &hits)
{
	unsigned long intersection = 0;
	Record *key = hits.getKey();
	CHRPOS keyStart = key->getStartPos();
	CHRPOS keyEnd = key->getEndPos();

	int hitIdx = 0;
	for (RecordKeyVector::iterator_type iter = hits.begin(); iter != hits.end(); iter = hits.next()) {
		Record *currRec = *iter;
		CHRPOS maxStart = max(currRec->getStartPos(), keyStart);
		CHRPOS minEnd = min(currRec->getEndPos(), keyEnd);
		if (_context->getObeySplits()) {
			intersection += upCast(_context)->getSplitBlockInfo()->getOverlapBases(hitIdx);
			hitIdx++;<commit_msg>change jaccard header to union<commit_after>/*
 * jaccard.cpp
 *
 *  Created on: Apr 24, 2015
 *      Author: nek3d
 */

#include "jaccard.h"

Jaccard::Jaccard(ContextJaccard *context)
: IntersectFile(context),
 _queryUnion(0),
 _dbUnion(0),
 _intersectionVal(0),
 _unionVal(0),
 _numIntersections(0)
{

}

bool Jaccard::findNext(RecordKeyVector &hits) {
	if (nextSortedFind(hits)) {
		checkSplits(hits);
		_intersectionVal += getTotalIntersection(hits);
		return true;
	}
	return false;
}

void Jaccard::cleanupHits(RecordKeyVector &hits)
{
//	upCastFRM(_queryFRM)->deleteMergedRecord(hits);
}


bool Jaccard::finalizeCalculations() {
	_sweep->closeOut(true);
	_queryUnion = _sweep->getQueryTotalRecordLength();
	_dbUnion = _sweep->getDatabaseTotalRecordLength();
	_unionVal = _queryUnion + _dbUnion;
	return true;
}

void  Jaccard::giveFinalReport(RecordOutputMgr *outputMgr) {
	// header
	outputMgr->checkForHeader();

	cout << "intersection\tunion\tjaccard\tn_intersections" << endl;

	unsigned long adjustedUnion = _unionVal - _intersectionVal;

	cout << _intersectionVal << "\t" << adjustedUnion << "\t" <<
			(float) _intersectionVal / (float)adjustedUnion << "\t" << _numIntersections << endl;
}

unsigned long Jaccard::getTotalIntersection(RecordKeyVector &hits)
{
	unsigned long intersection = 0;
	Record *key = hits.getKey();
	CHRPOS keyStart = key->getStartPos();
	CHRPOS keyEnd = key->getEndPos();

	int hitIdx = 0;
	for (RecordKeyVector::iterator_type iter = hits.begin(); iter != hits.end(); iter = hits.next()) {
		Record *currRec = *iter;
		CHRPOS maxStart = max(currRec->getStartPos(), keyStart);
		CHRPOS minEnd = min(currRec->getEndPos(), keyEnd);
		if (_context->getObeySplits()) {
			intersection += upCast(_context)->getSplitBlockInfo()->getOverlapBases(hitIdx);
			hitIdx++;
{'fraction_non_alphanumeric': 0.14560133630289532, 'fraction_numerical': 0.0077951002227171495, 'mean_word_length': 5.556569343065694, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3359739', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1273, 'n_words': 233}