<commit_before>#ifndef TESTLSH_H
#define TESTLSH_H
#define ARMA_64BIT_WORD
#include <cstdint>
#include <armadillo>
#include <unordered_map>
#include <algorithm>
#include <sys/types.h>
#include <sstream>
#include <random>
#include <set>
#include <list>
#include <sys/time.h>

//hash functions
#include "RandomProjectionHashFunction.h"
#include "RandomSubSamplingHashFunction.h"
#include "ShiftInvariantKernelHashFunction.h"
#include "DHHashFunction.h"
#include "RandomRotationHashFunction.h"

//data transformers
#include "NullTransformer.h"
#include "MeanTransformer.h"
#include "ThresholdTransformer.h"
#include "PCATransformer.h"
#include "DHHashTransformer.h"

//hash indices
#include "StdHashIndex.h"
#include "ResizeableHashIndex.h"
#include "LSHForestHashIndex.h"
#include "LSHForestHashIndexAsync.h"

//hash collection
#include "HashCollection.h"


double get_wall_time(){
    struct timeval time;
    if (gettimeofday(&time,NULL)){
        //  Handle error
        return 0;
    }
    return (double)time.tv_sec + (double)time.tv_usec * .000001;
}
#include <iostream>

template<class HashGenerator, class DataTransformer, class HashFunction, class HashIndex, class MAT>
class TestLSH {
public:
    const static std::tuple<double, double, double> TestLSHTimings(
        const MAT& data,
        const MAT& queries,
        const arma::Mat<uint32_t>& groundtruth, //this is the ground truth knn for the first m values of data against the rest of data
        const size_t& datadimensionality,
        const size_t& datalength,
        const size_t& querylength,
        const size_t& NumHashFunctions, //number of hash tables in the collection
        const size_t& NumHashBits, //number of bits per function
        const size_t& NumHashCandidates, //for trainable algorithms: number of candidates to optimise over
        const size_t& MaxSearchRadius, //maximum hamming ball / tree depth to explore
        const size_t& MaxQuerySize, //for the whole query
        const size_t& MaxRetrievalThreshold, //for each table
        const size_t& MinRetrievalThreshold, //for each table
        const size_t& KNeighbours) {

        //Step 1: build the hash functions
        double t0 = get_wall_time();
        HashCollection<DataTransformer,
                   HashFunction,
                   HashIndex,
                   MAT
                   > hashindex =
                   HashGenerator::GetHashes(
                            groundtruth,
                            data,
                            NumHashCandidates,
                            NumHashFunctions,
                            MinRetrievalThreshold,
                            MaxSearchRadius,
                            MaxQuerySize,
                            NumHashBits);
        double t1 = get_wall_time();

        //Step 2: build the hash index
        double t2 = get_wall_time();
        hashindex.buildChunkedDB(data);
        double t3 = get_wall_time();

        //Step 3: do the queries
        double t4 = get_wall_time();
        const std::pair<arma::Mat<uint32_t>,MAT> result__ = hashindex.batchQuery(queries,
                                                                            KNeighbours,
                                                                            data);
        double t5 = get_wall_time();

        //return the timings for all 3 parts
        return {t1 - t0, t3 - t2, t5 - t4};
    }


const static std::pair<arma::mat,arma::mat> TestLSHAccuracy(
        const MAT& data,
        const MAT& queries,<commit_msg>fix gcc constructor list compile issue<commit_after>#ifndef TESTLSH_H
#define TESTLSH_H
#define ARMA_64BIT_WORD
#include <cstdint>
#include <armadillo>
#include <unordered_map>
#include <algorithm>
#include <sys/types.h>
#include <sstream>
#include <random>
#include <set>
#include <list>
#include <sys/time.h>

//hash functions
#include "RandomProjectionHashFunction.h"
#include "RandomSubSamplingHashFunction.h"
#include "ShiftInvariantKernelHashFunction.h"
#include "DHHashFunction.h"
#include "RandomRotationHashFunction.h"

//data transformers
#include "NullTransformer.h"
#include "MeanTransformer.h"
#include "ThresholdTransformer.h"
#include "PCATransformer.h"
#include "DHHashTransformer.h"

//hash indices
#include "StdHashIndex.h"
#include "ResizeableHashIndex.h"
#include "LSHForestHashIndex.h"
#include "LSHForestHashIndexAsync.h"

//hash collection
#include "HashCollection.h"


double get_wall_time(){
    struct timeval time;
    if (gettimeofday(&time,NULL)){
        //  Handle error
        return 0;
    }
    return (double)time.tv_sec + (double)time.tv_usec * .000001;
}
#include <iostream>

template<class HashGenerator, class DataTransformer, class HashFunction, class HashIndex, class MAT>
class TestLSH {
public:
    const static std::tuple<double, double, double> TestLSHTimings(
        const MAT& data,
        const MAT& queries,
        const arma::Mat<uint32_t>& groundtruth, //this is the ground truth knn for the first m values of data against the rest of data
        const size_t& datadimensionality,
        const size_t& datalength,
        const size_t& querylength,
        const size_t& NumHashFunctions, //number of hash tables in the collection
        const size_t& NumHashBits, //number of bits per function
        const size_t& NumHashCandidates, //for trainable algorithms: number of candidates to optimise over
        const size_t& MaxSearchRadius, //maximum hamming ball / tree depth to explore
        const size_t& MaxQuerySize, //for the whole query
        const size_t& MaxRetrievalThreshold, //for each table
        const size_t& MinRetrievalThreshold, //for each table
        const size_t& KNeighbours) {

        //Step 1: build the hash functions
        double t0 = get_wall_time();
        HashCollection<DataTransformer,
                   HashFunction,
                   HashIndex,
                   MAT
                   > hashindex =
                   HashGenerator::GetHashes(
                            groundtruth,
                            data,
                            NumHashCandidates,
                            NumHashFunctions,
                            MinRetrievalThreshold,
                            MaxSearchRadius,
                            MaxQuerySize,
                            NumHashBits);
        double t1 = get_wall_time();

        //Step 2: build the hash index
        double t2 = get_wall_time();
        hashindex.buildChunkedDB(data);
        double t3 = get_wall_time();

        //Step 3: do the queries
        double t4 = get_wall_time();
        const std::pair<arma::Mat<uint32_t>,MAT> result__ = hashindex.batchQuery(queries,
                                                                            KNeighbours,
                                                                            data);
        double t5 = get_wall_time();

        //return the timings for all 3 parts
        return std::tuple<double,double,double>(t1 - t0, t3 - t2, t5 - t4);
    }


const static std::pair<arma::mat,arma::mat> TestLSHAccuracy(
        const MAT& data,
        const MAT& queries,
{'fraction_non_alphanumeric': 0.0845109836294949, 'fraction_numerical': 0.008115293129984609, 'mean_word_length': 2.1684397163120566, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '761601', 'n_tokens_mistral': 2054, 'n_tokens_neox': 1964, 'n_words': 560}