<commit_before>#include "hasher_obj.hpp"
#include "iotools.h"

#include <stdio.h>
#include <opencv2/opencv.hpp>
#include <fstream>

using namespace std;
using namespace cv;

int HasherObject::load_hashcodes() {
    cout << "Loading hashcodes... " << endl;
    // If we are updating
    itq.release();
    itq.create(data_num, int_num, CV_32SC1);
    char * read_pos = (char*)itq.data;
    for (int i=0; i < update_hash_files.size(); i++)
    {
        read_in.open(update_hash_files[i],ios::in|ios::binary);
        if (!read_in.is_open())
        {
            cout << "Cannot load the itq updates! File "<< update_hash_files[i] << endl;
            return -1;
        }
        read_size = sizeof(int)*data_nums[i]*int_num;
        read_in.read(read_pos, read_size);
        read_in.close();
        read_pos +=read_size;
    }
    cout << "Hashcodes dimensions are " << itq.rows << "x" << itq.cols << endl;
    return 0;
}


int HasherObject::load_itq_model() {
    // Read itq model
    // read W
    read_in.open(W_name, ios::in|ios::binary);
    if (!read_in.is_open())
    {
        cout << "Cannot load the W model from " << W_name << endl;
        return -1;
    }
    // If we are updating
    W.release();
    W.create(feature_dim, bit_num, CV_64F);
    read_size = sizeof(double)*feature_dim*bit_num;
    read_in.read((char*)W.data, read_size);
    read_in.close();
    // read mvec
    read_in.open(mvec_name, ios::in|ios::binary);
    if (!read_in.is_open())
    {
        cout << "Cannot load the mvec model from " << mvec_name << endl;
        return -1;
    }
    // If we are updating
    mvec.release();
    mvec.create(1, bit_num, CV_64F);
    read_size = sizeof(double)*bit_num;
    read_in.read((char*)mvec.data, read_size);
    read_in.close();
    return 0;
}


Mat HasherObject::read_feats_from_disk(string filename) {
    // Read count
    int feats_num = (int)filesize(filename)/4/feature_dim;
    cout << "[read_feats_from_disk] Reading " << feats_num << " features." << endl;
    // Check input file
    ifstream read_in(filename, ios::in|ios::binary);
    if (!read_in.is_open())
    {
        cout << "[read_feats_from_disk] Cannot load the feature file: " << filename << endl;
        return Mat();
    }
    // Allocate memory
    Mat feats_mat(feats_num, feature_dim, CV_32F);
    // Read features
    size_t read_size = sizeof(float)*feature_dim*feats_num;
    read_in.read((char*)feats_mat.data, read_size);
    // Finalize reading
    read_in.close();
    cout << "[read_feats_from_disk] Read " << read_size <<  " bytes for " << feats_num << " features." << endl;
    return feats_mat;
}

// // When would that be necessary?
// Mat HasherObject::read_hashcodes_from_disk(string filename) {

// }

void HasherObject::set_query_feats_from_disk(string filename) {
    query_feats = read_feats_from_disk(filename);
}


// compute hashcodes from feats
unsigned int* HasherObject::compute_hashcodes_from_feats(Mat feats_mat) {
    // hashing init
    if (norm) {
        for  (int k = 0; k < query_num; k++)
            normalize((float*)feats_mat.data + k*feature_dim, feature_dim);
    }
    // Allocate temporary matrices
    Mat feats_mat_double;
    feats_mat.convertTo(feats_mat_double, CV_64F);
    int feats_num = feats_mat.rows;
    cout << "[compute_hashcodes_from_feats] Computing hashcodes for " << feats_num << " features." << endl;
    mvec = repeat(mvec, feats_num,1);
    // Project features
    Mat realvalued_hash = feats_mat_double*W-mvec;
    // Binarizing features
    unsigned int * hash_mat = new unsigned int[int_num*query_num];
    for  (int k=0; k < query_num; k++)
    {
        for (int i=0; i < int_num; i++)
        {
            hash_mat[k*this->int_num+i] = 0;
            for (int j=0;j<32;j++)
                if (realvalued_hash.at<double>(k,i*32+j)>0)
                    hash_mat[k*this->int_num+i] += 1<<j;
        }
    }
    // Done hashing features
    return hash_mat;
}


// query methods
// use member query_feats and query_codes
void HasherObject::find_knn() {
    unsigned int* query = query_codes;
    float* query_feature = (float*)query_feats.data;
    query_num = query_feats.rows;
    vector<mypair> top_hamming;
    init_output_files();
    for (int k=0; k < query_num; k++)
    {
        cout <<  "Looking for similar images of query #" << k+1 << endl;
        // Compute hamming distances between query k and all DB hashcodes
        cout <<  "Computing hamming distances for query #" << k+1 << endl;
        top_hamming = compute_hamming_dist_onehash(query);
        // Rerank based on real valued features
        cout <<  "Reranking for query #" << k+1 << endl;
        postrank = rerank_knn_onesample(query_feature, top_hamming);
        // Write out results
        cout <<  "Writing output for query #" << k+1 << endl;
        write_to_output_file(postrank, hamming);
        // Move to next query
        cout <<  "Moving to next query" << endl;
        query += int_num;
        query_feature += feature_dim;
    }
    // Clean up
    delete[] query_codes;
    query_feats.release();
    close_output_files();
}

void HasherObject::find_knn_from_feats(Mat _query_feats) {
    query_codes = compute_hashcodes_from_feats(_query_feats);
    query_feats = _query_feats;
    find_knn();
}

// compute rerank top samples using real valued features
vector<mypairf> HasherObject::rerank_knn_onesample(float* query_feature, vector<mypair> top_hamming) {
    //read needed feature for post ranking
    //t[1]=get_wall_time();
    cout << "We have " << top_hamming.size() << " candidates to rerank." << endl;

    vector<mypairf> postrank(top_hamming.size());
    char* feature_p = (char*)top_feature_mat.data;
    read_size = sizeof(float)*feature_dim;
    int status = 0;
    int i;
    for (i = 0; i < top_hamming.size(); i++)
    {
        //cout << "Getting precomputed feature #" << top_hamming[i].second << endl;
        // accum, read_in_compfeatures, read_in_compidx should be set previously
        status = get_onefeatcomp(top_hamming[i].second, read_size, accum, read_in_compfeatures, read_in_compidx, feature_p);
        //status = get_onefeat(hamming[i].second,read_size,accum,read_in_features,feature_p);
        if (status == -1) {
            cout << "[rerank_knn_onesample] Could not get feature " << top_hamming[i].second << ". Exiting." << std::endl;
            // should clean here
            return vector<mypairf>(0);
        }
        feature_p += read_size;
    }

    cout << "[rerank_knn_onesample] Biggest hamming distance is: " << top_hamming[i-1].first << endl;<commit_msg>Is the feature read correctly<commit_after>#include "hasher_obj.hpp"
#include "iotools.h"

#include <stdio.h>
#include <opencv2/opencv.hpp>
#include <fstream>

using namespace std;
using namespace cv;

int HasherObject::load_hashcodes() {
    cout << "Loading hashcodes... " << endl;
    // If we are updating
    itq.release();
    itq.create(data_num, int_num, CV_32SC1);
    char * read_pos = (char*)itq.data;
    for (int i=0; i < update_hash_files.size(); i++)
    {
        read_in.open(update_hash_files[i],ios::in|ios::binary);
        if (!read_in.is_open())
        {
            cout << "Cannot load the itq updates! File "<< update_hash_files[i] << endl;
            return -1;
        }
        read_size = sizeof(int)*data_nums[i]*int_num;
        read_in.read(read_pos, read_size);
        read_in.close();
        read_pos +=read_size;
    }
    cout << "Hashcodes dimensions are " << itq.rows << "x" << itq.cols << endl;
    return 0;
}


int HasherObject::load_itq_model() {
    // Read itq model
    // read W
    read_in.open(W_name, ios::in|ios::binary);
    if (!read_in.is_open())
    {
        cout << "Cannot load the W model from " << W_name << endl;
        return -1;
    }
    // If we are updating
    W.release();
    W.create(feature_dim, bit_num, CV_64F);
    read_size = sizeof(double)*feature_dim*bit_num;
    read_in.read((char*)W.data, read_size);
    read_in.close();
    // read mvec
    read_in.open(mvec_name, ios::in|ios::binary);
    if (!read_in.is_open())
    {
        cout << "Cannot load the mvec model from " << mvec_name << endl;
        return -1;
    }
    // If we are updating
    mvec.release();
    mvec.create(1, bit_num, CV_64F);
    read_size = sizeof(double)*bit_num;
    read_in.read((char*)mvec.data, read_size);
    read_in.close();
    return 0;
}


Mat HasherObject::read_feats_from_disk(string filename) {
    // Read count
    int feats_num = (int)filesize(filename)/4/feature_dim;
    cout << "[read_feats_from_disk] Reading " << feats_num << " features." << endl;
    // Check input file
    ifstream read_in(filename, ios::in|ios::binary);
    if (!read_in.is_open())
    {
        cout << "[read_feats_from_disk] Cannot load the feature file: " << filename << endl;
        return Mat();
    }
    // Allocate memory
    Mat feats_mat(feats_num, feature_dim, CV_32F);
    // Read features
    size_t read_size = sizeof(float)*feature_dim*feats_num;
    read_in.read((char*)feats_mat.data, read_size);
    // Finalize reading
    read_in.close();
    cout << "[read_feats_from_disk] Read " << read_size <<  " bytes for " << feats_num << " features." << endl;
    cout << "[read_feats_from_disk] Features first value is: " << feats_mat.at<double>(0,0) << endl;
    return feats_mat;
}

// // When would that be necessary?
// Mat HasherObject::read_hashcodes_from_disk(string filename) {

// }

void HasherObject::set_query_feats_from_disk(string filename) {
    query_feats.release();
    query_feats = read_feats_from_disk(filename);
    cout << "[set_query_feats_from_disk] Features first value is: " << query_feats.at<double>(0,0) << endl;
}


// compute hashcodes from feats
unsigned int* HasherObject::compute_hashcodes_from_feats(Mat feats_mat) {
    cout << "[compute_hashcodes_from_feats] Features first value is: " << feats_mat.at<double>(0,0) << endl;
    // hashing init
    if (norm) {
        for  (int k = 0; k < query_num; k++)
            normalize((float*)feats_mat.data + k*feature_dim, feature_dim);
    }
    cout << "[compute_hashcodes_from_feats:after_norm] Features first value is: " << feats_mat.at<double>(0,0) << endl;
    // Allocate temporary matrices
    Mat feats_mat_double;
    feats_mat.convertTo(feats_mat_double, CV_64F);
    int feats_num = feats_mat.rows;
    cout << "[compute_hashcodes_from_feats] Computing hashcodes for " << feats_num << " features." << endl;
    mvec = repeat(mvec, feats_num,1);
    // Project features
    Mat realvalued_hash = feats_mat_double*W-mvec;
    // Binarizing features
    unsigned int * hash_mat = new unsigned int[int_num*query_num];
    for  (int k=0; k < query_num; k++)
    {
        for (int i=0; i < int_num; i++)
        {
            hash_mat[k*this->int_num+i] = 0;
            for (int j=0;j<32;j++)
                if (realvalued_hash.at<double>(k,i*32+j)>0)
                    hash_mat[k*this->int_num+i] += 1<<j;
        }
    }
    // Done hashing features
    return hash_mat;
}


// query methods
// use member query_feats and query_codes
void HasherObject::find_knn() {
    query_codes = compute_hashcodes_from_feats(query_feats);
    unsigned int* query = query_codes;
    float* query_feature = (float*)query_feats.data;
    query_num = query_feats.rows;
    vector<mypair> top_hamming;
    init_output_files();
    for (int k=0; k < query_num; k++)
    {
        cout <<  "Looking for similar images of query #" << k+1 << endl;
        // Compute hamming distances between query k and all DB hashcodes
        cout <<  "Computing hamming distances for query #" << k+1 << endl;
        top_hamming = compute_hamming_dist_onehash(query);
        // Rerank based on real valued features
        cout <<  "Reranking for query #" << k+1 << endl;
        postrank = rerank_knn_onesample(query_feature, top_hamming);
        // Write out results
        cout <<  "Writing output for query #" << k+1 << endl;
        write_to_output_file(postrank, hamming);
        // Move to next query
        cout <<  "Moving to next query" << endl;
        query += int_num;
        query_feature += feature_dim;
    }
    // Clean up
    delete[] query_codes;
    query_feats.release();
    close_output_files();
}

void HasherObject::find_knn_from_feats(Mat _query_feats) {
    query_feats.release();
    query_feats = _query_feats;
    find_knn();
}

// compute rerank top samples using real valued features
vector<mypairf> HasherObject::rerank_knn_onesample(float* query_feature, vector<mypair> top_hamming) {
    //read needed feature for post ranking
    //t[1]=get_wall_time();
    cout << "We have " << top_hamming.size() << " candidates to rerank." << endl;

    vector<mypairf> postrank(top_hamming.size());
    char* feature_p = (char*)top_feature_mat.data;
    read_size = sizeof(float)*feature_dim;
    int status = 0;
    int i;
    for (i = 0; i < top_hamming.size(); i++)
    {
        //cout << "Getting precomputed feature #" << top_hamming[i].second << endl;
        // accum, read_in_compfeatures, read_in_compidx should be set previously
        status = get_onefeatcomp(top_hamming[i].second, read_size, accum, read_in_compfeatures, read_in_compidx, feature_p);
        //status = get_onefeat(hamming[i].second,read_size,accum,read_in_features,feature_p);
        if (status == -1) {
            cout << "[rerank_knn_onesample] Could not get feature " << top_hamming[i].second << ". Exiting." << std::endl;
            // should clean here
            return vector<mypairf>(0);
        }
        feature_p += read_size;
    }

    cout << "[rerank_knn_onesample] Biggest hamming distance is: " << top_hamming[i-1].first << endl;
{'fraction_non_alphanumeric': 0.13471237482640158, 'fraction_numerical': 0.006870842774650976, 'mean_word_length': 2.955478461983232, 'pattern_counts': {'":': 0, '<': 311, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3740185', 'n_tokens_mistral': 4775, 'n_tokens_neox': 4677, 'n_words': 1041}