<commit_before>/* 
 * GIGA+ indexing implementation 
 *  by- <NAME> (svp at cs)
 *
 * TERMINOLOGY:
 *  - bitmap: indicates the presence/absence of a partition
 *  - hash: hash of the filename
 *  - index: position in the bitmap
 *
 */

#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <malloc.h>
#include <string.h>
#include <assert.h>

#include "trace.h"

#include "giga_index.h"
#include "sha.h"

#define ARRAY_LEN(array)(sizeof(array)/sizeof((array)[0])) 

//static int hash_compare(char hash_val_1[], char hash_val_2[], int len);

static index_t get_child_index(index_t index, int radix);
static index_t get_parent_index(index_t index);

static index_t compute_index(char hash_value[], int radix); 
static int get_highest_index(bitmap_t bitmap[]);

static int get_bit_status(bitmap_t bmap[], index_t index);

static int get_radix_from_bmap(bitmap_t bitmap[]);
static int get_radix_from_index(index_t index);

static void print_bitmap(bitmap_t bmap[], FILE *fp);

//static void struct giga_mapping_t_update_radix(struct giga_mapping_t *table);

// Compute the SHA-1 hash of the file name (or path name) 
//
void giga_hash_name(const char* hash_key, char hash_value[])
{
    uint8_t hash[SHA1_HASH_SIZE] = {0}; 
    assert(hash_key);
    assert(hash_value);
    int len = (int)strlen(hash_key);

#ifdef DBG_INDEXING
    dbg_msg(log_fp, "[%s] hash: key={%s} of len=%d",__func__, hash_key, len);
#endif

    shahash((uint8_t*) hash_key, len, hash);
    binary2hex(hash, SHA1_HASH_SIZE, hash_value);

#ifdef DBG_INDEXING
    int i;
    //fprintf(log_fp, "[%s] hash={%s}\n", __func__, hash_value);
    fprintf(log_fp, "[%s] hash={", __func__);
    for (i=0; i<HASH_LEN; i++)
        fprintf(log_fp, "%c", hash_value[i]);
    fprintf(log_fp, "} of len=%d\n", HASH_LEN);
#endif
}

// Initialize the mapping table: 
// - set the bitmap to all zeros, except for the first location to one which
//   indicates the presence of a zeroth bucket
// - set the radix to 1 (XXX: do we need radix??)
// - flag indicates the number of servers if you use static partitioning
//
void giga_init_mapping(struct giga_mapping_t *mapping, int flag)
{
    int i;
#ifdef DBG_INDEXING
    dbg_msg(log_fp, "[%s] initialize giga mapping (flag=%d)", __func__, flag);
#endif

    assert(mapping != NULL);

    memset(mapping->bitmap, 0, MAX_BMAP_LEN);
    
    if (flag == -1) {
        mapping->bitmap[0] = 1;
        mapping->curr_radix = 1;
        return;
    }
    
    switch(SPLIT_TYPE) {
        //case SPLIT_TYPE_KEEP_SPLITTING:
        case SPLIT_T_NO_BOUND:
            mapping->bitmap[0] = 1;
            mapping->curr_radix = 1;    
            break;
        //case SPLIT_TYPE_NEVER_SPLIT:
        case SPLIT_T_NO_SPLITTING_EVER:
            assert(flag != -1);
            if (flag < BITS_PER_MAP){<commit_msg>fixed giga_mapping initialization<commit_after>/* 
 * GIGA+ indexing implementation 
 *  by- <NAME> (svp at cs)
 *
 * TERMINOLOGY:
 *  - bitmap: indicates the presence/absence of a partition
 *  - hash: hash of the filename
 *  - index: position in the bitmap
 *
 */

#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <malloc.h>
#include <string.h>
#include <assert.h>

#include "trace.h"

#include "giga_index.h"
#include "sha.h"

#define ARRAY_LEN(array)(sizeof(array)/sizeof((array)[0])) 

//static int hash_compare(char hash_val_1[], char hash_val_2[], int len);

static index_t get_child_index(index_t index, int radix);
static index_t get_parent_index(index_t index);

static index_t compute_index(char hash_value[], int radix); 
static int get_highest_index(bitmap_t bitmap[]);

static int get_bit_status(bitmap_t bmap[], index_t index);

static int get_radix_from_bmap(bitmap_t bitmap[]);
static int get_radix_from_index(index_t index);

static void print_bitmap(bitmap_t bmap[], FILE *fp);

//static void struct giga_mapping_t_update_radix(struct giga_mapping_t *table);

// Compute the SHA-1 hash of the file name (or path name) 
//
void giga_hash_name(const char* hash_key, char hash_value[])
{
    uint8_t hash[SHA1_HASH_SIZE] = {0}; 
    assert(hash_key);
    assert(hash_value);
    int len = (int)strlen(hash_key);

#ifdef DBG_INDEXING
    dbg_msg(log_fp, "[%s] hash: key={%s} of len=%d",__func__, hash_key, len);
#endif

    shahash((uint8_t*) hash_key, len, hash);
    binary2hex(hash, SHA1_HASH_SIZE, hash_value);

#ifdef DBG_INDEXING
    int i;
    //fprintf(log_fp, "[%s] hash={%s}\n", __func__, hash_value);
    fprintf(log_fp, "[%s] hash={", __func__);
    for (i=0; i<HASH_LEN; i++)
        fprintf(log_fp, "%c", hash_value[i]);
    fprintf(log_fp, "} of len=%d\n", HASH_LEN);
#endif
}

// Initialize the mapping table: 
// - set the bitmap to all zeros, except for the first location to one which
//   indicates the presence of a zeroth bucket
// - set the radix to 1 (XXX: do we need radix??)
// - flag indicates the number of servers if you use static partitioning
//
void giga_init_mapping(struct giga_mapping_t *mapping, int flag)
{
    int i;
#ifdef DBG_INDEXING
    dbg_msg(log_fp, "[%s] initialize giga mapping (flag=%d)", __func__, flag);
#endif

    assert(mapping != NULL);

    memset(mapping->bitmap, 0, MAX_BMAP_LEN);
    
    if (flag == -1) {
        mapping->bitmap[0] = 1;
        mapping->curr_radix = 1;
        mapping->zeroth_server = 0;
        mapping->server_count = 1;
        return;
    }
    
    switch(SPLIT_TYPE) {
        //case SPLIT_TYPE_KEEP_SPLITTING:
        case SPLIT_T_NO_BOUND:
            mapping->bitmap[0] = 1;
            mapping->curr_radix = 1;    
            break;
        //case SPLIT_TYPE_NEVER_SPLIT:
        case SPLIT_T_NO_SPLITTING_EVER:
            assert(flag != -1);
            if (flag < BITS_PER_MAP){
{'fraction_non_alphanumeric': 0.1317115551694179, 'fraction_numerical': 0.007645525629887055, 'mean_word_length': 3.557403008709422, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3994027', 'n_tokens_mistral': 2137, 'n_tokens_neox': 2120, 'n_words': 541}