<commit_before>#include "intset.h"

#include "stdlib.h"
#include "stdio.h"
#include "string.h"

static void massert(int condition, const char* message) {
  if(condition == 0){
    printf("Assertion failed: %s\n", message);
    exit(1);
  }
}

IntSet* create_intset(const uint32_t bucket_count) {
  massert(bucket_count > 0, "Bucket count must not be zero");
  IntSet* intset = (IntSet*) calloc(1, sizeof(IntSet));
  intset->bucket_count = bucket_count;
  intset->buckets = (Bucket*) calloc(intset->bucket_count, sizeof(Bucket));
  // intset->filter = bf_create(1); 
  return intset;
}

void add_intset(IntSet* intset, uint64_t int_to_add) {
  Bucket* const bucket = &intset->buckets[int_to_add % intset->bucket_count];
  //bf_set(intset->filter, int_to_add % intset->filter->size, 1);
  if(bucket->slot_count == 0) {
    bucket->slot_count = 1;
    bucket->slots = (uint64_t*) malloc(bucket->slot_count * sizeof(uint64_t));
  } else if(bucket->slot_count == bucket->used_slots){
    // Bucket is full - expand it
    bucket->slot_count *= 2;
    //printf("Expanding bucket %d to size %d\n", int_to_add % intset->bucket_count, bucket->slot_count);
    bucket->slots = (uint32_t*) realloc(bucket->slots, bucket->slot_count * sizeof(uint64_t));
  }
  bucket->slots[bucket->used_slots++] = int_to_add;
}

inline int contains_intset(IntSet* const intset, const uint64_t int_to_find) {
  //if(bf_get(intset->filter, int_to_find % intset->filter->size) == 0){
  //  return 0;
  // }
  Bucket* const bucket = &intset->buckets[int_to_find % intset->bucket_count];
  for(unsigned i=0; i < bucket->used_slots; i++){
    if(bucket->slots[i] == int_to_find){
      return 1;
    }
  }
  return 0;
}

void destroy_intset(IntSet* intset) {
  for(int i=0; i < intset->bucket_count; i++){
    free(intset->buckets[i].slots);
  }
  free(intset->buckets);
  //bf_destroy(intset->filter);
  free(intset);
}<commit_msg>Optimized contains_intset()<commit_after>#include "intset.h"

#include "stdlib.h"
#include "stdio.h"
#include "string.h"
#include "stdint.h"

static void massert(int condition, const char* message) {
  if(condition == 0){
    printf("Assertion failed: %s\n", message);
    exit(1);
  }
}

static int is_power_of_2(const uint32_t n){
  uint32_t c = 1;
  for(unsigned i=0; i<32; i++){
    if(c == n)
      return 1;
    c <<= 1;
  }
  return 0;
}

#define qmod(a, b) ((a) & ((b)-1))

IntSet* create_intset(const uint32_t bucket_count) {
  massert(bucket_count > 0, "Bucket count must not be zero");
  massert(is_power_of_2(bucket_count), "Bucket count must be a power of 2");
  IntSet* intset = (IntSet*) calloc(1, sizeof(IntSet));
  intset->bucket_count = bucket_count;
  intset->buckets = (Bucket*) calloc(intset->bucket_count, sizeof(Bucket));
  // intset->filter = bf_create(1); 
  return intset;
}

void add_intset(IntSet* intset, uint64_t int_to_add) {
  Bucket* const bucket = &intset->buckets[qmod(int_to_add, intset->bucket_count)];
  //bf_set(intset->filter, int_to_add % intset->filter->size, 1);
  if(bucket->slot_count == 0) {
    bucket->slot_count = 1;
    bucket->slots = (uint64_t*) malloc(bucket->slot_count * sizeof(uint64_t));
  } else if(bucket->slot_count == bucket->used_slots){
    // Bucket is full - expand it
    bucket->slot_count *= 2;
    //printf("Expanding bucket %d to size %d\n", int_to_add % intset->bucket_count, bucket->slot_count);
    bucket->slots = (uint32_t*) realloc(bucket->slots, bucket->slot_count * sizeof(uint64_t));
  }
  bucket->slots[bucket->used_slots++] = int_to_add;
}

inline int contains_intset(IntSet* const intset, const uint64_t int_to_find) {
  //if(bf_get(intset->filter, int_to_find % intset->filter->size) == 0){
  //  return 0;
  // }
  Bucket* const bucket = &intset->buckets[qmod(int_to_find, intset->bucket_count)];
  switch(bucket->used_slots) {
    // Lets optimize this sucker...
  case 0:
    return 0;
  case 1:
    return bucket->slots[0] == int_to_find;
  case 2:
    return bucket->slots[0] == int_to_find || bucket->slots[1] == int_to_find;
  case 4:
    return bucket->slots[0] == int_to_find || bucket->slots[1] == int_to_find ||
      bucket->slots[2] == int_to_find || bucket->slots[3] == int_to_find;
  default:
    for(unsigned i=0; i < bucket->used_slots; i++){
      if(bucket->slots[i] == int_to_find){
	return 1;
      }
    }
    return 0;
  }
  massert(0, "contains_intset(): Should not get here");
}

void destroy_intset(IntSet* intset) {
  for(int i=0; i < intset->bucket_count; i++){
    free(intset->buckets[i].slots);
  }
  free(intset->buckets);
  //bf_destroy(intset->filter);
  free(intset);
}
{'fraction_non_alphanumeric': 0.16772082878953107, 'fraction_numerical': 0.01875681570338059, 'mean_word_length': 4.135498320268757, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1822815', 'n_tokens_mistral': 1827, 'n_tokens_neox': 1751, 'n_words': 354}