<commit_before>#define _GNU_SOURCE
#include <unordered_map>
#include <iostream>
#include <cstdlib>
#include <fstream>
#include <cstring>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#if _MSC_VER
#include <Windows.h>
#else
#include <sys/time.h>
#endif
class IPLookup {
public:
    IPLookup() {
        std::memset(tbl24_, 0, sizeof(uint16_t) * TBL24_SIZE);
        std::memset(tblLong_, 0, sizeof(uint16_t) * TBL24_SIZE);
        currentTBLLong_ = 0;
    }

    void AddRoute(uint32_t address, uint16_t len, uint16_t nextHop) {
        prefixTable_[len][address] = nextHop;
    }

    void DeleteRoute(uint32_t address, uint16_t len) {
        prefixTable_[len].erase(address);
    }

    void ConstructFIB() {
        for (int i = 0; i <= 24; i++) {
            for (auto const& kv : prefixTable_[i]) {
                uint32_t addr = kv.first;
                uint16_t dest = kv.second;
                uint32_t start = addr >> 8;
                uint32_t end = start + (1u << (24 - i));
                for (uint32_t pfx = start; pfx < end; pfx++) {
                    tbl24_[pfx] = dest;
                }
            }
        }
        for (int i = 25; i <= 32; i++) {
            for (auto kv : prefixTable_[i]) {
                uint32_t addr = kv.first;
                uint16_t dest = kv.second;
                uint16_t tblDest = tbl24_[addr >> 8];
                if ((tblDest & OVERFLOW_MASK) == 0) {
                    uint32_t start = currentTBLLong_ + (addr & 0xff);
                    uint32_t end = start + (1u << (32 - i));
                    for (uint32_t j = currentTBLLong_;
                                  j < currentTBLLong_ + 256;
                                  j++) {
                        if (j < start || j >= end) {
                            tblLong_[j] = tblDest;
                        } else {
                            tblLong_[j] = dest;
                        }
                    }
                    tbl24_[addr >> 8] =
                        (uint16_t)((currentTBLLong_ >> 8) |
                                   OVERFLOW_MASK);
                    currentTBLLong_ += 256;
                } else {
                    uint32_t start = ((uint32_t)(tblDest & 
                                (~OVERFLOW_MASK)) << 8) +
                                     (addr & 0xff);
                    uint32_t end = start + (1u << (32 - i));
                    for (uint32_t j = start; j < end; j++) {
                        tblLong_[j] = dest;
                    }
                }
            }
        }
    }

    uint16_t RouteLookup(uint32_t ip) {
        uint16_t tblDest = tbl24_[ip >> 8];
        if ((tblDest & OVERFLOW_MASK) > 0) {
            uint32_t index = (((uint32_t)(tblDest & (~OVERFLOW_MASK)) << 8)
                               + (ip & 0xff));
            return tblLong_[index];
        } else {
            return tblDest;
        }
    }
private:
    static const size_t TBL24_SIZE = (1u << 24) + 1;
    static const uint16_t OVERFLOW_MASK = 0x8000;
    std::unordered_map<uint32_t, uint16_t> prefixTable_[33];
    uint16_t tbl24_[TBL24_SIZE];
    uint16_t tblLong_[TBL24_SIZE];
    uint32_t currentTBLLong_;
};

static uint64_t get_sec(void) {
#if _MSC_VER
	return timeGetTime() / 1000;
#else
	struct timeval time;
	gettimeofday(&time, NULL);
	//long millis = (time.tv_sec * 1000) + (time.tv_usec / 1000);
	return (uint64_t)time.tv_sec;
#endif
}

static uint32_t rand_fast() {
    static uint64_t seed = 0;
    seed = seed * 1103515245 + 12345;
    return (uint32_t)(seed >> 32);
}

void Benchmark(
      IPLookup* lookup,
      uint64_t warm,
      uint32_t batch,
      uint64_t batches) {
    lookup->ConstructFIB();
    uint64_t lastSec = get_sec();
    while (get_sec() - lastSec < warm) {
        for (uint32_t i = 0; i < batch; i++) {
            lookup->RouteLookup(rand_fast());
        }
    }
    uint32_t batchComputed = 0;
    lastSec = get_sec();
    uint64_t lastLookups = 0;
    uint32_t lastIP = 0;
    while (batchComputed < batches) {
        for (int i = 0; i < batch; i++) {
            lastIP = rand_fast();
            lookup->RouteLookup(lastIP);
            lastLookups++;
        }
        if (lastSec != get_sec()) {
            uint64_t currSec = get_sec();
            batchComputed++;
            std::cout << (currSec - lastSec) << " "
                      << batch << " "
                      << (lastLookups / (currSec - lastSec)) << 
                      << " " << lastIP << std::endl;
            lastSec = get_sec();
            lastLookups = 0;
        }
    }
}

int main(int argc, char* argv[]) {
    IPLookup *lookup = new IPLookup();
    if (argc < 2) {
        std::cout << "Usage: lookup rib" << std::endl;<commit_msg>Update native<commit_after>#define _GNU_SOURCE
#include <unordered_map>
#include <iostream>
#include <cstdlib>
#include <fstream>
#include <cstring>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#if _MSC_VER
#include <Windows.h>
#else
#include <sys/time.h>
#endif
class IPLookup {
public:
    IPLookup() {
        std::memset(tbl24_, 0, sizeof(uint16_t) * TBL24_SIZE);
        std::memset(tblLong_, 0, sizeof(uint16_t) * TBL24_SIZE);
        currentTBLLong_ = 0;
    }

    void AddRoute(uint32_t address, uint16_t len, uint16_t nextHop) {
        prefixTable_[len][address] = nextHop;
    }

    void DeleteRoute(uint32_t address, uint16_t len) {
        prefixTable_[len].erase(address);
    }

    void ConstructFIB() {
        uint32_t entries = 0;
        uint32_t long_entries = 0;
        for (int i = 0; i <= 24; i++) {
            entries++;
            for (auto const& kv : prefixTable_[i]) {
                uint32_t addr = kv.first;
                uint16_t dest = kv.second;
                uint32_t start = addr >> 8;
                uint32_t end = start + (1u << (24 - i));
                for (uint32_t pfx = start; pfx < end; pfx++) {
                    tbl24_[pfx] = dest;
                }
            }
        }
        for (int i = 25; i <= 32; i++) {
            entries++;
            long_entries++;
            for (auto kv : prefixTable_[i]) {
                uint32_t addr = kv.first;
                uint16_t dest = kv.second;
                uint16_t tblDest = tbl24_[addr >> 8];
                if ((tblDest & OVERFLOW_MASK) == 0) {
                    uint32_t start = currentTBLLong_ + (addr & 0xff);
                    uint32_t end = start + (1u << (32 - i));
                    for (uint32_t j = currentTBLLong_;
                                  j < currentTBLLong_ + 256;
                                  j++) {
                        if (j < start || j >= end) {
                            tblLong_[j] = tblDest;
                        } else {
                            tblLong_[j] = dest;
                        }
                    }
                    tbl24_[addr >> 8] =
                        (uint16_t)((currentTBLLong_ >> 8) |
                                   OVERFLOW_MASK);
                    currentTBLLong_ += 256;
                } else {
                    uint32_t start = ((uint32_t)(tblDest & 
                                (~OVERFLOW_MASK)) << 8) +
                                     (addr & 0xff);
                    uint32_t end = start + (1u << (32 - i));
                    for (uint32_t j = start; j < end; j++) {
                        tblLong_[j] = dest;
                    }
                }
            }
        }
        std::cout << "Processed " << entries  << " entries and  " << long_entries << " long" << std::endl;
    }

    uint16_t RouteLookup(uint32_t ip) {
        uint16_t tblDest = tbl24_[ip >> 8];
        if ((tblDest & OVERFLOW_MASK) > 0) {
            uint32_t index = (((uint32_t)(tblDest & (~OVERFLOW_MASK)) << 8)
                               + (ip & 0xff));
            return tblLong_[index];
        } else {
            return tblDest;
        }
    }
private:
    static const size_t TBL24_SIZE = (1u << 24) + 1;
    static const uint16_t OVERFLOW_MASK = 0x8000;
    std::unordered_map<uint32_t, uint16_t> prefixTable_[33];
    uint16_t tbl24_[TBL24_SIZE];
    uint16_t tblLong_[TBL24_SIZE];
    uint32_t currentTBLLong_;
};

static uint64_t get_sec(void) {
#if _MSC_VER
	return timeGetTime() / 1000;
#else
	struct timeval time;
	gettimeofday(&time, NULL);
	//long millis = (time.tv_sec * 1000) + (time.tv_usec / 1000);
	return (uint64_t)time.tv_sec;
#endif
}

static uint32_t rand_fast() {
    static uint64_t seed = 0;
    seed = seed * 1103515245 + 12345;
    return (uint32_t)(seed >> 32);
}

void Benchmark(
      IPLookup* lookup,
      uint64_t warm,
      uint32_t batch,
      uint64_t batches) {
    lookup->ConstructFIB();
    uint64_t lastSec = get_sec();
    while (get_sec() - lastSec < warm) {
        for (uint32_t i = 0; i < batch; i++) {
            lookup->RouteLookup(rand_fast());
        }
    }
    uint32_t batchComputed = 0;
    lastSec = get_sec();
    uint64_t lastLookups = 0;
    uint32_t lastIP = 0;
    uint64_t dests = 0;
    while (batchComputed < batches) {
        for (int i = 0; i < batch; i++) {
            lastIP = rand_fast();
            dests = lookup->RouteLookup(lastIP);
            lastLookups++;
        }
        if (lastSec != get_sec()) {
            uint64_t currSec = get_sec();
            batchComputed++;
            std::cout << (currSec - lastSec) << " "
                      << batch << " "
                      << (lastLookups / (currSec - lastSec)) 
                      << " " << lastIP
                      << " " << dests << std::endl;
            lastSec = get_sec();
            lastLookups = 0;
        }
    }
}

int main(int argc, char* argv[]) {
    IPLookup *lookup = new IPLookup();
    if (argc < 2) {
        std::cout << "Usage: lookup rib" << std::endl;
{'fraction_non_alphanumeric': 0.12518763134193936, 'fraction_numerical': 0.04152907034924447, 'mean_word_length': 1.4941352632892437, 'pattern_counts': {'":': 0, '<': 137, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5611311', 'n_tokens_mistral': 3614, 'n_tokens_neox': 3183, 'n_words': 734}