<commit_before>/*
 * Copyright (c) 2016 Shanghai Jiao Tong University.
 *     All rights reserved.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an "AS
 *  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 *  express or implied.  See the License for the specific language
 *  governing permissions and limitations under the License.
 *
 * For more about this software visit:
 *
 *      http://ipads.se.sjtu.edu.cn/projects/wukong.html
 *
 */

#pragma once

#ifdef USE_GPU

#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/serialization/string.hpp>
#include <boost/serialization/map.hpp>
#include <map>
#include <sstream>
#include "type.hpp"

using namespace std;
using namespace boost::archive;


#define EXT_LIST_MAX_LEN 4
#define EXT_BUCKET_EXTENT_LEN(num_buckets) (num_buckets * 15 / 100 + 1)
#define PREDICATE_NSEGS 4

/**
 * A contiguous space in the indirect-header region
 */
struct ext_bucket_extent_t {
    uint64_t num_ext_buckets;   // capacity
    uint64_t off;       // current offset
    uint64_t start;     // start offset in the indirect header region

    ext_bucket_extent_t() : num_ext_buckets(0), off(0), start(0) {}
    ext_bucket_extent_t(uint64_t nbuckets, uint64_t start_off)
        : num_ext_buckets(nbuckets), off(0), start(start_off) { }

    template <typename Archive>
    void serialize(Archive &ar, const unsigned int version) {
        ar & num_ext_buckets;
        ar & off;
        ar & start;
    }
};

/**
 * Metadata of a segment
 *
 * Each predicate has four segments:
 * 1) Normal-OUT 2) Normal-IN 3) Index-OUT 4) Index-IN.
 * Normal segments are for normal vertices, while index segments for index vertices.
 * And the IN/OUT indicates the direction of triples in the segment.
 */
struct rdf_segment_meta_t {
    uint64_t num_keys;      // #key of the segment
    uint64_t num_buckets;   // allocated main headers (hash space)
    uint64_t bucket_start;  // start offset of main-header region
    ext_bucket_extent_t ext_bucket_list[EXT_LIST_MAX_LEN];
    int ext_list_sz;        // the length of ext_bucket_list
    uint64_t num_edges;     // #edge of the segment
    uint64_t edge_start;    // start offset in the entry region

    rdf_segment_meta_t() : num_keys(0), num_buckets(0), bucket_start(0),
        ext_list_sz(0), edge_start(0), num_edges(0) {
        memset(&ext_bucket_list, 0, sizeof(ext_bucket_list));
    }

    uint64_t get_ext_bucket() {
        for (int i = 0; i < ext_list_sz; ++i) {
            ext_bucket_extent_t &ext = ext_bucket_list[i];
            if (ext.off < ext.num_ext_buckets) {
                return ext.start + ext.off++;
            }
        }

        return 0;
    }

    void add_ext_buckets(const ext_bucket_extent_t &ext) {
        ext_bucket_list[ext_list_sz++] = ext;
    }

    uint64_t get_total_num_buckets() const {
        uint64_t total = num_buckets;
        for (int i = 0; i < ext_list_sz; ++i) {
            const auto &ext = ext_bucket_list[i];
            total += ext.num_ext_buckets;
        }
        return total;
    }

    int num_key_blocks() const {
        extern int global_block_num_buckets;
        static uint64_t n = get_total_num_buckets();
        return ceil(((double) n) / global_block_num_buckets);
    }

    int num_value_blocks() const {
        extern int global_block_num_edges;
        return ceil(((double) num_edges) / global_block_num_edges);
    }

    template <typename Archive>
    void serialize(Archive &ar, const unsigned int version) {
        ar & num_buckets;
        ar & bucket_start;
        ar & ext_bucket_list;
        ar & ext_list_sz;
        ar & num_edges;
        ar & edge_start;<commit_msg>bugfix: cannot use static variable for calculating object's state<commit_after>/*
 * Copyright (c) 2016 Shanghai Jiao Tong University.
 *     All rights reserved.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an "AS
 *  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 *  express or implied.  See the License for the specific language
 *  governing permissions and limitations under the License.
 *
 * For more about this software visit:
 *
 *      http://ipads.se.sjtu.edu.cn/projects/wukong.html
 *
 */

#pragma once

#ifdef USE_GPU

#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/serialization/string.hpp>
#include <boost/serialization/map.hpp>
#include <map>
#include <sstream>
#include "type.hpp"

using namespace std;
using namespace boost::archive;


#define EXT_LIST_MAX_LEN 1
#define EXT_BUCKET_EXTENT_LEN(num_buckets) (num_buckets * 15 / 100 + 1)
#define PREDICATE_NSEGS 4

/**
 * A contiguous space in the indirect-header region
 */
struct ext_bucket_extent_t {
    uint64_t num_ext_buckets;   // capacity
    uint64_t off;       // current offset
    uint64_t start;     // start offset in the indirect header region

    ext_bucket_extent_t() : num_ext_buckets(0), off(0), start(0) {}
    ext_bucket_extent_t(uint64_t nbuckets, uint64_t start_off)
        : num_ext_buckets(nbuckets), off(0), start(start_off) { }

    template <typename Archive>
    void serialize(Archive &ar, const unsigned int version) {
        ar & num_ext_buckets;
        ar & off;
        ar & start;
    }
};

/**
 * Metadata of a segment
 *
 * Each predicate has four segments:
 * 1) Normal-OUT 2) Normal-IN 3) Index-OUT 4) Index-IN.
 * Normal segments are for normal vertices, while index segments for index vertices.
 * And the IN/OUT indicates the direction of triples in the segment.
 */
struct rdf_segment_meta_t {
    uint64_t num_keys;      // #key of the segment
    uint64_t num_buckets;   // allocated main headers (hash space)
    uint64_t bucket_start;  // start offset of main-header region
    ext_bucket_extent_t ext_bucket_list[EXT_LIST_MAX_LEN];
    int ext_list_sz;        // the length of ext_bucket_list
    uint64_t num_edges;     // #edge of the segment
    uint64_t edge_start;    // start offset in the entry region

    rdf_segment_meta_t() : num_keys(0), num_buckets(0), bucket_start(0),
        ext_list_sz(0), edge_start(0), num_edges(0) {
        memset(&ext_bucket_list, 0, sizeof(ext_bucket_list));
    }

    uint64_t get_ext_bucket() {
        for (int i = 0; i < ext_list_sz; ++i) {
            ext_bucket_extent_t &ext = ext_bucket_list[i];
            if (ext.off < ext.num_ext_buckets) {
                return ext.start + ext.off++;
            }
        }

        return 0;
    }

    void add_ext_buckets(const ext_bucket_extent_t &ext) {
        ext_bucket_list[ext_list_sz++] = ext;
    }

    inline uint64_t get_total_num_buckets() const {
        uint64_t total = num_buckets;
        for (int i = 0; i < ext_list_sz; ++i) {
            const auto &ext = ext_bucket_list[i];
            total += ext.num_ext_buckets;
        }
        return total;
    }

    int num_key_blocks() const {
        extern int global_block_num_buckets;
        return ceil(((double) get_total_num_buckets()) / global_block_num_buckets);
    }

    int num_value_blocks() const {
        extern int global_block_num_edges;
        return ceil(((double) num_edges) / global_block_num_edges);
    }

    template <typename Archive>
    void serialize(Archive &ar, const unsigned int version) {
        ar & num_buckets;
        ar & bucket_start;
        ar & ext_bucket_list;
        ar & ext_list_sz;
        ar & num_edges;
        ar & edge_start;
{'fraction_non_alphanumeric': 0.09293129016312407, 'fraction_numerical': 0.015076618882847257, 'mean_word_length': 2.8964853153586905, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1409039', 'n_tokens_mistral': 2741, 'n_tokens_neox': 2596, 'n_words': 747}