<commit_before>/*
 * Copyright (c) 2016 Shanghai Jiao Tong University.
 *     All rights reserved.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an "AS
 *  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 *  express or implied.  See the License for the specific language
 *  governing permissions and limitations under the License.
 *
 * For more about this software visit:
 *
 *      http://ipads.se.sjtu.edu.cn/projects/wukong.html
 *
 */

#pragma once

#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/serialization/string.hpp>
#include <boost/serialization/map.hpp>
#include <map>
#include <sstream>

#include "type.hpp"
#include "vertex.hpp"

using namespace std;
using namespace boost::archive;


#define EXT_LIST_MAX_LEN 1
#define EXT_BUCKET_EXTENT_LEN(num_buckets) (num_buckets * 15 / 100 + 1)
#define PREDICATE_NSEGS 2
#ifdef VERSATILE
#define INDEX_NSEGS 4   // index(2) + vid's all preds(2)
#else // VERSATILE
#define INDEX_NSEGS 2   // index(2)
#endif // VERSATILE
/**
 * A contiguous space in the indirect-header region
 */
struct ext_bucket_extent_t {
    uint64_t num_ext_buckets;   // capacity
    uint64_t off;       // current offset
    uint64_t start;     // start offset in the indirect header region

    ext_bucket_extent_t() : num_ext_buckets(0), off(0), start(0) {}
    ext_bucket_extent_t(uint64_t nbuckets, uint64_t start_off)
        : num_ext_buckets(nbuckets), off(0), start(start_off) { }

    template <typename Archive>
    void serialize(Archive &ar, const unsigned int version) {
        ar & num_ext_buckets;
        ar & off;
        ar & start;
    }
};

/**
 * Metadata of a segment
 *
 * Each predicate has four segments:
 * 1) Normal-OUT 2) Normal-IN 3) Index-OUT 4) Index-IN.
 * Normal segments are for normal vertices, while index segments for index vertices.
 * And the IN/OUT indicates the direction of triples in the segment.
 */
struct rdf_seg_meta_t {
    uint64_t num_keys = 0;      // #keys of the segment
    uint64_t num_buckets = 0;   // allocated main headers (hash space)
    uint64_t bucket_start = 0;  // start offset of main-header region of gstore
    ext_bucket_extent_t ext_bucket_list[EXT_LIST_MAX_LEN];
    int ext_list_sz = 0;        // the length of ext_bucket_list
    uint64_t num_edges = 0;     // #edges of the segment
    uint64_t edge_start = 0;    // start offset in the entry region of gstore

    int num_key_blks = 0;       // #key-blocks needed in gcache
    int num_value_blks = 0;     // #value-blocks needed in gcache

    rdf_seg_meta_t() {
        memset(&ext_bucket_list, 0, sizeof(ext_bucket_list));
    }

    uint64_t get_ext_bucket() {
        for (int i = 0; i < ext_list_sz; ++i) {
            ext_bucket_extent_t &ext = ext_bucket_list[i];
            if (ext.off < ext.num_ext_buckets) {
                return ext.start + ext.off++;
            }
        }
        return 0;
    }

    void add_ext_buckets(const ext_bucket_extent_t &ext) {
        ASSERT_MSG(ext_list_sz < EXT_LIST_MAX_LEN, "ext_bucket_list overflow!");
        ext_bucket_list[ext_list_sz++] = ext;<commit_msg>[*][src] fix 1: wukongGPU doesn't support ASSERT_MSG<commit_after>/*
 * Copyright (c) 2016 Shanghai Jiao Tong University.
 *     All rights reserved.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing,
 *  software distributed under the License is distributed on an "AS
 *  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 *  express or implied.  See the License for the specific language
 *  governing permissions and limitations under the License.
 *
 * For more about this software visit:
 *
 *      http://ipads.se.sjtu.edu.cn/projects/wukong.html
 *
 */

#pragma once

#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/serialization/string.hpp>
#include <boost/serialization/map.hpp>
#include <map>
#include <sstream>

#include "type.hpp"
#include "vertex.hpp"

using namespace std;
using namespace boost::archive;


#define EXT_LIST_MAX_LEN 1
#define EXT_BUCKET_EXTENT_LEN(num_buckets) (num_buckets * 15 / 100 + 1)
#define PREDICATE_NSEGS 2
#ifdef VERSATILE
#define INDEX_NSEGS 4   // index(2) + vid's all preds(2)
#else // VERSATILE
#define INDEX_NSEGS 2   // index(2)
#endif // VERSATILE
/**
 * A contiguous space in the indirect-header region
 */
struct ext_bucket_extent_t {
    uint64_t num_ext_buckets;   // capacity
    uint64_t off;       // current offset
    uint64_t start;     // start offset in the indirect header region

    ext_bucket_extent_t() : num_ext_buckets(0), off(0), start(0) {}
    ext_bucket_extent_t(uint64_t nbuckets, uint64_t start_off)
        : num_ext_buckets(nbuckets), off(0), start(start_off) { }

    template <typename Archive>
    void serialize(Archive &ar, const unsigned int version) {
        ar & num_ext_buckets;
        ar & off;
        ar & start;
    }
};

/**
 * Metadata of a segment
 *
 * Each predicate has four segments:
 * 1) Normal-OUT 2) Normal-IN 3) Index-OUT 4) Index-IN.
 * Normal segments are for normal vertices, while index segments for index vertices.
 * And the IN/OUT indicates the direction of triples in the segment.
 */
struct rdf_seg_meta_t {
    uint64_t num_keys = 0;      // #keys of the segment
    uint64_t num_buckets = 0;   // allocated main headers (hash space)
    uint64_t bucket_start = 0;  // start offset of main-header region of gstore
    ext_bucket_extent_t ext_bucket_list[EXT_LIST_MAX_LEN];
    int ext_list_sz = 0;        // the length of ext_bucket_list
    uint64_t num_edges = 0;     // #edges of the segment
    uint64_t edge_start = 0;    // start offset in the entry region of gstore

    int num_key_blks = 0;       // #key-blocks needed in gcache
    int num_value_blks = 0;     // #value-blocks needed in gcache

    rdf_seg_meta_t() {
        memset(&ext_bucket_list, 0, sizeof(ext_bucket_list));
    }

    uint64_t get_ext_bucket() {
        for (int i = 0; i < ext_list_sz; ++i) {
            ext_bucket_extent_t &ext = ext_bucket_list[i];
            if (ext.off < ext.num_ext_buckets) {
                return ext.start + ext.off++;
            }
        }
        return 0;
    }

    void add_ext_buckets(const ext_bucket_extent_t &ext) {
        assert(ext_list_sz < EXT_LIST_MAX_LEN);
        ext_bucket_list[ext_list_sz++] = ext;
{'fraction_non_alphanumeric': 0.09624819624819625, 'fraction_numerical': 0.018037518037518036, 'mean_word_length': 3.055588063194851, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3698735', 'n_tokens_mistral': 2400, 'n_tokens_neox': 2271, 'n_words': 689}