<commit_before>#ifndef _IZENELIB_AM_SUCCINCT_HP_ARRAY_HPP
#define _IZENELIB_AM_SUCCINCT_HP_ARRAY_HPP

#include <am/succinct/constants.hpp>

#include <vector>
#include <iostream>
#include <cstdlib>
#include <cstring>
#include <sys/mman.h>

NS_IZENELIB_AM_BEGIN

namespace succinct
{

#define HUGE_PROTECTION (PROT_READ | PROT_WRITE)
#define HUGE_FLAGS (MAP_HUGETLB | MAP_ANONYMOUS | MAP_PRIVATE)

template <class T>
class HPArray
{
public:
    typedef T value_type;
    typedef value_type& reference;
    typedef const value_type& const_reference;
    typedef value_type* iterator;
    typedef const value_type* const_iterator;
    typedef ptrdiff_t difference_type;
    typedef size_t size_type;

    HPArray() : use_hp_(true), size_(), capacity_(), data_() {}

    HPArray(size_type size)
    {
        resize(size);
    }

    HPArray(const std::vector<value_type>& vec)
    {
        assign(vec);
    }

    ~HPArray()
    {
        clear();
    }

    size_type capacity()
    {
        return capacity_;
    }

    size_type size()
    {
        return size_;
    }

    value_type* data()
    {
        return data_;
    }
    const value_type* data() const
    {
        return data_;
    }

    reference operator[](size_type pos)
    {
        return data_[pos];
    }
    const_reference operator[](size_type pos) const
    {
        return data_[pos];
    }

    void clear()
    {
        if (data_ > 0)
        {
            size_ = 0;
            capacity_ = 0;
            if (use_hp_)
            {
                munmap(data_, sizeof(value_type) * capacity_);
            }
            else
            {
                free(data_);
            }
            data_ = 0;
        }
    }

    void resize(size_type size)
    {
        if (size > capacity_)
        {
            bool new_use_hp = true;
            size_type new_capacity = std::max(size, capacity_ * 2);
            value_type* new_data = (value_type*)mmap(NULL, sizeof(value_type) * new_capacity, HUGE_PROTECTION, HUGE_FLAGS, 0, 0);
            if (new_data == MAP_FAILED)
            {
                std::cout << "mmap was not successful" << std::endl;<commit_msg>tiny fix<commit_after>#ifndef _IZENELIB_AM_SUCCINCT_HP_ARRAY_HPP
#define _IZENELIB_AM_SUCCINCT_HP_ARRAY_HPP

#include <am/succinct/constants.hpp>

#include <vector>
#include <iostream>
#include <cstdlib>
#include <cstring>
#include <sys/mman.h>

NS_IZENELIB_AM_BEGIN

namespace succinct
{

#define HUGE_PROTECTION (PROT_READ | PROT_WRITE)
#define HUGE_FLAGS (MAP_HUGETLB | MAP_ANONYMOUS | MAP_PRIVATE)

template <class T>
class HPArray
{
public:
    typedef T value_type;
    typedef value_type& reference;
    typedef const value_type& const_reference;
    typedef value_type* iterator;
    typedef const value_type* const_iterator;
    typedef ptrdiff_t difference_type;
    typedef size_t size_type;

    HPArray() : use_hp_(true), size_(), capacity_(), data_() {}

    HPArray(size_type size)
    {
        resize(size);
    }

    HPArray(const std::vector<value_type>& vec)
    {
        assign(vec);
    }

    ~HPArray()
    {
        clear();
    }

    size_type capacity()
    {
        return capacity_;
    }

    size_type size()
    {
        return size_;
    }

    value_type* data()
    {
        return data_;
    }
    const value_type* data() const
    {
        return data_;
    }

    reference operator[](size_type pos)
    {
        return data_[pos];
    }
    const_reference operator[](size_type pos) const
    {
        return data_[pos];
    }

    void clear()
    {
        if (data_)
        {
            if (use_hp_)
            {
                munmap(data_, sizeof(value_type) * capacity_);
            }
            else
            {
                free(data_);
            }
            size_ = 0;
            capacity_ = 0;
            data_ = 0;
        }
    }

    void resize(size_type size)
    {
        if (size > capacity_)
        {
            bool new_use_hp = true;
            size_type new_capacity = std::max(size, capacity_ * 2);
            value_type* new_data = (value_type*)mmap(NULL, sizeof(value_type) * new_capacity, HUGE_PROTECTION, HUGE_FLAGS, 0, 0);
            if (new_data == MAP_FAILED)
            {
                std::cout << "mmap was not successful" << std::endl;
{'fraction_non_alphanumeric': 0.10034924330616997, 'fraction_numerical': 0.0030267753201396975, 'mean_word_length': 2.0338983050847457, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6623997', 'n_tokens_mistral': 1468, 'n_tokens_neox': 1393, 'n_words': 299}