<commit_before>#pragma once

#include <sdsl/int_vector.hpp>

#include <tudocomp/io.hpp>
#include <tudocomp/util.hpp>
#include <tudocomp/def.hpp>
#include "forward.hpp"

#include <tudocomp/util/divsufsort.hpp>

namespace tdc {

template<class T>
class SuffixArray {

public:
    typedef sdsl::int_vector<> iv_t;

private:
    iv_t m_sa;


public:
    inline iv_t& data() {
        return m_sa;
    }

    inline const iv_t& data() const {
        return m_sa;
    }

    inline iv_t::value_type operator[](iv_t::size_type i) const {
        return m_sa[i];
    }

    inline iv_t::size_type size() const {
        return m_sa.size();
    }

    inline void construct(T& t);
};

}//ns
namespace tdc {

template<class T>
void SuffixArray<T>::construct(T& t) {
	const size_t len = t.size();
	DCHECK_EQ(t[len-1],0);

	//Use divsufsort to construct
	std::vector<saidx_t> sa(len);
	divsufsort(t.text(), sa, len);

	//Bit compress using SDSL
	const size_t w = bits_for(len);
	m_sa = iv_t(len, 0, w);

	for(size_t i = 0; i < len; i++) {
		m_sa[i]  = sa[i];
        DCHECK_LT(m_sa[i], len);
	}
}

}//ns<commit_msg>Build SuffixArray directly on DynamicIntVector<commit_after>#pragma once

#include <tudocomp/io.hpp>
#include <tudocomp/util.hpp>
#include <tudocomp/def.hpp>
#include "forward.hpp"

#include <tudocomp/util/divsufsort.hpp>
#include <tudocomp/ds/IntVector.hpp>

namespace tdc {

template<class T>
class SuffixArray {

public:
    typedef DynamicIntVector iv_t;

private:
    iv_t m_sa;


public:
    inline iv_t& data() {
        return m_sa;
    }

    inline const iv_t& data() const {
        return m_sa;
    }

    inline len_t operator[](size_t i) const {
        return m_sa[i];
    }

    inline size_t size() const {
        return m_sa.size();
    }

    inline void construct(T& t);
};

}//ns
namespace tdc {

template<class T>
void SuffixArray<T>::construct(T& t) {
	const size_t len = t.size();
	DCHECK_EQ(t[len-1],0);

    //Allocate
    const size_t w = bits_for(len);
	m_sa = iv_t(len, 0, w + 1); //divsufsort needs one additional bit for signs

	//Use divsufsort to construct
	divsufsort(t.text(), m_sa, len);

    //Shrink to required width
    m_sa.width(w);
}

}//ns
{'fraction_non_alphanumeric': 0.14122486288848263, 'fraction_numerical': 0.003656307129798903, 'mean_word_length': 3.130188679245283, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4105309', 'n_tokens_mistral': 896, 'n_tokens_neox': 874, 'n_words': 205}