<commit_before>/*************************************************************************
 *
 * TIGHTDB CONFIDENTIAL
 * __________________
 *
 *  [2011] - [2012] TightDB Inc
 *  All Rights Reserved.
 *
 * NOTICE:  All information contained herein is, and remains
 * the property of TightDB Incorporated and its suppliers,
 * if any.  The intellectual and technical concepts contained
 * herein are proprietary to TightDB Incorporated
 * and its suppliers and may be covered by U.S. and Foreign Patents,
 * patents in process, and are protected by trade secret or copyright law.
 * Dissemination of this information or reproduction of this material
 * is strictly forbidden unless prior written permission is obtained
 * from TightDB Incorporated.
 *
 **************************************************************************/
#ifndef TIGHTDB_COLUMN_HPP
#define TIGHTDB_COLUMN_HPP

#include "array.hpp"

#ifdef _MSC_VER
#include <win32/stdint.h>
#else
#include <stdint.h> // unint8_t etc
#endif
//#include <climits> // size_t
#include <cstdlib> // size_t
#include <assert.h>

namespace tightdb {


// Pre-definitions
class Column;
class Index;

class ColumnBase {
public:
    virtual ~ColumnBase() {};

    virtual void SetHasRefs() {};

    virtual bool IsIntColumn() const {return false;}
    virtual bool IsStringColumn() const {return false;}
    virtual bool IsBinaryColumn() const {return false;}

    virtual size_t Size() const = 0;

    virtual bool add() = 0;
    virtual void insert(size_t ndx) = 0;
    virtual void Clear() = 0;
    virtual void Delete(size_t ndx) = 0;
    void Resize(size_t ndx) {m_array->Resize(ndx);}

    // Indexing
    virtual bool HasIndex() const = 0;
    //virtual Index& GetIndex() = 0;
    virtual void BuildIndex(Index& index) = 0;
    virtual void ClearIndex() = 0;
    virtual void SetIndexRef(size_t ref) { static_cast<void>(ref); }

    virtual size_t GetRef() const = 0;
    virtual void UpdateParentNdx(int diff) {m_array->UpdateParentNdx(diff);}
    virtual void UpdateFromParent() {m_array->UpdateFromParent();}

#ifdef _DEBUG
    virtual void Verify() const = 0; // Must be upper case to avoid conflict with macro in ObjC
    virtual void ToDot(std::ostream& out, const char* title=NULL) const;
#endif //_DEBUG

template<class C, class A> A* TreeGetArray(size_t start, size_t *first, size_t *last) const;
template<typename T, class C, class F> size_t TreeFind(T value, size_t start, size_t end) const;

protected:
    struct NodeChange {
        size_t ref1;
        size_t ref2;
        enum ChangeType {
            CT_ERROR,
            CT_NONE,
            CT_INSERT_BEFORE,
            CT_INSERT_AFTER,
            CT_SPLIT
        } type;
        NodeChange(ChangeType t, size_t r1=0, size_t r2=0) : ref1(r1), ref2(r2), type(t) {}
        NodeChange(bool success) : ref1(0), ref2(0), type(success ? CT_NONE : CT_ERROR) {}
    };

    // Tree functions
    template<typename T, class C> T TreeGet(size_t ndx) const;
    template<typename T, class C> bool TreeSet(size_t ndx, T value);
    template<typename T, class C> bool TreeInsert(size_t ndx, T value);
    template<typename T, class C> NodeChange DoInsert(size_t ndx, T value);
    template<typename T, class C> void TreeDelete(size_t ndx);
    template<typename T, class C> void TreeFindAll(Array &result, T value, size_t add_offset = 0, size_t start = 0, size_t end = -1) const;

    template<typename T, class C> void TreeVisitLeafs(size_t start, size_t end, size_t caller_offset, bool (*call)(T *arr, size_t start, size_t end, size_t caller_offset, void *state), void *state) const;


    template<typename T, class C, class S> size_t TreeWrite(S& out, size_t& pos) const;

    // Node functions
    bool IsNode() const {return m_array->IsNode();}
    const Array NodeGetOffsets() const;
    const Array NodeGetRefs() const;
    Array NodeGetOffsets();
    Array NodeGetRefs();
    template<class C> bool NodeInsert(size_t ndx, size_t ref);
    template<class C> bool NodeAdd(size_t ref);
public: // FIXME: I had to make NodeAddKey() public. GCC woul not compile it. Alexander?
    bool NodeAddKey(size_t ref);
protected:
    bool NodeUpdateOffsets(size_t ndx);
    template<class C> bool NodeInsertSplit(size_t ndx, size_t newRef);
    size_t GetRefSize(size_t ref) const;

#ifdef _DEBUG
    void ArrayToDot(std::ostream& out, const Array& array) const;
    virtual void LeafToDot(std::ostream& out, const Array& array) const;
#endif //_DEBUG

    // Member variables
    mutable Array* m_array;

    static std::size_t get_size_from_ref(std::size_t ref, Allocator&);
};

class Column : public ColumnBase {
public:
    Column(Allocator& alloc);
    Column(ColumnDef type, Allocator& alloc);
    Column(ColumnDef type=COLUMN_NORMAL, ArrayParent *parent=NULL, size_t pndx=0, Allocator& alloc=GetDefaultAllocator());<commit_msg>Made StringIndex friend of ColumnBase<commit_after>/*************************************************************************
 *
 * TIGHTDB CONFIDENTIAL
 * __________________
 *
 *  [2011] - [2012] TightDB Inc
 *  All Rights Reserved.
 *
 * NOTICE:  All information contained herein is, and remains
 * the property of TightDB Incorporated and its suppliers,
 * if any.  The intellectual and technical concepts contained
 * herein are proprietary to TightDB Incorporated
 * and its suppliers and may be covered by U.S. and Foreign Patents,
 * patents in process, and are protected by trade secret or copyright law.
 * Dissemination of this information or reproduction of this material
 * is strictly forbidden unless prior written permission is obtained
 * from TightDB Incorporated.
 *
 **************************************************************************/
#ifndef TIGHTDB_COLUMN_HPP
#define TIGHTDB_COLUMN_HPP

#include "array.hpp"

#ifdef _MSC_VER
#include <win32/stdint.h>
#else
#include <stdint.h> // unint8_t etc
#endif
//#include <climits> // size_t
#include <cstdlib> // size_t
#include <assert.h>

namespace tightdb {


// Pre-definitions
class Column;
class Index;
class StringIndex;

class ColumnBase {
public:
    virtual ~ColumnBase() {};

    virtual void SetHasRefs() {};

    virtual bool IsIntColumn() const {return false;}
    virtual bool IsStringColumn() const {return false;}
    virtual bool IsBinaryColumn() const {return false;}

    virtual size_t Size() const = 0;

    virtual bool add() = 0;
    virtual void insert(size_t ndx) = 0;
    virtual void Clear() = 0;
    virtual void Delete(size_t ndx) = 0;
    void Resize(size_t ndx) {m_array->Resize(ndx);}

    // Indexing
    virtual bool HasIndex() const = 0;
    //virtual Index& GetIndex() = 0;
    virtual void BuildIndex(Index& index) = 0;
    virtual void ClearIndex() = 0;
    virtual void SetIndexRef(size_t ref) { static_cast<void>(ref); }

    virtual size_t GetRef() const = 0;
    virtual void UpdateParentNdx(int diff) {m_array->UpdateParentNdx(diff);}
    virtual void UpdateFromParent() {m_array->UpdateFromParent();}

#ifdef _DEBUG
    virtual void Verify() const = 0; // Must be upper case to avoid conflict with macro in ObjC
    virtual void ToDot(std::ostream& out, const char* title=NULL) const;
#endif //_DEBUG

template<class C, class A> A* TreeGetArray(size_t start, size_t *first, size_t *last) const;
template<typename T, class C, class F> size_t TreeFind(T value, size_t start, size_t end) const;

protected:
    friend class StringIndex;

    struct NodeChange {
        size_t ref1;
        size_t ref2;
        enum ChangeType {
            CT_ERROR,
            CT_NONE,
            CT_INSERT_BEFORE,
            CT_INSERT_AFTER,
            CT_SPLIT
        } type;
        NodeChange(ChangeType t, size_t r1=0, size_t r2=0) : ref1(r1), ref2(r2), type(t) {}
        NodeChange(bool success) : ref1(0), ref2(0), type(success ? CT_NONE : CT_ERROR) {}
    };

    // Tree functions
    template<typename T, class C> T TreeGet(size_t ndx) const;
    template<typename T, class C> bool TreeSet(size_t ndx, T value);
    template<typename T, class C> bool TreeInsert(size_t ndx, T value);
    template<typename T, class C> NodeChange DoInsert(size_t ndx, T value);
    template<typename T, class C> void TreeDelete(size_t ndx);
    template<typename T, class C> void TreeFindAll(Array &result, T value, size_t add_offset = 0, size_t start = 0, size_t end = -1) const;

    template<typename T, class C> void TreeVisitLeafs(size_t start, size_t end, size_t caller_offset, bool (*call)(T *arr, size_t start, size_t end, size_t caller_offset, void *state), void *state) const;


    template<typename T, class C, class S> size_t TreeWrite(S& out, size_t& pos) const;

    // Node functions
    bool IsNode() const {return m_array->IsNode();}
    const Array NodeGetOffsets() const;
    const Array NodeGetRefs() const;
    Array NodeGetOffsets();
    Array NodeGetRefs();
    template<class C> bool NodeInsert(size_t ndx, size_t ref);
    template<class C> bool NodeAdd(size_t ref);
    bool NodeAddKey(size_t ref);
    bool NodeUpdateOffsets(size_t ndx);
    template<class C> bool NodeInsertSplit(size_t ndx, size_t newRef);
    size_t GetRefSize(size_t ref) const;

#ifdef _DEBUG
    void ArrayToDot(std::ostream& out, const Array& array) const;
    virtual void LeafToDot(std::ostream& out, const Array& array) const;
#endif //_DEBUG

    // Member variables
    mutable Array* m_array;

    static std::size_t get_size_from_ref(std::size_t ref, Allocator&);
};

class Column : public ColumnBase {
public:
    Column(Allocator& alloc);
    Column(ColumnDef type, Allocator& alloc);
    Column(ColumnDef type=COLUMN_NORMAL, ArrayParent *parent=NULL, size_t pndx=0, Allocator& alloc=GetDefaultAllocator());
{'fraction_non_alphanumeric': 0.12821043910521956, 'fraction_numerical': 0.008285004142502071, 'mean_word_length': 3.934593765968319, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1966049', 'n_tokens_mistral': 3090, 'n_tokens_neox': 2975, 'n_words': 999}