<commit_before>/**
 * Copyright (C) 2005-2007 <NAME> (<EMAIL>).
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2 of the License, or 
 * (at your option) any later version.
 *
 * See file COPYING.GPL2 and COPYING.GPL3 for License information.
 */

#include <stdexcept>
#include <errno.h>
#include <string.h>
#include <cppunit/extensions/HelperMacros.h>
#include <ham/hamsterdb.h>
#include "../src/db.h"
#include "../src/blob.h"
#include "../src/backend.h"
#include "../src/btree.h"
#include "memtracker.h"

class DupeTest : public CppUnit::TestFixture
{
    CPPUNIT_TEST_SUITE(DupeTest);
    CPPUNIT_TEST      (invalidFlagsTest);
    CPPUNIT_TEST      (insertDuplicatesTest);
    CPPUNIT_TEST      (overwriteDuplicatesTest);
    CPPUNIT_TEST      (overwriteVariousDuplicatesTest);
    CPPUNIT_TEST      (insertMoveForwardTest);
    CPPUNIT_TEST      (insertMoveBackwardTest);
    /*
    CPPUNIT_TEST      (insertEraseTest);
    CPPUNIT_TEST      (insertTest);
    CPPUNIT_TEST      (insertSkipDuplicatesTest);
    CPPUNIT_TEST      (insertOnlyDuplicatesTest);
    CPPUNIT_TEST      (coupleUncoupleTest);
    CPPUNIT_TEST      (reopenTest);
    CPPUNIT_TEST      (moveToLastDuplicateTest);
    CPPUNIT_TEST      (moveToPreviousDuplicateTest);
    CPPUNIT_TEST      (replaceTest);
    CPPUNIT_TEST      (overwriteTest);
    */
    CPPUNIT_TEST_SUITE_END();

protected:
    ham_u32_t m_flags;
    ham_db_t *m_db;
    std::vector<std::string> m_data;

public:
    DupeTest(ham_u32_t flags=0)
    :   m_flags(flags)
    {
    }

    void setUp()
    { 
#if WIN32
        (void)DeleteFileA((LPCSTR)".test");
#else
        if (unlink(".test")) {
            if (errno!=2)
                printf("failed to unlink .test: %s\n", strerror(errno));
        }
#endif
        CPPUNIT_ASSERT_EQUAL(0, ham_new(&m_db));
        CPPUNIT_ASSERT_EQUAL(0, ham_create(m_db, ".test", 
                    m_flags|HAM_ENABLE_DUPLICATES, 0664));

        m_data.resize(0);
    }

    void tearDown()
    {
        CPPUNIT_ASSERT_EQUAL(0, ham_close(m_db));
        CPPUNIT_ASSERT_EQUAL(0, ham_delete(m_db));
    }
    
    void insertDuplicatesTest(void)
    {
        ham_key_t key;
        ham_record_t rec, rec2;
        char data[16];

        ::memset(&rec2, 0, sizeof(rec2));

        for (int i=0; i<10; i++) {
            ::memset(&key, 0, sizeof(key));
            ::memset(&rec, 0, sizeof(rec));
            rec.data=data;
            rec.size=sizeof(data);
            ::memset(&data, i+0x15, sizeof(data));
            CPPUNIT_ASSERT_EQUAL(0, 
                    ham_insert(m_db, 0, &key, &rec, HAM_DUPLICATE));
        }

        ::memset(&data, 0x15, sizeof(data));
        CPPUNIT_ASSERT_EQUAL(0, ham_find(m_db, 0, &key, &rec2, 0));
        CPPUNIT_ASSERT_EQUAL((ham_size_t)sizeof(data), rec2.size);
        CPPUNIT_ASSERT_EQUAL(0, ::memcmp(data, rec2.data, sizeof(data)));
    }

    void overwriteDuplicatesTest(void)
    {
        ham_key_t key;
        ham_record_t rec, rec2;
        char data[16];

        ::memset(&rec2, 0, sizeof(rec2));

        for (int i=0; i<5; i++) {
            ::memset(&key, 0, sizeof(key));
            ::memset(&rec, 0, sizeof(rec));
            rec.data=data;
            rec.size=sizeof(data);
            ::memset(&data, i+0x15, sizeof(data));
            CPPUNIT_ASSERT_EQUAL(0, 
                    ham_insert(m_db, 0, &key, &rec, HAM_DUPLICATE));
        }

        ::memset(&key, 0, sizeof(key));
        ::memset(&rec, 0, sizeof(rec));
        rec.data=data;
        rec.size=sizeof(data);
        ::memset(&data, 0x99, sizeof(data));
        CPPUNIT_ASSERT_EQUAL(0, 
                ham_insert(m_db, 0, &key, &rec, HAM_OVERWRITE));

        CPPUNIT_ASSERT_EQUAL(0, ham_find(m_db, 0, &key, &rec2, 0));
        CPPUNIT_ASSERT_EQUAL((ham_size_t)sizeof(data), rec2.size);
        CPPUNIT_ASSERT_EQUAL(0, ::memcmp(data, rec2.data, sizeof(data)));
    }

    void overwriteVariousDuplicatesTest(void)
    {
        ham_key_t key;
        ham_record_t rec, rec2;
#define MAX 10
        unsigned sizes[MAX]={0, 1, 2, 3, 4, 5, 936, 5, 100, 50};
        char *data;
        ham_cursor_t *cursor;

        ::memset(&rec2, 0, sizeof(rec2));

        for (unsigned i=0; i<MAX; i++) {
            data=0;
            ::memset(&key, 0, sizeof(key));
            ::memset(&rec, 0, sizeof(rec));
            if (sizes[i]) {
                data=(char *)malloc(sizes[i]);
                ::memset(data, i+0x15, sizes[i]);
            }
            rec.data=sizes[i] ? data : 0;
            rec.size=sizes[i];
            CPPUNIT_ASSERT_EQUAL(0, 
                    ham_insert(m_db, 0, &key, &rec, HAM_DUPLICATE));
            CPPUNIT_ASSERT_EQUAL(sizes[i], rec.size);
            if (sizes[i]) {
                CPPUNIT_ASSERT_EQUAL(0, ::memcmp(data, rec.data, sizes[i]));
                free(data);
            }
        }

        CPPUNIT_ASSERT_EQUAL(0, ham_find(m_db, 0, &key, &rec2, 0));
        CPPUNIT_ASSERT_EQUAL((ham_size_t)0, rec2.size);

        CPPUNIT_ASSERT_EQUAL(0, ham_cursor_create(m_db, 0, 0, &cursor));

        for (unsigned i=0; i<MAX; i++) {
            ::memset(&key, 0, sizeof(key));
            ::memset(&rec, 0, sizeof(rec));
            if (sizes[i]) {
                data=(char *)malloc(sizes[i]);
                ::memset(data, i+0x15, sizes[i]);
            }
            rec.data=sizes[i] ? data : 0;
            rec.size=sizes[i];
            CPPUNIT_ASSERT_EQUAL(0, 
                    ham_cursor_move(cursor, &key, &rec, HAM_CURSOR_NEXT));
            CPPUNIT_ASSERT_EQUAL(sizes[i], rec.size);
            if (sizes[i]) {
                CPPUNIT_ASSERT_EQUAL(0, ::memcmp(data, rec.data, sizes[i]));
                free(data);
            }
        }

        CPPUNIT_ASSERT_EQUAL(0, ham_cursor_close(cursor));
        CPPUNIT_ASSERT_EQUAL(0, ham_cursor_create(m_db, 0, 0, &cursor));

        for (unsigned i=MAX-1; i>=0; i--) {
            ::memset(&key, 0, sizeof(key));
            ::memset(&rec, 0, sizeof(rec));
            if (sizes[i]) {
                data=(char *)malloc(sizes[i]);
                ::memset(data, i+0x15, sizes[i]);
            }
            rec.data=sizes[i] ? data : 0;
            rec.size=sizes[i];
            CPPUNIT_ASSERT_EQUAL(0, 
                    ham_cursor_move(cursor, &key, &rec, HAM_CURSOR_PREVIOUS));
            CPPUNIT_ASSERT_EQUAL(sizes[i], rec.size);
            if (sizes[i]) {
                CPPUNIT_ASSERT_EQUAL(0, ::memcmp(data, rec.data, sizes[i]));
                free(data);<commit_msg>fixed segfault in a for-loop<commit_after>/**
 * Copyright (C) 2005-2007 <NAME> (<EMAIL>).
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2 of the License, or 
 * (at your option) any later version.
 *
 * See file COPYING.GPL2 and COPYING.GPL3 for License information.
 */

#include <stdexcept>
#include <errno.h>
#include <string.h>
#include <cppunit/extensions/HelperMacros.h>
#include <ham/hamsterdb.h>
#include "../src/db.h"
#include "../src/blob.h"
#include "../src/backend.h"
#include "../src/btree.h"
#include "memtracker.h"

class DupeTest : public CppUnit::TestFixture
{
    CPPUNIT_TEST_SUITE(DupeTest);
    CPPUNIT_TEST      (invalidFlagsTest);
    CPPUNIT_TEST      (insertDuplicatesTest);
    CPPUNIT_TEST      (overwriteDuplicatesTest);
    CPPUNIT_TEST      (overwriteVariousDuplicatesTest);
    CPPUNIT_TEST      (insertMoveForwardTest);
    CPPUNIT_TEST      (insertMoveBackwardTest);
    /*
    CPPUNIT_TEST      (insertEraseTest);
    CPPUNIT_TEST      (insertTest);
    CPPUNIT_TEST      (insertSkipDuplicatesTest);
    CPPUNIT_TEST      (insertOnlyDuplicatesTest);
    CPPUNIT_TEST      (coupleUncoupleTest);
    CPPUNIT_TEST      (reopenTest);
    CPPUNIT_TEST      (moveToLastDuplicateTest);
    CPPUNIT_TEST      (moveToPreviousDuplicateTest);
    CPPUNIT_TEST      (replaceTest);
    CPPUNIT_TEST      (overwriteTest);
    */
    CPPUNIT_TEST_SUITE_END();

protected:
    ham_u32_t m_flags;
    ham_db_t *m_db;
    std::vector<std::string> m_data;

public:
    DupeTest(ham_u32_t flags=0)
    :   m_flags(flags)
    {
    }

    void setUp()
    { 
#if WIN32
        (void)DeleteFileA((LPCSTR)".test");
#else
        if (unlink(".test")) {
            if (errno!=2)
                printf("failed to unlink .test: %s\n", strerror(errno));
        }
#endif
        CPPUNIT_ASSERT_EQUAL(0, ham_new(&m_db));
        CPPUNIT_ASSERT_EQUAL(0, ham_create(m_db, ".test", 
                    m_flags|HAM_ENABLE_DUPLICATES, 0664));

        m_data.resize(0);
    }

    void tearDown()
    {
        CPPUNIT_ASSERT_EQUAL(0, ham_close(m_db));
        CPPUNIT_ASSERT_EQUAL(0, ham_delete(m_db));
    }
    
    void insertDuplicatesTest(void)
    {
        ham_key_t key;
        ham_record_t rec, rec2;
        char data[16];

        ::memset(&rec2, 0, sizeof(rec2));

        for (int i=0; i<10; i++) {
            ::memset(&key, 0, sizeof(key));
            ::memset(&rec, 0, sizeof(rec));
            rec.data=data;
            rec.size=sizeof(data);
            ::memset(&data, i+0x15, sizeof(data));
            CPPUNIT_ASSERT_EQUAL(0, 
                    ham_insert(m_db, 0, &key, &rec, HAM_DUPLICATE));
        }

        ::memset(&data, 0x15, sizeof(data));
        CPPUNIT_ASSERT_EQUAL(0, ham_find(m_db, 0, &key, &rec2, 0));
        CPPUNIT_ASSERT_EQUAL((ham_size_t)sizeof(data), rec2.size);
        CPPUNIT_ASSERT_EQUAL(0, ::memcmp(data, rec2.data, sizeof(data)));
    }

    void overwriteDuplicatesTest(void)
    {
        ham_key_t key;
        ham_record_t rec, rec2;
        char data[16];

        ::memset(&rec2, 0, sizeof(rec2));

        for (int i=0; i<5; i++) {
            ::memset(&key, 0, sizeof(key));
            ::memset(&rec, 0, sizeof(rec));
            rec.data=data;
            rec.size=sizeof(data);
            ::memset(&data, i+0x15, sizeof(data));
            CPPUNIT_ASSERT_EQUAL(0, 
                    ham_insert(m_db, 0, &key, &rec, HAM_DUPLICATE));
        }

        ::memset(&key, 0, sizeof(key));
        ::memset(&rec, 0, sizeof(rec));
        rec.data=data;
        rec.size=sizeof(data);
        ::memset(&data, 0x99, sizeof(data));
        CPPUNIT_ASSERT_EQUAL(0, 
                ham_insert(m_db, 0, &key, &rec, HAM_OVERWRITE));

        CPPUNIT_ASSERT_EQUAL(0, ham_find(m_db, 0, &key, &rec2, 0));
        CPPUNIT_ASSERT_EQUAL((ham_size_t)sizeof(data), rec2.size);
        CPPUNIT_ASSERT_EQUAL(0, ::memcmp(data, rec2.data, sizeof(data)));
    }

    void overwriteVariousDuplicatesTest(void)
    {
        ham_key_t key;
        ham_record_t rec, rec2;
#define MAX 10
        unsigned sizes[MAX]={0, 1, 2, 3, 4, 5, 936, 5, 100, 50};
        char *data;
        ham_cursor_t *cursor;

        ::memset(&rec2, 0, sizeof(rec2));

        for (unsigned i=0; i<MAX; i++) {
            data=0;
            ::memset(&key, 0, sizeof(key));
            ::memset(&rec, 0, sizeof(rec));
            if (sizes[i]) {
                data=(char *)malloc(sizes[i]);
                ::memset(data, i+0x15, sizes[i]);
            }
            rec.data=sizes[i] ? data : 0;
            rec.size=sizes[i];
            CPPUNIT_ASSERT_EQUAL(0, 
                    ham_insert(m_db, 0, &key, &rec, HAM_DUPLICATE));
            CPPUNIT_ASSERT_EQUAL(sizes[i], rec.size);
            if (sizes[i]) {
                CPPUNIT_ASSERT_EQUAL(0, ::memcmp(data, rec.data, sizes[i]));
                free(data);
            }
        }

        CPPUNIT_ASSERT_EQUAL(0, ham_find(m_db, 0, &key, &rec2, 0));
        CPPUNIT_ASSERT_EQUAL((ham_size_t)0, rec2.size);

        CPPUNIT_ASSERT_EQUAL(0, ham_cursor_create(m_db, 0, 0, &cursor));

        for (unsigned i=0; i<MAX; i++) {
            ::memset(&key, 0, sizeof(key));
            ::memset(&rec, 0, sizeof(rec));
            if (sizes[i]) {
                data=(char *)malloc(sizes[i]);
                ::memset(data, i+0x15, sizes[i]);
            }
            rec.data=sizes[i] ? data : 0;
            rec.size=sizes[i];
            CPPUNIT_ASSERT_EQUAL(0, 
                    ham_cursor_move(cursor, &key, &rec, HAM_CURSOR_NEXT));
            CPPUNIT_ASSERT_EQUAL(sizes[i], rec.size);
            if (sizes[i]) {
                CPPUNIT_ASSERT_EQUAL(0, ::memcmp(data, rec.data, sizes[i]));
                free(data);
            }
        }

        CPPUNIT_ASSERT_EQUAL(0, ham_cursor_close(cursor));
        CPPUNIT_ASSERT_EQUAL(0, ham_cursor_create(m_db, 0, 0, &cursor));

        for (int i=MAX-1; i>=0; i--) {
            ::memset(&key, 0, sizeof(key));
            ::memset(&rec, 0, sizeof(rec));
            if (sizes[i]) {
                data=(char *)malloc(sizes[i]);
                ::memset(data, i+0x15, sizes[i]);
            }
            rec.data=sizes[i] ? data : 0;
            rec.size=sizes[i];
            CPPUNIT_ASSERT_EQUAL(0, 
                    ham_cursor_move(cursor, &key, &rec, HAM_CURSOR_PREVIOUS));
            CPPUNIT_ASSERT_EQUAL(sizes[i], rec.size);
            if (sizes[i]) {
                CPPUNIT_ASSERT_EQUAL(0, ::memcmp(data, rec.data, sizes[i]));
                free(data);
{'fraction_non_alphanumeric': 0.14334034876728804, 'fraction_numerical': 0.022098616957306073, 'mean_word_length': 2.175417661097852, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4859511', 'n_tokens_mistral': 4865, 'n_tokens_neox': 4561, 'n_words': 853}