<commit_before>/*
 * Copyright (C) 2017 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */


#include <boost/range/adaptor/transformed.hpp>
#include <boost/range/algorithm/copy.hpp>
#include <boost/range/algorithm_ext/push_back.hpp>
#include <boost/range/size.hpp>
#include <seastar/core/thread.hh>
#include <seastar/util/defer.hh>

#include "partition_version.hh"
#include "partition_snapshot_row_cursor.hh"
#include "partition_snapshot_reader.hh"

#include "tests/test-utils.hh"
#include "tests/mutation_assertions.hh"
#include "tests/simple_schema.hh"
#include "tests/mutation_source_test.hh"
#include "tests/failure_injecting_allocation_strategy.hh"
#include "tests/range_tombstone_list_assertions.hh"
#include "real_dirty_memory_accounter.hh"

using namespace std::chrono_literals;


// Verifies that tombstones in "list" are monotonic, overlap with the requested range,
// and have information equivalent with "expected" in that range.
static
void check_tombstone_slice(const schema& s, std::vector<range_tombstone> list,
    const query::clustering_range& range,
    std::initializer_list<range_tombstone> expected)
{
    range_tombstone_list actual(s);
    position_in_partition::less_compare less(s);
    position_in_partition prev_pos = position_in_partition::before_all_clustered_rows();

    for (auto&& rt : list) {
        if (!less(rt.position(), position_in_partition::for_range_end(range))) {
            BOOST_FAIL(format("Range tombstone out of range: {}, range: {}", rt, range));
        }
        if (!less(position_in_partition::for_range_start(range), rt.end_position())) {
            BOOST_FAIL(format("Range tombstone out of range: {}, range: {}", rt, range));
        }
        if (!less(prev_pos, rt.position())) {
            BOOST_FAIL(format("Range tombstone breaks position monotonicity: {}, list: {}", rt, list));
        }
        prev_pos = position_in_partition(rt.position());
        actual.apply(s, rt);
    }

    actual.trim(s, query::clustering_row_ranges{range});

    range_tombstone_list expected_list(s);
    for (auto&& rt : expected) {
        expected_list.apply(s, rt);
    }
    expected_list.trim(s, query::clustering_row_ranges{range});

    assert_that(s, actual).is_equal_to(expected_list);
}

SEASTAR_TEST_CASE(test_range_tombstone_slicing) {
    return seastar::async([] {
        logalloc::region r;
        mutation_cleaner cleaner(r, no_cache_tracker);
        simple_schema table;
        auto s = table.schema();
        with_allocator(r.allocator(), [&] {
            logalloc::reclaim_lock l(r);

            auto rt1 = table.make_range_tombstone(table.make_ckey_range(1, 2));
            auto rt2 = table.make_range_tombstone(table.make_ckey_range(4, 7));
            auto rt3 = table.make_range_tombstone(table.make_ckey_range(6, 9));

            mutation_partition m1(s);
            m1.apply_delete(*s, rt1);
            m1.apply_delete(*s, rt2);
            m1.apply_delete(*s, rt3);

            partition_entry e(mutation_partition(*s, m1));

            auto snap = e.read(r, cleaner, s, no_cache_tracker);

            auto check_range = [&s] (partition_snapshot& snap, const query::clustering_range& range,
                    std::initializer_list<range_tombstone> expected) {
                auto tombstones = snap.range_tombstones(
                    position_in_partition::for_range_start(range),
                    position_in_partition::for_range_end(range));
                check_tombstone_slice(*s, tombstones, range, expected);
            };

            check_range(*snap, table.make_ckey_range(0, 0), {});
            check_range(*snap, table.make_ckey_range(1, 1), {rt1});
            check_range(*snap, table.make_ckey_range(3, 4), {rt2});
            check_range(*snap, table.make_ckey_range(3, 5), {rt2});
            check_range(*snap, table.make_ckey_range(3, 6), {rt2, rt3});
            check_range(*snap, table.make_ckey_range(6, 6), {rt2, rt3});
            check_range(*snap, table.make_ckey_range(7, 10), {rt2, rt3});
            check_range(*snap, table.make_ckey_range(8, 10), {rt3});
            check_range(*snap, table.make_ckey_range(10, 10), {});
            check_range(*snap, table.make_ckey_range(0, 10), {rt1, rt2, rt3});

            auto rt4 = table.make_range_tombstone(table.make_ckey_range(1, 2));
            auto rt5 = table.make_range_tombstone(table.make_ckey_range(5, 8));

            mutation_partition m2(s);
            m2.apply_delete(*s, rt4);
            m2.apply_delete(*s, rt5);

            auto&& v2 = e.add_version(*s, no_cache_tracker);
            v2.partition().apply_weak(*s, m2, *s);
            auto snap2 = e.read(r, cleaner, s, no_cache_tracker);

            check_range(*snap2, table.make_ckey_range(0, 0), {});
            check_range(*snap2, table.make_ckey_range(1, 1), {rt4});
            check_range(*snap2, table.make_ckey_range(3, 4), {rt2});
            check_range(*snap2, table.make_ckey_range(3, 5), {rt2, rt5});
            check_range(*snap2, table.make_ckey_range(3, 6), {rt2, rt3, rt5});
            check_range(*snap2, table.make_ckey_range(4, 4), {rt2});
            check_range(*snap2, table.make_ckey_range(5, 5), {rt2, rt5});
            check_range(*snap2, table.make_ckey_range(6, 6), {rt2, rt3, rt5});
            check_range(*snap2, table.make_ckey_range(7, 10), {rt2, rt3, rt5});
            check_range(*snap2, table.make_ckey_range(8, 8), {rt3, rt5});
            check_range(*snap2, table.make_ckey_range(9, 9), {rt3});
            check_range(*snap2, table.make_ckey_range(8, 10), {rt3, rt5});
            check_range(*snap2, table.make_ckey_range(10, 10), {});
            check_range(*snap2, table.make_ckey_range(0, 10), {rt4, rt2, rt3, rt5});
        });
    });

}

class mvcc_partition;

// Together with mvcc_partition abstracts memory management details of dealing with MVCC.
class mvcc_container {
    schema_ptr _schema;
    std::optional<cache_tracker> _tracker;
    std::optional<logalloc::region> _region_holder;
    std::optional<mutation_cleaner> _cleaner_holder;
    partition_snapshot::phase_type _phase = partition_snapshot::min_phase;
    dirty_memory_manager _mgr;
    std::optional<real_dirty_memory_accounter> _acc;
    logalloc::region* _region;
    mutation_cleaner* _cleaner;
public:
    struct no_tracker {};
    mvcc_container(schema_ptr s)
        : _schema(s)
        , _tracker(std::make_optional<cache_tracker>())
        , _acc(std::make_optional<real_dirty_memory_accounter>(_mgr, *_tracker, 0))
        , _region(&_tracker->region())
        , _cleaner(&_tracker->cleaner())
    { }
    mvcc_container(schema_ptr s, no_tracker)
        : _schema(s)
        , _region_holder(std::make_optional<logalloc::region>())
        , _cleaner_holder(std::make_optional<mutation_cleaner>(*_region_holder, nullptr))
        , _region(&*_region_holder)
        , _cleaner(&*_cleaner_holder)
    { }
    mvcc_container(mvcc_container&&) = delete;

    // Call only when this container was constructed with a tracker
    mvcc_partition make_evictable(const mutation_partition& mp);
    // Call only when this container was constructed without a tracker
    mvcc_partition make_not_evictable(const mutation_partition& mp);
    logalloc::region& region() { return *_region; }
    cache_tracker* tracker() { return &*_tracker; }
    mutation_cleaner& cleaner() { return *_cleaner; }
    partition_snapshot::phase_type next_phase() { return ++_phase; }
    partition_snapshot::phase_type phase() const { return _phase; }
    real_dirty_memory_accounter& accounter() { return *_acc; }

    mutation_partition squashed(partition_snapshot_ptr& snp) {
        logalloc::allocating_section as;
        return as(region(), [&] {
            return snp->squashed();
        });
    }
};
<commit_msg>tests: mvcc: Introduce mvcc_container::migrate()<commit_after>/*
 * Copyright (C) 2017 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */


#include <boost/range/adaptor/transformed.hpp>
#include <boost/range/algorithm/copy.hpp>
#include <boost/range/algorithm_ext/push_back.hpp>
#include <boost/range/size.hpp>
#include <seastar/core/thread.hh>
#include <seastar/util/defer.hh>

#include "partition_version.hh"
#include "partition_snapshot_row_cursor.hh"
#include "partition_snapshot_reader.hh"

#include "tests/test-utils.hh"
#include "tests/mutation_assertions.hh"
#include "tests/simple_schema.hh"
#include "tests/mutation_source_test.hh"
#include "tests/failure_injecting_allocation_strategy.hh"
#include "tests/range_tombstone_list_assertions.hh"
#include "real_dirty_memory_accounter.hh"

using namespace std::chrono_literals;


// Verifies that tombstones in "list" are monotonic, overlap with the requested range,
// and have information equivalent with "expected" in that range.
static
void check_tombstone_slice(const schema& s, std::vector<range_tombstone> list,
    const query::clustering_range& range,
    std::initializer_list<range_tombstone> expected)
{
    range_tombstone_list actual(s);
    position_in_partition::less_compare less(s);
    position_in_partition prev_pos = position_in_partition::before_all_clustered_rows();

    for (auto&& rt : list) {
        if (!less(rt.position(), position_in_partition::for_range_end(range))) {
            BOOST_FAIL(format("Range tombstone out of range: {}, range: {}", rt, range));
        }
        if (!less(position_in_partition::for_range_start(range), rt.end_position())) {
            BOOST_FAIL(format("Range tombstone out of range: {}, range: {}", rt, range));
        }
        if (!less(prev_pos, rt.position())) {
            BOOST_FAIL(format("Range tombstone breaks position monotonicity: {}, list: {}", rt, list));
        }
        prev_pos = position_in_partition(rt.position());
        actual.apply(s, rt);
    }

    actual.trim(s, query::clustering_row_ranges{range});

    range_tombstone_list expected_list(s);
    for (auto&& rt : expected) {
        expected_list.apply(s, rt);
    }
    expected_list.trim(s, query::clustering_row_ranges{range});

    assert_that(s, actual).is_equal_to(expected_list);
}

SEASTAR_TEST_CASE(test_range_tombstone_slicing) {
    return seastar::async([] {
        logalloc::region r;
        mutation_cleaner cleaner(r, no_cache_tracker);
        simple_schema table;
        auto s = table.schema();
        with_allocator(r.allocator(), [&] {
            logalloc::reclaim_lock l(r);

            auto rt1 = table.make_range_tombstone(table.make_ckey_range(1, 2));
            auto rt2 = table.make_range_tombstone(table.make_ckey_range(4, 7));
            auto rt3 = table.make_range_tombstone(table.make_ckey_range(6, 9));

            mutation_partition m1(s);
            m1.apply_delete(*s, rt1);
            m1.apply_delete(*s, rt2);
            m1.apply_delete(*s, rt3);

            partition_entry e(mutation_partition(*s, m1));

            auto snap = e.read(r, cleaner, s, no_cache_tracker);

            auto check_range = [&s] (partition_snapshot& snap, const query::clustering_range& range,
                    std::initializer_list<range_tombstone> expected) {
                auto tombstones = snap.range_tombstones(
                    position_in_partition::for_range_start(range),
                    position_in_partition::for_range_end(range));
                check_tombstone_slice(*s, tombstones, range, expected);
            };

            check_range(*snap, table.make_ckey_range(0, 0), {});
            check_range(*snap, table.make_ckey_range(1, 1), {rt1});
            check_range(*snap, table.make_ckey_range(3, 4), {rt2});
            check_range(*snap, table.make_ckey_range(3, 5), {rt2});
            check_range(*snap, table.make_ckey_range(3, 6), {rt2, rt3});
            check_range(*snap, table.make_ckey_range(6, 6), {rt2, rt3});
            check_range(*snap, table.make_ckey_range(7, 10), {rt2, rt3});
            check_range(*snap, table.make_ckey_range(8, 10), {rt3});
            check_range(*snap, table.make_ckey_range(10, 10), {});
            check_range(*snap, table.make_ckey_range(0, 10), {rt1, rt2, rt3});

            auto rt4 = table.make_range_tombstone(table.make_ckey_range(1, 2));
            auto rt5 = table.make_range_tombstone(table.make_ckey_range(5, 8));

            mutation_partition m2(s);
            m2.apply_delete(*s, rt4);
            m2.apply_delete(*s, rt5);

            auto&& v2 = e.add_version(*s, no_cache_tracker);
            v2.partition().apply_weak(*s, m2, *s);
            auto snap2 = e.read(r, cleaner, s, no_cache_tracker);

            check_range(*snap2, table.make_ckey_range(0, 0), {});
            check_range(*snap2, table.make_ckey_range(1, 1), {rt4});
            check_range(*snap2, table.make_ckey_range(3, 4), {rt2});
            check_range(*snap2, table.make_ckey_range(3, 5), {rt2, rt5});
            check_range(*snap2, table.make_ckey_range(3, 6), {rt2, rt3, rt5});
            check_range(*snap2, table.make_ckey_range(4, 4), {rt2});
            check_range(*snap2, table.make_ckey_range(5, 5), {rt2, rt5});
            check_range(*snap2, table.make_ckey_range(6, 6), {rt2, rt3, rt5});
            check_range(*snap2, table.make_ckey_range(7, 10), {rt2, rt3, rt5});
            check_range(*snap2, table.make_ckey_range(8, 8), {rt3, rt5});
            check_range(*snap2, table.make_ckey_range(9, 9), {rt3});
            check_range(*snap2, table.make_ckey_range(8, 10), {rt3, rt5});
            check_range(*snap2, table.make_ckey_range(10, 10), {});
            check_range(*snap2, table.make_ckey_range(0, 10), {rt4, rt2, rt3, rt5});
        });
    });

}

class mvcc_partition;

// Together with mvcc_partition abstracts memory management details of dealing with MVCC.
class mvcc_container {
    schema_ptr _schema;
    std::optional<cache_tracker> _tracker;
    std::optional<logalloc::region> _region_holder;
    std::optional<mutation_cleaner> _cleaner_holder;
    partition_snapshot::phase_type _phase = partition_snapshot::min_phase;
    dirty_memory_manager _mgr;
    std::optional<real_dirty_memory_accounter> _acc;
    logalloc::region* _region;
    mutation_cleaner* _cleaner;
public:
    struct no_tracker {};
    mvcc_container(schema_ptr s)
        : _schema(s)
        , _tracker(std::make_optional<cache_tracker>())
        , _acc(std::make_optional<real_dirty_memory_accounter>(_mgr, *_tracker, 0))
        , _region(&_tracker->region())
        , _cleaner(&_tracker->cleaner())
    { }
    mvcc_container(schema_ptr s, no_tracker)
        : _schema(s)
        , _region_holder(std::make_optional<logalloc::region>())
        , _cleaner_holder(std::make_optional<mutation_cleaner>(*_region_holder, nullptr))
        , _region(&*_region_holder)
        , _cleaner(&*_cleaner_holder)
    { }
    mvcc_container(mvcc_container&&) = delete;

    // Call only when this container was constructed with a tracker
    mvcc_partition make_evictable(const mutation_partition& mp);
    // Call only when this container was constructed without a tracker
    mvcc_partition make_not_evictable(const mutation_partition& mp);
    logalloc::region& region() { return *_region; }
    cache_tracker* tracker() { return &*_tracker; }
    mutation_cleaner& cleaner() { return *_cleaner; }
    partition_snapshot::phase_type next_phase() { return ++_phase; }
    partition_snapshot::phase_type phase() const { return _phase; }
    real_dirty_memory_accounter& accounter() { return *_acc; }

    mutation_partition squashed(partition_snapshot_ptr& snp) {
        logalloc::allocating_section as;
        return as(region(), [&] {
            return snp->squashed();
        });
    }

    // Merges other into this
    void merge(mvcc_container& other) {
        _region->merge(*other._region);
        _cleaner->merge(*other._cleaner);
    }
};

{'fraction_non_alphanumeric': 0.12673162714252173, 'fraction_numerical': 0.017374970650387414, 'mean_word_length': 3.433255269320843, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5173216', 'n_tokens_mistral': 5747, 'n_tokens_neox': 5560, 'n_words': 1128}