<commit_before>from __future__ import division

import subprocess
import tempfile
import time
from math import floor
from os.path import getsize

from dtest import Tester, debug
from tools import since


class TestSSTableSplit(Tester):

    def __init__(self, *args, **kwargs):
        kwargs['cluster_options'] = {'start_rpc': 'true'}
        Tester.__init__(self, *args, **kwargs)

    def split_test(self):
        """
        Check that after running compaction, sstablessplit can succesfully split
        The resultant sstable.  Check that split is reversable and that data is readable
        after carrying out these operations.
        """
        cluster = self.cluster
        cluster.populate(1).start(wait_for_binary_proto=True)
        node = cluster.nodelist()[0]
        version = cluster.version()

        debug("Run stress to insert data")
        if version < "2.1":
            node.stress(['-o', 'insert'])
        else:
            node.stress(['write', 'n=1000000', '-rate', 'threads=10'])

        self._do_compaction(node)
        self._do_split(node, version)
        self._do_compaction(node)
        self._do_split(node, version)

        debug("Run stress to ensure data is readable")
        if version < "2.1":
            node.stress(['-o', 'read'])
        else:
            node.stress(['read', 'n=1000000', '-rate', 'threads=25'])

    def _do_compaction(self, node):
        debug("Compact sstables.")
        node.flush()
        node.compact()
        node.flush()
        keyspace = 'keyspace1' if self.cluster.version() >= '2.1' else 'Keyspace1'
        sstables = node.get_sstables(keyspace, '')
        debug("Number of sstables after compaction: %s" % len(sstables))

    def _do_split(self, node, version):
        debug("Run sstablesplit")
        time.sleep(5.0)
        node.stop()

        # default split size is 50MB
        splitmaxsize = 10 
        expected_sstable_size = (10 * 1024 * 1024) 
        keyspace = 'keyspace1' if self.cluster.version() >= '2.1' else 'Keyspace1'

        # get the initial sstables and their total size
        origsstables = node.get_sstables(keyspace, '')
        origsstable_size = sum([getsize(sstable) for sstable in origsstables])
        debug("Original sstable and sizes before split: {}".format([(name, getsize(name)) for name in origsstables]))

        # calculate the expected number of sstables post-split
        expected_num_sstables = floor(origsstable_size / expected_sstable_size) 

        # split the sstables
        node.run_sstablesplit(keyspace=keyspace, size=splitmaxsize, no_snapshot=True)

        # get the sstables post-split and their total size
        sstables = node.get_sstables(keyspace, '')
        debug("Number of sstables after split: %s. expected %s" % (len(sstables), expected_num_sstables))
        self.assertLessEqual(expected_num_sstables, len(sstables) + 1)
        self.assertLessEqual(1, len(sstables))

        # make sure none of the tables are bigger than the max expected size
        sstable_sizes = [getsize(sstable) for sstable in sstables]<commit_msg>printing debug output on sstablesplit failure (sstablesplit_test.py)<commit_after>from __future__ import division

import subprocess
import tempfile
import time
from math import floor
from os.path import getsize

from dtest import Tester, debug
from tools import since


class TestSSTableSplit(Tester):

    def __init__(self, *args, **kwargs):
        kwargs['cluster_options'] = {'start_rpc': 'true'}
        Tester.__init__(self, *args, **kwargs)

    def split_test(self):
        """
        Check that after running compaction, sstablessplit can succesfully split
        The resultant sstable.  Check that split is reversable and that data is readable
        after carrying out these operations.
        """
        cluster = self.cluster
        cluster.populate(1).start(wait_for_binary_proto=True)
        node = cluster.nodelist()[0]
        version = cluster.version()

        debug("Run stress to insert data")
        if version < "2.1":
            node.stress(['-o', 'insert'])
        else:
            node.stress(['write', 'n=1000000', '-rate', 'threads=10'])

        self._do_compaction(node)
        self._do_split(node, version)
        self._do_compaction(node)
        self._do_split(node, version)

        debug("Run stress to ensure data is readable")
        if version < "2.1":
            node.stress(['-o', 'read'])
        else:
            node.stress(['read', 'n=1000000', '-rate', 'threads=25'])

    def _do_compaction(self, node):
        debug("Compact sstables.")
        node.flush()
        node.compact()
        node.flush()
        keyspace = 'keyspace1' if self.cluster.version() >= '2.1' else 'Keyspace1'
        sstables = node.get_sstables(keyspace, '')
        debug("Number of sstables after compaction: %s" % len(sstables))

    def _do_split(self, node, version):
        debug("Run sstablesplit")
        time.sleep(5.0)
        node.stop()

        # default split size is 50MB
        splitmaxsize = 10 
        expected_sstable_size = (10 * 1024 * 1024) 
        keyspace = 'keyspace1' if self.cluster.version() >= '2.1' else 'Keyspace1'

        # get the initial sstables and their total size
        origsstables = node.get_sstables(keyspace, '')
        origsstable_size = sum([getsize(sstable) for sstable in origsstables])
        debug("Original sstable and sizes before split: {}".format([(name, getsize(name)) for name in origsstables]))

        # calculate the expected number of sstables post-split
        expected_num_sstables = floor(origsstable_size / expected_sstable_size) 

        # split the sstables
        result = node.run_sstablesplit(keyspace=keyspace, size=splitmaxsize,
                                       no_snapshot=True, debug=True)

        for (out, error, rc) in result:
            debug("stdout: {}".format(out))
            debug("stderr: {}".format(error))
            debug("rc: {}".format(rc))

        # get the sstables post-split and their total size
        sstables = node.get_sstables(keyspace, '')
        debug("Number of sstables after split: %s. expected %s" % (len(sstables), expected_num_sstables))
        self.assertLessEqual(expected_num_sstables, len(sstables) + 1)
        self.assertLessEqual(1, len(sstables))

        # make sure none of the tables are bigger than the max expected size
        sstable_sizes = [getsize(sstable) for sstable in sstables]
{'fraction_non_alphanumeric': 0.10994278645430648, 'fraction_numerical': 0.015463120457708366, 'mean_word_length': 2.708715596330275, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1508962', 'n_tokens_mistral': 2001, 'n_tokens_neox': 1893, 'n_words': 521}