<commit_before>from dtest import Tester, debug, DISABLE_VNODES
import unittest
from tools import *
from ccmlib.cluster import Cluster
from ccmlib.node import NodeError
import time
from cql import OperationalError
from cql.cassandra.ttypes import UnavailableException
import sys

class NodeUnavailable(Exception):
    pass

class TestJson(Tester):

    def __init__(self, *args, **kwargs):
        # Ignore these log patterns:
        self.ignore_log_patterns = [
            # This one occurs when trying to send the migration to a
            # node that hasn't started yet, and when it does, it gets
            # replayed and everything is fine.
            r'Can\'t send migration request: node.*is down',
            # This is caused by starting a node improperly (replacing active/nonexistent)
            r'Exception encountered during startup',
            # This is caused by trying to replace a nonexistent node
            r'Exception in thread Thread'
        ]
        Tester.__init__(self, *args, **kwargs)

    def json_tools_test(self):

        debug("Starting cluster...")
        cluster = self.cluster
        cluster.populate(1).start()

        # time.sleep(1)

        debug("Version: " + cluster.version())

        debug("Getting nodes...")
        [node1] = cluster.nodelist()

        debug("Getting CQLSH...")
        cursor = self.patient_cql_connection(node1).cursor()

        debug("Inserting data...")
        self.create_ks(cursor, 'Test', 1)

        cursor.execute("""
            CREATE TABLE users (
                user_name varchar PRIMARY KEY,
                password varchar,
                gender varchar,
                state varchar,
                birth_year bigint
            );
        """)

        cursor.execute("INSERT INTO Test. users (user_name, password, gender, state, birth_year) VALUES('frodo', 'pass@', 'male', 'CA', 1985);")
        cursor.execute("INSERT INTO Test. users (user_name, password, gender, state, birth_year) VALUES('sam', '@pass', 'male', 'NY', 1980);")

        cursor.execute("SELECT * FROM users")
        res = cursor.fetchall()

        self.assertItemsEqual(res,
           [ [ u'frodo', 1985, u'male', u'pass@', u'CA' ],
              [u'sam', 1980, u'male', u'@pass', u'NY' ] ] )

        debug("Flushing and stopping cluster...")
        node1.flush()
        cluster.stop()

        debug("Creating JSON file...")
        # file = open("schema.json", "w")
        # file.write(node1.run_sstable2json())
        # print >> file, node1.run_sstable2json()

        # with open("schema.json", "w") as f:
        #     with f as sys.stdout:
        #         node1.run_sstable2json()
        # sys.stdout = sys.__stdout__

        sys.stdout = open("schema.json", 'w')
        #
        node1.run_sstable2json()
        sys.stdout = sys.__stdout__

        # file.close()
<commit_msg>working Json export/import<commit_after>from dtest import Tester, debug, DISABLE_VNODES
import unittest
from tools import *
from ccmlib.cluster import Cluster
from ccmlib.node import NodeError
import time
from cql import OperationalError
from cql.cassandra.ttypes import UnavailableException
import sys

class NodeUnavailable(Exception):
    pass

class TestJson(Tester):

    def __init__(self, *args, **kwargs):
        # Ignore these log patterns:
        self.ignore_log_patterns = [
            # This one occurs when trying to send the migration to a
            # node that hasn't started yet, and when it does, it gets
            # replayed and everything is fine.
            r'Can\'t send migration request: node.*is down',
            # This is caused by starting a node improperly (replacing active/nonexistent)
            r'Exception encountered during startup',
            # This is caused by trying to replace a nonexistent node
            r'Exception in thread Thread'
        ]
        Tester.__init__(self, *args, **kwargs)

    def json_tools_test(self):

        debug("Starting cluster...")
        cluster = self.cluster
        cluster.populate(1).start()

        # time.sleep(1)

        debug("Version: " + cluster.version())

        debug("Getting nodes...")
        [node1] = cluster.nodelist()

        debug("Getting CQLSH...")
        cursor = self.patient_cql_connection(node1).cursor()

        debug("Inserting data...")
        self.create_ks(cursor, 'Test', 1)

        cursor.execute("""
            CREATE TABLE users (
                user_name varchar PRIMARY KEY,
                password varchar,
                gender varchar,
                state varchar,
                birth_year bigint
            );
        """)

        cursor.execute("INSERT INTO Test. users (user_name, password, gender, state, birth_year) VALUES('frodo', 'pass@', 'male', 'CA', 1985);")
        cursor.execute("INSERT INTO Test. users (user_name, password, gender, state, birth_year) VALUES('sam', '@pass', 'male', 'NY', 1980);")

        cursor.execute("SELECT * FROM users")
        res = cursor.fetchall()

        self.assertItemsEqual(res,
           [ [ u'frodo', 1985, u'male', u'pass@', u'CA' ],
              [u'sam', 1980, u'male', u'@pass', u'NY' ] ] )

        debug("Flushing and stopping cluster...")
        node1.flush()
        # cluster.stop()

        debug("Exporting to JSON file...")

        out_file = open("schema.json", "w")
        node1.run_sstable2json(out_file)
        out_file.close()

        debug("Importing JSON file...")
        in_file = open("schema.json", "r")
        node1.run_json2sstable(in_file, "test", None, "users")
        in_file.close()
        os.remove("schema.json")

        debug("Verifying import...")
        # cluster.start()
        cursor.execute("SELECT * FROM users")
        res = cursor.fetchall()

        self.assertItemsEqual(res,
           [ [ u'frodo', 1985, u'male', u'pass@', u'CA' ],
              [u'sam', 1980, u'male', u'@pass', u'NY' ] ] )

{'fraction_non_alphanumeric': 0.12242137301318905, 'fraction_numerical': 0.010821778829895164, 'mean_word_length': 2.2111834961997827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2712292', 'n_tokens_mistral': 1794, 'n_tokens_neox': 1648, 'n_words': 467}