<commit_before>import os
import json
import sys
import unittest
import inspect
import shutil
import subprocess

from mtools.mlaunch.mlaunch import MLaunchTool, shutdown_host
from nose.tools import *
from nose.plugins.attrib import attr
from nose.plugins.skip import Skip, SkipTest
from pprint import pprint
from distutils.version import LooseVersion


class TestMLaunch(unittest.TestCase):

    # Setup & teardown functions

    def setUp(self):
        self.base_dir = 'data_test_mlaunch'
        self.tool = MLaunchTool(test=True)
        self.tool.args = {'verbose': False}
        self.mongod_version = self.tool.getMongoDVersion()

    def tearDown(self):
        self.tool = None
        if os.path.exists(self.base_dir):
            shutil.rmtree(self.base_dir)


    # Helper functions

    def run_tool(self, arg_str):
        ''' wrapper to call self.tool.run() with or without auth '''
        # name data directory according to test method name
        caller = inspect.stack()[1][3]
        self.data_dir = os.path.join(self.base_dir, caller)

        # add data directory to arguments for all commands
        arg_str += ' --dir %s' % self.data_dir

        with self.assertRaises(SystemExit):
            self.tool.run(arg_str)

    def read_config(self):
        ''' read the generated mlaunch startup file, get the command lines '''
        fp = open(self.data_dir + '/.mlaunch_startup', 'r')
        cfg = json.load(fp)
        cmd = [cfg['startup_info'][x] for x in cfg['startup_info'].keys()]
        return cfg, cmd

    def cmdlist_filter(self, cmdlist):
        ''' filter command lines to contain only [mongod|mongos] --parameter '''
        res = map(lambda cmd: set([param for param in cmd.split() if param.startswith('mongo') or param.startswith('--')]),
            [cmd for cmd in cmdlist if cmd.startswith('mongod') and '--configsvr' in cmd]
          + [cmd for cmd in cmdlist if cmd.startswith('mongod') and '--shardsvr' in cmd]
          + [cmd for cmd in cmdlist if cmd.startswith('mongod') and '--configsvr' not in cmd and '--shardsvr' not in cmd]
          + [cmd for cmd in cmdlist if cmd.startswith('mongos')]
        )
        return res

    def cmdlist_print(self):
        ''' print the generated command lines to console '''
        cfg, cmdlist = self.read_config()
        print '\n'
        print cmdlist
        print '\n'
        cmdset = self.cmdlist_filter(cmdlist)
        for cmd in cmdset:
            print cmd

    def cmdlist_assert(self, cmdlisttest):
        ''' assert helper for command lines '''
        cfg, cmdlist = self.read_config()
        cmdset = [set(x) for x in self.cmdlist_filter(cmdlist)]
        self.assertEqual(len(cmdlist), len(cmdlisttest), 'number of command lines is {0}, should be {1}'.format(len(cmdlisttest), len(cmdlist)))
        for cmd in zip(cmdset, cmdlisttest):
            self.assertSetEqual(cmd[0], cmd[1])

    def check_csrs(self):
        ''' check if CSRS is supported, skip test if unsupported '''
        if LooseVersion(self.mongod_version) < LooseVersion('3.1.0'):
            self.skipTest('CSRS not supported by MongoDB < 3.1.0')

    def check_sccc(self):
        ''' check if SCCC is supported, skip test if unsupported '''
        if LooseVersion(self.mongod_version) >= LooseVersion('3.3.0'):
            self.skipTest('SCCC not supported by MongoDB >= 3.3.0')

    def check_3_4(self):
        ''' check for MongoDB 3.4, skip test otherwise '''
        if LooseVersion(self.mongod_version) < LooseVersion('3.4.0'):
            self.skipTest('MongoDB version is < 3.4.0')


    # Tests

    def test_single(self):
        ''' mlaunch should start 1 node '''
        self.run_tool('init --single')
        cmdlist = [
            set(['mongod', '--dbpath', '--logpath', '--port', '--logappend', '--fork'])
        ]
        self.cmdlist_assert(cmdlist)

    def test_replicaset(self):
        ''' mlaunch should start 3 nodes replicaset '''
        self.run_tool('init --replicaset')
        cmdlist = (
            [ set(['mongod', '--replSet', '--dbpath', '--logpath', '--port', '--logappend', '--fork']) ] * 3
        )
        self.cmdlist_assert(cmdlist)

    def test_replicaset(self):
        ''' mlaunch should start 7 nodes replicaset '''
        self.run_tool('init --replicaset --nodes 7')
        cmdlist = (
            [ set(['mongod', '--replSet', '--dbpath', '--logpath', '--port', '--logappend', '--fork']) ] * 7
        )
        self.cmdlist_assert(cmdlist)

    def test_replicaset(self):
        ''' mlaunch should start 6 nodes + 1 arbiter replicaset '''
        self.run_tool('init --replicaset --nodes 6 --arbiter')
        cmdlist = (
            [ set(['mongod', '--replSet', '--dbpath', '--logpath', '--port', '--logappend', '--fork']) ] * 7
        )
        self.cmdlist_assert(cmdlist)

    def test_sharded_single(self):
        ''' mlaunch should start 1 config, 2 single shards 1 mongos '''<commit_msg>Test storage engine parameter, fixed replicaset test duplicated function names<commit_after>import os
import json
import sys
import unittest
import inspect
import shutil
import subprocess

from mtools.mlaunch.mlaunch import MLaunchTool, shutdown_host
from nose.tools import *
from nose.plugins.attrib import attr
from nose.plugins.skip import Skip, SkipTest
from pprint import pprint
from distutils.version import LooseVersion


class TestMLaunch(unittest.TestCase):

    # Setup & teardown functions

    def setUp(self):
        self.base_dir = 'data_test_mlaunch'
        self.tool = MLaunchTool(test=True)
        self.tool.args = {'verbose': False}
        self.mongod_version = self.tool.getMongoDVersion()

    def tearDown(self):
        self.tool = None
        if os.path.exists(self.base_dir):
            shutil.rmtree(self.base_dir)


    # Helper functions

    def run_tool(self, arg_str):
        ''' wrapper to call self.tool.run() with or without auth '''
        # name data directory according to test method name
        caller = inspect.stack()[1][3]
        self.data_dir = os.path.join(self.base_dir, caller)

        # add data directory to arguments for all commands
        arg_str += ' --dir %s' % self.data_dir

        with self.assertRaises(SystemExit):
            self.tool.run(arg_str)

    def read_config(self):
        ''' read the generated mlaunch startup file, get the command lines '''
        fp = open(self.data_dir + '/.mlaunch_startup', 'r')
        cfg = json.load(fp)
        cmd = [cfg['startup_info'][x] for x in cfg['startup_info'].keys()]
        return cfg, cmd

    def cmdlist_filter(self, cmdlist):
        ''' filter command lines to contain only [mongod|mongos] --parameter '''
        res = map(lambda cmd: set([param for param in cmd.split() if param.startswith('mongo') or param.startswith('--')]),
            [cmd for cmd in cmdlist if cmd.startswith('mongod') and '--configsvr' in cmd]
          + [cmd for cmd in cmdlist if cmd.startswith('mongod') and '--shardsvr' in cmd]
          + [cmd for cmd in cmdlist if cmd.startswith('mongod') and '--configsvr' not in cmd and '--shardsvr' not in cmd]
          + [cmd for cmd in cmdlist if cmd.startswith('mongos')]
        )
        return res

    def cmdlist_print(self):
        ''' print the generated command lines to console '''
        cfg, cmdlist = self.read_config()
        print '\n'
        print cmdlist
        print '\n'
        cmdset = self.cmdlist_filter(cmdlist)
        for cmd in cmdset:
            print cmd

    def cmdlist_assert(self, cmdlisttest):
        ''' assert helper for command lines '''
        cfg, cmdlist = self.read_config()
        cmdset = [set(x) for x in self.cmdlist_filter(cmdlist)]
        self.assertEqual(len(cmdlist), len(cmdlisttest), 'number of command lines is {0}, should be {1}'.format(len(cmdlisttest), len(cmdlist)))
        for cmd in zip(cmdset, cmdlisttest):
            self.assertSetEqual(cmd[0], cmd[1])

    def check_csrs(self):
        ''' check if CSRS is supported, skip test if unsupported '''
        if LooseVersion(self.mongod_version) < LooseVersion('3.1.0'):
            self.skipTest('CSRS not supported by MongoDB < 3.1.0')

    def check_sccc(self):
        ''' check if SCCC is supported, skip test if unsupported '''
        if LooseVersion(self.mongod_version) >= LooseVersion('3.3.0'):
            self.skipTest('SCCC not supported by MongoDB >= 3.3.0')

    def check_3_4(self):
        ''' check for MongoDB 3.4, skip test otherwise '''
        if LooseVersion(self.mongod_version) < LooseVersion('3.4.0'):
            self.skipTest('MongoDB version is < 3.4.0')


    # Tests

    def test_single(self):
        ''' mlaunch should start 1 node '''
        self.run_tool('init --single')
        cmdlist = [
            set(['mongod', '--dbpath', '--logpath', '--port', '--logappend', '--fork'])
        ]
        self.cmdlist_assert(cmdlist)

    def test_single_storage(self):
        ''' mlaunch should start 1 node with specified storage '''
        self.run_tool('init --single --storageEngine mmapv1')
        cmdlist = [
            set(['mongod', '--dbpath', '--logpath', '--port', '--logappend', '--fork', '--storageEngine'])
        ]
        self.cmdlist_assert(cmdlist)

    def test_replicaset_3(self):
        ''' mlaunch should start 3 nodes replicaset '''
        self.run_tool('init --replicaset')
        cmdlist = (
            [ set(['mongod', '--replSet', '--dbpath', '--logpath', '--port', '--logappend', '--fork']) ] * 3
        )
        self.cmdlist_assert(cmdlist)

    def test_replicaset_7(self):
        ''' mlaunch should start 7 nodes replicaset '''
        self.run_tool('init --replicaset --nodes 7')
        cmdlist = (
            [ set(['mongod', '--replSet', '--dbpath', '--logpath', '--port', '--logappend', '--fork']) ] * 7
        )
        self.cmdlist_assert(cmdlist)

    def test_replicaset_6_1(self):
        ''' mlaunch should start 6 nodes + 1 arbiter replicaset '''
        self.run_tool('init --replicaset --nodes 6 --arbiter')
        cmdlist = (
            [ set(['mongod', '--replSet', '--dbpath', '--logpath', '--port', '--logappend', '--fork']) ] * 7
        )
        self.cmdlist_assert(cmdlist)

    def test_sharded_single(self):
        ''' mlaunch should start 1 config, 2 single shards 1 mongos '''
{'fraction_non_alphanumeric': 0.13568376068376067, 'fraction_numerical': 0.008547008547008548, 'mean_word_length': 2.50357264375638, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4879053', 'n_tokens_mistral': 3263, 'n_tokens_neox': 3136, 'n_words': 864}