<commit_before>#!/usr/bin/python
# coding=utf-8

import logging
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
from uuid import uuid4
from singleton import Singleton
from container import Container
import tempfile
from hosts import Hosts
from rs import RS
import pymongo


class Shard(object):
    """class represents Sharding configuration"""

    def __init__(self, params):
        """init configuration acording params"""
        self.id = params.get('id', None) or 'sh-' + str(uuid4())
        self.login = params.get('login', '')
        self.password = params.get('password', '')
        self.auth_key = params.get('auth_key', None)
        self._configsvrs = []
        self._routers = []
        self._shards = {}
        self.tags = {}
        self.__init_configsvr(params.get('configsvrs', [{}]))
        map(self.router_add, params.get('routers', [{}]))
        for cfg in params.get('members', []):
            shard_params = cfg.get('shardParams', {})
            shard_tags = shard_params.pop('tags', None)
            info = self.member_add(cfg.get('id', None), shard_params)
            if shard_tags:
                self.tags[info['id']] = shard_tags

        if self.tags:
            for sh_id in self.tags:
                for tag in self.tags[sh_id]:
                    command = 'sh.addShardTag("{sh_id}", "{tag}")'.format(**locals())
                    logger.debug(command)
                    self.router_command(command=command, is_eval=True)

        if self.login:
            try:
                self.router_command(command="db.addUser('{login}', '{password}');".format(login=self.login, password=self.password), is_eval=True)
            except pymongo.errors.OperationFailure:
                pymongo.Connection(self.router['hostname']).admin.add_user(self.login, self.password)

    def __init_configsvr(self, params):
        """create and start config servers"""
        self._configsvrs = []
        for cfg in params:
            cfg.update({'configsvr': True})
            self._configsvrs.append(Hosts().create('mongod', cfg, autostart=True, auth_key=self.auth_key))

    def __len__(self):
        return len(self._shards)

    @property
    def configsvrs(self):
        """return list of config servers"""
        return [{'id': h_id, 'hostname': Hosts().hostname(h_id)} for h_id in self._configsvrs]

    @property
    def routers(self):
        """return list of routers"""
        return [{'id': h_id, 'hostname': Hosts().hostname(h_id)} for h_id in self._routers]

    @property
    def members(self):
        """return list of members"""
        # return [{'id': shard, 'hostname': Hosts().hostname(info['_id'])} for shard, info in self._shards.items()]
        return [self.member_info(item) for item in self._shards]

    @property
    def router(self):
        """return first available router"""
        for host in self._routers:
            info = Hosts().info(host)
            if info['procInfo'].get('alive', False):
                return {'id': host, 'hostname': Hosts().hostname(host)}

    def router_add(self, params):
        """add new router (mongos) into existing configuration"""
        cfgs = ','.join([Hosts().info(item)['uri'] for item in self._configsvrs])
        params.update({'configdb': cfgs})
        self._routers.append(Hosts().create('mongos', params, autostart=True, auth_key=self.auth_key))
        return {'id': self._routers[-1], 'hostname': Hosts().hostname(self._routers[-1])}

    def connection(self):
        c = pymongo.Connection(self.router['hostname'])
        self.login and self.password and c.admin.authenticate(self.login, self.password)
        return c

    def router_command(self, command, arg=None, is_eval=False):
        """run command on router host"""
        return Hosts().db_command(self.router['id'], command, arg, is_eval=is_eval)

    def _add(self, shard_uri, name):
        """execute addShard command"""
        return self.router_command("addShard", (shard_uri, {"name": name}), is_eval=False)

    def member_add(self, member_id=None, params=None):
        """add new member into existing configuration"""
        member_id = member_id or str(uuid4())
        if 'members' in params:
            # is replica set
            rs_params = params.copy()
            rs_params.update({'auth_key': self.auth_key})
            rs_id = RS().create(rs_params)
            members = RS().members(rs_id)
            cfgs = rs_id + r"/" + ','.join([item['host'] for item in members])
            result = self._add(cfgs, member_id)
            if result.get('ok', 0) == 1:
                self._shards[result['shardAdded']] = {'isReplicaSet': True, '_id': rs_id}
                # return self._shards[result['shardAdded']].copy()
                return self.member_info(member_id)

        else:
            # is single host
            host_id = Hosts().create('mongod', params, autostart=True, auth_key=self.auth_key)
            result = self._add(Hosts().info(host_id)['uri'], member_id)
            if result.get('ok', 0) == 1:
                self._shards[result['shardAdded']] = {'isHost': True, '_id': host_id}
                # return self._shards[result['shardAdded']]
                return self.member_info(member_id)<commit_msg>fix use auth on router<commit_after>#!/usr/bin/python
# coding=utf-8

import logging
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
from uuid import uuid4
from singleton import Singleton
from container import Container
import tempfile
from hosts import Hosts
from rs import RS
import pymongo


class Shard(object):
    """class represents Sharding configuration"""

    def __init__(self, params):
        """init configuration acording params"""
        self.id = params.get('id', None) or 'sh-' + str(uuid4())
        self.login = params.get('login', '')
        self.password = params.get('password', '')
        self.auth_key = params.get('auth_key', None)
        self._configsvrs = []
        self._routers = []
        self._shards = {}
        self.tags = {}
        self.__init_configsvr(params.get('configsvrs', [{}]))
        map(self.router_add, params.get('routers', [{}]))
        for cfg in params.get('members', []):
            shard_params = cfg.get('shardParams', {})
            shard_tags = shard_params.pop('tags', None)
            info = self.member_add(cfg.get('id', None), shard_params)
            if shard_tags:
                self.tags[info['id']] = shard_tags

        if self.tags:
            for sh_id in self.tags:
                for tag in self.tags[sh_id]:
                    command = 'sh.addShardTag("{sh_id}", "{tag}")'.format(**locals())
                    logger.debug(command)
                    self.router_command(command=command, is_eval=True)

        if self.login:
            try:
                self.router_command(command="db.addUser('{login}', '{password}');".format(login=self.login, password=self.password), is_eval=True)
            except pymongo.errors.OperationFailure:
                pymongo.Connection(self.router['hostname']).admin.add_user(self.login, self.password)

    def __init_configsvr(self, params):
        """create and start config servers"""
        self._configsvrs = []
        for cfg in params:
            cfg.update({'configsvr': True})
            self._configsvrs.append(Hosts().create('mongod', cfg, autostart=True, auth_key=self.auth_key))

    def __len__(self):
        return len(self._shards)

    @property
    def configsvrs(self):
        """return list of config servers"""
        return [{'id': h_id, 'hostname': Hosts().hostname(h_id)} for h_id in self._configsvrs]

    @property
    def routers(self):
        """return list of routers"""
        return [{'id': h_id, 'hostname': Hosts().hostname(h_id)} for h_id in self._routers]

    @property
    def members(self):
        """return list of members"""
        # return [{'id': shard, 'hostname': Hosts().hostname(info['_id'])} for shard, info in self._shards.items()]
        return [self.member_info(item) for item in self._shards]

    @property
    def router(self):
        """return first available router"""
        for host in self._routers:
            info = Hosts().info(host)
            if info['procInfo'].get('alive', False):
                return {'id': host, 'hostname': Hosts().hostname(host)}

    def router_add(self, params):
        """add new router (mongos) into existing configuration"""
        cfgs = ','.join([Hosts().info(item)['uri'] for item in self._configsvrs])
        params.update({'configdb': cfgs})
        self._routers.append(Hosts().create('mongos', params, autostart=True, auth_key=self.auth_key))
        return {'id': self._routers[-1], 'hostname': Hosts().hostname(self._routers[-1])}

    def connection(self):
        c = pymongo.Connection(self.router['hostname'])
        self.login and self.password and c.admin.authenticate(self.login, self.password)
        return c

    def router_command(self, command, arg=None, is_eval=False):
        """run command on the router host

        Args:
            command - command string
            arg - command argument
            is_eval - if True execute command as eval

        return command's result
        """
        mode = is_eval and 'eval' or 'command'

        if isinstance(arg, tuple):
            name, d = arg
        else:
            name, d = arg, {}

        result = getattr(self.connection().admin, mode)(command, name, **d)
        return result

    def _add(self, shard_uri, name):
        """execute addShard command"""
        return self.router_command("addShard", (shard_uri, {"name": name}), is_eval=False)

    def member_add(self, member_id=None, params=None):
        """add new member into existing configuration"""
        member_id = member_id or str(uuid4())
        if 'members' in params:
            # is replica set
            rs_params = params.copy()
            rs_params.update({'auth_key': self.auth_key})
            rs_id = RS().create(rs_params)
            members = RS().members(rs_id)
            cfgs = rs_id + r"/" + ','.join([item['host'] for item in members])
            result = self._add(cfgs, member_id)
            if result.get('ok', 0) == 1:
                self._shards[result['shardAdded']] = {'isReplicaSet': True, '_id': rs_id}
                # return self._shards[result['shardAdded']].copy()
                return self.member_info(member_id)

        else:
            # is single host
            host_id = Hosts().create('mongod', params, autostart=True, auth_key=self.auth_key)
            result = self._add(Hosts().info(host_id)['uri'], member_id)
            if result.get('ok', 0) == 1:
                self._shards[result['shardAdded']] = {'isHost': True, '_id': host_id}
                # return self._shards[result['shardAdded']]
                return self.member_info(member_id)
{'fraction_non_alphanumeric': 0.14403367496339678, 'fraction_numerical': 0.0018301610541727673, 'mean_word_length': 2.533462657613967, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1592556', 'n_tokens_mistral': 3301, 'n_tokens_neox': 3246, 'n_words': 745}