<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright 2014 Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################


import argparse
import atexit
import os
import pprint
import pymongo
import shutil
import signal
import six
import subprocess
import time

from six.moves import range


def getMongoClient(config, uri=None, init=False, timeout=30):
    """
    Get a mongodb client from a uri or an index into the Config list.  This can
    initiate a replica set if needed.

    :param config: list of servers.
    :param uri: the mongo uri.  If this is an integer, the uri is constructed
                from the config list.  If None, use the route entry in config
                if it exists, or the 0th entry if it does not.
    :param init: if set and a client connection is requested, initiate the
                 replica set with this configuration.
    :param timeout: timeout in seconds.
    :returns: the client connection.
    """
    if uri is None:
        for server in config:
            if server.get('route'):
                uri = 'mongodb://127.0.0.1:%d/admin' % server['port']
        if uri is None:
            uri = 0
    if isinstance(uri, int):
        uri = 'mongodb://localhost:%d/admin' % config[uri]['port']
    starttime = time.time()
    client = None
    while time.time() < starttime + timeout:
        try:
            client = pymongo.MongoClient(uri)
            if init:
                try:
                    client.admin.command({'replSetInitiate': init})
                    break
                except pymongo.errors.OperationFailure:
                    time.sleep(0.5)
            else:
                break
        except pymongo.errors.ConnectionFailure:
            time.sleep(0.5)
    if not client:
        raise
    return client


def getOrderedRSStatus(config, client, minMembers, baseIdx):
    """
    Get the status of the mongodb replica set and order the members to match
    the config list.

    :param config: list of servers.
    :param client: an existing client connection to a mongodb.
    :param minMembers: the number of members to ensure exist and are in the
                       desired order.
    :param baseIdx: base entry in the config list.
    :returns: current status of the replica set with the members in the order
              of the config list, or None.
    """
    if minMembers > len(config):
        return None
    try:
        stat = client.admin.command('replSetGetStatus')
        if 'members' not in stat or len(stat['members']) < minMembers:
            return None
        members = []
        for idx in range(minMembers):
            member = None
            for entry in stat['members']:
                if (entry['name'].split(':')[-1] ==
                        str(config[baseIdx + idx]['port'])):
                    member = entry
            if not member:
                return None
            members.append(member)
        stat['members'] = members
        return stat
    except (pymongo.errors.OperationFailure, pymongo.errors.AutoReconnect):
        return None


def makeConfig(port=27070, replicaset='replicaset', servers=3, shard=False,
               sharddb='gridfs', shardcollection='chunk', shardkey='uuid',
               dirroot='/tmp/girderdb', **kwargs):
    """
    Make a config array.

    :param port: the starting port number.
    :param replicaset: name for the replicaset.
    :param servers: number of servers to start.  For shards, this is the number
        of shards; there will also be a config server and a routing server.
    :param shard: True to make a shard configuration, false for a replica set.
    :param sharddb: if sharding is used, the name of a database to shard.
    :param shardcollection: if sharding is used, the name of a collection to
        shard.
    :param shardkey: if sharding is used, the key to shard on in a collection.
    :param dirroot: base temp directory name.
    :returns: a new configuration array.
    """
    try:
        cmd = [os.environ.get('MONGOD_EXECUTABLE', 'mongod'), '--version']
        version = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout.read()
        version = tuple(int(part) for part in version.split(
            'db version v')[1].split()[0].strip().split('.'))
    except Exception:
        version = (0, 0, 0)
    config = []
    if shard:
        config.append({
            'dir': '%scfg' % dirroot,
            'config': True,
            'port': port + 1,
            'replicaset': replicaset
        })
        if version < (3, 2):
            del config[-1]['replicaset']
        for i in range(servers):
            config.append({
                'dir': '%s%d' % (dirroot, i + 1),
                'port': port + 2 + i,
                'shard': True<commit_msg>Remove code for checking mongo version in sharding tests.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright 2014 Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################


import argparse
import atexit
import os
import pprint
import pymongo
import shutil
import signal
import six
import subprocess
import time

from six.moves import range


def getMongoClient(config, uri=None, init=False, timeout=30):
    """
    Get a mongodb client from a uri or an index into the Config list.  This can
    initiate a replica set if needed.

    :param config: list of servers.
    :param uri: the mongo uri.  If this is an integer, the uri is constructed
                from the config list.  If None, use the route entry in config
                if it exists, or the 0th entry if it does not.
    :param init: if set and a client connection is requested, initiate the
                 replica set with this configuration.
    :param timeout: timeout in seconds.
    :returns: the client connection.
    """
    if uri is None:
        for server in config:
            if server.get('route'):
                uri = 'mongodb://127.0.0.1:%d/admin' % server['port']
        if uri is None:
            uri = 0
    if isinstance(uri, int):
        uri = 'mongodb://localhost:%d/admin' % config[uri]['port']
    starttime = time.time()
    client = None
    while time.time() < starttime + timeout:
        try:
            client = pymongo.MongoClient(uri)
            if init:
                try:
                    client.admin.command({'replSetInitiate': init})
                    break
                except pymongo.errors.OperationFailure:
                    time.sleep(0.5)
            else:
                break
        except pymongo.errors.ConnectionFailure:
            time.sleep(0.5)
    if not client:
        raise
    return client


def getOrderedRSStatus(config, client, minMembers, baseIdx):
    """
    Get the status of the mongodb replica set and order the members to match
    the config list.

    :param config: list of servers.
    :param client: an existing client connection to a mongodb.
    :param minMembers: the number of members to ensure exist and are in the
                       desired order.
    :param baseIdx: base entry in the config list.
    :returns: current status of the replica set with the members in the order
              of the config list, or None.
    """
    if minMembers > len(config):
        return None
    try:
        stat = client.admin.command('replSetGetStatus')
        if 'members' not in stat or len(stat['members']) < minMembers:
            return None
        members = []
        for idx in range(minMembers):
            member = None
            for entry in stat['members']:
                if (entry['name'].split(':')[-1] ==
                        str(config[baseIdx + idx]['port'])):
                    member = entry
            if not member:
                return None
            members.append(member)
        stat['members'] = members
        return stat
    except (pymongo.errors.OperationFailure, pymongo.errors.AutoReconnect):
        return None


def makeConfig(port=27070, replicaset='replicaset', servers=3, shard=False,
               sharddb='gridfs', shardcollection='chunk', shardkey='uuid',
               dirroot='/tmp/girderdb', **kwargs):
    """
    Make a config array.

    :param port: the starting port number.
    :param replicaset: name for the replicaset.
    :param servers: number of servers to start.  For shards, this is the number
        of shards; there will also be a config server and a routing server.
    :param shard: True to make a shard configuration, false for a replica set.
    :param sharddb: if sharding is used, the name of a database to shard.
    :param shardcollection: if sharding is used, the name of a collection to
        shard.
    :param shardkey: if sharding is used, the key to shard on in a collection.
    :param dirroot: base temp directory name.
    :returns: a new configuration array.
    """
    config = []
    if shard:
        config.append({
            'dir': '%scfg' % dirroot,
            'config': True,
            'port': port + 1,
            'replicaset': replicaset
        })
        for i in range(servers):
            config.append({
                'dir': '%s%d' % (dirroot, i + 1),
                'port': port + 2 + i,
                'shard': True
{'fraction_non_alphanumeric': 0.11690154166272582, 'fraction_numerical': 0.006998959614111416, 'mean_word_length': 2.217894096165551, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2980309', 'n_tokens_mistral': 2892, 'n_tokens_neox': 2775, 'n_words': 1106}