<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright 2013 Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import bson
from hashlib import sha512
import pymongo
import six
from six import BytesIO
import time
import uuid

from girder import logger
from girder.api.rest import setResponseHeader
from girder.external.mongodb_proxy import MongoProxy
from girder.models import getDbConnection
from girder.models.model_base import ValidationException
from . import hash_state
from .abstract_assetstore_adapter import AbstractAssetstoreAdapter


# 2MB chunks. Clients must not send any chunks that are smaller than this
# unless they are sending the final chunk.
CHUNK_SIZE = 2097152

# Cache recent connections so we can skip some start up actions
RECENT_CONNECTION_CACHE_TIME = 600  # seconds
RECENT_CONNECTION_CACHE_MAX_SIZE = 100
_recentConnections = {}


def _ensureChunkIndices(collection):
    """
    Ensure that we have appropriate indices on the chunk collection.

    :param collection: a connection to a mongo collection.
    """
    collection.create_index([
        ('uuid', pymongo.ASCENDING),
        ('n', pymongo.ASCENDING)
    ], unique=True)


def _setupSharding(collection, keyname='uuid'):
    """
    If we are communicating with a sharded server, and the collection is not
    sharded, ask for it to be sharded based on a key.

    :param collection: a connection to a mongo collection.
    :param keyname: the name of the key to shard on.
    :returns: True if sharding was added, False if it could not be added, or
        'present' if already sharded.
    """
    database = collection.database
    client = database.client
    stat = client.admin.command('serverStatus')
    # sharding will be non-None if the client is communicating with a mongos
    # instance. For mongo 3.0 we have to check the process name for 'mongos'.
    if not stat.get('sharding') and 'mongos' not in stat.get('process', ''):
        return False
    if database.command('collstats', collection.name).get('sharded'):
        return 'present'
    try:
        client.admin.command('enableSharding', database.name)
    except pymongo.errors.OperationFailure:
        # sharding may already be enabled
        pass
    try:
        client.admin.command(
            'shardCollection', '%s.%s' % (database.name, collection.name),
            key={keyname: 1})
        return True
    except pymongo.errors.OperationFailure:
        pass
    return False


class GridFsAssetstoreAdapter(AbstractAssetstoreAdapter):
    """
    This assetstore type stores files within MongoDB using the GridFS data
    model.
    """

    @staticmethod
    def validateInfo(doc):
        """
        Validate the assetstore -- make sure we can connect to it and that the
        necessary indexes are set up.
        """
        if not doc.get('db', ''):
            raise ValidationException('Database name must not be empty.', 'db')
        if '.' in doc['db'] or ' ' in doc['db']:
            raise ValidationException('Database name cannot contain spaces'
                                      ' or periods.', 'db')

        try:
            chunkColl = getDbConnection(
                doc.get('mongohost', None), doc.get('replicaset', None),
                autoRetry=False,
                serverSelectionTimeoutMS=10000)[doc['db']].chunk
            _ensureChunkIndices(chunkColl)
        except pymongo.errors.ServerSelectionTimeoutError as e:
            raise ValidationException(
                'Could not connect to the database: %s' % str(e))

        return doc

    @staticmethod
    def fileIndexFields():
        return ['sha512', 'chunkUuid']

    def __init__(self, assetstore):
        """
        :param assetstore: The assetstore to act on.
        """
        super(GridFsAssetstoreAdapter, self).__init__(assetstore)
        recent = False
        self._connectionArgs = (self.assetstore.get('mongohost', None),
                                self.assetstore.get('replicaset', None),
                                self.assetstore.get('shard', None))
        try:
            # Guard in case the connectionArgs is unhashable
            key = self._connectionArgs
            if key in _recentConnections:
                recent = (time.time() - _recentConnections[key]['created'] <
                          RECENT_CONNECTION_CACHE_TIME)
        except TypeError:
            key = None
        try:
            # MongoClient automatically reuses connections from a pool, but we
            # want to avoid redoing ensureChunkIndices each time we get such a
            # connection.
            client = getDbConnection(*self._connectionArgs, quiet=recent)
            self.chunkColl = MongoProxy(client[self.assetstore['db']].chunk)
            if not recent:
                _ensureChunkIndices(self.chunkColl)
                if self.assetstore.get('shard') == 'auto':<commit_msg>Don't use shard as a connection argument; it doesn't belong there.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright 2013 Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import bson
from hashlib import sha512
import pymongo
import six
from six import BytesIO
import time
import uuid

from girder import logger
from girder.api.rest import setResponseHeader
from girder.external.mongodb_proxy import MongoProxy
from girder.models import getDbConnection
from girder.models.model_base import ValidationException
from . import hash_state
from .abstract_assetstore_adapter import AbstractAssetstoreAdapter


# 2MB chunks. Clients must not send any chunks that are smaller than this
# unless they are sending the final chunk.
CHUNK_SIZE = 2097152

# Cache recent connections so we can skip some start up actions
RECENT_CONNECTION_CACHE_TIME = 600  # seconds
RECENT_CONNECTION_CACHE_MAX_SIZE = 100
_recentConnections = {}


def _ensureChunkIndices(collection):
    """
    Ensure that we have appropriate indices on the chunk collection.

    :param collection: a connection to a mongo collection.
    """
    collection.create_index([
        ('uuid', pymongo.ASCENDING),
        ('n', pymongo.ASCENDING)
    ], unique=True)


def _setupSharding(collection, keyname='uuid'):
    """
    If we are communicating with a sharded server, and the collection is not
    sharded, ask for it to be sharded based on a key.

    :param collection: a connection to a mongo collection.
    :param keyname: the name of the key to shard on.
    :returns: True if sharding was added, False if it could not be added, or
        'present' if already sharded.
    """
    database = collection.database
    client = database.client
    stat = client.admin.command('serverStatus')
    # sharding will be non-None if the client is communicating with a mongos
    # instance. For mongo 3.0 we have to check the process name for 'mongos'.
    if not stat.get('sharding') and 'mongos' not in stat.get('process', ''):
        return False
    if database.command('collstats', collection.name).get('sharded'):
        return 'present'
    try:
        client.admin.command('enableSharding', database.name)
    except pymongo.errors.OperationFailure:
        # sharding may already be enabled
        pass
    try:
        client.admin.command(
            'shardCollection', '%s.%s' % (database.name, collection.name),
            key={keyname: 1})
        return True
    except pymongo.errors.OperationFailure:
        pass
    return False


class GridFsAssetstoreAdapter(AbstractAssetstoreAdapter):
    """
    This assetstore type stores files within MongoDB using the GridFS data
    model.
    """

    @staticmethod
    def validateInfo(doc):
        """
        Validate the assetstore -- make sure we can connect to it and that the
        necessary indexes are set up.
        """
        if not doc.get('db', ''):
            raise ValidationException('Database name must not be empty.', 'db')
        if '.' in doc['db'] or ' ' in doc['db']:
            raise ValidationException('Database name cannot contain spaces'
                                      ' or periods.', 'db')

        try:
            chunkColl = getDbConnection(
                doc.get('mongohost', None), doc.get('replicaset', None),
                autoRetry=False,
                serverSelectionTimeoutMS=10000)[doc['db']].chunk
            _ensureChunkIndices(chunkColl)
        except pymongo.errors.ServerSelectionTimeoutError as e:
            raise ValidationException(
                'Could not connect to the database: %s' % str(e))

        return doc

    @staticmethod
    def fileIndexFields():
        return ['sha512', 'chunkUuid']

    def __init__(self, assetstore):
        """
        :param assetstore: The assetstore to act on.
        """
        super(GridFsAssetstoreAdapter, self).__init__(assetstore)
        recent = False
        try:
            # Guard in case the connectionArgs is unhashable
            key = (self.assetstore.get('mongohost', None),
                   self.assetstore.get('replicaset', None),
                   self.assetstore.get('shard', None))
            if key in _recentConnections:
                recent = (time.time() - _recentConnections[key]['created'] <
                          RECENT_CONNECTION_CACHE_TIME)
        except TypeError:
            key = None
        try:
            # MongoClient automatically reuses connections from a pool, but we
            # want to avoid redoing ensureChunkIndices each time we get such a
            # connection.
            client = getDbConnection(self.assetstore.get('mongohost', None),
                                     self.assetstore.get('replicaset', None),
                                     quiet=recent)
            self.chunkColl = MongoProxy(client[self.assetstore['db']].chunk)
            if not recent:
                _ensureChunkIndices(self.chunkColl)
                if self.assetstore.get('shard') == 'auto':
{'fraction_non_alphanumeric': 0.10948711134732926, 'fraction_numerical': 0.006555053591992205, 'mean_word_length': 2.6454633516306103, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6191311', 'n_tokens_mistral': 2997, 'n_tokens_neox': 2876, 'n_words': 1014}