<commit_before># Copyright 2012 10gen, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file will be used with PyPi in order to package and distribute the final
# product.

"""Discovers the mongo cluster and starts the connector.
"""

import inspect
import logging
import oplog_manager
import optparse
import os
import pymongo
import re
import subprocess
import sys
import threading
import time
import util
import shutil

try:
    import simplejson as json
except:
    import json


class Connector(threading.Thread):
    """Checks the cluster for shards to tail.
    """
    def __init__(self, address, oplog_checkpoint, backend_url, ns_set,
                 u_key, auth_key, doc_manager=None, auth_username=None):
        file = inspect.getfile(inspect.currentframe())
        cmd_folder = os.path.realpath(os.path.abspath(os.path.split(file)[0]))
        if doc_manager is not None:
            if (doc_manager[0] is '/') or 'C:\\' in doc_manager:
                if ('win32' or 'win64') in sys.platform:
                    shutil.copy(doc_manager, cmd_folder + "\\doc_manager.py")
                    #CMD = ["cp " + doc_manager + " " +
                    #  + cmd_folder + "\doc_manager.py"]
                    #subprocess.Popen(CMD, shell=True)
                else:
                    shutil.copy(doc_manager, cmd_folder + "/doc_manager.py")
                    #CMD = ["cp " + doc_manager + " " +
                    #+ cmd_folder + "/doc_manager.py"]
                    #subprocess.Popen(CMD, shell=True)
            else:
                if ('win32' or 'win64') in sys.platform:
                    shutil.copy(cmd_folder + "\\doc_managers\\" + doc_manager,
                                cmd_folder + "\\doc_manager.py")
                    #CMD = ["cp " + cmd_folder +
                    #  "\\doc_managers\\" + doc_manager + " " +
                    #  cmd_folder + "\\doc_manager.py"]
                    #subprocess.Popen(CMD, shell=True)
                else:
                    shutil.copy(cmd_folder + "/doc_managers/" + doc_manager,
                                cmd_folder + "/doc_manager.py")
                    #CMD = ["cp " + cmd_folder +
                    #   "/doc_managers/" + doc_manager + " " +
                    #   cmd_folder + "/doc_manager.py"]
                    #subprocess.Popen(CMD, shell=True)

        time.sleep(1)
        from doc_manager import DocManager
        super(Connector, self).__init__()

        #can_run is set to false when we join the thread
        self.can_run = True

        #The name of the file that stores the progress of the OplogThreads
        self.oplog_checkpoint = oplog_checkpoint

        #main address - either mongos for sharded setups or a primary otherwise
        self.address = address

        #The URL of the target system
        self.backend_url = backend_url

        #The set of relevant namespaces to consider
        self.ns_set = ns_set

        #The key that is a unique document identifier for the backend system.
        #Not necessarily the mongo unique key.
        self.u_key = u_key

        #Password for authentication
        self.auth_key = auth_key

        #Username for authentication
        self.auth_username = auth_username

        #The set of OplogThreads created
        self.shard_set = {}

        #Dict of OplogThread/timestmap pairs to record progress
        self.oplog_progress_dict = {}

        if backend_url is None:
            self.doc_manager = DocManager(None)
        else:
            self.doc_manager = DocManager(self.backend_url)

        if self.doc_manager is None:
            logging.critical('Bad backend URL!')
            return

    def join(self):
        """ Joins thread, stops it from running<commit_msg>adjusted backend simulator manager to take a url<commit_after># Copyright 2012 10gen, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file will be used with PyPi in order to package and distribute the final
# product.

"""Discovers the mongo cluster and starts the connector.
"""

import inspect
import logging
import oplog_manager
import optparse
import os
import pymongo
import re
import subprocess
import sys
import threading
import time
import util
import shutil

try:
    import simplejson as json
except:
    import json


class Connector(threading.Thread):
    """Checks the cluster for shards to tail.
    """
    def __init__(self, address, oplog_checkpoint, backend_url, ns_set,
                 u_key, auth_key, doc_manager=None, auth_username=None):
        file = inspect.getfile(inspect.currentframe())
        cmd_folder = os.path.realpath(os.path.abspath(os.path.split(file)[0]))
        if doc_manager is not None:
            if (doc_manager[0] is '/') or 'C:\\' in doc_manager:
                if ('win32' or 'win64') in sys.platform:
                    shutil.copy(doc_manager, cmd_folder + "\\doc_manager.py")
                    #CMD = ["cp " + doc_manager + " " +
                    #  + cmd_folder + "\doc_manager.py"]
                    #subprocess.Popen(CMD, shell=True)
                else:
                    shutil.copy(doc_manager, cmd_folder + "/doc_manager.py")
                    #CMD = ["cp " + doc_manager + " " +
                    #+ cmd_folder + "/doc_manager.py"]
                    #subprocess.Popen(CMD, shell=True)
            else:
                if ('win32' or 'win64') in sys.platform:
                    shutil.copy(cmd_folder + "\\doc_managers\\" + doc_manager,
                                cmd_folder + "\\doc_manager.py")
                    #CMD = ["cp " + cmd_folder +
                    #  "\\doc_managers\\" + doc_manager + " " +
                    #  cmd_folder + "\\doc_manager.py"]
                    #subprocess.Popen(CMD, shell=True)
                else:
                    shutil.copy(cmd_folder + "/doc_managers/" + doc_manager,
                                cmd_folder + "/doc_manager.py")
                    #CMD = ["cp " + cmd_folder +
                    #   "/doc_managers/" + doc_manager + " " +
                    #   cmd_folder + "/doc_manager.py"]
                    #subprocess.Popen(CMD, shell=True)

        time.sleep(1)
        from doc_manager import DocManager
        super(Connector, self).__init__()

        #can_run is set to false when we join the thread
        self.can_run = True

        #The name of the file that stores the progress of the OplogThreads
        self.oplog_checkpoint = oplog_checkpoint

        #main address - either mongos for sharded setups or a primary otherwise
        self.address = address

        #The URL of the target system
        self.backend_url = backend_url

        #The set of relevant namespaces to consider
        self.ns_set = ns_set

        #The key that is a unique document identifier for the backend system.
        #Not necessarily the mongo unique key.
        self.u_key = u_key

        #Password for authentication
        self.auth_key = auth_key

        #Username for authentication
        self.auth_username = auth_username

        #The set of OplogThreads created
        self.shard_set = {}

        #Dict of OplogThread/timestmap pairs to record progress
        self.oplog_progress_dict = {}

        if backend_url is None:
            self.doc_manager = DocManager()
        else:
            self.doc_manager = DocManager(self.backend_url)

        if self.doc_manager is None:
            logging.critical('Bad backend URL!')
            return

    def join(self):
        """ Joins thread, stops it from running
{'fraction_non_alphanumeric': 0.08645869844608015, 'fraction_numerical': 0.004907115317209955, 'mean_word_length': 2.0014025245441793, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6325429', 'n_tokens_mistral': 2424, 'n_tokens_neox': 2311, 'n_words': 748}