<commit_before>import os
import re
import sys
import time
import types
import socket
import signal
import random
import threading
import cPickle as pickle

import OraDBProxy as DBProxy

from config      import panda_config
from JobSpec     import JobSpec
from FileSpec    import FileSpec
from DatasetSpec import DatasetSpec
from pandalogger.PandaLogger import PandaLogger

# logger
_logger = PandaLogger().getLogger('ConBridge')


# exception for normal termination
class HarmlessEx(Exception):
    pass
    

# terminate child process by itself when master has gone
class Terminator (threading.Thread):

    # constructor
    def __init__(self,consock):
        threading.Thread.__init__(self)
        self.consock = consock


    # main
    def run(self):
        # watching control socket
        try:
            rcvSize = self.consock.recv(1)
        except:
            pass
        # get PID
        pid = os.getpid()
        _logger.debug("child  %s received termination" % pid)
        # kill 
        try:
            os.kill(pid,signal.SIGTERM)
        except:
            pass
        try:
            os.kill(pid,signal.SIGKILL)
        except:
            pass


    
# connection bridge with with timeout
class ConBridge (object):

    # constructor
    def __init__(self):
        self.child_pid = 0
        self.isMaster  = False
        self.mysock    = None
        self.consock   = None
        self.pid       = os.getpid()
        # timeout
        if hasattr(panda_config,'dbtimeout'):
            self.timeout = int(panda_config.dbtimeout)
        else:
            self.timeout = 600
        # verbose    
        if hasattr(panda_config,'dbbridgeverbose'):
            self.verbose = panda_config.dbbridgeverbose
        else:
            self.verbose = False


    # destructor
    def __del__(self):
        # kill old child process
        self.bridge_killChild()

            
    # connect
    def connect(self,dbhost=panda_config.dbhost,dbpasswd=panda_config.dbpasswd,
                dbuser=panda_config.dbuser,dbname=panda_config.dbname,
                dbtimeout=None,reconnect=False):
        # kill old child process
        self.bridge_killChild()
        _logger.debug('master %s connecting' % self.pid)
        # reset child PID and sockets
        self.child_pid = 0     
        self.mysock    = None
        self.consock   = None
        # create socket
        datpair = socket.socketpair()
        conpair = socket.socketpair()
        # fork
        self.child_pid = os.fork()
        if self.child_pid == 0:
            # child
            self.isMaster = False
            self.pid = os.getpid()
            # keep socket
            self.mysock  = datpair[1]
            self.consock = conpair[1]
            datpair[0].close()
            conpair[0].close()
            # connect to database
            _logger.debug('child  %s connecting to database' % self.pid)            
            self.proxy = DBProxy.DBProxy()
            if not self.proxy.connect(dbhost=dbhost,dbpasswd=<PASSWORD>,dbtimeout=60):
                _logger.error('child  %s failed to connect : %s %s' % (self.pid,errType,errValue))
                # send error
                self.bridge_sendError((RuntimeError,'child  %s connection failed' % self.pid))
                # exit
                self.bridge_childExit()
            # send OK just for ACK
            _logger.debug('child  %s connection is ready' % self.pid)
            self.bridge_sendResponse(None)
            # start terminator
            Terminator(self.consock).start()
            # go main loop
            _logger.debug('child  %s going into the main loop' % self.pid)
            self.bridge_run()
            # exit
            self.bridge_childExit(0)
        else:
            # master<commit_msg>typo<commit_after>import os
import re
import sys
import time
import types
import socket
import signal
import random
import threading
import cPickle as pickle

import OraDBProxy as DBProxy

from config      import panda_config
from JobSpec     import JobSpec
from FileSpec    import FileSpec
from DatasetSpec import DatasetSpec
from pandalogger.PandaLogger import PandaLogger

# logger
_logger = PandaLogger().getLogger('ConBridge')


# exception for normal termination
class HarmlessEx(Exception):
    pass
    

# terminate child process by itself when master has gone
class Terminator (threading.Thread):

    # constructor
    def __init__(self,consock):
        threading.Thread.__init__(self)
        self.consock = consock


    # main
    def run(self):
        # watching control socket
        try:
            rcvSize = self.consock.recv(1)
        except:
            pass
        # get PID
        pid = os.getpid()
        _logger.debug("child  %s received termination" % pid)
        # kill 
        try:
            os.kill(pid,signal.SIGTERM)
        except:
            pass
        try:
            os.kill(pid,signal.SIGKILL)
        except:
            pass


    
# connection bridge with with timeout
class ConBridge (object):

    # constructor
    def __init__(self):
        self.child_pid = 0
        self.isMaster  = False
        self.mysock    = None
        self.consock   = None
        self.pid       = os.getpid()
        # timeout
        if hasattr(panda_config,'dbtimeout'):
            self.timeout = int(panda_config.dbtimeout)
        else:
            self.timeout = 600
        # verbose    
        if hasattr(panda_config,'dbbridgeverbose'):
            self.verbose = panda_config.dbbridgeverbose
        else:
            self.verbose = False


    # destructor
    def __del__(self):
        # kill old child process
        self.bridge_killChild()

            
    # connect
    def connect(self,dbhost=panda_config.dbhost,dbpasswd=<PASSWORD>,
                dbuser=panda_config.dbuser,dbname=panda_config.dbname,
                dbtimeout=None,reconnect=False):
        # kill old child process
        self.bridge_killChild()
        _logger.debug('master %s connecting' % self.pid)
        # reset child PID and sockets
        self.child_pid = 0     
        self.mysock    = None
        self.consock   = None
        # create socket
        datpair = socket.socketpair()
        conpair = socket.socketpair()
        # fork
        self.child_pid = os.fork()
        if self.child_pid == 0:
            # child
            self.isMaster = False
            self.pid = os.getpid()
            # keep socket
            self.mysock  = datpair[1]
            self.consock = conpair[1]
            datpair[0].close()
            conpair[0].close()
            # connect to database
            _logger.debug('child  %s connecting to database' % self.pid)            
            self.proxy = DBProxy.DBProxy()
            if not self.proxy.connect(dbhost=dbhost,dbpasswd=<PASSWORD>,dbtimeout=60):
                _logger.error('child  %s failed to connect' % self.pid)
                # send error
                self.bridge_sendError((RuntimeError,'child  %s connection failed' % self.pid))
                # exit
                self.bridge_childExit()
            # send OK just for ACK
            _logger.debug('child  %s connection is ready' % self.pid)
            self.bridge_sendResponse(None)
            # start terminator
            Terminator(self.consock).start()
            # go main loop
            _logger.debug('child  %s going into the main loop' % self.pid)
            self.bridge_run()
            # exit
            self.bridge_childExit(0)
        else:
            # master
{'fraction_non_alphanumeric': 0.0807626108830928, 'fraction_numerical': 0.0037071362372567192, 'mean_word_length': 1.784371544415776, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6701247', 'n_tokens_mistral': 2145, 'n_tokens_neox': 2076, 'n_words': 517}