<commit_before>#!/usr/bin/env python

# import statements
import sys, os
import traceback
from time import time, localtime, mktime
from stat import *
import signal
import thread
import ConfigParser
import re

# for SQL database functionality
# has to be installed on the running system, 
# package: "python-sqlobject"
from sqlobject import *

# setup search paths for python modules
sys.path.insert(0, os.path.expandvars('./happycore'))
sys.path.insert(0, os.path.expandvars('./modules'))
sys.path.insert(0, os.path.expandvars('./modules/examples'))
if os.path.exists('./local/modules'):
    sys.path.insert(0, os.path.expandvars('./local/modules'))
sys.path.insert(0, os.path.expandvars('./output/web'))

import DBWrapper

# import output module, e.g. website creator
from WebCreator import *

# import HappyFace services
from DownloadService import *
from CssService import *

##########################################################
# load the config file
# create storage directory for binary files
# initiate the database
# start the execution
# store data to database
# initiate the output creation module
##########################################################

def HappyFace():

    # load the config file
    config = ConfigParser.ConfigParser()

    # try to open standard config file, must be available
    try:
	config.readfp(open('./run.cfg'))
    except IOError:
        sys.stdout.write('Could not find configuration file run.cfg, aborting ...\n')
        sys.exit(-1)

    # try to open local config file if available.
    # default config settings in run.cfg will be overwritten
    try:
	config.readfp(open('./local/cfg/run.local'))
    except IOError:
        pass

    # get paths from config files
    output_dir	= config.get('setup','output_dir')
    tmp_dir     = config.get('setup','tmp_dir')
    try:
        tmp_keepalive = int(config.get('setup', 'tmp_keepalive'))
    except:
        # default
        tmp_keepalive = 7

    # convert to seconds:
    if tmp_keepalive > 0:
        tmp_keepalive = tmp_keepalive * 24 * 60 * 60

    # create timestamp (unixtime), set seconds to "0"
    time_tuple = localtime() 
    timestamp = int(mktime(time_tuple)) - time_tuple[5]

    # create archive directory for binary files
    archive_dir = output_dir + "/archive/" + str(time_tuple.tm_year) + "/" + ('%02d' % time_tuple.tm_mon) + "/" + ('%02d' % time_tuple.tm_mday) + "/" + str(timestamp)
    try:
        os.system("mkdir -p " + archive_dir)
    except:
        sys.stdout.write('Could not create archive directory ' + archive_dir + ', aborting ...\n')
        sys.exit(-1)

    try: # separate try for finally block so we are compatible with Python 2.4
        try:
            lockfile = output_dir + "/hf.lock"

            def signal_handler(signum, stack):
	        print('HappyFace: Terminating due to signal %s' % str(signum))
	        # Note that this raises an exception so the lock file will be
	        # removed by the finally clause of the outer try block.
	        sys.exit(-2)

            signal.signal(signal.SIGTERM, signal_handler)
            signal.signal(signal.SIGINT, signal_handler)
            psid=os.getpid()
            print "Current HappyFace process-id: ",psid
            # Check for lockfile existence
            try:
                ifile=os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR)
                os.write(ifile,str(psid)+"\n")
                os.close(ifile)
	    except:
                sys.stdout.write('Lock file exists. Checking age ...\n')
                # Prevent outer try block from removing the lock file 
                age=os.stat(lockfile)[ST_CTIME]
                timestamp = int(mktime(localtime()))
                diff=timestamp-age
                print "Lockfile age: ",diff,"s"
                maxAge=60*5;
                if (diff>maxAge):
                    print "Lock file older than ",maxAge," checking if process still exist..."
                    try:
                        rfile=open(lockfile,'r')
                        opid=rfile.readline()
                        print "Original Happyface: ",opid
                        dingens=os.getpgid(int(opid))
                        print "Original HF process still running, exiting..."
                    except:
                        print "Old HF process does not exist anymore. Removing lockfile..."
                        os.remove(lockfile)
                        lockfile = ''
                        sys.exit(-1)
                    print "Old HF process still running. Exiting here..."
                    lockfile = ''
                    sys.exit(-1)
                else:
                    print "Lockfile not older than ",maxAge,"s letting other HF instance relax..."
                    # Prevent outer try block from removing the lock file
                    lockfile = ''
                    sys.exit(-1)
             
            # get database wrapper
            dbWrapperModule = __import__(config.get('setup', 'db_wrapper_module'))
            DBWrapper.SelectedDBWrapper = dbWrapperModule.__dict__[config.get('setup', 'db_wrapper_class')]

            # try to initiate / create the database
            connection_string = config.get('setup', 'db_connection')
            # fetch special directory path
            result = re.match(r'(\w+://)\./(.+)', connection_string)
            if result is not None:
                connection_string = result.group(1) + os.path.join(os.getcwd(), output_dir, result.group(2))
            print connection_string
            try:
                # has to be the full path
                #connection_string = 'sqlite:' + os.getcwd() + "/" + database
                #connection_string = 'postgres://postgres:katze123@localhost/happyface'
                connection = connectionForURI(connection_string)
                sqlhub.processConnection = connection
            except Exception, ex:
                raise Exception('Could not initiate or create the database ' + connection_string + ': ' + str(ex))

            # definition of the global timeout
            timeout        =  int(config.get('setup','timeout_module'))
            timeoutDownload =  int(config.get('setup','timeout_download'))

            # definition of the holdback_time
            holdback_time  =  int(config.get('setup','holdback_time'))

            ##########################################################
            ################ start with the execution ################

            # create options dictionary
            module_options = {}
    
            module_options["timestamp"] = timestamp
            module_options["archive_dir"] = archive_dir
            module_options["holdback_time"] = holdback_time
    
            # dictionary collects the module objects
            modObj_list = {}

            # initialisation of modules
            print "HappyFace: Start with module preparation:"
            moduleCount = 0<commit_msg>Remove debug prints<commit_after>#!/usr/bin/env python

# import statements
import sys, os
import traceback
from time import time, localtime, mktime
from stat import *
import signal
import thread
import ConfigParser
import re

# for SQL database functionality
# has to be installed on the running system, 
# package: "python-sqlobject"
from sqlobject import *

# setup search paths for python modules
sys.path.insert(0, os.path.expandvars('./happycore'))
sys.path.insert(0, os.path.expandvars('./modules'))
sys.path.insert(0, os.path.expandvars('./modules/examples'))
if os.path.exists('./local/modules'):
    sys.path.insert(0, os.path.expandvars('./local/modules'))
sys.path.insert(0, os.path.expandvars('./output/web'))

import DBWrapper

# import output module, e.g. website creator
from WebCreator import *

# import HappyFace services
from DownloadService import *
from CssService import *

##########################################################
# load the config file
# create storage directory for binary files
# initiate the database
# start the execution
# store data to database
# initiate the output creation module
##########################################################

def HappyFace():

    # load the config file
    config = ConfigParser.ConfigParser()

    # try to open standard config file, must be available
    try:
	config.readfp(open('./run.cfg'))
    except IOError:
        sys.stdout.write('Could not find configuration file run.cfg, aborting ...\n')
        sys.exit(-1)

    # try to open local config file if available.
    # default config settings in run.cfg will be overwritten
    try:
	config.readfp(open('./local/cfg/run.local'))
    except IOError:
        pass

    # get paths from config files
    output_dir	= config.get('setup','output_dir')
    tmp_dir     = config.get('setup','tmp_dir')
    try:
        tmp_keepalive = int(config.get('setup', 'tmp_keepalive'))
    except:
        # default
        tmp_keepalive = 7

    # convert to seconds:
    if tmp_keepalive > 0:
        tmp_keepalive = tmp_keepalive * 24 * 60 * 60

    # create timestamp (unixtime), set seconds to "0"
    time_tuple = localtime() 
    timestamp = int(mktime(time_tuple)) - time_tuple[5]

    # create archive directory for binary files
    archive_dir = output_dir + "/archive/" + str(time_tuple.tm_year) + "/" + ('%02d' % time_tuple.tm_mon) + "/" + ('%02d' % time_tuple.tm_mday) + "/" + str(timestamp)
    try:
        os.system("mkdir -p " + archive_dir)
    except:
        sys.stdout.write('Could not create archive directory ' + archive_dir + ', aborting ...\n')
        sys.exit(-1)

    try: # separate try for finally block so we are compatible with Python 2.4
        try:
            lockfile = output_dir + "/hf.lock"

            def signal_handler(signum, stack):
	        print('HappyFace: Terminating due to signal %s' % str(signum))
	        # Note that this raises an exception so the lock file will be
	        # removed by the finally clause of the outer try block.
	        sys.exit(-2)

            signal.signal(signal.SIGTERM, signal_handler)
            signal.signal(signal.SIGINT, signal_handler)
            psid=os.getpid()
            print "Current HappyFace process-id: ",psid
            # Check for lockfile existence
            try:
                ifile=os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR)
                os.write(ifile,str(psid)+"\n")
                os.close(ifile)
	    except:
                sys.stdout.write('Lock file exists. Checking age ...\n')
                # Prevent outer try block from removing the lock file 
                age=os.stat(lockfile)[ST_CTIME]
                timestamp = int(mktime(localtime()))
                diff=timestamp-age
                print "Lockfile age: ",diff,"s"
                maxAge=60*5;
                if (diff>maxAge):
                    print "Lock file older than ",maxAge," checking if process still exist..."
                    try:
                        rfile=open(lockfile,'r')
                        opid=rfile.readline()
                        print "Original Happyface: ",opid
                        dingens=os.getpgid(int(opid))
                        print "Original HF process still running, exiting..."
                    except:
                        print "Old HF process does not exist anymore. Removing lockfile..."
                        os.remove(lockfile)
                        lockfile = ''
                        sys.exit(-1)
                    print "Old HF process still running. Exiting here..."
                    lockfile = ''
                    sys.exit(-1)
                else:
                    print "Lockfile not older than ",maxAge,"s letting other HF instance relax..."
                    # Prevent outer try block from removing the lock file
                    lockfile = ''
                    sys.exit(-1)
             
            # get database wrapper
            dbWrapperModule = __import__(config.get('setup', 'db_wrapper_module'))
            DBWrapper.SelectedDBWrapper = dbWrapperModule.__dict__[config.get('setup', 'db_wrapper_class')]

            # try to initiate / create the database
            connection_string = config.get('setup', 'db_connection')
            # fetch special directory path
            result = re.match(r'(\w+://)\./(.+)', connection_string)
            if result is not None:
                connection_string = result.group(1) + os.path.join(os.getcwd(), output_dir, result.group(2))
            try:
                connection = connectionForURI(connection_string)
                sqlhub.processConnection = connection
            except Exception, ex:
                raise Exception('Could not initiate or create the database ' + connection_string + ': ' + str(ex))

            # definition of the global timeout
            timeout        =  int(config.get('setup','timeout_module'))
            timeoutDownload =  int(config.get('setup','timeout_download'))

            # definition of the holdback_time
            holdback_time  =  int(config.get('setup','holdback_time'))

            ##########################################################
            ################ start with the execution ################

            # create options dictionary
            module_options = {}
    
            module_options["timestamp"] = timestamp
            module_options["archive_dir"] = archive_dir
            module_options["holdback_time"] = holdback_time
    
            # dictionary collects the module objects
            modObj_list = {}

            # initialisation of modules
            print "HappyFace: Start with module preparation:"
            moduleCount = 0
{'fraction_non_alphanumeric': 0.12832310838445807, 'fraction_numerical': 0.005039439088518843, 'mean_word_length': 2.0972630626555078, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7101389', 'n_tokens_mistral': 3682, 'n_tokens_neox': 3465, 'n_words': 1007}