<commit_before>import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))

import logging
import socket
import urlparse
from StringIO import StringIO
from string import Template
from datetime import datetime
import MySQLdb
import psycopg2
import sqlite3

from library.models import *
from cmudbac.settings import *
import utils

## =====================================================================
## LOGGING CONFIGURATION
## =====================================================================
LOG = logging.getLogger()
LOG_handler = logging.StreamHandler()
LOG_formatter = logging.Formatter(fmt='%(asctime)s [%(filename)s:%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s',
                                  datefmt='%m-%d-%Y %H:%M:%S')
LOG_handler.setFormatter(LOG_formatter)
LOG.addHandler(LOG_handler)
LOG.setLevel(logging.INFO)

## =====================================================================
## BASE DEPLOYER
## =====================================================================
class BaseDeployer(object):
    TMP_ZIP_FILE = "tmp.zip"
    TMP_DEPLOY_PATH = "/tmp/crawler"
    
    def __init__(self, repo, database, deploy_id, database_config = None, runtime = None):
        self.repo = repo
        self.database = database
        self.requirement_files = None
        self.packages_from_database = []
        self.packages_from_file = []
        self.deploy_id = deploy_id
        self.zip_file = self.TMP_ZIP_FILE + str(self.deploy_id)
        self.base_path = self.TMP_DEPLOY_PATH + str(self.deploy_id)
        self.setting_path = None
        self.port = int(self.repo.project_type.default_port) + int(self.deploy_id)
        self.runtime = None
        self.log_file = None

        if database_config == None:
            if self.database.name == 'MySQL':
                self.database_config = {<commit_msg>add a file handler to LOG<commit_after>import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))

import logging
import socket
import urlparse
from StringIO import StringIO
from string import Template
from datetime import datetime
import MySQLdb
import psycopg2
import sqlite3

from library.models import *
from cmudbac.settings import *
import utils

## =====================================================================
## LOGGING CONFIGURATION
## =====================================================================
LOG = logging.getLogger()
streamHandler = logging.StreamHandler()
LOG_formatter = logging.Formatter(fmt='%(asctime)s [%(filename)s:%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s',
                                  datefmt='%m-%d-%Y %H:%M:%S')
streamHandler.setFormatter(LOG_formatter)
LOG.addHandler(streamHandler)
LOG.setLevel(logging.INFO)

## =====================================================================
## BASE DEPLOYER
## =====================================================================
class BaseDeployer(object):
    TMP_ZIP_FILE = "tmp.zip"
    TMP_DEPLOY_PATH = "/tmp/crawler"
    
    def __init__(self, repo, database, deploy_id, database_config = None, runtime = None):
        self.repo = repo
        self.database = database
        self.requirement_files = None
        self.packages_from_database = []
        self.packages_from_file = []
        self.deploy_id = deploy_id
        self.zip_file = self.TMP_ZIP_FILE + str(self.deploy_id)
        self.base_path = self.TMP_DEPLOY_PATH + str(self.deploy_id)
        self.setting_path = None
        self.port = int(self.repo.project_type.default_port) + int(self.deploy_id)
        self.runtime = None
        self.log_file = None

        # add file handler to LOG
        fileHandler = logging.FileHandler(os.path.join('/vagrant', str(self.deploy_id) + '.log'), 'w')
        fileHandler.setFormatter(LOG_formatter)
        LOG.addHandler(fileHandler)

        if database_config == None:
            if self.database.name == 'MySQL':
                self.database_config = {
{'fraction_non_alphanumeric': 0.23569277108433734, 'fraction_numerical': 0.0025100401606425703, 'mean_word_length': 3.919753086419753, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7563979', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1109, 'n_words': 221}