<commit_before>#!/usr/bin/env python
# encoding: utf-8
'''
run_feature_mp - feature extraction multiprocessing

@author: <NAME>

@license: MIT

@contact: <EMAIL>

'''
import glob,os,time,logging
import logging.config
from multiprocessing import Pool
import ConfigParser
########################################
# LOAD CONFOGURATION
########################################

# init config parser
config = ConfigParser.ConfigParser()
config.read("init.cfg")
# get localhost names
lh_names = config.get('Hostname','local').split(',')
# determine section
section = "Local" if os.uname()[1] in lh_names else "Server"
params = dict(config.items(section))
# number of processes
PROCESSES = config.getint('Feature','processes')
# feature extraction program
PROG=params['bin']+config.get('Feature','program')
########################################
# LOGGER CONFIGURATION
########################################
logging.config.fileConfig(params['logcfg'])
# create logger
logger_feature = logging.getLogger('feature')
logger_fail = logging.getLogger("fail")


def run_cmd(cat):
    """run c++ program by calling shell"""
    CMD = PROG + '-c ' + cat
    logger.debug("%s"%CMD)
    status = os.system(CMD)
    if status==0:
        logger_feature.info('[%s] Done (Elasped Time: %f)' % \
                           (current_in_file,time.time()-start))
        with open(PID_FILE , 'a') as pid:
            pid.write(cat + '\n')
    else:
        logger_feature.error('[%s] Fail' % cat)
        logger_fail.error('[%s](Search) Fail' % cat)
    return status

if __name__ == '__main__':

    logger_feature.info("\tStart")
    start = time.time()
    # get categories download from hive
    raw_cats =[x.split('/')[-1].split('.')[0] for x in \
               glob.glob(params['sqldata'] + '/*')]
    with open(PID_FILE,'r') as exist:
        exist_cats = [x.strip() for x in exist]
    cats = [x for x in raw_cats if not (x in exist_cats)]
    logger_feature.info("number of categories: %d" % len(cats))

    pool = Pool(PROCESSES)
    processes = pool._pool
    pool.map(run_cmd , cats)

    pool.close()
    time.sleep(1)
    try:
        for worker in processes:
            logger_feature.info('\t%s'%worker)
            assert not worker.is_alive()
        logger_feature.info('\tGarbage Collection Succeeded')
    except AssertionError:
        pool.terminate()
        sys.exit(0)

    logger_feature.info('\tEnd (Elasped Time : %s)\n' % str(time.time() - start))
<commit_msg>fix logger name<commit_after>#!/usr/bin/env python
# encoding: utf-8
'''
run_feature_mp - feature extraction multiprocessing

@author: <NAME>

@license: MIT

@contact: <EMAIL>

'''
import glob,os,time,logging
import logging.config
from multiprocessing import Pool
import ConfigParser
########################################
# LOAD CONFOGURATION
########################################

# init config parser
config = ConfigParser.ConfigParser()
config.read("init.cfg")
# get localhost names
lh_names = config.get('Hostname','local').split(',')
# determine section
section = "Local" if os.uname()[1] in lh_names else "Server"
params = dict(config.items(section))
# number of processes
PROCESSES = config.getint('Feature','processes')
# feature extraction program
PROG=params['bin']+config.get('Feature','program')
########################################
# LOGGER CONFIGURATION
########################################
logging.config.fileConfig(params['logcfg'])
# create logger
logger_feature = logging.getLogger('feature')
logger_fail = logging.getLogger("fail")


def run_cmd(cat):
    """run c++ program by calling shell"""
    CMD = PROG + '-c ' + cat
    logger_feature.debug("%s"%CMD)
    status = os.system(CMD)
    if status==0:
        logger_feature.info('[%s] Done (Elasped Time: %f)' % \
                           (current_in_file,time.time()-start))
        with open(PID_FILE , 'a') as pid:
            pid.write(cat + '\n')
    else:
        logger_feature.error('[%s] Fail' % cat)
        logger_fail.error('[%s](Search) Fail' % cat)
    return status

if __name__ == '__main__':

    logger_feature.info("\tStart")
    start = time.time()
    # get categories download from hive
    raw_cats =[x.split('/')[-1].split('.')[0] for x in \
               glob.glob(params['sqldata'] + '/*')]
    with open(PID_FILE,'r') as exist:
        exist_cats = [x.strip() for x in exist]
    cats = [x for x in raw_cats if not (x in exist_cats)]
    logger_feature.info("number of categories: %d" % len(cats))

    pool = Pool(PROCESSES)
    processes = pool._pool
    pool.map(run_cmd , cats)

    pool.close()
    time.sleep(1)
    try:
        for worker in processes:
            logger_feature.info('\t%s'%worker)
            assert not worker.is_alive()
        logger_feature.info('\tGarbage Collection Succeeded')
    except AssertionError:
        pool.terminate()
        sys.exit(0)

    logger_feature.info('\tEnd (Elasped Time : %s)\n' % str(time.time() - start))

{'fraction_non_alphanumeric': 0.20387565603552685, 'fraction_numerical': 0.002825999192571659, 'mean_word_length': 3.6007428040854226, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2436900', 'n_tokens_mistral': 1577, 'n_tokens_neox': 1540, 'n_words': 360}