<commit_before>#!/usr/bin/env python
# encoding: utf-8
'''
run_upload - push computed result to redis

@author:     <NAME>
@modified by:   <NAME>

@license:    MIT

@contact:    <EMAIL>

'''

import sys,os,logging,time,glob
import logging.config
import ConfigParser
from string import Template

# SAFTY FLAG #
PID = time.strftime("_%d_%m_%Y")

########################################
# LOAD CONFIGURATION
########################################
# init config parser
config = ConfigParser.ConfigParser()
config.read("init.cfg")
# get localhost names
lh_names = config.get('Hostname','local').split(',')
# determine section
section = "Local" if os.uname()[1] in lh_names else "Server"
params = dict(config.items(section))

# min lines of file
MIN_ROW_SIZE = config.getint('Hive','minrow')
########################################
# LOGGER CONFIGURATION
########################################
logging.config.fileConfig(params['logcfg'])
# create logger
logger_hive = logging.getLogger('hive')
logger_fail = logging.getLogger("fail")


with open(params['conf'] + config.get('Hive','categoryfile'),'r') as cate_file:
    categories = ','.join(["'"+cat.strip()+"'" for cat in cate_file])
with open(params['conf'] + config.get('Hive','sqlfile'),'r') as hive_file:
    hive_temp=Template(" ".join([line.strip() for line in hive_file]))

HIVE_QL = hive_temp.safe_substitute({"CATES":categories})

logger_hive.debug("HIVE_QL:")
logger_hive.debug(HIVE_QL)

result_file = params['conf'] + "sql" + PID + ".txt"
logger_hive.info("\tStart")

# first to download data file from hadoop


# status=os.system("touch "+LOADING_FILE)

# status=os.system("chmod a+rw "+LOADING_FILE)

# if status==0:
#     logger.info("chmod a+rw"+LOADING_FILE+" finished.")
# else:
#     logger.error("chmod a+rw"+LOADING_FILE+" failed.")
#     sys.exit(1)

status=os.system("su - hadoop-cdh -c \' hive -S -e \"" + HIVE_QL + \
                 "\" > "+ result_file+"\'")

if status==0:
    logger_hive.info("Write data from Hive into %s" % result_file)
else:
    logger_hive.error("Fail to donwload data from hive")
    logger_fail.error("Fail to donwload data from hive")

lines=os.popen("wc -l "+result_file,'r').readline()
if lines < MIN_ROW_SIZE:
    logger_hive.error("Number of items download from hive not enough")
    logger_fail.error("Number of items download from hive not enough")
    sys.exit(-1)
logger_hive.info("Number of items download from hive: %d" % lines)

if len(glob.glob(params['sqldata'] + '/*')) > 0:
    logger_hive.error("[sqldata] directory not empty")<commit_msg>fix %d bug<commit_after>#!/usr/bin/env python
# encoding: utf-8
'''
run_upload - push computed result to redis

@author:     <NAME>
@modified by:   <NAME>

@license:    MIT

@contact:    <EMAIL>

'''

import sys,os,logging,time,glob
import logging.config
import ConfigParser
from string import Template

# SAFTY FLAG #
PID = time.strftime("_%d_%m_%Y")

########################################
# LOAD CONFIGURATION
########################################
# init config parser
config = ConfigParser.ConfigParser()
config.read("init.cfg")
# get localhost names
lh_names = config.get('Hostname','local').split(',')
# determine section
section = "Local" if os.uname()[1] in lh_names else "Server"
params = dict(config.items(section))

# min lines of file
MIN_ROW_SIZE = config.getint('Hive','minrow')
########################################
# LOGGER CONFIGURATION
########################################
logging.config.fileConfig(params['logcfg'])
# create logger
logger_hive = logging.getLogger('hive')
logger_fail = logging.getLogger("fail")


with open(params['conf'] + config.get('Hive','categoryfile'),'r') as cate_file:
    categories = ','.join(["'"+cat.strip()+"'" for cat in cate_file])
with open(params['conf'] + config.get('Hive','sqlfile'),'r') as hive_file:
    hive_temp=Template(" ".join([line.strip() for line in hive_file]))

HIVE_QL = hive_temp.safe_substitute({"CATES":categories})

logger_hive.debug("HIVE_QL:")
logger_hive.debug(HIVE_QL)

result_file = params['conf'] + "sql" + PID + ".txt"
logger_hive.info("\tStart")

# first to download data file from hadoop


# status=os.system("touch "+LOADING_FILE)

# status=os.system("chmod a+rw "+LOADING_FILE)

# if status==0:
#     logger.info("chmod a+rw"+LOADING_FILE+" finished.")
# else:
#     logger.error("chmod a+rw"+LOADING_FILE+" failed.")
#     sys.exit(1)

status=os.system("su - hadoop-cdh -c \' hive -S -e \"" + HIVE_QL + \
                 "\" > "+ result_file+"\'")

if status==0:
    logger_hive.info("Write data from Hive into %s" % result_file)
else:
    logger_hive.error("Fail to donwload data from hive")
    logger_fail.error("Fail to donwload data from hive")

lines=os.popen("wc -l "+result_file,'r').readline()
if lines < MIN_ROW_SIZE:
    logger_hive.error("Number of items download from hive not enough")
    logger_fail.error("Number of items download from hive not enough")
    sys.exit(-1)
logger_hive.info("Number of items download from hive: %s" % lines)

if len(glob.glob(params['sqldata'] + '/*')) > 0:
    logger_hive.error("[sqldata] directory not empty")
{'fraction_non_alphanumeric': 0.21759530791788856, 'fraction_numerical': 0.002737047898338221, 'mean_word_length': 5.1862152357920195, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1824769', 'n_tokens_mistral': 1748, 'n_tokens_neox': 1715, 'n_words': 391}