<commit_before>#!/usr/bin/env python

#Copyright (C) 2013 by <NAME>
#
#Released under the MIT license, see LICENSE.txt
import sys
import os
import argparse
import subprocess
from multiprocessing import Pool
import logging
import resource
import logging.handlers
import collections
import numpy as np
import string
import random
from argparse import ArgumentParser
from optparse import OptionParser, OptionContainer, OptionGroup
import pybedtools

LOGZERO = -1e100
EPSILON = np.finfo(float).eps

def __myLogFloat(x):
    if np.abs(x) < EPSILON:
        return LOGZERO
    return np.log(x)

""" Replace np.log to accept zero """
myLog = np.vectorize(__myLogFloat)
    
def runShellCommand(command):
    try:
        logger.info("Running %s" % command)
        process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE,
                                   stderr=sys.stderr, bufsize=-1)
        output, nothing = process.communicate()
        sts = process.wait()
        if sts != 0:
            raise RuntimeError("Command: %s exited with non-zero status %i" %
                               (command, sts))
        return output
    except KeyboardInterrupt:
        raise RuntimeError("Aborting %s" % command)

def runParallelShellCommands(cmdList, numProc):
    if numProc == 1 or len(cmdList) == 1:
        map(runShellCommand, cmdList)
    elif len(cmdList) > 0:
        mpPool = Pool(processes=min(numProc, len(cmdList)))
        result = mpPool.map_async(runShellCommand, cmdList)
        # specifying a timeout allows keyboard interrupts to work?!
        # http://stackoverflow.com/questions/1408356/keyboard-interrupts-with-pythons-multiprocessing-pool
        try:
            result.get(sys.maxint)
        except KeyboardInterrupt:
            mpPool.terminate()
            raise RuntimeError("Keyboard interrupt")
        if not result.successful():
            raise "One or more of commands %s failed" % str(cmdList)

def getLocalTempPath(prefix="", extension="", tagLen=5):
    S = string.ascii_uppercase + string.digits
    tag = ''.join(random.choice(S) for x in range(tagLen))
    tempPath = os.path.join(os.getcwd(), "%s%s%s" % (prefix, tag, extension))
    return tempPath
        
def initBedTool(tempPrefix=""):
    # keep temporary files in current directory, to make it a little harder to
    # lose track of them and clog up the system....
    S = string.ascii_uppercase + string.digits
    tag = ''.join(random.choice(S) for x in range(5))
    tempPath = os.path.join(os.getcwd(), "%sTempBedTool_%s" % (tempPrefix, tag))
    logger.info("Temporary directory for BedTools (you may need to manually"
                 " erase in event of crash): %s" % tempPath)
    return tempPath

def cleanBedTool(tempPath):
    # do best to erase temporary bedtool files if necessary
    # (tempPath argument must have been created with initBedTool())
    assert "TempBedTool_" in tempPath
    pybedtools.cleanup(remove_all=True)
    runShellCommand("rm -rf %s" % tempPath)

        
#########################################################<commit_msg>fix initbedtool<commit_after>#!/usr/bin/env python

#Copyright (C) 2013 by <NAME>
#
#Released under the MIT license, see LICENSE.txt
import sys
import os
import argparse
import subprocess
from multiprocessing import Pool
import logging
import resource
import logging.handlers
import collections
import numpy as np
import string
import random
from argparse import ArgumentParser
from optparse import OptionParser, OptionContainer, OptionGroup
import pybedtools

LOGZERO = -1e100
EPSILON = np.finfo(float).eps

def __myLogFloat(x):
    if np.abs(x) < EPSILON:
        return LOGZERO
    return np.log(x)

""" Replace np.log to accept zero """
myLog = np.vectorize(__myLogFloat)
    
def runShellCommand(command):
    try:
        logger.info("Running %s" % command)
        process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE,
                                   stderr=sys.stderr, bufsize=-1)
        output, nothing = process.communicate()
        sts = process.wait()
        if sts != 0:
            raise RuntimeError("Command: %s exited with non-zero status %i" %
                               (command, sts))
        return output
    except KeyboardInterrupt:
        raise RuntimeError("Aborting %s" % command)

def runParallelShellCommands(cmdList, numProc):
    if numProc == 1 or len(cmdList) == 1:
        map(runShellCommand, cmdList)
    elif len(cmdList) > 0:
        mpPool = Pool(processes=min(numProc, len(cmdList)))
        result = mpPool.map_async(runShellCommand, cmdList)
        # specifying a timeout allows keyboard interrupts to work?!
        # http://stackoverflow.com/questions/1408356/keyboard-interrupts-with-pythons-multiprocessing-pool
        try:
            result.get(sys.maxint)
        except KeyboardInterrupt:
            mpPool.terminate()
            raise RuntimeError("Keyboard interrupt")
        if not result.successful():
            raise "One or more of commands %s failed" % str(cmdList)

def getLocalTempPath(prefix="", extension="", tagLen=5):
    S = string.ascii_uppercase + string.digits
    tag = ''.join(random.choice(S) for x in range(tagLen))
    tempPath = os.path.join(os.getcwd(), "%s%s%s" % (prefix, tag, extension))
    return tempPath
        
def initBedTool(tempPrefix=""):
    # keep temporary files in current directory, to make it a little harder to
    # lose track of them and clog up the system....
    S = string.ascii_uppercase + string.digits
    tag = ''.join(random.choice(S) for x in range(5))
    tempPath = os.path.join(os.getcwd(), "%sTempBedTool_%s" % (tempPrefix, tag))
    logger.info("Temporary directory for BedTools (you may need to manually"
                 " erase in event of crash): %s" % tempPath)
    try:
        os.makedirs(tempPath)
    except:
        pass
    pybedtools.set_tempdir(tempPath)
    return tempPath

def cleanBedTool(tempPath):
    # do best to erase temporary bedtool files if necessary
    # (tempPath argument must have been created with initBedTool())
    assert "TempBedTool_" in tempPath
    pybedtools.cleanup(remove_all=True)
    runShellCommand("rm -rf %s" % tempPath)

        
#########################################################
{'fraction_non_alphanumeric': 0.11603004634809014, 'fraction_numerical': 0.007032124021096372, 'mean_word_length': 3.060999351070733, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2576814', 'n_tokens_mistral': 1882, 'n_tokens_neox': 1819, 'n_words': 538}