<commit_before># Author : hdj <<EMAIL>>
# Date   : 2014.07
# Project: maaps
# Description:
#   TODO: add some description!
# 
import os, re, sys
import logging, logging.config
# -----------------------------------------------------------------------------
# import global shared definitions
parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if parentdir not in os.sys.path:
    os.sys.path.insert(0,parentdir)
from shared import *
# -----------------------------------------------------------------------------
# configure logger
logging.config.fileConfig("%s/../logging.conf" % os.path.dirname(__file__))


class PyCode(object):
    def __init__(self, name, filename, code):
        self.logger   = logging.getLogger('maaps.pycode')
        self.name     = name
        self.filename = filename
        # if the code is in a file dont touch it, but if it's inline-code then
        # we have to reformat it we need to remove leading withspaces (else the
        # interpreter would complain about unexpected indentation.
        self.logger.debug("creating code for name '%s' filename '%s' code '%r'", name, filename, code)
        if filename.startswith('inline'):
            self.code = PyCode.tidy_source_code(code)
        else:
            self.code = code

        # compile the code
        try:
            self._compiled = compile(self.code, self.name, 'exec')
        except Exception, e:
            self.logger.error("%s: compilation of %s failed.", self.name, self.filename)
            self.logger.error("%s: code is: [%r]", self.name, self.code)
            self.logger.exception("Exception is:")
            raise e

    @staticmethod
    def tidy_source_code(source_code):
        regex           = re.compile("^(\s+)[^\s]*")
        found_to_skip   = False
        pattern_to_skip = ""
        new_code        = []
        for line in source_code.split('\n'):
            if not found_to_skip:
                m = regex.match(line)
                if m:
                    pattern_to_skip   = "^%s" % (m.group(1),)
                    found_to_skip = True
            line = re.sub(pattern_to_skip, "", line)
            if len( line.strip() ) == 0:
               continue
            new_code.append( re.sub(pattern_to_skip, "", line) )
        return "\n".join(new_code,)

    def run(self, exception_queue, global_context, local_context=None):

        assert( CTX_PAYLOAD in global_context )

        import os, sys
        cwd = os.getcwd()
        if cwd not in sys.path:
            sys.path.append( os.getcwd() )
        save_logger = None
        if CTX_LOGGER not in global_context:
            save_logger = global_context.get(CTX_LOGGER, None)<commit_msg>Fix: Bug with comments. There was a bug with realigning the code block if there was a line starting with a '#'. The solution is to ignore comments while searching the right alignement.<commit_after># Author : hdj <<EMAIL>>
# Date   : 2014.07
# Project: maaps
# Description:
#   TODO: add some description!
# 
import os, re, sys
import logging, logging.config
# -----------------------------------------------------------------------------
# import global shared definitions
parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if parentdir not in os.sys.path:
    os.sys.path.insert(0,parentdir)
from shared import *
# -----------------------------------------------------------------------------
# configure logger
logging.config.fileConfig("%s/../logging.conf" % os.path.dirname(__file__))


class PyCode(object):
    def __init__(self, name, filename, code):
        self.logger   = logging.getLogger('maaps.pycode')
        self.name     = name
        self.filename = filename
        # if the code is in a file dont touch it, but if it's inline-code then
        # we have to reformat it we need to remove leading withspaces (else the
        # interpreter would complain about unexpected indentation.
        self.logger.debug("creating code for name '%s' filename '%s' code '%r'", name, filename, code)
        if filename.startswith('inline'):
            self.code = PyCode.tidy_source_code(code)
        else:
            self.code = code

        # compile the code
        try:
            self._compiled = compile(self.code, self.name, 'exec')
        except Exception, e:
            self.logger.error("%s: compilation of %s failed.", self.name, self.filename)
            self.logger.error("%s: code is: [%r]", self.name, self.code)
            self.logger.exception("Exception is:")
            raise e

    @staticmethod
    def tidy_source_code(source_code):
        regex           = re.compile("^(\s+)[^\s]*")
        comment_regex   = re.compile(r'^\s*#.*$')
        found_to_skip   = False
        pattern_to_skip = ""
        new_code        = []
        for line in source_code.split('\n'):
            if not found_to_skip:
                # if the whole line is acomment ignore it
                if comment_regex.match(line):
                    continue
                m = regex.match(line)
                if m:
                    pattern_to_skip   = "^%s" % (m.group(1),)
                    found_to_skip = True
            line = re.sub(pattern_to_skip, "", line)
            # skip empty lines
            if len( line.strip() ) == 0:
               continue
            new_code.append( re.sub(pattern_to_skip, "", line) )
        return "\n".join(new_code,)

    def run(self, exception_queue, global_context, local_context=None):

        assert( CTX_PAYLOAD in global_context )

        import os, sys
        cwd = os.getcwd()
        if cwd not in sys.path:
            sys.path.append( os.getcwd() )
        save_logger = None
        if CTX_LOGGER not in global_context:
            save_logger = global_context.get(CTX_LOGGER, None)
{'fraction_non_alphanumeric': 0.15946271741002238, 'fraction_numerical': 0.0030997072498708455, 'mean_word_length': 1.8180494905385736, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4358510', 'n_tokens_mistral': 1656, 'n_tokens_neox': 1611, 'n_words': 450}