<commit_before># This module is responsible for producing the pre-processed form of each file
# modified by a given edit.
import os
import subprocess
import utility as util

# Compiles the source code for a project, together with its pre-processed
# files.
#
# We need to run a whole bunch of things on our Docker instance, then save
# it using the Storage handler.
class Preprocessor(object):
    def __init__(self, threads = 8):
        self.__threads = threads
        self.__script = os.path.dirname(os.path.realpath(__file__))

    def preprocess(self, version):
        repo = version.repository().repository()

        # Find the commit ID for this program version
        fix = version.fix()
        commit = version.identifier()
        if version.is_faulty():
            commit = "%s~1" % commit

        # Switch the repository's preprocessing branch to point at the
        # correct commit for this program version.
        current_branch_name = repo.active_branch_name
        try:         
            repo.git.reset('--hard')
            repo.git.checkout(commit, b='preprocessing')

            # Compute and execute the pre-process command
            cmd = os.path.join(os.path.dirname(os.path.realpath(__file__)),\
                               "../lib/preprocess")
            cmd  = "%s '%s'" % repo.working_dir
            with open(os.devnull, "w") as f:
                subprocess.call(cmd, stdout=f)

        # Ensure the repository is returned to its original state, prior to
        # pre-processing
        finally:
            repo.git.clean('-f', '-x')
            repo.git.reset('--hard')
            repo.git.checkout(current_branch_name)
            repo.git.branch('-D', 'preprocessing')

        # Save each of the modified source files to storage
        for fn in fix.modified_source_files():
            pass
            # shutil.copyfile(cp_from, cp_to)
<commit_msg>more work on preprocessed file extraction<commit_after># This module is responsible for producing the pre-processed form of each file
# modified by a given edit.
import os
import subprocess
import utility as util

# Compiles the source code for a project, together with its pre-processed
# files.
#
# We need to run a whole bunch of things on our Docker instance, then save
# it using the Storage handler.
class Preprocessor(object):
    def __init__(self, threads = 8):
        self.__threads = threads

    def preprocess(self, version):
        repo = version.repository().repository()

        # Find the commit ID for this program version
        fix = version.fix()
        commit = version.identifier()
        if version.is_faulty():
            commit = "%s~1" % commit

        # Switch the repository's preprocessing branch to point at the
        # correct commit for this program version.
        current_branch_name = repo.active_branch_name
        try:         
            repo.git.reset('--hard')
            repo.git.checkout(commit, b='preprocessing')

            # Compute and execute the pre-process command
            cmd = os.path.join(os.path.dirname(os.path.realpath(__file__)),\
                               "../lib/preprocess")
            cmd  = "%s '%s'" % repo.working_dir
            with open(os.devnull, "w") as f:
                subprocess.call(cmd, stdout=f)

        # Ensure the repository is returned to its original state, prior to
        # pre-processing
        finally:
            repo.git.clean('-f', '-x')
            repo.git.reset('--hard')
            repo.git.checkout(current_branch_name)
            repo.git.branch('-D', 'preprocessing')

        # Save each of the modified source files to storage
        for fn in fix.modified_source_files():

            # Find where the file resides. For some projects, it will be in the
            # same directory as the original source code file; for others, it
            # may reside at the root of the repository.
            pp_fn = '%s.i' % fn[:-2]
            if os.path.isfile(os.path.join(repo.working_dir,\
                                           os.path.basename(pp_fn))):
                cp_from = os.path.join(repo.working_dir, os.path.basename(pp_fn)) 
            elif os.path.isfile(os.path.join(repo.working_dir, pp_fn)):
                cp_from = os.path.join(repo.working_dir, pp_fn)
            else:
                cp_from = None

            # shutil.copyfile(cp_from, cp_to)

{'fraction_non_alphanumeric': 0.09496826835902085, 'fraction_numerical': 0.0011332728921124207, 'mean_word_length': 2.056094182825485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6051175', 'n_tokens_mistral': 1204, 'n_tokens_neox': 1171, 'n_words': 384}