<commit_before># Copyright (c) <NAME>, TU Delft.
# All rights reserved.
# See COPYRIGHT for details.

import getopt
import os
import re
import shutil
import sys

###################################################################
# the following programmes should either be on your path, or you
# should specify the full paths here.
# Microsoft utility to rebase files.
REBASE = "rebase"

# end of programmes ###############################################

PPF = "[*** DeVIDE make_dist ***]"
S_PPF = "%s =====>>>" % (PPF,) # used for stage headers

S_CLEAN_PYI = 'clean_pyi'
S_RUN_PYI = 'run_pyi'
S_WRAPITK_TREE = 'wrapitk_tree'
S_REBASE_DLLS = 'rebase_dlls'
S_PACKAGE_DIST = 'package_dist'
DEFAULT_STAGES = '%s, %s, %s, %s, %s' % \
        (S_CLEAN_PYI, S_RUN_PYI, S_WRAPITK_TREE, 
                S_REBASE_DLLS, S_PACKAGE_DIST)





####################################################################
class MDPaths:
    """Initialise all directories required for building DeVIDE
    distributables.
    """
    def __init__(self, specfile, pyinstaller_script):
        self.specfile = os.path.normpath(specfile)
        self.pyinstaller_script = os.path.normpath(pyinstaller_script)

        # devide/installer
        self.specfile_dir = os.path.normpath(
                os.path.abspath(os.path.dirname(self.specfile)))

        self.pyi_dist_dir = os.path.join(self.specfile_dir,
                'distdevide')
        self.pyi_build_dir = os.path.join(self.specfile_dir,
                'builddevide')

        # devide
        self.devide_dir = \
            os.path.normpath(
                    os.path.join(self.specfile_dir, '..'))


####################################################################
# UTILITY METHODS
####################################################################
def get_status_output(command):
    """Run command, return output of command and exit code in status.
    In general, status is None for success and 1 for command not
    found.

    Method taken from johannes.utils.
    """

    ph = os.popen(command)
    output = ph.read()
    status = ph.close()
    return (status, output)


def find_command_with_ver(name, command, ver_re):
    """Try to run command, use ver_re regular expression to parse for
    the version string.  This will print for example:
    CVS: version 2.11 found.

    @return: True if command found, False if not or if version could
    not be parsed. 

    Method taken from johannes.utils.
    """

    retval = False
    s,o = get_status_output(command)

    if s:
        msg2 = 'NOT FOUND!'

    else:
        mo = re.search(ver_re, o, re.MULTILINE) 
        if mo:
            msg2 = 'version %s found.' % (mo.groups()[0],)
            retval = True
        else:
            msg2 = 'could not extract version.'


    print PPF, "%s: %s" % (name, msg2)

    return retval

def find_files(start_dir, re_pattern='.*\.(pyd|dll)'):
    """Recursively find all files (not directories) with filenames 
    matching given regular expression.  Case is ignored.

    @param start_dir: search starts in this directory
    @param re_pattern: regular expression with which all found files
    will be matched. example: re_pattern = '.*\.(pyd|dll)' will match
    all filenames ending in pyd or dll.
    @return: list of fully qualified filenames that satisfy the
    pattern
    """

    cpat = re.compile(re_pattern, re.IGNORECASE)
    found_files = []

    for dirpath, dirnames, filenames in os.walk(start_dir):
        ndirpath = os.path.normpath(os.path.abspath(dirpath))
        for fn in filenames:
            if cpat.match(fn):
                found_files.append(os.path.join(ndirpath,fn))

    return found_files

####################################################################
# METHODS CALLED FROM MAIN()
####################################################################
def usage():
    message = """
make_dist.py - build DeVIDE distributables.


Invoke as follows:

python make_dist.py -s specfile -i installer_script

where specfile is the pyinstaller spec file and installer_script
refers to the full path of the pyinstaller Build.py 

The specfile should be in the directory devide/installer, where devide
is the directory containing the devide source that you are using to
build the distributables.

Other switches:
--stages : by default all stages are run.  With this parameter, a
           subset of the stages can be specified.  The full list is: 
           %s
""" % (DEFAULT_STAGES,)

    print message

def clean_pyinstaller(md_paths):
    """Clean out pyinstaller dist and build directories so that it has
    to do everything from scratch.  We usually do this before building
    full release versions.
    """

    print S_PPF, "clean_pyinstaller"
    
    if os.path.isdir(md_paths.pyi_dist_dir):
        print PPF, "Removing distdevide..."
        shutil.rmtree(md_paths.pyi_dist_dir)

    if os.path.isdir(md_paths.pyi_build_dir):
        print PPF, "Removing builddevide..."
        shutil.rmtree(md_paths.pyi_build_dir)

def run_pyinstaller(md_paths):
    """Run pyinstaller with the given parameters.  This does not clean
    out the dist and build directories before it begins<commit_msg>Minor changes to make structure clearer.<commit_after># Copyright (c) <NAME>, TU Delft.
# All rights reserved.
# See COPYRIGHT for details.

###################################################################
# the following programmes should either be on your path, or you
# should specify the full paths here.

# Microsoft utility to rebase files.
REBASE = "rebase"

# end of programmes ###############################################



# NOTHING TO CONFIGURE BELOW THIS LINE ############################

import getopt
import os
import re
import shutil
import sys


PPF = "[*** DeVIDE make_dist ***]"
S_PPF = "%s =====>>>" % (PPF,) # used for stage headers

S_CLEAN_PYI = 'clean_pyi'
S_RUN_PYI = 'run_pyi'
S_WRAPITK_TREE = 'wrapitk_tree'
S_REBASE_DLLS = 'rebase_dlls'
S_PACKAGE_DIST = 'package_dist'
DEFAULT_STAGES = '%s, %s, %s, %s, %s' % \
        (S_CLEAN_PYI, S_RUN_PYI, S_WRAPITK_TREE, 
                S_REBASE_DLLS, S_PACKAGE_DIST)

HELP_MESSAGE = """
make_dist.py - build DeVIDE distributables.


Invoke as follows:

python make_dist.py -s specfile -i installer_script

where specfile is the pyinstaller spec file and installer_script
refers to the full path of the pyinstaller Build.py 

The specfile should be in the directory devide/installer, where devide
is the directory containing the devide source that you are using to
build the distributables.

Other switches:
--stages : by default all stages are run.  With this parameter, a
           subset of the stages can be specified.  The full list is: 
           %s
""" % (DEFAULT_STAGES,)


####################################################################
class MDPaths:
    """Initialise all directories required for building DeVIDE
    distributables.
    """
    def __init__(self, specfile, pyinstaller_script):
        self.specfile = os.path.normpath(specfile)
        self.pyinstaller_script = os.path.normpath(pyinstaller_script)

        # devide/installer
        self.specfile_dir = os.path.normpath(
                os.path.abspath(os.path.dirname(self.specfile)))

        self.pyi_dist_dir = os.path.join(self.specfile_dir,
                'distdevide')
        self.pyi_build_dir = os.path.join(self.specfile_dir,
                'builddevide')

        # devide
        self.devide_dir = \
            os.path.normpath(
                    os.path.join(self.specfile_dir, '..'))


####################################################################
# UTILITY METHODS
####################################################################
def get_status_output(command):
    """Run command, return output of command and exit code in status.
    In general, status is None for success and 1 for command not
    found.

    Method taken from johannes.utils.
    """

    ph = os.popen(command)
    output = ph.read()
    status = ph.close()
    return (status, output)


def find_command_with_ver(name, command, ver_re):
    """Try to run command, use ver_re regular expression to parse for
    the version string.  This will print for example:
    CVS: version 2.11 found.

    @return: True if command found, False if not or if version could
    not be parsed. 

    Method taken from johannes.utils.
    """

    retval = False
    s,o = get_status_output(command)

    if s:
        msg2 = 'NOT FOUND!'

    else:
        mo = re.search(ver_re, o, re.MULTILINE) 
        if mo:
            msg2 = 'version %s found.' % (mo.groups()[0],)
            retval = True
        else:
            msg2 = 'could not extract version.'


    print PPF, "%s: %s" % (name, msg2)

    return retval

def find_files(start_dir, re_pattern='.*\.(pyd|dll)'):
    """Recursively find all files (not directories) with filenames 
    matching given regular expression.  Case is ignored.

    @param start_dir: search starts in this directory
    @param re_pattern: regular expression with which all found files
    will be matched. example: re_pattern = '.*\.(pyd|dll)' will match
    all filenames ending in pyd or dll.
    @return: list of fully qualified filenames that satisfy the
    pattern
    """

    cpat = re.compile(re_pattern, re.IGNORECASE)
    found_files = []

    for dirpath, dirnames, filenames in os.walk(start_dir):
        ndirpath = os.path.normpath(os.path.abspath(dirpath))
        for fn in filenames:
            if cpat.match(fn):
                found_files.append(os.path.join(ndirpath,fn))

    return found_files

####################################################################
# METHODS CALLED FROM MAIN()
####################################################################
def usage():

    print HELP_MESSAGE

def clean_pyinstaller(md_paths):
    """Clean out pyinstaller dist and build directories so that it has
    to do everything from scratch.  We usually do this before building
    full release versions.
    """

    print S_PPF, "clean_pyinstaller"
    
    if os.path.isdir(md_paths.pyi_dist_dir):
        print PPF, "Removing distdevide..."
        shutil.rmtree(md_paths.pyi_dist_dir)

    if os.path.isdir(md_paths.pyi_build_dir):
        print PPF, "Removing builddevide..."
        shutil.rmtree(md_paths.pyi_build_dir)

def run_pyinstaller(md_paths):
    """Run pyinstaller with the given parameters.  This does not clean
    out the dist and build directories before it begins
{'fraction_non_alphanumeric': 0.17317584301771766, 'fraction_numerical': 0.0017146123071061154, 'mean_word_length': 3.1895450917797286, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5249253', 'n_tokens_mistral': 3246, 'n_tokens_neox': 3080, 'n_words': 956}