<commit_before>import sys, os, fnmatch
import copy
import genUtils
import modules
import mutex
from random import choice

class metaModule:
    """Class used to store module-related information.
    """
    
    def __init__(self, instance, instanceName):
        """Instance is the actual class instance and instanceName is a unique
        name that has been chosen by the user or automatically.
        """
        self.instance = instance        
        self.instanceName = instanceName
        self.resetInputsOutputs()

    def close(self):
        del self.instance
        del self.inputs
        del self.outputs

    def resetInputsOutputs(self):
        numIns = len(self.instance.getInputDescriptions())
        numOuts = len(self.instance.getOutputDescriptions())
        # numIns list of tuples of (supplierModule, supplierOutputIdx)
        self.inputs = [None] * numIns
        # numOuts list of lists of tuples of (consumerModule, consumerInputIdx)
        # be careful with list concatenation, it makes copies, which are mostly
        # shallow!!!
        self.outputs = [[] for _ in range(numOuts)]

class pickledModuleState:
    def __init__(self):
        self.moduleConfig = None
        self.moduleName = None
        self.instanceName = None

class pickledConnection:
    def __init__(self, sourceInstanceName=None, outputIdx=None,
                 targetInstanceName=None, inputIdx=None, connectionType=None):
        
        self.sourceInstanceName = sourceInstanceName
        self.outputIdx = outputIdx
        self.targetInstanceName = targetInstanceName
        self.inputIdx = inputIdx
        self.connectionType = connectionType

class moduleManager:
    """This class in responsible for picking up new modules in the modules 
    directory and making them available to the rest of the program."""
    
    def __init__(self, dscas3_app):
	"""Initialise module manager by fishing .py dscas3 modules from
	all pertinent directories."""
	
        self._dscas3_app = dscas3_app
        # module dictionary, keyed on instance... cool.
        # values are metaModules
	self._moduleDict = {}

        appdir = self._dscas3_app.get_appdir()
        self._modules_dir = os.path.join(appdir, 'modules')
        self._userModules_dir = os.path.join(appdir, 'userModules')
        
	# make first scan of available modules
	self.scanModules()

        # this is a list of modules that have the ability to start a network
        # executing all by themselves and usually do... when we break down
        # a network, we should take these out first.  when we build a network
        # we should put them down last
        self.dangerousConsumerModules = ['slice3dVWR']

        # we'll use this to perform mutex-based locking on the progress
        # callback... (there SHOULD only be ONE moduleManager instance)
        self._inVTKProgressCallback = mutex.mutex()

    def close(self):
        """Iterates through each module and closes it.

        This is only called during dscas3 application shutdown.
        """

        # this is fine because .items() makes a copy of the dict
        for mModule in self._moduleDict.items():
            try:
                self.deleteModule(mModule.instance)
            except:
                # we can't allow a module to stop us
                pass

    def scanModules(self):
	"""(Re)Check the modules directory for *.py files and put them in
	the list self.module_files."""
        self._availableModuleList = []

        def recursiveDirectoryD3MNSearch(adir, curModulePath, mnList):
            """Iterate recursively starting at adir and make a list of
            all available modules and networks.  We do not traverse into dirs
            that are named 'resources' or that end with 'modules'.
            """

            if curModulePath:
                wildCard = "*.py"
            else:
                wildCard = "*.d3n"
                
            fileNames = os.listdir(adir)
            for fileName in fileNames:
                completeName = os.path.join(adir, fileName)
                if os.path.isdir(completeName) and \
                       fileName.strip('/') != 'resources' and \
                       not fileName.strip('/').lower().endswith('modules'):
                    # fileName is just a directory name then
                    # make sure it has no /'s at the end and append
                    # it to the curModulePath when recursing
                    newCurModulePath = None
                    if curModulePath:
                        newCurModulePath = curModulePath + '.' + \
                                           fileName.strip('/')

                    recursiveDirectoryD3MNSearch(
                        completeName,
                        newCurModulePath,
                        mnList)

                elif os.path.isfile(completeName) and \
                         fnmatch.fnmatch(fileName, wildCard) and \
                         not fnmatch.fnmatch(fileName, "_*"):
                    if curModulePath:
                        mnList.append(
                            "%s.%s" % (curModulePath,
                                       os.path.splitext(fileName)[0]))
                    else:
                        mnList.append(completeName)

        appDir = self._dscas3_app.get_appdir()
        userModuleList = []
        recursiveDirectoryD3MNSearch(os.path.join(appDir,
                                                    'userModules'),
                                       'userModules', userModuleList)


        # first add the core modules to our central list
        for mn in modules.moduleList:
            self._availableModuleList.append('modules.%s' % (mn,))

        # then all the user modules
        self._availableModuleList += userModuleList

        networkSegmentList = []
        recursiveDirectoryD3MNSearch(os.path.join(appDir, 'networkSegments'),
                                     None, networkSegmentList)
        self.availableSegmentsList = networkSegmentList

    def get_app_dir(self):
        return self._dscas3_app.get_appdir()
	
    def getAvailableModuleList(self):
	return self._availableModuleList

    def getInstanceName(self, instance):
        """Given the actual instance, return its unique instance.  If the
        instance doesn't exist in self._moduleDict, return the currently
        halfborn instance.
        """

        try:
            return self._moduleDict[instance].instanceName
        except Exception:<commit_msg>Changed directory name from networkSegments to segments.<commit_after>import sys, os, fnmatch
import copy
import genUtils
import modules
import mutex
from random import choice

class metaModule:
    """Class used to store module-related information.
    """
    
    def __init__(self, instance, instanceName):
        """Instance is the actual class instance and instanceName is a unique
        name that has been chosen by the user or automatically.
        """
        self.instance = instance        
        self.instanceName = instanceName
        self.resetInputsOutputs()

    def close(self):
        del self.instance
        del self.inputs
        del self.outputs

    def resetInputsOutputs(self):
        numIns = len(self.instance.getInputDescriptions())
        numOuts = len(self.instance.getOutputDescriptions())
        # numIns list of tuples of (supplierModule, supplierOutputIdx)
        self.inputs = [None] * numIns
        # numOuts list of lists of tuples of (consumerModule, consumerInputIdx)
        # be careful with list concatenation, it makes copies, which are mostly
        # shallow!!!
        self.outputs = [[] for _ in range(numOuts)]

class pickledModuleState:
    def __init__(self):
        self.moduleConfig = None
        self.moduleName = None
        self.instanceName = None

class pickledConnection:
    def __init__(self, sourceInstanceName=None, outputIdx=None,
                 targetInstanceName=None, inputIdx=None, connectionType=None):
        
        self.sourceInstanceName = sourceInstanceName
        self.outputIdx = outputIdx
        self.targetInstanceName = targetInstanceName
        self.inputIdx = inputIdx
        self.connectionType = connectionType

class moduleManager:
    """This class in responsible for picking up new modules in the modules 
    directory and making them available to the rest of the program."""
    
    def __init__(self, dscas3_app):
	"""Initialise module manager by fishing .py dscas3 modules from
	all pertinent directories."""
	
        self._dscas3_app = dscas3_app
        # module dictionary, keyed on instance... cool.
        # values are metaModules
	self._moduleDict = {}

        appdir = self._dscas3_app.get_appdir()
        self._modules_dir = os.path.join(appdir, 'modules')
        self._userModules_dir = os.path.join(appdir, 'userModules')
        
	# make first scan of available modules
	self.scanModules()

        # this is a list of modules that have the ability to start a network
        # executing all by themselves and usually do... when we break down
        # a network, we should take these out first.  when we build a network
        # we should put them down last
        self.dangerousConsumerModules = ['slice3dVWR']

        # we'll use this to perform mutex-based locking on the progress
        # callback... (there SHOULD only be ONE moduleManager instance)
        self._inVTKProgressCallback = mutex.mutex()

    def close(self):
        """Iterates through each module and closes it.

        This is only called during dscas3 application shutdown.
        """

        # this is fine because .items() makes a copy of the dict
        for mModule in self._moduleDict.items():
            try:
                self.deleteModule(mModule.instance)
            except:
                # we can't allow a module to stop us
                pass

    def scanModules(self):
	"""(Re)Check the modules directory for *.py files and put them in
	the list self.module_files."""
        self._availableModuleList = []

        def recursiveDirectoryD3MNSearch(adir, curModulePath, mnList):
            """Iterate recursively starting at adir and make a list of
            all available modules and networks.  We do not traverse into dirs
            that are named 'resources' or that end with 'modules'.
            """

            if curModulePath:
                wildCard = "*.py"
            else:
                wildCard = "*.d3n"
                
            fileNames = os.listdir(adir)
            for fileName in fileNames:
                completeName = os.path.join(adir, fileName)
                if os.path.isdir(completeName) and \
                       fileName.strip('/') != 'resources' and \
                       not fileName.strip('/').lower().endswith('modules'):
                    # fileName is just a directory name then
                    # make sure it has no /'s at the end and append
                    # it to the curModulePath when recursing
                    newCurModulePath = None
                    if curModulePath:
                        newCurModulePath = curModulePath + '.' + \
                                           fileName.strip('/')

                    recursiveDirectoryD3MNSearch(
                        completeName,
                        newCurModulePath,
                        mnList)

                elif os.path.isfile(completeName) and \
                         fnmatch.fnmatch(fileName, wildCard) and \
                         not fnmatch.fnmatch(fileName, "_*"):
                    if curModulePath:
                        mnList.append(
                            "%s.%s" % (curModulePath,
                                       os.path.splitext(fileName)[0]))
                    else:
                        mnList.append(completeName)

        appDir = self._dscas3_app.get_appdir()
        userModuleList = []
        recursiveDirectoryD3MNSearch(os.path.join(appDir,
                                                    'userModules'),
                                       'userModules', userModuleList)


        # first add the core modules to our central list
        for mn in modules.moduleList:
            self._availableModuleList.append('modules.%s' % (mn,))

        # then all the user modules
        self._availableModuleList += userModuleList

        # we should move this functionality to the graphEditor.  "segments"
        # are _probably_ only valid there... alternatively, we should move
        # the concept here
        segmentList = []
        recursiveDirectoryD3MNSearch(os.path.join(appDir, 'segments'),
                                     None, segmentList)
        self.availableSegmentsList = segmentList

    def get_app_dir(self):
        return self._dscas3_app.get_appdir()
	
    def getAvailableModuleList(self):
	return self._availableModuleList

    def getInstanceName(self, instance):
        """Given the actual instance, return its unique instance.  If the
        instance doesn't exist in self._moduleDict, return the currently
        halfborn instance.
        """

        try:
            return self._moduleDict[instance].instanceName
        except Exception:
{'fraction_non_alphanumeric': 0.07310448887212373, 'fraction_numerical': 0.002263296869105998, 'mean_word_length': 1.8636854612227263, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1497986', 'n_tokens_mistral': 3347, 'n_tokens_neox': 3228, 'n_words': 1040}