<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- 

import os,sys,re,shutil,time
import webbrowser,threading
from RunSubprocess import RunSubprocess
from SphinxLogger import SphinxLogger
from Logging import Logger
from Controller import Controller


class NoGuiAnalysis():

    def __init__(self,controller=None,mainWindow=None):
      
        ## input variables
        if controller == None:
            self.controller = Controller()
        else:
            self.controller = controller
        self.mainWindow = mainWindow
        self.validLanguages = ['r','python']
        self.sphinxLog = None
        self.goFlag = False
        self.reload_program_paths()
        
    def reload_program_paths(self):
        """
        function to update the paths to programs for system calls
        """
        
        ## ensure that the log file is loaded
        if len(self.controller.log.log.items()) < 1:
           self.controller.log = Logger() 

        self.rPath = self.controller.get_r_path(mainWindow=self.mainWindow)
        self.pythonPath = self.controller.get_python_path(mainWindow=self.mainWindow)
        self.sphinxPath = self.controller.get_sphinx_path(mainWindow=self.mainWindow)
        self.latexPath = self.controller.get_latex_path()
        self.parsePath = os.path.join(self.controller.baseDir,"ParseEmbedded.py")
        self.latex2htmlPath = self.controller.get_latex2html_path()

    def build(self,fileName=None,verbose=False):
        """
        generic command to build a rst,nw or rnw document
        """

        ## variables
        self.goFlag = False
        if self.mainWindow != None:
            self.mainWindow.ensure_tab_is_current()
        
        ## if a fileName is specified the set the current index
        if fileName != None and fileName in self.controller.fileNameList:
            currentIdx = self.controller.fileNameList.index(fileName)
            self.controller.currentFileIndex = currentIdx
        
        currentIdx = self.controller.currentFileIndex

        filePath = self.controller.filePathList[currentIdx]
        fileName = self.controller.fileNameList[currentIdx]
        fileLanguage = self.controller.fileLangList[currentIdx]
        self.reload_program_paths()
        goFlag = None

        if self.mainWindow != None:
            editor = self.mainWindow.controller.editorList[currentIdx] 
            editor.clear_messages()
        
        self.output_text("BUILDING... %s"%fileName)

        ## error checking
        isClean = self.controller.sanitize_check(filePath)
        if isClean == False:
            msg = "Invalid file/dir name\n"
            msg += "%s\nremove characters like '$','#' and '&' then try again"%(filePath) 
            self.display_error(msg)
            return

        isActive = self.is_active_project(filePath)
        if not isActive:
            return
        isTemplate = self.is_template()
        if isTemplate == True:
            return

        fileLanguage = fileLanguage.lower()
        if fileLanguage not in self.validLanguages:
            msg = "Valid languages are %s not %s"%(self.validLanguages,fileLanguage)
            self.display_error(msg)
            return

        ## save file
        if self.mainWindow != None:
            self.mainWindow.file_save(display=False)

        ## run the appropriate builder
        if re.search("\.rst",fileName,flags=re.IGNORECASE):
            self.build_rst(verbose=verbose)
        elif re.search("\.rnw|\.nw",fileName,flags=re.IGNORECASE):
            self.build_nw(verbose=verbose)
        else:
            msg = "Invalid file name extension\n"
            msg +="...lpEdit cannot build\n"
            msg +=filePath
            self.display_error(msg)

    def load_file(self,filePath,fileLang=None):
        '''
        use the controller to load a specified file
        '''

        filePath = os.path.realpath(filePath)

        if os.path.exists(filePath) == False:
            print("ERROR: invalid file path specified for load")
            print("... %s"%filePath)
            return
        
        isValid = self.controller.load_file(filePath,fileLang=fileLang)
        if isValid == False:
            print "ERROR: controller could not load specified file"
            print "\t", filePath

    def clean_file(self,filePath,verbose=False):
        '''
        cleans a directory of its files -- function used with unittests
        '''
        
        self.controller.clean_file(filePath)

    def output_text(self,txt):
        currentIdx = self.controller.currentFileIndex
        
        if self.mainWindow != None:
            if self.mainWindow.controller.editorList[currentIdx].textScreen.showMessages == False:
                self.mainWindow.controller.editorList[currentIdx].textScreen.toggle_message_btn()
            self.mainWindow.controller.editorList[currentIdx].textScreen.add_text(txt)
        else:
            print txt

    def copy_included_files(self,filePath,dirPath):
        """
        Copy files that are included with the keyword INCLUDE
        The source file path is given and it is scanned for INCLUDE
        Each include may be multiple file paths (comma delim)
        """
        
        baseDir = os.path.split(filePath)[0]
        
        ## adjust for subdirectories
        if os.path.split(baseDir)[1] != os.path.split(os.path.split(dirPath)[0])[1]:
            subdir = os.path.split(baseDir)[1]
            targetDir = os.path.join(dirPath,subdir)
        else:
            targetDir = dirPath
            subdir = None

        fid = open(filePath,'r')
        includedFiles = []
        found = False
        for linja in fid:
            if re.search("^%|^\.\. ",linja) and re.search("INCLUDE",linja):
                linja = re.sub("INCLUDE[\W|\w]","",linja)
                linja = re.sub("\%|\s+|^\.\.","",linja)
                found = True
                linja = re.split(",",linja)
                for includedFile in linja:
                    includedFilePath = os.path.join(baseDir,includedFile)
                    includedFiles.append(os.path.realpath(includedFilePath))

        for includedFilePath in includedFiles:
            includedFilePath = includedFilePath
            includedFileName = os.path.split(includedFilePath)[-1]
            includedFileDir = os.path.split(includedFilePath)[-1]
            newFilePath = os.path.join(targetDir,includedFileName)

            if not os.path.exists(includedFilePath):
                msg = "Invalid INCLUDE path...\n"         
                msg += includedFilePath
                if self.mainWindow != None:
                    self.mainWindow.display_warning(msg)
                else:
                    print msg
                return False

            if os.path.islink(newFilePath):
                os.remove(newFilePath)
            if os.path.exists(newFilePath):
                os.remove(newFilePath)
            os.symlink(includedFilePath,newFilePath)

        return True

    def is_active_project(self,currentFilePath):<commit_msg>added print line<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- 

import os,sys,re,shutil,time
import webbrowser,threading
from RunSubprocess import RunSubprocess
from SphinxLogger import SphinxLogger
from Logging import Logger
from Controller import Controller


class NoGuiAnalysis():

    def __init__(self,controller=None,mainWindow=None):
      
        ## input variables
        if controller == None:
            self.controller = Controller()
        else:
            self.controller = controller
        self.mainWindow = mainWindow
        self.validLanguages = ['r','python']
        self.sphinxLog = None
        self.goFlag = False
        self.reload_program_paths()
        
    def reload_program_paths(self):
        """
        function to update the paths to programs for system calls
        """
        
        ## ensure that the log file is loaded
        if len(self.controller.log.log.items()) < 1:
           self.controller.log = Logger() 

        self.rPath = self.controller.get_r_path(mainWindow=self.mainWindow)
        self.pythonPath = self.controller.get_python_path(mainWindow=self.mainWindow)
        self.sphinxPath = self.controller.get_sphinx_path(mainWindow=self.mainWindow)
        self.latexPath = self.controller.get_latex_path()
        self.parsePath = os.path.join(self.controller.baseDir,"ParseEmbedded.py")
        self.latex2htmlPath = self.controller.get_latex2html_path()

    def build(self,fileName=None,verbose=False):
        """
        generic command to build a rst,nw or rnw document
        """

        ## variables
        self.goFlag = False
        if self.mainWindow != None:
            self.mainWindow.ensure_tab_is_current()
        
        ## if a fileName is specified the set the current index
        if fileName != None and fileName in self.controller.fileNameList:
            currentIdx = self.controller.fileNameList.index(fileName)
            self.controller.currentFileIndex = currentIdx
        
        currentIdx = self.controller.currentFileIndex

        filePath = self.controller.filePathList[currentIdx]
        fileName = self.controller.fileNameList[currentIdx]
        fileLanguage = self.controller.fileLangList[currentIdx]
        self.reload_program_paths()
        goFlag = None

        if self.mainWindow != None:
            editor = self.mainWindow.controller.editorList[currentIdx] 
            editor.clear_messages()
        
        self.output_text("BUILDING... %s"%fileName)

        ## error checking
        isClean = self.controller.sanitize_check(filePath)
        if isClean == False:
            msg = "Invalid file/dir name\n"
            msg += "%s\nremove characters like '$','#' and '&' then try again"%(filePath) 
            self.display_error(msg)
            return

        isActive = self.is_active_project(filePath)
        if not isActive:
            return
        isTemplate = self.is_template()
        if isTemplate == True:
            return

        fileLanguage = fileLanguage.lower()
        if fileLanguage not in self.validLanguages:
            msg = "Valid languages are %s not %s"%(self.validLanguages,fileLanguage)
            self.display_error(msg)
            return

        ## save file
        if self.mainWindow != None:
            self.mainWindow.file_save(display=False)

        ## run the appropriate builder
        if re.search("\.rst",fileName,flags=re.IGNORECASE):
            self.build_rst(verbose=verbose)
        elif re.search("\.rnw|\.nw",fileName,flags=re.IGNORECASE):
            self.build_nw(verbose=verbose)
        else:
            msg = "Invalid file name extension\n"
            msg +="...lpEdit cannot build\n"
            msg +=filePath
            self.display_error(msg)

    def load_file(self,filePath,fileLang=None):
        '''
        use the controller to load a specified file
        '''

        filePath = os.path.realpath(filePath)

        if os.path.exists(filePath) == False:
            print("ERROR: invalid file path specified for load")
            print("... %s"%filePath)
            return
        
        isValid = self.controller.load_file(filePath,fileLang=fileLang)
        if isValid == False:
            print "ERROR: controller could not load specified file"
            print "\t", filePath

    def clean_file(self,filePath,verbose=False):
        '''
        cleans a directory of its files -- function used with unittests
        '''
        
        self.controller.clean_file(filePath)

    def output_text(self,txt):
        currentIdx = self.controller.currentFileIndex
        
        if self.mainWindow != None:
            if self.mainWindow.controller.editorList[currentIdx].textScreen.showMessages == False:
                self.mainWindow.controller.editorList[currentIdx].textScreen.toggle_message_btn()
            self.mainWindow.controller.editorList[currentIdx].textScreen.add_text(txt)
        else:
            print txt

    def copy_included_files(self,filePath,dirPath):
        """
        Copy files that are included with the keyword INCLUDE
        The source file path is given and it is scanned for INCLUDE
        Each include may be multiple file paths (comma delim)
        """
        
        baseDir = os.path.split(filePath)[0]
        
        ## adjust for subdirectories
        if os.path.split(baseDir)[1] != os.path.split(os.path.split(dirPath)[0])[1]:
            subdir = os.path.split(baseDir)[1]
            targetDir = os.path.join(dirPath,subdir)
        else:
            targetDir = dirPath
            subdir = None

        fid = open(filePath,'r')
        includedFiles = []
        found = False
        for linja in fid:
            if re.search("^%|^\.\. ",linja) and re.search("INCLUDE",linja):
                linja = re.sub("INCLUDE[\W|\w]","",linja)
                linja = re.sub("\%|\s+|^\.\.","",linja)
                found = True
                linja = re.split(",",linja)
                for includedFile in linja:
                    includedFilePath = os.path.join(baseDir,includedFile)
                    includedFiles.append(os.path.realpath(includedFilePath))

        for includedFilePath in includedFiles:
            includedFilePath = includedFilePath
            includedFileName = os.path.split(includedFilePath)[-1]
            includedFileDir = os.path.split(includedFilePath)[-1]
            newFilePath = os.path.join(targetDir,includedFileName)

            if not os.path.exists(includedFilePath):
                msg = "Invalid INCLUDE path...\n"         
                msg += includedFilePath
                if self.mainWindow != None:
                    self.mainWindow.display_warning(msg)
                else:
                    print msg
                return False

            if os.path.islink(newFilePath):
                os.remove(newFilePath)
            if os.path.exists(newFilePath):
                os.remove(newFilePath)

            print("INCLUDE: %s \n...%s"%(includedFilePath,newFilePath))

            os.symlink(includedFilePath,newFilePath)

        return True

    def is_active_project(self,currentFilePath):
{'fraction_non_alphanumeric': 0.09822119102861562, 'fraction_numerical': 0.0015467904098994587, 'mean_word_length': 2.0484354907843976, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '320474', 'n_tokens_mistral': 3901, 'n_tokens_neox': 3777, 'n_words': 820}