<commit_before>import StringIO
import zipfile
import re
import os
from os.path import join as pathjoin
from os import makedirs, remove
from flask import session, request, send_file

import prepare.scrubber as scrubber
import prepare.cutter as cutter

import helpers.general_functions as general_functions
import helpers.session_functions as session_functions
import helpers.constants as constants

import analyze.dendrogrammer as dendrogrammer
import analyze.rw_analyzer as rw_analyzer

import codecs

class FileManager:
    PREVIEW_NORMAL = 1
    PREVIEW_CUT = 2

    def __init__(self, sessionFolder):
        self.files = {}
        self.lastID = 0
        self.noActiveFiles = True

        makedirs(pathjoin(sessionFolder, constants.FILECONTENTS_FOLDER))

    def addFile(self, newFile):
        if newFile.id == None:
            newFile.id = self.lastID


        self.files[newFile.id] = newFile

        self.lastID += 1

        return newFile.id

    def addFile(self, fileName, fileString):
        newFile = LexosFile(fileName, fileString)

        return self.addFile(newFile)

    def deleteActiveFiles(self):
        # Delete the contents and mark them for removal from list
        for fileID, lFile in self.files.items(): # Using an underscore is a convention for not using that variable
            if lFile.active:
                lFile.cleanAndDelete()
                del self.files[fileID] # Delete the entry

    def fileExists(self, fileID):
        for lFile in self.files.values():
            if lFile.id == fileID:
                return True

        return False

    def disableAll(self):
        for lFile in self.files.values():
            lFile.disable()

    def enableAll(self):
        for lFile in self.files.values():
            lFile.enable()

    def getPreviewsOfActive(self):
        previews = []

        for lFile in self.files.values():
            if lFile.active:
                previews.append((lFile.id, lFile.label, lFile.classLabel, lFile.getPreview()))

        return previews

    def getPreviewsOfInactive(self):
        previews = []

        for lFile in self.files.values():
            if not lFile.active:
                previews.append((lFile.id, lFile.label, lFile.classLabel, lFile.getPreview()))

        return previews

    def numActiveFiles(self):
        numActive = 0
        for lFile in self.files.values():
            if lFile.active:
                numActive += 1

        return numActive

    def toggleFile(self, fileID):
        numActive = 0

        for lFile in self.files.values():
            if lFile.id == fileID:
                if lFile.active:
                    lFile.disable()
                    numActive -= 1
                else:
                    lFile.enable()
                    numActive += 1

            elif lFile.active:
                numActive += 1

        if numActive == 0:
            self.noActiveFiles = True

    def classifyActiveFiles(self):
        for lFile in self.files.values():
            if lFile.active:
                lFile.setClassLabel(request.data)

    def scrubFiles(self, savingChanges):
        previews = []

        for lFile in self.files.values():
            if lFile.active:
                previews.append((lFile.id, lFile.label, lFile.classLabel, lFile.scrubContents(savingChanges)))

        return previews

    def cutFiles(self, savingChanges):
        previews = []

        #print request.form

        activeFiles = []
        for lFile in self.files.values():
            if lFile.active:
                activeFiles.append(lFile)

        for lFile in activeFiles:
            subFileTuples = lFile.cutContents()
            lFile.active = False

            if savingChanges:
                for i, (fileLabel, fileString) in enumerate(subFileTuples):
                    fileID = self.addFile(fileLabel + '_' + str(i+1) + '.txt', fileString)
                    self.files[fileID].saveCutOptions(parentID=lFile.id)

            else:
                cutPreview = []
                for i, (fileLabel, fileString) in enumerate(subFileTuples):
                    cutPreview.append(('Chunk ' + str(i+1), general_functions.makePreviewFrom(fileString)))

                previews.append((lFile.id, lFile.label, lFile.classLabel, cutPreview))

        if savingChanges:
            previews = self.getPreviewsOfActive()

        return previews

    def zipActiveFiles(self, fileName):
        zipstream = StringIO.StringIO()
        zfile = zipfile.ZipFile(file=zipstream, mode='w')
        for lFile in self.files.values():
            if lFile.active:
                zfile.write(lFile.savePath, arcname=lFile.name, compress_type=zipfile.ZIP_STORED)
        zfile.close()
        zipstream.seek(0)

        return send_file(zipstream, attachment_filename=fileName, as_attachment=True)

    def checkActivesTags(self):
        foundTags = False
        foundDOE = False

        for lFile in self.files.values():
            if not lFile.active:
                continue # with the looping, do not do the rest of current loop
                
            if lFile.type == lFile.TYPE_DOE:
                foundDOE = True
                foundTags = True
            if lFile.hasTags:
                foundTags = True

            if foundDOE and foundTags:
                break

        return foundTags, foundDOE

    def updateLabel(self, fileID, fileLabel):
        for lFile in self.files.values():
            if lFile.id == fileID:
                lFile.label = fileLabel
                return

    def getActiveLabels(self):
        labels = {}
        for lFile in self.files.values():
            if lFile.active:
                labels[lFile.id] = lFile.label

        return labels

    def generateDataMatrix(self, labels, useFreq):
        countDictDict = {} # Dictionary of dictionaries, keys are ids, values are count dictionaries of {'word' : number of occurances}
        totalWordCountDict = {}
        allWords = set()
        for lFile in self.files.values():
            if lFile.active:
                countDictDict[lFile.id] = lFile.getWordCounts()
                totalWordCountDict[lFile.id] = lFile.length()
                allWords.update(countDictDict[lFile.id].keys()) # Update the master list of all words from the word in each file

        countMatrix = [[''] + sorted(allWords)]
        for fileID, fileCountDict in countDictDict.items():
            countMatrix.append([labels[fileID]])
            for word in sorted(allWords):
                if word in fileCountDict:
                    if useFreq:
                        countMatrix[-1].append(fileCountDict[word] / float(totalWordCountDict[fileID]))
                    else:
                        countMatrix[-1].append(fileCountDict[word])
                else:
                    countMatrix[-1].append(0)

        for i in xrange(len(countMatrix)):
            row = countMatrix[i]
            for j in xrange(len(row)):
                element = countMatrix[i][j]
                if isinstance(element, unicode):
                    countMatrix[i][j] = element.encode('utf-8')

        return countMatrix


    def generateCSV(self, tempLabels):
        useCounts = request.form['csvdata'] == 'count'
        transpose = request.form['csvorientation'] == 'filecolumn'
        useTSV = request.form['csvdelimiter'] == 'tab'

        extension = '.tsv' if useTSV else '.csv'

        matrix = self.generateDataMatrix(labels=tempLabels, useFreq = not useCounts)

        delimiter = '\t' if useTSV else ','
        outFilePath = pathjoin(session_functions.session_folder(), constants.ANALYZER_FOLDER, 'csvfile'+extension)

        if transpose:
            matrix = zip(*matrix)

        with open(outFilePath, 'w') as outFile:
            for row in matrix:
                rowStr = delimiter.join([str(x) for x in row])
                outFile.write(rowStr + '\n')
        outFile.close()

        return outFilePath, extension


    def getMatrix(self, tempLabels):
        useFreq = request.form['matrixData'] == 'freq'
        countDictDict = {} # Dictionary of dictionaries, keys are ids, values are count dictionaries of {'word' : number of occurances}
        totalWordCountDict = {}
        allWords = set()
        for lFile in self.files.values():
            if lFile.active:
                countDictDict[lFile.id] = lFile.getWordCounts()
                totalWordCountDict[lFile.id] = lFile.length()
                allWords.update(countDictDict[lFile.id].keys()) # Update the master list of all words from the word in each file

        countMatrix = [[''] + sorted(allWords)]
        for fileID, fileCountDict in countDictDict.items():
            countMatrix.append([tempLabels[fileID]])
            for word in sorted(allWords):
                if word in fileCountDict:
                    if useFreq:
                        countMatrix[-1].append(fileCountDict[word] / float(totalWordCountDict[fileID]))
                    else:
                        countMatrix[-1].append(fileCountDict[word])
                else:
                    countMatrix[-1].append(0)
       
        matrix = []
        fileNumber = len(countMatrix)
        totalWords = len(countMatrix[0])

        for row in range(1,fileNumber):
            wordCount = []
            for col in range(1,totalWords):
                wordCount.append(countMatrix[row][col])
            matrix.append(wordCount)

        fileName = []
        for eachLabel in tempLabels:
            fileName.append(tempLabels[eachLabel])
            
        return matrix, fileName


    def generateDendrogram(self,tempLabels):
        orientation = str(request.form['orientation'])
        title       = request.form['title'] 
        pruning     = request.form['pruning']
        pruning     = int(request.form['pruning']) if pruning else 0
        linkage     = str(request.form['linkage'])
        metric      = str(request.form['metric'])
        folderPath    = pathjoin(session_functions.session_folder(),constants.ANALYZER_FOLDER)
   
        if (not os.path.isdir(folderPath)):
            makedirs(folderPath)

        matrix, fileName = self.getMatrix(tempLabels)
        return dendrogrammer.dendrogram(orientation, title, pruning, linkage, metric, fileName, matrix, folderPath)


    def getDendroLegend(self):
        for lFile in self.files.values():
            if lFile.active:
                # -------- store dendrogram options ----------
                lFile.optionsDic["dendrogram"]['metric']   = request.form['metric']
                lFile.optionsDic["dendrogram"]['linkage']  = request.form['linkage']
                lFile.optionsDic["dendrogram"]['format']   = request.form['matrixData']


    def generateRWA(self):

        fileID        = int(request.form['filetorollinganalyze'])    # file the user selected to use for generating the grpah
        fileString    = self.files[fileID].fetchContents()

        #user inputed option choices
        analysisType  = request.form['analysistype']
        inputType     = request.form['inputtype']
        windowType    = request.form['windowtype']
        keyWord       = request.form['rollingsearchword']
        secondKeyWord = request.form['rollingsearchwordopt']
        windowSize    = request.form['rollingwindowsize']

        """Calls rw_analyzer, which 1) returns session['rwadatagenerated'] true
                                    2) generates and returns dataList, a list of single average or ratio values
                                    3) returns label (ex: "Average number of e's in a window of 207 characters")
        all according to the user inputed options"""
        dataList, label = rw_analyzer.rw_analyze(fileString, analysisType, inputType, windowType, keyWord, secondKeyWord, windowSize)

        """Creates a list of two-item lists using previously generated dataList. These are our x and y values for
            our graph, ex: [0, 4.3], [1, 3.9], [2, 8.5], etc. """
        dataPoints = [[i, dataList[i]] for i in xrange(len(dataList))]

        return dataPoints, label


    def getAllContents(self):
        chosenFileIDs = [int(x) for x in request.form.getlist('segmentlist')]

        allWordsString = ""

        if chosenFileIDs:
            for ID in chosenFileIDs:
                allWordsString += " " + self.files[ID].getWords()

        else:
            for lFile in self.files.values():
                if lFile.active:
                    allWordsString += " " + lFile.getWords()

        return allWordsString


    def generateJSONForD3(self, mergedSet):
        chosenFileIDs = [int(x) for x in request.form.getlist('segmentlist')]

        activeFiles = []
        if chosenFileIDs:
            for ID in chosenFileIDs:
                activeFiles.append(self.files[ID])
        else:
            for lFile in self.files.values():
                if lFile.active:
                    activeFiles.append(lFile)

        if mergedSet: # Create one JSON Object across all the chunks
            minimumLength = int(request.form['minlength'])
            masterWordCounts = {}
            
            for lFile in activeFiles:
                wordCounts = lFile.getWordCounts()

                for key in wordCounts:
                    if len(key) <= minimumLength:
                        continue

                    if key in masterWordCounts:
                        masterWordCounts[key] += wordCounts[key]
                    else:
                        masterWordCounts[key] = wordCounts[key]



            returnObj = general_functions.generateD3Object(masterWordCounts, objectLabel="tokens", wordLabel="name", countLabel="size")

        else: # Create a JSON object for each chunk
            returnObj = []
            for lFile in activeFiles:
                returnObj.append(lFile.generateD3JSONObject(wordLabel="text", countLabel="size"))

        return returnObj

class LexosFile:
    TYPE_TXT = 1
    TYPE_HTML = 2
    TYPE_XML = 3
    TYPE_SGML = 4
    TYPE_DOE = 5

    def __init__(self, fileName, fileString):
        self.contents = fileString
        self.id = None # Starts out without an id - later assigned one from FileManager
        self.name = fileName
        self.contentsPreview = ''
        self.savePath = pathjoin(session_functions.session_folder(), constants.FILECONTENTS_FOLDER, str(self.id) + '.txt')
        self.active = True
        self.isChild = False
        self.classLabel = ''

        splitName = self.name.split('.')

        self.label = self.updateLabel()
        self.updateType(splitName[-1])
        self.hasTags = self.checkForTags()
        self.generatePreview()
        self.dumpContents()

        # -------- store all options -----------
        self.optionsDic = {}
        
        # -------- store scrubbing options ----------
        self.optionsDic["scrub"] = {}

        self.optionsDic["scrub"]['punctuationbox'] = False
        self.optionsDic["scrub"]['lowercasebox']   = False
        self.optionsDic["scrub"]['digitsbox']      = False
        self.optionsDic["scrub"]['tagbox']         = False
        self.optionsDic["scrub"]['hyphensbox']     = False
        self.optionsDic["scrub"]['aposbox']        = False

        for box in constants.TEXTAREAS:
            self.optionsDic["scrub"][box] = ''
        for box in constants.OPTUPLOADNAMES:
            self.optionsDic["scrub"][box] = ''
        self.optionsDic["scrub"]['entityrules'] = 'none'


        # ------- store cutting options ---------
        self.optionsDic["cut"] = {}

        self.optionsDic["cut"]['cut_type']      = 'number'
        self.optionsDic["cut"]['cutting_value'] = 1
        self.optionsDic["cut"]['overlap']       = 0 
        self.optionsDic["cut"]['lastprop']      = 0
        self.optionsDic["cut"]['cutsetnaming']  = ''

        # ------- store dendrogram options ---------
        self.optionsDic["dendrogram"] = {}

        self.optionsDic["dendrogram"]['metric']   = 'euclidean'
        self.optionsDic["dendrogram"]['linkage']  = 'average'
        self.optionsDic["dendrogram"]['format']   = 'frequency percentage'


    def cleanAndDelete(self):
        # Delete the file on the hard drive where the LexosFile saves its contents string
        remove(self.savePath)
<commit_msg>Removing default values for preparation from LexosFile constructor<commit_after>import StringIO
import zipfile
import re
import os
from os.path import join as pathjoin
from os import makedirs, remove
from flask import session, request, send_file

import prepare.scrubber as scrubber
import prepare.cutter as cutter

import helpers.general_functions as general_functions
import helpers.session_functions as session_functions
import helpers.constants as constants

import analyze.dendrogrammer as dendrogrammer
import analyze.rw_analyzer as rw_analyzer

import codecs

class FileManager:
    PREVIEW_NORMAL = 1
    PREVIEW_CUT = 2

    def __init__(self, sessionFolder):
        self.files = {}
        self.lastID = 0
        self.noActiveFiles = True

        makedirs(pathjoin(sessionFolder, constants.FILECONTENTS_FOLDER))

    def addFile(self, newFile):
        if newFile.id == None:
            newFile.id = self.lastID


        self.files[newFile.id] = newFile

        self.lastID += 1

        return newFile.id

    def addFile(self, fileName, fileString):
        newFile = LexosFile(fileName, fileString)

        return self.addFile(newFile)

    def deleteActiveFiles(self):
        # Delete the contents and mark them for removal from list
        for fileID, lFile in self.files.items(): # Using an underscore is a convention for not using that variable
            if lFile.active:
                lFile.cleanAndDelete()
                del self.files[fileID] # Delete the entry

    def fileExists(self, fileID):
        for lFile in self.files.values():
            if lFile.id == fileID:
                return True

        return False

    def disableAll(self):
        for lFile in self.files.values():
            lFile.disable()

    def enableAll(self):
        for lFile in self.files.values():
            lFile.enable()

    def getPreviewsOfActive(self):
        previews = []

        for lFile in self.files.values():
            if lFile.active:
                previews.append((lFile.id, lFile.label, lFile.classLabel, lFile.getPreview()))

        return previews

    def getPreviewsOfInactive(self):
        previews = []

        for lFile in self.files.values():
            if not lFile.active:
                previews.append((lFile.id, lFile.label, lFile.classLabel, lFile.getPreview()))

        return previews

    def numActiveFiles(self):
        numActive = 0
        for lFile in self.files.values():
            if lFile.active:
                numActive += 1

        return numActive

    def toggleFile(self, fileID):
        numActive = 0

        for lFile in self.files.values():
            if lFile.id == fileID:
                if lFile.active:
                    lFile.disable()
                    numActive -= 1
                else:
                    lFile.enable()
                    numActive += 1

            elif lFile.active:
                numActive += 1

        if numActive == 0:
            self.noActiveFiles = True

    def classifyActiveFiles(self):
        for lFile in self.files.values():
            if lFile.active:
                lFile.setClassLabel(request.data)

    def scrubFiles(self, savingChanges):
        previews = []

        for lFile in self.files.values():
            if lFile.active:
                previews.append((lFile.id, lFile.label, lFile.classLabel, lFile.scrubContents(savingChanges)))

        return previews

    def cutFiles(self, savingChanges):
        previews = []

        #print request.form

        activeFiles = []
        for lFile in self.files.values():
            if lFile.active:
                activeFiles.append(lFile)

        for lFile in activeFiles:
            subFileTuples = lFile.cutContents()
            lFile.active = False

            if savingChanges:
                for i, (fileLabel, fileString) in enumerate(subFileTuples):
                    fileID = self.addFile(fileLabel + '_' + str(i+1) + '.txt', fileString)
                    self.files[fileID].saveCutOptions(parentID=lFile.id)

            else:
                cutPreview = []
                for i, (fileLabel, fileString) in enumerate(subFileTuples):
                    cutPreview.append(('Chunk ' + str(i+1), general_functions.makePreviewFrom(fileString)))

                previews.append((lFile.id, lFile.label, lFile.classLabel, cutPreview))

        if savingChanges:
            previews = self.getPreviewsOfActive()

        return previews

    def zipActiveFiles(self, fileName):
        zipstream = StringIO.StringIO()
        zfile = zipfile.ZipFile(file=zipstream, mode='w')
        for lFile in self.files.values():
            if lFile.active:
                zfile.write(lFile.savePath, arcname=lFile.name, compress_type=zipfile.ZIP_STORED)
        zfile.close()
        zipstream.seek(0)

        return send_file(zipstream, attachment_filename=fileName, as_attachment=True)

    def checkActivesTags(self):
        foundTags = False
        foundDOE = False

        for lFile in self.files.values():
            if not lFile.active:
                continue # with the looping, do not do the rest of current loop
                
            if lFile.type == lFile.TYPE_DOE:
                foundDOE = True
                foundTags = True
            if lFile.hasTags:
                foundTags = True

            if foundDOE and foundTags:
                break

        return foundTags, foundDOE

    def updateLabel(self, fileID, fileLabel):
        for lFile in self.files.values():
            if lFile.id == fileID:
                lFile.label = fileLabel
                return

    def getActiveLabels(self):
        labels = {}
        for lFile in self.files.values():
            if lFile.active:
                labels[lFile.id] = lFile.label

        return labels

    def generateDataMatrix(self, labels, useFreq):
        countDictDict = {} # Dictionary of dictionaries, keys are ids, values are count dictionaries of {'word' : number of occurances}
        totalWordCountDict = {}
        allWords = set()
        for lFile in self.files.values():
            if lFile.active:
                countDictDict[lFile.id] = lFile.getWordCounts()
                totalWordCountDict[lFile.id] = lFile.length()
                allWords.update(countDictDict[lFile.id].keys()) # Update the master list of all words from the word in each file

        countMatrix = [[''] + sorted(allWords)]
        for fileID, fileCountDict in countDictDict.items():
            countMatrix.append([labels[fileID]])
            for word in sorted(allWords):
                if word in fileCountDict:
                    if useFreq:
                        countMatrix[-1].append(fileCountDict[word] / float(totalWordCountDict[fileID]))
                    else:
                        countMatrix[-1].append(fileCountDict[word])
                else:
                    countMatrix[-1].append(0)

        for i in xrange(len(countMatrix)):
            row = countMatrix[i]
            for j in xrange(len(row)):
                element = countMatrix[i][j]
                if isinstance(element, unicode):
                    countMatrix[i][j] = element.encode('utf-8')

        return countMatrix


    def generateCSV(self, tempLabels):
        useCounts = request.form['csvdata'] == 'count'
        transpose = request.form['csvorientation'] == 'filecolumn'
        useTSV = request.form['csvdelimiter'] == 'tab'

        extension = '.tsv' if useTSV else '.csv'

        matrix = self.generateDataMatrix(labels=tempLabels, useFreq = not useCounts)

        delimiter = '\t' if useTSV else ','
        outFilePath = pathjoin(session_functions.session_folder(), constants.ANALYZER_FOLDER, 'csvfile'+extension)

        if transpose:
            matrix = zip(*matrix)

        with open(outFilePath, 'w') as outFile:
            for row in matrix:
                rowStr = delimiter.join([str(x) for x in row])
                outFile.write(rowStr + '\n')
        outFile.close()

        return outFilePath, extension


    def getMatrix(self, tempLabels):
        useFreq = request.form['matrixData'] == 'freq'
        countDictDict = {} # Dictionary of dictionaries, keys are ids, values are count dictionaries of {'word' : number of occurances}
        totalWordCountDict = {}
        allWords = set()
        for lFile in self.files.values():
            if lFile.active:
                countDictDict[lFile.id] = lFile.getWordCounts()
                totalWordCountDict[lFile.id] = lFile.length()
                allWords.update(countDictDict[lFile.id].keys()) # Update the master list of all words from the word in each file

        countMatrix = [[''] + sorted(allWords)]
        for fileID, fileCountDict in countDictDict.items():
            countMatrix.append([tempLabels[fileID]])
            for word in sorted(allWords):
                if word in fileCountDict:
                    if useFreq:
                        countMatrix[-1].append(fileCountDict[word] / float(totalWordCountDict[fileID]))
                    else:
                        countMatrix[-1].append(fileCountDict[word])
                else:
                    countMatrix[-1].append(0)
       
        matrix = []
        fileNumber = len(countMatrix)
        totalWords = len(countMatrix[0])

        for row in range(1,fileNumber):
            wordCount = []
            for col in range(1,totalWords):
                wordCount.append(countMatrix[row][col])
            matrix.append(wordCount)

        fileName = []
        for eachLabel in tempLabels:
            fileName.append(tempLabels[eachLabel])
            
        return matrix, fileName


    def generateDendrogram(self,tempLabels):
        orientation = str(request.form['orientation'])
        title       = request.form['title'] 
        pruning     = request.form['pruning']
        pruning     = int(request.form['pruning']) if pruning else 0
        linkage     = str(request.form['linkage'])
        metric      = str(request.form['metric'])
        folderPath    = pathjoin(session_functions.session_folder(),constants.ANALYZER_FOLDER)
   
        if (not os.path.isdir(folderPath)):
            makedirs(folderPath)

        matrix, fileName = self.getMatrix(tempLabels)
        return dendrogrammer.dendrogram(orientation, title, pruning, linkage, metric, fileName, matrix, folderPath)


    def getDendroLegend(self):
        for lFile in self.files.values():
            if lFile.active:
                # -------- store dendrogram options ----------
                lFile.optionsDic["dendrogram"]['metric']   = request.form['metric']
                lFile.optionsDic["dendrogram"]['linkage']  = request.form['linkage']
                lFile.optionsDic["dendrogram"]['format']   = request.form['matrixData']


    def generateRWA(self):

        fileID        = int(request.form['filetorollinganalyze'])    # file the user selected to use for generating the grpah
        fileString    = self.files[fileID].fetchContents()

        #user inputed option choices
        analysisType  = request.form['analysistype']
        inputType     = request.form['inputtype']
        windowType    = request.form['windowtype']
        keyWord       = request.form['rollingsearchword']
        secondKeyWord = request.form['rollingsearchwordopt']
        windowSize    = request.form['rollingwindowsize']

        """Calls rw_analyzer, which 1) returns session['rwadatagenerated'] true
                                    2) generates and returns dataList, a list of single average or ratio values
                                    3) returns label (ex: "Average number of e's in a window of 207 characters")
        all according to the user inputed options"""
        dataList, label = rw_analyzer.rw_analyze(fileString, analysisType, inputType, windowType, keyWord, secondKeyWord, windowSize)

        """Creates a list of two-item lists using previously generated dataList. These are our x and y values for
            our graph, ex: [0, 4.3], [1, 3.9], [2, 8.5], etc. """
        dataPoints = [[i, dataList[i]] for i in xrange(len(dataList))]

        return dataPoints, label


    def getAllContents(self):
        chosenFileIDs = [int(x) for x in request.form.getlist('segmentlist')]

        allWordsString = ""

        if chosenFileIDs:
            for ID in chosenFileIDs:
                allWordsString += " " + self.files[ID].getWords()

        else:
            for lFile in self.files.values():
                if lFile.active:
                    allWordsString += " " + lFile.getWords()

        return allWordsString


    def generateJSONForD3(self, mergedSet):
        chosenFileIDs = [int(x) for x in request.form.getlist('segmentlist')]

        activeFiles = []
        if chosenFileIDs:
            for ID in chosenFileIDs:
                activeFiles.append(self.files[ID])
        else:
            for lFile in self.files.values():
                if lFile.active:
                    activeFiles.append(lFile)

        if mergedSet: # Create one JSON Object across all the chunks
            minimumLength = int(request.form['minlength'])
            masterWordCounts = {}
            
            for lFile in activeFiles:
                wordCounts = lFile.getWordCounts()

                for key in wordCounts:
                    if len(key) <= minimumLength:
                        continue

                    if key in masterWordCounts:
                        masterWordCounts[key] += wordCounts[key]
                    else:
                        masterWordCounts[key] = wordCounts[key]



            returnObj = general_functions.generateD3Object(masterWordCounts, objectLabel="tokens", wordLabel="name", countLabel="size")

        else: # Create a JSON object for each chunk
            returnObj = []
            for lFile in activeFiles:
                returnObj.append(lFile.generateD3JSONObject(wordLabel="text", countLabel="size"))

        return returnObj

class LexosFile:
    TYPE_TXT = 1
    TYPE_HTML = 2
    TYPE_XML = 3
    TYPE_SGML = 4
    TYPE_DOE = 5

    def __init__(self, fileName, fileString):
        self.contents = fileString
        self.id = None # Starts out without an id - later assigned one from FileManager
        self.name = fileName
        self.contentsPreview = ''
        self.savePath = pathjoin(session_functions.session_folder(), constants.FILECONTENTS_FOLDER, str(self.id) + '.txt')
        self.active = True
        self.isChild = False
        self.classLabel = ''

        splitName = self.name.split('.')

        self.label = self.updateLabel()
        self.updateType(splitName[-1])
        self.hasTags = self.checkForTags()
        self.generatePreview()
        self.dumpContents()

        # -------- stores all preparation options -----------
        self.optionsDic = {}


    def cleanAndDelete(self):
        # Delete the file on the hard drive where the LexosFile saves its contents string
        remove(self.savePath)

{'fraction_non_alphanumeric': 0.09911894273127753, 'fraction_numerical': 0.0033518483049224286, 'mean_word_length': 1.9578887734869228, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5794399', 'n_tokens_mistral': 8858, 'n_tokens_neox': 8355, 'n_words': 2082}