<commit_before>from flask import Flask, make_response, redirect, render_template, request, url_for, send_file, session
from werkzeug import secure_filename
from shutil import rmtree
import os, sys, zipfile, StringIO, pickle, re
from collections import OrderedDict
from scrubber import scrubber, minimal_scrubber
from cutter import cutter
from analysis import analyze
from rwanalysis import rollinganalyze

from werkzeug.contrib.profiler import ProfilerMiddleware

""" Constants """
UPLOAD_FOLDER = '/tmp/Lexos/'
FILES_FOLDER = 'active_files/'
INACTIVE_FOLDER = 'disabled_files/'
PREVIEWSIZE = 100 # note: number of words
PREVIEW_FILENAME = 'preview.p'
FILELABELSFILENAME = 'filelabels.p'
SETIDENTIFIER_FILENAME = 'identifierlist.p'
ALLOWED_EXTENSIONS = set(['txt', 'html', 'xml', 'sgml'])
SCRUBBOXES = ('punctuationbox', 'aposbox', 'hyphensbox', 'digitsbox', 'lowercasebox', 'tagbox')
TEXTAREAS = ('manualstopwords', 'manualspecialchars', 'manualconsolidations', 'manuallemmas')
ANALYZEOPTIONS = ('orientation', 'title', 'metric', 'pruning', 'linkage')

app = Flask(__name__)
app.jinja_env.filters['type'] = type
app.jinja_env.filters['str'] = str
app.config['MAX_CONTENT_LENGTH'] = 4 * 1024 * 1024

@app.route("/", methods=["GET", "POST"])
def base():
	if 'noactivefiles' not in session:
		return redirect(url_for('upload'))
	elif session['noactivefiles']:
		return redirect(url_for('manage'))
	elif not session['noactivefiles']:
		return redirect(url_for('manage'))

@app.route("/reset", methods=["GET"])
def reset():
	print '\nWiping session and old files...'
	try:
		rmtree(os.path.join(UPLOAD_FOLDER, session['id']))
	except:
		pass
	session.clear()
	return init()

@app.route("/upload", methods=["GET", "POST"])
def upload():
	print session
	"""
	Handles the functionality of the upload page. It uploads files to be used
	in the current session.

	*upload() is called with a 'GET' request when a new lexos session is started or the 'Upload' 
	button is clicked in the navigation bar.

	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == "GET":
		# 'GET' request occurs when the page is first loaded.
		if 'id' not in session:
			# init() is called, initializing session variables
			init()
		return render_template('upload.html')
	if 'X_FILENAME' in request.headers:
		# File upload through javascript
		filename = request.headers['X_FILENAME']
		filetype = find_type(filename)
		doe_pattern = re.compile("<publisher>Dictionary of Old English")
		if doe_pattern.search(request.data) != None:
			filename = re.sub('.sgml','.doe',filename)
			filetype = find_type(filename)
		filepath = os.path.join(UPLOAD_FOLDER, session['id'], FILES_FOLDER, filename)
		for existingfilename in getAllFilenames():
			if filename == existingfilename:
				return 'redundant_fail'
		with open(filepath, 'w') as fout:
			fout.write(request.data)
		previewfilepath = os.path.join(UPLOAD_FOLDER, session['id'], PREVIEW_FILENAME)
		preview = pickle.load(open(previewfilepath, 'rb'))
		preview[filename] = makePreviewString(request.data.decode('utf-8'))
		pickle.dump(preview, open(previewfilepath, 'wb'))
		session['noactivefiles'] = False
		return 'success'

@app.route("/manage", methods=["GET", "POST"])
def manage():
	"""
	Handles the functionality of the manage page. It activates/deactivates specific files depending
	on the user.

	*manage() is called with a 'GET' request when the 'Manage' button is clicked in the 
	navigation bar.

	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == "GET":
		preview = makeManagePreview()
		identifierfilepath = os.path.join(UPLOAD_FOLDER, session['id'], SETIDENTIFIER_FILENAME)
		setnames = pickle.load(open(identifierfilepath, 'rb')).keys()
		x, y, active_files = next(os.walk(os.path.join(UPLOAD_FOLDER, session['id'], FILES_FOLDER)))
		return render_template('manage.html', preview=preview, active=active_files, sets=setnames)
	if 'getSubchunks' in request.headers:
		key = request.data
		identifierfilepath = os.path.join(UPLOAD_FOLDER, session['id'], SETIDENTIFIER_FILENAME)
		set_identifier = pickle.load(open(identifierfilepath, 'rb'))
		subchunknames = set_identifier[key]
		numEnabled = 0
		numTotal = len(subchunknames)
		for filename in subchunknames:
			filepath = getFilepath(filename)
			if filepath.find(FILES_FOLDER) != -1:
				numEnabled += 1
		if float(numEnabled) / numTotal > 0.5:
			for filename in subchunknames:
				filepath = getFilepath(filename)
				os.rename(filepath, filepath.replace(FILES_FOLDER, INACTIVE_FOLDER))
				result = 'disable'
		else:
			for filename in subchunknames:
				filepath = getFilepath(filename)
				os.rename(filepath, filepath.replace(INACTIVE_FOLDER, FILES_FOLDER))
				result = 'enable'
		return ','.join(subchunknames) + ',' + result
	if 'disableAll' in request.headers:
		allFiles = getAllFilenames()
		for filename in allFiles:
			filepath = getFilepath(filename)
			os.rename(filepath, filepath.replace(FILES_FOLDER, INACTIVE_FOLDER))
		return ''
	if request.method == "POST":
		# Catch-all for any POST request.
		# In Manage, POSTs come from JavaScript AJAX XHRequests.
		filename = request.data
		if filename in paths():
			filepath = os.path.join(UPLOAD_FOLDER, session['id'], FILES_FOLDER, filename)
			newfilepath = filepath.replace(FILES_FOLDER, INACTIVE_FOLDER)
		else:
			filepath = os.path.join(UPLOAD_FOLDER, session['id'], INACTIVE_FOLDER, filename)
			newfilepath = filepath.replace(INACTIVE_FOLDER, FILES_FOLDER)
		os.rename(filepath, newfilepath)
		activeFiles = getAllFilenames(activeOnly=True).keys()
		if len(activeFiles) == 0:
			session['noactivefiles'] = True
		return ''


@app.route("/scrub", methods=["GET", "POST"])
def scrub():
	"""
	Handles the functionality of the scrub page. It scrubs the files depending on the 
	specifications chosen by the user, and sends the scrubbed files.

	*scrub() is called with a 'GET' request after the 'Scrub' button is clicked in the navigation bar.
	
	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == "GET":
		# 'GET' request occurs when the page is first loaded.
		if session['scrubbingoptions'] == {}:
			for box in SCRUBBOXES:
				session['scrubbingoptions'][box] = False
			for box in TEXTAREAS:
				session['scrubbingoptions'][box] = ''
			session['scrubbingoptions']['optuploadnames'] = { 'swfileselect[]': '', 
															  'lemfileselect[]': '', 
															  'consfileselect[]': '', 
															  'scfileselect[]': '' }
			session['scrubbingoptions']['entityrules'] = 'default'
		session['scrubbingoptions']['keeptags'] = True
		session.modified = True # Letting Flask know that it needs to update session
		# calls makePreviewDict() in helpful functions
		preview = makePreviewDict()
		session['DOE'] = False
		session['hastags'] = False
		for name in preview.keys():
			if find_type(name) == 'doe':
				session['DOE'] = True
			else: # find_type(name) != 'doe'
				session['hastags'] = True
		return render_template('scrub.html', preview=preview)
	if request.method == "POST":
		# 'POST' request occur when html form is submitted (i.e. 'Preview Scrubbing', 'Apply Scrubbing', 'Restore Previews', 'Download...')
		for filetype in request.files:
			filename = request.files[filetype].filename
			if filename != '':
				session['scrubbingoptions']['optuploadnames'][filetype] = filename
		session.modified = True # Necessary to tell Flask that the mutable object (dict) has changed
	if 'preview' in request.form:
		#The 'Preview Scrubbing' button is clicked on scrub.html.
		previewfilepath = os.path.join(UPLOAD_FOLDER, session['id'], PREVIEW_FILENAME)
		preview = pickle.load(open(previewfilepath, 'rb'))
		for filename, path in paths().items():
			filetype = find_type(filename)
			if filetype == 'doe':
				with open(path, 'r') as edit:
					text = edit.read().decode('utf-8')
				text = minimal_scrubber(text,
										tags = session['scrubbingoptions']['tagbox'], 
										keeptags = session['scrubbingoptions']['keeptags'],
										filetype = filetype)
				preview[filename] = (' '.join(text.split()[:PREVIEWSIZE]))
		pickle.dump(preview, open(previewfilepath, 'wb'))
		# calls makePreviewDict() in helpful functions
		preview = makePreviewDict(scrub=True)
		return render_template('scrub.html', preview=preview)
		# scrub.html is rendered again with the scrubbed preview (depending on the chosen settings)
		return render_template('scrub.html', preview=preview)
	if 'apply' in request.form:
		# The 'Apply Scrubbing' button is clicked on scrub.html.
		storeScrubbingOptions()
		for filename, path in paths().items():
			with open(path, 'r') as edit:
				text = edit.read().decode('utf-8')
			filetype = find_type(path)
			text = call_scrubber(text, filetype, previewing=False)
			with open(path, 'w') as edit:
				edit.write(text.encode('utf-8'))
		preview = fullReplacePreview()
		session['scrubbed'] = True
		return render_template('scrub.html', preview=preview)
	if 'download' in request.form:
		# The 'Download Scrubbed Files' button is clicked on scrub.html.
		# sends zipped files to downloads folder.
		return sendActiveFilesAsZip(sentFilename='scrubbed.zip')

@app.route("/cut", methods=["GET", "POST"])
def cut():
	"""
	Handles the functionality of the cut page. It cuts the files into various segments 
	depending on the specifications chosen by the user, and sends the text segments.

	*cut() is called with a 'GET' request after the 'Cut' button is clicked in the navigation bar.
	
	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == "GET":
		# 'GET' request occurs when the page is first loaded.
		preview = makePreviewDict()
		defaultCuts = {'cuttingType': 'Size', 
					   'cuttingValue': '', 
					   'overlap': '0', 
					   'lastProp': '50%'}
		if 'overall' not in session['cuttingoptions']:
			session['cuttingoptions']['overall'] = defaultCuts
		session.modified = True
		return render_template('cut.html', preview=preview)
	if 'downloadchunks' in request.form:
		# The 'Download Segmented Files' button is clicked on cut.html
		# sends zipped files to downloads folder
		return sendActiveFilesAsZip(sentFilename='chunk_files.zip')
	if 'preview' in request.form:
		# The 'Preview Cuts' button is clicked on cut.html.
		preview = call_cutter(previewOnly=True)
		return render_template('cut.html', preview=preview)
	if 'apply' in request.form:
		# The 'Apply Cuts' button is clicked on cut.html.
		storeCuttingOptions()
		preview = call_cutter(previewOnly=False)
		return render_template('cut.html', preview=preview)

@app.route("/analysis", methods=["GET", "POST"])
def analysis():
	"""
	Handles the functionality on the analysis page. It presents various analysis options.

	*analysis() is called with a 'GET' request after the 'Analyze' button is clicked in the navigation bar.

	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == 'GET':
		#'GET' request occurs when the page is first loaded.
		return render_template('analysis.html')
	if 'dendrogram' in request.form:
		return redirect(url_for('dendrogram'))

@app.route("/csvgenerator", methods=["GET", "POST"])
def csvgenerator():
	"""
	Handles the functionality on the csvgenerator page. It analyzes the texts to produce
	and send various frequency matrices.

	*csvgenerator() is called with a 'GET' request after the 'CSV-Generator' button is clicked in the navigation bar.

	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == 'GET':
		# 'GET' request occurs when the page is first loaded.
		filelabels = generateNewLabels()
		return render_template('csvgenerator.html', labels=filelabels)
	if 'get-csv' in request.form:
		#The 'Generate and Download Matrix' button is clicked on csvgenerator.html.
		masterlist = getAllFilenames()
		filelabelsfilepath = os.path.join(UPLOAD_FOLDER, session['id'], FILELABELSFILENAME)
		filelabels = pickle.load(open(filelabelsfilepath, 'rb'))
		for field in request.form:
			if field in masterlist.keys():
				filelabels[field] = request.form[field]
		pickle.dump(filelabels, open(filelabelsfilepath, 'wb'))
		reverse = 'csvorientation' not in request.form
		tsv = 'usetabdelimiter' in request.form
		counts = 'csvtype' in request.form
		if tsv:
			extension = '.tsv'
		else:
			extension = '.csv'
		analyze(orientation=None,
				title=None,
				pruning=None,
				linkage=None,
				metric=None,
				filelabels=filelabels,
				files=os.path.join(UPLOAD_FOLDER, session['id'], FILES_FOLDER), 
				folder=os.path.join(UPLOAD_FOLDER, session['id']),
				forCSV=True,
				orientationReversed=reverse,
				tsv=tsv,
				counts=counts)
		return send_file(os.path.join(UPLOAD_FOLDER, session['id'], 'frequency_matrix'+extension), attachment_filename="frequency_matrix"+extension, as_attachment=True)



@app.route("/dendrogram", methods=["GET", "POST"])
def dendrogram():
	"""
	Handles the functionality on the dendrogram page. It analyzes the various texts and 
	displays a dendrogram.

	*dendrogram() is called with a 'GET' request after the 'Dendrogram' button is clicked in the navigation bar.

	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == 'GET':
		# 'GET' request occurs when the page is first loaded.
		filelabels = generateNewLabels()
		session['denfilepath'] = False
		return render_template('dendrogram.html', labels=filelabels)
	if 'dendro_download' in request.form:
		# The 'Download Dendrogram' button is clicked on dendrogram.html.
		# sends pdf file to downloads folder.
		return send_file(os.path.join(UPLOAD_FOLDER, session['id'], "dendrogram.pdf"), attachment_filename="dendrogram.pdf", as_attachment=True)
	if 'getdendro' in request.form:
		#The 'Get Dendrogram' button is clicked on dendrogram.html.
		session['analyzingoptions']['orientation'] = request.form['orientation']
		session['analyzingoptions']['linkage'] = request.form['linkage']
		session['analyzingoptions']['metric'] = request.form['metric']
		filelabelsfilepath = os.path.join(UPLOAD_FOLDER, session['id'], FILELABELSFILENAME)
		filelabels = pickle.load(open(filelabelsfilepath, 'rb'))
		masterlist = getAllFilenames().keys()
		for field in request.form:
			if field in masterlist:
				filelabels[field] = request.form[field]
		pickle.dump(filelabels, open(filelabelsfilepath, 'wb'))
		session.modified = True
		session['denfilepath'] = analyze(orientation=request.form['orientation'],
										 title = request.form['title'],
										 pruning=request.form['pruning'],
										 linkage=request.form['linkage'],
										 metric=request.form['metric'],
										 filelabels=filelabels,
										 files=os.path.join(UPLOAD_FOLDER, session['id'], FILES_FOLDER), 
										 folder=os.path.join(UPLOAD_FOLDER, session['id']))
		return render_template('dendrogram.html', labels=filelabels)

@app.route("/dendrogramimage", methods=["GET", "POST"])
def dendrogramimage():
	"""
	Reads the png image of the dendrogram and displays it on the web browser.

	*dendrogramimage() is called in analysis.html, displaying the dendrogram.png (if session['denfilepath'] != False).

	Note: Returns a response object with the dendrogram png to flask and eventually to the browser.
	"""
	# dendrogramimage() is called in analysis.html, displaying the dendrogram.png (if session['denfilepath'] != False).
	resp = make_response(open(session['denfilepath']).read())
	resp.content_type = "image/png"
	return resp

@app.route("/rwanalysis", methods=["GET", "POST"])
def rwanalysis():
	"""
	Handles the functionality on the rwanalysis page. It analyzes the various
	texts using a rolling window of analysis.

	*rwanalysis() is called with a 'GET' request after the 'Rolling Analysis' 
	button is clicked in the navigation bar.

	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == 'GET':
		#'GET' request occurs when the page is first loaded.
		filepathDict = paths()
		session['rollanafilepath'] = False
		return render_template('rwanalysis.html', paths=filepathDict)
	if 'rollinganalyze' in request.form:
		# The 'Submit' button is clicked on rwanalysis.html
		filepath = request.form['filetorollinganalyze']
		filestring = open(filepath, 'r').read().decode('utf-8')

		session['rollanafilepath'] = rollinganalyze(fileString=filestring,
						analysisType=request.form['analysistype'],
						inputType=request.form['inputtype'],
						windowType=request.form['windowtype'],
						keyWord=request.form['rollingsearchword'],
						secondKeyWord=request.form['rollingsearchwordopt'],
						windowSize=request.form['rollingwindowsize'],
						folder=os.path.join(UPLOAD_FOLDER, session['id']),
						widthWarp=request.form['rollinggraphwidth'])
		filepathDict = paths()
		return render_template('rwanalysis.html', paths=filepathDict)

@app.route("/rwanalysisimage", methods=["GET", "POST"])
def rwanalysisimage():
	"""
	Reads the png image of the dendrogram and displays it on the web browser.

	*dendrogramimage() is called in analysis.html, displaying the dendrogram.png (if session['denfilepath'] != False).

	Note: Returns a response object with the dendrogram png to flask and eventually to the browser.
	"""
	# rwanalysisimage() is called in analysis.html, displaying the rollingaverage.png (if session['denfilepath'] != False).
	resp = make_response(open(session['rollanafilepath']).read())
	resp.content_type = "image/png"
	return resp
<commit_msg>updated documentation on rolling analysis<commit_after>from flask import Flask, make_response, redirect, render_template, request, url_for, send_file, session
from werkzeug import secure_filename
from shutil import rmtree
import os, sys, zipfile, StringIO, pickle, re
from collections import OrderedDict
from scrubber import scrubber, minimal_scrubber
from cutter import cutter
from analysis import analyze
from rwanalysis import rollinganalyze

from werkzeug.contrib.profiler import ProfilerMiddleware

""" Constants """
UPLOAD_FOLDER = '/tmp/Lexos/'
FILES_FOLDER = 'active_files/'
INACTIVE_FOLDER = 'disabled_files/'
PREVIEWSIZE = 100 # note: number of words
PREVIEW_FILENAME = 'preview.p'
FILELABELSFILENAME = 'filelabels.p'
SETIDENTIFIER_FILENAME = 'identifierlist.p'
ALLOWED_EXTENSIONS = set(['txt', 'html', 'xml', 'sgml'])
SCRUBBOXES = ('punctuationbox', 'aposbox', 'hyphensbox', 'digitsbox', 'lowercasebox', 'tagbox')
TEXTAREAS = ('manualstopwords', 'manualspecialchars', 'manualconsolidations', 'manuallemmas')
ANALYZEOPTIONS = ('orientation', 'title', 'metric', 'pruning', 'linkage')

app = Flask(__name__)
app.jinja_env.filters['type'] = type
app.jinja_env.filters['str'] = str
app.config['MAX_CONTENT_LENGTH'] = 4 * 1024 * 1024

@app.route("/", methods=["GET", "POST"])
def base():
	if 'noactivefiles' not in session:
		return redirect(url_for('upload'))
	elif session['noactivefiles']:
		return redirect(url_for('manage'))
	elif not session['noactivefiles']:
		return redirect(url_for('manage'))

@app.route("/reset", methods=["GET"])
def reset():
	print '\nWiping session and old files...'
	try:
		rmtree(os.path.join(UPLOAD_FOLDER, session['id']))
	except:
		pass
	session.clear()
	return init()

@app.route("/upload", methods=["GET", "POST"])
def upload():
	print session
	"""
	Handles the functionality of the upload page. It uploads files to be used
	in the current session.

	*upload() is called with a 'GET' request when a new lexos session is started or the 'Upload' 
	button is clicked in the navigation bar.

	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == "GET":
		# 'GET' request occurs when the page is first loaded.
		if 'id' not in session:
			# init() is called, initializing session variables
			init()
		return render_template('upload.html')
	if 'X_FILENAME' in request.headers:
		# File upload through javascript
		filename = request.headers['X_FILENAME']
		filetype = find_type(filename)
		doe_pattern = re.compile("<publisher>Dictionary of Old English")
		if doe_pattern.search(request.data) != None:
			filename = re.sub('.sgml','.doe',filename)
			filetype = find_type(filename)
		filepath = os.path.join(UPLOAD_FOLDER, session['id'], FILES_FOLDER, filename)
		for existingfilename in getAllFilenames():
			if filename == existingfilename:
				return 'redundant_fail'
		with open(filepath, 'w') as fout:
			fout.write(request.data)
		previewfilepath = os.path.join(UPLOAD_FOLDER, session['id'], PREVIEW_FILENAME)
		preview = pickle.load(open(previewfilepath, 'rb'))
		preview[filename] = makePreviewString(request.data.decode('utf-8'))
		pickle.dump(preview, open(previewfilepath, 'wb'))
		session['noactivefiles'] = False
		return 'success'

@app.route("/manage", methods=["GET", "POST"])
def manage():
	"""
	Handles the functionality of the manage page. It activates/deactivates specific files depending
	on the user.

	*manage() is called with a 'GET' request when the 'Manage' button is clicked in the 
	navigation bar.

	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == "GET":
		preview = makeManagePreview()
		identifierfilepath = os.path.join(UPLOAD_FOLDER, session['id'], SETIDENTIFIER_FILENAME)
		setnames = pickle.load(open(identifierfilepath, 'rb')).keys()
		x, y, active_files = next(os.walk(os.path.join(UPLOAD_FOLDER, session['id'], FILES_FOLDER)))
		return render_template('manage.html', preview=preview, active=active_files, sets=setnames)
	if 'getSubchunks' in request.headers:
		key = request.data
		identifierfilepath = os.path.join(UPLOAD_FOLDER, session['id'], SETIDENTIFIER_FILENAME)
		set_identifier = pickle.load(open(identifierfilepath, 'rb'))
		subchunknames = set_identifier[key]
		numEnabled = 0
		numTotal = len(subchunknames)
		for filename in subchunknames:
			filepath = getFilepath(filename)
			if filepath.find(FILES_FOLDER) != -1:
				numEnabled += 1
		if float(numEnabled) / numTotal > 0.5:
			for filename in subchunknames:
				filepath = getFilepath(filename)
				os.rename(filepath, filepath.replace(FILES_FOLDER, INACTIVE_FOLDER))
				result = 'disable'
		else:
			for filename in subchunknames:
				filepath = getFilepath(filename)
				os.rename(filepath, filepath.replace(INACTIVE_FOLDER, FILES_FOLDER))
				result = 'enable'
		return ','.join(subchunknames) + ',' + result
	if 'disableAll' in request.headers:
		allFiles = getAllFilenames()
		for filename in allFiles:
			filepath = getFilepath(filename)
			os.rename(filepath, filepath.replace(FILES_FOLDER, INACTIVE_FOLDER))
		return ''
	if request.method == "POST":
		# Catch-all for any POST request.
		# In Manage, POSTs come from JavaScript AJAX XHRequests.
		filename = request.data
		if filename in paths():
			filepath = os.path.join(UPLOAD_FOLDER, session['id'], FILES_FOLDER, filename)
			newfilepath = filepath.replace(FILES_FOLDER, INACTIVE_FOLDER)
		else:
			filepath = os.path.join(UPLOAD_FOLDER, session['id'], INACTIVE_FOLDER, filename)
			newfilepath = filepath.replace(INACTIVE_FOLDER, FILES_FOLDER)
		os.rename(filepath, newfilepath)
		activeFiles = getAllFilenames(activeOnly=True).keys()
		if len(activeFiles) == 0:
			session['noactivefiles'] = True
		return ''


@app.route("/scrub", methods=["GET", "POST"])
def scrub():
	"""
	Handles the functionality of the scrub page. It scrubs the files depending on the 
	specifications chosen by the user, and sends the scrubbed files.

	*scrub() is called with a 'GET' request after the 'Scrub' button is clicked in the navigation bar.
	
	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == "GET":
		# 'GET' request occurs when the page is first loaded.
		if session['scrubbingoptions'] == {}:
			for box in SCRUBBOXES:
				session['scrubbingoptions'][box] = False
			for box in TEXTAREAS:
				session['scrubbingoptions'][box] = ''
			session['scrubbingoptions']['optuploadnames'] = { 'swfileselect[]': '', 
															  'lemfileselect[]': '', 
															  'consfileselect[]': '', 
															  'scfileselect[]': '' }
			session['scrubbingoptions']['entityrules'] = 'default'
		session['scrubbingoptions']['keeptags'] = True
		session.modified = True # Letting Flask know that it needs to update session
		# calls makePreviewDict() in helpful functions
		preview = makePreviewDict()
		session['DOE'] = False
		session['hastags'] = False
		for name in preview.keys():
			if find_type(name) == 'doe':
				session['DOE'] = True
			else: # find_type(name) != 'doe'
				session['hastags'] = True
		return render_template('scrub.html', preview=preview)
	if request.method == "POST":
		# 'POST' request occur when html form is submitted (i.e. 'Preview Scrubbing', 'Apply Scrubbing', 'Restore Previews', 'Download...')
		for filetype in request.files:
			filename = request.files[filetype].filename
			if filename != '':
				session['scrubbingoptions']['optuploadnames'][filetype] = filename
		session.modified = True # Necessary to tell Flask that the mutable object (dict) has changed
	if 'preview' in request.form:
		#The 'Preview Scrubbing' button is clicked on scrub.html.
		previewfilepath = os.path.join(UPLOAD_FOLDER, session['id'], PREVIEW_FILENAME)
		preview = pickle.load(open(previewfilepath, 'rb'))
		for filename, path in paths().items():
			filetype = find_type(filename)
			if filetype == 'doe':
				with open(path, 'r') as edit:
					text = edit.read().decode('utf-8')
				text = minimal_scrubber(text,
										tags = session['scrubbingoptions']['tagbox'], 
										keeptags = session['scrubbingoptions']['keeptags'],
										filetype = filetype)
				preview[filename] = (' '.join(text.split()[:PREVIEWSIZE]))
		pickle.dump(preview, open(previewfilepath, 'wb'))
		# calls makePreviewDict() in helpful functions
		preview = makePreviewDict(scrub=True)
		return render_template('scrub.html', preview=preview)
		# scrub.html is rendered again with the scrubbed preview (depending on the chosen settings)
		return render_template('scrub.html', preview=preview)
	if 'apply' in request.form:
		# The 'Apply Scrubbing' button is clicked on scrub.html.
		storeScrubbingOptions()
		for filename, path in paths().items():
			with open(path, 'r') as edit:
				text = edit.read().decode('utf-8')
			filetype = find_type(path)
			text = call_scrubber(text, filetype, previewing=False)
			with open(path, 'w') as edit:
				edit.write(text.encode('utf-8'))
		preview = fullReplacePreview()
		session['scrubbed'] = True
		return render_template('scrub.html', preview=preview)
	if 'download' in request.form:
		# The 'Download Scrubbed Files' button is clicked on scrub.html.
		# sends zipped files to downloads folder.
		return sendActiveFilesAsZip(sentFilename='scrubbed.zip')

@app.route("/cut", methods=["GET", "POST"])
def cut():
	"""
	Handles the functionality of the cut page. It cuts the files into various segments 
	depending on the specifications chosen by the user, and sends the text segments.

	*cut() is called with a 'GET' request after the 'Cut' button is clicked in the navigation bar.
	
	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == "GET":
		# 'GET' request occurs when the page is first loaded.
		preview = makePreviewDict()
		defaultCuts = {'cuttingType': 'Size', 
					   'cuttingValue': '', 
					   'overlap': '0', 
					   'lastProp': '50%'}
		if 'overall' not in session['cuttingoptions']:
			session['cuttingoptions']['overall'] = defaultCuts
		session.modified = True
		return render_template('cut.html', preview=preview)
	if 'downloadchunks' in request.form:
		# The 'Download Segmented Files' button is clicked on cut.html
		# sends zipped files to downloads folder
		return sendActiveFilesAsZip(sentFilename='chunk_files.zip')
	if 'preview' in request.form:
		# The 'Preview Cuts' button is clicked on cut.html.
		preview = call_cutter(previewOnly=True)
		return render_template('cut.html', preview=preview)
	if 'apply' in request.form:
		# The 'Apply Cuts' button is clicked on cut.html.
		storeCuttingOptions()
		preview = call_cutter(previewOnly=False)
		return render_template('cut.html', preview=preview)

@app.route("/analysis", methods=["GET", "POST"])
def analysis():
	"""
	Handles the functionality on the analysis page. It presents various analysis options.

	*analysis() is called with a 'GET' request after the 'Analyze' button is clicked in the navigation bar.

	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == 'GET':
		#'GET' request occurs when the page is first loaded.
		return render_template('analysis.html')
	if 'dendrogram' in request.form:
		return redirect(url_for('dendrogram'))

@app.route("/csvgenerator", methods=["GET", "POST"])
def csvgenerator():
	"""
	Handles the functionality on the csvgenerator page. It analyzes the texts to produce
	and send various frequency matrices.

	*csvgenerator() is called with a 'GET' request after the 'CSV-Generator' button is clicked in the navigation bar.

	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == 'GET':
		# 'GET' request occurs when the page is first loaded.
		filelabels = generateNewLabels()
		return render_template('csvgenerator.html', labels=filelabels)
	if 'get-csv' in request.form:
		#The 'Generate and Download Matrix' button is clicked on csvgenerator.html.
		masterlist = getAllFilenames()
		filelabelsfilepath = os.path.join(UPLOAD_FOLDER, session['id'], FILELABELSFILENAME)
		filelabels = pickle.load(open(filelabelsfilepath, 'rb'))
		for field in request.form:
			if field in masterlist.keys():
				filelabels[field] = request.form[field]
		pickle.dump(filelabels, open(filelabelsfilepath, 'wb'))
		reverse = 'csvorientation' not in request.form
		tsv = 'usetabdelimiter' in request.form
		counts = 'csvtype' in request.form
		if tsv:
			extension = '.tsv'
		else:
			extension = '.csv'
		analyze(orientation=None,
				title=None,
				pruning=None,
				linkage=None,
				metric=None,
				filelabels=filelabels,
				files=os.path.join(UPLOAD_FOLDER, session['id'], FILES_FOLDER), 
				folder=os.path.join(UPLOAD_FOLDER, session['id']),
				forCSV=True,
				orientationReversed=reverse,
				tsv=tsv,
				counts=counts)
		return send_file(os.path.join(UPLOAD_FOLDER, session['id'], 'frequency_matrix'+extension), attachment_filename="frequency_matrix"+extension, as_attachment=True)



@app.route("/dendrogram", methods=["GET", "POST"])
def dendrogram():
	"""
	Handles the functionality on the dendrogram page. It analyzes the various texts and 
	displays a dendrogram.

	*dendrogram() is called with a 'GET' request after the 'Dendrogram' button is clicked in the navigation bar.

	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == 'GET':
		# 'GET' request occurs when the page is first loaded.
		filelabels = generateNewLabels()
		session['denfilepath'] = False
		return render_template('dendrogram.html', labels=filelabels)
	if 'dendro_download' in request.form:
		# The 'Download Dendrogram' button is clicked on dendrogram.html.
		# sends pdf file to downloads folder.
		return send_file(os.path.join(UPLOAD_FOLDER, session['id'], "dendrogram.pdf"), attachment_filename="dendrogram.pdf", as_attachment=True)
	if 'getdendro' in request.form:
		#The 'Get Dendrogram' button is clicked on dendrogram.html.
		session['analyzingoptions']['orientation'] = request.form['orientation']
		session['analyzingoptions']['linkage'] = request.form['linkage']
		session['analyzingoptions']['metric'] = request.form['metric']
		filelabelsfilepath = os.path.join(UPLOAD_FOLDER, session['id'], FILELABELSFILENAME)
		filelabels = pickle.load(open(filelabelsfilepath, 'rb'))
		masterlist = getAllFilenames().keys()
		for field in request.form:
			if field in masterlist:
				filelabels[field] = request.form[field]
		pickle.dump(filelabels, open(filelabelsfilepath, 'wb'))
		session.modified = True
		session['denfilepath'] = analyze(orientation=request.form['orientation'],
										 title = request.form['title'],
										 pruning=request.form['pruning'],
										 linkage=request.form['linkage'],
										 metric=request.form['metric'],
										 filelabels=filelabels,
										 files=os.path.join(UPLOAD_FOLDER, session['id'], FILES_FOLDER), 
										 folder=os.path.join(UPLOAD_FOLDER, session['id']))
		return render_template('dendrogram.html', labels=filelabels)

@app.route("/dendrogramimage", methods=["GET", "POST"])
def dendrogramimage():
	"""
	Reads the png image of the dendrogram and displays it on the web browser.

	*dendrogramimage() is called in analysis.html, displaying the dendrogram.png (if session['denfilepath'] != False).

	Note: Returns a response object with the dendrogram png to flask and eventually to the browser.
	"""
	# dendrogramimage() is called in analysis.html, displaying the dendrogram.png (if session['denfilepath'] != False).
	resp = make_response(open(session['denfilepath']).read())
	resp.content_type = "image/png"
	return resp

@app.route("/rwanalysis", methods=["GET", "POST"])
def rwanalysis():
	"""
	Handles the functionality on the rwanalysis page. It analyzes the various
	texts using a rolling window of analysis.

	*rwanalysis() is called with a 'GET' request after the 'Rolling Analysis' 
	button is clicked in the navigation bar.

	Note: Returns a response object (often a render_template call) to flask and eventually
		  to the browser.
	"""
	if request.method == 'GET':
		#'GET' request occurs when the page is first loaded.
		filepathDict = paths()
		session['rollanafilepath'] = False
		return render_template('rwanalysis.html', paths=filepathDict)
	if 'rollinganalyze' in request.form:
		# The 'Submit' button is clicked on rwanalysis.html
		filepath = request.form['filetorollinganalyze']
		filestring = open(filepath, 'r').read().decode('utf-8')
		print request.form
		session['rollanafilepath'] = rollinganalyze(fileString=filestring,
						analysisType=request.form['analysistype'],
						inputType=request.form['inputtype'],
						windowType=request.form['windowtype'],
						keyWord=request.form['rollingsearchword'],
						secondKeyWord=request.form['rollingsearchwordopt'],
						windowSize=request.form['rollingwindowsize'],
						folder=os.path.join(UPLOAD_FOLDER, session['id']),
						widthWarp=request.form['rollinggraphwidth'])
		filepathDict = paths()
		return render_template('rwanalysis.html', paths=filepathDict)

@app.route("/rwanalysisimage", methods=["GET", "POST"])
def rwanalysisimage():
	"""
	Reads the png image of the dendrogram and displays it on the web browser.

	*dendrogramimage() is called in analysis.html, displaying the dendrogram.png (if session['denfilepath'] != False).

	Note: Returns a response object with the dendrogram png to flask and eventually to the browser.
	"""
	# rwanalysisimage() is called in analysis.html, displaying the rollingaverage.png (if session['denfilepath'] != False).
	resp = make_response(open(session['rollanafilepath']).read())
	resp.content_type = "image/png"
	return resp

{'fraction_non_alphanumeric': 0.1237075747358957, 'fraction_numerical': 0.0014610024724657227, 'mean_word_length': 4.847379661573846, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3118805', 'n_tokens_mistral': 12293, 'n_tokens_neox': 11078, 'n_words': 3154}