<commit_before>#!/usr/bin/python2.7

# include blocks/ directory
import sys
sys.path.insert(0, './blocks')
sys.path.insert(0, './libs')

# public imports
import base64
from Crypto.Cipher import AES # pycrypto
import flask
from flask import Flask
from flask import jsonify
import hashlib
import json
import mimetypes
mimetypes.init()
import os
import os.path
from random import choice
import re
import requests
import urllib
import shutil
from string import ascii_uppercase
import threading
import yaml # pyyaml

# private imports
import parseblocks
import loadblocks as lb
import qhelper

# start up & load configuration

with open('./configuration.yaml','r') as f:
	configuration = yaml.load(f.read())
	
	try:
		QENGINE_CACHE_DIR = configuration['QENGINE_CACHE_DIR']
		QENGINE_SALT = configuration['QENGINE_SALT']
	except Exception as e:
		print e
		sys.exit()
		
	validSaltLengths = [16,24,32]
	if len(QENGINE_SALT) not in validSaltLengths:
		print 'QENGINE_SALT must be 16, 24, or 32 bytes long'
		sys.exit()
	
	try:
		QENGINE_IV = configuration['QENGINE_IV']
		if len(QENGINE_SALT) not in validSaltLengths:
			print 'QENGINE_IV must be 16 bytes long'
			sys.exit()
	except:
		QENGINE_IV = ''.join(choice(ascii_uppercase) for i in range(16))
	
	# optional environment variables
	try:
		QENGINE_PASSKEY = configuration['QENGINE_PASSKEY']
		if len(QENGINE_PASSKEY) == 0:
			QENGINE_PASSKEY = None
	except:
		QENGINE_PASSKEY = None
	
	try:
		QENGINE_NO_CACHE = configuration['QENGINE_NO_CACHE']
		if not QENGINE_NO_CACHE:
			QENGINE_NO_CACHE = False
	except:
		QENGINE_NO_CACHE = False
	
	ENGINEINFO = {}
	
	# engine info
	for key in configuration:
		if 'ENGINEINFO_' in key:
			ekey = key.split('_',1)[1]
			if len(ekey) > 0:
				ENGINEINFO[ekey] = configuration[key]
	
	BLOCKS = lb.loadblocks(configuration)

app = Flask(__name__)

# helper functions

def log(err):
	with open('log.txt','a') as file:
		file.write("%s" % err)

class SaveToCache(threading.Thread):
	def __init__(self, qenginevars, genfiles, sessionDataPath, sessionDataDir):
		threading.Thread.__init__(self)
		self.qenginevars = qenginevars
		self.genfiles = genfiles
		self.sessionDataPath = sessionDataPath
		self.sessionDataDir = sessionDataDir
	
	def run(self):
		with open(self.sessionDataPath, 'w') as sessionjsonfile:
			json.dump(self.qenginevars, sessionjsonfile)
		
		for file in self.genfiles:
			with open(self.sessionDataDir + '/'+ file['filename'],'wb') as sessionfile:
				sessionfile.write(base64.b64decode(file['content']))

# routes

@app.route('/info',methods=['GET'])
def getEngineInfo():
	engineinfo = {'engineinfo':ENGINEINFO}
	return jsonify(engineinfo)

@app.route('/metadata/<string:base>/<string:id>/<string:version>',methods=['GET'])
def getQuestionMetadata(base,id,version):
	qdirname = "./questions/" + urllib.unquote(base) + "/" + urllib.unquote(id) + "/" + urllib.unquote(version)
	
	if not os.path.isdir(qdirname):
		badfile = {'error':'The question you requested does not exist'}
		return flask.make_response(jsonify(badfile), 404)
	try:
		metadatafile = "./questions/" + base + "/" + id + "/" + "metadata"
		file = qhelper.get_first_file(metadatafile)
		if file is not None:
			with open(file,'r') as f:
				metadata = yaml.load(f.read())
				return jsonify(metadata)
		else:
			nometadata = {'metadata':'none'}
			return jsonify(nometadata)
	except:
		nometadata = {'metadata':'none'}
		return jsonify(nometadata)

@app.route('/session',methods=['POST'])<commit_msg>opaque protocol fixes<commit_after>#!/usr/bin/python2.7

# include blocks/ directory
import sys
sys.path.insert(0, './blocks')
sys.path.insert(0, './libs')

# public imports
import base64
from Crypto.Cipher import AES # pycrypto
import flask
from flask import Flask
from flask import jsonify
import hashlib
import json
import mimetypes
mimetypes.init()
import os
import os.path
from random import choice
import re
import requests
import urllib
import shutil
from string import ascii_uppercase
import threading
import yaml # pyyaml

# private imports
import parseblocks
import loadblocks as lb
import qhelper

# start up & load configuration

with open('./configuration.yaml','r') as f:
	configuration = yaml.load(f.read())
	
	try:
		QENGINE_CACHE_DIR = configuration['QENGINE_CACHE_DIR']
		QENGINE_SALT = configuration['QENGINE_SALT']
	except Exception as e:
		print e
		sys.exit()
		
	validSaltLengths = [16,24,32]
	if len(QENGINE_SALT) not in validSaltLengths:
		print 'QENGINE_SALT must be 16, 24, or 32 bytes long'
		sys.exit()
	
	try:
		QENGINE_IV = configuration['QENGINE_IV']
		if len(QENGINE_SALT) not in validSaltLengths:
			print 'QENGINE_IV must be 16 bytes long'
			sys.exit()
	except:
		QENGINE_IV = ''.join(choice(ascii_uppercase) for i in range(16))
	
	# optional environment variables
	try:
		QENGINE_PASSKEY = configuration['QENGINE_PASSKEY']
		if len(QENGINE_PASSKEY) == 0:
			QENGINE_PASSKEY = None
	except:
		QENGINE_PASSKEY = None
	
	try:
		QENGINE_NO_CACHE = configuration['QENGINE_NO_CACHE']
		if not QENGINE_NO_CACHE:
			QENGINE_NO_CACHE = False
	except:
		QENGINE_NO_CACHE = False
	
	ENGINEINFO = {}
	
	# engine info
	for key in configuration:
		if 'ENGINEINFO_' in key:
			ekey = key.split('_',1)[1]
			if len(ekey) > 0:
				ENGINEINFO[ekey] = configuration[key]
	
	BLOCKS = lb.loadblocks(configuration)

app = Flask(__name__)

# helper functions

def log(err):
	with open('log.txt','a') as file:
		file.write("%s" % err)

class SaveToCache(threading.Thread):
	def __init__(self, qenginevars, genfiles, sessionDataPath, sessionDataDir):
		threading.Thread.__init__(self)
		self.qenginevars = qenginevars
		self.genfiles = genfiles
		self.sessionDataPath = sessionDataPath
		self.sessionDataDir = sessionDataDir
	
	def run(self):
		with open(self.sessionDataPath, 'w') as sessionjsonfile:
			json.dump(self.qenginevars, sessionjsonfile)
		
		for file in self.genfiles:
			with open(self.sessionDataDir + '/'+ file['filename'],'wb') as sessionfile:
				sessionfile.write(base64.b64decode(file['content']))

# routes

@app.route('/info',methods=['GET'])
def getEngineInfo():
	engineinfo = {'engineinfo':ENGINEINFO}
	return jsonify(engineinfo)

@app.route('/question/<string:base>/<string:id>/<string:version>',methods=['GET'])
def getQuestionMetadata(base,id,version):
	qdirname = "./questions/" + urllib.unquote(base) + "/" + urllib.unquote(id) + "/" + urllib.unquote(version)
	
	if not os.path.isdir(qdirname):
		badfile = {'error':'The question you requested does not exist'}
		return flask.make_response(jsonify(badfile), 404)
	try:
		metadatafile = "./questions/" + base + "/" + id + "/" + "metadata"
		file = qhelper.get_first_file(metadatafile)
		if file is not None:
			with open(file,'r') as f:
				metadata = yaml.load(f.read())
				return jsonify({'questionmetadata':metadata})
		else:
			nometadata = {'questionmetadata':'none'}
			return jsonify(nometadata)
	except:
		nometadata = {'questionmetadata':'none'}
		return jsonify(nometadata)

@app.route('/session',methods=['POST'])
{'fraction_non_alphanumeric': 0.12440944881889764, 'fraction_numerical': 0.009448818897637795, 'mean_word_length': 4.855825649622799, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4897201', 'n_tokens_mistral': 2691, 'n_tokens_neox': 2375, 'n_words': 579}