<commit_before>import web
import shlex
import subprocess
import os
import re
import uuid
import json
import mimetypematch
import copy
import importlib

DATABASE=None	# The database itself. Will be set by main module
COMMANDS=None	# The command processor. Will be set by the main module.

urls = (
	'/scripts/(.*)', 'runScript',
	'/data/(.*)', 'xmlDatabaseAccess',
	'/internal/(.*)', 'runCommand',
	'/plugin/(.*)', 'runPlugin',
)
app = web.application(urls, globals())

class runScript:
	"""Run a shell script"""
		
	def GET(self, command):
		allArgs = web.input()
		if '/' in command:
			raise web.HTTPError("401 Cannot use / in command")
			
		if allArgs.has_key('args'):
			args = shlex.split(allArgs.args)
		else:
			args = []
			
		# Setup userdata for pull script, if available
		env = None
		if allArgs.has_key('user'):
			env = copy.deepcopy(os.environ)
			user = allArgs['user']
			env['user'] = user
			tmpDB = xmlDatabaseAccess()
			try:
				userData = tmpDB.get_key('identities/%s/scriptData/%s' % (user, command), 'application/json', 'content')
			except:
				userData = None
			if userData:
				env['userData'] = userData
				
		command = "./scripts/" + command
		try:
			linked = os.readlink(command)
			command = os.path.join(os.path.dirname(command), linked)
		except OSError:
			pass
		try:
			rv = subprocess.check_output([command] + args, stderr=subprocess.STDOUT, env=env)
		except subprocess.CalledProcessError, arg:
			msg = "502 Command %s exited with status code=%d" % (command, arg.returncode)
			raise web.HTTPError(msg, {"Content-type": "text/plain"}, msg+'\n\n' + arg.output)
		except OSError, arg:
			msg = "502 Error running command: %s: %s" % (command, arg.strerror)
			raise web.HTTPError(msg, {"Content-type": "text/plain"}, msg+'\n\n')
		return rv

class runCommand:
	"""Call an internal method"""
	
	def GET(self, command):
		if not COMMANDS:
			raise web.notfound()
		try:
			method = getattr(COMMANDS, command)
		except AttributeError:
			raise web.notfound()
		allArgs = dict(web.input())
		try:
			rv = method(**allArgs)
		except TypeError, arg:
			raise web.HTTPError("401 Error calling command method %s: %s" % (command, arg))
		return rv

class runPlugin:
	"""Call a plugin method"""
	
	def GET(self, command):
		try:
			mod = importlib.import_module(command)
		except ImportError:
			raise web.notfound()
		try:
			method = getattr(mod, command)
		except AttributeError:
			raise web.notfound()
			
		# Communicate database to the plugin module
		mod.DATABASE=DATABASE
		mod.COMMANDS=COMMANDS
		
		allArgs = dict(web.input())
		try:
			rv = method(**allArgs)
		except TypeError, arg:
			raise web.HTTPError("401 Error calling plugin method %s: %s" % (command, arg))
		return rv
		
class AbstractDatabaseAccess(object):
	"""Abstract database that handles the high-level HTTP primitives.
	"""
	def GET(self, name):
		"""If no query get the content of a section of the database.
		If there is a query can be used as a 1-url shortcut for POST."""
		optArgs = web.input()

		# See whether we have a variant request
		variant = None
		if '.VARIANT' in optArgs:
			variant = optArgs['.VARIANT']
			del optArgs['.VARIANT']
			
		if optArgs:
			# GET with a query is treated as POST with the query as JSON data,
			# unless the .METHOD argument states it should be treaded as another method.
			optArgs = dict(optArgs)
			method = self.POST
			if '.METHOD' in optArgs:
				method = getattr(self, optArgs['.METHOD'])
				del optArgs['.METHOD']
			return method(name, optArgs, mimetype="application/x-www-form-urlencoded")
			
		returnType = self.best_return_mimetype()
		if not returnType:
			raise web.NotAcceptable()
		web.header("Content-Type", returnType)
		rv = self.get_key(name, self.best_return_mimetype(), variant)
		return rv

	def PUT(self, name, data=None, mimetype=None, replace=True):
		"""Replace part of the document with new data, or inster new data
		in a specific location.
		"""
		optArgs = web.input()

		# See whether we have a variant request
		variant = None
		if '.VARIANT' in optArgs:
			variant = optArgs['.VARIANT']
			del optArgs['.VARIANT']
		
		if not data:
			# We either have a url-encoded query in optArgs or read raw data
			if optArgs:
				data = dict(optArgs)
				mimetype = "application/x-www-form-urlencoded"
			else:
				data = web.data()
				mimetype = web.ctx.env.get('CONTENT_TYPE', 'application/unknown')
		rv = self.put_key(name, self.best_return_mimetype(), variant, data, mimetype, replace=replace)
		return rv

	def POST(self, name, data=None, mimetype=None):
		return self.PUT(name, data, mimetype, replace=False)

	def DELETE(self, name, data=None, mimetype=None):
		return self.delete_key(str(name))

	def is_acceptable_mimetype(self, mimetype=None):
		"""Test whether the mimetype in the request is of an acceptable type"""
		if not self.MIMETYPES:
			return True
		if not mimetype:
			mimetype = web.ctx.env.get("CONTENT_TYPE")
		if not mimetype:
			return True
		return mimetype in self.MIMETYPES
		
	def best_return_mimetype(self):
		"""Return the best mimetype in which to encode the return data, or None"""
		if not self.MIMETYPES:
			return None
		acceptable = web.ctx.env.get("HTTP_ACCEPT")
		if not acceptable:
			return self.MIMETYPES[0]
		return mimetypematch.match(acceptable, self.MIMETYPES)

class xmlDatabaseAccess(AbstractDatabaseAccess):
	MIMETYPES = ["application/xml", "application/json", "text/plain"]
	
	def __init__(self):
		self.db = DATABASE
		assert DATABASE
		self.rootTag = self.db.getDocument().tagName
		
	def get_key(self, key, mimetype, variant):
		"""Get subtree for 'key' as 'mimetype'. Variant can be used
		to state which data should be returned (single node, multinode,
		including refs, etc)"""
		if not key:
			rv = [self.db.getDocument()]
			# This always returns XML, so just continue
		else:
			key = '/%s/%s' % (self.rootTag, key)
			rv = self.db.getElements(key)
		rv = self.convertto(rv, mimetype, variant)
		return rv
		
	def put_key(self, key, mimetype, variant, data, datamimetype, replace=True):
		key = '/%s/%s' % (self.rootTag, key)
		if not variant: variant = 'ref'
		nodesToSignal = []
		with self.db:
			parentPath, tag = self.db.splitXPath(key)
			element = self.convertfrom(data, tag, datamimetype)
			oldElements = self.db.getElements(key)
			if not oldElements:
				#
				# Does not exist yet. See if we can create it
				#
				if not parentPath or not tag:
					raise web.notfound()
				#
				# Find parent
				#
				parentElements = self.db.getElements(parentPath)
				if not parentElements:
					raise web.notfound()
				if len(parentElements) > 1:
					raise web.BadRequest("Bad request, XPath parent selects multiple items")
				parent = parentElements[0]
				#
				# Add new node to the end of the parent
				#
				parent.appendChild(element)
				#
				# Signal both parent and new node
				#
				nodesToSignal.append(element)
				nodesToSignal.append(parent)
			else:
				#
				# Already exists, possibly multiple times. First make sure that if there are
				# multiple matches they all have the same parent (we will replace all of them by
				# the single new node, or append it).
				#
				if len(oldElements) > 1:
					parent1 = oldElements[0].parentNode
					for otherNode in oldElements[1:]:
						if otherNode.parentNode != parent1:
							raise web.BadRequest("Bad Request, XPath selects multiple items from multiple parents")
							
				oldElement = oldElements[0]
				if replace:
					#
					# We should really do a selective replace here: change only the subtrees that need replacing.
					# That will make the signalling much more fine-grained. Will do so, at some point in the future.
					#
					# For now we replace the first matching node and delete its siblings.
					#
					parent = oldElement.parentNode
					parent.replaceChild(element, oldElement)
					for otherNode in oldElements[1:]:
						# Delete other nodes with the same tag
						parent.removeChild(otherNode)
				else:
					#
					# Simply add the new node to the parent (and signal that parent)
					parent = oldElement.parentNode
					parent.appendChild(element)
					nodesToSignal.append(parent)
				#
				# We want to signal the new node
				nodesToSignal.append(element)
				
			self.db.signalNodelist(nodesToSignal)
			path = self.db.getXPathForElement(element)
			return self.convertto(path, mimetype, variant)
		
	def delete_key(self, key):
		key = '/%s/%s' % (self.rootTag, key)
		self.db.delValues(key)
		return ''
		
	def convertto(self, value, mimetype, variant):
		if variant == 'ref':
			if not isinstance(value, basestring):
				raise web.BadRequest("Bad request, cannot use .VARIANT=ref for this operation")
			if mimetype == "application/json":
				return json.dumps({"ref":value})+'\n'
			elif mimetype == "text/plain":<commit_msg>Added /evaluate to evaluate any xpath expression<commit_after>import web
import shlex
import subprocess
import os
import re
import uuid
import json
import mimetypematch
import copy
import importlib
import xpath

DATABASE=None	# The database itself. Will be set by main module
COMMANDS=None	# The command processor. Will be set by the main module.

urls = (
	'/scripts/(.*)', 'runScript',
	'/data/(.*)', 'xmlDatabaseAccess',
	'/evaluate/(.*)', 'xmlDatabaseEvaluate',
	'/internal/(.*)', 'runCommand',
	'/plugin/(.*)', 'runPlugin',
)
app = web.application(urls, globals())

class runScript:
	"""Run a shell script"""
		
	def GET(self, command):
		allArgs = web.input()
		if '/' in command:
			raise web.HTTPError("401 Cannot use / in command")
			
		if allArgs.has_key('args'):
			args = shlex.split(allArgs.args)
		else:
			args = []
			
		# Setup userdata for pull script, if available
		env = None
		if allArgs.has_key('user'):
			env = copy.deepcopy(os.environ)
			user = allArgs['user']
			env['user'] = user
			tmpDB = xmlDatabaseAccess()
			try:
				userData = tmpDB.get_key('identities/%s/scriptData/%s' % (user, command), 'application/json', 'content')
			except:
				userData = None
			if userData:
				env['userData'] = userData
				
		command = "./scripts/" + command
		try:
			linked = os.readlink(command)
			command = os.path.join(os.path.dirname(command), linked)
		except OSError:
			pass
		try:
			rv = subprocess.check_output([command] + args, stderr=subprocess.STDOUT, env=env)
		except subprocess.CalledProcessError, arg:
			msg = "502 Command %s exited with status code=%d" % (command, arg.returncode)
			raise web.HTTPError(msg, {"Content-type": "text/plain"}, msg+'\n\n' + arg.output)
		except OSError, arg:
			msg = "502 Error running command: %s: %s" % (command, arg.strerror)
			raise web.HTTPError(msg, {"Content-type": "text/plain"}, msg+'\n\n')
		return rv

class runCommand:
	"""Call an internal method"""
	
	def GET(self, command):
		if not COMMANDS:
			raise web.notfound()
		try:
			method = getattr(COMMANDS, command)
		except AttributeError:
			raise web.notfound()
		allArgs = dict(web.input())
		try:
			rv = method(**allArgs)
		except TypeError, arg:
			raise web.HTTPError("401 Error calling command method %s: %s" % (command, arg))
		return rv

class runPlugin:
	"""Call a plugin method"""
	
	def GET(self, command):
		try:
			mod = importlib.import_module(command)
		except ImportError:
			raise web.notfound()
		try:
			method = getattr(mod, command)
		except AttributeError:
			raise web.notfound()
			
		# Communicate database to the plugin module
		mod.DATABASE=DATABASE
		mod.COMMANDS=COMMANDS
		
		allArgs = dict(web.input())
		try:
			rv = method(**allArgs)
		except TypeError, arg:
			raise web.HTTPError("401 Error calling plugin method %s: %s" % (command, arg))
		return rv

class xmlDatabaseEvaluate:
	"""Evaluate an XPath expression and return the result as plaintext"""
	def GET(self, command):
		tmpDB = xmlDatabaseAccess()
		return tmpDB.get_value(command)
		
class AbstractDatabaseAccess(object):
	"""Abstract database that handles the high-level HTTP primitives.
	"""
	def GET(self, name):
		"""If no query get the content of a section of the database.
		If there is a query can be used as a 1-url shortcut for POST."""
		optArgs = web.input()

		# See whether we have a variant request
		variant = None
		if '.VARIANT' in optArgs:
			variant = optArgs['.VARIANT']
			del optArgs['.VARIANT']
			
		if optArgs:
			# GET with a query is treated as POST with the query as JSON data,
			# unless the .METHOD argument states it should be treaded as another method.
			optArgs = dict(optArgs)
			method = self.POST
			if '.METHOD' in optArgs:
				method = getattr(self, optArgs['.METHOD'])
				del optArgs['.METHOD']
			return method(name, optArgs, mimetype="application/x-www-form-urlencoded")
			
		returnType = self.best_return_mimetype()
		if not returnType:
			raise web.NotAcceptable()
		web.header("Content-Type", returnType)
		rv = self.get_key(name, self.best_return_mimetype(), variant)
		return rv

	def PUT(self, name, data=None, mimetype=None, replace=True):
		"""Replace part of the document with new data, or inster new data
		in a specific location.
		"""
		optArgs = web.input()

		# See whether we have a variant request
		variant = None
		if '.VARIANT' in optArgs:
			variant = optArgs['.VARIANT']
			del optArgs['.VARIANT']
		
		if not data:
			# We either have a url-encoded query in optArgs or read raw data
			if optArgs:
				data = dict(optArgs)
				mimetype = "application/x-www-form-urlencoded"
			else:
				data = web.data()
				mimetype = web.ctx.env.get('CONTENT_TYPE', 'application/unknown')
		rv = self.put_key(name, self.best_return_mimetype(), variant, data, mimetype, replace=replace)
		return rv

	def POST(self, name, data=None, mimetype=None):
		return self.PUT(name, data, mimetype, replace=False)

	def DELETE(self, name, data=None, mimetype=None):
		return self.delete_key(str(name))

	def is_acceptable_mimetype(self, mimetype=None):
		"""Test whether the mimetype in the request is of an acceptable type"""
		if not self.MIMETYPES:
			return True
		if not mimetype:
			mimetype = web.ctx.env.get("CONTENT_TYPE")
		if not mimetype:
			return True
		return mimetype in self.MIMETYPES
		
	def best_return_mimetype(self):
		"""Return the best mimetype in which to encode the return data, or None"""
		if not self.MIMETYPES:
			return None
		acceptable = web.ctx.env.get("HTTP_ACCEPT")
		if not acceptable:
			return self.MIMETYPES[0]
		return mimetypematch.match(acceptable, self.MIMETYPES)

class xmlDatabaseAccess(AbstractDatabaseAccess):
	MIMETYPES = ["application/xml", "application/json", "text/plain"]
	
	def __init__(self):
		self.db = DATABASE
		assert DATABASE
		self.rootTag = self.db.getDocument().tagName
		
	def get_key(self, key, mimetype, variant):
		"""Get subtree for 'key' as 'mimetype'. Variant can be used
		to state which data should be returned (single node, multinode,
		including refs, etc)"""
		try:
			if not key:
				rv = [self.db.getDocument()]
				# This always returns XML, so just continue
			else:
				key = '/%s/%s' % (self.rootTag, key)
				rv = self.db.getElements(key)
			rv = self.convertto(rv, mimetype, variant)
			return rv
		except xpath.XPathError, arg:
			msg = "401 XPath error: %s" % str(arg)
			raise web.HTTPError(msg, {"Content-type": "text/plain"}, msg+'\n\n')
		
	def get_value(self, expression):
		"""Evaluate a general expression and return the string value"""
		try:
			return self.db.getValue(expression)
		except xpath.XPathError, arg:
			msg = "401 XPath error: %s" % str(arg)
			raise web.HTTPError(msg, {"Content-type": "text/plain"}, msg+'\n\n')
		
	def put_key(self, key, mimetype, variant, data, datamimetype, replace=True):
		try:
			key = '/%s/%s' % (self.rootTag, key)
			if not variant: variant = 'ref'
			nodesToSignal = []
			with self.db:
				parentPath, tag = self.db.splitXPath(key)
				element = self.convertfrom(data, tag, datamimetype)
				oldElements = self.db.getElements(key)
				if not oldElements:
					#
					# Does not exist yet. See if we can create it
					#
					if not parentPath or not tag:
						raise web.notfound()
					#
					# Find parent
					#
					parentElements = self.db.getElements(parentPath)
					if not parentElements:
						raise web.notfound()
					if len(parentElements) > 1:
						raise web.BadRequest("Bad request, XPath parent selects multiple items")
					parent = parentElements[0]
					#
					# Add new node to the end of the parent
					#
					parent.appendChild(element)
					#
					# Signal both parent and new node
					#
					nodesToSignal.append(element)
					nodesToSignal.append(parent)
				else:
					#
					# Already exists, possibly multiple times. First make sure that if there are
					# multiple matches they all have the same parent (we will replace all of them by
					# the single new node, or append it).
					#
					if len(oldElements) > 1:
						parent1 = oldElements[0].parentNode
						for otherNode in oldElements[1:]:
							if otherNode.parentNode != parent1:
								raise web.BadRequest("Bad Request, XPath selects multiple items from multiple parents")
							
					oldElement = oldElements[0]
					if replace:
						#
						# We should really do a selective replace here: change only the subtrees that need replacing.
						# That will make the signalling much more fine-grained. Will do so, at some point in the future.
						#
						# For now we replace the first matching node and delete its siblings.
						#
						parent = oldElement.parentNode
						parent.replaceChild(element, oldElement)
						for otherNode in oldElements[1:]:
							# Delete other nodes with the same tag
							parent.removeChild(otherNode)
					else:
						#
						# Simply add the new node to the parent (and signal that parent)
						parent = oldElement.parentNode
						parent.appendChild(element)
						nodesToSignal.append(parent)
					#
					# We want to signal the new node
					nodesToSignal.append(element)
				
				self.db.signalNodelist(nodesToSignal)
				path = self.db.getXPathForElement(element)
				return self.convertto(path, mimetype, variant)
		except xpath.XPathError, arg:
			msg = "401 XPath error: %s" % str(arg)
			raise web.HTTPError(msg, {"Content-type": "text/plain"}, msg+'\n\n')
		
	def delete_key(self, key):
		try:
			key = '/%s/%s' % (self.rootTag, key)
			self.db.delValues(key)
			return ''
		except xpath.XPathError, arg:
			msg = "401 XPath error: %s" % str(arg)
			raise web.HTTPError(msg, {"Content-type": "text/plain"}, msg+'\n\n')
		
	def convertto(self, value, mimetype, variant):
		if variant == 'ref':
			if not isinstance(value, basestring):
				raise web.BadRequest("Bad request, cannot use .VARIANT=ref for this operation")
			if mimetype == "application/json":
				return json.dumps({"ref":value})+'\n'
			elif mimetype == "text/plain":
{'fraction_non_alphanumeric': 0.11626660951564509, 'fraction_numerical': 0.0034290612944706386, 'mean_word_length': 3.4451059776137174, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5416725', 'n_tokens_mistral': 7179, 'n_tokens_neox': 5996, 'n_words': 1878}