<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  AdvancedHTTPServer.py
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following disclaimer
#    in the documentation and/or other materials provided with the
#    distribution.
#  * Neither the name of the project nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#  pylint: disable=too-many-lines

#  Homepage: https://github.com/zeroSteiner/AdvancedHTTPServer
#  Author:   <NAME> (zeroSteiner)

# Config file example
FILE_CONFIG = """
[server]
ip = 0.0.0.0
port = 8080
web_root = /var/www/html
list_directories = True
# Set an ssl_cert to enable SSL
# ssl_cert = /path/to/cert.pem
"""

# The AdvancedHTTPServer systemd service unit file
# Quick how to:
#   1. Copy this file to /etc/systemd/system/pyhttpd.service
#   2. Edit the run parameters appropriately in the ExecStart option
#   3. Set configuration settings in /etc/pyhttpd.conf
#   4. Run "systemctl daemon-reload"
FILE_SYSTEMD_SERVICE_UNIT = """
[Unit]
Description=Python Advanced HTTP Server
After=network.target

[Service]
Type=simple
ExecStart=/sbin/runuser -l nobody -c "/usr/bin/python -m AdvancedHTTPServer -c /etc/pyhttpd.conf"
ExecStop=/bin/kill -INT $MAINPID

[Install]
WantedBy=multi-user.target
"""

__version__ = '0.4.3'
__all__ = [
	'AdvancedHTTPServer',
	'AdvancedHTTPServerRegisterPath',
	'AdvancedHTTPServerRequestHandler',
	'AdvancedHTTPServerRPCClient',
	'AdvancedHTTPServerRPCClientCached',
	'AdvancedHTTPServerRPCError',
	'AdvancedHTTPServerTestCase',
	'build_server_from_argparser',
	'build_server_from_config'
]

import base64
import binascii
import datetime
import hashlib
import hmac
import io
import json
import logging
import logging.handlers
import mimetypes
import os
import posixpath
import random
import re
import shutil
import socket
import sqlite3
import ssl
import string
import sys
import threading
import time
import traceback
import unittest
import urllib
import zlib

if sys.version_info[0] < 3:
	import BaseHTTPServer
	import cgi as html
	import Cookie
	import httplib
	import SocketServer as socketserver
	import urlparse
	http = type('http', (), {'client': httplib, 'cookies': Cookie, 'server': BaseHTTPServer})
	urllib.parse = urlparse
	urllib.parse.quote = urllib.quote
	urllib.parse.unquote = urllib.unquote
	from ConfigParser import ConfigParser
else:
	import html
	import http.client
	import http.cookies
	import http.server
	import socketserver
	import urllib.parse
	from configparser import ConfigParser

GLOBAL_HANDLER_MAP = {}

def _serialize_ext_dump(obj):
	if obj.__class__ == datetime.date:
		return 'datetime.date', obj.isoformat()
	elif obj.__class__ == datetime.datetime:
		return 'datetime.datetime', obj.isoformat()
	elif obj.__class__ == datetime.time:
		return 'datetime.time', obj.isoformat()
	raise TypeError('Unknown type: ' + repr(obj))

def _serialize_ext_load(obj_type, obj_value, default):
	if obj_type == 'datetime.date':
		return datetime.datetime.strptime(obj_value, '%Y-%m-%d').date()
	elif obj_type == 'datetime.datetime':
		return datetime.datetime.strptime(obj_value, '%Y-%m-%dT%H:%M:%S' + ('.%f' if '.' in obj_value else ''))
	elif obj_type == 'datetime.time':
		return datetime.datetime.strptime(obj_value, '%H:%M:%S' + ('.%f' if '.' in obj_value else '')).time()
	return default

def _json_default(obj):
	obj_type, obj_value = _serialize_ext_dump(obj)
	return {'__complex_type__': obj_type, 'value': obj_value}

def _json_object_hook(obj):
	return _serialize_ext_load(obj.get('__complex_type__'), obj.get('value'), obj)

SERIALIZER_DRIVERS = {}
"""Dictionary of available drivers for serialization."""
SERIALIZER_DRIVERS['application/json'] = {'loads': lambda d, e: json.loads(d, object_hook=_json_object_hook), 'dumps': lambda d: json.dumps(d, default=_json_default)}

try:
	import msgpack
except ImportError:
	has_msgpack = False
else:
	has_msgpack = True
	_MSGPACK_EXT_TYPES = {10: 'datetime.datetime', 11: 'datetime.date', 12: 'datetime.time'}
	def _msgpack_default(obj):
		obj_type, obj_value = _serialize_ext_dump(obj)
		obj_type = next(i[0] for i in _MSGPACK_EXT_TYPES.items() if i[1] == obj_type)
		if sys.version_info[0] == 3:
			obj_value = obj_value.encode('utf-8')
		return msgpack.ExtType(obj_type, obj_value)

	def _msgpack_ext_hook(code, obj_value):
		default = msgpack.ExtType(code, obj_value)
		if sys.version_info[0] == 3:
			obj_value = obj_value.decode('utf-8')
		obj_type = _MSGPACK_EXT_TYPES.get(code)
		return _serialize_ext_load(obj_type, obj_value, default)
	SERIALIZER_DRIVERS['binary/message-pack'] = {'loads': lambda d, e: msgpack.loads(d, encoding=e, ext_hook=_msgpack_ext_hook), 'dumps': lambda d: msgpack.dumps(d, default=_msgpack_default)}

if hasattr(logging, 'NullHandler'):
	logging.getLogger('AdvancedHTTPServer').addHandler(logging.NullHandler())

def random_string(size):
	"""
	Generate a random string of *size* length consisting of both letters
	and numbers. This function is not meant for cryptographic purposes
	and should not be used to generate security tokens.

	:param int size: The length of the string to return.
	:return: A string consisting of random characters.
	:rtype: str
	"""
	return ''.join(random.choice(string.ascii_letters + string.digits) for x in range(size))

def resolve_ssl_protocol_version(version=None):
	"""
	Look up an SSL protocol version by name. If *version* is not specified, then
	the strongest protocol available will be returned.

	:param str version: The name of the version to look up.
	:return: A protocol constant from the :py:mod:`ssl` module.
	:rtype: int
	"""
	if version == None:
		protocol_preference = ('TLSv1_2', 'TLSv1_1', 'TLSv1', 'SSLv3', 'SSLv23', 'SSLv2')
		for protocol in protocol_preference:
			if hasattr(ssl, 'PROTOCOL_' + protocol):
				return getattr(ssl, 'PROTOCOL_' + protocol)
		raise RuntimeError('could not find a suitable ssl PROTOCOL_ version constant')
	elif isinstance(version, str):
		if not hasattr(ssl, 'PROTOCOL_' + version):
			raise ValueError('invalid ssl protocol version: ' + version)
		return getattr(ssl, 'PROTOCOL_' + version)
	raise TypeError("ssl_version() argument 1 must be str, not {0}".format(type(version).__name__))

def build_serializer_from_content_type(content_type):
	"""
	Build a serializer object from a MIME Content-Type string.

	:param str content_type: The Content-Type string to parse.
	:return: A new configured serializer instance.
	:rtype: :py:class:`.AdvancedHTTPServerSerializer`
	"""
	name = content_type
	options = {}
	if ';' in content_type:
		name, options_str = content_type.split(';', 1)
		for part in options_str.split(';'):
			part = part.strip()
			if '=' in part:
				key, value = part.split('=')
			else:
				key, value = (part, None)
			options[key] = value
	# old style compatibility
	if name.endswith('+zlib'):
		options['compression'] = 'zlib'
		name = name[:-5]
	return AdvancedHTTPServerSerializer(name, charset=options.get('charset', 'UTF-8'), compression=options.get('compression'))

def build_server_from_argparser(description=None, ServerClass=None, HandlerClass=None):
	"""
	Build a server from command line arguments. If a ServerClass or
	HandlerClass is specified, then the object must inherit from the
	corresponding AdvancedHTTPServer base class.

	:param str description: Description string to be passed to the argument parser.
	:param ServerClass: Alternative server class to use.
	:type ServerClass: :py:class:`.AdvancedHTTPServer`
	:param HandlerClass: Alternative handler class to use.
	:type HandlerClass: :py:class:`.AdvancedHTTPServerRequestHandler`
	:return: A configured server instance.
	:rtype: :py:class:`.AdvancedHTTPServer`
	"""
	import argparse

	def _argp_dir_type(arg):
		if not os.path.isdir(arg):
			raise argparse.ArgumentTypeError("{0} is not a valid directory".format(repr(arg)))
		return arg

	def _argp_port_type(arg):
		if not arg.isdigit():
			raise argparse.ArgumentTypeError("{0} is not a valid port".format(repr(arg)))
		arg = int(arg)
		if arg < 0 or arg > 65535:
			raise argparse.ArgumentTypeError("{0} is not a valid port".format(repr(arg)))
		return arg

	description = (description or 'HTTP Server')
	ServerClass = (ServerClass or AdvancedHTTPServer)
	HandlerClass = (HandlerClass or AdvancedHTTPServerRequestHandler)

	parser = argparse.ArgumentParser(conflict_handler='resolve', description=description, fromfile_prefix_chars='@')
	parser.epilog = 'When a config file is specified with --config the --ip, --port and --web-root options are all ignored.'
	parser.add_argument('-w', '--web-root', dest='web_root', action='store', default='.', type=_argp_dir_type, help='path to the web root directory')
	parser.add_argument('-p', '--port', dest='port', action='store', default=8080, type=_argp_port_type, help='port to serve on')
	parser.add_argument('-i', '--ip', dest='ip', action='store', default='0.0.0.0', help='the ip address to serve on')
	parser.add_argument('--password', dest='password', action='store', default=None, help='password to use for basic authentication')
	parser.add_argument('--log-file', dest='log_file', action='store', default=None, help='log information to a file')
	parser.add_argument('-c', '--conf', dest='config', action='store', default=None, type=argparse.FileType('r'), help='read settings from a config file')
	parser.add_argument('-v', '--version', action='version', version=parser.prog + ' Version: ' + __version__)
	parser.add_argument('-L', '--log', dest='loglvl', action='store', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], default='INFO', help='set the logging level')
	arguments = parser.parse_args()

	logging.getLogger('').setLevel(logging.DEBUG)
	console_log_handler = logging.StreamHandler()
	console_log_handler.setLevel(getattr(logging, arguments.loglvl))
	console_log_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-8s %(message)s"))
	logging.getLogger('').addHandler(console_log_handler)

	if arguments.log_file:
		main_file_handler = logging.handlers.RotatingFileHandler(arguments.log_file, maxBytes=262144, backupCount=5)
		main_file_handler.setLevel(logging.DEBUG)
		main_file_handler.setFormatter(logging.Formatter("%(asctime)s %(name)-30s %(levelname)-10s %(message)s"))
		logging.getLogger('').setLevel(logging.DEBUG)
		logging.getLogger('').addHandler(main_file_handler)

	if arguments.config:
		config = ConfigParser()
		config.readfp(arguments.config)
		server = build_server_from_config(config, 'server', ServerClass=ServerClass, HandlerClass=HandlerClass)
	else:
		server = ServerClass(HandlerClass, address=(arguments.ip, arguments.port))
		server.serve_files_root = arguments.web_root

	if arguments.password:
		server.auth_add_creds('', arguments.password)
	return server

def build_server_from_config(config, section_name, ServerClass=None, HandlerClass=None):
	"""
	Build a server from a provided :py:class:`configparser.ConfigParser`
	instance. If a ServerClass or HandlerClass is specified, then the
	object must inherit from the corresponding AdvancedHTTPServer base
	class.

	:param config: Configuration to retrieve settings from.
	:type config: :py:class:`configparser.ConfigParser`
	:param str section_name: The section name of the configuration to use.
	:param ServerClass: Alternative server class to use.
	:type ServerClass: :py:class:`.AdvancedHTTPServer`
	:param HandlerClass: Alternative handler class to use.
	:type HandlerClass: :py:class:`.AdvancedHTTPServerRequestHandler`
	:return: A configured server instance.
	:rtype: :py:class:`.AdvancedHTTPServer`
	"""
	ServerClass = (ServerClass or AdvancedHTTPServer)
	HandlerClass = (HandlerClass or AdvancedHTTPServerRequestHandler)
	port = config.getint(section_name, 'port')
	web_root = None
	if config.has_option(section_name, 'web_root'):
		web_root = config.get(section_name, 'web_root')

	if config.has_option(section_name, 'ip'):
		ip = config.get(section_name, 'ip')
	else:
		ip = '0.0.0.0'
	ssl_certfile = None
	if config.has_option(section_name, 'ssl_cert'):
		ssl_certfile = config.get(section_name, 'ssl_cert')
	ssl_keyfile = None
	if config.has_option(section_name, 'ssl_key'):
		ssl_keyfile = config.get(section_name, 'ssl_key')
	server = ServerClass(HandlerClass, address=(ip, port), ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile)

	if config.has_option(section_name, 'password_type'):
		password_type = config.get(section_name, 'password_type')
	else:
		password_type = '<PASSWORD>'
	if config.has_option(section_name, 'password'):
		password = config.get(section_name, 'password')
		if config.has_option(section_name, 'username'):
			username = config.get(section_name, 'username')
		else:
			username = ''
		server.auth_add_creds(username, password, pwtype=password_type)
	cred_idx = 0
	while config.has_option(section_name, 'password' + str(cred_idx)):
		password = config.get(section_name, 'password' + str(cred_idx))
		if not config.has_option(section_name, 'username' + str(cred_idx)):
			break
		username = config.get(section_name, 'username' + str(cred_idx))
		server.auth_add_creds(username, password, pwtype=password_type)
		cred_idx += 1

	if web_root == None:
		server.serve_files = False
	else:
		server.serve_files = True
		server.serve_files_root = web_root
		if config.has_option(section_name, 'list_directories'):
			server.serve_files_list_directories = config.getboolean(section_name, 'list_directories')
	return server

class AdvancedHTTPServerRegisterPath(object):
	"""
	Register a path and handler with the global handler map. This can be
	used as a decorator. If no handler is specified then the path and
	function will be registered with all :py:class:`.AdvancedHTTPServerRequestHandler`
	instances.

	.. code-block:: python

	  @AdvancedHTTPServerRegisterPath('^test$')
	  def handle_test(handler, query):
	      pass
	"""
	def __init__(self, path, handler=None, is_rpc=False):
		"""
		:param str path: The path regex to register the function to.
		:param str handler: A specific :py:class:`.AdvancedHTTPServerRequestHandler` class to register the handler with.
		:param bool is_rpc: Whether the handler is an RPC handler or not.
		"""
		self.path = path
		self.is_rpc = is_rpc
		if handler == None or isinstance(handler, str):
			self.handler = handler
		elif hasattr(handler, '__name__'):
			self.handler = handler.__name__
		elif hasattr(handler, '__class__'):
			self.handler = handler.__class__.__name__
		else:
			raise ValueError('unknown handler: ' + repr(handler))

	def __call__(self, function):
		handler_map = GLOBAL_HANDLER_MAP.get(self.handler, {})
		handler_map[self.path] = (function, self.is_rpc)
		GLOBAL_HANDLER_MAP[self.handler] = handler_map
		return function

class AdvancedHTTPServerRPCError(Exception):
	"""
	This class represents an RPC error either local or remote. Any errors
	in routines executed on the server will raise this error.
	"""
	def __init__(self, message, status, remote_exception=None):
		super(AdvancedHTTPServerRPCError, self).__init__()
		self.message = message
		self.status = status
		self.remote_exception = remote_exception

	def __repr__(self):
		return "{0}(message='{1}', status={2}, remote_exception={3})".format(self.__class__.__name__, self.message, self.status, self.is_remote_exception)

	def __str__(self):
		if self.is_remote_exception:
			return 'a remote exception occurred'
		return "the server responded with {0} '{1}'".format(self.status, self.message)

	@property
	def is_remote_exception(self):
		"""
		This is true if the represented error resulted from an exception on the
		remote server.

		:type: bool
		"""
		return bool(self.remote_exception != None)

class AdvancedHTTPServerRPCClient(object):
	"""
	This object facilitates communication with remote RPC methods as
	provided by a :py:class:`.AdvancedHTTPServerRequestHandler` instance.
	Once created this object can be called directly, doing so is the same
	as using the call method.

	This object uses locks internally to be thread safe. Only one thread
	can execute a function at a time.
	"""
	def __init__(self, address, use_ssl=False, username=None, password=<PASSWORD>, uri_base='/', hmac_key=None):
		"""
		:param tuple address: The address of the server to conenct to as (host, port).
		:param bool use_ssl: Whether to connect with SSL or not.
		:param str username: The username to authenticate with.
		:param str password: The password to authenticate with.
		:param str uri_base: An optional prefix for all methods.
		:param str hmac_key: An HMAC key to use for request authentication.
		"""
		self.host = str(address[0])
		self.port = int(address[1])
		if not hasattr(self, 'logger'):
			self.logger = logging.getLogger('AdvancedHTTPServer.RPCClient')

		self.use_ssl = bool(use_ssl)
		self.uri_base = str(uri_base)
		self.username = (str(username) if username != None else None)
		self.password = (str(password) if password != None else None)
		if isinstance(hmac_key, str):
			hmac_key = hmac_key.encode('UTF-8')
		self.hmac_key = hmac_key
		self.lock = threading.Lock()
		self.set_serializer('application/json')
		self.reconnect()

	def __reduce__(self):
		address = (self.host, self.port)
		return (self.__class__, (address, self.use_ssl, self.username, self.password, self.uri_base, self.hmac_key))

	def set_serializer(self, serializer_name, compression=None):
		"""
		Configure the serializer to use for communication with the server.
		The serializer specified must be valid and in the
		:py:data:`.SERIALIZER_DRIVERS` map.

		:param str serializer_name: The name of the serializer to use.
		:param str compression: The name of a compression library to use.
		"""
		self.serializer = AdvancedHTTPServerSerializer(serializer_name, charset='UTF-8', compression=compression)
		self.logger.debug('using serializer: ' + serializer_name)

	def __call__(self, *args, **kwargs):
		return self.call(*args, **kwargs)

	def encode(self, data):
		"""Encode data with the configured serializer."""
		return self.serializer.dumps(data)

	def decode(self, data):
		"""Decode data with the configured serializer."""
		return self.serializer.loads(data)

	def reconnect(self):
		"""Reconnect to the remote server."""
		self.lock.acquire()
		if self.use_ssl:
			self.client = http.client.HTTPSConnection(self.host, self.port)
		else:
			self.client = http.client.HTTPConnection(self.host, self.port)
		self.lock.release()

	def call(self, method, *options):
		"""
		Issue a call to the remote end point to execute the specified
		procedure.

		:param str method: The name of the remote procedure to execute.
		:return: The return value from the remote function.
		"""
		options = self.encode(options)

		headers = {}
		headers['Content-Type'] = self.serializer.content_type
		headers['Content-Length'] = str(len(options))

		if self.hmac_key != None:
			hmac_calculator = hmac.new(self.hmac_key, digestmod=hashlib.sha1)
			hmac_calculator.update(options)
			headers['X-RPC-HMAC'] = hmac_calculator.hexdigest()

		if self.username != None and self.password != None:
			headers['Authorization'] = 'Basic ' + base64.b64encode((self.username + ':' + self.password).encode('UTF-8')).decode('UTF-8')

		method = os.path.join(self.uri_base, method)
		self.logger.debug('calling RPC method: ' + method[1:])
		with self.lock:
			self.client.request("RPC", method, options, headers)
			resp = self.client.getresponse()
		if resp.status != 200:
			raise AdvancedHTTPServerRPCError(resp.reason, resp.status)

		resp_data = resp.read()
		if self.hmac_key != None:
			hmac_digest = resp.getheader('X-RPC-HMAC')
			if not isinstance(hmac_digest, str):
				raise AdvancedHTTPServerRPCError('hmac validation error', resp.status)
			hmac_digest = hmac_digest.lower()
			hmac_calculator = hmac.new(self.hmac_key, digestmod=hashlib.sha1)
			hmac_calculator.update(resp_data)
			if hmac_digest != hmac_calculator.hexdigest():
				raise AdvancedHTTPServerRPCError('hmac validation error', resp.status)
		resp_data = self.decode(resp_data)
		if not ('exception_occurred' in resp_data and 'result' in resp_data):
			raise AdvancedHTTPServerRPCError('missing response information', resp.status)
		if resp_data['exception_occurred']:
			raise AdvancedHTTPServerRPCError('remote method incured an exception', resp.status, remote_exception=resp_data['exception'])
		return resp_data['result']

class AdvancedHTTPServerRPCClientCached(AdvancedHTTPServerRPCClient):
	"""
	This object builds upon :py:class:`.AdvancedHTTPServerRPCClient` and
	provides additional methods for cacheing results in memory.
	"""
	def __init__(self, *args, **kwargs):
		cache_db = kwargs.pop('cache_db', ':memory:')
		super(AdvancedHTTPServerRPCClientCached, self).__init__(*args, **kwargs)
		self.cache_db = sqlite3.connect(cache_db, check_same_thread=False)
		cursor = self.cache_db.cursor()
		cursor.execute('CREATE TABLE IF NOT EXISTS cache (method TEXT NOT NULL, options_hash BLOB NOT NULL, return_value TEXT NOT NULL)')
		self.cache_db.commit()
		self.cache_serializer_loads = SERIALIZER_DRIVERS['application/json']['loads']
		self.cache_serializer_dumps = SERIALIZER_DRIVERS['application/json']['dumps']
		self.cache_lock = threading.Lock()

	def cache_call(self, method, *options):
		"""
		Call a remote method and store the result locally. Subsequent
		calls to the same method with the same arguments will return the
		cached result without invoking the remote procedure. Cached results are
		kept indefinitely and must be manually refreshed with a call to
		:py:meth:`.cache_call_refresh`.

		:param str method: The name of the remote procedure to execute.
		:return: The return value from the remote function.
		"""
		options_hash = self.encode(options)
		if len(options_hash) > 20:
			options_hash = hashlib.new('sha1', options_hash).digest()
		if sys.version_info[0] == 2:
			options_hash = buffer(options_hash)

		with self.cache_lock:
			cursor = self.cache_db.cursor()
			cursor.execute('SELECT return_value FROM cache WHERE method = ? AND options_hash = ?', (method, options_hash))
			return_value = cursor.fetchone()
		if return_value:
			return self.cache_serializer_loads(return_value[0], 'UTF-8')
		return_value = self.call(method, *options)
		with self.cache_lock:
			cursor = self.cache_db.cursor()
			cursor.execute('INSERT INTO cache (method, options_hash, return_value) VALUES (?, ?, ?)', (method, options_hash, self.cache_serializer_dumps(return_value)))
			self.cache_db.commit()
		return return_value

	def cache_call_refresh(self, method, *options):
		"""
		Call a remote method and update the local cache with the result
		if it already existed.

		:param str method: The name of the remote procedure to execute.
		:return: The return value from the remote function.
		"""
		options_hash = self.encode(options)
		if len(options_hash) > 20:
			options_hash = hashlib.new('sha1', options).digest()
		if sys.version_info[0] == 2:
			options_hash = buffer(options_hash)

		with self.cache_lock:
			cursor = self.cache_db.cursor()
			cursor.execute('DELETE FROM cache WHERE method = ? AND options_hash = ?', (method, options_hash))
		return_value = self.call(method, *options)
		with self.cache_lock:
			cursor = self.cache_db.cursor()
			cursor.execute('INSERT INTO cache (method, options_hash, return_value) VALUES (?, ?, ?)', (method, options_hash, self.cache_serializer_dumps(return_value)))
			self.cache_db.commit()
		return return_value

	def cache_clear(self):
		"""Purge the local store of all cached function information."""
		with self.cache_lock:
			cursor = self.cache_db.cursor()
			cursor.execute('DELETE FROM cache')
			self.cache_db.commit()<commit_msg>Remove the cache encoders and store ret values as blobs<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  AdvancedHTTPServer.py
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following disclaimer
#    in the documentation and/or other materials provided with the
#    distribution.
#  * Neither the name of the project nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#  pylint: disable=too-many-lines

#  Homepage: https://github.com/zeroSteiner/AdvancedHTTPServer
#  Author:   <NAME> (zeroSteiner)

# Config file example
FILE_CONFIG = """
[server]
ip = 0.0.0.0
port = 8080
web_root = /var/www/html
list_directories = True
# Set an ssl_cert to enable SSL
# ssl_cert = /path/to/cert.pem
"""

# The AdvancedHTTPServer systemd service unit file
# Quick how to:
#   1. Copy this file to /etc/systemd/system/pyhttpd.service
#   2. Edit the run parameters appropriately in the ExecStart option
#   3. Set configuration settings in /etc/pyhttpd.conf
#   4. Run "systemctl daemon-reload"
FILE_SYSTEMD_SERVICE_UNIT = """
[Unit]
Description=Python Advanced HTTP Server
After=network.target

[Service]
Type=simple
ExecStart=/sbin/runuser -l nobody -c "/usr/bin/python -m AdvancedHTTPServer -c /etc/pyhttpd.conf"
ExecStop=/bin/kill -INT $MAINPID

[Install]
WantedBy=multi-user.target
"""

__version__ = '0.4.3'
__all__ = [
	'AdvancedHTTPServer',
	'AdvancedHTTPServerRegisterPath',
	'AdvancedHTTPServerRequestHandler',
	'AdvancedHTTPServerRPCClient',
	'AdvancedHTTPServerRPCClientCached',
	'AdvancedHTTPServerRPCError',
	'AdvancedHTTPServerTestCase',
	'build_server_from_argparser',
	'build_server_from_config'
]

import base64
import binascii
import datetime
import hashlib
import hmac
import io
import json
import logging
import logging.handlers
import mimetypes
import os
import posixpath
import random
import re
import shutil
import socket
import sqlite3
import ssl
import string
import sys
import threading
import time
import traceback
import unittest
import urllib
import zlib

if sys.version_info[0] < 3:
	import BaseHTTPServer
	import cgi as html
	import Cookie
	import httplib
	import SocketServer as socketserver
	import urlparse
	http = type('http', (), {'client': httplib, 'cookies': Cookie, 'server': BaseHTTPServer})
	urllib.parse = urlparse
	urllib.parse.quote = urllib.quote
	urllib.parse.unquote = urllib.unquote
	from ConfigParser import ConfigParser
else:
	import html
	import http.client
	import http.cookies
	import http.server
	import socketserver
	import urllib.parse
	from configparser import ConfigParser

GLOBAL_HANDLER_MAP = {}

def _serialize_ext_dump(obj):
	if obj.__class__ == datetime.date:
		return 'datetime.date', obj.isoformat()
	elif obj.__class__ == datetime.datetime:
		return 'datetime.datetime', obj.isoformat()
	elif obj.__class__ == datetime.time:
		return 'datetime.time', obj.isoformat()
	raise TypeError('Unknown type: ' + repr(obj))

def _serialize_ext_load(obj_type, obj_value, default):
	if obj_type == 'datetime.date':
		return datetime.datetime.strptime(obj_value, '%Y-%m-%d').date()
	elif obj_type == 'datetime.datetime':
		return datetime.datetime.strptime(obj_value, '%Y-%m-%dT%H:%M:%S' + ('.%f' if '.' in obj_value else ''))
	elif obj_type == 'datetime.time':
		return datetime.datetime.strptime(obj_value, '%H:%M:%S' + ('.%f' if '.' in obj_value else '')).time()
	return default

def _json_default(obj):
	obj_type, obj_value = _serialize_ext_dump(obj)
	return {'__complex_type__': obj_type, 'value': obj_value}

def _json_object_hook(obj):
	return _serialize_ext_load(obj.get('__complex_type__'), obj.get('value'), obj)

SERIALIZER_DRIVERS = {}
"""Dictionary of available drivers for serialization."""
SERIALIZER_DRIVERS['application/json'] = {'loads': lambda d, e: json.loads(d, object_hook=_json_object_hook), 'dumps': lambda d: json.dumps(d, default=_json_default)}

try:
	import msgpack
except ImportError:
	has_msgpack = False
else:
	has_msgpack = True
	_MSGPACK_EXT_TYPES = {10: 'datetime.datetime', 11: 'datetime.date', 12: 'datetime.time'}
	def _msgpack_default(obj):
		obj_type, obj_value = _serialize_ext_dump(obj)
		obj_type = next(i[0] for i in _MSGPACK_EXT_TYPES.items() if i[1] == obj_type)
		if sys.version_info[0] == 3:
			obj_value = obj_value.encode('utf-8')
		return msgpack.ExtType(obj_type, obj_value)

	def _msgpack_ext_hook(code, obj_value):
		default = msgpack.ExtType(code, obj_value)
		if sys.version_info[0] == 3:
			obj_value = obj_value.decode('utf-8')
		obj_type = _MSGPACK_EXT_TYPES.get(code)
		return _serialize_ext_load(obj_type, obj_value, default)
	SERIALIZER_DRIVERS['binary/message-pack'] = {'loads': lambda d, e: msgpack.loads(d, encoding=e, ext_hook=_msgpack_ext_hook), 'dumps': lambda d: msgpack.dumps(d, default=_msgpack_default)}

if hasattr(logging, 'NullHandler'):
	logging.getLogger('AdvancedHTTPServer').addHandler(logging.NullHandler())

def random_string(size):
	"""
	Generate a random string of *size* length consisting of both letters
	and numbers. This function is not meant for cryptographic purposes
	and should not be used to generate security tokens.

	:param int size: The length of the string to return.
	:return: A string consisting of random characters.
	:rtype: str
	"""
	return ''.join(random.choice(string.ascii_letters + string.digits) for x in range(size))

def resolve_ssl_protocol_version(version=None):
	"""
	Look up an SSL protocol version by name. If *version* is not specified, then
	the strongest protocol available will be returned.

	:param str version: The name of the version to look up.
	:return: A protocol constant from the :py:mod:`ssl` module.
	:rtype: int
	"""
	if version == None:
		protocol_preference = ('TLSv1_2', 'TLSv1_1', 'TLSv1', 'SSLv3', 'SSLv23', 'SSLv2')
		for protocol in protocol_preference:
			if hasattr(ssl, 'PROTOCOL_' + protocol):
				return getattr(ssl, 'PROTOCOL_' + protocol)
		raise RuntimeError('could not find a suitable ssl PROTOCOL_ version constant')
	elif isinstance(version, str):
		if not hasattr(ssl, 'PROTOCOL_' + version):
			raise ValueError('invalid ssl protocol version: ' + version)
		return getattr(ssl, 'PROTOCOL_' + version)
	raise TypeError("ssl_version() argument 1 must be str, not {0}".format(type(version).__name__))

def build_serializer_from_content_type(content_type):
	"""
	Build a serializer object from a MIME Content-Type string.

	:param str content_type: The Content-Type string to parse.
	:return: A new configured serializer instance.
	:rtype: :py:class:`.AdvancedHTTPServerSerializer`
	"""
	name = content_type
	options = {}
	if ';' in content_type:
		name, options_str = content_type.split(';', 1)
		for part in options_str.split(';'):
			part = part.strip()
			if '=' in part:
				key, value = part.split('=')
			else:
				key, value = (part, None)
			options[key] = value
	# old style compatibility
	if name.endswith('+zlib'):
		options['compression'] = 'zlib'
		name = name[:-5]
	return AdvancedHTTPServerSerializer(name, charset=options.get('charset', 'UTF-8'), compression=options.get('compression'))

def build_server_from_argparser(description=None, ServerClass=None, HandlerClass=None):
	"""
	Build a server from command line arguments. If a ServerClass or
	HandlerClass is specified, then the object must inherit from the
	corresponding AdvancedHTTPServer base class.

	:param str description: Description string to be passed to the argument parser.
	:param ServerClass: Alternative server class to use.
	:type ServerClass: :py:class:`.AdvancedHTTPServer`
	:param HandlerClass: Alternative handler class to use.
	:type HandlerClass: :py:class:`.AdvancedHTTPServerRequestHandler`
	:return: A configured server instance.
	:rtype: :py:class:`.AdvancedHTTPServer`
	"""
	import argparse

	def _argp_dir_type(arg):
		if not os.path.isdir(arg):
			raise argparse.ArgumentTypeError("{0} is not a valid directory".format(repr(arg)))
		return arg

	def _argp_port_type(arg):
		if not arg.isdigit():
			raise argparse.ArgumentTypeError("{0} is not a valid port".format(repr(arg)))
		arg = int(arg)
		if arg < 0 or arg > 65535:
			raise argparse.ArgumentTypeError("{0} is not a valid port".format(repr(arg)))
		return arg

	description = (description or 'HTTP Server')
	ServerClass = (ServerClass or AdvancedHTTPServer)
	HandlerClass = (HandlerClass or AdvancedHTTPServerRequestHandler)

	parser = argparse.ArgumentParser(conflict_handler='resolve', description=description, fromfile_prefix_chars='@')
	parser.epilog = 'When a config file is specified with --config the --ip, --port and --web-root options are all ignored.'
	parser.add_argument('-w', '--web-root', dest='web_root', action='store', default='.', type=_argp_dir_type, help='path to the web root directory')
	parser.add_argument('-p', '--port', dest='port', action='store', default=8080, type=_argp_port_type, help='port to serve on')
	parser.add_argument('-i', '--ip', dest='ip', action='store', default='0.0.0.0', help='the ip address to serve on')
	parser.add_argument('--password', dest='password', action='store', default=None, help='password to use for basic authentication')
	parser.add_argument('--log-file', dest='log_file', action='store', default=None, help='log information to a file')
	parser.add_argument('-c', '--conf', dest='config', action='store', default=None, type=argparse.FileType('r'), help='read settings from a config file')
	parser.add_argument('-v', '--version', action='version', version=parser.prog + ' Version: ' + __version__)
	parser.add_argument('-L', '--log', dest='loglvl', action='store', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], default='INFO', help='set the logging level')
	arguments = parser.parse_args()

	logging.getLogger('').setLevel(logging.DEBUG)
	console_log_handler = logging.StreamHandler()
	console_log_handler.setLevel(getattr(logging, arguments.loglvl))
	console_log_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-8s %(message)s"))
	logging.getLogger('').addHandler(console_log_handler)

	if arguments.log_file:
		main_file_handler = logging.handlers.RotatingFileHandler(arguments.log_file, maxBytes=262144, backupCount=5)
		main_file_handler.setLevel(logging.DEBUG)
		main_file_handler.setFormatter(logging.Formatter("%(asctime)s %(name)-30s %(levelname)-10s %(message)s"))
		logging.getLogger('').setLevel(logging.DEBUG)
		logging.getLogger('').addHandler(main_file_handler)

	if arguments.config:
		config = ConfigParser()
		config.readfp(arguments.config)
		server = build_server_from_config(config, 'server', ServerClass=ServerClass, HandlerClass=HandlerClass)
	else:
		server = ServerClass(HandlerClass, address=(arguments.ip, arguments.port))
		server.serve_files_root = arguments.web_root

	if arguments.password:
		server.auth_add_creds('', arguments.password)
	return server

def build_server_from_config(config, section_name, ServerClass=None, HandlerClass=None):
	"""
	Build a server from a provided :py:class:`configparser.ConfigParser`
	instance. If a ServerClass or HandlerClass is specified, then the
	object must inherit from the corresponding AdvancedHTTPServer base
	class.

	:param config: Configuration to retrieve settings from.
	:type config: :py:class:`configparser.ConfigParser`
	:param str section_name: The section name of the configuration to use.
	:param ServerClass: Alternative server class to use.
	:type ServerClass: :py:class:`.AdvancedHTTPServer`
	:param HandlerClass: Alternative handler class to use.
	:type HandlerClass: :py:class:`.AdvancedHTTPServerRequestHandler`
	:return: A configured server instance.
	:rtype: :py:class:`.AdvancedHTTPServer`
	"""
	ServerClass = (ServerClass or AdvancedHTTPServer)
	HandlerClass = (HandlerClass or AdvancedHTTPServerRequestHandler)
	port = config.getint(section_name, 'port')
	web_root = None
	if config.has_option(section_name, 'web_root'):
		web_root = config.get(section_name, 'web_root')

	if config.has_option(section_name, 'ip'):
		ip = config.get(section_name, 'ip')
	else:
		ip = '0.0.0.0'
	ssl_certfile = None
	if config.has_option(section_name, 'ssl_cert'):
		ssl_certfile = config.get(section_name, 'ssl_cert')
	ssl_keyfile = None
	if config.has_option(section_name, 'ssl_key'):
		ssl_keyfile = config.get(section_name, 'ssl_key')
	server = ServerClass(HandlerClass, address=(ip, port), ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile)

	if config.has_option(section_name, 'password_type'):
		password_type = config.get(section_name, 'password_type')
	else:
		password_type = '<PASSWORD>'
	if config.has_option(section_name, 'password'):
		password = config.get(section_name, 'password')
		if config.has_option(section_name, 'username'):
			username = config.get(section_name, 'username')
		else:
			username = ''
		server.auth_add_creds(username, password, pwtype=password_type)
	cred_idx = 0
	while config.has_option(section_name, 'password' + str(cred_idx)):
		password = config.get(section_name, 'password' + str(cred_idx))
		if not config.has_option(section_name, 'username' + str(cred_idx)):
			break
		username = config.get(section_name, 'username' + str(cred_idx))
		server.auth_add_creds(username, password, pwtype=password_type)
		cred_idx += 1

	if web_root == None:
		server.serve_files = False
	else:
		server.serve_files = True
		server.serve_files_root = web_root
		if config.has_option(section_name, 'list_directories'):
			server.serve_files_list_directories = config.getboolean(section_name, 'list_directories')
	return server

class AdvancedHTTPServerRegisterPath(object):
	"""
	Register a path and handler with the global handler map. This can be
	used as a decorator. If no handler is specified then the path and
	function will be registered with all :py:class:`.AdvancedHTTPServerRequestHandler`
	instances.

	.. code-block:: python

	  @AdvancedHTTPServerRegisterPath('^test$')
	  def handle_test(handler, query):
	      pass
	"""
	def __init__(self, path, handler=None, is_rpc=False):
		"""
		:param str path: The path regex to register the function to.
		:param str handler: A specific :py:class:`.AdvancedHTTPServerRequestHandler` class to register the handler with.
		:param bool is_rpc: Whether the handler is an RPC handler or not.
		"""
		self.path = path
		self.is_rpc = is_rpc
		if handler == None or isinstance(handler, str):
			self.handler = handler
		elif hasattr(handler, '__name__'):
			self.handler = handler.__name__
		elif hasattr(handler, '__class__'):
			self.handler = handler.__class__.__name__
		else:
			raise ValueError('unknown handler: ' + repr(handler))

	def __call__(self, function):
		handler_map = GLOBAL_HANDLER_MAP.get(self.handler, {})
		handler_map[self.path] = (function, self.is_rpc)
		GLOBAL_HANDLER_MAP[self.handler] = handler_map
		return function

class AdvancedHTTPServerRPCError(Exception):
	"""
	This class represents an RPC error either local or remote. Any errors
	in routines executed on the server will raise this error.
	"""
	def __init__(self, message, status, remote_exception=None):
		super(AdvancedHTTPServerRPCError, self).__init__()
		self.message = message
		self.status = status
		self.remote_exception = remote_exception

	def __repr__(self):
		return "{0}(message='{1}', status={2}, remote_exception={3})".format(self.__class__.__name__, self.message, self.status, self.is_remote_exception)

	def __str__(self):
		if self.is_remote_exception:
			return 'a remote exception occurred'
		return "the server responded with {0} '{1}'".format(self.status, self.message)

	@property
	def is_remote_exception(self):
		"""
		This is true if the represented error resulted from an exception on the
		remote server.

		:type: bool
		"""
		return bool(self.remote_exception != None)

class AdvancedHTTPServerRPCClient(object):
	"""
	This object facilitates communication with remote RPC methods as
	provided by a :py:class:`.AdvancedHTTPServerRequestHandler` instance.
	Once created this object can be called directly, doing so is the same
	as using the call method.

	This object uses locks internally to be thread safe. Only one thread
	can execute a function at a time.
	"""
	def __init__(self, address, use_ssl=False, username=None, password=<PASSWORD>, uri_base='/', hmac_key=None):
		"""
		:param tuple address: The address of the server to conenct to as (host, port).
		:param bool use_ssl: Whether to connect with SSL or not.
		:param str username: The username to authenticate with.
		:param str password: <PASSWORD>.
		:param str uri_base: An optional prefix for all methods.
		:param str hmac_key: An HMAC key to use for request authentication.
		"""
		self.host = str(address[0])
		self.port = int(address[1])
		if not hasattr(self, 'logger'):
			self.logger = logging.getLogger('AdvancedHTTPServer.RPCClient')

		self.use_ssl = bool(use_ssl)
		self.uri_base = str(uri_base)
		self.username = (str(username) if username != None else None)
		self.password = (str(password) if password != None else None)
		if isinstance(hmac_key, str):
			hmac_key = hmac_key.encode('UTF-8')
		self.hmac_key = hmac_key
		self.lock = threading.Lock()
		self.set_serializer('application/json')
		self.reconnect()

	def __reduce__(self):
		address = (self.host, self.port)
		return (self.__class__, (address, self.use_ssl, self.username, self.password, self.uri_base, self.hmac_key))

	def set_serializer(self, serializer_name, compression=None):
		"""
		Configure the serializer to use for communication with the server.
		The serializer specified must be valid and in the
		:py:data:`.SERIALIZER_DRIVERS` map.

		:param str serializer_name: The name of the serializer to use.
		:param str compression: The name of a compression library to use.
		"""
		self.serializer = AdvancedHTTPServerSerializer(serializer_name, charset='UTF-8', compression=compression)
		self.logger.debug('using serializer: ' + serializer_name)

	def __call__(self, *args, **kwargs):
		return self.call(*args, **kwargs)

	def encode(self, data):
		"""Encode data with the configured serializer."""
		return self.serializer.dumps(data)

	def decode(self, data):
		"""Decode data with the configured serializer."""
		return self.serializer.loads(data)

	def reconnect(self):
		"""Reconnect to the remote server."""
		self.lock.acquire()
		if self.use_ssl:
			self.client = http.client.HTTPSConnection(self.host, self.port)
		else:
			self.client = http.client.HTTPConnection(self.host, self.port)
		self.lock.release()

	def call(self, method, *options):
		"""
		Issue a call to the remote end point to execute the specified
		procedure.

		:param str method: The name of the remote procedure to execute.
		:return: The return value from the remote function.
		"""
		options = self.encode(options)

		headers = {}
		headers['Content-Type'] = self.serializer.content_type
		headers['Content-Length'] = str(len(options))

		if self.hmac_key != None:
			hmac_calculator = hmac.new(self.hmac_key, digestmod=hashlib.sha1)
			hmac_calculator.update(options)
			headers['X-RPC-HMAC'] = hmac_calculator.hexdigest()

		if self.username != None and self.password != None:
			headers['Authorization'] = 'Basic ' + base64.b64encode((self.username + ':' + self.password).encode('UTF-8')).decode('UTF-8')

		method = os.path.join(self.uri_base, method)
		self.logger.debug('calling RPC method: ' + method[1:])
		with self.lock:
			self.client.request("RPC", method, options, headers)
			resp = self.client.getresponse()
		if resp.status != 200:
			raise AdvancedHTTPServerRPCError(resp.reason, resp.status)

		resp_data = resp.read()
		if self.hmac_key != None:
			hmac_digest = resp.getheader('X-RPC-HMAC')
			if not isinstance(hmac_digest, str):
				raise AdvancedHTTPServerRPCError('hmac validation error', resp.status)
			hmac_digest = hmac_digest.lower()
			hmac_calculator = hmac.new(self.hmac_key, digestmod=hashlib.sha1)
			hmac_calculator.update(resp_data)
			if hmac_digest != hmac_calculator.hexdigest():
				raise AdvancedHTTPServerRPCError('hmac validation error', resp.status)
		resp_data = self.decode(resp_data)
		if not ('exception_occurred' in resp_data and 'result' in resp_data):
			raise AdvancedHTTPServerRPCError('missing response information', resp.status)
		if resp_data['exception_occurred']:
			raise AdvancedHTTPServerRPCError('remote method incured an exception', resp.status, remote_exception=resp_data['exception'])
		return resp_data['result']

class AdvancedHTTPServerRPCClientCached(AdvancedHTTPServerRPCClient):
	"""
	This object builds upon :py:class:`.AdvancedHTTPServerRPCClient` and
	provides additional methods for cacheing results in memory.
	"""
	def __init__(self, *args, **kwargs):
		cache_db = kwargs.pop('cache_db', ':memory:')
		super(AdvancedHTTPServerRPCClientCached, self).__init__(*args, **kwargs)
		self.cache_db = sqlite3.connect(cache_db, check_same_thread=False)
		cursor = self.cache_db.cursor()
		cursor.execute('CREATE TABLE IF NOT EXISTS cache (method TEXT NOT NULL, options_hash BLOB NOT NULL, return_value BLOB NOT NULL)')
		self.cache_db.commit()
		self.cache_lock = threading.Lock()

	def cache_call(self, method, *options):
		"""
		Call a remote method and store the result locally. Subsequent
		calls to the same method with the same arguments will return the
		cached result without invoking the remote procedure. Cached results are
		kept indefinitely and must be manually refreshed with a call to
		:py:meth:`.cache_call_refresh`.

		:param str method: The name of the remote procedure to execute.
		:return: The return value from the remote function.
		"""
		options_hash = self.encode(options)
		if len(options_hash) > 20:
			options_hash = hashlib.new('sha1', options_hash).digest()
		options_hash = sqlite3.Binary(options_hash)

		with self.cache_lock:
			cursor = self.cache_db.cursor()
			cursor.execute('SELECT return_value FROM cache WHERE method = ? AND options_hash = ?', (method, options_hash))
			return_value = cursor.fetchone()
		if return_value:
			return_value = bytes(return_value[0])
			return self.decode(return_value)
		return_value = self.call(method, *options)
		store_return_value = sqlite3.Binary(self.encode(return_value))
		with self.cache_lock:
			cursor = self.cache_db.cursor()
			cursor.execute('INSERT INTO cache (method, options_hash, return_value) VALUES (?, ?, ?)', (method, options_hash, store_return_value))
			self.cache_db.commit()
		return return_value

	def cache_call_refresh(self, method, *options):
		"""
		Call a remote method and update the local cache with the result
		if it already existed.

		:param str method: The name of the remote procedure to execute.
		:return: The return value from the remote function.
		"""
		options_hash = self.encode(options)
		if len(options_hash) > 20:
			options_hash = hashlib.new('sha1', options).digest()
		options_hash = sqlite3.Binary(options_hash)

		with self.cache_lock:
			cursor = self.cache_db.cursor()
			cursor.execute('DELETE FROM cache WHERE method = ? AND options_hash = ?', (method, options_hash))
		return_value = self.call(method, *options)
		return_value = sqlite3.Binary(self.encode(return_value))
		with self.cache_lock:
			cursor = self.cache_db.cursor()
			cursor.execute('INSERT INTO cache (method, options_hash, return_value) VALUES (?, ?, ?)', (method, options_hash, return_value))
			self.cache_db.commit()
		return return_value

	def cache_clear(self):
		"""Purge the local store of all cached function information."""
		with self.cache_lock:
			cursor = self.cache_db.cursor()
			cursor.execute('DELETE FROM cache')
			self.cache_db.commit()
{'fraction_non_alphanumeric': 0.11396224886584576, 'fraction_numerical': 0.004799902786779002, 'mean_word_length': 5.250253164556962, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '195368', 'n_tokens_mistral': 16408, 'n_tokens_neox': 15452, 'n_words': 4399}