<commit_before>
import tornadoredis
import os
import signal
import shutil
import tempfile
import logging
import subprocess
import json
import time

class ManagedRedisError(Exception):
	pass

class ManagedRedis(object):
	redis_server_config = """
daemonize yes
pidfile %(working_dir)s/redis.pid
dir %(working_dir)s
port %(port)d
bind %(host)s

databases 16

# Default set of save entries from Ubuntu/Debian.
save 900 1
save 300 10
save 60 10000

dbfilename dump.rdb

# Optional password line.
%(auth_line)s

timeout 300
loglevel notice
logfile %(working_dir)s/redis.log
appendonly no
vm-enabled no
"""

	def __init__(self, configuration):
		self.configuration = configuration
		self.parameters = {}
		self.client = None

	def configure(self, working_dir, port, bind_host, password=<PASSWORD>):
		"""
		Configure this instance.
		"""
		# TODO: Allow a memory limit.
		# TODO: Allow custom configuration entries.
		self.parameters['working_dir'] = working_dir
		self.parameters['port'] = port
		self.parameters['host'] = bind_host
		self.parameters['password'] = password

		# Create the working dir. If this fails, let it bubble up.
		if not os.path.exists(working_dir):
			os.makedirs(working_dir)

		self.save_parameters()

	def load_parameters(self, working_dir):
		"""
		Load the parameters from the working directory.

		Throws an exception if that directory is not configured.
		"""
		parameters_path = self.get_parameters_path(working_dir)
		config_path = self.get_configuration_path(working_dir)
		if os.path.exists(config_path):
			# TODO: Some more error checking.
			fp = open(config_path, 'r')
			self.parameters = json.loads(fp.read())
			fp.close()
		else:
			raise ManagedRedisError('No configuration file found in the working directory. Looking for %s.' % config_path)

	def save_parameters(self):
		"""
		Save the parameters file to disk.
		"""
		# TODO: More error checking.
		fp = open(self.get_parameters_path(self.parameters['working_dir']), 'w')
		fp.write(json.dumps(self.parameters))
		fp.close()

	def get_parameters_path(self, working_dir):
		"""
		Get the parameters configuration path.
		"""
		return os.path.join(working_dir, 'redis.json')

	def get_configuration_path(self, working_dir):
		"""
		Get the redis server configuration path.
		"""
		return os.path.join(working_dir, 'redis.conf')

	def get_pid(self):
		"""
		Gets the PID of the running instance. This just reads the on disk
		PID file, and may not as such indicate that it's running. Returns
		None if no pid file found.
		"""
		pidpath = os.path.join(self.parameters['working_dir'], 'redis.pid')
		if os.path.exists(pidpath):
			fp = open(pidpath, 'r')
			pid = int(fp.read())
			fp.close()
			return pid
		else:
			return None

	def start_if_not_running(self):
		if not self.is_running():
			self.start()

	def start(self):
		"""
		Start up the server for this instance.

		Throws an exception if the redis server fails to start.
		"""
		# Write out the configuration.
		configfile = self.get_configuration_path(self.parameters['working_dir'])
		if self.parameters['password']:
			self.parameters['auth_line'] = "requirepass %s" % self.parameters['password']
		else:
			self.parameters['auth_line'] = ''
		redisconfig = self.redis_server_config % self.parameters
		fp = open(configfile, 'w')
		fp.write(redisconfig)
		fp.close()

		# Fire up the server.
		logging.info("Starting up redis server on port %d." % self.parameters['port'])
		subprocess.check_call([self.configuration.get_flat('redis_binary'), self.get_configuration_path(self.parameters['working_dir'])])

	def get_client(self, io_loop=None):
		"""
		Get a redis client object for this instance.
		"""
		if self.client:
			return self.client

		# Create a client for it.
		self.client = tornadoredis.Client(host=self.parameters['host'],
			port=self.parameters['port'], io_loop=io_loop)
		self.client.connect()

		return self.client

	def is_running(self):
		# TODO: Make this more robust than checking if the PID file exists.
		if self.get_pid():
			return True
		else:
			return False

	def stop(self, sig=signal.SIGTERM):
		"""
		Stop this instance of the redis server, allowing for it to be restarted later.
		"""
		if self.client:
			self.client.disconnect()
			self.client = None<commit_msg>Updated the Managed Redis to more thoroughly check if the process is running when asked.<commit_after>
import tornadoredis
import os
import signal
import shutil
import tempfile
import logging
import subprocess
import json
import time

from processcheck import ProcessCheck

class ManagedRedisError(Exception):
	pass

class ManagedRedis(object):
	redis_server_config = """
daemonize yes
pidfile %(working_dir)s/redis.pid
dir %(working_dir)s
port %(port)d
bind %(host)s

databases 16

# Default set of save entries from Ubuntu/Debian.
save 900 1
save 300 10
save 60 10000

dbfilename dump.rdb

# Optional password line.
%(auth_line)s

timeout 300
loglevel notice
logfile %(working_dir)s/redis.log
appendonly no
vm-enabled no
"""

	def __init__(self, configuration):
		self.configuration = configuration
		self.parameters = {}
		self.client = None

	def configure(self, working_dir, port, bind_host, password=None):
		"""
		Configure this instance.
		"""
		# TODO: Allow a memory limit.
		# TODO: Allow custom configuration entries.
		self.parameters['working_dir'] = working_dir
		self.parameters['port'] = port
		self.parameters['host'] = bind_host
		self.parameters['password'] = password

		# Create the working dir. If this fails, let it bubble up.
		if not os.path.exists(working_dir):
			os.makedirs(working_dir)

		self.save_parameters()

	def load_parameters(self, working_dir):
		"""
		Load the parameters from the working directory.

		Throws an exception if that directory is not configured.
		"""
		parameters_path = self.get_parameters_path(working_dir)
		config_path = self.get_configuration_path(working_dir)
		if os.path.exists(config_path):
			# TODO: Some more error checking.
			fp = open(config_path, 'r')
			self.parameters = json.loads(fp.read())
			fp.close()
		else:
			raise ManagedRedisError('No configuration file found in the working directory. Looking for %s.' % config_path)

	def save_parameters(self):
		"""
		Save the parameters file to disk.
		"""
		# TODO: More error checking.
		fp = open(self.get_parameters_path(self.parameters['working_dir']), 'w')
		fp.write(json.dumps(self.parameters))
		fp.close()

	def get_parameters_path(self, working_dir):
		"""
		Get the parameters configuration path.
		"""
		return os.path.join(working_dir, 'redis.json')

	def get_configuration_path(self, working_dir):
		"""
		Get the redis server configuration path.
		"""
		return os.path.join(working_dir, 'redis.conf')

	def get_pid(self):
		"""
		Gets the PID of the running instance. This just reads the on disk
		PID file, and may not as such indicate that it's running. Returns
		None if no pid file found.
		"""
		pidpath = os.path.join(self.parameters['working_dir'], 'redis.pid')
		if os.path.exists(pidpath):
			fp = open(pidpath, 'r')
			pid = int(fp.read())
			fp.close()
			return pid
		else:
			return None

	def start_if_not_running(self):
		if not self.is_running():
			self.start()

	def start(self):
		"""
		Start up the server for this instance.

		Throws an exception if the redis server fails to start.
		"""
		# Write out the configuration.
		configfile = self.get_configuration_path(self.parameters['working_dir'])
		if self.parameters['password']:
			self.parameters['auth_line'] = "requirepass %s" % self.parameters['password']
		else:
			self.parameters['auth_line'] = ''
		redisconfig = self.redis_server_config % self.parameters
		fp = open(configfile, 'w')
		fp.write(redisconfig)
		fp.close()

		# Fire up the server.
		logging.info("Starting up redis server on port %d." % self.parameters['port'])
		subprocess.check_call([self.configuration.get_flat('redis_binary'), self.get_configuration_path(self.parameters['working_dir'])])

	def get_client(self, io_loop=None):
		"""
		Get a redis client object for this instance.
		"""
		if self.client:
			return self.client

		# Create a client for it.
		self.client = tornadoredis.Client(host=self.parameters['host'],
			port=self.parameters['port'], io_loop=io_loop)
		self.client.connect()

		return self.client

	def is_running(self):
		pid = self.get_pid()
		if pid:
			# Do an advanced check on pid.
			return ProcessCheck.is_running(pid, 'redis-server')
		else:
			# No PID at all. Not running.
			return False

	def stop(self, sig=signal.SIGTERM):
		"""
		Stop this instance of the redis server, allowing for it to be restarted later.
		"""
		if self.client:
			self.client.disconnect()
			self.client = None
{'fraction_non_alphanumeric': 0.11303544625331947, 'fraction_numerical': 0.0048493245583650845, 'mean_word_length': 4.224366706875754, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3926783', 'n_tokens_mistral': 3088, 'n_tokens_neox': 2755, 'n_words': 800}