<commit_before><?php
class Log
{
	private static $classes = null;
	private static $handle_id = 0;
	private static $last_timestamp = null;
	private static $last_timestring = null;
	
	public $line = null;
	public $line_offset = null;

	public $client_ip = null;
	public $client_uid = null;
	public $error_level = null;
	public $syslog_timestamp = null;
	public $syslog_host = null;
	public $syslog_program = null;

	public $extra_data = null;


	public function __construct($line)
	{
		$fields = array();

		$this->line = $line;
		$this->error_level = 'unknown';

		# Regular Expression Map:
		#	'(Sep 11 13:07:27) (10.0.3.8) (.*)'
		#	'(Sep 11 13:07:27) (10.0.3.8/10.0.0.16) (.*)'
		if (preg_match('!^([a-z]{3} +\d+ +[0-9:]{8}) +([0-9./]+) +(.*)\n?$!i', $this->line, $fields))
		{
			$this->syslog_timestamp = strtotime($fields[1]);
			$this->syslog_host = $fields[2];

			$this->extra_data = $fields[3];
		}
		else
		{
			# This is note a default syslog line... we really don't know what to do
			# with it.
			return;
		}

		# Regular Expression Map:
		#	'(php-site:) (.*)'
		#	'(nexopia-parent - initializing:) (.*)'
		if (preg_match('!^([^:]+:) +(.*)!', $this->extra_data, $fields))
		{
			$this->syslog_program = $fields[1];

			$this->extra_data = $fields[2];
		}

		# Regular Expression Map:
		#  '([3443893/)(172.16.58.3]) (.*)'
		if (preg_match('!^(\[[0-9-]+/)([0-9.]+\]) +(.*)!', $this->extra_data, $fields))
		{
			$this->client_uid = $fields[1];
			$this->client_ip = $fields[2];

			$this->extra_data = $fields[3];
		}
		# Regular Expression Map:
		#  '([172.16.58.3]) (.*)'
		elseif (preg_match('!^(\[[0-9.]+\]) +(.*)!', $this->extra_data, $fields))
		{
			$this->client_ip = $fields[1];

			$this->extra_data = $fields[2];
		}
	}


	public function __toString()
	{
		if (is_null($this->error_level))
		{
			return $this->line;
		}

		$string  = '';
		if (DEBUG)
		{
			$string .= '<span class="debug">Begin ' . __CLASS__ . '</span>';
		}
		if (self::$last_timestamp != $this->syslog_timestamp)
		{
			self::$last_timestamp = $this->syslog_timestamp;
			self::$last_timestring = htmlspecialchars(strftime('%b %e %H:%M:%S', $this->syslog_timestamp), ENT_QUOTES);			
		}
		$string .= '<span class="date">' . self::$last_timestring . '</span> ';
		$string .= '<span class="host">' . htmlspecialchars($this->syslog_host, ENT_QUOTES) . '</span> ';
		$string .= '<span class="program">' . htmlspecialchars($this->syslog_program, ENT_QUOTES) . '</span> ';
		if (! is_null($this->client_uid))
		{
			$string .= '<span class="uid">' . htmlspecialchars($this->client_uid, ENT_QUOTES) . '</span>';
		}
		if (! is_null($this->client_ip))
		{
			$string .= '<span class="ip">' . htmlspecialchars($this->client_ip, ENT_QUOTES) . '</span> ';
		}
		$string .= htmlspecialchars($this->extra_data, ENT_QUOTES);
		if (DEBUG)
		{
			$string .= '<span class="debug">End ' . __CLASS__ . '</span>';
		}
		return $string;
	}

<commit_msg>* Performance improvement: cache input time parsing.<commit_after><?php
class Log
{
	private static $classes = null;
	private static $handle_id = 0;
	private static $last_input_timestamp = null;
	private static $last_input_timestring = null;
	private static $last_output_timestamp = null;
	private static $last_output_timestring = null;
	
	public $line = null;
	public $line_offset = null;

	public $client_ip = null;
	public $client_uid = null;
	public $error_level = null;
	public $syslog_timestamp = null;
	public $syslog_host = null;
	public $syslog_program = null;

	public $extra_data = null;


	public function __construct($line)
	{
		$fields = array();

		$this->line = $line;
		$this->error_level = 'unknown';

		# Regular Expression Map:
		#	'(Sep 11 13:07:27) (10.0.3.8) (.*)'
		#	'(Sep 11 13:07:27) (10.0.3.8/10.0.0.16) (.*)'
		if (preg_match('!^([a-z]{3} +\d+ +[0-9:]{8}) +([0-9./]+) +(.*)\n?$!i', $this->line, $fields))
		{
			if (strcmp(self::$last_input_timestring, $fields[1]))
			{
				self::$last_input_timestring = $fields[1];
				self::$last_input_timestamp = strtotime($fields[1]);
			}
			$this->syslog_timestamp = self::$last_input_timestamp;
			$this->syslog_host = $fields[2];

			$this->extra_data = $fields[3];
		}
		else
		{
			# This is note a default syslog line... we really don't know what to do
			# with it.
			return;
		}

		# Regular Expression Map:
		#	'(php-site:) (.*)'
		#	'(nexopia-parent - initializing:) (.*)'
		if (preg_match('!^([^:]+:) +(.*)!', $this->extra_data, $fields))
		{
			$this->syslog_program = $fields[1];

			$this->extra_data = $fields[2];
		}

		# Regular Expression Map:
		#  '([3443893/)(172.16.58.3]) (.*)'
		if (preg_match('!^(\[[0-9-]+/)([0-9.]+\]) +(.*)!', $this->extra_data, $fields))
		{
			$this->client_uid = $fields[1];
			$this->client_ip = $fields[2];

			$this->extra_data = $fields[3];
		}
		# Regular Expression Map:
		#  '([172.16.58.3]) (.*)'
		elseif (preg_match('!^(\[[0-9.]+\]) +(.*)!', $this->extra_data, $fields))
		{
			$this->client_ip = $fields[1];

			$this->extra_data = $fields[2];
		}
	}


	public function __toString()
	{
		if (is_null($this->error_level))
		{
			return $this->line;
		}

		$string  = '';
		if (DEBUG)
		{
			$string .= '<span class="debug">Begin ' . __CLASS__ . '</span>';
		}
		if (self::$last_output_timestamp != $this->syslog_timestamp)
		{
			self::$last_output_timestamp = $this->syslog_timestamp;
			self::$last_output_timestring = htmlspecialchars(strftime('%b %e %H:%M:%S', $this->syslog_timestamp), ENT_QUOTES);			
		}
		$string .= '<span class="date">' . self::$last_output_timestring . '</span> ';
		$string .= '<span class="host">' . htmlspecialchars($this->syslog_host, ENT_QUOTES) . '</span> ';
		$string .= '<span class="program">' . htmlspecialchars($this->syslog_program, ENT_QUOTES) . '</span> ';
		if (! is_null($this->client_uid))
		{
			$string .= '<span class="uid">' . htmlspecialchars($this->client_uid, ENT_QUOTES) . '</span>';
		}
		if (! is_null($this->client_ip))
		{
			$string .= '<span class="ip">' . htmlspecialchars($this->client_ip, ENT_QUOTES) . '</span> ';
		}
		$string .= htmlspecialchars($this->extra_data, ENT_QUOTES);
		if (DEBUG)
		{
			$string .= '<span class="debug">End ' . __CLASS__ . '</span>';
		}
		return $string;
	}


{'fraction_non_alphanumeric': 0.22240128928283642, 'fraction_numerical': 0.02546333601933924, 'mean_word_length': 3.708649468892261, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 10, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1377234', 'n_tokens_mistral': 2747, 'n_tokens_neox': 2435, 'n_words': 439}