<commit_before><?php namespace Dotink\Sage {

	use TokenReflection;

	/**
	 * Provides documentation generation service by reading a particular source directory
	 *
	 * @copyright Copyright (c) 2013, <NAME>
	 * @author <NAME> [mjs] <<EMAIL>>
	 *
	 * @license Please reference the LICENSE.txt file at the root of this distribution
	 *
	 * @package Sage
	 */
	class Generator
	{
		/**
		 * The token reflection broker of the generator
		 *
		 * @access private
		 * @var TokenReflection\Broker
		 */
		private $broker = NULL;


		/**
		 * A collection of documents
		 *
		 * @access private
		 * @var array
		 */
		private $documents = array();


		/**
		 * The input path we will be scanning for source code
		 *
		 * @access private
		 * @var string
		 */
		private $inputPath = NULL;


		/**
		 * Options and values for the generator
		 *
		 * @access private
		 * @var array
		 */
		private $options = array();


		/**
		 * Creates a new generator
		 *
		 * @access public
		 * @param TokenReflection\Broker $broker The token reflection broker to use for reflection
		 * @return void
		 */
		public function __construct(TokenReflection\Broker $broker)
		{
			$this->broker = $broker;
		}


		/**
		 * Gets the token parser class for a given token
		 *
		 * @access public
		 * @param string $token The token to get a parser for
		 * @return string The class for parsing the token, or NULL if not available
		 */
		public function getTokenParser($token)
		{
			$default = __NAMESPACE__ . '\\TokenParser\\Token' . ucwords($token);

			if (isset($this->tokenParsers[$token])) {
				return $this->tokenParsers[$token];
			}

			if (class_exists($default)) {
				return $default;
			}

			return NULL;
		}


		/**
		 * Runs the documentation generator
		 *
		 * @access public
		 * @param string $input_path A relative or absolute directory to scan
		 * @param array $options An array of options and their values
		 * @return void
		 */
		public function run($input_path)
		{
			$this->setInputPath($input_path);

			$config  = $this->inputPath . DIRECTORY_SEPARATOR . 'sage.config.php';
			$options = file_exists($config)
				? include $config
				: array();

			$reflections   = $this->broker->processDirectory($this->inputPath, [], TRUE);
			$sort_by_type  = !empty($options['sort_by_type']);
			$token_parsers = !empty($options['token_parsers'])
				? $options['token_parsers']
				: array();

			$this->configTokenParsers($token_parsers);
			$this->makeDocumentCollection($reflections, $sort_by_type);

			foreach ($reflections as $reflection) {
				foreach ($reflection->getNamespaces() as $namespace) {

					//
					// Add our class type structures
					//

					foreach ($namespace->getClasses() as $structure) {
						$root =& $this->documents;

						if ($sort_by_type) {
							if ($structure->isTrait()) {
								$root =& $this->documents['traits'];

							} elseif ($structure->isInterface()) {
								$root =& $this->documents['interfaces'];
<commit_msg>Sage config renamed to sage.config (no .php extension), added seeking<commit_after><?php namespace Dotink\Sage {

	use TokenReflection;

	/**
	 * Provides documentation generation service by reading a particular source directory
	 *
	 * @copyright Copyright (c) 2013, <NAME>
	 * @author <NAME> [mjs] <<EMAIL>>
	 *
	 * @license Please reference the LICENSE.txt file at the root of this distribution
	 *
	 * @package Sage
	 */
	class Generator
	{
		/**
		 * The token reflection broker of the generator
		 *
		 * @access private
		 * @var TokenReflection\Broker
		 */
		private $broker = NULL;


		/**
		 * A collection of documents
		 *
		 * @access private
		 * @var array
		 */
		private $documents = array();


		/**
		 * The input path we will be scanning for source code
		 *
		 * @access private
		 * @var string
		 */
		private $inputPath = NULL;


		/**
		 * Options and values for the generator
		 *
		 * @access private
		 * @var array
		 */
		private $options = array();


		/**
		 * Creates a new generator
		 *
		 * @access public
		 * @param TokenReflection\Broker $broker The token reflection broker to use for reflection
		 * @return void
		 */
		public function __construct(TokenReflection\Broker $broker)
		{
			$this->broker = $broker;
		}


		/**
		 * Gets the token parser class for a given token
		 *
		 * @access public
		 * @param string $token The token to get a parser for
		 * @return string The class for parsing the token, or NULL if not available
		 */
		public function getTokenParser($token)
		{
			$default = __NAMESPACE__ . '\\TokenParser\\Token' . ucwords($token);

			if (isset($this->tokenParsers[$token])) {
				return $this->tokenParsers[$token];
			}

			if (class_exists($default)) {
				return $default;
			}

			return NULL;
		}


		/**
		 * Runs the documentation generator
		 *
		 * @access public
		 * @param string $input_path A relative or absolute directory to scan
		 * @param array $options An array of options and their values
		 * @return void
		 */
		public function run($input_path)
		{
			$this->setInputPath($input_path);

			for (
				$config_path  = realpath($this->inputPath);

				$config_path != realpath($config_path . DIRECTORY_SEPARATOR . '..')
				&& !is_readable($config_path . DIRECTORY_SEPARATOR . 'sage.config');

				$config_path  = realpath($config_path . DIRECTORY_SEPARATOR . '..')
			);

			$options = file_exists($config_path . DIRECTORY_SEPARATOR . 'sage.config')
				? include $config_path . DIRECTORY_SEPARATOR . 'sage.config'
				: array();

			$reflections   = $this->broker->processDirectory($this->inputPath, [], TRUE);
			$sort_by_type  = !empty($options['sort_by_type']);
			$token_parsers = !empty($options['token_parsers'])
				? $options['token_parsers']
				: array();

			$this->configTokenParsers($token_parsers);
			$this->makeDocumentCollection($reflections, $sort_by_type);

			foreach ($reflections as $reflection) {
				foreach ($reflection->getNamespaces() as $namespace) {

					//
					// Add our class type structures
					//

					foreach ($namespace->getClasses() as $structure) {
						$root =& $this->documents;

						if ($sort_by_type) {
							if ($structure->isTrait()) {
								$root =& $this->documents['traits'];

							} elseif ($structure->isInterface()) {
								$root =& $this->documents['interfaces'];

{'fraction_non_alphanumeric': 0.13022738114167595, 'fraction_numerical': 0.0012720623310542216, 'mean_word_length': 2.98858592263792, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3695492', 'n_tokens_mistral': 2321, 'n_tokens_neox': 1814, 'n_words': 527}