<commit_before><?php

class GrammatistaScannerFilesystem extends FilterIterator implements IGrammatistaScanner
{
	public function __construct(array $options)
	{
		if(!isset($options['filesystem.path'])) {
			throw new GrammatistaException('No path given for GrammatistaScannerFilesystem');
		}
		
		$this->innerIterator = new RecursiveIteratorIterator(new GrammatistaScannerFilesystemRecursivedirectoryiterator($options), RecursiveIteratorIterator::LEAVES_ONLY | RecursiveIteratorIterator::CHILD_FIRST);
		
		parent::__construct($this->innerIterator);
	}
	
	public function __call($name, $args)
	{
		return call_user_func_array(array($this->innerIterator, $name), $args);
	}
	
	public function getInnerIterator()
	{
		return $this->innerIterator;
	}
	
	public function accept()
	{
		return $this->innerIterator->isFile();
	}
	
	public function current()
	{
		$current = $this->innerIterator->current();
		
		$retval = new GrammatistaEntity(array(
			'ident' => $current->getRealpath(),
			'type' => pathinfo($current->getPathname(), PATHINFO_EXTENSION),
			'content' => file_get_contents($current->getRealpath()),
		));
		
		return $retval;
	}
}

?><commit_msg>added ability for filesystem scanner to strip parts of the path handed to parsers as ident, refs #6<commit_after><?php

class GrammatistaScannerFilesystem extends FilterIterator implements IGrammatistaScanner
{
	protected $options = array();
	
	public function __construct(array $options)
	{
		if(!isset($options['filesystem.path'])) {
			throw new GrammatistaException('No path given for GrammatistaScannerFilesystem');
		}
		
		if(!isset($options['filesystem.ident.strip'])) {
			$options['filesystem.ident.strip'] = $options['filesystem.path'] . '/';
		}
		
		$this->options = $options;
		
		$this->innerIterator = new RecursiveIteratorIterator(new GrammatistaScannerFilesystemRecursivedirectoryiterator($options), RecursiveIteratorIterator::LEAVES_ONLY | RecursiveIteratorIterator::CHILD_FIRST);
		
		parent::__construct($this->innerIterator);
	}
	
	public function __call($name, $args)
	{
		return call_user_func_array(array($this->innerIterator, $name), $args);
	}
	
	public function getInnerIterator()
	{
		return $this->innerIterator;
	}
	
	public function accept()
	{
		return $this->innerIterator->isFile();
	}
	
	public function current()
	{
		$current = $this->innerIterator->current();
		
		$retval = new GrammatistaEntity(array(
			'ident' => preg_replace('#^' . preg_quote($this->options['filesystem.ident.strip'], '#') . '#', '', $current->getRealpath()),
			'type' => pathinfo($current->getPathname(), PATHINFO_EXTENSION),
			'content' => file_get_contents($current->getRealpath()),
		));
		
		return $retval;
	}
}

?>
{'fraction_non_alphanumeric': 0.14169139465875372, 'fraction_numerical': 0.000370919881305638, 'mean_word_length': 5.6923076923076925, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1449977', 'n_tokens_mistral': 905, 'n_tokens_neox': 804, 'n_words': 149}