<commit_before><?php
/**
 * Yarrow {{version}}
 * Simple Documentation Generator
 * <http://yarrowdoc.org>
 *
 * Copyright (c) 2010-2011, <NAME> <http://maetl.net>
 * All rights reserved.
 *
 * This library is free software; refer to the terms in the LICENSE file found
 * with this source code for details about modification and redistribution.
 */

abstract class FilePatternFilter extends RecursiveFilterIterator {
	protected $pattern;

	public function __construct(RecursiveIterator $it, $pattern) {
		$this->pattern = $pattern;
		parent::__construct($it);
	}
}

class FileIncludeFilter extends FilePatternFilter {

	public function accept() {
		return ($this->isFile() && fnmatch($this->pattern, $this->getFilename()));
	}
}

class FileExcludeFilter extends FilePatternFilter {

	public function accept() {
		return ($this->isFile() && !fnmatch($this->pattern, $this->getFilename()));
	}
}

class FilePatternIncludeFilter extends FilePatternFilter {

	public function accept() {
		return ($this->isFile() && preg_match($this->pattern, $this->getFilename()));
	}
}

class FilePatternExcludeFilter extends FilePatternFilter {

	public function accept() {
		return ($this->isFile() && !preg_match($this->pattern, $this->getFilename()));
	}
}

class FileCollector {
	private $base_path;
	private $base_dir;
	private $collection;

	function __construct($path) {
		$this->base_path = $path;
		$this->base_dir = basename($path);
		$this->collection = new RecursiveDirectoryIterator($path);
	}
	
	function includeByMatch($pattern) {
		$this->collection = new FileIncludeFilter($this->collection, $pattern);
	}
	
	function excludeByMatch($pattern) {
		$this->collection = new FileExcludeFilter($this->collection, $pattern);
	}

	function includeByPattern($pattern) {
		$this->collection = new FilePatternIncludeFilter($this->collection, $pattern);
	}

	function excludeByPattern($pattern) {
		$this->collection = new FilePatternExcludeFilter($this->collection, $pattern);
	}

	function getManifest() {
		$map = array();
		foreach(new RecursiveIteratorIterator($this->collection) as $file) {
			$map[] = array(
				"filename" => $file->getFilename(),
				"relative_path" => $this->base_dir . str_replace($this->base_path, '', $file->getPathname()),
				"absolute_path" => $file->getRealPath()
			);
		}
		return $map;
	}

}<commit_msg>fix recursion over subdirectories with provided pattern filters<commit_after><?php
/**
 * Yarrow {{version}}
 * Simple Documentation Generator
 * <http://yarrowdoc.org>
 *
 * Copyright (c) 2010-2011, <NAME> <http://maetl.net>
 * All rights reserved.
 *
 * This library is free software; refer to the terms in the LICENSE file found
 * with this source code for details about modification and redistribution.
 */

abstract class FilePatternFilter extends RecursiveRegexIterator {
	protected $pattern;

	public function __construct(RecursiveIterator $it, $pattern) {
		$this->pattern = $pattern;
		parent::__construct($it, $pattern);
	}

}

class FilePatternIncludeFilter extends FilePatternFilter {

	public function accept() {		
		return ($this->isDir() || preg_match($this->pattern, $this->getFilename()));
	}
}

class FilePatternExcludeFilter extends FilePatternFilter {

	public function accept() {
		return ($this->isDir() || !preg_match($this->pattern, $this->getFilename()));
	}
}

class FileCollector {
	private $base_path;
	private $base_dir;
	private $collection;

	function __construct($path) {
		$this->base_path = $path;
		$this->base_dir = basename($path);
		$this->collection = new RecursiveDirectoryIterator($path);
	}
	
	function includeByMatch($pattern) {
		$pattern = str_replace('.', '\.', $pattern);
		$pattern = str_replace('*', '.+', $pattern);
		$this->includeByPattern("/$pattern$/");
	}
	
	function excludeByMatch($pattern) {
		$pattern = str_replace('.', '\.', $pattern);
		$pattern = str_replace('*', '.+', $pattern);
		$this->excludeByPattern("/$pattern$/");
	}

	function includeByPattern($pattern) {
		$this->collection = new FilePatternIncludeFilter($this->collection, $pattern);
	}

	function excludeByPattern($pattern) {
		$this->collection = new FilePatternExcludeFilter($this->collection, $pattern);
	}
	
	function getIterator() {
		return new RecursiveIteratorIterator($this->collection);
	}

	function getManifest() {
		$map = array();
		$files = new RecursiveIteratorIterator($this->collection);
		foreach($files as $file) {
			$map[] = array(
				"filename" => $file->getFilename(),
				"relative_path" => $this->base_dir . str_replace($this->base_path, '', $file->getPathname()),
				"absolute_path" => $file->getRealPath()
			);
		}
		return $map;
	}

}
{'fraction_non_alphanumeric': 0.15592560553633217, 'fraction_numerical': 0.0034602076124567475, 'mean_word_length': 5.077529566360052, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1039215', 'n_tokens_mistral': 1493, 'n_tokens_neox': 1386, 'n_words': 317}