<commit_before><?php

Class Helpers {
	
	static function rglob($pattern, $flags = 0, $path = '') {
		if (!$path && ($dir = dirname($pattern)) != '.') {
			if ($dir == '\\' || $dir == '/') $dir = '';
			return self::rglob(basename($pattern), $flags, $dir . '/');
		}
		$paths = glob($path . '*', GLOB_ONLYDIR | GLOB_NOSORT);
		$files = glob($path . $pattern, $flags);
		foreach ($paths as $p) $files = array_merge($files, self::rglob($pattern, $flags, $p . '/'));
		return $files;
	}
	
	static function sort_by_length($a,$b){
		if($a == $b) return 0;
		return (strlen($a) > strlen($b) ? -1 : 1);
	}
	
	static function file_path_to_url($file_path) {
		return preg_replace(array('/\d+?\./', '/\.\/content(\/)?/'), '', $file_path);
	}
	
	static function url_to_file_path($url) {
		$file_path = './content';
		# Split the url and recursively unclean the parts into folder names
		$url_parts = explode('/', $url);
		foreach($url_parts as $u) {
				# Look for a folder at the current $path
				$matches = array_keys(Helpers::list_files($file_path, '/^\d+?\.'.$u.'$/', true));
				# No matches means a bad url
				if(empty($matches)) return false; 
				else $file_path .=  '/'.$matches[0];
		}
		return $file_path;
	}
	
	static function has_children($dir) {
		# check if this folder contains inner folders - if it does, then it is a category
		$inner_folders = Helpers::list_files($dir, '/.*/', true);
		return !empty($inner_folders);
	}

	static function list_files($dir, $regex, $folders_only = false) {
		if(!is_dir($dir)) return array();
		$glob = ($folders_only) ? glob($dir."/*", GLOB_ONLYDIR) : glob($dir."/*");
		if(!$glob) return array();
		# loop through each glob result and push it to $dirs if it matches the passed regexp
		$files = array();<commit_msg>Converting a file path of './content' to 'index' should be handled by the 'file_path_to_url()' function<commit_after><?php

Class Helpers {
	
	static function rglob($pattern, $flags = 0, $path = '') {
		if (!$path && ($dir = dirname($pattern)) != '.') {
			if ($dir == '\\' || $dir == '/') $dir = '';
			return self::rglob(basename($pattern), $flags, $dir . '/');
		}
		$paths = glob($path . '*', GLOB_ONLYDIR | GLOB_NOSORT);
		$files = glob($path . $pattern, $flags);
		foreach ($paths as $p) $files = array_merge($files, self::rglob($pattern, $flags, $p . '/'));
		return $files;
	}
	
	static function sort_by_length($a,$b){
		if($a == $b) return 0;
		return (strlen($a) > strlen($b) ? -1 : 1);
	}
	
	static function file_path_to_url($file_path) {
	  $url = preg_replace(array('/\d+?\./', '/\.\/content(\/)?/'), '', $file_path);
		return $url ? $url : 'index';
	}
	
	static function url_to_file_path($url) {
		$file_path = './content';
		# Split the url and recursively unclean the parts into folder names
		$url_parts = explode('/', $url);
		foreach($url_parts as $u) {
				# Look for a folder at the current $path
				$matches = array_keys(Helpers::list_files($file_path, '/^\d+?\.'.$u.'$/', true));
				# No matches means a bad url
				if(empty($matches)) return false; 
				else $file_path .=  '/'.$matches[0];
		}
		return $file_path;
	}
	
	static function has_children($dir) {
		# check if this folder contains inner folders - if it does, then it is a category
		$inner_folders = Helpers::list_files($dir, '/.*/', true);
		return !empty($inner_folders);
	}

	static function list_files($dir, $regex, $folders_only = false) {
		if(!is_dir($dir)) return array();
		$glob = ($folders_only) ? glob($dir."/*", GLOB_ONLYDIR) : glob($dir."/*");
		if(!$glob) return array();
		# loop through each glob result and push it to $dirs if it matches the passed regexp
		$files = array();
{'fraction_non_alphanumeric': 0.20032976092333057, 'fraction_numerical': 0.0027480076944215444, 'mean_word_length': 3.770642201834862, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '68906', 'n_tokens_mistral': 1385, 'n_tokens_neox': 1278, 'n_words': 347}