<commit_before><?php defined('BLUDIT') or die('Bludit CMS.');

class Filesystem {

	// Returns an array with the absolutes directories.
	public static function listDirectories($path, $regex='*', $sortByDate=false)
	{
		$directories = glob($path.$regex, GLOB_ONLYDIR);

		if(empty($directories)) {
			return array();
		}

		if($sortByDate) {
			usort($directories,
			      function($a, $b) {
				      return filemtime($b) - filemtime($a);
			      }
			);
		}

		return $directories;
	}

	// Returns an array with the list of files with the absolute path
	// $sortByDate = TRUE, the first file is the newer file
	// $chunk = amount of chunks, FALSE if you don't want to chunk
	public static function listFiles($path, $regex='*', $extension='*', $sortByDate=false, $chunk=false)
	{
		$files = glob($path.$regex.'.'.$extension);

		if (empty($files)) {
			return array();
		}

		if ($sortByDate) {
			usort($files,
				function($a, $b) {
					return filemtime($b) - filemtime($a);
				}
			);
		}

		// Split the list of files into chunks
		// http://php.net/manual/en/function.array-chunk.php
		if ($chunk) {
			return array_chunk($files, $chunk);
		}

		return $files;
	}

	public static function mkdir($pathname, $recursive=false)
	{
		return mkdir($pathname, DIR_PERMISSIONS, $recursive);
	}

	public static function rmdir($pathname)
	{
		return rmdir($pathname);
	}

	public static function mv($oldname, $newname)
	{
		return rename($oldname, $newname);
	}

	public static function rmfile($filename)
	{
		return unlink($filename);
	}

	public static function fileExists($filename)
	{
		return file_exists($filename);
	}

	public static function directoryExists($path)
	{
		return file_exists($path);
	}

	// Copy recursive a directory to another
	// If the destination directory not exists is created
	// $source = /home/diego/example or /home/diego/example/
	// $destination = /home/diego/newplace or /home/diego/newplace/
	public static function copyRecursive($source, $destination, $skipDirectory=false)
	{
		$source 	= rtrim($source, DS);
		$destination 	= rtrim($destination, DS);

		// Check $source directory if exists
		if (!self::directoryExists($source)) {
			return false;
		}

		// Check $destionation directory if exists
		if (!self::directoryExists($destination)) {
			// Create the $destination directory
			if (!mkdir($destination, DIR_PERMISSIONS, true)) {
				return false;
			}
		}

		foreach ($iterator = new RecursiveIteratorIterator(
				new RecursiveDirectoryIterator($source, RecursiveDirectoryIterator::SKIP_DOTS),
				RecursiveIteratorIterator::SELF_FIRST) as $item) {

			$currentDirectory = dirname($item->getPathName());
			if ($skipDirectory !== $currentDirectory) {
				if ($item->isDir()) {
					@mkdir($destination.DS.$iterator->getSubPathName());
				} else {
					copy($item, $destination.DS.$iterator->getSubPathName());
				}
			}
		}

		return true;
	}

	// Delete a file or directory recursive
	// The directory is delete
	public static function deleteRecursive($source, $deleteDirectory=true)
	{
		if (!self::directoryExists($source)) {
			return false;
		}

		foreach (new RecursiveIteratorIterator(
			new RecursiveDirectoryIterator($source, FilesystemIterator::SKIP_DOTS),
			RecursiveIteratorIterator::CHILD_FIRST) as $item) {
				if ($item->isFile()) {
					unlink($item);
				} else {
					rmdir($item);
				}
		}

		if ($deleteDirectory) {
			return rmdir($source);
		}
		return true;
	}

	// Compress a file or directory
	// $source = /home/diego/example
	// $destionation = /tmp/example.zip
	public static function zip($source, $destination)
	{
		if (!extension_loaded('zip')) {
			return false;
		}

		if (!file_exists($source)) {
			return false;
		}

		$zip = new ZipArchive();
		if (!$zip->open($destination, ZIPARCHIVE::CREATE)) {
			return false;
		}

		if (is_dir($source) === true) {<commit_msg>added some logs<commit_after><?php defined('BLUDIT') or die('Bludit CMS.');

class Filesystem {

	// Returns an array with the absolutes directories.
	public static function listDirectories($path, $regex='*', $sortByDate=false)
	{
		$directories = glob($path.$regex, GLOB_ONLYDIR);

		if(empty($directories)) {
			return array();
		}

		if($sortByDate) {
			usort($directories,
			      function($a, $b) {
				      return filemtime($b) - filemtime($a);
			      }
			);
		}

		return $directories;
	}

	// Returns an array with the list of files with the absolute path
	// $sortByDate = TRUE, the first file is the newer file
	// $chunk = amount of chunks, FALSE if you don't want to chunk
	public static function listFiles($path, $regex='*', $extension='*', $sortByDate=false, $chunk=false)
	{
		$files = glob($path.$regex.'.'.$extension);

		if (empty($files)) {
			return array();
		}

		if ($sortByDate) {
			usort($files,
				function($a, $b) {
					return filemtime($b) - filemtime($a);
				}
			);
		}

		// Split the list of files into chunks
		// http://php.net/manual/en/function.array-chunk.php
		if ($chunk) {
			return array_chunk($files, $chunk);
		}

		return $files;
	}

	public static function mkdir($pathname, $recursive=false)
	{
		return mkdir($pathname, DIR_PERMISSIONS, $recursive);
	}

	public static function rmdir($pathname)
	{
		Log::set('rmdir = '.$pathname, LOG_TYPE_INFO);
		return rmdir($pathname);
	}

	public static function mv($oldname, $newname)
	{
		Log::set('mv '.$oldname.' '.$newname, LOG_TYPE_INFO);
		return rename($oldname, $newname);
	}

	public static function rmfile($filename)
	{
		Log::set('rmfile = '.$filename, LOG_TYPE_INFO);
		return unlink($filename);
	}

	public static function fileExists($filename)
	{
		return file_exists($filename);
	}

	public static function directoryExists($path)
	{
		return file_exists($path);
	}

	// Copy recursive a directory to another
	// If the destination directory not exists is created
	// $source = /home/diego/example or /home/diego/example/
	// $destination = /home/diego/newplace or /home/diego/newplace/
	public static function copyRecursive($source, $destination, $skipDirectory=false)
	{
		$source 	= rtrim($source, DS);
		$destination 	= rtrim($destination, DS);

		// Check $source directory if exists
		if (!self::directoryExists($source)) {
			return false;
		}

		// Check $destionation directory if exists
		if (!self::directoryExists($destination)) {
			// Create the $destination directory
			if (!mkdir($destination, DIR_PERMISSIONS, true)) {
				return false;
			}
		}

		foreach ($iterator = new RecursiveIteratorIterator(
				new RecursiveDirectoryIterator($source, RecursiveDirectoryIterator::SKIP_DOTS),
				RecursiveIteratorIterator::SELF_FIRST) as $item) {

			$currentDirectory = dirname($item->getPathName());
			if ($skipDirectory !== $currentDirectory) {
				if ($item->isDir()) {
					@mkdir($destination.DS.$iterator->getSubPathName());
				} else {
					copy($item, $destination.DS.$iterator->getSubPathName());
				}
			}
		}

		return true;
	}

	// Delete a file or directory recursive
	// The directory is delete
	public static function deleteRecursive($source, $deleteDirectory=true)
	{
		Log::set('deleteRecursive = '.$source, LOG_TYPE_INFO);

		if (!self::directoryExists($source)) {
			return false;
		}

		foreach (new RecursiveIteratorIterator(
			new RecursiveDirectoryIterator($source, FilesystemIterator::SKIP_DOTS),
			RecursiveIteratorIterator::CHILD_FIRST) as $item) {
				if ($item->isFile() || $item->isLink()) {
					unlink($item);
				} else {
					rmdir($item);
				}
		}

		if ($deleteDirectory) {
			return rmdir($source);
		}
		return true;
	}

	// Compress a file or directory
	// $source = /home/diego/example
	// $destionation = /tmp/example.zip
	public static function zip($source, $destination)
	{
		if (!extension_loaded('zip')) {
			return false;
		}

		if (!file_exists($source)) {
			return false;
		}

		$zip = new ZipArchive();
		if (!$zip->open($destination, ZIPARCHIVE::CREATE)) {
			return false;
		}

		if (is_dir($source) === true) {
{'fraction_non_alphanumeric': 0.14736974578404227, 'fraction_numerical': 0.0, 'mean_word_length': 3.7444776119402987, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3000922', 'n_tokens_mistral': 2964, 'n_tokens_neox': 2577, 'n_words': 616}