<commit_before><?php

namespace A7;

/**
 * Recursively loads all php files in all subdirectories of the given path
 *
 * @param $directory
 *
 * @throws \Exception
 */
function autoload( $directory ) {

	// Get a listing of the current directory
	$scanned_dir = scandir( $directory );

	if ( empty( $scanned_dir ) ) {
		return;
	}

	if ( count( $scanned_dir ) > 200 ) {
		throw new \Exception( 'Too many files attempted to load via autoload' );
	}

	// Ignore these items from scandir
	$ignore = [
		'.',
		'..'
	];

	// Remove the ignored items
	$scanned_dir = array_diff( $scanned_dir, $ignore );

	foreach ( $scanned_dir as $item ) {

		$filename  = $directory . '/' . $item;
		$real_path = realpath( $filename );

		if ( false === $real_path ) {
			continue;
		}

		$filetype = filetype( $real_path );

		if ( empty( $filetype ) ) {
			continue;
		}

		// If it's a directory then recursively load it
		if ( 'dir' === $filetype ) {

			autoload( $real_path );
		}

		// If it's a file, let's try to load it
		else if ( 'file' === $filetype ) {

			// Don't allow files that have been uploaded
			if ( is_uploaded_file( $real_path ) ) {
				continue;
			}

			// Don't load any files that are not the proper mime type
			if ( 'text/x-php' !== mime_content_type( $real_path ) ) {
				continue;
			}

			$filesize = filesize( $real_path );
			// Don't include empty or negative sized files
			if ( $filesize <= 0 ) {
				continue;
			}

			// Don't include files that are greater than 100kb
			if ( $filesize > 100000 ) {
				continue;
			}

			$pathinfo = pathinfo( $real_path );

			// An empty filename wouldn't be a good idea
			if ( empty( $pathinfo['filename'] ) ) {
				continue;
			}

			// Sorry, need an extension
			if ( empty( $pathinfo['extension'] ) ) {
				continue;
			}<commit_msg>Update autoload.php<commit_after><?php

namespace A7;

/**
 * Recursively loads all php files in all subdirectories of the given path
 *
 * @param $directory
 *
 * @throws \Exception
 */
function autoload( $directory ) {

	// Get a listing of the current directory
	$scanned_dir = scandir( $directory );

	if ( empty( $scanned_dir ) ) {
		return;
	}

	if ( count( $scanned_dir ) > 200 ) {
		throw new \Exception( 'Too many files attempted to load via autoload' );
	}

	// Ignore these items from scandir
	$ignore = [
		'.',
		'..'
	];

	// Remove the ignored items
	$scanned_dir = array_diff( $scanned_dir, $ignore );

	foreach ( $scanned_dir as $item ) {

		$filename  = $directory . '/' . $item;
		$real_path = realpath( $filename );

		if ( false === $real_path ) {
			continue;
		}

		$filetype = filetype( $real_path );

		if ( empty( $filetype ) ) {
			continue;
		}

		// If it's a directory then recursively load it
		if ( 'dir' === $filetype ) {

			autoload( $real_path );
		}

		// If it's a file, let's try to load it
		else if ( 'file' === $filetype ) {

			// Don't allow files that have been uploaded
			if ( is_uploaded_file( $real_path ) ) {
				continue;
			}

			$filesize = filesize( $real_path );
			// Don't include empty or negative sized files
			if ( $filesize <= 0 ) {
				continue;
			}

			// Don't include files that are greater than 100kb
			if ( $filesize > 100000 ) {
				continue;
			}

			$pathinfo = pathinfo( $real_path );

			// An empty filename wouldn't be a good idea
			if ( empty( $pathinfo['filename'] ) ) {
				continue;
			}

			// Sorry, need an extension
			if ( empty( $pathinfo['extension'] ) ) {
				continue;
			}
{'fraction_non_alphanumeric': 0.13712471131639722, 'fraction_numerical': 0.008083140877598153, 'mean_word_length': 2.6473684210526316, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7414164', 'n_tokens_mistral': 1406, 'n_tokens_neox': 1169, 'n_words': 361}