<commit_before><?php
/**
 * @author <NAME> <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 * @author <EMAIL> <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 *
 * @copyright Copyright (c) 2016, ownCloud, Inc.
 * @license AGPL-3.0
 *
 * This code is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License, version 3,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License, version 3,
 * along with this program.  If not, see <http://www.gnu.org/licenses/>
 *
 */

namespace OCA\Files\Command;

use OC\Core\Command\Base;
use OC\ForbiddenException;
use OCP\Files\StorageNotAvailableException;
use OCP\IUserManager;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Helper\Table;

class Scan extends Base {

	/** @var IUserManager $userManager */
	private $userManager;
	/** @var float */
	protected $execTime = 0;
	/** @var int */
	protected $foldersCounter = 0;
	/** @var int */
	protected $filesCounter = 0;

	public function __construct(IUserManager $userManager) {
		$this->userManager = $userManager;
		parent::__construct();
	}

	protected function configure() {
		parent::configure();

		$this
			->setName('files:scan')
			->setDescription('rescan filesystem')
			->addArgument(
				'user_id',
				InputArgument::OPTIONAL | InputArgument::IS_ARRAY,
				'will rescan all files of the given user(s)'
			)
			->addOption(
				'path',
				'p',
				InputArgument::OPTIONAL,
				'limit rescan to this path, eg. --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored'
			)
			->addOption(
				'quiet',
				'q',
				InputOption::VALUE_NONE,
				'suppress any output'
			)
			->addOption(
				'verbose',
				'-v|vv|vvv',
				InputOption::VALUE_NONE,
				'verbose the output'
			)
			->addOption(
				'all',
				null,
				InputOption::VALUE_NONE,
				'will rescan all files of all known users'
			);
	}

	protected function scanFiles($user, $path, $verbose, OutputInterface $output) {
		$scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection(), \OC::$server->getLogger());
		# check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception
		# printout and count
		if ($verbose) {
			$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) {
				$output->writeln("\tFile   <info>$path</info>");
				$this->filesCounter += 1;
				if ($this->hasBeenInterrupted()) {
					throw new \Exception('ctrl-c');
				}
			});
			$scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output) {
				$output->writeln("\tFolder <info>$path</info>");
				$this->foldersCounter += 1;
				if ($this->hasBeenInterrupted()) {
					throw new \Exception('ctrl-c');
				}
			});
			$scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function (StorageNotAvailableException $e) use ($output) {
				$output->writeln("Error while scanning, storage not available (" . $e->getMessage() . ")");
			});
		# count only
		} else {
			$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function () use ($output) {
				$this->filesCounter += 1;
				if ($this->hasBeenInterrupted()) {
					throw new \Exception('ctrl-c');
				}
			});
			$scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function () use ($output) {
				$this->foldersCounter += 1;
				if ($this->hasBeenInterrupted()) {
					throw new \Exception('ctrl-c');
				}
			});
		}

		try {
			$scanner->scan($path);
		} catch (ForbiddenException $e) {
			$output->writeln("<error>Home storage for user $user not writable</error>");
			$output->writeln("Make sure you're running the scan command only as the user the web server runs as");
		} catch (\Exception $e) {
			# exit the function if ctrl-c has been pressed 
			return;
		}<commit_msg>Add files:scan warning when NFD or incompatible encoding found<commit_after><?php
/**
 * @author <NAME> <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 * @author <EMAIL> <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 * @author <NAME> <<EMAIL>>
 *
 * @copyright Copyright (c) 2016, ownCloud, Inc.
 * @license AGPL-3.0
 *
 * This code is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License, version 3,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License, version 3,
 * along with this program.  If not, see <http://www.gnu.org/licenses/>
 *
 */

namespace OCA\Files\Command;

use OC\Core\Command\Base;
use OC\ForbiddenException;
use OCP\Files\StorageNotAvailableException;
use OCP\IUserManager;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Helper\Table;

class Scan extends Base {

	/** @var IUserManager $userManager */
	private $userManager;
	/** @var float */
	protected $execTime = 0;
	/** @var int */
	protected $foldersCounter = 0;
	/** @var int */
	protected $filesCounter = 0;

	public function __construct(IUserManager $userManager) {
		$this->userManager = $userManager;
		parent::__construct();
	}

	protected function configure() {
		parent::configure();

		$this
			->setName('files:scan')
			->setDescription('rescan filesystem')
			->addArgument(
				'user_id',
				InputArgument::OPTIONAL | InputArgument::IS_ARRAY,
				'will rescan all files of the given user(s)'
			)
			->addOption(
				'path',
				'p',
				InputArgument::OPTIONAL,
				'limit rescan to this path, eg. --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored'
			)
			->addOption(
				'quiet',
				'q',
				InputOption::VALUE_NONE,
				'suppress any output'
			)
			->addOption(
				'verbose',
				'-v|vv|vvv',
				InputOption::VALUE_NONE,
				'verbose the output'
			)
			->addOption(
				'all',
				null,
				InputOption::VALUE_NONE,
				'will rescan all files of all known users'
			);
	}

	public function checkScanWarning($fullPath, OutputInterface $output) {
		$normalizedPath = basename(\OC\Files\Filesystem::normalizePath($fullPath));
		$path = basename($fullPath);

		if ($normalizedPath !== $path) {
			$output->writeln("\t<error>Entry \"" . $fullPath . '" will not be accessible due to incompatible encoding</error>');
		}
	}

	protected function scanFiles($user, $path, $verbose, OutputInterface $output) {
		$scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection(), \OC::$server->getLogger());
		# check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception
		# printout and count
		if ($verbose) {
			$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) {
				$output->writeln("\tFile   <info>$path</info>");
				$this->filesCounter += 1;
				if ($this->hasBeenInterrupted()) {
					throw new \Exception('ctrl-c');
				}
			});
			$scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output) {
				$output->writeln("\tFolder <info>$path</info>");
				$this->foldersCounter += 1;
				if ($this->hasBeenInterrupted()) {
					throw new \Exception('ctrl-c');
				}
			});
			$scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function (StorageNotAvailableException $e) use ($output) {
				$output->writeln("Error while scanning, storage not available (" . $e->getMessage() . ")");
			});
		# count only
		} else {
			$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function () use ($output) {
				$this->filesCounter += 1;
				if ($this->hasBeenInterrupted()) {
					throw new \Exception('ctrl-c');
				}
			});
			$scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function () use ($output) {
				$this->foldersCounter += 1;
				if ($this->hasBeenInterrupted()) {
					throw new \Exception('ctrl-c');
				}
			});
		}
		$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function($path) use ($output) {
			$this->checkScanWarning($path, $output);
		});
		$scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function($path) use ($output) {
			$this->checkScanWarning($path, $output);
		});

		try {
			$scanner->scan($path);
		} catch (ForbiddenException $e) {
			$output->writeln("<error>Home storage for user $user not writable</error>");
			$output->writeln("Make sure you're running the scan command only as the user the web server runs as");
		} catch (\Exception $e) {
			if ($e->getMessage() !== 'ctrl-c') {
				$output->writeln('<error>Exception while scanning: ' . $e->getMessage() . "\n" . $e->getTraceAsString() . '</error>');
			}
			return;
		}
{'fraction_non_alphanumeric': 0.15749187375484952, 'fraction_numerical': 0.0031456432840515887, 'mean_word_length': 3.7857501254390367, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 126, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5740205', 'n_tokens_mistral': 3400, 'n_tokens_neox': 2984, 'n_words': 818}