<commit_before><?php
error_reporting(2147483647);

try {
	/*
	 * If we are run from another directory, try to change the current
	 * working directory to a directory the script is in
	 */
	if (@!file_exists(getcwd() . '/' . basename($argv[0]))) {
		chdir(dirname(__FILE__));
	} # if

	require_once "lib/SpotClassAutoload.php";

	/*
	 * Initialize the Spotweb base classes
	 */
	$bootstrap = new Bootstrap();
	list($settings, $daoFactory, $req) = $bootstrap->boot();

	/*
	 * disable timing, all queries which are ran by retrieve this would make it use
	 * large amounts of memory
	 */
	SpotTiming::disable();

	# Initialize commandline arguments
	SpotCommandline::initialize(array('force', 'debug', 'retro', 'timing'), array('force' => false, 'timing' => false, 'debug' => false, 'retro' => false));

    # Allow for timing to be displayed after retrieval of spots
    $showTiming = SpotCommandline::get('timing');
    if ($showTiming) {
        SpotTiming::enable();
        SpotTiming::enableHtml(false);
        SpotTiming::disableExtra(true);
    } # if

    # Initialize translation to english
	SpotTranslation::initialize('en_US');

	/*
	 * When PHP is running in safe mode, max execution time cannot be set,
	 * which is necessary on slow systems for retrieval and statistics generation
	 */
	if (ini_get('safe_mode')) {
		echo "WARNING: PHP safemode is enabled, maximum execution cannot be reset! Turn off safemode if this causes problems" . PHP_EOL . PHP_EOL;
	} # if

	/*
	 * When retrieval is run from the webinterface, we want to make
	 * sure this user is actually allowed to run retrieval.
	 */
	$svcUserRecord = new Services_User_Record($daoFactory, $settings);
	$svcUserAuth = new Services_User_Authentication($daoFactory, $settings);
	if (!SpotCommandline::isCommandline()) {
		/*
		 * An API key is required, so request it and try to
		 * create a session with it which we can use to validate
		 * the user with
		 */
		$apiKey = $req->getDef('apikey', '');
		$userSession = $svcUserAuth->verifyApi($apiKey);

		/*
		 * If the session failed or the the user doesn't have access
		 * to retrieve spots, let the user know
		 */
		if (($userSession == false) || (!$userSession['security']->allowed(SpotSecurity::spotsec_retrieve_spots, ''))) { 
			throw new PermissionDeniedException(SpotSecurity::spotsec_retrieve_spots, '');
		} # if
		
		# Add the user's ip addres, we need it for sending notifications
		$userSession['session'] = array('ipaddr' => '');
	} else {
		$userSession['user'] = $svcUserRecord->getUser(SPOTWEB_ADMIN_USERID);
		$userSession['security'] = new SpotSecurity($daoFactory->getUserDao(),
													$daoFactory->getAuditDao(),
													$settings, 
													$userSession['user'], 
													'');
		$userSession['session'] = array('ipaddr' => '');
	} # if

	/*
	 * We normally check whether we are not running already, because
	 * this would mean it will mess up all sorts of things like
	 * comment calculation, but a user can force our hand
	 */
	$forceMode = SpotCommandline::get('force');

	/*
	 * Do we need to debuglog this session? Generates loads of
	 * output
	 */
	$debugLog = SpotCommandline::get('debug');

	/*
	 * Retro mode will allow os to start from the beginning and retrieve
	 * all spots starting from scratch
	 */
	$retroMode = SpotCommandline::get('retro');

	/*
	 * Retention cleanup. Basically when we ask for Spotweb to only
	 * keep spots for 'xx' days (eg: 30 days), we either have to delete
	 * everyting older than 'xx' days, or delete all 'full' resources
	 * older than the specified time period.
	 *
	 * The full resources are everything beyond the bare minimum to 
	 * display the spots, so we delete nzb's, images, comments, etc.
	 */
	if (($settings->get('retention') > 0) && (!$retroMode)) {
		$spotDao = $daoFactory->getSpotDao();
        $cacheDao = $daoFactory->getCacheDao();

		switch ($settings->get('retentiontype')) {
			case 'everything'		: {
				$spotDao->deleteSpotsRetention($settings->get('retention'));
			} # case everything

			case 'fullonly'			: {
				$cacheDao->expireCache($settings->get('retention'));
				$spotDao->expireCommentsFull($settings->get('retention'));<commit_msg>Use the correct DAO for expiring comments<commit_after><?php
error_reporting(2147483647);

try {
	/*
	 * If we are run from another directory, try to change the current
	 * working directory to a directory the script is in
	 */
	if (@!file_exists(getcwd() . '/' . basename($argv[0]))) {
		chdir(dirname(__FILE__));
	} # if

	require_once "lib/SpotClassAutoload.php";

	/*
	 * Initialize the Spotweb base classes
	 */
	$bootstrap = new Bootstrap();
	list($settings, $daoFactory, $req) = $bootstrap->boot();

	/*
	 * disable timing, all queries which are ran by retrieve this would make it use
	 * large amounts of memory
	 */
	SpotTiming::disable();

	# Initialize commandline arguments
	SpotCommandline::initialize(array('force', 'debug', 'retro', 'timing'), array('force' => false, 'timing' => false, 'debug' => false, 'retro' => false));

    # Allow for timing to be displayed after retrieval of spots
    $showTiming = SpotCommandline::get('timing');
    if ($showTiming) {
        SpotTiming::enable();
        SpotTiming::enableHtml(false);
        SpotTiming::disableExtra(true);
    } # if

    # Initialize translation to english
	SpotTranslation::initialize('en_US');

	/*
	 * When PHP is running in safe mode, max execution time cannot be set,
	 * which is necessary on slow systems for retrieval and statistics generation
	 */
	if (ini_get('safe_mode')) {
		echo "WARNING: PHP safemode is enabled, maximum execution cannot be reset! Turn off safemode if this causes problems" . PHP_EOL . PHP_EOL;
	} # if

	/*
	 * When retrieval is run from the webinterface, we want to make
	 * sure this user is actually allowed to run retrieval.
	 */
	$svcUserRecord = new Services_User_Record($daoFactory, $settings);
	$svcUserAuth = new Services_User_Authentication($daoFactory, $settings);
	if (!SpotCommandline::isCommandline()) {
		/*
		 * An API key is required, so request it and try to
		 * create a session with it which we can use to validate
		 * the user with
		 */
		$apiKey = $req->getDef('apikey', '');
		$userSession = $svcUserAuth->verifyApi($apiKey);

		/*
		 * If the session failed or the the user doesn't have access
		 * to retrieve spots, let the user know
		 */
		if (($userSession == false) || (!$userSession['security']->allowed(SpotSecurity::spotsec_retrieve_spots, ''))) { 
			throw new PermissionDeniedException(SpotSecurity::spotsec_retrieve_spots, '');
		} # if
		
		# Add the user's ip addres, we need it for sending notifications
		$userSession['session'] = array('ipaddr' => '');
	} else {
		$userSession['user'] = $svcUserRecord->getUser(SPOTWEB_ADMIN_USERID);
		$userSession['security'] = new SpotSecurity($daoFactory->getUserDao(),
													$daoFactory->getAuditDao(),
													$settings, 
													$userSession['user'], 
													'');
		$userSession['session'] = array('ipaddr' => '');
	} # if

	/*
	 * We normally check whether we are not running already, because
	 * this would mean it will mess up all sorts of things like
	 * comment calculation, but a user can force our hand
	 */
	$forceMode = SpotCommandline::get('force');

	/*
	 * Do we need to debuglog this session? Generates loads of
	 * output
	 */
	$debugLog = SpotCommandline::get('debug');

	/*
	 * Retro mode will allow os to start from the beginning and retrieve
	 * all spots starting from scratch
	 */
	$retroMode = SpotCommandline::get('retro');

	/*
	 * Retention cleanup. Basically when we ask for Spotweb to only
	 * keep spots for 'xx' days (eg: 30 days), we either have to delete
	 * everyting older than 'xx' days, or delete all 'full' resources
	 * older than the specified time period.
	 *
	 * The full resources are everything beyond the bare minimum to 
	 * display the spots, so we delete nzb's, images, comments, etc.
	 */
	if (($settings->get('retention') > 0) && (!$retroMode)) {
		$spotDao = $daoFactory->getSpotDao();
        $cacheDao = $daoFactory->getCacheDao();
        $commentDao = $daoFactory->getCommentDao();

		switch ($settings->get('retentiontype')) {
			case 'everything'		: {
				$spotDao->deleteSpotsRetention($settings->get('retention'));
			} # case everything

			case 'fullonly'			: {
				$cacheDao->expireCache($settings->get('retention'));
				$commentDao->expireCommentsFull($settings->get('retention'));
{'fraction_non_alphanumeric': 0.1306972889783355, 'fraction_numerical': 0.0033147863146679294, 'mean_word_length': 3.835718374356039, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5579105', 'n_tokens_mistral': 2783, 'n_tokens_neox': 2473, 'n_words': 811}