<commit_before><?php
class SpotRetriever_Spots extends SpotRetriever_Abs {
		private $_rsakeys;
		private $_outputType;
		private $_retrieveFull;
		private $_prefetch_image;
		private $_prefetch_nzb;

		/**
		 * Server is the server array we are expecting to connect to
		 * db - database object
		 */
		function __construct($server, SpotDb $db, SpotSettings $settings, $outputType, $debug, $retro) {
			parent::__construct($server, $db, $settings, $debug, $retro);
			
			$this->_rsakeys = $this->_settings->get('rsa_keys');
			$this->_outputType = $outputType;
			$this->_retrieveFull = $this->_settings->get('retrieve_full');
			$this->_prefetch_image = $this->_settings->get('prefetch_image');
			$this->_prefetch_nzb = $this->_settings->get('prefetch_nzb');
		} # ctor


		/*
		 * Returns the status in either xml or text format 
		 */
		function displayStatus($cat, $txt) {
			if ($this->_outputType != 'xml') {
				switch($cat) {
					case 'start'			: echo "Retrieving new Spots from server " . $txt . "..." . PHP_EOL; break;
					case 'lastretrieve'		: echo strftime("Last retrieve at %c", $txt) . PHP_EOL; break;
					case 'done'			: echo "Finished retrieving spots." . PHP_EOL . PHP_EOL; break;
					case 'groupmessagecount': echo "Appr. Message count: 	" . $txt . "" . PHP_EOL; break;
					case 'firstmsg'			: echo "First message number:	" . $txt . "" . PHP_EOL; break;
					case 'lastmsg'			: echo "Last message number:	" . $txt . "" . PHP_EOL; break;
					case 'curmsg'			: echo "Current message:	" . $txt . "" . PHP_EOL; break;
					case 'progress'			: echo "Retrieving " . $txt; break;
					case 'hdrparsed'		: echo " (parsed " . $txt . " headers, "; break;
					case 'fullretrieved'		: echo $txt . " full, "; break;
					case 'verified'			: echo "verified " . $txt . ", "; break;
					case 'modcount'		: echo "moderated " . $txt . ", "; break;
					case 'skipcount'		: echo "skipped " . $txt . " of "; break;
					case 'loopcount'		: echo $txt . " total messages)"; break;
					case 'timer'			: echo " in " . $txt . " seconds" . PHP_EOL; break;
					case 'totalprocessed'		: echo "Processed a total of " . $txt . " spots" . PHP_EOL; break;
					case 'searchmsgid'		: echo "Looking for articlenumber for messageid" . PHP_EOL; break;
					case ''				: echo PHP_EOL; break;
					
					default				: echo $cat . $txt;
				} # switch
			} else {
			
				switch($cat) {
					case 'start'			: echo "<spots>"; break;
					case 'done'				: echo "</spots>"; break;
					case 'totalprocessed'	: echo "<totalprocessed>" . $txt . "</totalprocessed>"; break;
					case 'skipcount'		: echo "<totalskipped> " . $txt . "</totalskipped>"; break;
					case 'totalremoved'		: echo "<totalremoved>" . $txt . "</totalremoved>"; break;
					default					: break;
				} # switch
			} # else xmloutput
		} # displayStatus
		
		/*
		 * Remove any extraneous reports from the database because we assume
		 * the highest messgeid in the database is the latest on the server.
		 */
		function updateLastRetrieved($highestMessageId) {
			$this->debug('Highest messageid found: ' . $highestMessageId);

			/*
			 * Remove any extraneous spots from the database because we assume
			 * the highest messgeid in the database is the latest on the server.
			 *
			 * If the server is marked as buggy, the last 'x' amount of spot are
			 * always checked so we do not have to do this 
			 */
			if (!$this->_server['buggy']) {
				$this->_db->removeExtraSpots($highestMessageId);
			} # if
		} # updateLastRetrieved
		
		/*
		 * Actually process the retrieved headers from XOVER
		 */
		function process($hdrList, $curMsg, $endMsg) {
			$this->displayStatus("progress", ($curMsg) . " till " . ($endMsg));

			$spotParser = new SpotParser();
			$signedCount = 0;
			$hdrsRetrieved = 0;
			$fullsRetrieved = 0;
			$modCount = 0;
			$skipCount = 0;
			$lastProcessedId = '';
			$fullSpotDbList = array();
			$spotDbList = array();
			$moderationList = array();
			$timer = microtime(true);

			/*
			 * Determine the cutoff date (unixtimestamp) from whereon we do not want to 
			 * load the spots
			 */
			if ($this->_settings->get('retention') > 0) {
				$retentionStamp = time() - ($this->_settings->get('retention') * 24 * 60 * 60);
			} else {
				$retentionStamp = 0;
			} # else
			$this->debug('retentionStamp=' . $retentionStamp);
			$this->debug('hdrList=' . serialize($hdrList));
			
			/**
			 * We ask the database to match our messageid's we just retrieved with
			 * the list of id's we have just retrieved from the server
			 */
			$dbIdList = $this->_db->matchSpotMessageIds($hdrList);

			$this->debug('dbIdList=' . serialize($dbIdList));

			# if we need to fetch images or nzb files, we need an spotsoverview instance
			if ($this->_retrieveFull && ($this->_prefetch_image) || ($this->_prefetch_nzb)) {
				$spotsOverview = new SpotsOverview($this->_db, $this->_settings);
				$spotsOverview->setActiveRetriever(true);
				
				/*
				 * Only create a new NZB instance if the server differs from the
				 * header host, else re-use the connection
				 */
				$settings_nntp_nzb = $this->_settings->get('nntp_nzb'); 
				if ($this->_server['host'] == $settings_nntp_nzb['host']) {
					$nntp_nzb = $this->_spotnntp;
				} else {
					$nntp_nzb = new SpotNntp($settings_nntp_nzb);
				} # else
			} # if
			
			foreach($hdrList as $msgid => $msgheader) {
				$this->debug('foreach-loop, start. msgId= ' . $msgid);

				/* <commit_msg>Make sure the proper newsgroup is used for keeping the NZB server alive<commit_after><?php
class SpotRetriever_Spots extends SpotRetriever_Abs {
		private $_rsakeys;
		private $_outputType;
		private $_retrieveFull;
		private $_prefetch_image;
		private $_prefetch_nzb;

		/**
		 * Server is the server array we are expecting to connect to
		 * db - database object
		 */
		function __construct($server, SpotDb $db, SpotSettings $settings, $outputType, $debug, $retro) {
			parent::__construct($server, $db, $settings, $debug, $retro);
			
			$this->_rsakeys = $this->_settings->get('rsa_keys');
			$this->_outputType = $outputType;
			$this->_retrieveFull = $this->_settings->get('retrieve_full');
			$this->_prefetch_image = $this->_settings->get('prefetch_image');
			$this->_prefetch_nzb = $this->_settings->get('prefetch_nzb');
		} # ctor


		/*
		 * Returns the status in either xml or text format 
		 */
		function displayStatus($cat, $txt) {
			if ($this->_outputType != 'xml') {
				switch($cat) {
					case 'start'			: echo "Retrieving new Spots from server " . $txt . "..." . PHP_EOL; break;
					case 'lastretrieve'		: echo strftime("Last retrieve at %c", $txt) . PHP_EOL; break;
					case 'done'			: echo "Finished retrieving spots." . PHP_EOL . PHP_EOL; break;
					case 'groupmessagecount': echo "Appr. Message count: 	" . $txt . "" . PHP_EOL; break;
					case 'firstmsg'			: echo "First message number:	" . $txt . "" . PHP_EOL; break;
					case 'lastmsg'			: echo "Last message number:	" . $txt . "" . PHP_EOL; break;
					case 'curmsg'			: echo "Current message:	" . $txt . "" . PHP_EOL; break;
					case 'progress'			: echo "Retrieving " . $txt; break;
					case 'hdrparsed'		: echo " (parsed " . $txt . " headers, "; break;
					case 'fullretrieved'		: echo $txt . " full, "; break;
					case 'verified'			: echo "verified " . $txt . ", "; break;
					case 'modcount'		: echo "moderated " . $txt . ", "; break;
					case 'skipcount'		: echo "skipped " . $txt . " of "; break;
					case 'loopcount'		: echo $txt . " total messages)"; break;
					case 'timer'			: echo " in " . $txt . " seconds" . PHP_EOL; break;
					case 'totalprocessed'		: echo "Processed a total of " . $txt . " spots" . PHP_EOL; break;
					case 'searchmsgid'		: echo "Looking for articlenumber for messageid" . PHP_EOL; break;
					case ''				: echo PHP_EOL; break;
					
					default				: echo $cat . $txt;
				} # switch
			} else {
			
				switch($cat) {
					case 'start'			: echo "<spots>"; break;
					case 'done'				: echo "</spots>"; break;
					case 'totalprocessed'	: echo "<totalprocessed>" . $txt . "</totalprocessed>"; break;
					case 'skipcount'		: echo "<totalskipped> " . $txt . "</totalskipped>"; break;
					case 'totalremoved'		: echo "<totalremoved>" . $txt . "</totalremoved>"; break;
					default					: break;
				} # switch
			} # else xmloutput
		} # displayStatus
		
		/*
		 * Remove any extraneous reports from the database because we assume
		 * the highest messgeid in the database is the latest on the server.
		 */
		function updateLastRetrieved($highestMessageId) {
			$this->debug('Highest messageid found: ' . $highestMessageId);

			/*
			 * Remove any extraneous spots from the database because we assume
			 * the highest messgeid in the database is the latest on the server.
			 *
			 * If the server is marked as buggy, the last 'x' amount of spot are
			 * always checked so we do not have to do this 
			 */
			if (!$this->_server['buggy']) {
				$this->_db->removeExtraSpots($highestMessageId);
			} # if
		} # updateLastRetrieved
		
		/*
		 * Actually process the retrieved headers from XOVER
		 */
		function process($hdrList, $curMsg, $endMsg) {
			$this->displayStatus("progress", ($curMsg) . " till " . ($endMsg));

			$spotParser = new SpotParser();
			$signedCount = 0;
			$hdrsRetrieved = 0;
			$fullsRetrieved = 0;
			$modCount = 0;
			$skipCount = 0;
			$lastProcessedId = '';
			$fullSpotDbList = array();
			$spotDbList = array();
			$moderationList = array();
			$timer = microtime(true);

			/*
			 * Determine the cutoff date (unixtimestamp) from whereon we do not want to 
			 * load the spots
			 */
			if ($this->_settings->get('retention') > 0) {
				$retentionStamp = time() - ($this->_settings->get('retention') * 24 * 60 * 60);
			} else {
				$retentionStamp = 0;
			} # else
			$this->debug('retentionStamp=' . $retentionStamp);
			$this->debug('hdrList=' . serialize($hdrList));
			
			/**
			 * We ask the database to match our messageid's we just retrieved with
			 * the list of id's we have just retrieved from the server
			 */
			$dbIdList = $this->_db->matchSpotMessageIds($hdrList);

			$this->debug('dbIdList=' . serialize($dbIdList));

			# if we need to fetch images or nzb files, we need an spotsoverview instance
			if ($this->_retrieveFull && ($this->_prefetch_image) || ($this->_prefetch_nzb)) {
				$spotsOverview = new SpotsOverview($this->_db, $this->_settings);
				$spotsOverview->setActiveRetriever(true);
				
				/*
				 * Only create a new NZB instance if the server differs from the
				 * header host, else re-use the connection
				 */
				$settings_nntp_nzb = $this->_settings->get('nntp_nzb'); 
				if ($this->_server['host'] == $settings_nntp_nzb['host']) {
					$nntp_nzb = $this->_spotnntp;
				} else {
					$nntp_nzb = new SpotNntp($settings_nntp_nzb);
					$nntp_nzb->selectGroup($this->_settings->get('nzb_group'));
				} # else
			} # if
			
			foreach($hdrList as $msgid => $msgheader) {
				$this->debug('foreach-loop, start. msgId= ' . $msgid);

				/* 
{'fraction_non_alphanumeric': 0.14855633482102196, 'fraction_numerical': 0.00236815739138355, 'mean_word_length': 2.995633187772926, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 104, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6254941', 'n_tokens_mistral': 4411, 'n_tokens_neox': 3567, 'n_words': 1001}