<commit_before><?php

require_once(dirname(__FILE__) . "/../DatabaseTable.php");
require_once(dirname(__FILE__) . "/ResilientMongoIterator.php");

class MongoDatabaseTable extends DatabaseTable {
	
	protected static function perfmon($enter) {
		global $PERFMON;
		if (@$PERFMON) {
			if ($enter)
				$PERFMON->enter("database");
			else
				$PERFMON->leave("database");
		}
	}

	private $collection;
	
	private function getCollection() {
		if (!$this->collection) {
			static::perfmon(true);
			$this->collection = $this->getDatabase()->getDatabase()->selectCollection($this->getTablename());
			static::perfmon(false);
		}
		return $this->collection;<commit_msg>Memory limit check and manual garbage collection<commit_after><?php

require_once(dirname(__FILE__) . "/../DatabaseTable.php");
require_once(dirname(__FILE__) . "/ResilientMongoIterator.php");

class MongoDatabaseTable extends DatabaseTable {
	
	protected static function perfmon($enter) {
		global $PERFMON;
		if (@$PERFMON) {
			if ($enter)
				$PERFMON->enter("database");
			else
				$PERFMON->leave("database");
		}
		/*
		 * Putting this here because it's used by most of the query functions.
		 *
		 * We check if the memory usage by the script is greater than the limit - 2MB. If it is, we force the garbage
		 * collection. 2MB seems like a safe threshold as we're dealing with DB records and that would mean a huge increase
		 * in each cycle.
		 */
		if (memory_get_usage() >= ((intval(ini_get("memory_limit")) * 1024 * 1024)) - (2 * 1024 * 1024)) {
			gc_collect_cycles();
		}
	}

	private $collection;
	
	private function getCollection() {
		if (!$this->collection) {
			static::perfmon(true);
			$this->collection = $this->getDatabase()->getDatabase()->selectCollection($this->getTablename());
			static::perfmon(false);
		}
		return $this->collection;
{'fraction_non_alphanumeric': 0.14598141060688902, 'fraction_numerical': 0.01038819026790596, 'mean_word_length': 4.350877192982456, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4212465', 'n_tokens_mistral': 634, 'n_tokens_neox': 542, 'n_words': 141}