<commit_before><?php

class MongoDBStorage extends StorageProvider {

	protected static $name = "mongodb";
	
	protected $mongo;
	protected $db;

	public function putData($provider, $metadata, $rawdata) {
		$provider = $provider->getName();
		$timestamp = new MongoDate();
		// Metadata
		$metacollection = $this->db->$provider->meta;
		$metadata["_timestamp"] = $timestamp;
		$metacollection->insert($metadata);

		// Rawdata
		$rawcollection = $this->db->$provider->raw;
		$rawdata["_timestamp"] = $timestamp;
		$rawcollection->insert($rawdata);
	}

	public function init(){
		$this->mongo = new Mongo();
	 	$this->db = $this->mongo->derptracker; 
	}
}
<commit_msg>check if data changed. closes #24<commit_after><?php

class MongoDBStorage extends StorageProvider {

	protected static $name = "mongodb";
	
	protected $mongo;
	protected $db;

	public function putData($provider, $metadata, $rawdata) {
		$provider = $provider->getName();
		$timestamp = new MongoDate();
		// Metadata
		$metacollection = $this->db->$provider->meta;
		$lastmeta = $metacollection->find()->sort(array('_timestamp' => -1))->limit(1);
		foreach ($lastmeta as $val) {
			unset($val['_timestamp'], $val['_id']);
			$lastmeta = $val;
		}
		if($metadata != $lastmeta){
			$metadata["_timestamp"] = $timestamp;
			$metacollection->insert($metadata);
		}

		// Rawdata
		$rawcollection = $this->db->$provider->raw;
		$lastraw = $rawcollection->find()->sort(array('_timestamp' => -1))->limit(1);
		foreach ($lastraw as $val) {
			unset($val['_timestamp'], $val['_id']);
			$lastraw = $val;
		}
		if($rawdata != $lastraw){
			$rawdata["_timestamp"] = $timestamp;
			$rawcollection->insert($rawdata);
		}
	}

	public function init(){
		$this->mongo = new Mongo();
	 	$this->db = $this->mongo->derptracker; 
	}
}

{'fraction_non_alphanumeric': 0.1919191919191919, 'fraction_numerical': 0.003367003367003367, 'mean_word_length': 4.520123839009288, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2206138', 'n_tokens_mistral': 675, 'n_tokens_neox': 600, 'n_words': 117}