<commit_before><?php
class DataHandler {
	protected $data_folder = 'data/';
	protected $id;
	protected $version;
	
	public function __construct($id = '') {
		if ($id === '') $id = $this->getNewId();
		$this->id = preg_replace('/[^0-9a-zA-Z]/', '', $id); // remove every char, except allowed
	}
	
	public function _get() {
		$return = new stdClass();
		$return->id = $this->id;
		
		$data = $this->readData();
		if ($data === false) {
			$return->result = false;
			$return->errorMsg = 'there is no data with this identifier or data could not be read';
			return $return;
		}
		
		$return->result = true;
		$return->version = md5($data);
		$return->data = $data;
		return $return;
	}

	public function _set($data, $version = '') {
		$data = (string) $data;
		$version = (string) $version;
		
		$return = new stdClass();
		$return->id = $this->id;
		
		// check if version is out of date
		$data_org = $this->readData();
		if ($data_org !== false && md5($data_org) !== $version) {
			$return->result = false;
			$return->errorMsg = 'used version is out of date';
			return $return;
		}
		
		// write data
		if(file_put_contents($this->data_folder.$this->id, $data, LOCK_EX ) === false) {
			$return->result = false;
			$return->errorMsg = 'data could not be written';
			return $return;
		}
		
		$return->version = md5($data);
		$return->result = true;
		return $return;
	}
	
	protected function getNewId() {
		$characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
		$length = 10;
		
    $randomString = '';
    for ($i = 0; $i < $length; $i++) {
        $randomString .= $characters[rand(0, strlen($characters) - 1)];
    }
		
		// check if id is already used, generate new one if necessary
		if (file_exists($this->data_folder.$randomString)) $randomString = getNewId();
		
    return $randomString;
	}
	
	protected function readData() {
		return file_exists($this->data_folder.$this->id) ? file_get_contents($this->data_folder.$this->id) : false;
	}
}
?><commit_msg>adding traficLimiter saving config in class constants<commit_after><?php
class DataHandler {
	// (string) folder to store data in
	// webserver has to have write access to this folder
	// must end with a slash
	const DATA_FOLDER = 'data/';
	
	// (string) filename to store trafic_limiter data
	// could lead to data disclosure if filename extension is not php 
	const IP_LIST_FILENAME = 'trafic_limiter.php';
	
	// (int) limit request per IP in seconds; no limit if zero
	const TRAFIC_LIMITER = 30;
	
	protected $id;
	protected $version;
	
	public function __construct($id = '') {
		if ($id === '') $id = $this->getNewId();
		$this->id = preg_replace('/[^0-9a-zA-Z]/', '', $id); // remove every char, except allowed
	}
	
	public function _get() {
		$return = new stdClass();
		$return->id = $this->id;
		
		$data = $this->readData();
		if ($data === false) {
			$return->result = false;
			$return->errorMsg = 'there is no data with this identifier or data could not be read';
			return $return;
		}
		
		$return->result = true;
		$return->version = md5($data);
		$return->data = $data;
		return $return;
	}

	public function _set($data, $version = '') {
		$data = (string) $data;
		$version = (string) $version;
		
		$return = new stdClass();
		$return->id = $this->id;
		
		// try to read existing data
		$data_org = $this->readData();
		if ($data_org !== false) {
			// check if version is out of date
			if (md5($data_org) !== $version) {
				$return->result = false;
				$return->errorMsg = 'used version is out of date';
				return $return;
			}
		}
		else {
			// check traficLimiter
			if (!$this->traficLimiterCanPass()) {
				$return->result = false;
				$return->errorMsg = 'to many request in last ' . self::TRAFIC_LIMITER . ' seconds from your IP address';
				return $return;
			}
		}
		
		// write data
		if(file_put_contents(self::DATA_FOLDER.$this->id, $data, LOCK_EX ) === false) {
			$return->result = false;
			$return->errorMsg = 'data could not be written';
			return $return;
		}
		
		$return->version = md5($data);
		$return->result = true;
		return $return;
	}
	
	protected function getNewId() {
		$characters = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
		$length = 10;
		
    $randomString = '';
    for ($i = 0; $i < $length; $i++) {
        $randomString .= $characters[rand(0, strlen($characters) - 1)];
    }
		
		// check if id is already used, generate new one if necessary
		if (file_exists(self::DATA_FOLDER.$randomString)) $randomString = getNewId();
		
    return $randomString;
	}
	
	/*
	 * limit request per IP
	 * returns true on valid request
	 * returns false on invalid request or error
	 */
	protected function traficLimiterCanPass() {
		if (self::TRAFIC_LIMITER === 0) return true;
		
		$ip = $_SERVER['REMOTE_ADDR'];
		$filename = self::DATA_FOLDER.self::IP_LIST_FILENAME;
		
		// create storage for ip_list if not exists
		if (file_exists($filename) === false) {
			if (file_put_contents($filename, "<?php\n\$ip_list=array();\n?>") === false) {
				return false;
			}
		}
		
		// load ip_list
		require $filename;
		
		// check if IP can pass
		if (isset($ip_list[$ip]) && (($ip_list[$ip] + self::TRAFIC_LIMITER) > time())) return false;
		
		// delete expired IPs
		foreach ($ip_list as $ip => $time) {
			if (($time + self::TRAFIC_LIMITER) < time()) unset($ip_list[$ip]);
		}
		
		// add new IP
		$ip_list[$ip] = time();
		
		// save ip_list
		if (file_put_contents($filename, "<?php\n\$ip_list=" . var_export($ip_list, true) . "\n?>") === false) {
			return false;
		}
		
		return true;
	}
	
	protected function readData() {
		return file_exists(self::DATA_FOLDER.$this->id) ? file_get_contents(self::DATA_FOLDER.$this->id) : false;
	}
}
?>
{'fraction_non_alphanumeric': 0.16438836312653346, 'fraction_numerical': 0.007535927094286716, 'mean_word_length': 3.3631498470948014, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '749715', 'n_tokens_mistral': 2236, 'n_tokens_neox': 1936, 'n_words': 514}