<commit_before><?php

    /**
     * This file contains all the treatments functions for PHP
    **/


    /**
     * Return a cut string (a resume)
    **/
	function resume($string, $max = 50) { 
		$string = strip_tags($string);
		$array = explode(" ", $string);
		if(count($array) < $max):
			$resume = $string;
		else:
			$resume = null;
			for($i=0; $i<$max; $i++) {
				$resume .= ' '.$array[$i]; 
			}
		endif;
	   
		if(count($array) > $max):
			$resume .= ' ...';
		endif;
		return $resume; 
	}


    /**
     * Generate keywords from a string
    **/
	function keywords($string, $sensitivity = 4, $min = 2, $max = 8, $limit = 10) {
	
		// Delete HTML tags, codes, punctuation ...
		$string = strip_tags($string);
		$string = preg_replace('#<(/)?(.+)(/)?>#', '', $string);
		$string = preg_replace('#([[:punct:]])#isU', '', $string);
		
		// Sort words
		$words = explode(' ', $string);
		$words = array_count_values($words);
		asort($words);
		$words = array_reverse($words);
		
		// Remove words where length < sensitivity
		foreach($words as $word => $count) {
			if(strlen($word) < $sensitivity):
				unset($words[$word]);
			endif;
		}
		
		// Total words
		$total = count($words);
		
		// Check words density
		foreach($words as $word => $count) {
			$density = ($count/$total)*100;
			if($density < $min || $density > $max):
				unset($words[$word]);
			endif;
		}
		
		// Apply the limit
		$words = array_splice($words, 0, $limit, false);
		
		$result = array();
		foreach($words as $word => $count) {
			$result[] = $word;
		}
		
		return $result;
	}


    /**
     * Remove Javascript actions in a string
    **/
    function strip_scripts($string, $exceptions = array()) {
		$events = array('onabort', 'onblur', 'onchange', 'onclick', 'ondblclick', 'ondragdrop', 'onerror', 'onfocus', 'onkeydown', 'onkeypress', 'onkeyup', 'onload', 'onmouseover', 'onmouseout', 'onreset', 'onresize', 'onselect', 'onsubmit', 'onunload');
		
		foreach($events as $i => $name) {
			if(in_array($name, $exceptions)):
				unset($events[$i]);
			endif;
		}
		
        $string = preg_replace('#<script(.+)?>(.+)</script>#isU', '', $string);
		$string = preg_replace('# ('.implode('|',$events).')="(.+)"#isU', null, $string);
		$string = preg_replace('# ('.implode('|',$events).')=\'(.+)\'#isU', null, $string);
		
		return $string;
	}
    
    
    /**
     * Generate an URL accepted format from a string
    **/
    function nl2url($string) {
		// Accents \\
		$string = str_replace(array('à', 'â', 'ä', 'á', 'ã', 'å',
									'î', 'ï', 'ì', 'í', 
									'ô', 'ö', 'ò', 'ó', 'õ', 'ø', 
									'ù', 'û', 'ü', 'ú', 
									'é', 'è', 'ê', 'ë', 
									'ç', 'ÿ', 'ñ', 'ý',
									'À', 'Â', 'Ä', 'Á', 'Ã', 'Å',<commit_msg>Remove strip_scripts function from strings library<commit_after><?php

    /**
     * This file contains all the treatments functions for PHP
    **/


    /**
     * Return a cut string (a resume)
    **/
	function resume($string, $max = 50) { 
		$string = strip_tags($string);
		$array = explode(" ", $string);
		if(count($array) < $max):
			$resume = $string;
		else:
			$resume = null;
			for($i=0; $i<$max; $i++) {
				$resume .= ' '.$array[$i]; 
			}
		endif;
	   
		if(count($array) > $max):
			$resume .= ' ...';
		endif;
		return $resume; 
	}


    /**
     * Generate keywords from a string
    **/
	function keywords($string, $sensitivity = 4, $min = 2, $max = 8, $limit = 10) {
	
		// Delete HTML tags, codes, punctuation ...
		$string = strip_tags($string);
		$string = preg_replace('#<(/)?(.+)(/)?>#', '', $string);
		$string = preg_replace('#([[:punct:]])#isU', '', $string);
		
		// Sort words
		$words = explode(' ', $string);
		$words = array_count_values($words);
		asort($words);
		$words = array_reverse($words);
		
		// Remove words where length < sensitivity
		foreach($words as $word => $count) {
			if(strlen($word) < $sensitivity):
				unset($words[$word]);
			endif;
		}
		
		// Total words
		$total = count($words);
		
		// Check words density
		foreach($words as $word => $count) {
			$density = ($count/$total)*100;
			if($density < $min || $density > $max):
				unset($words[$word]);
			endif;
		}
		
		// Apply the limit
		$words = array_splice($words, 0, $limit, false);
		
		$result = array();
		foreach($words as $word => $count) {
			$result[] = $word;
		}
		
		return $result;
	}

    
    /**
     * Generate an URL accepted format from a string
    **/
    function nl2url($string) {
		// Accents \\
		$string = str_replace(array('à', 'â', 'ä', 'á', 'ã', 'å',
									'î', 'ï', 'ì', 'í', 
									'ô', 'ö', 'ò', 'ó', 'õ', 'ø', 
									'ù', 'û', 'ü', 'ú', 
									'é', 'è', 'ê', 'ë', 
									'ç', 'ÿ', 'ñ', 'ý',
									'À', 'Â', 'Ä', 'Á', 'Ã', 'Å',
{'fraction_non_alphanumeric': 0.2326910861740419, 'fraction_numerical': 0.0055049756510692355, 'mean_word_length': 2.728492501973165, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6848347', 'n_tokens_mistral': 2029, 'n_tokens_neox': 1711, 'n_words': 413}