<commit_before><?php
class htmlConverter {

	public function __construct() {
		
	}	

	public function convertFromArray($array,$text) {
		foreach( $array as $key => $value )
                	$text = str_replace($key, $value, $text);
                return $text;
        }

	public function getODF($text) {
                $convertArray = array( 
                  '&ldquo;' => '“', 
		  '&bdquo;' => '„', 
	          '&uuml;'  => 'ü', 
	          '&Uuml;'  => 'Ü', 
                  '&ouml;'  => 'ö', 
                  '&Ouml;'  => 'Ö', 
                  '&auml;'  => 'ä', 
                  '&Auml;'  => 'Ä', 
                  '&szlig;' => 'ß', 
                  '&nbsp;'  => ' ', 
                  '&acute;'  => '´', 
		  '<strong>'  => '<text:span text:style-name="T5">',
		  '</strong>' => '</text:span>',
                  '<em>'    => '<text:span text:style-name="T1">',
                  '</em>'   => '</text:span>',
//	   	  '<li>' => '<text:list><text:list-item>',
//                '</li>' => '</text:list-item></text:list>',
                  '<p>'     => '', 
                  '</p>'    => "\n");
		$result = strip_tags($text, '<strong><em><p>');
                $result = $this->convertFromArray($convertArray,$result);<commit_msg>htmlConverter: extended filter array, introduced two more filtering methods to ensure good results ;)<commit_after><?php
class htmlConverter {

	public function __construct() {
		
	}	

	public function convertFromArray($array,$text) {
		foreach( $array as $key => $value )
                	$text = str_replace($key, $value, $text);
                return $text;
        }

	public function removeSpecialChars($text) {
		
		$result = html_entity_decode($text, ENT_QUOTES, 'UTF-8');		   	     return $result;
	}

	function cleanRemainingString($in,$offset=null) 
	{ 
	    $out = trim($in); 
	    if (!empty($out)) 
	    { 
		$entity_start = strpos($out,'&',$offset); 
		// nichts weiter gefunden
		if ($entity_start === false) 
		{ 
		    return $out;    
		} 
		else 
		{ 
		    $entity_end = strpos($out,';',$entity_start); 
		    if ($entity_end === false) 
		    { 
			 return $out; 
		    } 
		    // gefunden, aber zu lang um eine entity zu sein 
		    else if ($entity_end > $entity_start+7) 
		    { 
			 $out = cleanString($out,$entity_start+1); 
		    } 
		    // weiteres vorkommen gefunden 
		    else 
		    { 
			 $clean = substr($out,0,$entity_start); 
			 $subst = substr($out,$entity_start+1,1); 
			 
			 $clean .= ($subst != "#") ? $subst : "_"; 
			 $clean .= substr($out,$entity_end+1); 
			 
			 $out = cleanString($clean,$entity_start+1); 
		    } 
		} 
	    } 
	    return $out; 
	} 

	public function getODF($text) {
                $convertArray = array( 
                  '&ldquo;'  => '“', 
		  '&bdquo;'  => '„', 
	          '&quot;'   => '"',
		  '&uuml;'   => 'ü', 
	          '&Uuml;'   => 'Ü', 
                  '&ouml;'   => 'ö', 
                  '&Ouml;'   => 'Ö', 
                  '&auml;'   => 'ä', 
                  '&Auml;'   => 'Ä', 
                  '&szlig;'  => 'ß',
		  '&egrave;' => 'è',
		  '&eacute;' => 'é',
		  '&ecirc;'  => 'ê',
                  '&ntilde;' => 'ñ',
		  '&nbsp;'   => ' ', 
                  '&acute;'  => '´',
                  '&gt;'     => '>',
                  '&lt;'     => '<', 
		  '<strong>'  => '<text:span text:style-name="T5">',
		  '</strong>' => '</text:span>',
                  '<em>'     => '<text:span text:style-name="T1">',
                  '</em>'    => '</text:span>',
                  '<p>'      => '', 
                  '</p>'     => "\n");
		$result = strip_tags($text, '<strong><em><p>');
                $result = $this->convertFromArray($convertArray,$result);
		$result = $this->removeSpecialChars($result);
		$result = $this->cleanRemainingString($result,null);
{'fraction_non_alphanumeric': 0.21089861147498035, 'fraction_numerical': 0.003143830233167409, 'mean_word_length': 1.6422145328719724, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '922741', 'n_tokens_mistral': 1453, 'n_tokens_neox': 1288, 'n_words': 220}