<commit_before><?php
/* Usage
 Grab some XML data, either from a file, URL, etc. however you want. Assume storage in $strYourXML;

 $objXML = new xml2Array();
 $arrOutput = $objXML->parse($strYourXML);
 print_r($arrOutput); //print it out, or do whatever!
  
*/
class xml2Array {
   
   var $arrOutput = array();
   var $resParser;
   var $strXmlData;
   
   function utf8_to_entities($str) {
        $entities = '';
        $values = array();
        $lookingfor = 1;
        
        for ($i = 0; $i < strlen($str); $i++) {
            $thisvalue = ord($str[$i]);
            if ($thisvalue < 128) {
                $entities .= chr($thisvalue);
            } else {
                if (count($values) == 0) {
                    $lookingfor = ($thisvalue < 224) ? 2 : 3;
                }
                $values[] = $thisvalue;
                if (count($values) == $lookingfor) {
                    $number = ($lookingfor == 3) ?
                        (($values[0] % 16) * 4096) + (($values[1] % 64) * 64) + ($values[2] % 64):
                        (($values[0] % 32) * 64) + ($values[1] % 64);
                    $entities .= '&#' . $number . ';';
                    $values = array();
                    $lookingfor = 1;
                }
            }
        }
        return $entities;
    }
   
   function parse($strInputXML) {
           $this->resParser = xml_parser_create ('UTF-8');
           xml_set_object($this->resParser,$this);
           xml_set_element_handler($this->resParser, "tagOpen", "tagClosed");
           
           xml_set_character_data_handler($this->resParser, "tagData");
       
           $this->strXmlData = xml_parse($this->resParser,$strInputXML );
           if(!$this->strXmlData) {
               die(sprintf("XML error: %s at line %d",
                           xml_error_string(xml_get_error_code($this->resParser)),
                           xml_get_current_line_number($this->resParser)));
           }
                           
           xml_parser_free($this->resParser);
           
           return $this->arrOutput;
   }
   
   function tagOpen($parser, $name, $attrs) {
       $tag=array("name"=>$name,"attrs"=>$attrs); 
       array_push($this->arrOutput,$tag);
   }<commit_msg>Removed double char conversion and added PHPDoc comments<commit_after><?php
/* Usage
 Grab some XML data, either from a file, URL, etc. however you want. Assume storage in $strYourXML;

 $objXML = new xml2Array();
 $arrOutput = $objXML->parse($strYourXML);
 print_r($arrOutput); //print it out, or do whatever!
  
*/
class xml2Array {
   
   var $arrOutput = array();
   var $resParser;
   var $strXmlData;
   
   /**
   * Convert a utf-8 string to html entities
   *
   * @param string $str The UTF-8 string
   * @return string
   */
   function utf8_to_entities($str) {
       $entities = '';
       $values = array();
       $lookingfor = 1;

       for ($i = 0; $i < strlen($str); $i++) {
           $thisvalue = ord($str[$i]);
           if ($thisvalue < 128) {
               $entities .= $str[$i]; // Leave ASCII chars unchanged 
           } else {
               if (count($values) == 0) {
                   $lookingfor = ($thisvalue < 224) ? 2 : 3;
               }
               $values[] = $thisvalue;
               if (count($values) == $lookingfor) {
                   $number = ($lookingfor == 3) ?
                       (($values[0] % 16) * 4096) + (($values[1] % 64) * 64) + ($values[2] % 64):
                       (($values[0] % 32) * 64) + ($values[1] % 64);
                   $entities .= '&#' . $number . ';';
                   $values = array();
                   $lookingfor = 1;
               }
           }
       }
       return $entities;
   }

   /**
   * Parse an XML text string and create an array tree that rapresent the XML structure
   *
   * @param string $strInputXML The XML string
   * @return array
   */
   function parse($strInputXML) {
           $this->resParser = xml_parser_create ('UTF-8');
           xml_set_object($this->resParser,$this);
           xml_set_element_handler($this->resParser, "tagOpen", "tagClosed");
           
           xml_set_character_data_handler($this->resParser, "tagData");
       
           $this->strXmlData = xml_parse($this->resParser,$strInputXML );
           if(!$this->strXmlData) {
               die(sprintf("XML error: %s at line %d",
                           xml_error_string(xml_get_error_code($this->resParser)),
                           xml_get_current_line_number($this->resParser)));
           }
                           
           xml_parser_free($this->resParser);
           
           return $this->arrOutput;
   }
   
   function tagOpen($parser, $name, $attrs) {
       $tag=array("name"=>$name,"attrs"=>$attrs); 
       array_push($this->arrOutput,$tag);
   }
{'fraction_non_alphanumeric': 0.14965845580625128, 'fraction_numerical': 0.016973711446905404, 'mean_word_length': 1.6949247071946458, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5805447', 'n_tokens_mistral': 1505, 'n_tokens_neox': 1421, 'n_words': 307}