<commit_before>#include <algorithm>
#include <iostream>
#include <fstream>
#include <libxml/xmlmemory.h>
#include <stdexcept>
#include <string>
#include <vector>
#include "retriever.h"
#include "../node.h"
#include "../node_util.h"
#include "../string_util.h"
#include "../tree.hh"

using std::ifstream;
using std::invalid_argument;
using std::runtime_error;
using std::string;
using std::cout;
using std::endl;
using std::vector;
using string_util::starts_with;

typedef vector<string> StringVec;

static string get_type(const string &location){
  static const string CHAR("CHAR");
  if(starts_with(location,CHAR))
    return "NX_CHAR";

  static const string INT8("INT8");
  if(starts_with(location,INT8))
    return "NX_INT8";

  static const string INT16("INT16");
  if(starts_with(location,INT16))
    return "NX_INT16";

  static const string INT32("INT32");
  if(starts_with(location,INT32))
    return "NX_INT32";

  static const string UINT8("UINT8");
  if(starts_with(location,UINT8))
    return "NX_UINT8";

  static const string UINT16("UINT16");
  if(starts_with(location,UINT16))
    return "NX_UINT16";

  static const string UINT32("UINT32");
  if(starts_with(location,UINT32))
    return "NX_UINT32";

  static const string FLOAT32("FLOAT32");
  if(starts_with(location,FLOAT32))
    return "NX_FLOAT32";

  static const string FLOAT64("FLOAT64");
  if(starts_with(location,FLOAT64))
    return "NX_FLOAT64";

  throw invalid_argument("Cannot determine type in location: "+location);
}

static string xmlChar_to_str(const xmlChar *ch, int len){
  string result((char *)ch);
  if( (len>0) && ((unsigned int)len<result.size()) )
    result.erase(result.begin()+len,result.end());

  return string_util::trim(result);
}

static bool is_right_square_bracket(const char c){
  static const string RIGHT="]";
  return find(RIGHT.begin(),RIGHT.end(),c)!=RIGHT.end();
}

static string get_dims(const string &location){
  using std::find;
  static const string LEFT("[");

  if(!starts_with(location,LEFT))
    return "";

  string result="";
  for(string::const_iterator it=location.begin() ; it!=location.end() ; it++ ){
    result+=(*it);
    if(is_right_square_bracket(*it))
      break;
  }

  if(result.size()==location.size())
    return "";
  else
    return result;
}

static xmlNode* find_element(xmlNode *a_node, const string &name){
  xmlNode *cur_node=NULL;

  string nodeName;
  for( cur_node=a_node ; cur_node!=NULL ; cur_node=cur_node->next ){
    nodeName=xmlChar_to_str(cur_node->name,-1);
    if(nodeName==name)
      return cur_node->xmlChildrenNode;
  }
  return NULL;
}

static void print_element_names(xmlNode * a_node){
    xmlNode *cur_node = NULL;

    for( cur_node=a_node ; cur_node!=NULL ; cur_node=cur_node->next ){
        if(cur_node->type == XML_ELEMENT_NODE){
          cout << "node type: Element, name: " << cur_node->name << endl;
        }
        
        //print_element_names(cur_node->children);
    }
}

static xmlNode* open_path(xmlNode *node,StringVec::iterator begin, StringVec::iterator end){
  // error check input
  if(begin==end)
    return NULL;
  
  // locate the next part of the path
  node=find_element(node,*begin);

  // if it returned poorly then get out now
  if(node==NULL)
    return NULL;

  // go to next step
  begin++;
  if(begin==end)
    return node;

  // recursively call self<commit_msg>Removed ^M characters from the file. No other changes from previous version.<commit_after>#include <algorithm>
#include <iostream>
#include <fstream>
#include <libxml/xmlmemory.h>
#include <stdexcept>
#include <string>
#include <vector>
#include "retriever.h"
#include "../node.h"
#include "../node_util.h"
#include "../string_util.h"
#include "../tree.hh"

using std::ifstream;
using std::invalid_argument;
using std::runtime_error;
using std::string;
using std::cout;
using std::endl;
using std::vector;
using string_util::starts_with;

typedef vector<string> StringVec;

static string get_type(const string &location){
  static const string CHAR("CHAR");
  if(starts_with(location,CHAR))
    return "NX_CHAR";

  static const string INT8("INT8");
  if(starts_with(location,INT8))
    return "NX_INT8";

  static const string INT16("INT16");
  if(starts_with(location,INT16))
    return "NX_INT16";

  static const string INT32("INT32");
  if(starts_with(location,INT32))
    return "NX_INT32";

  static const string UINT8("UINT8");
  if(starts_with(location,UINT8))
    return "NX_UINT8";

  static const string UINT16("UINT16");
  if(starts_with(location,UINT16))
    return "NX_UINT16";

  static const string UINT32("UINT32");
  if(starts_with(location,UINT32))
    return "NX_UINT32";

  static const string FLOAT32("FLOAT32");
  if(starts_with(location,FLOAT32))
    return "NX_FLOAT32";

  static const string FLOAT64("FLOAT64");
  if(starts_with(location,FLOAT64))
    return "NX_FLOAT64";

  throw invalid_argument("Cannot determine type in location: "+location);
}

static string xmlChar_to_str(const xmlChar *ch, int len){
  string result((char *)ch);
  if( (len>0) && ((unsigned int)len<result.size()) )
    result.erase(result.begin()+len,result.end());

  return string_util::trim(result);
}

static bool is_right_square_bracket(const char c){
  static const string RIGHT="]";
  return find(RIGHT.begin(),RIGHT.end(),c)!=RIGHT.end();
}

static string get_dims(const string &location){
  using std::find;
  static const string LEFT("[");

  if(!starts_with(location,LEFT))
    return "";

  string result="";
  for(string::const_iterator it=location.begin() ; it!=location.end() ; it++ ){
    result+=(*it);
    if(is_right_square_bracket(*it))
      break;
  }

  if(result.size()==location.size())
    return "";
  else
    return result;
}

static xmlNode* find_element(xmlNode *a_node, const string &name){
  xmlNode *cur_node=NULL;

  string nodeName;
  for( cur_node=a_node ; cur_node!=NULL ; cur_node=cur_node->next ){
    nodeName=xmlChar_to_str(cur_node->name,-1);
    if(nodeName==name)
      return cur_node->xmlChildrenNode;
  }
  return NULL;
}

static void print_element_names(xmlNode * a_node){
    xmlNode *cur_node = NULL;

    for( cur_node=a_node ; cur_node!=NULL ; cur_node=cur_node->next ){
        if(cur_node->type == XML_ELEMENT_NODE){
          cout << "node type: Element, name: " << cur_node->name << endl;
        }

        //print_element_names(cur_node->children);
    }
}

static xmlNode* open_path(xmlNode *node,StringVec::iterator begin, StringVec::iterator end){
  // error check input
  if(begin==end)
    return NULL;

  // locate the next part of the path
  node=find_element(node,*begin);

  // if it returned poorly then get out now
  if(node==NULL)
    return NULL;

  // go to next step
  begin++;
  if(begin==end)
    return node;

  // recursively call self
{'fraction_non_alphanumeric': 0.14466892788674238, 'fraction_numerical': 0.017106621442265154, 'mean_word_length': 4.513821138211382, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 26}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2974804', 'n_tokens_mistral': 2393, 'n_tokens_neox': 2298, 'n_words': 526}