<commit_before>/* this program takes in Tetra code and produces a parse tree which allows it
 * to be tested and visualized.  It now produces a structured text dump.
 * TODO add graphical output with graphiz or similar */

#include <string>
#include <iostream>
#include <fstream>
#include <sstream>
#include <cstdlib>
#include <cstdio>
#include "tetra.hpp"
#include "parser.gen.hpp"

using namespace std;

/* returns a string representation of a node type */
string stringType(Node* node) {
  stringstream ss;

  switch (node->node_type) {
    /* statements and groups */
    case NODE_FUNCTION: return "FUNC " + node->stringval;
    case NODE_FUNCTION_LIST: return "FUNCS";
    case NODE_STATEMENT: return "STMTS";
    case NODE_FORMAL_PARAM: return "PARAM: " + node->stringval;
    case NODE_FORMAL_PARAM_LIST: return "PARAMS";
    case NODE_PASS: return "PASS";
    case NODE_RETURN: return "RETURN";
    case NODE_BREAK: return "BREAK";
    case NODE_CONTINUE: return "CONTINUE";
    case NODE_IF: return "IF";
    case NODE_WHILE: return "WHILE";
    case NODE_ELIF: return "ELIF";
    case NODE_ELIF_CHAIN: return "ELIF CHAIN";
    case NODE_ELIF_CLAUSE: return "ELIF CLAUSE";

    /* operators */
    case NODE_ASSIGN: return "=";
    case NODE_OR: return "or";
    case NODE_AND: return "and";
    case NODE_LT: return "<";
    case NODE_LTE: return "<=";
    case NODE_GT: return ">";
    case NODE_GTE: return ">=";
    case NODE_EQ: return "==";
    case NODE_NEQ: return "!=";
    case NODE_NOT: return "not";
    case NODE_BITXOR: return "^";
    case NODE_BITAND: return "&";
    case NODE_BITOR: return "|";
    case NODE_BITNOT: return "~";
    case NODE_SHIFTL: return "<<";
    case NODE_SHIFTR: return ">>";
    case NODE_PLUS: return "+";
    case NODE_MINUS: return "-";
    case NODE_TIMES: return "*";
    case NODE_DIVIDE: return "/";
    case NODE_MODULUS: return "%";
    case NODE_EXP: return "EXP";

    /* functions */
    case NODE_FUNCALL: return "CALL: " + node->stringval;
    case NODE_ACTUAL_PARAM_LIST: return "ARGS";

    /* leafs */
    case NODE_INTVAL:
      ss << "INT: " << node->intval;
      return ss.str( );
    case NODE_REALVAL:
      ss << "REAL: " << node->realval;
      return ss.str( );
    case NODE_STRINGVAL: return "\\\"" +  node->stringval + "\\\"";
    case NODE_IDENTIFIER: return "ID " + node->stringval;
    case NODE_BOOLVAL:
      if (node->boolval) {
        return "true";
      } else {
        return "false";
      }
    default:
      throw Error("Unsupported node type!");
  }
}

/* generate a unique label for a node */
string genId( ) {
  static int count = 0;
  count++;
  stringstream ss;
  ss << "n" << count;
  return ss.str( );
}


void dumpNodeGraphviz(Node* node, string id, ofstream& out) {
  if (!node) {
    return;
  }

  /* dump the node designator */
  out << "  " << id << "[label=\"";
  out << stringType(node);
  out << "\"];\n";

  /* for each child */
  for (unsigned int i = 0; i < node->children.size( ); i++) {
    /* generate a new id */
    string childId = genId( );

    /* dump the connection */
    out << "  " << id << " -> " << childId << ";\n";

    /* dump the child itself */
    dumpNodeGraphviz(node->children[i], childId, out);
  }
}


void dumpTreeGraphviz(Node* node) {
  ofstream out("tree.gv");
  out << "digraph G {\n";
  dumpNodeGraphviz(node, "n0", out);
  out << "}\n";
  out.close( );

  system("rm tree.png");
  system("dot -Tpng tree.gv -o tree.png");
  system("rm tree.gv");<commit_msg>Changed font of graphs.<commit_after>/* this program takes in Tetra code and produces a parse tree which allows it
 * to be tested and visualized.  It now produces a structured text dump.
 * along with graphical output with graphiz */

#include <string>
#include <iostream>
#include <fstream>
#include <sstream>
#include <cstdlib>
#include <cstdio>
#include "tetra.hpp"
#include "parser.gen.hpp"

using namespace std;

/* returns a string representation of a node type */
string stringType(Node* node) {
  stringstream ss;

  switch (node->node_type) {
    /* statements and groups */
    case NODE_FUNCTION: return "FUNC " + node->stringval;
    case NODE_FUNCTION_LIST: return "FUNCS";
    case NODE_STATEMENT: return "STMTS";
    case NODE_FORMAL_PARAM: return "PARAM: " + node->stringval;
    case NODE_FORMAL_PARAM_LIST: return "PARAMS";
    case NODE_PASS: return "PASS";
    case NODE_RETURN: return "RETURN";
    case NODE_BREAK: return "BREAK";
    case NODE_CONTINUE: return "CONTINUE";
    case NODE_IF: return "IF";
    case NODE_WHILE: return "WHILE";
    case NODE_ELIF: return "ELIF";
    case NODE_ELIF_CHAIN: return "ELIF CHAIN";
    case NODE_ELIF_CLAUSE: return "ELIF CLAUSE";

    /* operators */
    case NODE_ASSIGN: return "=";
    case NODE_OR: return "or";
    case NODE_AND: return "and";
    case NODE_LT: return "<";
    case NODE_LTE: return "<=";
    case NODE_GT: return ">";
    case NODE_GTE: return ">=";
    case NODE_EQ: return "==";
    case NODE_NEQ: return "!=";
    case NODE_NOT: return "not";
    case NODE_BITXOR: return "^";
    case NODE_BITAND: return "&";
    case NODE_BITOR: return "|";
    case NODE_BITNOT: return "~";
    case NODE_SHIFTL: return "<<";
    case NODE_SHIFTR: return ">>";
    case NODE_PLUS: return "+";
    case NODE_MINUS: return "-";
    case NODE_TIMES: return "*";
    case NODE_DIVIDE: return "/";
    case NODE_MODULUS: return "%";
    case NODE_EXP: return "EXP";

    /* functions */
    case NODE_FUNCALL: return "CALL: " + node->stringval;
    case NODE_ACTUAL_PARAM_LIST: return "ARGS";

    /* leafs */
    case NODE_INTVAL:
      ss << "INT: " << node->intval;
      return ss.str( );
    case NODE_REALVAL:
      ss << "REAL: " << node->realval;
      return ss.str( );
    case NODE_STRINGVAL: return "\\\"" +  node->stringval + "\\\"";
    case NODE_IDENTIFIER: return "ID " + node->stringval;
    case NODE_BOOLVAL:
      if (node->boolval) {
        return "true";
      } else {
        return "false";
      }
    default:
      throw Error("Unsupported node type!");
  }
}

/* generate a unique label for a node */
string genId( ) {
  static int count = 0;
  count++;
  stringstream ss;
  ss << "n" << count;
  return ss.str( );
}


void dumpNodeGraphviz(Node* node, string id, ofstream& out) {
  if (!node) {
    return;
  }

  /* dump the node designator */
  out << "  " << id << "[label=\"";
  out << stringType(node);
  out << "\", fontname=\"courier\"];\n";

  /* for each child */
  for (unsigned int i = 0; i < node->children.size( ); i++) {
    /* generate a new id */
    string childId = genId( );

    /* dump the connection */
    out << "  " << id << " -> " << childId << ";\n";

    /* dump the child itself */
    dumpNodeGraphviz(node->children[i], childId, out);
  }
}


void dumpTreeGraphviz(Node* node) {
  ofstream out("tree.gv");
  out << "digraph G {\n";
  dumpNodeGraphviz(node, "n0", out);
  out << "}\n";
  out.close( );

  system("rm tree.png");
  system("dot -Tpng tree.gv -o tree.png");
  system("rm tree.gv");
{'fraction_non_alphanumeric': 0.1513859275053305, 'fraction_numerical': 0.0008528784648187633, 'mean_word_length': 2.975141242937853, 'pattern_counts': {'":': 0, '<': 97, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7208109', 'n_tokens_mistral': 2514, 'n_tokens_neox': 2456, 'n_words': 698}