<commit_before>#include "cInCLexer.h"
#include "cInCParser.h"

char testip[] = "float x;";

void processTree(pANTLR3_BASE_TREE node, int depth)
{
int count = node->getChildCount(node); 
  for(int i=0; i<depth; i++)
    printf("  ");

  printf("Type %u Children %u\n", node->getType(node), count);
  for(int i=0; i<count; i++)
    processTree(node->getChild(node,i), depth+1);
}

int main(int argc, char **argv)
{
pANTLR3_INPUT_STREAM ip;
pANTLR3_COMMON_TOKEN_STREAM tokens;
pcInCLexer lex;
pcInCParser parser;
cInCParser_translationUnit_return retVal;

  ip = antlr3NewAsciiStringInPlaceStream(testip, strlen(testip), NULL);
  lex = cInCLexerNew(ip);
  tokens = antlr3CommonTokenStreamSourceNew(ANTLR3_SIZE_HINT, TOKENSOURCE(lex));
  parser = cInCParserNew(tokens);
  retVal = parser->translationUnit(parser);
  processTree(retVal.tree,0);

}<commit_msg>Text contents of nodes in the tree.<commit_after>#include "cInCLexer.h"
#include "cInCParser.h"

char testip[] = "float x;\nstatic int z;\nchar text[64] = \"hello\"; int bob(char x, char *harry) { stuff  { inside } }";

void processTree(pANTLR3_BASE_TREE node, int depth)
{
int count = node->getChildCount(node); 
  for(int i=0; i<depth; i++)
    printf("  ");

  printf("Type %u Children %u ", node->getType(node), count);
  if(node->getText(node)!=NULL)
    printf("%s\n", node->getText(node)->chars);
  else
    printf("empty\n");
  for(int i=0; i<count; i++)
    processTree(node->getChild(node,i), depth+1);
}

int main(int argc, char **argv)
{
pANTLR3_INPUT_STREAM ip;
pANTLR3_COMMON_TOKEN_STREAM tokens;
pcInCLexer lex;
pcInCParser parser;
cInCParser_translationUnit_return retVal;

  ip = antlr3NewAsciiStringInPlaceStream(testip, strlen(testip), NULL);
  lex = cInCLexerNew(ip);
  tokens = antlr3CommonTokenStreamSourceNew(ANTLR3_SIZE_HINT, TOKENSOURCE(lex));
  parser = cInCParserNew(tokens);
  retVal = parser->translationUnit(parser);
  processTree(retVal.tree,0);

}
{'fraction_non_alphanumeric': 0.14847387480600102, 'fraction_numerical': 0.011381272633212623, 'mean_word_length': 5.882562277580071, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1724554', 'n_tokens_mistral': 750, 'n_tokens_neox': 733, 'n_words': 146}