<commit_before>/**
 *  Dump ast in dot format.  Ast is dumped at each step of the
 *  transformation in a different file descriptor:
 *
 *  3: parsing
 *  4: desugaring
 *  5: rescoping
 *  6: flowing
 *  7: binding
 *
 *  For instance, you might use it like this to see ast after
 *  desugaring:
 *
 *  _build/src/ast-dump 4> ast.dot && dot -Tpng ast.dot > ast.png
 *
 */

#include <iostream>

#include <ast/dot-printer.hh>
#include <ast/nary.hh>

#include <binder/binder.hh>

#include <flower/flower.hh>

#include <parser/parse.hh>

#include <rewrite/desugarer.hh>
#include <rewrite/rescoper.hh>

using namespace ast;
using namespace parser;

static const int sz = 4096;

static void
print(rAst ast)
{
  static int fd = 2;
  ++fd;
  std::stringstream output;
  DotPrinter p(output);
  p(ast.get());
  write(fd, output.str().c_str(), output.str().length());
}

int
main()
{
  char buf[sz + 1];
  std::string source;
  while (!std::cin.eof())
  {
    std::cin.read(buf, sz);
    buf[std::cin.gcount()] = 0;
    source += buf;
  }

  rAst res = parse(source)->ast_get();;

  print(res);

  rewrite::Desugarer desugar;
  desugar(res.get());
  res = desugar.result_get();
  print(res);

  rewrite::Rescoper rescope;
  rescope(res.get());
  res = rescope.result_get();
  print(res);

  flower::Flower flow;
  res = ast::analyze(flow, res.unsafe_cast<ast::Nary>().get());
  print(res);

  binder::Binder bind;
  res = ast::analyze(bind, res.unsafe_cast<ast::Nary>().get());
  print(res);
}<commit_msg>Update ast-dump with new parse signature.<commit_after>/**
 *  Dump ast in dot format.  Ast is dumped at each step of the
 *  transformation in a different file descriptor:
 *
 *  3: parsing
 *  4: desugaring
 *  5: rescoping
 *  6: flowing
 *  7: binding
 *
 *  For instance, you might use it like this to see ast after
 *  desugaring:
 *
 *  _build/src/ast-dump 4> ast.dot && dot -Tpng ast.dot > ast.png
 *
 */

#include <iostream>

#include <ast/dot-printer.hh>
#include <ast/nary.hh>

#include <binder/binder.hh>

#include <flower/flower.hh>

#include <parser/parse.hh>

#include <rewrite/desugarer.hh>
#include <rewrite/rescoper.hh>

using namespace ast;
using namespace parser;

static const int sz = 4096;

static void
print(rAst ast)
{
  static int fd = 2;
  ++fd;
  std::stringstream output;
  DotPrinter p(output);
  p(ast.get());
  write(fd, output.str().c_str(), output.str().length());
}

int
main()
{
  char buf[sz + 1];
  std::string source;
  while (!std::cin.eof())
  {
    std::cin.read(buf, sz);
    buf[std::cin.gcount()] = 0;
    source += buf;
  }

  rAst res = parse(source, __HERE__)->ast_get();;

  print(res);

  rewrite::Desugarer desugar;
  desugar(res.get());
  res = desugar.result_get();
  print(res);

  rewrite::Rescoper rescope;
  rescope(res.get());
  res = rescope.result_get();
  print(res);

  flower::Flower flow;
  res = ast::analyze(flow, res.unsafe_cast<ast::Nary>().get());
  print(res);

  binder::Binder bind;
  res = ast::analyze(bind, res.unsafe_cast<ast::Nary>().get());
  print(res);
}
{'fraction_non_alphanumeric': 0.16930693069306932, 'fraction_numerical': 0.008580858085808581, 'mean_word_length': 3.6846986089644513, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1252233', 'n_tokens_mistral': 1194, 'n_tokens_neox': 1157, 'n_words': 280}