<commit_before>// This file is distributed under the BSD License.
// See "license.txt" for details.
// Copyright 2009-2010, <NAME> (<EMAIL>)
// and <NAME> (<EMAIL>)
// http://www.chaiscript.com

#ifndef CHAISCRIPT_PARSER_HPP_
#define CHAISCRIPT_PARSER_HPP_

#include <boost/assign/std/vector.hpp>

#include <exception>
#include <fstream>
#include <sstream>

#include "chaiscript_prelude.hpp"
#include "chaiscript_common.hpp"

namespace chaiscript
{
  class ChaiScript_Parser {

    std::string::const_iterator input_pos, input_end;
    int line, col;
    std::string multiline_comment_begin, multiline_comment_end;
    std::string singleline_comment;
    const char *filename;
    std::vector<AST_NodePtr> match_stack;

    std::vector<std::vector<std::string> > operator_matches;
    std::vector<AST_Node_Type::Type> operators;

  public:
    ChaiScript_Parser() {           
      multiline_comment_begin = "/*";
      multiline_comment_end = "*/";
      singleline_comment = "//";

      setup_operators();
    }

    ChaiScript_Parser(const ChaiScript_Parser &); // explicitly unimplemented copy constructor
    ChaiScript_Parser &operator=(const ChaiScript_Parser &); // explicitly unimplemented assignment operator

    void setup_operators() {
      using namespace boost::assign;
                        
      operators.push_back(AST_Node_Type::Logical_Or);
      std::vector<std::string> logical_or;
      logical_or += "||";
      operator_matches.push_back(logical_or);

      operators.push_back(AST_Node_Type::Logical_And);
      std::vector<std::string> logical_and;
      logical_and += "&&";
      operator_matches.push_back(logical_and);

      operators.push_back(AST_Node_Type::Bitwise_Or);
      std::vector<std::string> bitwise_or;
      bitwise_or += "|";
      operator_matches.push_back(bitwise_or);

      operators.push_back(AST_Node_Type::Bitwise_Xor);
      std::vector<std::string> bitwise_xor;
      bitwise_xor += "^";
      operator_matches.push_back(bitwise_xor);           

      operators.push_back(AST_Node_Type::Bitwise_And);
      std::vector<std::string> bitwise_and;
      bitwise_and += "&";
      operator_matches.push_back(bitwise_and);

      operators.push_back(AST_Node_Type::Equality);
      std::vector<std::string> equality;
      equality += "==", "!=";
      operator_matches.push_back(equality);           

      operators.push_back(AST_Node_Type::Comparison);
      std::vector<std::string> comparison;
      comparison += "<", "<=", ">", ">=";
      operator_matches.push_back(comparison);

      operators.push_back(AST_Node_Type::Shift);
      std::vector<std::string> shift;
      shift += "<<", ">>";
      operator_matches.push_back(shift);           

      operators.push_back(AST_Node_Type::Additive);
      std::vector<std::string> additive;
      additive += "+", "-";
      operator_matches.push_back(additive);

      operators.push_back(AST_Node_Type::Multiplicative);
      std::vector<std::string> multiplicative;
      multiplicative += "*", "/", "%";
      operator_matches.push_back(multiplicative);

      operators.push_back(AST_Node_Type::Dot_Access);
      std::vector<std::string> dot_access;
      dot_access += ".";
      operator_matches.push_back(dot_access);
    }
    /**
     * Prints the parsed ast_nodes as a tree
     */
    void debug_print(AST_NodePtr t, std::string prepend = "") {
      std::cout << prepend << "(" << ast_node_type_to_string(t->identifier) << ") " << t->text << " : " << t->start.line << ", " << t->start.column << std::endl;
      for (unsigned int j = 0; j < t->children.size(); ++j) {
        debug_print(t->children[j], prepend + "  ");
      }
    }

    /**
     * Shows the current stack of matched ast_nodes
     */
    void show_match_stack() {
      for (unsigned int i = 0; i < match_stack.size(); ++i) {
        debug_print(match_stack[i]);
      }
    }

    /**
     * Clears the stack of matched ast_nodes
     */
    void clear_match_stack() {
      match_stack.clear();
    }

    /**
     * Returns the front-most AST node
     */
    AST_NodePtr ast() {
      return match_stack.front();
    }

    /**
     * Helper function that collects ast_nodes from a starting position to the top of the stack into a new AST node
     */
    void build_match(AST_NodePtr t, int match_start) {
      int pos_line_start, pos_col_start, pos_line_stop, pos_col_stop;
      int is_deep = false;
      
      //so we want to take everything to the right of this and make them children
      if (match_start != int(match_stack.size())) {
        pos_line_start = match_stack[match_start]->start.line;
        pos_col_start = match_stack[match_start]->start.column;
        pos_line_stop = line;
        pos_col_stop = col;
        is_deep = true;
      }
      else {
        pos_line_start = line;
        pos_col_start = col;
        pos_line_stop = line;
        pos_col_stop = col;
      }

      t->filename = filename;
      t->start.line = pos_line_start;
      t->start.column = pos_col_start;
      t->end.line = pos_line_stop;
      t->end.column = pos_col_stop;
      
      if (is_deep) {
        t->children.assign(match_stack.begin() + (match_start), match_stack.end());
        match_stack.erase(match_stack.begin() + (match_start), match_stack.end());
        match_stack.push_back(t);
      }
      else {
        //todo: fix the fact that a successful match that captured no ast_nodes doesn't have any real start position
        match_stack.push_back(t);
      }
    }

    /**
     * Does ranged char check
     */
    inline bool char_between(char start, char end) {
      if ((*input_pos >= start) && (*input_pos <= end)) {
        return true;
      }
      else {
        return false;
      }
    }

    /**
     * Check to see if there is more text parse
     */
    inline bool has_more_input() {
      return (input_pos != input_end);
    }
                                
    /**
     * Skips any multi-line or single-line comment
     */
    bool SkipComment() {
      bool retval = false;

      if (Symbol_(multiline_comment_begin.c_str())) {
        while (input_pos != input_end) {
          if (Symbol_(multiline_comment_end.c_str())) {
            break;
          }
          else if (!Eol_()) {
            ++col;
            ++input_pos;
          }
        }
        retval = true;
      }
      else if (Symbol_(singleline_comment.c_str())) {
        while (input_pos != input_end) {
          if (Symbol_("\r\n") || Char_('\n')) {
            ++line;
            col = 1;
            break;
          }
          else {
            ++col;
            ++input_pos;
          }<commit_msg>Fix to parsing of single-line comments where the comment at the end of a line was merging statements separated by a single line comment<commit_after>// This file is distributed under the BSD License.
// See "license.txt" for details.
// Copyright 2009-2010, <NAME> (<EMAIL>)
// and <NAME> (<EMAIL>)
// http://www.chaiscript.com

#ifndef CHAISCRIPT_PARSER_HPP_
#define CHAISCRIPT_PARSER_HPP_

#include <boost/assign/std/vector.hpp>

#include <exception>
#include <fstream>
#include <sstream>

#include "chaiscript_prelude.hpp"
#include "chaiscript_common.hpp"

namespace chaiscript
{
  class ChaiScript_Parser {

    std::string::const_iterator input_pos, input_end;
    int line, col;
    std::string multiline_comment_begin, multiline_comment_end;
    std::string singleline_comment;
    const char *filename;
    std::vector<AST_NodePtr> match_stack;

    std::vector<std::vector<std::string> > operator_matches;
    std::vector<AST_Node_Type::Type> operators;

  public:
    ChaiScript_Parser() {           
      multiline_comment_begin = "/*";
      multiline_comment_end = "*/";
      singleline_comment = "//";

      setup_operators();
    }

    ChaiScript_Parser(const ChaiScript_Parser &); // explicitly unimplemented copy constructor
    ChaiScript_Parser &operator=(const ChaiScript_Parser &); // explicitly unimplemented assignment operator

    void setup_operators() {
      using namespace boost::assign;
                        
      operators.push_back(AST_Node_Type::Logical_Or);
      std::vector<std::string> logical_or;
      logical_or += "||";
      operator_matches.push_back(logical_or);

      operators.push_back(AST_Node_Type::Logical_And);
      std::vector<std::string> logical_and;
      logical_and += "&&";
      operator_matches.push_back(logical_and);

      operators.push_back(AST_Node_Type::Bitwise_Or);
      std::vector<std::string> bitwise_or;
      bitwise_or += "|";
      operator_matches.push_back(bitwise_or);

      operators.push_back(AST_Node_Type::Bitwise_Xor);
      std::vector<std::string> bitwise_xor;
      bitwise_xor += "^";
      operator_matches.push_back(bitwise_xor);           

      operators.push_back(AST_Node_Type::Bitwise_And);
      std::vector<std::string> bitwise_and;
      bitwise_and += "&";
      operator_matches.push_back(bitwise_and);

      operators.push_back(AST_Node_Type::Equality);
      std::vector<std::string> equality;
      equality += "==", "!=";
      operator_matches.push_back(equality);           

      operators.push_back(AST_Node_Type::Comparison);
      std::vector<std::string> comparison;
      comparison += "<", "<=", ">", ">=";
      operator_matches.push_back(comparison);

      operators.push_back(AST_Node_Type::Shift);
      std::vector<std::string> shift;
      shift += "<<", ">>";
      operator_matches.push_back(shift);           

      operators.push_back(AST_Node_Type::Additive);
      std::vector<std::string> additive;
      additive += "+", "-";
      operator_matches.push_back(additive);

      operators.push_back(AST_Node_Type::Multiplicative);
      std::vector<std::string> multiplicative;
      multiplicative += "*", "/", "%";
      operator_matches.push_back(multiplicative);

      operators.push_back(AST_Node_Type::Dot_Access);
      std::vector<std::string> dot_access;
      dot_access += ".";
      operator_matches.push_back(dot_access);
    }
    /**
     * Prints the parsed ast_nodes as a tree
     */
    void debug_print(AST_NodePtr t, std::string prepend = "") {
      std::cout << prepend << "(" << ast_node_type_to_string(t->identifier) << ") " << t->text << " : " << t->start.line << ", " << t->start.column << std::endl;
      for (unsigned int j = 0; j < t->children.size(); ++j) {
        debug_print(t->children[j], prepend + "  ");
      }
    }

    /**
     * Shows the current stack of matched ast_nodes
     */
    void show_match_stack() {
      for (unsigned int i = 0; i < match_stack.size(); ++i) {
        debug_print(match_stack[i]);
      }
    }

    /**
     * Clears the stack of matched ast_nodes
     */
    void clear_match_stack() {
      match_stack.clear();
    }

    /**
     * Returns the front-most AST node
     */
    AST_NodePtr ast() {
      return match_stack.front();
    }

    /**
     * Helper function that collects ast_nodes from a starting position to the top of the stack into a new AST node
     */
    void build_match(AST_NodePtr t, int match_start) {
      int pos_line_start, pos_col_start, pos_line_stop, pos_col_stop;
      int is_deep = false;
      
      //so we want to take everything to the right of this and make them children
      if (match_start != int(match_stack.size())) {
        pos_line_start = match_stack[match_start]->start.line;
        pos_col_start = match_stack[match_start]->start.column;
        pos_line_stop = line;
        pos_col_stop = col;
        is_deep = true;
      }
      else {
        pos_line_start = line;
        pos_col_start = col;
        pos_line_stop = line;
        pos_col_stop = col;
      }

      t->filename = filename;
      t->start.line = pos_line_start;
      t->start.column = pos_col_start;
      t->end.line = pos_line_stop;
      t->end.column = pos_col_stop;
      
      if (is_deep) {
        t->children.assign(match_stack.begin() + (match_start), match_stack.end());
        match_stack.erase(match_stack.begin() + (match_start), match_stack.end());
        match_stack.push_back(t);
      }
      else {
        //todo: fix the fact that a successful match that captured no ast_nodes doesn't have any real start position
        match_stack.push_back(t);
      }
    }

    /**
     * Does ranged char check
     */
    inline bool char_between(char start, char end) {
      if ((*input_pos >= start) && (*input_pos <= end)) {
        return true;
      }
      else {
        return false;
      }
    }

    /**
     * Check to see if there is more text parse
     */
    inline bool has_more_input() {
      return (input_pos != input_end);
    }
                                
    /**
     * Skips any multi-line or single-line comment
     */
    bool SkipComment() {
      bool retval = false;

      if (Symbol_(multiline_comment_begin.c_str())) {
        while (input_pos != input_end) {
          if (Symbol_(multiline_comment_end.c_str())) {
            break;
          }
          else if (!Eol_()) {
            ++col;
            ++input_pos;
          }
        }
        retval = true;
      }
      else if (Symbol_(singleline_comment.c_str())) {
        while (input_pos != input_end) {
          if (Symbol_("\r\n")) {
            input_pos -= 2;
            break;
          }
          else if (Char_('\n')) {
            --input_pos;
            break;
          }
          else {
            ++col;
            ++input_pos;
          }
{'fraction_non_alphanumeric': 0.12926937926937926, 'fraction_numerical': 0.0016335016335016334, 'mean_word_length': 2.4821613236814892, 'pattern_counts': {'":': 0, '<': 103, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6713366', 'n_tokens_mistral': 4314, 'n_tokens_neox': 4216, 'n_words': 808}