<commit_before>/******************************************************************************
* Copyright (c) 2011, <NAME> (<EMAIL>)
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following
* conditions are met:
*
*     * Redistributions of source code must retain the above copyright
*       notice, this list of conditions and the following disclaimer.
*     * Redistributions in binary form must reproduce the above copyright
*       notice, this list of conditions and the following disclaimer in
*       the documentation and/or other materials provided
*       with the distribution.
*     * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
*       names of its contributors may be used to endorse or promote
*       products derived from this software without specific prior
*       written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
* OF SUCH DAMAGE.
****************************************************************************/

#include <pdal/pdal_internal.hpp>
#include <pdal/plang/Parser.hpp>

#include <pdal/plang/AstNode.hpp>
#include <pdal/plang/Program.hpp>
#include <pdal/plang/SymbolTable.hpp>

#ifdef PDAL_COMPILER_MSVC
#  pragma warning(push)
#  pragma warning(disable: 4512)  // assignment operator could not be generated
#  pragma warning(disable: 4127)  // conditional expression is constant
#  pragma warning(disable: 4100)  // unreferenced formal
#endif

#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_object.hpp>

#ifdef PDAL_COMPILER_MSVC
#  pragma warning(pop)
#endif


#include <iostream>
#include <string>
#include <complex>
#include <vector>


namespace qi = boost::spirit::qi;
#ifdef PDAL_EMBED_BOOST
namespace pho = boost::pdalboostphoenix;
#else
namespace pho = boost::phoenix;
#endif
namespace ascii = boost::spirit::ascii;


namespace pdal { namespace plang {


struct PlangParser : qi::grammar<std::string::const_iterator, Program*(), ascii::space_type>
{
    PlangParser();

    qi::rule<std::string::const_iterator, Program*(), ascii::space_type> program;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> assignments;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> assignment;
    qi::rule<std::string::const_iterator, SymbolTable*(), ascii::space_type> declarations;
    qi::rule<std::string::const_iterator, Symbol*(), ascii::space_type> declaration;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> lhs;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> rhs;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> constant;
    qi::rule<std::string::const_iterator, SymbolName*(), ascii::space_type> variable;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> predicate;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> predicate_tail;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> predicate_tail_star;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> expr;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> expr_tail;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> expr_tail_star;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> term;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> term_tail;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> term_tail_star;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> factor;
};


PlangParser::PlangParser() : PlangParser::base_type(program)
{
#define SET_P(T, p)      qi::_val = pho::new_<T>(p)
#define SET_1(T)         qi::_val = pho::new_<T>(qi::_1)
#define SET_1_P(T, p)    qi::_val = pho::new_<T>(qi::_1, p)
#define SET_1_2(T)       qi::_val = pho::new_<T>(qi::_1, qi::_2)
#define SET_1_2_P(T, p)  qi::_val = pho::new_<T>(qi::_1, qi::_2, p)
#define PASS             qi::_val = qi::_1

    qi::real_parser<float, qi::strict_real_policies<float> > strict_float;
    qi::real_parser<double, qi::strict_real_policies<double> > strict_double;

    constant = ( ("0x" >> qi::hex >> 'l')       [ SET_1(AstConstant) ]
               | ("0x" >> qi::hex)              [ SET_1(AstConstant) ]
               | (strict_float >> 'f')          [ SET_1(AstConstant) ]
               | strict_double                  [ SET_1(AstConstant) ]
               | (qi::ulong_long >> "ul")       [ SET_1(AstConstant) ]
               | (qi::ulong_long >> "lu")       [ SET_1(AstConstant) ]
               | (qi::long_long >> 'l')         [ SET_1(AstConstant) ]
               | (qi::uint_ >> 'u')             [ SET_1(AstConstant) ]
               | qi::int_                       [ SET_1(AstConstant) ]
               | qi::lit("true")                [ SET_P(AstConstant, true) ]
               | qi::lit("false")               [ SET_P(AstConstant, false) ]
               )
             ;

    // (letter or _) followed by *(letter or number or _)
    variable = ( (qi::alpha|qi::char_('_'))  >> *(qi::alpha | qi::digit | qi::char_('_')))    [ SET_1_2(SymbolName) ]
             ;

    predicate_tail = ( ">" >> expr     [ SET_1_P(AstTempOp1, NodeType_Greater1) ]
                     | ">=" >> expr    [ SET_1_P(AstTempOp1, NodeType_GreaterEq1) ]
                     | "<" >> expr     [ SET_1_P(AstTempOp1, NodeType_Less1) ]
                     | "<=" >> expr    [ SET_1_P(AstTempOp1, NodeType_LessEq1) ]
                     | "==" >> expr    [ SET_1_P(AstTempOp1, NodeType_Equal1) ]
                     | "!=" >> expr    [ SET_1_P(AstTempOp1, NodeType_NotEqual1) ]
                     | "&&" >> expr    [ SET_1_P(AstTempOp1, NodeType_LogicalAnd1) ]
                     | "||" >> expr    [ SET_1_P(AstTempOp1, NodeType_LogicalOr1) ]
                     ) 
                   ;

    predicate_tail_star = (*predicate_tail)    [ SET_1(AstTempVector) ]
                        ;
                  
    predicate = (expr >> predicate_tail_star)    [ SET_1_2(AstTempVector) ]
              ;<commit_msg>turn off 64bit integer types for plang for now<commit_after>/******************************************************************************
* Copyright (c) 2011, <NAME> (<EMAIL>)
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following
* conditions are met:
*
*     * Redistributions of source code must retain the above copyright
*       notice, this list of conditions and the following disclaimer.
*     * Redistributions in binary form must reproduce the above copyright
*       notice, this list of conditions and the following disclaimer in
*       the documentation and/or other materials provided
*       with the distribution.
*     * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
*       names of its contributors may be used to endorse or promote
*       products derived from this software without specific prior
*       written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
* OF SUCH DAMAGE.
****************************************************************************/

#include <pdal/pdal_internal.hpp>
#include <pdal/plang/Parser.hpp>

#include <pdal/plang/AstNode.hpp>
#include <pdal/plang/Program.hpp>
#include <pdal/plang/SymbolTable.hpp>

#ifdef PDAL_COMPILER_MSVC
#  pragma warning(push)
#  pragma warning(disable: 4512)  // assignment operator could not be generated
#  pragma warning(disable: 4127)  // conditional expression is constant
#  pragma warning(disable: 4100)  // unreferenced formal
#endif

#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_object.hpp>

#ifdef PDAL_COMPILER_MSVC
#  pragma warning(pop)
#endif


#include <iostream>
#include <string>
#include <complex>
#include <vector>


namespace qi = boost::spirit::qi;
#ifdef PDAL_EMBED_BOOST
namespace pho = boost::pdalboostphoenix;
#else
namespace pho = boost::phoenix;
#endif
namespace ascii = boost::spirit::ascii;


namespace pdal { namespace plang {


struct PlangParser : qi::grammar<std::string::const_iterator, Program*(), ascii::space_type>
{
    PlangParser();

    qi::rule<std::string::const_iterator, Program*(), ascii::space_type> program;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> assignments;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> assignment;
    qi::rule<std::string::const_iterator, SymbolTable*(), ascii::space_type> declarations;
    qi::rule<std::string::const_iterator, Symbol*(), ascii::space_type> declaration;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> lhs;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> rhs;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> constant;
    qi::rule<std::string::const_iterator, SymbolName*(), ascii::space_type> variable;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> predicate;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> predicate_tail;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> predicate_tail_star;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> expr;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> expr_tail;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> expr_tail_star;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> term;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> term_tail;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> term_tail_star;
    qi::rule<std::string::const_iterator, AstNode*(), ascii::space_type> factor;
};


PlangParser::PlangParser() : PlangParser::base_type(program)
{
#define SET_P(T, p)      qi::_val = pho::new_<T>(p)
#define SET_1(T)         qi::_val = pho::new_<T>(qi::_1)
#define SET_1_P(T, p)    qi::_val = pho::new_<T>(qi::_1, p)
#define SET_1_2(T)       qi::_val = pho::new_<T>(qi::_1, qi::_2)
#define SET_1_2_P(T, p)  qi::_val = pho::new_<T>(qi::_1, qi::_2, p)
#define PASS             qi::_val = qi::_1

    qi::real_parser<float, qi::strict_real_policies<float> > strict_float;
    qi::real_parser<double, qi::strict_real_policies<double> > strict_double;

    constant = ( ("0x" >> qi::hex >> 'l')       [ SET_1(AstConstant) ]
               | ("0x" >> qi::hex)              [ SET_1(AstConstant) ]
               | (strict_float >> 'f')          [ SET_1(AstConstant) ]
               | strict_double                  [ SET_1(AstConstant) ]
#if notdef
// This doesn't work on the jenkins build.  
/* I'm getting an error of boost/variant/variant.hpp:1326:9: error: call of overloaded ‘initialize(void*, long long int&)’ is ambiguous and boost/variant/variant.hpp:1326:9: error: call of overloaded ‘initialize(void*, long long unsigned int&)’ is ambiguous yet BOOST_HAS_LONG_LONG is defined for my setup.  Pointers to settings/macros other than BOOST_HAS_LONG_LONG that I might look at would be greatly appreciated
[10:49am] hobu: The note there says: "Compile error here indicates that the given type not is unambiguously convertible to one of the variant's types" but I'm not sure what that means...
[10:50am] hobu: http://stackoverflow.com/questions/6571837/why-does-boostspiritqiparse-not-set-this-boostvariants-value seems pertinent, as this is a qi situation for me as well
[10:54am] mdupont: hobu, cast it
[10:54am] mdupont: you have the tsecond param is either int& or unsigned int ^
[10:54am] mdupont: you have the tsecond param is either int& or unsigned int &
[10:54am] VeXocide: http://www.boost.org/doc/libs/1_48_0/libs/spirit/doc/html/spirit/qi/reference/auxiliary/attr_cast.html
[10:54am] mdupont: you can declare a variable of the right type before and pass that in
*/
               | (qi::ulong_long >> "ul")       [ SET_1(AstConstant) ]
               | (qi::ulong_long >> "lu")       [ SET_1(AstConstant) ]
               | (qi::long_long >> 'l')         [ SET_1(AstConstant) ]
#endif
               | (qi::uint_ >> 'u')             [ SET_1(AstConstant) ]
               | qi::int_                       [ SET_1(AstConstant) ]
               | qi::lit("true")                [ SET_P(AstConstant, true) ]
               | qi::lit("false")               [ SET_P(AstConstant, false) ]
               )
             ;

    // (letter or _) followed by *(letter or number or _)
    variable = ( (qi::alpha|qi::char_('_'))  >> *(qi::alpha | qi::digit | qi::char_('_')))    [ SET_1_2(SymbolName) ]
             ;

    predicate_tail = ( ">" >> expr     [ SET_1_P(AstTempOp1, NodeType_Greater1) ]
                     | ">=" >> expr    [ SET_1_P(AstTempOp1, NodeType_GreaterEq1) ]
                     | "<" >> expr     [ SET_1_P(AstTempOp1, NodeType_Less1) ]
                     | "<=" >> expr    [ SET_1_P(AstTempOp1, NodeType_LessEq1) ]
                     | "==" >> expr    [ SET_1_P(AstTempOp1, NodeType_Equal1) ]
                     | "!=" >> expr    [ SET_1_P(AstTempOp1, NodeType_NotEqual1) ]
                     | "&&" >> expr    [ SET_1_P(AstTempOp1, NodeType_LogicalAnd1) ]
                     | "||" >> expr    [ SET_1_P(AstTempOp1, NodeType_LogicalOr1) ]
                     ) 
                   ;

    predicate_tail_star = (*predicate_tail)    [ SET_1(AstTempVector) ]
                        ;
                  
    predicate = (expr >> predicate_tail_star)    [ SET_1_2(AstTempVector) ]
              ;
{'fraction_non_alphanumeric': 0.15058219853956978, 'fraction_numerical': 0.012433392539964476, 'mean_word_length': 3.601089588377724, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 165, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4976225', 'n_tokens_mistral': 5120, 'n_tokens_neox': 4708, 'n_words': 1165}