<commit_before>/*
   Copyright 2012 <NAME> <<EMAIL>>
   All rights reserved.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/


#ifndef P1_SMALLS_PARSER_SYNTAX_HPP
#define	P1_SMALLS_PARSER_SYNTAX_HPP

#include "p1/smalls/common/SourceCoords.hpp"
#include <vector>
#include <utility>
#include <cstring>
#include <cassert>
#include <stdint.h>

namespace p1 {
namespace smalls {

class Scope;

struct Mark
{
  int32_t const value; // -1 means anti-mark
  Scope * const scope;
  Mark * const next;

  Mark ( int32_t value_, Scope * scope_, Mark * next_ )
    : value(value_), scope(scope_), next( next_ )
  {
    assert( value < 0 || scope != NULL );
  }

  bool isAntiMark () const { return value < 0; };
  bool isMark () const { return value > 0; };

  bool equal ( const Mark * m ) const;
  void toStream ( std::ostream & os ) const;
};

inline std::ostream & operator << ( std::ostream & os, const Mark & m )
{
  m.toStream( os );
  return os;
}

Mark * concat ( Mark * first, Mark * second );
bool equal ( const Mark * a, const Mark * b );

#define _DEF_SYNTAX_CLASSES \
   _MK_ENUM(NONE) \
   _MK_ENUM(DEOF) \
   _MK_ENUM(COMMENT) \
   _MK_ENUM(REAL) \
   _MK_ENUM(INTEGER) \
   _MK_ENUM(BOOL) \
   _MK_ENUM(STR) \
   _MK_ENUM(SYMBOL) \
   _MK_ENUM(BINDING) \
   _MK_ENUM(PAIR) \
   _MK_ENUM(NIL) \
   _MK_ENUM(VECTOR) \


struct SyntaxKind
{
  #define _MK_ENUM(name)  name,
  enum Enum
  {
    _DEF_SYNTAX_CLASSES
  };
  #undef _MK_ENUM

  static const char * name ( Enum x ) { return s_names[x]; }

private:
  static const char * s_names[];
};

class Syntax : public gc
{
public:
  SyntaxKind::Enum const skind;
  SourceCoords coords;
  Syntax ( SyntaxKind::Enum skind_, const SourceCoords & coords_ ) : skind( skind_ ), coords( coords_ ) {}

  bool isNil () const { return this->skind == SyntaxKind::NIL; }

  virtual Syntax * wrap ( Mark * mark );

  virtual void toStream ( std::ostream & os ) const;
  virtual bool equal ( const Syntax * x ) const;

  static void toStreamIndented ( std::ostream & os, unsigned indent, const Syntax * dat );
};

Syntax * unwrapCompletely ( Syntax * syntax, Mark * mark = NULL );

inline std::ostream & operator << ( std::ostream & os, const Syntax & dat )
{
  dat.toStream(os);
  return os;
}

class Symbol;
class Binding;

class SyntaxValue : public Syntax
{
public:
  union
  {
    double real;
    int64_t integer;
    bool     vbool;
    const gc_char * str;
  } u;

  SyntaxValue ( SyntaxKind::Enum skind_, const SourceCoords & coords_, double real )
    : Syntax( skind_, coords_ ) { assert(skind == SyntaxKind::REAL); u.real = real; }
  SyntaxValue ( SyntaxKind::Enum skind_, const SourceCoords & coords_, int64_t integer )
    : Syntax( skind_, coords_ ) { assert(skind == SyntaxKind::INTEGER); u.integer = integer; }
  SyntaxValue ( SyntaxKind::Enum skind_, const SourceCoords & coords_, bool vbool )
    : Syntax( skind_, coords_ ) { assert(skind == SyntaxKind::BOOL); u.vbool = vbool; }<commit_msg>smalls/parser/Syntax.hpp: move class forward definitions to top<commit_after>/*
   Copyright 2012 <NAME> <<EMAIL>>
   All rights reserved.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/


#ifndef P1_SMALLS_PARSER_SYNTAX_HPP
#define	P1_SMALLS_PARSER_SYNTAX_HPP

#include "p1/smalls/common/SourceCoords.hpp"
#include <vector>
#include <utility>
#include <cstring>
#include <cassert>
#include <stdint.h>

namespace p1 {
namespace smalls {
  class Scope;
  class Symbol;
  class Binding;
}}

namespace p1 {
namespace smalls {

struct Mark
{
  int32_t const value; // -1 means anti-mark
  Scope * const scope;
  Mark * const next;

  Mark ( int32_t value_, Scope * scope_, Mark * next_ )
    : value(value_), scope(scope_), next( next_ )
  {
    assert( value < 0 || scope != NULL );
  }

  bool isAntiMark () const { return value < 0; };
  bool isMark () const { return value > 0; };

  bool equal ( const Mark * m ) const;
  void toStream ( std::ostream & os ) const;
};

inline std::ostream & operator << ( std::ostream & os, const Mark & m )
{
  m.toStream( os );
  return os;
}

Mark * concat ( Mark * first, Mark * second );
bool equal ( const Mark * a, const Mark * b );

#define _DEF_SYNTAX_CLASSES \
   _MK_ENUM(NONE) \
   _MK_ENUM(DEOF) \
   _MK_ENUM(COMMENT) \
   _MK_ENUM(REAL) \
   _MK_ENUM(INTEGER) \
   _MK_ENUM(BOOL) \
   _MK_ENUM(STR) \
   _MK_ENUM(SYMBOL) \
   _MK_ENUM(BINDING) \
   _MK_ENUM(PAIR) \
   _MK_ENUM(NIL) \
   _MK_ENUM(VECTOR) \


struct SyntaxKind
{
  #define _MK_ENUM(name)  name,
  enum Enum
  {
    _DEF_SYNTAX_CLASSES
  };
  #undef _MK_ENUM

  static const char * name ( Enum x ) { return s_names[x]; }

private:
  static const char * s_names[];
};

class Syntax : public gc
{
public:
  SyntaxKind::Enum const skind;
  SourceCoords coords;
  Syntax ( SyntaxKind::Enum skind_, const SourceCoords & coords_ ) : skind( skind_ ), coords( coords_ ) {}

  bool isNil () const { return this->skind == SyntaxKind::NIL; }

  virtual Syntax * wrap ( Mark * mark );

  virtual void toStream ( std::ostream & os ) const;
  virtual bool equal ( const Syntax * x ) const;

  static void toStreamIndented ( std::ostream & os, unsigned indent, const Syntax * dat );
};

Syntax * unwrapCompletely ( Syntax * syntax, Mark * mark = NULL );

inline std::ostream & operator << ( std::ostream & os, const Syntax & dat )
{
  dat.toStream(os);
  return os;
}

class SyntaxValue : public Syntax
{
public:
  union
  {
    double real;
    int64_t integer;
    bool     vbool;
    const gc_char * str;
  } u;

  SyntaxValue ( SyntaxKind::Enum skind_, const SourceCoords & coords_, double real )
    : Syntax( skind_, coords_ ) { assert(skind == SyntaxKind::REAL); u.real = real; }
  SyntaxValue ( SyntaxKind::Enum skind_, const SourceCoords & coords_, int64_t integer )
    : Syntax( skind_, coords_ ) { assert(skind == SyntaxKind::INTEGER); u.integer = integer; }
  SyntaxValue ( SyntaxKind::Enum skind_, const SourceCoords & coords_, bool vbool )
    : Syntax( skind_, coords_ ) { assert(skind == SyntaxKind::BOOL); u.vbool = vbool; }
{'fraction_non_alphanumeric': 0.11193603655054254, 'fraction_numerical': 0.006996002284408909, 'mean_word_length': 3.4903846153846154, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6803137', 'n_tokens_mistral': 2483, 'n_tokens_neox': 2408, 'n_words': 732}