<commit_before>
#include "yacas/yacasprivate.h"
#include "yacas/tokenizer.h"

/* Default parse algorithm:
 *
 * Numbers: start with + or -, and continues to have digits or .
 * ( and ) as brackets
 * Alpha atom: First char isalpha, and after that isalnum.
 * Oper atom:  all chars have to be symbolic
 *
 */

#include "yacas/yacasprivate.h"
#include "yacas/lisperror.h"
#include "yacas/numbers.h"

#include <cassert>

static const char symbolics[] = "~`!@#$^&*-=+:<>?/\\|";

bool IsSymbolic(LispChar c)
{
    for (const char *ptr = symbolics; *ptr; ++ptr)
        if (*ptr == c)
            return true;

    return false;
}

const LispString* LispTokenizer::NextToken(LispInput& aInput,
                                           LispHashTable& aHashTable)
{
  LispChar c;
  LispInt firstpos;

REDO:
  firstpos = aInput.Position();

  // End of stream: return empty string
  if (aInput.EndOfStream())
    goto FINISH;

  c = aInput.Next();
  //printf("%c",c);

  //Parse brackets
  if (c == '(')      {}
  else if (c == ')') {}
  else if (c == '{') {}
  else if (c == '}') {}
  else if (c == '[') {}
  else if (c == ']') {}
  else if (c == ',') {}
  else if (c == ';') {}
  else if (c == '%') {}
  else if (c == '.' && !std::isdigit(aInput.Peek()) )
  {
    while (aInput.Peek() == '.')
    {
      aInput.Next();
    }
  }
  // parse comments
  else if (c == '/' && aInput.Peek() == '*')
  {
    aInput.Next(); //consume *
FALSEALARM:
    while (aInput.Next() != '*' && !aInput.EndOfStream());

    if (aInput.EndOfStream())
        throw LispErrCommentToEndOfFile();

    if (aInput.Peek() == '/')
    {
      aInput.Next();  // consume /
      goto REDO;
    }
    goto FALSEALARM;
  }
  else if (c == '/' && aInput.Peek() == '/')
  {
    aInput.Next(); //consume /
    while (aInput.Next() != '\n' && !aInput.EndOfStream());
    goto REDO;
  }
  // parse literal strings
  else if (c == '\"')
  {
    std::string str;
    str.push_back(c);
    while (aInput.Peek() != '\"')
    {
      if (aInput.Peek() == '\\')
      {
        aInput.Next();

        if (aInput.EndOfStream())
            throw LispErrParsingInput();
      }
      str.push_back(aInput.Next());

      if (aInput.EndOfStream())
          throw LispErrParsingInput();
    }
    str.push_back(aInput.Next()); // consume the close quote
    return aHashTable.LookUp(str);
  }
  //parse atoms
  else if (IsAlpha(c))
  {
    while (IsAlNum( aInput.Peek()))
    {
      aInput.Next();
    }
  }

  else if (IsSymbolic(c))
  {
    while (IsSymbolic( aInput.Peek()))
    {
      aInput.Next();
    }
  }
  else if (c == '_')
  {
    while (aInput.Peek() == '_')
    {
      aInput.Next();
    }
  }
  else if (std::isdigit(c) || c == '.')
  {
    while (std::isdigit( aInput.Peek())) aInput.Next();
    if (aInput.Peek() == '.')
    {
      aInput.Next();
      while (std::isdigit( aInput.Peek())) aInput.Next();
    }
    if (aInput.Peek() == 'e' || aInput.Peek() == 'E') {
        aInput.Next();
        if (aInput.Peek() == '-' || aInput.Peek() == '+')
            aInput.Next();
        while (std::isdigit( aInput.Peek()))
            aInput.Next();
    }
  }
  // Treat the char as a space.
  else
  {
    goto REDO;
  }

FINISH:
  return aHashTable.LookUp(std::string(&aInput.StartPtr()[firstpos],aInput.Position()-firstpos));
}<commit_msg>do not use StartPtr in LispTokenizer, related #75<commit_after>#include "yacas/tokenizer.h"

#include "yacas/lisperror.h"

namespace {
    static const char symbolics[] = "~`!@#$^&*-=+:<>?/\\|";
}

bool IsSymbolic(LispChar c)
{
    for (const char *ptr = symbolics; *ptr; ++ptr)
        if (*ptr == c)
            return true;

    return false;
}

const LispString* LispTokenizer::NextToken(LispInput& aInput,
                                           LispHashTable& aHashTable)
{
    char c;

    // skip whitespaces and comments
    for (;;) {
        // End of stream: return empty string
        if (aInput.EndOfStream())
            return aHashTable.LookUp("");

        c = aInput.Next();

        if (std::isspace(c))
            continue;

        // parse comments
        if (c == '/' && aInput.Peek() == '*') {
            aInput.Next();
            for (;;) {
                while (aInput.Next() != '*' && !aInput.EndOfStream())
                    ;

                if (aInput.EndOfStream())
                    throw LispErrCommentToEndOfFile();

                if (aInput.Peek() == '/') {
                    aInput.Next();
                    break;
                }
            }

            continue;
        }

        if (c == '/' && aInput.Peek() == '/') {
            aInput.Next();
            while (aInput.Next() != '\n' && !aInput.EndOfStream())
                ;
            continue;
        }

        break;
    }

    // parse brackets
    if (c == '(' || c == ')' || c == '{' || c == '}' || c == '[' || c == ']')
        return aHashTable.LookUp(std::string(1, c));

    // percent
    if (c == '%')
        return aHashTable.LookUp(std::string(1, c));

    // comma and semicolon
    if (c == ',' || c == ';')
        return aHashTable.LookUp(std::string(1, c));

    // parse . or ..
    if (c == '.' && !std::isdigit(aInput.Peek())) {
        std::string token;
        token.push_back(c);
        while (aInput.Peek() == '.')
            token.push_back(aInput.Next());
        return aHashTable.LookUp(token);
    }

    // parse literal strings
    if (c == '\"') {
        std::string str;
        str.push_back(c);
        while (aInput.Peek() != '\"') {
            if (aInput.Peek() == '\\') {
                aInput.Next();

                if (aInput.EndOfStream())
                    throw LispErrParsingInput();
            }
            str.push_back(aInput.Next());

            if (aInput.EndOfStream())
                throw LispErrParsingInput();
        }
        str.push_back(aInput.Next());
        return aHashTable.LookUp(str);
    }

    // parse atoms
    if (IsAlpha(c)) {
        std::string atom;
        atom.push_back(c);
        while (IsAlNum(aInput.Peek()))
            atom.push_back(aInput.Next());
        return aHashTable.LookUp(atom);
    }

    // parse operators
    if (IsSymbolic(c)) {
        std::string op;
        op.push_back(c);
        while (IsSymbolic(aInput.Peek()))
            op.push_back(aInput.Next());
        return aHashTable.LookUp(op);
    }

    // parse subscripts
    if (c == '_') {
        std::string token;
        token.push_back(c);
        while (aInput.Peek() == '_')
            token.push_back(aInput.Next());
        return aHashTable.LookUp(token);
    }

    // parse numbers
    if (std::isdigit(c) || c == '.') {
        std::string number;
        number.push_back(c);

        while (std::isdigit(aInput.Peek()))
            number.push_back(aInput.Next());

        if (aInput.Peek() == '.') {
            number.push_back(aInput.Next());
            while (std::isdigit(aInput.Peek()))
                number.push_back(aInput.Next());
        }

        if (aInput.Peek() == 'e' || aInput.Peek() == 'E') {
            number.push_back(aInput.Next());
            if (aInput.Peek() == '-' || aInput.Peek() == '+')
                number.push_back(aInput.Next());
            while (std::isdigit(aInput.Peek()))
                number.push_back(aInput.Next());
        }

        return aHashTable.LookUp(number);
    }

    throw InvalidToken();
}
{'fraction_non_alphanumeric': 0.18665403356794802, 'fraction_numerical': 0.0006767731456415809, 'mean_word_length': 1.8234619793656859, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1602232', 'n_tokens_mistral': 2605, 'n_tokens_neox': 2534, 'n_words': 482}