<commit_before>#include "Parser.hpp"
#include <algorithm>
using namespace std;
using namespace std::experimental;
//-------------------------------------------------------------------
optional<string> Parser::getNextToken() {
  const auto whitespace = string{" \t\n"};
  const auto seperators = string{" \t\n,)(;"};
  //ignore leading whitespace
  while(in->good() && whitespace.find(in->peek()) != string::npos) {
    if(in->get() == '\n') lineno++;
  }
  //parse the actual token
  string token;
  char firstChar;
  if(!in->get(firstChar)) {
    return nullopt;
  }
  token.push_back(firstChar);
  if(seperators.find(firstChar) == string::npos) {
    while(in->good() && seperators.find(in->peek()) == string::npos) {
      token.push_back(in->get());
    }
  }
  std::transform(token.begin(), token.end(), token.begin(), ::tolower);
  return token;
}
//-------------------------------------------------------------------
string Parser::expectToken() {
  auto token = getNextToken();
  if(!token) {
    throw ParserError(lineno, "Unexpected end of stream");
  }
  return *token;
}
//-------------------------------------------------------------------
void Parser::expectToken(const char* expected) {
  auto token = expectToken();
  if(token != expected){
    throw ParserError(lineno, string{"Expected token \""} + expected + "\"; found: \"" + token + "\"");
  }
}
//-------------------------------------------------------------------
bool Parser::isIdentifier(string token) {
  if(token == "create"    ||
     token == "table"     ||
     token == "integer"   ||
     token == "numeric"   ||
     token == "char"      ||
     token == "varchar"   ||
     token == "timestamp" ||
     token == "not"       ||
     token == "null"      ||
     token == "primary"   ||
     token == "key") {<commit_msg>Parser: support for quoted identifiers<commit_after>#include "Parser.hpp"
#include <algorithm>
using namespace std;
using namespace std::experimental;
//-------------------------------------------------------------------
optional<string> Parser::getNextToken() {
  const auto whitespace = string{" \t\n"};
  const auto seperators = string{" \t\n,)(;\""};
  //ignore leading whitespace
  while(in->good() && whitespace.find(in->peek()) != string::npos) {
    if(in->get() == '\n') lineno++;
  }
  //parse the actual token
  string token;
  char firstChar;
  if(!in->get(firstChar)) {
    return nullopt;
  }
  if(firstChar == '"') {
    while(in->get(firstChar) && firstChar != '"') {
      token.push_back(firstChar);
    }
  } else {
    token.push_back(firstChar);
    if(seperators.find(firstChar) == string::npos) {
      while(in->good() && seperators.find(in->peek()) == string::npos) {
        token.push_back(in->get());
      }
    }
    std::transform(token.begin(), token.end(), token.begin(), ::tolower);
  }
  return token;
}
//-------------------------------------------------------------------
string Parser::expectToken() {
  auto token = getNextToken();
  if(!token) {
    throw ParserError(lineno, "Unexpected end of stream");
  }
  return *token;
}
//-------------------------------------------------------------------
void Parser::expectToken(const char* expected) {
  auto token = expectToken();
  if(token != expected){
    throw ParserError(lineno, string{"Expected token \""} + expected + "\"; found: \"" + token + "\"");
  }
}
//-------------------------------------------------------------------
bool Parser::isIdentifier(string token) {
  if(token == "create"    ||
     token == "table"     ||
     token == "integer"   ||
     token == "numeric"   ||
     token == "char"      ||
     token == "varchar"   ||
     token == "timestamp" ||
     token == "not"       ||
     token == "null"      ||
     token == "primary"   ||
     token == "key") {
{'fraction_non_alphanumeric': 0.3109288171473935, 'fraction_numerical': 0.0, 'mean_word_length': 1.8506787330316743, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4965130', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1088, 'n_words': 229}