<commit_before>#include <tree_sitter/parser.h>
#include <string>
#include <cwctype>

namespace {

using std::wstring;
using std::iswspace;

enum TokenType {
  RAW_STRING_LITERAL,
};

struct Scanner {
  bool scan(TSLexer *lexer, const bool *valid_symbols) {
    while (iswspace(lexer->lookahead)) {
      lexer->advance(lexer, true);
    }

    lexer->result_symbol = RAW_STRING_LITERAL;

    // Consume 'R'
    if (lexer->lookahead != 'R') return false;
    lexer->advance(lexer, false);

    // Consume '"'
    if (lexer->lookahead != '"') return false;
    lexer->advance(lexer, false);

    // Consume '(', delimiter
    wstring delimiter;
    for (;;) {
      if (lexer->lookahead == 0 || lexer->lookahead == '\\' || iswspace(lexer->lookahead)) {
        return false;
      }
      if (lexer->lookahead == '(') {
        lexer->advance(lexer, false);
        break;
      }
      delimiter += lexer->lookahead;
      lexer->advance(lexer, false);
    }

    // Consume content, delimiter, ')', '"'
    int delimiter_index = -1;
    for (;;) {
      if (lexer->lookahead == 0) return false;

      if (delimiter_index >= 0) {
        if (delimiter_index == delimiter.size()) {
          if (lexer->lookahead == '"') {
            lexer->advance(lexer, false);
            return true;
          } else {
            delimiter_index = -1;
          }
        } else {
          if (lexer->lookahead == delimiter[delimiter_index]) {
            delimiter_index++;
          } else {
            delimiter_index = -1;
          }
        }<commit_msg>Fix signed/unsigned comparison in scanner.cc<commit_after>#include <tree_sitter/parser.h>
#include <string>
#include <cwctype>

namespace {

using std::wstring;
using std::iswspace;

enum TokenType {
  RAW_STRING_LITERAL,
};

struct Scanner {
  bool scan(TSLexer *lexer, const bool *valid_symbols) {
    while (iswspace(lexer->lookahead)) {
      lexer->advance(lexer, true);
    }

    lexer->result_symbol = RAW_STRING_LITERAL;

    // Consume 'R'
    if (lexer->lookahead != 'R') return false;
    lexer->advance(lexer, false);

    // Consume '"'
    if (lexer->lookahead != '"') return false;
    lexer->advance(lexer, false);

    // Consume '(', delimiter
    wstring delimiter;
    for (;;) {
      if (lexer->lookahead == 0 || lexer->lookahead == '\\' || iswspace(lexer->lookahead)) {
        return false;
      }
      if (lexer->lookahead == '(') {
        lexer->advance(lexer, false);
        break;
      }
      delimiter += lexer->lookahead;
      lexer->advance(lexer, false);
    }

    // Consume content, delimiter, ')', '"'
    int delimiter_index = -1;
    for (;;) {
      if (lexer->lookahead == 0) return false;

      if (delimiter_index >= 0) {
        if (static_cast<unsigned>(delimiter_index) == delimiter.size()) {
          if (lexer->lookahead == '"') {
            lexer->advance(lexer, false);
            return true;
          } else {
            delimiter_index = -1;
          }
        } else {
          if (lexer->lookahead == delimiter[delimiter_index]) {
            delimiter_index++;
          } else {
            delimiter_index = -1;
          }
        }
{'fraction_non_alphanumeric': 0.1538948701709943, 'fraction_numerical': 0.0037998733375554147, 'mean_word_length': 2.1941354903943378, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '602423', 'n_tokens_mistral': 1123, 'n_tokens_neox': 1054, 'n_words': 204}