<commit_before>#include <cctype>
#include <iostream>
#include "constants.hpp"
#include "prelexer.hpp"


namespace Sass {
  using namespace Constants;

  namespace Prelexer {
    using std::cerr; using std::endl;
    // Matches zero characters (always succeeds without consuming input).
    const char* epsilon(char *src) {
      return src;
    }
    // Matches the empty string.
    const char* empty(char *src) {
      return *src ? 0 : src;
    }

    // Match any single character.
    const char* any_char(const char* src) { return *src ? src+1 : src; }

    // Match a single character satisfying the ctype predicates.
    const char* space(const char* src) { return std::isspace(*src) ? src+1 : 0; }
    const char* alpha(const char* src) { return std::isalpha(*src) ? src+1 : 0; }
    const char* digit(const char* src) { return std::isdigit(*src) ? src+1 : 0; }
    const char* xdigit(const char* src) { return std::isxdigit(*src) ? src+1 : 0; }
    const char* alnum(const char* src) { return std::isalnum(*src) ? src+1 : 0; }
    const char* punct(const char* src) { return std::ispunct(*src) ? src+1 : 0; }
    // Match multiple ctype characters.
    const char* spaces(const char* src) { return one_plus<space>(src); }
    const char* alphas(const char* src) { return one_plus<alpha>(src); }
    const char* digits(const char* src) { return one_plus<digit>(src); }
    const char* xdigits(const char* src) { return one_plus<xdigit>(src); }
    const char* alnums(const char* src) { return one_plus<alnum>(src); }
    const char* puncts(const char* src) { return one_plus<punct>(src); }

    // Match a line comment.
    const char* line_comment(const char* src) { return to_endl<slash_slash>(src); }
    // Match a line comment prefix.
    const char* line_comment_prefix(const char* src) { return exactly<slash_slash>(src); }


    // Match a block comment.
    const char* block_comment(const char* src) {
      return sequence< optional_spaces, delimited_by<slash_star, star_slash, false> >(src);<commit_msg>Treat all non ascii chars (utf-8) as alpha characters<commit_after>#include <cctype>
#include <iostream>
#include "constants.hpp"
#include "prelexer.hpp"


namespace Sass {
  using namespace Constants;

  namespace Prelexer {
    using std::cerr; using std::endl;
    // Matches zero characters (always succeeds without consuming input).
    const char* epsilon(char *src) {
      return src;
    }
    // Matches the empty string.
    const char* empty(char *src) {
      return *src ? 0 : src;
    }

    // Match any single character.
    const char* any_char(const char* src) { return *src ? src+1 : src; }

    // Match a single character satisfying the ctype predicates.
    const char* space(const char* src) { return std::isspace(*src) ? src+1 : 0; }
    const char* alpha(const char* src) { return std::isalpha(*src) || !isascii(*src) ? src+1 : 0; }
    const char* digit(const char* src) { return std::isdigit(*src) ? src+1 : 0; }
    const char* xdigit(const char* src) { return std::isxdigit(*src) ? src+1 : 0; }
    const char* alnum(const char* src) { return std::isalnum(*src) || !isascii(*src) ? src+1 : 0; }
    const char* punct(const char* src) { return std::ispunct(*src) ? src+1 : 0; }
    // Match multiple ctype characters.
    const char* spaces(const char* src) { return one_plus<space>(src); }
    const char* alphas(const char* src) { return one_plus<alpha>(src); }
    const char* digits(const char* src) { return one_plus<digit>(src); }
    const char* xdigits(const char* src) { return one_plus<xdigit>(src); }
    const char* alnums(const char* src) { return one_plus<alnum>(src); }
    const char* puncts(const char* src) { return one_plus<punct>(src); }

    // Match a line comment.
    const char* line_comment(const char* src) { return to_endl<slash_slash>(src); }
    // Match a line comment prefix.
    const char* line_comment_prefix(const char* src) { return exactly<slash_slash>(src); }


    // Match a block comment.
    const char* block_comment(const char* src) {
      return sequence< optional_spaces, delimited_by<slash_star, star_slash, false> >(src);
{'fraction_non_alphanumeric': 0.13603297769156159, 'fraction_numerical': 0.007032007759456838, 'mean_word_length': 3.64527027027027, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5579073', 'n_tokens_mistral': 1373, 'n_tokens_neox': 1323, 'n_words': 419}