<commit_before>#include <cstdlib>
#include <iostream>
#include "parser.hpp"
#include "file.hpp"
#include "inspect.hpp"
#include "to_string.hpp"
#include "constants.hpp"

#ifndef SASS_PRELEXER
#include "prelexer.hpp"
#endif

#include <typeinfo>

namespace Sass {
  using namespace std;
  using namespace Constants;

  Parser Parser::from_c_str(const char* str, Context& ctx, string path, Position source_position)
  {
    Parser p(ctx, path, source_position);
    p.source   = str;
    p.position = p.source;
    p.end      = str + strlen(str);
    return p;
  }

  Parser Parser::from_token(Token t, Context& ctx, string path, Position source_position)
  {
    Parser p(ctx, path, source_position);
    p.source   = t.begin;
    p.position = p.source;
    p.end      = t.end;
    return p;
  }

  Block* Parser::parse()
  {
    Block* root = new (ctx.mem) Block(path, source_position);
    root->is_root(true);
    read_bom();
    lex< optional_spaces >();
    Selector_Lookahead lookahead_result;
    while (position < end) {
      if (lex< block_comment >()) {
        String*  contents = parse_interpolated_chunk(lexed);
        Comment* comment  = new (ctx.mem) Comment(path, source_position, contents);
        (*root) << comment;
      }
      else if (peek< import >()) {
        Import* imp = parse_import();
        if (!imp->urls().empty()) (*root) << imp;
        if (!imp->files().empty()) {
          for (size_t i = 0, S = imp->files().size(); i < S; ++i) {
            (*root) << new (ctx.mem) Import_Stub(path, source_position, imp->files()[i]);
          }
        }
        if (!lex< exactly<';'> >()) error("top-level @import directive must be terminated by ';'");
      }
      else if (peek< mixin >() || peek< function >()) {
        (*root) << parse_definition();
      }
      else if (peek< variable >()) {
        (*root) << parse_assignment();
        if (!lex< exactly<';'> >()) error("top-level variable binding must be terminated by ';'");
      }
      else if (peek< sequence< optional< exactly<'*'> >, alternatives< identifier_schema, identifier >, optional_spaces, exactly<':'>, optional_spaces, exactly<'{'> > >(position)) {
        (*root) << parse_propset();
      }
      else if (peek< include >() /* || peek< exactly<'+'> >() */) {
        Mixin_Call* mixin_call = parse_mixin_call();
        (*root) << mixin_call;
        if (!mixin_call->block() && !lex< exactly<';'> >()) error("top-level @include directive must be terminated by ';'");
      }
      else if (peek< if_directive >()) {
        (*root) << parse_if_directive();
      }
      else if (peek< for_directive >()) {
        (*root) << parse_for_directive();
      }
      else if (peek< each_directive >()) {
        (*root) << parse_each_directive();
      }
      else if (peek< while_directive >()) {
        (*root) << parse_while_directive();
      }
      else if (peek< media >()) {
        (*root) << parse_media_block();
      }
      else if (peek< warn >()) {
        (*root) << parse_warning();
        if (!lex< exactly<';'> >()) error("top-level @warn directive must be terminated by ';'");
      }
      // ignore the @charset directive for now
      else if (lex< exactly< charset_kwd > >()) {
        lex< string_constant >();
        lex< exactly<';'> >();
      }
      else if (peek< at_keyword >()) {
        At_Rule* at_rule = parse_at_rule();
        (*root) << at_rule;
        if (!at_rule->block() && !lex< exactly<';'> >()) error("top-level directive must be terminated by ';'");
      }
      else if ((lookahead_result = lookahead_for_selector(position)).found) {
        (*root) << parse_ruleset(lookahead_result);
      }
      else {
        lex< spaces_and_comments >();
        if (position >= end) break;
        error("invalid top-level expression");
      }
      lex< optional_spaces >();
    }
    return root;
  }

  Import* Parser::parse_import()
  {
    lex< import >();
    Import* imp = new (ctx.mem) Import(path, source_position);
    bool first = true;
    do {
      if (lex< string_constant >()) {
        string import_path(lexed);
        string extension;
        if (import_path.length() > 4) {
          // a string constant is guaranteed to end with a quote mark, so make sure to skip it when indexing from the end
          extension = import_path.substr(import_path.length() - 5, 4);
        }
        if (extension == ".css") {
          String_Constant* loc = new (ctx.mem) String_Constant(path, source_position, import_path);
          Argument* loc_arg = new (ctx.mem) Argument(path, source_position, loc);
          Arguments* loc_args = new (ctx.mem) Arguments(path, source_position);
          (*loc_args) << loc_arg;
          Function_Call* new_url = new (ctx.mem) Function_Call(path, source_position, "url", loc_args);<commit_msg>let's bump up the length check<commit_after>#include <cstdlib>
#include <iostream>
#include "parser.hpp"
#include "file.hpp"
#include "inspect.hpp"
#include "to_string.hpp"
#include "constants.hpp"

#ifndef SASS_PRELEXER
#include "prelexer.hpp"
#endif

#include <typeinfo>

namespace Sass {
  using namespace std;
  using namespace Constants;

  Parser Parser::from_c_str(const char* str, Context& ctx, string path, Position source_position)
  {
    Parser p(ctx, path, source_position);
    p.source   = str;
    p.position = p.source;
    p.end      = str + strlen(str);
    return p;
  }

  Parser Parser::from_token(Token t, Context& ctx, string path, Position source_position)
  {
    Parser p(ctx, path, source_position);
    p.source   = t.begin;
    p.position = p.source;
    p.end      = t.end;
    return p;
  }

  Block* Parser::parse()
  {
    Block* root = new (ctx.mem) Block(path, source_position);
    root->is_root(true);
    read_bom();
    lex< optional_spaces >();
    Selector_Lookahead lookahead_result;
    while (position < end) {
      if (lex< block_comment >()) {
        String*  contents = parse_interpolated_chunk(lexed);
        Comment* comment  = new (ctx.mem) Comment(path, source_position, contents);
        (*root) << comment;
      }
      else if (peek< import >()) {
        Import* imp = parse_import();
        if (!imp->urls().empty()) (*root) << imp;
        if (!imp->files().empty()) {
          for (size_t i = 0, S = imp->files().size(); i < S; ++i) {
            (*root) << new (ctx.mem) Import_Stub(path, source_position, imp->files()[i]);
          }
        }
        if (!lex< exactly<';'> >()) error("top-level @import directive must be terminated by ';'");
      }
      else if (peek< mixin >() || peek< function >()) {
        (*root) << parse_definition();
      }
      else if (peek< variable >()) {
        (*root) << parse_assignment();
        if (!lex< exactly<';'> >()) error("top-level variable binding must be terminated by ';'");
      }
      else if (peek< sequence< optional< exactly<'*'> >, alternatives< identifier_schema, identifier >, optional_spaces, exactly<':'>, optional_spaces, exactly<'{'> > >(position)) {
        (*root) << parse_propset();
      }
      else if (peek< include >() /* || peek< exactly<'+'> >() */) {
        Mixin_Call* mixin_call = parse_mixin_call();
        (*root) << mixin_call;
        if (!mixin_call->block() && !lex< exactly<';'> >()) error("top-level @include directive must be terminated by ';'");
      }
      else if (peek< if_directive >()) {
        (*root) << parse_if_directive();
      }
      else if (peek< for_directive >()) {
        (*root) << parse_for_directive();
      }
      else if (peek< each_directive >()) {
        (*root) << parse_each_directive();
      }
      else if (peek< while_directive >()) {
        (*root) << parse_while_directive();
      }
      else if (peek< media >()) {
        (*root) << parse_media_block();
      }
      else if (peek< warn >()) {
        (*root) << parse_warning();
        if (!lex< exactly<';'> >()) error("top-level @warn directive must be terminated by ';'");
      }
      // ignore the @charset directive for now
      else if (lex< exactly< charset_kwd > >()) {
        lex< string_constant >();
        lex< exactly<';'> >();
      }
      else if (peek< at_keyword >()) {
        At_Rule* at_rule = parse_at_rule();
        (*root) << at_rule;
        if (!at_rule->block() && !lex< exactly<';'> >()) error("top-level directive must be terminated by ';'");
      }
      else if ((lookahead_result = lookahead_for_selector(position)).found) {
        (*root) << parse_ruleset(lookahead_result);
      }
      else {
        lex< spaces_and_comments >();
        if (position >= end) break;
        error("invalid top-level expression");
      }
      lex< optional_spaces >();
    }
    return root;
  }

  Import* Parser::parse_import()
  {
    lex< import >();
    Import* imp = new (ctx.mem) Import(path, source_position);
    bool first = true;
    do {
      if (lex< string_constant >()) {
        string import_path(lexed);
        string extension;
        if (import_path.length() > 6) { // 2 quote marks + the 4 chars in .css
          // a string constant is guaranteed to end with a quote mark, so make sure to skip it when indexing from the end
          extension = import_path.substr(import_path.length() - 5, 4);
        }
        if (extension == ".css") {
          String_Constant* loc = new (ctx.mem) String_Constant(path, source_position, import_path);
          Argument* loc_arg = new (ctx.mem) Argument(path, source_position, loc);
          Arguments* loc_args = new (ctx.mem) Arguments(path, source_position);
          (*loc_args) << loc_arg;
          Function_Call* new_url = new (ctx.mem) Function_Call(path, source_position, "url", loc_args);
{'fraction_non_alphanumeric': 0.1610064195485608, 'fraction_numerical': 0.0010354110581901014, 'mean_word_length': 2.6435307431158055, 'pattern_counts': {'":': 0, '<': 161, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5119228', 'n_tokens_mistral': 3130, 'n_tokens_neox': 3041, 'n_words': 788}