<commit_before>#include <cstdlib>
#include <iostream>
#include <vector>
#include "parser.hpp"
#include "file.hpp"
#include "inspect.hpp"
#include "to_string.hpp"
#include "constants.hpp"
#include "util.hpp"

#ifndef SASS_PRELEXER
#include "prelexer.hpp"
#endif

#include "sass_functions.h"

#include <typeinfo>

namespace Sass {
  using namespace std;
  using namespace Constants;

  Parser Parser::from_c_str(const char* str, Context& ctx, string path, Position source_position)
  {
    Parser p(ctx, path, source_position);
    p.source   = str;
    p.position = p.source;
    p.end      = str + strlen(str);
    return p;
  }

  Parser Parser::from_token(Token t, Context& ctx, string path, Position source_position)
  {
    Parser p(ctx, path, source_position);
    p.source   = t.begin;
    p.position = p.source;
    p.end      = t.end;
    return p;
  }

  Block* Parser::parse()
  {
    Block* root = new (ctx.mem) Block(path, source_position);
    root->is_root(true);
    read_bom();
    lex< optional_spaces >();
    Selector_Lookahead lookahead_result;
    while (position < end) {
      if (lex< block_comment >()) {
        String*  contents = parse_interpolated_chunk(lexed);
        Comment* comment  = new (ctx.mem) Comment(path, source_position, contents);
        (*root) << comment;
      }
      else if (peek< import >()) {
        Import* imp = parse_import();
        if (!imp->urls().empty()) (*root) << imp;
        if (!imp->files().empty()) {
          for (size_t i = 0, S = imp->files().size(); i < S; ++i) {
            (*root) << new (ctx.mem) Import_Stub(path, source_position, imp->files()[i]);
          }
        }
        if (!lex< exactly<';'> >()) error("top-level @import directive must be terminated by ';'");
      }
      else if (peek< mixin >() || peek< function >()) {
        (*root) << parse_definition();
      }
      else if (peek< variable >()) {
        (*root) << parse_assignment();
        if (!lex< exactly<';'> >()) error("top-level variable binding must be terminated by ';'");
      }
      else if (peek< sequence< optional< exactly<'*'> >, alternatives< identifier_schema, identifier >, optional_spaces, exactly<':'>, optional_spaces, exactly<'{'> > >(position)) {
        (*root) << parse_propset();
      }
      else if (peek< include >() /* || peek< exactly<'+'> >() */) {
        Mixin_Call* mixin_call = parse_mixin_call();
        (*root) << mixin_call;
        if (!mixin_call->block() && !lex< exactly<';'> >()) error("top-level @include directive must be terminated by ';'");
      }
      else if (peek< if_directive >()) {
        (*root) << parse_if_directive();
      }
      else if (peek< for_directive >()) {
        (*root) << parse_for_directive();
      }
      else if (peek< each_directive >()) {
        (*root) << parse_each_directive();
      }
      else if (peek< while_directive >()) {
        (*root) << parse_while_directive();
      }
      else if (peek< media >()) {
        (*root) << parse_media_block();
      }
      else if (peek< supports >()) {
        (*root) << parse_feature_block();
      }
      else if (peek< warn >()) {
        (*root) << parse_warning();
        if (!lex< exactly<';'> >()) error("top-level @warn directive must be terminated by ';'");
      }
      else if (peek< err >()) {
        (*root) << parse_error();
        if (!lex< exactly<';'> >()) error("top-level @error directive must be terminated by ';'");
      }
      else if (peek< dbg >()) {
        (*root) << parse_debug();
        if (!lex< exactly<';'> >()) error("top-level @debug directive must be terminated by ';'");
      }
      // ignore the @charset directive for now
      else if (lex< exactly< charset_kwd > >()) {
        lex< string_constant >();
        lex< exactly<';'> >();
      }
      else if (peek< at_keyword >()) {
        At_Rule* at_rule = parse_at_rule();
        (*root) << at_rule;
        if (!at_rule->block() && !lex< exactly<';'> >()) error("top-level directive must be terminated by ';'");
      }
      else if ((lookahead_result = lookahead_for_selector(position)).found) {
        (*root) << parse_ruleset(lookahead_result);
      }
      else {
        lex< spaces_and_comments >();
        if (position >= end) break;
        error("invalid top-level expression");
      }
      lex< optional_spaces >();
    }
    return root;
  }

  void Parser::add_single_file (Import* imp, string import_path) {

    string extension;
    string unquoted(unquote(import_path));
    if (unquoted.length() > 4) { // 2 quote marks + the 4 chars in .css
      // a string constant is guaranteed to end with a quote mark, so make sure to skip it when indexing from the end
      extension = unquoted.substr(unquoted.length() - 4, 4);
    }

    if (extension == ".css") {
      String_Constant* loc = new (ctx.mem) String_Constant(path, source_position, import_path, true);
      Argument* loc_arg = new (ctx.mem) Argument(path, source_position, loc);
      Arguments* loc_args = new (ctx.mem) Arguments(path, source_position);
      (*loc_args) << loc_arg;
      Function_Call* new_url = new (ctx.mem) Function_Call(path, source_position, "url", loc_args);
      imp->urls().push_back(new_url);
    }
    else {
      string current_dir = File::dir_name(path);
      string resolved(ctx.add_file(current_dir, unquoted));
      if (resolved.empty()) error("file to import not found or unreadable: " + unquoted + "\nCurrent dir: " + current_dir);
      imp->files().push_back(resolved);
    }

  }

  Import* Parser::parse_import()
  {
    lex< import >();
    Import* imp = new (ctx.mem) Import(path, source_position);
    bool first = true;
    do {
      if (lex< string_constant >()) {
        string import_path(lexed);

        // struct Sass_Options opt = sass_context_get_options(ctx)
        Sass_C_Import_Callback importer = ctx.importer;
        // custom importer
        if (importer) {
          Sass_Import* current = ctx.import_stack.back();
          Sass_C_Import_Fn fn = sass_import_get_function(importer);
          void* cookie = sass_import_get_cookie(importer);
          // create a new import entry
          string inc_path = unquote(import_path);
          struct Sass_Import** includes = fn(
            inc_path.c_str(),
            sass_import_get_path(current),
            cookie);
          if (includes) {
            struct Sass_Import** list = includes;
            while (*includes) {
              struct Sass_Import* include = *includes;
              const char *file = sass_import_get_path(include);
              char *source = sass_import_take_source(include);
              // char *srcmap = sass_import_take_srcmap(include);
              if (source) {
                if (file) {
                  ctx.add_source(file, inc_path, source);
                  imp->files().push_back(file);
                } else {
                  ctx.add_source(inc_path, inc_path, source);
                  imp->files().push_back(inc_path);
                }
              } else if(file) {
                add_single_file(imp, file);
              }
              ++includes;
            }
            // deallocate returned memory
            sass_delete_import_list(list);
            // parse next import
            continue;
          }
        }

        add_single_file(imp, import_path);

      }
      else if (peek< uri_prefix >()) {
        imp->urls().push_back(parse_value());
      }
      else {
        if (first) error("@import directive requires a url or quoted path");
        else error("expecting another url or quoted path in @import list");
      }
      first = false;
    } while (lex< exactly<','> >());
    return imp;
  }

  Definition* Parser::parse_definition()
  {
    Definition::Type which_type = Definition::MIXIN;
    if      (lex< mixin >())    which_type = Definition::MIXIN;
    else if (lex< function >()) which_type = Definition::FUNCTION;
    string which_str(lexed);
    if (!lex< identifier >()) error("invalid name in " + which_str + " definition");
    string name(Util::normalize_underscores(lexed));
    if (which_type == Definition::FUNCTION && (name == "and" || name == "or" || name == "not"))
    { error("Invalid function name \"" + name + "\"."); }
    Position source_position_of_def = source_position;
    Parameters* params = parse_parameters();
    if (!peek< exactly<'{'> >()) error("body for " + which_str + " " + name + " must begin with a '{'");
    if (which_type == Definition::MIXIN) stack.push_back(mixin_def);
    else stack.push_back(function_def);
    Block* body = parse_block();
    stack.pop_back();
    Definition* def = new (ctx.mem) Definition(path, source_position_of_def, name, params, body, which_type);
    return def;
  }

  Parameters* Parser::parse_parameters()
  {
    string name(lexed); // for the error message
    Parameters* params = new (ctx.mem) Parameters(path, source_position);
    if (lex< exactly<'('> >()) {
      // if there's anything there at all
      if (!peek< exactly<')'> >()) {
        do (*params) << parse_parameter();
        while (lex< exactly<','> >());
      }
      if (!lex< exactly<')'> >()) error("expected a variable name (e.g. $x) or ')' for the parameter list for " + name);
    }
    return params;
  }

  Parameter* Parser::parse_parameter()
  {
    lex< variable >();
    string name(Util::normalize_underscores(lexed));
    Position pos = source_position;
    Expression* val = 0;
    bool is_rest = false;
    if (lex< exactly<':'> >()) { // there's a default value
      val = parse_space_list();
      val->is_delayed(false);
    }
    else if (lex< exactly< ellipsis > >()) {
      is_rest = true;
    }
    Parameter* p = new (ctx.mem) Parameter(path, pos, name, val, is_rest);
    return p;
  }

  Mixin_Call* Parser::parse_mixin_call()
  {
    lex< include >() /* || lex< exactly<'+'> >() */;
    if (!lex< identifier >()) error("invalid name in @include directive");
    Position source_position_of_call = source_position;
    string name(Util::normalize_underscores(lexed));
    Arguments* args = parse_arguments();
    Block* content = 0;
    if (peek< exactly<'{'> >()) {
      content = parse_block();
    }
    Mixin_Call* the_call = new (ctx.mem) Mixin_Call(path, source_position_of_call, name, args, content);
    return the_call;
  }

  Arguments* Parser::parse_arguments()
  {
    string name(lexed);
    Arguments* args = new (ctx.mem) Arguments(path, source_position);

    if (lex< exactly<'('> >()) {
      // if there's anything there at all
      if (!peek< exactly<')'> >()) {
        do (*args) << parse_argument();
        while (lex< exactly<','> >());
      }
      if (!lex< exactly<')'> >()) error("expected a variable name (e.g. $x) or ')' for the parameter list for " + name);
    }

    return args;
  }

  Argument* Parser::parse_argument()
  {
    Argument* arg;
    if (peek< sequence < variable, spaces_and_comments, exactly<':'> > >()) {
      lex< variable >();
      string name(Util::normalize_underscores(lexed));
      Position p = source_position;
      lex< exactly<':'> >();
      Expression* val = parse_space_list();
      val->is_delayed(false);
      arg = new (ctx.mem) Argument(path, p, val, name);
    }
    else {
      bool is_arglist = false;
      bool is_keyword = false;
      Expression* val = parse_space_list();
      val->is_delayed(false);
      if (lex< exactly< ellipsis > >()) {
        if (val->concrete_type() == Expression::MAP) is_keyword = true;
        else is_arglist = true;
      }
      arg = new (ctx.mem) Argument(path, source_position, val, "", is_arglist, is_keyword);
    }
    return arg;
  }

  Assignment* Parser::parse_assignment()
  {
    lex< variable >();
    string name(Util::normalize_underscores(lexed));
    Position var_source_position = source_position;
    if (!lex< exactly<':'> >()) error("expected ':' after " + name + " in assignment statement");
    Expression* val = parse_list();
    val->is_delayed(false);
    bool is_guarded = lex< default_flag >() != NULL;
    bool is_global = lex< global_flag >() != NULL;
    Assignment* var = new (ctx.mem) Assignment(path, var_source_position, name, val, is_guarded, is_global);
    return var;
  }

  Propset* Parser::parse_propset()
  {
    String* property_segment;
    if (peek< sequence< optional< exactly<'*'> >, identifier_schema > >()) {
      property_segment = parse_identifier_schema();
    }
    else {
      lex< sequence< optional< exactly<'*'> >, identifier > >();
      property_segment = new (ctx.mem) String_Constant(path, source_position, lexed);
    }
    Propset* propset = new (ctx.mem) Propset(path, source_position, property_segment);
    lex< exactly<':'> >();

    if (!peek< exactly<'{'> >()) error("expected a '{' after namespaced property");

    propset->block(parse_block());

    return propset;<commit_msg>Correctly parse default and global flags in assignments<commit_after>#include <cstdlib>
#include <iostream>
#include <vector>
#include "parser.hpp"
#include "file.hpp"
#include "inspect.hpp"
#include "to_string.hpp"
#include "constants.hpp"
#include "util.hpp"

#ifndef SASS_PRELEXER
#include "prelexer.hpp"
#endif

#include "sass_functions.h"

#include <typeinfo>

namespace Sass {
  using namespace std;
  using namespace Constants;

  Parser Parser::from_c_str(const char* str, Context& ctx, string path, Position source_position)
  {
    Parser p(ctx, path, source_position);
    p.source   = str;
    p.position = p.source;
    p.end      = str + strlen(str);
    return p;
  }

  Parser Parser::from_token(Token t, Context& ctx, string path, Position source_position)
  {
    Parser p(ctx, path, source_position);
    p.source   = t.begin;
    p.position = p.source;
    p.end      = t.end;
    return p;
  }

  Block* Parser::parse()
  {
    Block* root = new (ctx.mem) Block(path, source_position);
    root->is_root(true);
    read_bom();
    lex< optional_spaces >();
    Selector_Lookahead lookahead_result;
    while (position < end) {
      if (lex< block_comment >()) {
        String*  contents = parse_interpolated_chunk(lexed);
        Comment* comment  = new (ctx.mem) Comment(path, source_position, contents);
        (*root) << comment;
      }
      else if (peek< import >()) {
        Import* imp = parse_import();
        if (!imp->urls().empty()) (*root) << imp;
        if (!imp->files().empty()) {
          for (size_t i = 0, S = imp->files().size(); i < S; ++i) {
            (*root) << new (ctx.mem) Import_Stub(path, source_position, imp->files()[i]);
          }
        }
        if (!lex< exactly<';'> >()) error("top-level @import directive must be terminated by ';'");
      }
      else if (peek< mixin >() || peek< function >()) {
        (*root) << parse_definition();
      }
      else if (peek< variable >()) {
        (*root) << parse_assignment();
        if (!lex< exactly<';'> >()) error("top-level variable binding must be terminated by ';'");
      }
      else if (peek< sequence< optional< exactly<'*'> >, alternatives< identifier_schema, identifier >, optional_spaces, exactly<':'>, optional_spaces, exactly<'{'> > >(position)) {
        (*root) << parse_propset();
      }
      else if (peek< include >() /* || peek< exactly<'+'> >() */) {
        Mixin_Call* mixin_call = parse_mixin_call();
        (*root) << mixin_call;
        if (!mixin_call->block() && !lex< exactly<';'> >()) error("top-level @include directive must be terminated by ';'");
      }
      else if (peek< if_directive >()) {
        (*root) << parse_if_directive();
      }
      else if (peek< for_directive >()) {
        (*root) << parse_for_directive();
      }
      else if (peek< each_directive >()) {
        (*root) << parse_each_directive();
      }
      else if (peek< while_directive >()) {
        (*root) << parse_while_directive();
      }
      else if (peek< media >()) {
        (*root) << parse_media_block();
      }
      else if (peek< supports >()) {
        (*root) << parse_feature_block();
      }
      else if (peek< warn >()) {
        (*root) << parse_warning();
        if (!lex< exactly<';'> >()) error("top-level @warn directive must be terminated by ';'");
      }
      else if (peek< err >()) {
        (*root) << parse_error();
        if (!lex< exactly<';'> >()) error("top-level @error directive must be terminated by ';'");
      }
      else if (peek< dbg >()) {
        (*root) << parse_debug();
        if (!lex< exactly<';'> >()) error("top-level @debug directive must be terminated by ';'");
      }
      // ignore the @charset directive for now
      else if (lex< exactly< charset_kwd > >()) {
        lex< string_constant >();
        lex< exactly<';'> >();
      }
      else if (peek< at_keyword >()) {
        At_Rule* at_rule = parse_at_rule();
        (*root) << at_rule;
        if (!at_rule->block() && !lex< exactly<';'> >()) error("top-level directive must be terminated by ';'");
      }
      else if ((lookahead_result = lookahead_for_selector(position)).found) {
        (*root) << parse_ruleset(lookahead_result);
      }
      else {
        lex< spaces_and_comments >();
        if (position >= end) break;
        error("invalid top-level expression");
      }
      lex< optional_spaces >();
    }
    return root;
  }

  void Parser::add_single_file (Import* imp, string import_path) {

    string extension;
    string unquoted(unquote(import_path));
    if (unquoted.length() > 4) { // 2 quote marks + the 4 chars in .css
      // a string constant is guaranteed to end with a quote mark, so make sure to skip it when indexing from the end
      extension = unquoted.substr(unquoted.length() - 4, 4);
    }

    if (extension == ".css") {
      String_Constant* loc = new (ctx.mem) String_Constant(path, source_position, import_path, true);
      Argument* loc_arg = new (ctx.mem) Argument(path, source_position, loc);
      Arguments* loc_args = new (ctx.mem) Arguments(path, source_position);
      (*loc_args) << loc_arg;
      Function_Call* new_url = new (ctx.mem) Function_Call(path, source_position, "url", loc_args);
      imp->urls().push_back(new_url);
    }
    else {
      string current_dir = File::dir_name(path);
      string resolved(ctx.add_file(current_dir, unquoted));
      if (resolved.empty()) error("file to import not found or unreadable: " + unquoted + "\nCurrent dir: " + current_dir);
      imp->files().push_back(resolved);
    }

  }

  Import* Parser::parse_import()
  {
    lex< import >();
    Import* imp = new (ctx.mem) Import(path, source_position);
    bool first = true;
    do {
      if (lex< string_constant >()) {
        string import_path(lexed);

        // struct Sass_Options opt = sass_context_get_options(ctx)
        Sass_C_Import_Callback importer = ctx.importer;
        // custom importer
        if (importer) {
          Sass_Import* current = ctx.import_stack.back();
          Sass_C_Import_Fn fn = sass_import_get_function(importer);
          void* cookie = sass_import_get_cookie(importer);
          // create a new import entry
          string inc_path = unquote(import_path);
          struct Sass_Import** includes = fn(
            inc_path.c_str(),
            sass_import_get_path(current),
            cookie);
          if (includes) {
            struct Sass_Import** list = includes;
            while (*includes) {
              struct Sass_Import* include = *includes;
              const char *file = sass_import_get_path(include);
              char *source = sass_import_take_source(include);
              // char *srcmap = sass_import_take_srcmap(include);
              if (source) {
                if (file) {
                  ctx.add_source(file, inc_path, source);
                  imp->files().push_back(file);
                } else {
                  ctx.add_source(inc_path, inc_path, source);
                  imp->files().push_back(inc_path);
                }
              } else if(file) {
                add_single_file(imp, file);
              }
              ++includes;
            }
            // deallocate returned memory
            sass_delete_import_list(list);
            // parse next import
            continue;
          }
        }

        add_single_file(imp, import_path);

      }
      else if (peek< uri_prefix >()) {
        imp->urls().push_back(parse_value());
      }
      else {
        if (first) error("@import directive requires a url or quoted path");
        else error("expecting another url or quoted path in @import list");
      }
      first = false;
    } while (lex< exactly<','> >());
    return imp;
  }

  Definition* Parser::parse_definition()
  {
    Definition::Type which_type = Definition::MIXIN;
    if      (lex< mixin >())    which_type = Definition::MIXIN;
    else if (lex< function >()) which_type = Definition::FUNCTION;
    string which_str(lexed);
    if (!lex< identifier >()) error("invalid name in " + which_str + " definition");
    string name(Util::normalize_underscores(lexed));
    if (which_type == Definition::FUNCTION && (name == "and" || name == "or" || name == "not"))
    { error("Invalid function name \"" + name + "\"."); }
    Position source_position_of_def = source_position;
    Parameters* params = parse_parameters();
    if (!peek< exactly<'{'> >()) error("body for " + which_str + " " + name + " must begin with a '{'");
    if (which_type == Definition::MIXIN) stack.push_back(mixin_def);
    else stack.push_back(function_def);
    Block* body = parse_block();
    stack.pop_back();
    Definition* def = new (ctx.mem) Definition(path, source_position_of_def, name, params, body, which_type);
    return def;
  }

  Parameters* Parser::parse_parameters()
  {
    string name(lexed); // for the error message
    Parameters* params = new (ctx.mem) Parameters(path, source_position);
    if (lex< exactly<'('> >()) {
      // if there's anything there at all
      if (!peek< exactly<')'> >()) {
        do (*params) << parse_parameter();
        while (lex< exactly<','> >());
      }
      if (!lex< exactly<')'> >()) error("expected a variable name (e.g. $x) or ')' for the parameter list for " + name);
    }
    return params;
  }

  Parameter* Parser::parse_parameter()
  {
    lex< variable >();
    string name(Util::normalize_underscores(lexed));
    Position pos = source_position;
    Expression* val = 0;
    bool is_rest = false;
    if (lex< exactly<':'> >()) { // there's a default value
      val = parse_space_list();
      val->is_delayed(false);
    }
    else if (lex< exactly< ellipsis > >()) {
      is_rest = true;
    }
    Parameter* p = new (ctx.mem) Parameter(path, pos, name, val, is_rest);
    return p;
  }

  Mixin_Call* Parser::parse_mixin_call()
  {
    lex< include >() /* || lex< exactly<'+'> >() */;
    if (!lex< identifier >()) error("invalid name in @include directive");
    Position source_position_of_call = source_position;
    string name(Util::normalize_underscores(lexed));
    Arguments* args = parse_arguments();
    Block* content = 0;
    if (peek< exactly<'{'> >()) {
      content = parse_block();
    }
    Mixin_Call* the_call = new (ctx.mem) Mixin_Call(path, source_position_of_call, name, args, content);
    return the_call;
  }

  Arguments* Parser::parse_arguments()
  {
    string name(lexed);
    Arguments* args = new (ctx.mem) Arguments(path, source_position);

    if (lex< exactly<'('> >()) {
      // if there's anything there at all
      if (!peek< exactly<')'> >()) {
        do (*args) << parse_argument();
        while (lex< exactly<','> >());
      }
      if (!lex< exactly<')'> >()) error("expected a variable name (e.g. $x) or ')' for the parameter list for " + name);
    }

    return args;
  }

  Argument* Parser::parse_argument()
  {
    Argument* arg;
    if (peek< sequence < variable, spaces_and_comments, exactly<':'> > >()) {
      lex< variable >();
      string name(Util::normalize_underscores(lexed));
      Position p = source_position;
      lex< exactly<':'> >();
      Expression* val = parse_space_list();
      val->is_delayed(false);
      arg = new (ctx.mem) Argument(path, p, val, name);
    }
    else {
      bool is_arglist = false;
      bool is_keyword = false;
      Expression* val = parse_space_list();
      val->is_delayed(false);
      if (lex< exactly< ellipsis > >()) {
        if (val->concrete_type() == Expression::MAP) is_keyword = true;
        else is_arglist = true;
      }
      arg = new (ctx.mem) Argument(path, source_position, val, "", is_arglist, is_keyword);
    }
    return arg;
  }

  Assignment* Parser::parse_assignment()
  {
    lex< variable >();
    string name(Util::normalize_underscores(lexed));
    Position var_source_position = source_position;
    if (!lex< exactly<':'> >()) error("expected ':' after " + name + " in assignment statement");
    Expression* val = parse_list();
    val->is_delayed(false);
    bool is_guarded = false;
    bool is_global = false;
    while (peek< default_flag >() || peek< global_flag >()) {
      is_guarded = lex< default_flag >() || is_guarded;
      is_global = lex< global_flag >() || is_global;
    }
    Assignment* var = new (ctx.mem) Assignment(path, var_source_position, name, val, is_guarded, is_global);
    return var;
  }

  Propset* Parser::parse_propset()
  {
    String* property_segment;
    if (peek< sequence< optional< exactly<'*'> >, identifier_schema > >()) {
      property_segment = parse_identifier_schema();
    }
    else {
      lex< sequence< optional< exactly<'*'> >, identifier > >();
      property_segment = new (ctx.mem) String_Constant(path, source_position, lexed);
    }
    Propset* propset = new (ctx.mem) Propset(path, source_position, property_segment);
    lex< exactly<':'> >();

    if (!peek< exactly<'{'> >()) error("expected a '{' after namespaced property");

    propset->block(parse_block());

    return propset;
{'fraction_non_alphanumeric': 0.1542684812867306, 'fraction_numerical': 0.0006186204763377668, 'mean_word_length': 2.6918355695118468, 'pattern_counts': {'":': 0, '<': 319, '<?xml version=': 0, '>': 271, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6757334', 'n_tokens_mistral': 8257, 'n_tokens_neox': 8045, 'n_words': 2015}