<commit_before>CodeMirror.defineMode("pyret", function(config, parserConfig) {
  var ERRORCLASS = 'error';
  function wordRegexp(words) {
    return new RegExp("^((" + words.join(")|(") + "))(?![a-zA-Z0-9-_])");
  }
  
  const pyret_indent_regex = new RegExp("^[a-zA-Z_][a-zA-Z0-9$_\\-]*");
  const pyret_keywords = 
    wordRegexp(["fun", "method", "var", "when", "import", "provide", 
                "data", "end", "except", "for", "from", 
                "and", "or", "not", "as", "if", "else", "cases"]);
  const pyret_keywords_colon = 
    wordRegexp(["doc", "try", "with", "sharing", "where", "check"]);
  const pyret_single_punctuation = 
    new RegExp("^([" + ["\\:", "\\.", "<", ">", ",", "^", 
                        ";", "|", "=", "+", "*", "/", "\\", // NOTE: No minus
                        "\\(", "\\)", "{", "}", "\\[", "\\]"].join('') + "])");
  const pyret_double_punctuation = 
    new RegExp("^((" + ["::", "==", ">=", "<=", "=>", "->", ":=", "<>"].join(")|(") + "))");
  const initial_operators = { "-": true, "+": true, "*": true, "/": true, "<": true, "<=": true,
                              ">": true, ">=": true, "==": true, "<>": true, ".": true, "^": true }
  
  
  function ret(state, tokType, content, style) {
    state.lastToken = tokType; state.lastContent = content;
    return style;
  }

  function tokenBase(stream, state) { 
    if (stream.eatSpace())
      return "IGNORED-SPACE";

    var ch = stream.peek();
    

    // Handle Comments
    if (ch === '#') {
      stream.skipToEnd();
      return ret(state, state.lastToken, state.lastContent, 'comment');
    }
    // Handle Number Literals
    if (stream.match(/^[0-9]+(\.[0-9]+)?/))
      return ret(state, 'number', stream.current(), 'number');
    
    if (ch === '"') {
      state.tokenizer = tokenString;
      state.lastToken = '"';
      stream.eat('"');
      return state.tokenizer(stream, state);
    }
    // Level 1
    var match;
    if ((match = stream.match(pyret_double_punctuation, true)) || 
        (match = stream.match(pyret_single_punctuation, true))) {
      return ret(state, match[0], match[0], 'builtin');
    }
    if (match = stream.match(pyret_keywords, true)) {
      return ret(state, match[0], match[0], 'keyword');
    }
    if (match = stream.match(pyret_keywords_colon, true)) {
      if (stream.peek() === ":")
        return ret(state, match[0], match[0], 'keyword');
      else
        return ret(state, 'name', match[0], 'variable');
    }
    // Level 2
    if (match = stream.match(pyret_indent_regex)) {
      if (state.lastToken === "|") {
        if (stream.match(/\s*\(/, false))
          return ret(state, 'name', match[0],'function-name');
        else if (stream.match(/\s*=>/, false))
          return ret(state, 'name', match[0], 'variable');
        else if (stream.match(/\s*($|with\b|\()/, false))
          return ret(state, 'name', match[0], 'type');
      } else if (state.lastToken === "::")
        return ret(state, 'name', match[0], 'type');
      else if (state.lastToken === "data")
        return ret(state, 'name', match[0], 'type');
      else if (stream.match(/\s*\(/, false))
        return ret(state, 'name', match[0], 'function-name');
      return ret(state, 'name', match[0], 'variable');
    }
    if (stream.eat("-"))
      return ret(state, '-', '-', 'builtin');
    stream.next();
    return null;
  }
  function tokenString(stream, state) { 
    while (!stream.eol()) {
      stream.eatWhile(/[^"\\]/);
      if (stream.eat('\\')) {
        stream.next();
        if (stream.eol())
          return ret(state, 'string', stream.current(), 'string');
      } else if (stream.eat('"')) {
        state.tokenizer = tokenBase;
        return ret(state, 'string', stream.current(), 'string');
      } else
        stream.eat(/"/);
    }
    return ret(state, 'string', stream.current(), 'string');
  }

  // Parsing

  function Indent(funs, cases, data, shared, trys, except, parens, objects, vars, fields, initial) {
    this.fn = funs || 0;
    this.c = cases || 0;
    this.d = data || 0;
    this.s = shared || 0;
    this.t = trys || 0;
    this.e = except || 0;
    this.p = parens || 0;
    this.o = objects || 0;
    this.v = vars || 0;
    this.f = fields || 0;
    this.i = initial || 0;
  }
  Indent.prototype.toString = function() {
    return ("Fun " + this.fn + ", Cases " + this.c + ", Data " + this.d + ", Shared " + this.s
            + ", Try " + this.t + ", Except " + this.e + ", Parens " + this.p 
            + ", Object " + this.o + ", Vars " + this.v + ", Fields " + this.f + ", Initial " + this.i);
  }
  Indent.prototype.copy = function() {
    return new Indent(this.fn, this.c, this.d, this.s, this.t, this.e, this.p, this.o, this.v, this.f, this.i);
  }
  Indent.prototype.zeroOut = function() {
    this.fn = this.c = this.d = this.s = this.t = this.e = this.p = this.o = this.v = this.f = this.i = 0;<commit_msg>single-quote strings<commit_after>CodeMirror.defineMode("pyret", function(config, parserConfig) {
  var ERRORCLASS = 'error';
  function wordRegexp(words) {
    return new RegExp("^((" + words.join(")|(") + "))(?![a-zA-Z0-9-_])");
  }
  
  const pyret_indent_regex = new RegExp("^[a-zA-Z_][a-zA-Z0-9$_\\-]*");
  const pyret_keywords = 
    wordRegexp(["fun", "method", "var", "when", "import", "provide", 
                "data", "end", "except", "for", "from", 
                "and", "or", "not", "as", "if", "else", "cases"]);
  const pyret_keywords_colon = 
    wordRegexp(["doc", "try", "with", "sharing", "where", "check"]);
  const pyret_single_punctuation = 
    new RegExp("^([" + ["\\:", "\\.", "<", ">", ",", "^", 
                        ";", "|", "=", "+", "*", "/", "\\", // NOTE: No minus
                        "\\(", "\\)", "{", "}", "\\[", "\\]"].join('') + "])");
  const pyret_double_punctuation = 
    new RegExp("^((" + ["::", "==", ">=", "<=", "=>", "->", ":=", "<>"].join(")|(") + "))");
  const initial_operators = { "-": true, "+": true, "*": true, "/": true, "<": true, "<=": true,
                              ">": true, ">=": true, "==": true, "<>": true, ".": true, "^": true }
  
  
  function ret(state, tokType, content, style) {
    state.lastToken = tokType; state.lastContent = content;
    return style;
  }

  function tokenBase(stream, state) { 
    if (stream.eatSpace())
      return "IGNORED-SPACE";

    var ch = stream.peek();
    

    // Handle Comments
    if (ch === '#') {
      stream.skipToEnd();
      return ret(state, state.lastToken, state.lastContent, 'comment');
    }
    // Handle Number Literals
    if (stream.match(/^[0-9]+(\.[0-9]+)?/))
      return ret(state, 'number', stream.current(), 'number');
    
    if (ch === '"') {
      state.tokenizer = tokenStringDouble;
      state.lastToken = '"';
      stream.eat('"');
      return state.tokenizer(stream, state);
    }
    if (ch === "'") {
      state.tokenizer = tokenStringSingle;
      state.lastToken = "'";
      stream.eat("'");
      return state.tokenizer(stream, state);
    }
    // Level 1
    var match;
    if ((match = stream.match(pyret_double_punctuation, true)) || 
        (match = stream.match(pyret_single_punctuation, true))) {
      return ret(state, match[0], match[0], 'builtin');
    }
    if (match = stream.match(pyret_keywords, true)) {
      return ret(state, match[0], match[0], 'keyword');
    }
    if (match = stream.match(pyret_keywords_colon, true)) {
      if (stream.peek() === ":")
        return ret(state, match[0], match[0], 'keyword');
      else
        return ret(state, 'name', match[0], 'variable');
    }
    // Level 2
    if (match = stream.match(pyret_indent_regex)) {
      if (state.lastToken === "|") {
        if (stream.match(/\s*\(/, false))
          return ret(state, 'name', match[0],'function-name');
        else if (stream.match(/\s*=>/, false))
          return ret(state, 'name', match[0], 'variable');
        else if (stream.match(/\s*($|with\b|\()/, false))
          return ret(state, 'name', match[0], 'type');
      } else if (state.lastToken === "::")
        return ret(state, 'name', match[0], 'type');
      else if (state.lastToken === "data")
        return ret(state, 'name', match[0], 'type');
      else if (stream.match(/\s*\(/, false))
        return ret(state, 'name', match[0], 'function-name');
      return ret(state, 'name', match[0], 'variable');
    }
    if (stream.eat("-"))
      return ret(state, '-', '-', 'builtin');
    stream.next();
    return null;
  }
  function mkTokenString(singleOrDouble) {
    return function(stream, state) {
      var insideRE = singleOrDouble === "'" ? /[^'\\]/ : /[^"\\]/;
      var endRE = singleOrDouble === "'" ? /'/ : /"/;
      while (!stream.eol()) {
        stream.eatWhile(insideRE);
        if (stream.eat('\\')) {
          stream.next();
          if (stream.eol())
            return ret(state, 'string', stream.current(), 'string');
        } else if (stream.eat(singleOrDouble)) {
          state.tokenizer = tokenBase;
          return ret(state, 'string', stream.current(), 'string');
        } else
          stream.eat(endRE);
      }
      return ret(state, 'string', stream.current(), 'string');
    };
  }

  var tokenStringDouble = mkTokenString('"');
  var tokenStringSingle = mkTokenString("'");

  // Parsing

  function Indent(funs, cases, data, shared, trys, except, parens, objects, vars, fields, initial) {
    this.fn = funs || 0;
    this.c = cases || 0;
    this.d = data || 0;
    this.s = shared || 0;
    this.t = trys || 0;
    this.e = except || 0;
    this.p = parens || 0;
    this.o = objects || 0;
    this.v = vars || 0;
    this.f = fields || 0;
    this.i = initial || 0;
  }
  Indent.prototype.toString = function() {
    return ("Fun " + this.fn + ", Cases " + this.c + ", Data " + this.d + ", Shared " + this.s
            + ", Try " + this.t + ", Except " + this.e + ", Parens " + this.p 
            + ", Object " + this.o + ", Vars " + this.v + ", Fields " + this.f + ", Initial " + this.i);
  }
  Indent.prototype.copy = function() {
    return new Indent(this.fn, this.c, this.d, this.s, this.t, this.e, this.p, this.o, this.v, this.f, this.i);
  }
  Indent.prototype.zeroOut = function() {
    this.fn = this.c = this.d = this.s = this.t = this.e = this.p = this.o = this.v = this.f = this.i = 0;
{'fraction_non_alphanumeric': 0.22865735351183547, 'fraction_numerical': 0.006984866123399301, 'mean_word_length': 2.439773106439773, 'pattern_counts': {'":': 32, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2598666', 'n_tokens_mistral': 3652, 'n_tokens_neox': 3511, 'n_words': 759}