<commit_before>CodeMirror.defineMode("pyret", function(config, parserConfig) {
  var ERRORCLASS = 'error';
  function wordRegexp(words) {
    return new RegExp("^((" + words.join(")|(") + "))(?![a-zA-Z0-9-_])");
  }
  
  const pyret_indent_regex = new RegExp("^[a-zA-Z_][a-zA-Z0-9$_\\-]*");
  const pyret_keywords = 
    wordRegexp(["fun", "lam", "method", "var", "when", "import", "provide", "type", "newtype",
                "data", "end", "for", "from", "lazy",
                "shadow", "ref",
                "and", "or", "as", "if", "else", "cases", "is", "satisfies", "raises",
                "raises-satisfies", "violates", "raises-violates",
                "does-not-raise", "raises-other-than",
                "check", "examples"]);
  const pyret_keywords_hyphen =
    wordRegexp(["provide-types", "type-let"]);
  const pyret_keywords_colon = 
    wordRegexp(["doc", "try", "ask", "otherwise", "then", "with", "sharing", "where", "ref-graph", "block"]);
  const pyret_single_punctuation = 
    new RegExp("^([" + [":", ".", "<", ">", ",", "^", "!",
                        ";", "|", "=", "+", "*", "/", "\\\\", // NOTE: No minus
                        "(", ")", "{", "}", "\\[", "\\]"].join('') + "])");
  const pyret_double_punctuation = 
    new RegExp("^((" + ["<=>", "::", "=~", "==", ">=", "<=", "=>", "->", ":=", "<>"].join(")|(") + "))");
  const initial_operators = { "-": true, "+": true, "*": true, "/": true, "<": true, "<=": true,
                              ">": true, ">=": true, "==": true, "<>": true, ".": true, "^": true,
                              "<=>": true, "=~": true,
                              "is": true, "raises": true, "satisfies": true }
  <commit_msg>rec keyword<commit_after>CodeMirror.defineMode("pyret", function(config, parserConfig) {
  var ERRORCLASS = 'error';
  function wordRegexp(words) {
    return new RegExp("^((" + words.join(")|(") + "))(?![a-zA-Z0-9-_])");
  }
  
  const pyret_indent_regex = new RegExp("^[a-zA-Z_][a-zA-Z0-9$_\\-]*");
  const pyret_keywords = 
    wordRegexp(["fun", "lam", "method", "var", "rec", "when", "import", "provide", "type", "newtype",
                "data", "end", "for", "from", "lazy",
                "shadow", "ref",
                "and", "or", "as", "if", "else", "cases", "is", "satisfies", "raises",
                "raises-satisfies", "violates", "raises-violates",
                "does-not-raise", "raises-other-than",
                "check", "examples"]);
  const pyret_keywords_hyphen =
    wordRegexp(["provide-types", "type-let"]);
  const pyret_keywords_colon = 
    wordRegexp(["doc", "try", "ask", "otherwise", "then", "with", "sharing", "where", "ref-graph", "block"]);
  const pyret_single_punctuation = 
    new RegExp("^([" + [":", ".", "<", ">", ",", "^", "!",
                        ";", "|", "=", "+", "*", "/", "\\\\", // NOTE: No minus
                        "(", ")", "{", "}", "\\[", "\\]"].join('') + "])");
  const pyret_double_punctuation = 
    new RegExp("^((" + ["<=>", "::", "=~", "==", ">=", "<=", "=>", "->", ":=", "<>"].join(")|(") + "))");
  const initial_operators = { "-": true, "+": true, "*": true, "/": true, "<": true, "<=": true,
                              ">": true, ">=": true, "==": true, "<>": true, ".": true, "^": true,
                              "<=>": true, "=~": true,
                              "is": true, "raises": true, "satisfies": true }
  
{'fraction_non_alphanumeric': 0.2973923234690888, 'fraction_numerical': 0.0023439789041898623, 'mean_word_length': 2.4450050454086782, 'pattern_counts': {'":': 40, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2181931', 'n_tokens_mistral': 1265, 'n_tokens_neox': 1186, 'n_words': 206}