<commit_before>(function(mod) {
  if (typeof exports == "object" && typeof module == "object") // CommonJS
    mod(require("../../lib/codemirror"));
  else if (typeof define == "function" && define.amd) // AMD
    define(["../../lib/codemirror"], mod);
  else // Plain browser env
    mod(CodeMirror);
})(function(CodeMirror) {
  "use strict";

  /*
   * Logic for Keyword/Bracket Matching
   */

  /* =========== WARNING ============
   * CodeMirror mangles 0 and 1-indexed
   * values to a certain extent. CodeMirror.Pos
   * (aliased as just "Pos") expects the
   * character index to be 1-indexed, while
   * token objects returned by methods such as
   * cm.getTokenAt have `start` and `end` character
   * indices which are 0-indexed. As such,
   * some things in this file might look
   * a little weird, but bear the above
   * in mind before tweaking.
   */

  var Pos = CodeMirror.Pos;

  /**
   * Returns the difference between two {@link CodeMirror.Pos} objects
   */
  function cmp(a, b) { return a.line - b.line || a.ch - b.ch; }

  /**
   * Like cmp(...), but accepts two positions representing
   * a range and returns the distance to the closest point
   * of [a, b]. (If a <= c <= b, returns 0).
   */
  function cmpClosest(a, b, c) {
    var ret = a.line - c.line
    if (ret) { return ret; }
    if (a.ch <= c.ch && c.ch <= b.ch) { return 0; }
    if (Math.abs(a.ch - c.ch) < Math.abs(b.ch - c.ch)) { return a.ch - c.ch; }
    return b.ch - c.ch;
  }

  var pyretMode = CodeMirror.getMode({},"pyret");
  if (pyretMode.name === "null") {
    throw Error("Pyret Mode not Defined");
  } else if (!pyretMode.delimiters || // Make sure delimiters exist
             !pyretMode.delimiters.opening ||      // and are valid
             !pyretMode.delimiters.closing) {
    throw Error("No correct delimiters defined in Pyret Mode");
  }
  // Opening Delimiter Tokens
  var DELIMS = pyretMode.delimiters.opening;
  // Closing Delimiter Tokents
  var ENDDELIM = pyretMode.delimiters.closing;
  // Tokens with closing tokens other than "end"
  var SPECIALDELIM = [{start: "(", end: ")"},
                      {start: "[", end: "]"},
                      {start: "{", end: "}"}];
  // Matches against any token text
  var delimrx = new RegExp("(" + DELIMS.join("|") + "|" +
                           ENDDELIM.join("|") + "|\\(|\\)|\\[|\\]|{|})", "g");

  // Encapsulates parent->sub-keyword relationship
  var SIMPLESUBKEYWORDS = {
    "if": ["else if", "else"], "fun": ["where"],
    "data": ["sharing", "where"], "method": ["where"]
  };

  // Represents subkeywords which cannot be followed
  // by any other keywords
  var LASTSUBKEYWORDS = {
    "if": "else"
  };

  // Like SIMPLESUBKEYWORDS, but goes from sub-keyword->parent
  var INV_SIMPLESUBKEYWORDS = {};
  Object.keys(SIMPLESUBKEYWORDS).forEach(function(key){
    var arr = SIMPLESUBKEYWORDS[key];
    arr.forEach(function(skw) {
      INV_SIMPLESUBKEYWORDS[skw] = INV_SIMPLESUBKEYWORDS[skw] || [];
      INV_SIMPLESUBKEYWORDS[skw].push(key);
    });
  });

  // Inverse mapping from LASTSUBKEYWORDS
  var INV_LASTSUBKEYWORDS = {};
  Object.keys(LASTSUBKEYWORDS).forEach(function(key){
    var kw = LASTSUBKEYWORDS[key];
    // Needs to be an array since mapping is (potentially) non-bijective
    INV_LASTSUBKEYWORDS[kw] = INV_LASTSUBKEYWORDS[kw] || [];
    INV_LASTSUBKEYWORDS[kw].push(key);
  });

  // Utility Dictionary which is similar to INV_LASTSUBKEYWORDS,
  // except that each subkeyword `s` maps to an array `a` of *subkeywords*.
  // If any subkeyword `t` in `a` follows `s` at the same nesting level
  // (in the same context, of course), then `t` cannot match to the parent
  // keyword of `s` (i.e. `t` is implicitly said to be one nesting level
  // higher).
  // P.S. Good thing we're only dealing with tiny lists
  var ILLEGAL_LATER_SIBLINGS = {};
  Object.keys(INV_LASTSUBKEYWORDS).forEach(function(key){
    ILLEGAL_LATER_SIBLINGS[key] = [];
    var parents = INV_SIMPLESUBKEYWORDS[key];
    if (!parents) throw new Error("Entries in LASTSUBKEYWORDS must also be in SIMPLESUBKEYWORDS");
    parents.forEach(function(parent){
      // NB: children == children of *parent* (so *siblings* of key)
      var children = SIMPLESUBKEYWORDS[parent];
      children.forEach(function(sibling){
        ILLEGAL_LATER_SIBLINGS[key].push(sibling);
      });
    });
  });

  /**
   * Checks the given text for whether it is an opening keyword
   * (Done textually...assumption is that the text originates from
   * a keyword or builtin token type)
   * @param {string} text - The text to check
   * @returns {boolean} Whether the given text is an opening delimiter
   */
  function isOpening(text) {
    text = (typeof(text) === 'string') ? text : text.string;
    if (DELIMS.indexOf(text) != -1) {
      return true;
    }
    for (var i = 0; i < SPECIALDELIM.length; i++) {
      if (text === SPECIALDELIM[i].start) return true;
    }
    return false;
  }

  /**
   * Checks the given text for whether it is a closing keyword
   * (Done textually...assumption is that the text originates from
   * a keyword or builtin token type)
   * @param {string} text - The text to check
   * @returns {boolean} Whether the given text is a closing delimiter
   */
  function isClosing(text) {
    text = (typeof(text) === 'string') ? text : text.string;
    if (ENDDELIM.indexOf(text) != -1) {<commit_msg>Forgot to remove this behemoth<commit_after>(function(mod) {
  if (typeof exports == "object" && typeof module == "object") // CommonJS
    mod(require("../../lib/codemirror"));
  else if (typeof define == "function" && define.amd) // AMD
    define(["../../lib/codemirror"], mod);
  else // Plain browser env
    mod(CodeMirror);
})(function(CodeMirror) {
  "use strict";

  /*
   * Logic for Keyword/Bracket Matching
   */

  /* =========== WARNING ============
   * CodeMirror mangles 0 and 1-indexed
   * values to a certain extent. CodeMirror.Pos
   * (aliased as just "Pos") expects the
   * character index to be 1-indexed, while
   * token objects returned by methods such as
   * cm.getTokenAt have `start` and `end` character
   * indices which are 0-indexed. As such,
   * some things in this file might look
   * a little weird, but bear the above
   * in mind before tweaking.
   */

  var Pos = CodeMirror.Pos;

  /**
   * Returns the difference between two {@link CodeMirror.Pos} objects
   */
  function cmp(a, b) { return a.line - b.line || a.ch - b.ch; }

  /**
   * Like cmp(...), but accepts two positions representing
   * a range and returns the distance to the closest point
   * of [a, b]. (If a <= c <= b, returns 0).
   */
  function cmpClosest(a, b, c) {
    var ret = a.line - c.line
    if (ret) { return ret; }
    if (a.ch <= c.ch && c.ch <= b.ch) { return 0; }
    if (Math.abs(a.ch - c.ch) < Math.abs(b.ch - c.ch)) { return a.ch - c.ch; }
    return b.ch - c.ch;
  }

  var pyretMode = CodeMirror.getMode({},"pyret");
  if (pyretMode.name === "null") {
    throw Error("Pyret Mode not Defined");
  } else if (!pyretMode.delimiters || // Make sure delimiters exist
             !pyretMode.delimiters.opening ||      // and are valid
             !pyretMode.delimiters.closing) {
    throw Error("No correct delimiters defined in Pyret Mode");
  }
  // Opening Delimiter Tokens
  var DELIMS = pyretMode.delimiters.opening;
  // Closing Delimiter Tokents
  var ENDDELIM = pyretMode.delimiters.closing;
  // Tokens with closing tokens other than "end"
  var SPECIALDELIM = [{start: "(", end: ")"},
                      {start: "[", end: "]"},
                      {start: "{", end: "}"}];
  // Matches against any token text
  var delimrx = new RegExp("(" + DELIMS.join("|") + "|" +
                           ENDDELIM.join("|") + "|\\(|\\)|\\[|\\]|{|})", "g");

  // Encapsulates parent->sub-keyword relationship
  var SIMPLESUBKEYWORDS = {
    "if": ["else if", "else"], "fun": ["where"],
    "data": ["sharing", "where"], "method": ["where"]
  };

  // Represents subkeywords which cannot be followed
  // by any other keywords
  var LASTSUBKEYWORDS = {
    "if": "else"
  };

  // Like SIMPLESUBKEYWORDS, but goes from sub-keyword->parent
  var INV_SIMPLESUBKEYWORDS = {};
  Object.keys(SIMPLESUBKEYWORDS).forEach(function(key){
    var arr = SIMPLESUBKEYWORDS[key];
    arr.forEach(function(skw) {
      INV_SIMPLESUBKEYWORDS[skw] = INV_SIMPLESUBKEYWORDS[skw] || [];
      INV_SIMPLESUBKEYWORDS[skw].push(key);
    });
  });

  // Inverse mapping from LASTSUBKEYWORDS
  var INV_LASTSUBKEYWORDS = {};
  Object.keys(LASTSUBKEYWORDS).forEach(function(key){
    var kw = LASTSUBKEYWORDS[key];
    // Needs to be an array since mapping is (potentially) non-bijective
    INV_LASTSUBKEYWORDS[kw] = INV_LASTSUBKEYWORDS[kw] || [];
    INV_LASTSUBKEYWORDS[kw].push(key);
  });

  /**
   * Checks the given text for whether it is an opening keyword
   * (Done textually...assumption is that the text originates from
   * a keyword or builtin token type)
   * @param {string} text - The text to check
   * @returns {boolean} Whether the given text is an opening delimiter
   */
  function isOpening(text) {
    text = (typeof(text) === 'string') ? text : text.string;
    if (DELIMS.indexOf(text) != -1) {
      return true;
    }
    for (var i = 0; i < SPECIALDELIM.length; i++) {
      if (text === SPECIALDELIM[i].start) return true;
    }
    return false;
  }

  /**
   * Checks the given text for whether it is a closing keyword
   * (Done textually...assumption is that the text originates from
   * a keyword or builtin token type)
   * @param {string} text - The text to check
   * @returns {boolean} Whether the given text is a closing delimiter
   */
  function isClosing(text) {
    text = (typeof(text) === 'string') ? text : text.string;
    if (ENDDELIM.indexOf(text) != -1) {
{'fraction_non_alphanumeric': 0.14943350005103603, 'fraction_numerical': 0.0018372971317750333, 'mean_word_length': 2.846878680800942, 'pattern_counts': {'":': 10, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4003931', 'n_tokens_mistral': 3365, 'n_tokens_neox': 3238, 'n_words': 961}