<commit_before>CodeMirror.multiplexingMode = function(outer /*, others */) {
  // Others should be {open, close, mode [, delimStyle]} objects
  var others = Array.prototype.slice.call(arguments, 1);
  var n_others = others.length;

  function indexOf(string, pattern, from) {
    if (typeof pattern == "string") return string.indexOf(pattern, from);
    var m = pattern.exec(from ? string.slice(from) : string);
    return m ? m.index + from : -1;
  }

  return {
    startState: function() {
      return {
        outer: CodeMirror.startState(outer),
        innerActive: null,
        inner: null
      };
    },

    copyState: function(state) {
      return {
        outer: CodeMirror.copyState(outer, state.outer),
        innerActive: state.innerActive,
        inner: state.innerActive && CodeMirror.copyState(state.innerActive.mode, state.inner)
      };
    },

    token: function(stream, state) {
      if (!state.innerActive) {
        var cutOff = Infinity, oldContent = stream.string;
        for (var i = 0; i < n_others; ++i) {
          var other = others[i];
          var found = indexOf(oldContent, other.open, stream.pos);
          if (found == stream.pos) {
            stream.match(other.open);
            state.innerActive = other;
            state.inner = CodeMirror.startState(other.mode, outer.indent ? outer.indent(state.outer, "") : 0);
            return other.delimStyle;
          } else if (found != -1 && found < cutOff) {
            cutOff = found;
          }
        }
        if (cutOff != Infinity) stream.string = oldContent.slice(0, cutOff);
        var outerToken = outer.token(stream, state.outer);
        if (cutOff != Infinity) stream.string = oldContent;
        return outerToken;
      } else {
        var curInner = state.innerActive, oldContent = stream.string;
        var found = indexOf(oldContent, curInner.close, stream.pos);
        if (found == stream.pos) {
          stream.match(curInner.close);
          state.innerActive = state.inner = null;
          return curInner.delimStyle;
        }
        if (found > -1) stream.string = oldContent.slice(0, found);
        var innerToken = curInner.mode.token(stream, state.inner);
        if (found > -1) stream.string = oldContent;
        var cur = stream.current(), found = cur.indexOf(curInner.close);
        if (found > -1) stream.backUp(cur.length - found);
        return innerToken;
      }
    },
    
    indent: function(state, textAfter) {
      var mode = state.innerActive ? state.innerActive.mode : outer;
      if (!mode.indent) return CodeMirror.Pass;
      return mode.indent(state.innerActive ? state.inner : state.outer, textAfter);
    },

    electricChars: outer.electricChars,

    innerMode: function(state) {
      return state.inner ? {state: state.inner, mode: state.innerActive.mode} : {state: state.outer, mode: outer};
    }
  };
};<commit_msg>Patch codemirror's lib/util/multiplex.js plugin to work properly with modes that use blankLine<commit_after>CodeMirror.multiplexingMode = function(outer /*, others */) {
  // Others should be {open, close, mode [, delimStyle]} objects
  var others = Array.prototype.slice.call(arguments, 1);
  var n_others = others.length;

  function indexOf(string, pattern, from) {
    if (typeof pattern == "string") return string.indexOf(pattern, from);
    var m = pattern.exec(from ? string.slice(from) : string);
    return m ? m.index + from : -1;
  }

  return {
    startState: function() {
      return {
        outer: CodeMirror.startState(outer),
        innerActive: null,
        inner: null
      };
    },

    copyState: function(state) {
      return {
        outer: CodeMirror.copyState(outer, state.outer),
        innerActive: state.innerActive,
        inner: state.innerActive && CodeMirror.copyState(state.innerActive.mode, state.inner)
      };
    },

    token: function(stream, state) {
      if (!state.innerActive) {
        var cutOff = Infinity, oldContent = stream.string;
        for (var i = 0; i < n_others; ++i) {
          var other = others[i];
          var found = indexOf(oldContent, other.open, stream.pos);
          if (found == stream.pos) {
            stream.match(other.open);
            state.innerActive = other;
            state.inner = CodeMirror.startState(other.mode, outer.indent ? outer.indent(state.outer, "") : 0);
            return other.delimStyle;
          } else if (found != -1 && found < cutOff) {
            cutOff = found;
          }
        }
        if (cutOff != Infinity) stream.string = oldContent.slice(0, cutOff);
        var outerToken = outer.token(stream, state.outer);
        if (cutOff != Infinity) stream.string = oldContent;
        return outerToken;
      } else {
        var curInner = state.innerActive, oldContent = stream.string;
        var found = indexOf(oldContent, curInner.close, stream.pos);
        if (found == stream.pos) {
          stream.match(curInner.close);
          state.innerActive = state.inner = null;
          return curInner.delimStyle;
        }
        if (found > -1) stream.string = oldContent.slice(0, found);
        var innerToken = curInner.mode.token(stream, state.inner);
        if (found > -1) stream.string = oldContent;
        var cur = stream.current(), found = cur.indexOf(curInner.close);
        if (found > -1) stream.backUp(cur.length - found);
        return innerToken;
      }
    },
    
    indent: function(state, textAfter) {
      var mode = state.innerActive ? state.innerActive.mode : outer;
      if (!mode.indent) return CodeMirror.Pass;
      return mode.indent(state.innerActive ? state.inner : state.outer, textAfter);
    },

    blankLine: function(state) {
       var mode = state.innerActive ? state.innerActive.mode : outer;
       if (mode.blankLine) {
          mode.blankLine(state.innerActive ? state.inner : state.outer);
       }
       if (!state.innerActive) {
          for (var i = 0; i < n_others; ++i) {
             var other = others[i];
             if (other.open === "\n") {
                state.innerActive = other;
                state.inner = CodeMirror.startState(other.mode, mode.indent ? mode.indent(state.outer, "") : 0);
             }
         }
       } else if (mode.close === "\n") {
          state.innerActive = state.inner = null;
       }
    },

    electricChars: outer.electricChars,

    innerMode: function(state) {
      return state.inner ? {state: state.inner, mode: state.innerActive.mode} : {state: state.outer, mode: outer};
    }
  };
};
{'fraction_non_alphanumeric': 0.1285823754789272, 'fraction_numerical': 0.003371647509578544, 'mean_word_length': 2.3295918367346937, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5884031', 'n_tokens_mistral': 1945, 'n_tokens_neox': 1895, 'n_words': 456}