<commit_before>/* global define, brackets, console */

define(function () {
  'use strict';

  var LanguageManager = brackets.getModule('language/LanguageManager');
  var codeMirror = brackets.getModule('thirdparty/CodeMirror2/lib/codemirror');

  // Include `overlayMode` code mirror plugin
  if (!codeMirror.overlayMode) {
    codeMirror.overlayMode = function (base, overlay, combine) {
      return {
        startState: function () {
          return {
            base: codeMirror.startState(base),
            overlay: codeMirror.startState(overlay),
            basePos: 0,
            baseCur: null,
            overlayPos: 0,
            overlayCur: null
          };
        },
        copyState: function (state) {
          return {
            base: codeMirror.copyState(base, state.base),
            overlay: codeMirror.copyState(overlay, state.overlay),
            basePos: state.basePos,
            baseCur: null,
            overlayPos: state.overlayPos,
            overlayCur: null
          };
        },

        token: function (stream, state) {
          if (stream.start === state.basePos) {
            state.baseCur = base.token(stream, state.base);
            state.basePos = stream.pos;
          }
          if (stream.start === state.overlayPos) {
            stream.pos = stream.start;
            state.overlayCur = overlay.token(stream, state.overlay);
            state.overlayPos = stream.pos;
          }
          stream.pos = Math.min(state.basePos, state.overlayPos);
          if (stream.eol()) {
            state.basePos = state.overlayPos = 0;
          }

          if (state.overlayCur === null) {
            return state.baseCur;
          }
          if (state.baseCur !== null && combine) {
            return state.baseCur + ' ' + state.overlayCur;
          }
          return state.overlayCur;
        },

        indent: base.indent && function (state, textAfter) {
          return base.indent(state.base, textAfter);
        },
        electricChars: base.electricChars,

        innerMode: function (state) {
          return {
            state: state.base,
            mode: base
          };
        },

        blankLine: function (state) {
          if (base.blankLine) {
            base.blankLine(state.base);
          }
          if (overlay.blankLine) {
            overlay.blankLine(state.overlay);
          }
        }
      };
    };
  }

  codeMirror.defineMode('handlebars', function (config, parserConfig) {
    var mustacheOverlay = {
      startState: function () {
        return {
          moustacheStack: [],
          hasError: false,
          errorTerminatesOn: null,
          opening: false,
          closing: false,
          helperName: false,
          comment: false,
          safeComment: false,
          doNotEscape: false,
          blockParam: false
        };
      },
      token: function (stream, state) {
        /*jslint regexp:true */
        stream.eatSpace();
        if (!state.inHandlebar) {
          if (stream.match('{{')) {// || stream.match(/<((?:\w[\w\$:\d]++(?:\-[\w\$:\d]++)+))(?:\s|>)/, false)) {
            stream.eatSpace();
            state.inHandlebar = true;
            state.helperName = true;
            state.closing = false;
            state.opening = false;
            state.doNotEscape = false;
            state.comment = false;

            if (stream.eat('{')) {
              //output HTML
              state.doNotEscape = true;
              stream.eatSpace();
              return 'operator';
            }
            if (stream.eat('!')) {
              stream.backUp(1);
              state.comment = true;
              if (stream.match('!--')) {
                stream.backUp(3);
                state.safeComment = true;
              }
              return 'comment';
            }
            if (stream.eat('#')) {
              //tag start
              state.opening = true;
            } else if (stream.eat('/')) {
              //tag end
              state.closing = true;
            }
            stream.eatSpace();
            return 'bracket';
          }
          if (stream.next() === null && state.moustacheStack.length > 0) {
            console.log('Unclosed tags: ', state.moustacheStack);
            return 'invalidchar';
          }
          return null;
        }
        // Since comments can contain }} it needs to be processed first
        if(state.comment) {
          if((state.safeComment === true && stream.match('--}}')) || (state.safeComment === false && stream.match('}}'))) {
            state.comment = false;
            state.inHandlebar = false;
            state.helperName = false;
            state.safeComment = false;
          } else {
            stream.next();
          }

          return 'comment';
        }
        if (state.helperName) {
          state.helperName = false;

          if (!state.opening && !state.closing && stream.match(/^else\s*\}\}/, false)) {
            stream.match(/^else/, true);
            stream.eatSpace();
            return 'keyword';
          }
          
          if (!state.opening && !state.closing && stream.match(/^[\w\d\-\_\$\.\/\@]+\s*\}\}/, false)) {
            stream.match(/^[\w\d\-\_\$\.\/\@]+/, true);
            stream.eatSpace();
            return 'variable';
          }
          if (stream.match(/^[\w\d\-\_\$]+/, false)) {
            var reserved = stream.match(/^(if|unless|else|each|with|view|action|component|yield)/, false);
            stream.match(/^[\w\d\-\_\$]+/, true);
            state.helperName = false;
            if (state.closing) {
              state.closing = false;
              state.endOnly = true;
              if (state.moustacheStack.pop() !== stream.current()) {
                console.log('Mismatched tags');
                return 'invalidchar';
              }
              return reserved ? 'keyword':'tag';
            }
            if (state.opening) {<commit_msg>Added partial as a keyword<commit_after>/* global define, brackets, console */

define(function () {
  'use strict';

  var LanguageManager = brackets.getModule('language/LanguageManager');
  var codeMirror = brackets.getModule('thirdparty/CodeMirror2/lib/codemirror');

  // Include `overlayMode` code mirror plugin
  if (!codeMirror.overlayMode) {
    codeMirror.overlayMode = function (base, overlay, combine) {
      return {
        startState: function () {
          return {
            base: codeMirror.startState(base),
            overlay: codeMirror.startState(overlay),
            basePos: 0,
            baseCur: null,
            overlayPos: 0,
            overlayCur: null
          };
        },
        copyState: function (state) {
          return {
            base: codeMirror.copyState(base, state.base),
            overlay: codeMirror.copyState(overlay, state.overlay),
            basePos: state.basePos,
            baseCur: null,
            overlayPos: state.overlayPos,
            overlayCur: null
          };
        },

        token: function (stream, state) {
          if (stream.start === state.basePos) {
            state.baseCur = base.token(stream, state.base);
            state.basePos = stream.pos;
          }
          if (stream.start === state.overlayPos) {
            stream.pos = stream.start;
            state.overlayCur = overlay.token(stream, state.overlay);
            state.overlayPos = stream.pos;
          }
          stream.pos = Math.min(state.basePos, state.overlayPos);
          if (stream.eol()) {
            state.basePos = state.overlayPos = 0;
          }

          if (state.overlayCur === null) {
            return state.baseCur;
          }
          if (state.baseCur !== null && combine) {
            return state.baseCur + ' ' + state.overlayCur;
          }
          return state.overlayCur;
        },

        indent: base.indent && function (state, textAfter) {
          return base.indent(state.base, textAfter);
        },
        electricChars: base.electricChars,

        innerMode: function (state) {
          return {
            state: state.base,
            mode: base
          };
        },

        blankLine: function (state) {
          if (base.blankLine) {
            base.blankLine(state.base);
          }
          if (overlay.blankLine) {
            overlay.blankLine(state.overlay);
          }
        }
      };
    };
  }

  codeMirror.defineMode('handlebars', function (config, parserConfig) {
    var mustacheOverlay = {
      startState: function () {
        return {
          moustacheStack: [],
          hasError: false,
          errorTerminatesOn: null,
          opening: false,
          closing: false,
          helperName: false,
          comment: false,
          safeComment: false,
          doNotEscape: false,
          blockParam: false
        };
      },
      token: function (stream, state) {
        /*jslint regexp:true */
        stream.eatSpace();
        if (!state.inHandlebar) {
          if (stream.match('{{')) {// || stream.match(/<((?:\w[\w\$:\d]++(?:\-[\w\$:\d]++)+))(?:\s|>)/, false)) {
            stream.eatSpace();
            state.inHandlebar = true;
            state.helperName = true;
            state.closing = false;
            state.opening = false;
            state.doNotEscape = false;
            state.comment = false;

            if (stream.eat('{')) {
              //output HTML
              state.doNotEscape = true;
              stream.eatSpace();
              return 'operator';
            }
            if (stream.eat('!')) {
              stream.backUp(1);
              state.comment = true;
              if (stream.match('!--')) {
                stream.backUp(3);
                state.safeComment = true;
              }
              return 'comment';
            }
            if (stream.eat('#')) {
              //tag start
              state.opening = true;
            } else if (stream.eat('/')) {
              //tag end
              state.closing = true;
            }
            stream.eatSpace();
            return 'bracket';
          }
          if (stream.next() === null && state.moustacheStack.length > 0) {
            console.log('Unclosed tags: ', state.moustacheStack);
            return 'invalidchar';
          }
          return null;
        }
        // Since comments can contain }} it needs to be processed first
        if(state.comment) {
          if((state.safeComment === true && stream.match('--}}')) || (state.safeComment === false && stream.match('}}'))) {
            state.comment = false;
            state.inHandlebar = false;
            state.helperName = false;
            state.safeComment = false;
          } else {
            stream.next();
          }

          return 'comment';
        }
        if (state.helperName) {
          state.helperName = false;

          if (!state.opening && !state.closing && stream.match(/^else\s*\}\}/, false)) {
            stream.match(/^else/, true);
            stream.eatSpace();
            return 'keyword';
          }
          
          if (!state.opening && !state.closing && stream.match(/^[\w\d\-\_\$\.\/\@]+\s*\}\}/, false)) {
            stream.match(/^[\w\d\-\_\$\.\/\@]+/, true);
            stream.eatSpace();
            return 'variable';
          }
          if (stream.match(/^[\w\d\-\_\$]+/, false)) {
            var reserved = stream.match(/^(if|unless|else|each|with|view|action|component|yield|partial)/, false);
            stream.match(/^[\w\d\-\_\$]+/, true);
            state.helperName = false;
            if (state.closing) {
              state.closing = false;
              state.endOnly = true;
              if (state.moustacheStack.pop() !== stream.current()) {
                console.log('Mismatched tags');
                return 'invalidchar';
              }
              return reserved ? 'keyword':'tag';
            }
            if (state.opening) {
{'fraction_non_alphanumeric': 0.1403934700711595, 'fraction_numerical': 0.0011720385098367518, 'mean_word_length': 1.4484525517524083, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6994404', 'n_tokens_mistral': 3321, 'n_tokens_neox': 3170, 'n_words': 636}