<commit_before>/*jslint
  vars: true,
  plusplus: true,
  devel: true,
  nomen: true,
  regexp: true,
  indent: 2,
  maxerr: 50
*/
/*global
  define,
  brackets,
  CodeMirror
*/
define(function (require, exports, module) {
  'use strict';

  var LanguageManager = brackets.getModule("language/LanguageManager");
  var Quotes = {
    SINGLE: 1,
    DOUBLE: 2
  };

  var regex = {
    keywords: /(Feature| {2}(Scenario|In order to|As|I)| {4}(Given|When|Then|And))/
  };

  CodeMirror.defineMode("gherkin", function () {
    return {
      startState: function () {
        return {
          allowFeature: true,
          allowBackground: false,
          allowScenario: false,
          allowSteps: false,
          inMultilineArgument: false,
          inMultilineString: false
        };
      },
      token: function (stream, state) {
        stream.eatSpace();

        // INSIDE OF MULTILINE ARGUMENTS
        if (state.inMultilineArgument) {

          // STRING
          if (state.inMultilineString) {
            if (stream.match('"""')) {
              state.inMultilineString = false;
              state.inMultilineArgument = false;
            } else {
              stream.match(/.*/);
            }
            return "string";
          }

          // TABLE
          if (state.inMultilineTable) {
            if (stream.match(/\|\s*/)) {
              if (stream.eol()) {
                state.inMultilineTable = false;
              }
              return "keyword";
            } else {
              stream.match(/[^\|]/);
              return "string";
            }
          }

          // DETECT START
          if (stream.match('"""')) {
            // String
            state.inMultilineString = true;
            return "string";
          } else if (stream.match("|")) {
            // Table
            state.inMultilineTable = true;
            return "keyword";
          } else {
            state.inMultilineArgument = false;
          }


          return null;
        }

        // LINE COMMENT
        if (stream.match(/#.*/)) {
          return "comment";

        // TAG
        } else if (stream.match(/@\S+/)) {
          return "def";

        // FEATURE
        } else if (state.allowFeature && stream.match(/Feature:/)) {
          state.allowScenario = true;<commit_msg>Not a keyword<commit_after>/*jslint
  vars: true,
  plusplus: true,
  devel: true,
  nomen: true,
  regexp: true,
  indent: 2,
  maxerr: 50
*/
/*global
  define,
  brackets,
  CodeMirror
*/
define(function (require, exports, module) {
  'use strict';

  var LanguageManager = brackets.getModule("language/LanguageManager");
  var Quotes = {
    SINGLE: 1,
    DOUBLE: 2
  };

  var regex = {
    keywords: /(Feature| {2}(Scenario|In order to|As|I)| {4}(Given|When|Then|And))/
  };

  CodeMirror.defineMode("gherkin", function () {
    return {
      startState: function () {
        return {
          allowFeature: true,
          allowBackground: false,
          allowScenario: false,
          allowSteps: false,
          inMultilineArgument: false,
          inMultilineString: false
        };
      },
      token: function (stream, state) {
        stream.eatSpace();

        // INSIDE OF MULTILINE ARGUMENTS
        if (state.inMultilineArgument) {

          // STRING
          if (state.inMultilineString) {
            if (stream.match('"""')) {
              state.inMultilineString = false;
              state.inMultilineArgument = false;
            } else {
              stream.match(/.*/);
            }
            return "string";
          }

          // TABLE
          if (state.inMultilineTable) {
            if (stream.match(/\|\s*/)) {
              if (stream.eol()) {
                state.inMultilineTable = false;
              }
              return null;
            } else {
              stream.match(/[^\|]/);
              return "string";
            }
          }

          // DETECT START
          if (stream.match('"""')) {
            // String
            state.inMultilineString = true;
            return "string";
          } else if (stream.match("|")) {
            // Table
            state.inMultilineTable = true;
            return null;
          } else {
            state.inMultilineArgument = false;
          }


          return null;
        }

        // LINE COMMENT
        if (stream.match(/#.*/)) {
          return "comment";

        // TAG
        } else if (stream.match(/@\S+/)) {
          return "def";

        // FEATURE
        } else if (state.allowFeature && stream.match(/Feature:/)) {
          state.allowScenario = true;
{'fraction_non_alphanumeric': 0.13050993949870354, 'fraction_numerical': 0.003025064822817632, 'mean_word_length': 1.4299212598425197, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '576491', 'n_tokens_mistral': 1397, 'n_tokens_neox': 1303, 'n_words': 281}