<commit_before>var CodeMirror = require("codemirror"), tokenUtils = require("./tokenUtils.js");

("use strict");
var lookFor = "PREFIX ";
module.exports = {
  findFirstPrefixLine: function(cm) {
    var lastLine = cm.lastLine();
    for (var i = 0; i <= lastLine; ++i) {
      if (findFirstPrefix(cm, i) >= 0) {
        return i;
      }
    }
  }
};

function findFirstPrefix(cm, line, ch, lineText) {
  if (!ch) ch = 0;
  if (!lineText) lineText = cm.getLine(line);
  lineText = lineText.toUpperCase();
  for (var at = ch, pass = 0; ; ) {
    var found = lineText.indexOf(lookFor, at);
    if (found == -1) {
      //no prefix on this line
      if (pass == 1) break;
      pass = 1;
      at = lineText.length;
      continue;
    }
    if (pass == 1 && found < ch) break;
    var tokenType = cm.getTokenTypeAt(CodeMirror.Pos(line, found + 1));
    if (!/^(comment|string)/.test(tokenType)) return found + 1;
    at = found - 1;
    //Could not find a prefix, no use looping any further. Probably invalid query
    if (at === pass) break;
  }
}

CodeMirror.registerHelper("fold", "prefix", function(cm, start) {
  var line = start.line, lineText = cm.getLine(line);

  var startCh, tokenType;

  function hasPreviousPrefix() {
    var hasPreviousPrefix = false;
    for (var i = line - 1; i >= 0; i--) {
      if (cm.getLine(i).toUpperCase().indexOf(lookFor) >= 0) {
        hasPreviousPrefix = true;
        break;
      }
    }
    return hasPreviousPrefix;
  }<commit_msg>Avoid infinite loops when the line is incorrect<commit_after>var CodeMirror = require("codemirror"), tokenUtils = require("./tokenUtils.js");

("use strict");
var lookFor = "PREFIX ";
module.exports = {
  findFirstPrefixLine: function(cm) {
    var lastLine = cm.lastLine();
    for (var i = 0; i <= lastLine; ++i) {
      if (findFirstPrefix(cm, i) >= 0) {
        return i;
      }
    }
  }
};

function findFirstPrefix(cm, line, ch, lineText) {
  if (!ch) ch = 0;
  if (!lineText) lineText = cm.getLine(line);
  lineText = lineText.toUpperCase();
  for (var at = ch, pass = 0; ; ) {
    var found = lineText.indexOf(lookFor, at);
    if (found == -1) {
      //no prefix on this line
      if (pass == 1) break;
      pass = 1;
      at = lineText.length;
      continue;
    }
    if (pass == 1 && found < ch) break;
    var tokenType = cm.getTokenTypeAt(CodeMirror.Pos(line, found + 1));
    if (!/^(comment|string)/.test(tokenType)) return found + 1;
    if (!/(comment)/.test(tokenType)) break;
    
    //Could not find a prefix, no use looping any further. Probably invalid query
    if (at === pass) break;
  }
}

CodeMirror.registerHelper("fold", "prefix", function(cm, start) {
  var line = start.line, lineText = cm.getLine(line);

  var startCh, tokenType;

  function hasPreviousPrefix() {
    var hasPreviousPrefix = false;
    for (var i = line - 1; i >= 0; i--) {
      if (cm.getLine(i).toUpperCase().indexOf(lookFor) >= 0) {
        hasPreviousPrefix = true;
        break;
      }
    }
    return hasPreviousPrefix;
  }
{'fraction_non_alphanumeric': 0.15298013245033112, 'fraction_numerical': 0.008940397350993378, 'mean_word_length': 2.7434944237918217, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1730196', 'n_tokens_mistral': 1047, 'n_tokens_neox': 1010, 'n_words': 287}