<commit_before>var HaskellParser = Editor.Parser = (function() {
  var tokenizeHaskell = (function() {
  
    // These should all be Unicode extended, as per the Haskell 2010 report
    var smallRE = /[a-z_]/;
    var largeRE = /[A-Z]/;
    var digitRE = /[0-9]/;
    var hexitRE = /[0-9A-Fa-f]/;
    var octitRE = /[0-7]/;
    var idRE = /[a-z_A-Z0-9']/;
    var symbolRE = /[-!#$%&*+.\/<=>?@\\^|~:]/;
    var whiteCharRE = /[ \t\v\f]/; // newlines are handled in tokenizer
    
    
    function nextIfLiteralChar(delimiter) {
      if (source.endOfLine()) return false;
      if (source.equals(delimiter)) return false;
      if (source.equals('\\')) {
        source.next();
        source.next();
      }
      else {
        source.next();
      }
      return true
    }

    function normal(source, setState) {
      var ch = source.next();
      
      if (/[(),;[\]`{}]/.test(ch)) {
        if (ch == '{' && source.equals('-')) {
          source.next();
          var t = "hs-comment";
          if (source.equals('#')) {
            source.next();
            t = "hs-pragma";
          }
          setState(ncomment(t, 1));
          return null;
        }
        return "hs-special";
      }
      
      if (ch == '\'') {
        if (source.next == '\\') {
          source.next();  // should do more here
          if (source.equals('\'')) {
            source.next();
            return "hs-char";
          }
        }
        return "hs-error";
      }
      
      if (ch == '"') {
        setState(stringLiteral);
        return null;
      }
      
      if (largeRE.test(ch)) {
        source.nextWhileMatches(idRE);
        if (source.equals('.')) {
          source.next();
          return "hs-qualifier";
        }
        return "hs-conid";
      }
      
      if (smallRE.test(ch)) {
        source.nextWhileMatches(idRE);
        return "hs-varid";
      }
      
      if (digitRE.test(ch)) {
        if (ch == '0') {
          if (source.matches(/[xX]/)) {
            source.next();
            source.nextWhileMatches(hexitRE); // should require at least 1
            return "hs-integer";
          }
          if (source.matches(/[oO]/)) {
            source.next();
            source.nextWhileMatches(octitRE); // should require at least 1
            return "hs-integer";
          }
        }
        source.nextWhileMatches(digitRE);
        var t = "hs-integer";
        if (source.equals('.')) {
          t = "hs-float";
          source.next();
          source.nextWhileMatches(digitRE); // should require at least 1
        }
        if (source.matches(/[eE]/)) {
          t = "hs-float";
          source.next();
          if (source.matches(/[-+]/)) {
            source.next();
          }
          source.nextWhileMatches(digitRE); // should require at least 1
        }
        return t;
      }
      
      if (symbolRE.test(ch)) {
        if (ch == '-' && source.matches(/-/)) {
          source.nextWhileMatches(/-/);
          if (!source.matches(symbolRE)) {
            while (!source.endOfLine()) {
              source.next();
            }
            return "hs-comment";
          }
        }
        var t = "hs-varsym";
        if (ch == ':') {
          t = "hs-consym";
        }
        source.nextWhileMatches(symbolRE);
        return t;    
      }
      
      return "hs-error";
    }
    
    function in_ncomment(type, nest) {
      if (nest == 0) {
        return normal;
      }
      return function(source, setState) {
        while (source.more() && !source.endOfLine()) {
          ch = source.next();
          if (ch == '{' && source.equals('-')) {
            source.next();
            setState(ncomment(type, nest+1));
            return null;
          }
          else if (ch == '-' && source.equals('}')) {
            source.next();
            setState(ncomment(type, nest-1));
            return (nest == 1) ? type : null;
          }
        }<commit_msg>fix bug in hilighting block comments<commit_after>var HaskellParser = Editor.Parser = (function() {
  var tokenizeHaskell = (function() {
  
    // These should all be Unicode extended, as per the Haskell 2010 report
    var smallRE = /[a-z_]/;
    var largeRE = /[A-Z]/;
    var digitRE = /[0-9]/;
    var hexitRE = /[0-9A-Fa-f]/;
    var octitRE = /[0-7]/;
    var idRE = /[a-z_A-Z0-9']/;
    var symbolRE = /[-!#$%&*+.\/<=>?@\\^|~:]/;
    var whiteCharRE = /[ \t\v\f]/; // newlines are handled in tokenizer
    
    
    function nextIfLiteralChar(delimiter) {
      if (source.endOfLine()) return false;
      if (source.equals(delimiter)) return false;
      if (source.equals('\\')) {
        source.next();
        source.next();
      }
      else {
        source.next();
      }
      return true
    }

    function normal(source, setState) {
      var ch = source.next();
      
      if (/[(),;[\]`{}]/.test(ch)) {
        if (ch == '{' && source.equals('-')) {
          source.next();
          var t = "hs-comment";
          if (source.equals('#')) {
            source.next();
            t = "hs-pragma";
          }
          setState(ncomment(t, 1));
          return null;
        }
        return "hs-special";
      }
      
      if (ch == '\'') {
        if (source.next == '\\') {
          source.next();  // should do more here
          if (source.equals('\'')) {
            source.next();
            return "hs-char";
          }
        }
        return "hs-error";
      }
      
      if (ch == '"') {
        setState(stringLiteral);
        return null;
      }
      
      if (largeRE.test(ch)) {
        source.nextWhileMatches(idRE);
        if (source.equals('.')) {
          source.next();
          return "hs-qualifier";
        }
        return "hs-conid";
      }
      
      if (smallRE.test(ch)) {
        source.nextWhileMatches(idRE);
        return "hs-varid";
      }
      
      if (digitRE.test(ch)) {
        if (ch == '0') {
          if (source.matches(/[xX]/)) {
            source.next();
            source.nextWhileMatches(hexitRE); // should require at least 1
            return "hs-integer";
          }
          if (source.matches(/[oO]/)) {
            source.next();
            source.nextWhileMatches(octitRE); // should require at least 1
            return "hs-integer";
          }
        }
        source.nextWhileMatches(digitRE);
        var t = "hs-integer";
        if (source.equals('.')) {
          t = "hs-float";
          source.next();
          source.nextWhileMatches(digitRE); // should require at least 1
        }
        if (source.matches(/[eE]/)) {
          t = "hs-float";
          source.next();
          if (source.matches(/[-+]/)) {
            source.next();
          }
          source.nextWhileMatches(digitRE); // should require at least 1
        }
        return t;
      }
      
      if (symbolRE.test(ch)) {
        if (ch == '-' && source.matches(/-/)) {
          source.nextWhileMatches(/-/);
          if (!source.matches(symbolRE)) {
            while (!source.endOfLine()) {
              source.next();
            }
            return "hs-comment";
          }
        }
        var t = "hs-varsym";
        if (ch == ':') {
          t = "hs-consym";
        }
        source.nextWhileMatches(symbolRE);
        return t;    
      }
      
      return "hs-error";
    }
    
    function ncomment(type, nest) {
      if (nest == 0) {
        return normal;
      }
      return function(source, setState) {
        while (source.more() && !source.endOfLine()) {
          ch = source.next();
          if (ch == '{' && source.equals('-')) {
            source.next();
            setState(ncomment(type, nest+1));
            return null;
          }
          else if (ch == '-' && source.equals('}')) {
            source.next();
            setState(ncomment(type, nest-1));
            return (nest == 1) ? type : null;
          }
        }
{'fraction_non_alphanumeric': 0.1680545041635125, 'fraction_numerical': 0.0055513499873832955, 'mean_word_length': 1.3690974297668859, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2964519', 'n_tokens_mistral': 2515, 'n_tokens_neox': 2450, 'n_words': 497}