<commit_before>/* 
Syntax highlighting with language autodetection.  
http://softwaremaniacs.org/soft/highlight/
*/

var hljs = new function() {

  var DEFAULT_LANGUAGES = ['python', 'ruby', 'perl', 'php', 'css', 'xml', 'html', 'django', 'javascript', 'java', 'cpp', 'sql', 'smalltalk'];
  var ALL_LANGUAGES = (DEFAULT_LANGUAGES.join(',') + ',' + ['1c', 'axapta', 'delphi', 'rib', 'rsl', 'vbscript', 'profile'].join(',')).split(',');
  var LANGUAGE_GROUPS = {
    'xml': 'www',
    'html': 'www',
    'css': 'www',
    'django': 'www',
    'python': 'dynamic',
    'perl': 'dynamic',
    'php': 'dynamic',
    'ruby': 'dynamic',
    'cpp': 'static',
    'java': 'static',
    'delphi': 'static',
    'rib': 'renderman',
    'rsl': 'renderman'
  }

  var LANGUAGES = {}
  var selected_languages = {};
  
  function escape(value) {
    return value.replace(/&/gm, '&amp;').replace(/</gm, '&lt;').replace(/>/gm, '&gt;');
  }//escape
  
  function contains(array, item) {
    if (!array)
      return false;
    for (var key in array)
      if (array[key] == item)
        return true;
    return false;
  }//contains
  
  function highlight(language_name, value) {
    function subMode(lexem) {
      if (!modes[modes.length - 1].contains)
        return null;
      for (var i in modes[modes.length - 1].contains) {
        var className = modes[modes.length - 1].contains[i];
        for (var key in language.modes)
          if (language.modes[key].className == className && language.modes[key].beginRe.test(lexem))
            return language.modes[key];
      }//for
      return null;
    }//subMode
    
    function endOfMode(mode_index, lexem) {
      if (modes[mode_index].end && modes[mode_index].endRe.test(lexem))
        return 1;
      if (modes[mode_index].endsWithParent) {
        var level = endOfMode(mode_index - 1, lexem);
        return level ? level + 1 : 0;
      }//if
      return 0;
    }//endOfMode
    
    function isIllegal(lexem) {
      if (!modes[modes.length - 1].illegalRe)
        return false;
      return modes[modes.length - 1].illegalRe.test(lexem);
    }//isIllegal
    
    function compileTerminators(modes, language) {
      var terminators = [];
      
      function addTerminator(re) {
        if (!contains(terminators, re)) {
          terminators[terminators.length] = re;
        }//if
      }//addTerminator
      
      var index = modes.length - 1;
      var mode = modes[index];
      
      if (mode.contains)
        for (var key in language.modes) {
          if (contains(mode.contains, language.modes[key].className)) {
            addTerminator(language.modes[key].begin);
          }//if
        }//for
      
      do {
        if (modes[index].end) {
          addTerminator(modes[index].end);
        }//if
        index--;
      } while (modes[index + 1].endsWithParent);
      
      if (mode.illegal) {
        addTerminator(mode.illegal);
      }//if
      
      var terminator_re = '(' + terminators[0];
      for (var i = 0; i < terminators.length; i++)
        terminator_re += '|' + terminators[i];
      terminator_re += ')';
      return langRe(language, terminator_re);
    }//compileTerminators

    function eatModeChunk(value, index) {
      if (!modes[modes.length - 1].terminators) {
        modes[modes.length - 1].terminators = compileTerminators(modes, language);
      }//if
      value = value.substr(index);
      var match = modes[modes.length - 1].terminators.exec(value);
      if (!match) 
        return [value, '', true];
      if (match.index == 0)
        return ['', match[0], false];
      else
        return [value.substr(0, match.index), match[0], false];
    }//eatModeChunk
    
    function keywordMatch(mode, match) {
      var match_str = language.case_insensitive ? match[0].toLowerCase() : match[0]
      for (var className in mode.keywordGroups) {
        var value = mode.keywordGroups[className].hasOwnProperty(match_str);
        if (value)
          return [className, value];
      }//for
      return false;
    }//keywordMatch
    
    function processKeywords(buffer) {
      var mode = modes[modes.length - 1];
      if (!mode.keywords || !mode.lexems)
        return escape(buffer);
      if (!mode.lexemsRe) {
        var lexems = [];
        for (var key in mode.lexems)
          if (!contains(lexems, mode.lexems[key]))
            lexems[lexems.length] = mode.lexems[key];
        var lexems_re = '(' + lexems[0];
        for (var i = 1; i < lexems.length; i++)
          lexems_re += '|' + lexems[i];
        lexems_re += ')';
        mode.lexemsRe = langRe(language, lexems_re, true);
      }//if
      var result = '';
      var last_index = 0;
      mode.lexemsRe.lastIndex = 0;
      var match = mode.lexemsRe.exec(buffer);
      while (match) {
        result += escape(buffer.substr(last_index, match.index - last_index));
        keyword_match = keywordMatch(mode, match);
        if (keyword_match) {
          keyword_count += keyword_match[1];
          result += '<span class="'+ keyword_match[0] +'">' + escape(match[0]) + '</span>';
        } else {
          result += escape(match[0]);
        }//if
        last_index = mode.lexemsRe.lastIndex;
        match = mode.lexemsRe.exec(buffer);
      }//while
      result += escape(buffer.substr(last_index, buffer.length - last_index));
      return result;
    }//processKeywords
    
    function processModeInfo(buffer, lexem, end) {
      if (end) {
        result += processKeywords(modes[modes.length - 1].buffer + buffer);
        return;
      }//if
      if (isIllegal(lexem))
        throw 'Illegal';
      var new_mode = subMode(lexem);
      if (new_mode) {
        modes[modes.length - 1].buffer += buffer;
        result += processKeywords(modes[modes.length - 1].buffer);
        if (new_mode.excludeBegin) {
          result += lexem + '<span class="' + new_mode.className + '">';
          new_mode.buffer = '';
        } else {
          result += '<span class="' + new_mode.className + '">';
          new_mode.buffer = lexem;
        }//if
        modes[modes.length] = new_mode;
        relevance += modes[modes.length - 1].relevance != undefined ? modes[modes.length - 1].relevance : 1;<commit_msg>Небольшой рефакторинг modes[modes.length - 1]<commit_after>/* 
Syntax highlighting with language autodetection.  
http://softwaremaniacs.org/soft/highlight/
*/

var hljs = new function() {

  var DEFAULT_LANGUAGES = ['python', 'ruby', 'perl', 'php', 'css', 'xml', 'html', 'django', 'javascript', 'java', 'cpp', 'sql', 'smalltalk'];
  var ALL_LANGUAGES = (DEFAULT_LANGUAGES.join(',') + ',' + ['1c', 'axapta', 'delphi', 'rib', 'rsl', 'vbscript', 'profile'].join(',')).split(',');
  var LANGUAGE_GROUPS = {
    'xml': 'www',
    'html': 'www',
    'css': 'www',
    'django': 'www',
    'python': 'dynamic',
    'perl': 'dynamic',
    'php': 'dynamic',
    'ruby': 'dynamic',
    'cpp': 'static',
    'java': 'static',
    'delphi': 'static',
    'rib': 'renderman',
    'rsl': 'renderman'
  }

  var LANGUAGES = {}
  var selected_languages = {};
  
  function escape(value) {
    return value.replace(/&/gm, '&amp;').replace(/</gm, '&lt;').replace(/>/gm, '&gt;');
  }//escape
  
  function contains(array, item) {
    if (!array)
      return false;
    for (var key in array)
      if (array[key] == item)
        return true;
    return false;
  }//contains
  
  function highlight(language_name, value) {
    function subMode(lexem, mode) {
      if (!mode.contains)
        return null;
      for (var i in mode.contains) {
        var className = mode.contains[i];
        for (var key in language.modes)
          if (language.modes[key].className == className && language.modes[key].beginRe.test(lexem))
            return language.modes[key];
      }//for
      return null;
    }//subMode
    
    function endOfMode(mode_index, lexem) {
      if (modes[mode_index].end && modes[mode_index].endRe.test(lexem))
        return 1;
      if (modes[mode_index].endsWithParent) {
        var level = endOfMode(mode_index - 1, lexem);
        return level ? level + 1 : 0;
      }//if
      return 0;
    }//endOfMode
    
    function isIllegal(lexem, mode) {
      return mode.illegalRe && mode.illegalRe.test(lexem);
    }//isIllegal
    
    function compileTerminators(mode, language) {
      var terminators = [];
      
      function addTerminator(re) {
        if (!contains(terminators, re)) {
          terminators[terminators.length] = re;
        }//if
      }//addTerminator
      
      if (mode.contains)
        for (var key in language.modes) {
          if (contains(mode.contains, language.modes[key].className)) {
            addTerminator(language.modes[key].begin);
          }//if
        }//for
      
      var index = modes.length - 1;
      do {
        if (modes[index].end) {
          addTerminator(modes[index].end);
        }//if
        index--;
      } while (modes[index + 1].endsWithParent);
      
      if (mode.illegal) {
        addTerminator(mode.illegal);
      }//if
      
      var terminator_re = '(' + terminators[0];
      for (var i = 0; i < terminators.length; i++)
        terminator_re += '|' + terminators[i];
      terminator_re += ')';
      return langRe(language, terminator_re);
    }//compileTerminators

    function eatModeChunk(value, index) {
      var mode = modes[modes.length - 1];
      if (!mode.terminators) {
        mode.terminators = compileTerminators(mode, language);
      }//if
      value = value.substr(index);
      var match = mode.terminators.exec(value);
      if (!match) 
        return [value, '', true];
      if (match.index == 0)
        return ['', match[0], false];
      else
        return [value.substr(0, match.index), match[0], false];
    }//eatModeChunk
    
    function keywordMatch(mode, match) {
      var match_str = language.case_insensitive ? match[0].toLowerCase() : match[0]
      for (var className in mode.keywordGroups) {
        var value = mode.keywordGroups[className].hasOwnProperty(match_str);
        if (value)
          return [className, value];
      }//for
      return false;
    }//keywordMatch
    
    function processKeywords(buffer) {
      var mode = modes[modes.length - 1];
      if (!mode.keywords || !mode.lexems)
        return escape(buffer);
      if (!mode.lexemsRe) {
        var lexems = [];
        for (var key in mode.lexems)
          if (!contains(lexems, mode.lexems[key]))
            lexems[lexems.length] = mode.lexems[key];
        var lexems_re = '(' + lexems[0];
        for (var i = 1; i < lexems.length; i++)
          lexems_re += '|' + lexems[i];
        lexems_re += ')';
        mode.lexemsRe = langRe(language, lexems_re, true);
      }//if
      var result = '';
      var last_index = 0;
      mode.lexemsRe.lastIndex = 0;
      var match = mode.lexemsRe.exec(buffer);
      while (match) {
        result += escape(buffer.substr(last_index, match.index - last_index));
        keyword_match = keywordMatch(mode, match);
        if (keyword_match) {
          keyword_count += keyword_match[1];
          result += '<span class="'+ keyword_match[0] +'">' + escape(match[0]) + '</span>';
        } else {
          result += escape(match[0]);
        }//if
        last_index = mode.lexemsRe.lastIndex;
        match = mode.lexemsRe.exec(buffer);
      }//while
      result += escape(buffer.substr(last_index, buffer.length - last_index));
      return result;
    }//processKeywords
    
    function processModeInfo(buffer, lexem, end) {
      if (end) {
        result += processKeywords(modes[modes.length - 1].buffer + buffer);
        return;
      }//if
      if (isIllegal(lexem, modes[modes.length - 1]))
        throw 'Illegal';
      var new_mode = subMode(lexem, modes[modes.length - 1]);
      if (new_mode) {
        modes[modes.length - 1].buffer += buffer;
        result += processKeywords(modes[modes.length - 1].buffer);
        if (new_mode.excludeBegin) {
          result += lexem + '<span class="' + new_mode.className + '">';
          new_mode.buffer = '';
        } else {
          result += '<span class="' + new_mode.className + '">';
          new_mode.buffer = lexem;
        }//if
        modes[modes.length] = new_mode;
        relevance += modes[modes.length - 1].relevance != undefined ? modes[modes.length - 1].relevance : 1;
{'fraction_non_alphanumeric': 0.15737811308509775, 'fraction_numerical': 0.006003082664070739, 'mean_word_length': 2.289220917822839, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3892154', 'n_tokens_mistral': 4086, 'n_tokens_neox': 3908, 'n_words': 858}