<commit_before>/*
highlight.js v1.0.0
Available via the MIT or new BSD license.
see: http://github.com/jianghai/highlight for details
*/

(function() {

  /**
   * Parse normal code string new "marked" string by all kinds of
   * syntax rules.
   */
  var parse = function(source, lang) {

    // Remove \n in the start
    source = source.replace(/^\n/g, '');

    // Replace whitespace with entity
    // '\s' contains '\n', so just use ' '
    // source = source.replace(/ /g, '\u00a0');

    var reg = getReg(lang);
    if (reg) {
      var rules = highlight.rules[lang];
      var reg = new RegExp(getReg(lang), 'g');
      source = source.replace(reg, function() {
        var args = arguments;
        var len = args.length - 1;
        var i = 1;
        for (; i < len; i++) {
          if (args[i] && rules[i - 1]) {
            if (rules[i - 1].callback) {
              args[i] = rules[i - 1].callback.call(null, args[i])
            }
            return '<span class="' + rules[i - 1].name + '">' + args[i] + '</span>';
          }
        }
      });
    }
    
    return source.replace(/\n/g, '<br>');
  };


  /**
   * Get all regexp rules of this lang.
   */
  var getReg = function(lang) {
    if (getReg[lang]) return getReg[lang];
    var reg = [];
    var rules = highlight.rules[lang];
    if (rules) {
      for (var i = 0; i < rules.length; i++) {
        reg.push(rules[i].rule);
      }
    }
    reg = reg.join('|');

    getReg[lang] = reg;

    return reg;
  };


  /**
   * Outer interface
   */
  var highlight = function(source, lang) {
    return parse(source, lang);
  };


  /**
   * Language syntax highlight rules.
   * Just wrap the matched or the return of callback with span element, the span
   * element has a className which is value of key name for custom your css style.
   * You can add or edit by yourself.
   */
  highlight.rules = {

    markup: [{
      name: 'com',
      rule: '(<!--[\\s\\S]*?-->)',
      callback: function(str) {
        return str.replace(/</g, '&lt;');
      }
    }, {
      name: 'tag',
      rule: '(<[\\/!]?[\\w-]+|>)',
      callback: function(str) {
        return str.replace('<', '&lt;');
      }
    }, {
      name: 'attr',
      rule: '\\b([\\w-:]+[=>])'
    }, {
      name: 'str', // attribute value
      rule: '(\"[\\s\\S]*?\")'
    }],

    css: [{
      name: 'tag',
      rule: '(<[\\/!]?[\\w\\d]+|>)',
      callback: function(str) {
        return str.replace('<', '&lt;');
      }
    }, {
      name: 'str', // attribute value
      rule: '(\"[\\s\\S]*?\")'
    }],

    js: [{
      name: 'com', // comment
      rule: '(\\/\\/.*|\\/\\*[\\s\\S]*?\\*\\/)',
      callback: function(str) {
        // Some js comment has html tags
        return str.replace(/</g, '&lt;');
      }
    }, {
      name: 'str', // string
      rule: '(\'[\\s\\S]*?\'|\"[\\s\\S]*?\")',
      callback: function(str) {
        // Some js string has html tags
        return str.replace(/</g, '&lt;');
      }
    }, {
      name: 'kwd', // keyword
      rule: '\\b(function|break|case|catch|continue|debugger|default|delete|do|else|finally|for|function|if|in|instanceof|new|return|switch|this|throw|try|typeof|var|void|while|with)\\b'
    }, {
      name: 'kc', // key constant
      rule: '\\b(true|false|undefined|null|Infinity)\\b'
    }, {
      name: 'nb', // built-in object
      rule: '\\b(Array|console|Date|document|Function|isFinite|isNaN|Math|Object|parseInt|parseFloat|RegExp|string|window)\\b'
    }, {
      name: 'opt', // operator
      rule: '(\\+|-|\\*|\\/|%|<|>|=|==|===|!=|!==|!|&&|\\|\\||&|\\|)'
    }, {
      name: 'num', // number
      rule: '(\\d+)'
    }]

    // Another language<commit_msg>add in & fix number rules<commit_after>/*
highlight.js v1.0.0
Available via the MIT or new BSD license.
see: http://github.com/jianghai/highlight for details
*/

(function() {

  /**
   * Parse normal code string new "marked" string by all kinds of
   * syntax rules.
   */
  var parse = function(source, lang) {

    // Remove \n in the start
    source = source.replace(/^\n/g, '');

    // Replace whitespace with entity
    // '\s' contains '\n', so just use ' '
    // source = source.replace(/ /g, '\u00a0');

    var reg = getReg(lang);
    if (reg) {
      var rules = highlight.rules[lang];
      var reg = new RegExp(getReg(lang), 'g');
      source = source.replace(reg, function() {
        var args = arguments;
        var len = args.length - 1;
        var i = 1;
        for (; i < len; i++) {
          if (args[i] && rules[i - 1]) {
            if (rules[i - 1].callback) {
              args[i] = rules[i - 1].callback.call(null, args[i])
            }
            return '<span class="' + rules[i - 1].name + '">' + args[i] + '</span>';
          }
        }
      });
    }
    
    return source.replace(/\n/g, '<br>');
  };


  /**
   * Get all regexp rules of this lang.
   */
  var getReg = function(lang) {
    if (getReg[lang]) return getReg[lang];
    var reg = [];
    var rules = highlight.rules[lang];
    if (rules) {
      for (var i = 0; i < rules.length; i++) {
        reg.push(rules[i].rule);
      }
    }
    reg = reg.join('|');

    getReg[lang] = reg;

    return reg;
  };


  /**
   * Outer interface
   */
  var highlight = function(source, lang) {
    return parse(source, lang);
  };


  /**
   * Language syntax highlight rules.
   * Just wrap the matched or the return of callback with span element, the span
   * element has a className which is value of key name for custom your css style.
   * You can add or edit by yourself.
   */
  highlight.rules = {

    markup: [{
      name: 'com',
      rule: '(<!--[\\s\\S]*?-->)',
      callback: function(str) {
        return str.replace(/</g, '&lt;');
      }
    }, {
      name: 'tag',
      rule: '(<[\\/!]?[\\w-]+|>)',
      callback: function(str) {
        return str.replace('<', '&lt;');
      }
    }, {
      name: 'attr',
      rule: '\\b([\\w-:]+[=>])'
    }, {
      name: 'str', // attribute value
      rule: '(\"[\\s\\S]*?\")'
    }],

    css: [{
      name: 'tag',
      rule: '(<[\\/!]?[\\w\\d]+|>)',
      callback: function(str) {
        return str.replace('<', '&lt;');
      }
    }, {
      name: 'str', // attribute value
      rule: '(\"[\\s\\S]*?\")'
    }],

    js: [{
      name: 'com', // comment
      rule: '(\\/\\/.*|\\/\\*[\\s\\S]*?\\*\\/)',
      callback: function(str) {
        // Some js comment has html tags
        return str.replace(/</g, '&lt;');
      }
    }, {
      name: 'str', // string
      rule: '(\'[\\s\\S]*?\'|\"[\\s\\S]*?\")',
      callback: function(str) {
        // Some js string has html tags
        return str.replace(/</g, '&lt;');
      }
    }, {
      name: 'kwd', // keyword
      rule: '\\b(function|break|case|catch|continue|debugger|default|delete|do|else|finally|for|function|if|in|instanceof|new|return|switch|this|throw|try|typeof|var|void|while|with)\\b'
    }, {
      name: 'kc', // key constant
      rule: '\\b(true|false|undefined|null|Infinity)\\b'
    }, {
      name: 'in', // build-in keyword
      rule: '\\b(window|require|console)\\b'
    }, {
      name: 'nb', // built-in object
      rule: '\\b(Array|console|Date|document|Function|isFinite|isNaN|Math|Object|parseInt|parseFloat|RegExp|string|window)\\b'
    }, {
      name: 'opt', // operator
      rule: '(\\+|-|\\*|\\/|%|<|>|=|==|===|!=|!==|!|&&|\\|\\||&|\\|)'
    }, {
      name: 'num', // number
      rule: '\\b(\\d+)\\b'
    }]

    // Another language
{'fraction_non_alphanumeric': 0.22325953587623365, 'fraction_numerical': 0.0034675913576953854, 'mean_word_length': 1.9020897832817338, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '705785', 'n_tokens_mistral': 2692, 'n_tokens_neox': 2609, 'n_words': 601}