<commit_before>// Code block parser plugin
//

'use strict';


var hljs = require('highlight.js/lib/highlight.js');


module.exports = function () {

  return function (parser) {
    parser.md.enable([ 'fence', 'code', 'backticks' ]);

    hljs.registerLanguage('actionscript', require('highlight.js/lib/languages/actionscript'));
    hljs.registerLanguage('apache',       require('highlight.js/lib/languages/apache'));
    hljs.registerLanguage('arduino',      require('highlight.js/lib/languages/arduino'));
    hljs.registerLanguage('armasm',       require('highlight.js/lib/languages/armasm'));
    hljs.registerLanguage('xml',          require('highlight.js/lib/languages/xml'));
    hljs.registerLanguage('asciidoc',     require('highlight.js/lib//languages/asciidoc'));
    hljs.registerLanguage('avrasm',       require('highlight.js/lib/languages/avrasm'));
    hljs.registerLanguage('bash',         require('highlight.js/lib/languages/bash'));
    hljs.registerLanguage('clojure',      require('highlight.js/lib/languages/clojure'));
    hljs.registerLanguage('cmake',        require('highlight.js/lib/languages/cmake'));
    hljs.registerLanguage('coffeescript', require('highlight.js/lib/languages/coffeescript'));
    hljs.registerLanguage('cpp',          require('highlight.js/lib/languages/cpp'));
    hljs.registerLanguage('css',          require('highlight.js/lib/languages/css'));
    hljs.registerLanguage('diff',         require('highlight.js/lib/languages/diff'));
    hljs.registerLanguage('django',       require('highlight.js/lib/languages/django'));
    hljs.registerLanguage('dockerfile',   require('highlight.js/lib/languages/dockerfile'));
    hljs.registerLanguage('ruby',         require('highlight.js/lib/languages/ruby'));
    hljs.registerLanguage('fortran',      require('highlight.js/lib/languages/fortran'));
    hljs.registerLanguage('glsl',         require('highlight.js/lib/languages/glsl'));
    hljs.registerLanguage('go',           require('highlight.js/lib/languages/go'));
    hljs.registerLanguage('groovy',       require('highlight.js/lib/languages/groovy'));
    hljs.registerLanguage('handlebars',   require('highlight.js/lib/languages/handlebars'));
    hljs.registerLanguage('haskell',      require('highlight.js/lib/languages/haskell'));
    hljs.registerLanguage('ini',          require('highlight.js/lib/languages/ini'));
    hljs.registerLanguage('java',         require('highlight.js/lib/languages/java'));
    hljs.registerLanguage('javascript',   require('highlight.js/lib/languages/javascript'));
    hljs.registerLanguage('json',         require('highlight.js/lib/languages/json'));
    hljs.registerLanguage('less',         require('highlight.js/lib/languages/less'));
    hljs.registerLanguage('lisp',         require('highlight.js/lib/languages/lisp'));
    hljs.registerLanguage('livescript',   require('highlight.js/lib/languages/livescript'));
    hljs.registerLanguage('lua',          require('highlight.js/lib/languages/lua'));
    hljs.registerLanguage('makefile',     require('highlight.js/lib/languages/makefile'));
    hljs.registerLanguage('matlab',       require('highlight.js/lib/languages/matlab'));
    hljs.registerLanguage('mipsasm',      require('highlight.js/lib/languages/mipsasm'));
    hljs.registerLanguage('perl',         require('highlight.js/lib/languages/perl'));
    hljs.registerLanguage('nginx',        require('highlight.js/lib/languages/nginx'));
    hljs.registerLanguage('objectivec',   require('highlight.js/lib/languages/objectivec'));
    hljs.registerLanguage('php',          require('highlight.js/lib/languages/php'));
    hljs.registerLanguage('python',       require('highlight.js/lib/languages/python'));
    hljs.registerLanguage('rust',         require('highlight.js/lib/languages/rust'));
    hljs.registerLanguage('scala',        require('highlight.js/lib/languages/scala'));
    hljs.registerLanguage('scheme',       require('highlight.js/lib/languages/scheme'));<commit_msg>Fixed highlight.js: "arduino" depends on "cpp" and should be loaded later<commit_after>// Code block parser plugin
//

'use strict';


var hljs = require('highlight.js/lib/highlight.js');


module.exports = function () {

  return function (parser) {
    parser.md.enable([ 'fence', 'code', 'backticks' ]);

    hljs.registerLanguage('actionscript', require('highlight.js/lib/languages/actionscript'));
    hljs.registerLanguage('apache',       require('highlight.js/lib/languages/apache'));
    hljs.registerLanguage('armasm',       require('highlight.js/lib/languages/armasm'));
    hljs.registerLanguage('xml',          require('highlight.js/lib/languages/xml'));
    hljs.registerLanguage('asciidoc',     require('highlight.js/lib//languages/asciidoc'));
    hljs.registerLanguage('avrasm',       require('highlight.js/lib/languages/avrasm'));
    hljs.registerLanguage('bash',         require('highlight.js/lib/languages/bash'));
    hljs.registerLanguage('clojure',      require('highlight.js/lib/languages/clojure'));
    hljs.registerLanguage('cmake',        require('highlight.js/lib/languages/cmake'));
    hljs.registerLanguage('coffeescript', require('highlight.js/lib/languages/coffeescript'));
    hljs.registerLanguage('cpp',          require('highlight.js/lib/languages/cpp'));
    hljs.registerLanguage('arduino',      require('highlight.js/lib/languages/arduino'));
    hljs.registerLanguage('css',          require('highlight.js/lib/languages/css'));
    hljs.registerLanguage('diff',         require('highlight.js/lib/languages/diff'));
    hljs.registerLanguage('django',       require('highlight.js/lib/languages/django'));
    hljs.registerLanguage('dockerfile',   require('highlight.js/lib/languages/dockerfile'));
    hljs.registerLanguage('ruby',         require('highlight.js/lib/languages/ruby'));
    hljs.registerLanguage('fortran',      require('highlight.js/lib/languages/fortran'));
    hljs.registerLanguage('glsl',         require('highlight.js/lib/languages/glsl'));
    hljs.registerLanguage('go',           require('highlight.js/lib/languages/go'));
    hljs.registerLanguage('groovy',       require('highlight.js/lib/languages/groovy'));
    hljs.registerLanguage('handlebars',   require('highlight.js/lib/languages/handlebars'));
    hljs.registerLanguage('haskell',      require('highlight.js/lib/languages/haskell'));
    hljs.registerLanguage('ini',          require('highlight.js/lib/languages/ini'));
    hljs.registerLanguage('java',         require('highlight.js/lib/languages/java'));
    hljs.registerLanguage('javascript',   require('highlight.js/lib/languages/javascript'));
    hljs.registerLanguage('json',         require('highlight.js/lib/languages/json'));
    hljs.registerLanguage('less',         require('highlight.js/lib/languages/less'));
    hljs.registerLanguage('lisp',         require('highlight.js/lib/languages/lisp'));
    hljs.registerLanguage('livescript',   require('highlight.js/lib/languages/livescript'));
    hljs.registerLanguage('lua',          require('highlight.js/lib/languages/lua'));
    hljs.registerLanguage('makefile',     require('highlight.js/lib/languages/makefile'));
    hljs.registerLanguage('matlab',       require('highlight.js/lib/languages/matlab'));
    hljs.registerLanguage('mipsasm',      require('highlight.js/lib/languages/mipsasm'));
    hljs.registerLanguage('perl',         require('highlight.js/lib/languages/perl'));
    hljs.registerLanguage('nginx',        require('highlight.js/lib/languages/nginx'));
    hljs.registerLanguage('objectivec',   require('highlight.js/lib/languages/objectivec'));
    hljs.registerLanguage('php',          require('highlight.js/lib/languages/php'));
    hljs.registerLanguage('python',       require('highlight.js/lib/languages/python'));
    hljs.registerLanguage('rust',         require('highlight.js/lib/languages/rust'));
    hljs.registerLanguage('scala',        require('highlight.js/lib/languages/scala'));
    hljs.registerLanguage('scheme',       require('highlight.js/lib/languages/scheme'));
{'fraction_non_alphanumeric': 0.16874376869391824, 'fraction_numerical': 0.0, 'mean_word_length': 4.184108527131783, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1330196', 'n_tokens_mistral': 2518, 'n_tokens_neox': 2229, 'n_words': 215}