<commit_before>'use strict';

module.exports = function(grunt) {

    var metaRemarkable = require('meta-remarkable');

    var ejs = require('ejs');
    var path = require('path');
    var highlight = require('highlight.js');
    var _ = require('lodash');

    var md = new metaRemarkable('commonmark', {
        html: true,
        langPrefix: 'hljs ',
        highlight: function(code, lang) {
            return highlight.highlightAuto(code, [lang]).value;
        }
    });

    // converts [preview] and [pattern] tags to properly classed divs
    var replaceMarkup = function (text) {
        text = text.replace(/\[preview\]/g, '<div class="pl-preview"><div class="pl-toggle-code"><i class="icon icon-code" style="opacity: .5;"></i></div>\n');
        text = text.replace(/\[\/preview\]/g, '</div>\n');
        text = text.replace(/\[pattern\]/g, '<div class="pl-pattern">\n');
        text = text.replace(/\[\/pattern\]/g, '</div>\n');
        return text;
    };

    // pre-processes markdown text and converts ### to h3s with anchors in them.
    var linkifyHeadings = function (text) {
        var re = /^###\s?([^\#\n\r]+)/gm;
        var result;
        while ((result = re.exec(text)) !== null) {
            var escapedTitle = result[1].toLowerCase().replace(/[^\w]+/g, '-');
            text = text.replace(result[0], '<h3><a class="pl-heading-link" id="' + escapedTitle + '" href="#' + escapedTitle + '"><i class="icon icon-link"></i></a> ' + result[1] + '</h3>');
        }
        return text;
    };

    grunt.registerMultiTask('patterns', 'Gather and compile patterns.', function() {

        var options = this.options({
            patternRoot: 'content',
            urlRoot: ''
        });

        options.patternRoot = appendTrailingSlash(options.patternRoot);

        var tplString = grunt.file.read(options.template, 'utf8');

        // generate content map
        var contentMap = {};
        var nav = {};
        this.files.forEach(function (f) {

            var validFiles = removeInvalidFiles(f);
            var paths = createOutputPaths(f);<commit_msg>Trim whitespace in pattern headings<commit_after>'use strict';

module.exports = function(grunt) {

    var metaRemarkable = require('meta-remarkable');

    var ejs = require('ejs');
    var path = require('path');
    var highlight = require('highlight.js');
    var _ = require('lodash');

    var md = new metaRemarkable('commonmark', {
        html: true,
        langPrefix: 'hljs ',
        highlight: function(code, lang) {
            return highlight.highlightAuto(code, [lang]).value;
        }
    });

    // converts [preview] and [pattern] tags to properly classed divs
    var replaceMarkup = function (text) {
        text = text.replace(/\[preview\]/g, '<div class="pl-preview"><div class="pl-toggle-code"><i class="icon icon-code" style="opacity: .5;"></i></div>\n');
        text = text.replace(/\[\/preview\]/g, '</div>\n');
        text = text.replace(/\[pattern\]/g, '<div class="pl-pattern">\n');
        text = text.replace(/\[\/pattern\]/g, '</div>\n');
        return text;
    };

    // pre-processes markdown text and converts ### to h3s with anchors in them.
    var linkifyHeadings = function (text) {
        var re = /^###\s?([^\#\n\r]+)/gm;
        var result;
        while ((result = re.exec(text)) !== null) {
            var escapedTitle = result[1].toLowerCase().replace(/[^\w]+/g, '-');
            text = text.replace(result[0], '<h3><a class="pl-heading-link" id="' + escapedTitle + '" href="#' + escapedTitle + '"><i class="icon icon-link"></i></a>' + result[1].trim() + '</h3>');
        }
        return text;
    };

    grunt.registerMultiTask('patterns', 'Gather and compile patterns.', function() {

        var options = this.options({
            patternRoot: 'content',
            urlRoot: ''
        });

        options.patternRoot = appendTrailingSlash(options.patternRoot);

        var tplString = grunt.file.read(options.template, 'utf8');

        // generate content map
        var contentMap = {};
        var nav = {};
        this.files.forEach(function (f) {

            var validFiles = removeInvalidFiles(f);
            var paths = createOutputPaths(f);
{'fraction_non_alphanumeric': 0.18175355450236966, 'fraction_numerical': 0.0037914691943127963, 'mean_word_length': 2.468364831552999, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5478337', 'n_tokens_mistral': 1289, 'n_tokens_neox': 1245, 'n_words': 285}