<commit_before>/*

Use Commonmark to parse *.md files and make them available via grunt.data.markdown

*/

var path = require("path");
var typo = require("typogr");
var stmd = require("commonmark");
var writer = new stmd.HtmlRenderer();
var reader = new stmd.Parser();

//monkey-patch writer to handle typographical entities
var escape = writer.escape;
writer.escape = function(str) {
  return escape(str, true);
};

module.exports = function(grunt) {

  grunt.registerTask("markdown", "Renders .md to HTML on grunt.data.markdown", function() {

    grunt.task.requires("state");

    //ignore markdown files inside the JS folder that come from Bower or libraries
    var files = grunt.file.expand("src/**/*.md", "!src/js/**/*.md");
    grunt.data.markdown = {};

    files.forEach(function(filename) {
      var input = grunt.file.read(filename);

      //run this through the template system
      input = grunt.template.process(input);

      var parsed = reader.parse(input);

      var walker = parsed.walker();
      //merge text nodes together
      var e;
      var previous;
      while (e = walker.next()) {
        var node = e.node;
        //is this an adjacent text node?
        if (node && previous && previous.parent == node.parent && previous.type == "Text" && node.type == "Text") {
          previous.literal += node.literal;
          // grunt.log.oklns(previous.literal);
          node.unlink();
        } else {
          previous = node;
        }
      }
      //second pass, run Typogr on the text
      walker = parsed.walker();
      while (e = walker.next()) {
        if (e.node && e.node.type == "Text") {
          e.node.literal = typo.smartypants(e.node.literal);
        }
      }


      var output = writer.render(parsed);
      //strip HTML block hack
      output = output.replace(/\<\?|\?\>/g, "");
      
      var sansExtension = path.basename(filename).replace(/\..*?$/, "");
      grunt.data.markdown[sansExtension] = output;
    });

  });

};<commit_msg>Handle markdown templates at runtime<commit_after>/*

Use Commonmark to parse *.md files and make them available via grunt.data.markdown

*/

var path = require("path");
var typo = require("typogr");
var stmd = require("commonmark");
var writer = new stmd.HtmlRenderer();
var reader = new stmd.Parser();

//monkey-patch writer to handle typographical entities
var escape = writer.escape;
writer.escape = function(str) {
  return escape(str, true);
};

module.exports = function(grunt) {

  grunt.registerTask("markdown", "Renders .md to HTML on grunt.data.markdown", function() {

    grunt.task.requires("state");

    //ignore markdown files inside the JS folder that come from Bower or libraries
    var files = grunt.file.expand("src/**/*.md", "!src/js/**/*.md");
    grunt.data.markdown = {};

    files.forEach(function(filename) {
      var input = grunt.file.read(filename);
      var sansExtension = path.basename(filename).replace(/\..*?$/, "");

      //define these as getters, so that templating can be processed out-of-order at runtime

      Object.defineProperty(grunt.data.markdown, sansExtension, {
        get: function() {
          //run this through the template system
          input = grunt.template.process(input);

          var parsed = reader.parse(input);

          var walker = parsed.walker();
          //merge text nodes together
          var e;
          var previous;
          while (e = walker.next()) {
            var node = e.node;
            //is this an adjacent text node?
            if (node && previous && previous.parent == node.parent && previous.type == "Text" && node.type == "Text") {
              previous.literal += node.literal;
              // grunt.log.oklns(previous.literal);
              node.unlink();
            } else {
              previous = node;
            }
          }
          //second pass, run Typogr on the text
          walker = parsed.walker();
          while (e = walker.next()) {
            if (e.node && e.node.type == "Text") {
              e.node.literal = typo.smartypants(e.node.literal);
            }
          }


          var output = writer.render(parsed);
          //strip HTML block hack
          output = output.replace(/\<\?|\?\>/g, "");
          return output;
        }
      });


    });

    console.log(grunt.data.markdown);

  });

};
{'fraction_non_alphanumeric': 0.14548379682831533, 'fraction_numerical': 0.0, 'mean_word_length': 2.245339299030574, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1321316', 'n_tokens_mistral': 1293, 'n_tokens_neox': 1250, 'n_words': 340}