<commit_before>var fs = require('fs');
var markzero = require('markzero');
var manual = markzero.manual;
var marked = markzero.marked;
var Parser = markzero.Parser;
var ProgramRenderer = require('./renderer');

var options = new RegExp('^' + manual.OPTIONS + '$', 'i');
var commands = new RegExp('^' + manual.COMMANDS + '$', 'i');

function parse(markdown, def, callback) {
  // TODO: get lexer configuration
  var lexer = new marked.Lexer();
  var tokens = lexer.lex(markdown);
  //console.dir(tokens);
  // take name from h1
  if(tokens.length
    && tokens[0].type === 'heading'
    && tokens[0].depth === 1) {
    this.name(tokens[0].text);
    tokens.shift();
  }

  // remove subsequent heading
  // taking care not to remove specially
  // recognized headings
  if(tokens.length
    && tokens[0].type === 'heading'
    && tokens[0].depth === 2
    && !options.test(tokens[0].text)
    && !commands.test(tokens[0].text)) {
    tokens.shift();
  }

  // take description from first paragraph
  if(tokens.length
    && tokens[0].type === 'paragraph') {
    this.description(tokens[0].text);
    tokens.shift();
  }
  //console.dir(tokens);
  var renderer = new ProgramRenderer(null, this, def);
  var parser = new Parser({renderer: renderer});
  var md =  parser.parse(tokens);
  callback();
}

function loader(file, def, callback) {
  var scope = this;
  fs.exists(file, function(exists) {
    if(!exists) return callback(scope.errors.EEXIST, [file]);
    fs.readFile(file, function(err, contents) {
      if(err) return callback(err);
      contents = '' + contents;
      parse.call(scope, contents, def, callback);
    })
  })
}

module.exports = loader;<commit_msg>Gobble description paragraphs.<commit_after>var EOL = require('os').EOL;
var fs = require('fs');
var markzero = require('markzero');
var manual = markzero.manual;
var marked = markzero.marked;
var Parser = markzero.Parser;
var ProgramRenderer = require('./renderer');

var options = new RegExp('^' + manual.OPTIONS + '$', 'i');
var commands = new RegExp('^' + manual.COMMANDS + '$', 'i');

function parse(markdown, def, callback) {
  // TODO: get lexer configuration
  var lexer = new marked.Lexer();
  var tokens = lexer.lex(markdown);
  //console.dir(tokens);
  // take name from h1
  if(tokens.length
    && tokens[0].type === 'heading'
    && tokens[0].depth === 1) {
    this.name(tokens[0].text);
    tokens.shift();
  }

  // remove subsequent heading
  // taking care not to remove specially
  // recognized headings
  if(tokens.length
    && tokens[0].type === 'heading'
    && tokens[0].depth === 2
    && !options.test(tokens[0].text)
    && !commands.test(tokens[0].text)) {
    tokens.shift();
  }

  // take description from initial paragraph(s)
  var description = '';
  while(tokens.length
    && tokens[0].type === 'paragraph') {
    description +=  !description ? tokens[0].text  : EOL + EOL + tokens[0].text;
    tokens.shift();
  }
  if(description) this.description(description);
  //console.dir(tokens);
  var renderer = new ProgramRenderer(null, this, def);
  var parser = new Parser({renderer: renderer});
  var md =  parser.parse(tokens);
  callback();
}

function loader(file, def, callback) {
  var scope = this;
  fs.exists(file, function(exists) {
    if(!exists) return callback(scope.errors.EEXIST, [file]);
    fs.readFile(file, function(err, contents) {
      if(err) return callback(err);
      contents = '' + contents;
      parse.call(scope, contents, def, callback);
    })
  })
}

module.exports = loader;
{'fraction_non_alphanumeric': 0.17083570210346788, 'fraction_numerical': 0.007106310403638431, 'mean_word_length': 3.58203125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3263193', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1117, 'n_words': 271}