<commit_before>"use strict";
var md = require("html-md");
var _ = require("lodash");
var re = "\\[([^\\[]+)\\]\\(([^\\)]+)\\)";

module.exports = function ($) {

  var options = {};
  var row = [];
  var markdown = md($("center .wikitable").html(), {
    inline: true
  });
  var markdownSplit = markdown.split("\n\n");
  var $headers = $("center .wikitable").find("tr").first().find("th");

  options.rowHeaders = [];
  generateHeaders($headers, options, $);

  var destinationsMarkdown = _.reduce(markdownSplit, function (destinations, block) {
    var lines = block.split("\n");
    var linksRe = new RegExp(re);

    if (lines.length === options.rowHeaders.length) {
      _.map(lines, function (line) {
        if (/#/.test(line)) {
          return;
        }
        if (linksRe.test(line)) {
          destinations.push(line);
        }
        getLinkStrings(line);
      });
    }
    return destinations;
  }, []);
  console.log("destinationsMarkdown: %o",destinationsMarkdown);
  // possible use match method stringmd.match(\[(\w*\s*\w*)*\])
  // $("center .wikitable").map(function () {

  //   var $headers = $(this).find("th");
  //   var $rowtable = $(this).find("tr");
  //   var options = {
  //     "defaultName": "",
  //     "defaultLink": "",
  //     "sharedAirport": 0,
  //     "numberMissingCells": 0,
  //     "rowSpanAttribute": 0,
  //     "lenghtRow": 0
  //   };

  //   for (var l = 1; l < $rowtable.length; l += 1) {
  //     var $rowTableContent = $($rowtable[l]).find("td");

  //     options.lenghtRow = $rowTableContent.length;
  //     if (options.lenghtRow < 2) {continue;}
  //     for (var m = 0; m < options.lenghtRow; m += 1) {
  //       options.textHeader = $($headers[m]).text().toLowerCase();

  //       options.textTableContent = $($rowTableContent[m]).text() || options.defaultName;
  //       options.linkTableContent = $($rowTableContent[m]).find("a[href^='/']").attr("href") || options.defaultLink;

  //       options.textHeader = filterTextHeader(options);
  //       options.rowSpanAttribute = $($rowTableContent[m]).attr("rowspan");

  //       options.rowNumber = row.length;

  //       checkRowSpan(options);<commit_msg>mkd notes<commit_after>"use strict";
var md = require("html-md");
var _ = require("lodash");
var re = "\\[([^\\[]+)\\]\\(([^\\)]+)\\)";

module.exports = function ($) {

  var options = {};
  var row = [];
  var markdown = md($("center .wikitable").html(), {
    inline: true
  });
  var markdownSplit = markdown.split("\n\n");
  var $headers = $("center .wikitable").find("tr").first().find("th");

  options.rowHeaders = [];
  generateHeaders($headers, options, $);

  var destinationsMarkdown = _.reduce(markdownSplit, function (destinations, block) {
    var lines = block.split("\n");
    var linksRe = new RegExp(re);

    if (lines.length === options.rowHeaders.length) {
      _.map(lines, function (line) {
        if (/#/.test(line)) {
          return;
        }
        if (linksRe.test(line)) {
          destinations.push(line);
        }
        getLinkStrings(line);
      });
    }
    return destinations;
  }, []);
  console.log("destinationsMarkdown: %o",destinationsMarkdown);
  // possible use match method stringmd.match(\[(\w*\s*\w*)*\])
  // var txt='[Saint Denis](/wiki/Saint-Denis,_RÃ©union "Saint-Denis, Réunion")';

  // var re1='(\\[.*?\\])';  // Square Braces 1

  // var p = new RegExp(re1,["i"]);
  // var m = p.exec(txt);
  // if (m != null)
  // {
  //     var sbraces1=m[1];
  //     document.write("("+sbraces1.replace(/</,"&lt;")+")"+"\n");
  // }
  // $("center .wikitable").map(function () {
  // var txt='[Saint Denis](/wiki/Saint-Denis,_RÃ©union "Saint-Denis, Réunion")';
  // This is for the link
  // var re1='.*?';  // Non-greedy match on filler
  // var re2='((?:\\/[\\w\\.\\-]+)+)'; // Unix Path 1
  // var re3='(.)';  // Any Single Character 1

  // var p = new RegExp(re1+re2+re3,["i"]);
  // var m = p.exec(txt);
  // if (m != null)
  // {
  //     var unixpath1=m[1];
  //     var c1=m[2];
  //     document.write("("+unixpath1.replace(/</,"&lt;")+")"+"("+c1.replace(/</,"&lt;")+")"+"\n");
  // }

  //   var $headers = $(this).find("th");
  //   var $rowtable = $(this).find("tr");
  //   var options = {
  //     "defaultName": "",
  //     "defaultLink": "",
  //     "sharedAirport": 0,
  //     "numberMissingCells": 0,
  //     "rowSpanAttribute": 0,
  //     "lenghtRow": 0
  //   };

  //   for (var l = 1; l < $rowtable.length; l += 1) {
  //     var $rowTableContent = $($rowtable[l]).find("td");

  //     options.lenghtRow = $rowTableContent.length;
  //     if (options.lenghtRow < 2) {continue;}
  //     for (var m = 0; m < options.lenghtRow; m += 1) {
  //       options.textHeader = $($headers[m]).text().toLowerCase();

  //       options.textTableContent = $($rowTableContent[m]).text() || options.defaultName;
  //       options.linkTableContent = $($rowTableContent[m]).find("a[href^='/']").attr("href") || options.defaultLink;

  //       options.textHeader = filterTextHeader(options);
  //       options.rowSpanAttribute = $($rowTableContent[m]).attr("rowspan");

  //       options.rowNumber = row.length;

  //       checkRowSpan(options);
{'fraction_non_alphanumeric': 0.2315242494226328, 'fraction_numerical': 0.007313317936874519, 'mean_word_length': 2.5474402730375427, 'pattern_counts': {'":': 12, '<': 12, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4193607', 'n_tokens_mistral': 1850, 'n_tokens_neox': 1785, 'n_words': 317}