<commit_before>/*
* µmarkdown.js
* markdown in under 5kb
*
* Copyright 2013, <NAME> - http://simon.waldherr.eu/
* Released under the MIT Licence
* http://simon.waldherr.eu/license/mit/
*
* Github:  https://github.com/simonwaldherr/micromarkdown.js/
* Version: 0.1.5
*/

/*jslint browser: true, plusplus: true, indent: 2, regexp: true, ass: true */
/*global ActiveXObject */

var micromarkdown = {
  useajax: false,
  regexobject: {
    headline:   /^(\#{1,6})([^\#\n]+)\n/m,
    code:       /\s\`\`\`\n?([^`]+)\`\`\`/g,
    hr:         /\n(?:([\*\-_] ?)+)\1\1\n/gm,
    lists:      /^(( *(\*|\d\.) [^\n]+)\n)+/gm,
    bolditalic: /(?:([\*_~]{1,3}))([^\*_~\n]+[^\*_~\s])\1/g,
    links:      /!?\[([^\]<>]+)\]\(([^ \)<>]+)( "[^\(\)\"]+")?\)/g,
    reflinks:   /\[([^\]]+)\]\[([^\]]+)\]/g,
    mail:       /<(([a-z0-9_\-\.])+\@([a-z0-9_\-\.])+\.([a-z]{2,7}))>/gmi,
    tables:     /\n(([^|\n]+ *\| *)+([^|\n]+\n))(\-+\|)+(\-+\n)((([^|\n]+ *\| *)+([^|\n]+)\n)+)/g,
    include:    /[\[<]include (\S+) from (https?:\/\/[a-z0-9\.\-]+\.[a-z]{2,9}[a-z0-9\.\-\?\&\/]+)[\]>]/gi,
    url:        /<([a-zA-Z0-9@:%_\+.~#?&\/\/=]{2,256}\.[a-z]{2,4}\b(\/[\-a-zA-Z0-9@:%_\+.~#?&\/\/=]*)?)>/g
  },
  parse: function (str) {
    'use strict';
    var line, nstatus, status, helper, helper1, helper2, count, repstr, stra, trashgc = [], casca = 0, i = 0, j = 0;
    str = '\n' + str + '\n';

    /* code */
    while ((stra = micromarkdown.regexobject.code.exec(str)) !== null) {
      str = str.replace(stra[0], '<code>\n' + micromarkdown.htmlEncode(stra[1]).replace(/\n/gm, '<br/>').replace(/\ /gm, '&nbsp;') + '</code>\n');
    }

    /* headlines */
    while ((stra = micromarkdown.regexobject.headline.exec(str)) !== null) {
      count = stra[1].length;
      str = str.replace(stra[0], '<h' + count + '>' + stra[2] + '</h' + count + '>' + '\n');
    }

    /* horizontal line */
    while ((stra = micromarkdown.regexobject.hr.exec(str)) !== null) {
      str = str.replace(stra[0], '\n<hr/>\n');
    }

    /* lists */
    while ((stra = micromarkdown.regexobject.lists.exec(str)) !== null) {
      casca = 0;
      if (stra[0].trim().substr(0, 1) === '*') {
        repstr = '<ul>';
      } else {
        repstr = '<ol>';
      }
      helper = stra[0].split('\n');
      status = 0;
      for (i = 0; i < helper.length; i++) {
        if ((line = /^(( )*(\*|\d\.) ([^\n]+))/.exec(helper[i])) !== null) {
          if (line[2] === undefined) {
            nstatus = 0;
          } else {
            nstatus = line[2].length;
          }
          if (status > nstatus) {
            repstr += '</ul>';
            status = nstatus;
            casca--;
          }
          if (status < nstatus) {
            repstr += '<ul>';
            status = nstatus;
            casca++;
          }
          repstr += '<li>' + line[4] + '</li>' + '\n';
        }
      }
      while (casca > 0) {
        repstr += '</ul>';
        casca--;
      }
      if (stra[0].trim().substr(0, 1) === '*') {
        repstr += '</ul>';
      } else {
        repstr += '</ol>';
      }
      str = str.replace(stra[0], repstr + '\n');
    }

    /* bold and italic */
    for (i = 0; i < 3; i++) {
      while ((stra = micromarkdown.regexobject.bolditalic.exec(str)) !== null) {
        repstr = [];
        if (stra[1] === '~~') {
          str = str.replace(stra[0], '<del>' + stra[2] + '</del>');
        } else {
          switch (stra[1].length) {
          case 1:
            repstr = ['<i>', '</i>'];
            break;
          case 2:
            repstr = ['<b>', '</b>'];
            break;
          case 3:
            repstr = ['<i><b>', '</b></i>'];
            break;
          }
          str = str.replace(stra[0], repstr[0] + stra[2] + repstr[1]);
        }
      }
    }

    /* tables */
    while ((stra = micromarkdown.regexobject.tables.exec(str)) !== null) {
      repstr = '<table><tr>';
      helper = stra[1].split('|');
      for (i = 0; i < helper.length; i++) {
        repstr += '<th>' + helper[i] + '</th>';
      }
      repstr += '</tr>';
      helper1 = stra[6].split('\n');
      for (i = 0; i < helper1.length; i++) {
        helper2 = helper1[i].split('|');
        if (helper2[0].length !== 0) {
          repstr += '<tr>';
          for (j = 0; j < helper2.length; j++) {
            repstr += '<td>' + helper2[j] + '</td>';
          }
          repstr += '</tr>' + '\n';<commit_msg>added gfm format table alignment<commit_after>/*
* µmarkdown.js
* markdown in under 5kb
*
* Copyright 2013, <NAME> - http://simon.waldherr.eu/
* Released under the MIT Licence
* http://simon.waldherr.eu/license/mit/
*
* Github:  https://github.com/simonwaldherr/micromarkdown.js/
* Version: 0.1.5
*/

/*jslint browser: true, plusplus: true, indent: 2, regexp: true, ass: true */
/*global ActiveXObject */

var micromarkdown = {
  useajax: false,
  regexobject: {
    headline:   /^(\#{1,6})([^\#\n]+)\n/m,
    code:       /\s\`\`\`\n?([^`]+)\`\`\`/g,
    hr:         /\n(?:([\*\-_] ?)+)\1\1\n/gm,
    lists:      /^(( *(\*|\d\.) [^\n]+)\n)+/gm,
    bolditalic: /(?:([\*_~]{1,3}))([^\*_~\n]+[^\*_~\s])\1/g,
    links:      /!?\[([^\]<>]+)\]\(([^ \)<>]+)( "[^\(\)\"]+")?\)/g,
    reflinks:   /\[([^\]]+)\]\[([^\]]+)\]/g,
    mail:       /<(([a-z0-9_\-\.])+\@([a-z0-9_\-\.])+\.([a-z]{2,7}))>/gmi,
    tables:     /\n(([^|\n]+ *\| *)+([^|\n]+\n))((:?\-+:?\|)+(:?\-+:?)*\n)((([^|\n]+ *\| *)+([^|\n]+)\n)+)/g,
    include:    /[\[<]include (\S+) from (https?:\/\/[a-z0-9\.\-]+\.[a-z]{2,9}[a-z0-9\.\-\?\&\/]+)[\]>]/gi,
    url:        /<([a-zA-Z0-9@:%_\+.~#?&\/\/=]{2,256}\.[a-z]{2,4}\b(\/[\-a-zA-Z0-9@:%_\+.~#?&\/\/=]*)?)>/g
  },
  parse: function (str) {
    'use strict';
    var line, nstatus, status, cel, calign, helper, helper1, helper2, count, repstr, stra, trashgc = [], casca = 0, i = 0, j = 0;
    str = '\n' + str + '\n';

    /* code */
    while ((stra = micromarkdown.regexobject.code.exec(str)) !== null) {
      str = str.replace(stra[0], '<code>\n' + micromarkdown.htmlEncode(stra[1]).replace(/\n/gm, '<br/>').replace(/\ /gm, '&nbsp;') + '</code>\n');
    }

    /* headlines */
    while ((stra = micromarkdown.regexobject.headline.exec(str)) !== null) {
      count = stra[1].length;
      str = str.replace(stra[0], '<h' + count + '>' + stra[2] + '</h' + count + '>' + '\n');
    }

    /* horizontal line */
    while ((stra = micromarkdown.regexobject.hr.exec(str)) !== null) {
      str = str.replace(stra[0], '\n<hr/>\n');
    }

    /* lists */
    while ((stra = micromarkdown.regexobject.lists.exec(str)) !== null) {
      casca = 0;
      if (stra[0].trim().substr(0, 1) === '*') {
        repstr = '<ul>';
      } else {
        repstr = '<ol>';
      }
      helper = stra[0].split('\n');
      status = 0;
      for (i = 0; i < helper.length; i++) {
        if ((line = /^(( )*(\*|\d\.) ([^\n]+))/.exec(helper[i])) !== null) {
          if (line[2] === undefined) {
            nstatus = 0;
          } else {
            nstatus = line[2].length;
          }
          if (status > nstatus) {
            repstr += '</ul>';
            status = nstatus;
            casca--;
          }
          if (status < nstatus) {
            repstr += '<ul>';
            status = nstatus;
            casca++;
          }
          repstr += '<li>' + line[4] + '</li>' + '\n';
        }
      }
      while (casca > 0) {
        repstr += '</ul>';
        casca--;
      }
      if (stra[0].trim().substr(0, 1) === '*') {
        repstr += '</ul>';
      } else {
        repstr += '</ol>';
      }
      str = str.replace(stra[0], repstr + '\n');
    }

    /* bold and italic */
    for (i = 0; i < 3; i++) {
      while ((stra = micromarkdown.regexobject.bolditalic.exec(str)) !== null) {
        repstr = [];
        if (stra[1] === '~~') {
          str = str.replace(stra[0], '<del>' + stra[2] + '</del>');
        } else {
          switch (stra[1].length) {
          case 1:
            repstr = ['<i>', '</i>'];
            break;
          case 2:
            repstr = ['<b>', '</b>'];
            break;
          case 3:
            repstr = ['<i><b>', '</b></i>'];
            break;
          }
          str = str.replace(stra[0], repstr[0] + stra[2] + repstr[1]);
        }
      }
    }

    /* tables */
    while ((stra = micromarkdown.regexobject.tables.exec(str)) !== null) {
      repstr = '<table><tr>';
      helper = stra[1].split('|');
      calign = stra[4].split('|');
      for (i=0; i<helper.length; i++) {
        if (calign.length <= i) {
          calign.push(0)
        } else if (calign[i].trimRight().slice(-1) == ':') {
          if (calign[i][0] == ':') calign[i] = 3;
          else calign[i] = 2;
        } else {
          if (calign[i][0] == ':') calign[i] = 1;
          else calign[i] = 0;
        }
      }
      cel = ['<th>', '<th align="left">', '<th align="right">', '<th align="center">'];
      for (i = 0; i < helper.length; i++) {
        repstr += cel[calign[i]] + helper[i] + '</th>';
      }
      repstr += '</tr>';
      cel = ['<td>', '<td align="left">', '<td align="right">', '<td align="center">'];
      helper1 = stra[7].split('\n');
      for (i = 0; i < helper1.length; i++) {
        helper2 = helper1[i].split('|');
        if (helper2[0].length !== 0) {
          while (calign.length < helper2.length) calign.push(0);
          repstr += '<tr>';
          for (j = 0; j < helper2.length; j++) {
            repstr += cel[calign[j]] + helper2[j] + '</td>';
          }
          repstr += '</tr>' + '\n';
{'fraction_non_alphanumeric': 0.2851274489151043, 'fraction_numerical': 0.020644617653254687, 'mean_word_length': 1.7649970879440886, 'pattern_counts': {'":': 0, '<': 104, '<?xml version=': 0, '>': 93, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4392464', 'n_tokens_mistral': 3766, 'n_tokens_neox': 3648, 'n_words': 659}