<commit_before>var PEG = require('pegjs'),
    fs = require('fs'),
    util = require('./util')

var parser;
var loaded = false; // singleton parser

var self = exports;

exports.walk = function(tree) {
  var res = "";
  for (var i=0;i<tree.length;i++) {
    if (tree[i].tag == "") {
      res += tree[i].content;
    } else {
      if (self.tags[tree[i].tag]) {
        var stage = self.tags[tree[i].tag](self.walk(tree[i].content), tree[i].attr);
        if (stage) {
          res += stage;
          continue;
        }
      }
      res += tree[i].raw + self.walk(tree[i].content) + '[/' + tree[i].tag + ']';
    }
  }
  return res;
}

exports.parse = function(input, callback) {
  if (!loaded) {
    fs.readFile('./bbcode.pegjs', 'utf8', function(err, data) {
      if (err) {
        return callback(err, null)
      }

      parser = PEG.buildParser(data);
      loaded = true;
      self.parse(input, callback)
    })
  } else {
    var tree = parser.parse(util.nl2br_escape(input));

    callback(null, self.walk(tree))
  }
}

exports.validURL = function(url) {
  return /^(https?|ftp):\/\/(((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:)*@)?(((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5]))|((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.?)(:\d*)?)(\/((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)+(\/(([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)*)*)?)?(\?((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|[\uE000-\uF8FF]|\/|\?)*)?(\#((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|\/|\?)*)?$/i.test(url);
}

/*
  if a tag doesn't match, return bool false.
*/
exports.tags = {
  "ol":function(content, attr) {
    return "<ol>" + content + "</ol>"
  },
  "ul":function(content, attr) {
    return "<ul>" + content + "</ul>"
  },
  "li":function(content, attr) {
    return "<li>" + content + "</li>"
  },<commit_msg>Fixed BBcode layout weirdness with newlines between list items, etc.<commit_after>var PEG = require('pegjs'),
    fs = require('fs'),
    util = require('./util')

var parser;
var loaded = false; // singleton parser

var self = exports;

exports.walk = function(tree, parent) {
  var res = "";
  for (var i=0;i<tree.length;i++) {
    if (tree[i].tag == "") {
      if (parent) {
        res += self.filter(parent, tree[i].content);
      } else {
        res += tree[i].content;
      }
    } else {
      if (self.tags[tree[i].tag]) {
        var stage = self.tags[tree[i].tag](self.walk(tree[i].content, tree[i].tag), tree[i].attr);
        if (stage) {
          res += stage;
          continue;
        }
      }
      res += tree[i].raw + self.walk(tree[i].content, null) + '[/' + tree[i].tag + ']';
    }
  }
  return res;
}

exports.filter = function(tag, content) {
  if (tag == "ol" || tag == "ul") {
    return ""; // nothing between <li>'s, including newlines
  }
  if (tag == "li") {
    return content.replace("<br />", "")
  }
  return content;
}

exports.parse = function(input, callback) {
  if (!loaded) {
    fs.readFile('./bbcode.pegjs', 'utf8', function(err, data) {
      if (err) {
        return callback(err, null)
      }

      parser = PEG.buildParser(data);
      loaded = true;
      self.parse(input, callback)
    })
  } else {
    var tree = parser.parse(util.nl2br_escape(input));

    callback(null, self.walk(tree, null))
  }
}

exports.validURL = function(url) {
  return /^(https?|ftp):\/\/(((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:)*@)?(((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5]))|((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.?)(:\d*)?)(\/((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)+(\/(([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)*)*)?)?(\?((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|[\uE000-\uF8FF]|\/|\?)*)?(\#((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!\$&'\(\)\*\+,;=]|:|@)|\/|\?)*)?$/i.test(url);
}

/*
  if a tag doesn't match, return bool false.
*/
exports.tags = {
  "ol":function(content, attr) {
    return "<ol>" + content + "</ol>"
  },
  "ul":function(content, attr) {
    return "<ul>" + content + "</ul>"
  },
  "li":function(content, attr) {
    return "<li>" + content + "</li>"
  },
{'fraction_non_alphanumeric': 0.34660714285714284, 'fraction_numerical': 0.055714285714285716, 'mean_word_length': 2.903135888501742, 'pattern_counts': {'":': 6, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4166689', 'n_tokens_mistral': 2965, 'n_tokens_neox': 2786, 'n_words': 246}