<commit_before>var Cheerio = require('cheerio'),
  marked = require('marked'),
  sanitizeHtml = require('sanitize-html');

var sanitize = function(text) {
  if (!text) return '';

  // Remove leading &nbsp; from html. (a side-effect of contenteditable is the leading &nbsp;)
  var strippedText = text.replace(/<p>&nbsp;|<p>&NBSP;/g, "<p>");

  var cleanText = sanitizeHtml(strippedText, {
    allowedTags: [ 'a', 'p' ],
    allowedAttributes: {
      'a': [ 'href', 'data-user-id' ]
    },

    // Removes empty paragraphs<commit_msg>allow line breaks in user-generated text<commit_after>var Cheerio = require('cheerio'),
  marked = require('marked'),
  sanitizeHtml = require('sanitize-html');

var sanitize = function(text) {
  if (!text) return '';

  // Remove leading &nbsp; from html. (a side-effect of contenteditable is the leading &nbsp;)
  var strippedText = text.replace(/<p>&nbsp;|<p>&NBSP;/g, "<p>");

  var cleanText = sanitizeHtml(strippedText, {
    allowedTags: ['a', 'p', 'br'],
    allowedAttributes: {
      'a': [ 'href', 'data-user-id' ]
    },

    // Removes empty paragraphs
{'fraction_non_alphanumeric': 0.17470427661510465, 'fraction_numerical': 0.0, 'mean_word_length': 3.9107142857142856, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5182412', 'n_tokens_mistral': 371, 'n_tokens_neox': 360, 'n_words': 91}