<commit_before>var assert = require('assert');

module.exports = function formatText(obj, options) {
  assert(obj.shift() == 'text');

  str = obj.map(formatBlock).join(',\n\n');
  if (str.length > 0) {
    str = ',\n\n' + str; // first comma after "text"
  }

  return '["text"' + str + '\n\n]\n';
};

var formatBlock = function (block) {
  var type = block.shift();
  var attr = block.shift();

  var header = JSON.stringify(type) + ', { "level": ' + attr.level + ' }';
  var str = block.map(formatSpan).join(',\n');
  if (str.length > 0) {
    str = ',\n' + str;
  }

  return '  [' + header + str + '\n  ]';
};

var formatSpan = function (span) {
  assert(span.length == 2);
  var type = span[0];
  var text = span[1];

  return '    [' + JSON.stringify(type) + ', ' + JSON.stringify(text) + ']';
};<commit_msg>Output all attributes in JSON formatter.<commit_after>var assert = require('assert');

module.exports = function formatText(obj, options) {
  assert(obj.shift() == 'text');

  str = obj.map(formatBlock).join(',\n\n');
  if (str.length > 0) {
    str = ',\n\n' + str; // first comma after "text"
  }

  return '["text"' + str + '\n\n]\n';
};

var formatBlock = function (block) {
  var type = block.shift();
  var attr = block.shift();

  var header = JSON.stringify(type) + ', ' + formatAttributes(attr);
  var str = block.map(formatSpan).join(',\n');
  if (str.length > 0) {
    str = ',\n' + str;
  }

  return '  [' + header + str + '\n  ]';
};

var formatSpan = function (span) {
  assert(span.length == 2);
  var type = span[0];
  var text = span[1];

  return '    [' + JSON.stringify(type) + ', ' + JSON.stringify(text) + ']';
};

var formatAttributes = function(attr) {
  return JSON.stringify(attr);
};
{'fraction_non_alphanumeric': 0.21436848203939746, 'fraction_numerical': 0.005793742757821553, 'mean_word_length': 3.1414868105515588, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2312392', 'n_tokens_mistral': 639, 'n_tokens_neox': 614, 'n_words': 145}