<commit_before>import o from 'component-dom';
import debug from 'debug';

let log = debug('democracyos:body-serializer');

function divToObject (div, position) {
  var obj = {};
  if (div.attr('data-id')) {
    obj.id = div.attr('data-id');
    div.removeAttr('data-id');
  }

  obj.markup = div[0].outerHTML;
  obj.position = position;
  return obj;
}

function paragraphToHTML (paragraph, options) {
  var el = o(paragraph.markup);
  el.attr('data-id', paragraph.id);
  if ('object' === typeof options && options.className) {
    options.className
      .filter(cl => !!cl)
      .forEach(cl => el.addClass(cl));
  }
  if ('function' === typeof options) {
    options(el);
  }

  return el[0].outerHTML;
}

function mergeDuplicates (arr) {
  // If the array is empty, return it as is
  if (!arr.length) return arr;

  // The cleaned-up array
  var merged = [];

  // Push the first item
  var mergedIdx = 0;
  merged.push(arr[0]);

  for (var i = 1; i < arr.length; i++) {
    var current = arr[i];
    var previous = arr[i - 1];
    if (current.id === previous.id) {<commit_msg>[body-serializer] Changed data attribute to match the one SideComments needs #482<commit_after>import o from 'component-dom';
import debug from 'debug';

let log = debug('democracyos:body-serializer');
const DATA_ID_ATTR = 'data-section-id';

function divToObject (div, position) {
  let obj = {};
  let id = div.attr(DATA_ID_ATTR);
  if (id) {
    obj.id = id;
    div.removeAttr(DATA_ID_ATTR);
  }

  obj.markup = div[0].outerHTML;
  obj.position = position;
  return obj;
}

function paragraphToHTML (paragraph, options) {
  var el = o(paragraph.markup);
  el.attr(DATA_ID_ATTR, paragraph.id);
  if ('object' === typeof options && options.className) {
    options.className
      .filter(cl => !!cl)
      .forEach(cl => el.addClass(cl));
  }
  if ('function' === typeof options) {
    options(el);
  }

  return el[0].outerHTML;
}

function mergeDuplicates (arr) {
  // If the array is empty, return it as is
  if (!arr.length) return arr;

  // The cleaned-up array
  var merged = [];

  // Push the first item
  var mergedIdx = 0;
  merged.push(arr[0]);

  for (var i = 1; i < arr.length; i++) {
    var current = arr[i];
    var previous = arr[i - 1];
    if (current.id === previous.id) {
{'fraction_non_alphanumeric': 0.1585419411506368, 'fraction_numerical': 0.006587615283267457, 'mean_word_length': 3.1952117863720075, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '222092', 'n_tokens_mistral': 800, 'n_tokens_neox': 775, 'n_words': 216}