<commit_before>const transferAttributes = (src, dest) => {
    for(let i = 0; i < src.attributes.length; i++) {
        let attr = src.attributes[i];
        dest.setAttribute(attr.name, attr.value);
    }
};

const replaceTag = (el, newTag) => {
  let newEl = document.createElement(newTag);
  transferAttributes(el, newEl);
  newEl.innerHTML = el.innerHTML;
  el.parentNode.replaceChild(newEl, el);
};

const wrap = (el, wrapperTag) => {
  let wrapper = document.createElement(wrapperTag);
  el.parentNode.insertBefore(wrapper, el);
  wrapper.appendChild(el);
};

const unwrap = (el) => {
  let parent = el.parentNode;
  while (el.firstChild) parent.insertBefore(el.firstChild, el);
  parent.removeChild(el);
};

export const unwrapUndesiredTags = (doc) => {
  doc.querySelectorAll("article, section, aside").forEach(unwrap);
  return doc;
};

export const emptyULToP = (doc) => {
  Array.from(doc.querySelectorAll("ul"))
       .filter((ul) => ul.querySelector(":not(li)") || ul.children.length == 0)
    .forEach(ul => replaceTag(ul, "p"));
  return doc;
};

export const wrapBareInlineTags = (doc) => {
  doc.querySelectorAll("body > :not(p):not(center):not(blockquote):not(article)")
      .forEach(el => wrap(el, "p"));
  return doc;
};

export const removeEmptyNodes = (doc) => {
  while(doc.querySelectorAll(":empty").length){
    doc.querySelectorAll(":empty").forEach(el => el.remove());
  };
  return doc;
};

const BLOCK_LEVEL_ELEMENTS = [
  "ADDRESS",
  "ARTICLE",
  "ASIDE",
  "BLOCKQUOTE",
  "DETAILS",
  "DIALOG",
  "DD",
  "DIV",
  "DL",
  "DT",
  "FIELDSET",
  "FIGCAPTION",
  "FIGURE",
  "FOOTER",
  "FORM",
  "H6",
  "HEADER",
  "HGROUP",
  "HR",
  "LI",<commit_msg>Add missing H elements to block list<commit_after>const transferAttributes = (src, dest) => {
    for(let i = 0; i < src.attributes.length; i++) {
        let attr = src.attributes[i];
        dest.setAttribute(attr.name, attr.value);
    }
};

const replaceTag = (el, newTag) => {
  let newEl = document.createElement(newTag);
  transferAttributes(el, newEl);
  newEl.innerHTML = el.innerHTML;
  el.parentNode.replaceChild(newEl, el);
};

const wrap = (el, wrapperTag) => {
  let wrapper = document.createElement(wrapperTag);
  el.parentNode.insertBefore(wrapper, el);
  wrapper.appendChild(el);
};

const unwrap = (el) => {
  let parent = el.parentNode;
  while (el.firstChild) parent.insertBefore(el.firstChild, el);
  parent.removeChild(el);
};

export const unwrapUndesiredTags = (doc) => {
  doc.querySelectorAll("article, section, aside").forEach(unwrap);
  return doc;
};

export const emptyULToP = (doc) => {
  Array.from(doc.querySelectorAll("ul"))
       .filter((ul) => ul.querySelector(":not(li)") || ul.children.length == 0)
    .forEach(ul => replaceTag(ul, "p"));
  return doc;
};

export const wrapBareInlineTags = (doc) => {
  doc.querySelectorAll("body > :not(p):not(center):not(blockquote):not(article)")
      .forEach(el => wrap(el, "p"));
  return doc;
};

export const removeEmptyNodes = (doc) => {
  while(doc.querySelectorAll(":empty").length){
    doc.querySelectorAll(":empty").forEach(el => el.remove());
  };
  return doc;
};

const BLOCK_LEVEL_ELEMENTS = [
  "ADDRESS",
  "ARTICLE",
  "ASIDE",
  "BLOCKQUOTE",
  "DETAILS",
  "DIALOG",
  "DD",
  "DIV",
  "DL",
  "DT",
  "FIELDSET",
  "FIGCAPTION",
  "FIGURE",
  "FOOTER",
  "FORM",
  "H1",
  "H2",
  "H3",
  "H4",
  "H5",
  "H6",
  "HEADER",
  "HGROUP",
  "HR",
  "LI",
{'fraction_non_alphanumeric': 0.1888274658132092, 'fraction_numerical': 0.003200465522257783, 'mean_word_length': 4.026315789473684, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6520226', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1258, 'n_words': 238}