<commit_before>/*
  Normilize Footnotes contents.
  Removes everything except textual paragraphs from Footnotes.
*/
export default class NormalizeFn {

  import(dom) {
    let fns = dom.findAll('fn')
    fns.forEach(fn => {
      let paragraphs = []
      let ptags = fn.findAll('p')
      ptags.forEach(p => {
        if(p.children.length === 0) {
          paragraphs.push(p.getTextContent())
        }
      })

      fn.children.forEach(child => {
        fn.removeChild(child)
      })
      paragraphs.forEach(text => {
        fn.append(
          dom.createElement('p').append(text)
        )
      })
    })
  }

  export() {
    // nothing
  }
}<commit_msg>Make transformer more robust, was loosing annotations.<commit_after>/*
  Normalize footnotes contents.
  Removes everything except textual paragraphs from footnotes.
*/
export default class NormalizeFn {

  import(dom) {
    let fns = dom.findAll('fn')
    fns.forEach(fn => {
      // Find all ptags that are nested in another p tag
      let ptags = fn.findAll('p p')
      // If any nested paragraphs are found we need to take action
      if (ptags.length > 0) {
        fn.empty()
        fn.append(ptags)
      }
    })
  }

  export() {
    // nothing
  }
}
{'fraction_non_alphanumeric': 0.12347684809098294, 'fraction_numerical': 0.0016246953696181965, 'mean_word_length': 2.142857142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1057951', 'n_tokens_mistral': 392, 'n_tokens_neox': 375, 'n_words': 94}