<commit_before>export default class TableConverter {
  get tagName () { return 'table' }

  get type () { return 'table' }

  import (el, node, importer) {
    const doc = importer.state.doc
    const $$ = doc.createElement.bind(doc)
    let rows = el.findAll('tr')
    let newRows = rows.map(tr => {
      return {
        id: tr.id,
        children: []
      }
    })
    for (let i = 0; i < rows.length; i++) {
      let tr = rows[i]
      let newRow = newRows[i]
      let children = tr.getChildren()
      for (let j = 0, k = 0; j < children.length; j++, k++) {
        // skipping spanned cells which is necessary
        // because HTML tables have a sparse representation w.r.t. span
        while (newRow.children[k]) k++
        let c = children[j]
        let attributes = {}
        if (c.is('th')) attributes.heading = true
        let rowspan = c.attr('rowspan')
        if (rowspan) {
          rowspan = Math.max(1, parseInt(rowspan, 10))
          if (rowspan > 1) {
            attributes.rowspan = String(rowspan)
          }
        }
        let colspan = c.attr('colspan')
        if (colspan) {
          colspan = Math.max(1, parseInt(colspan, 10))
          if (colspan > 1) {
            attributes.colspan = String(colspan)
          }
        }
        // flag all spanned cells so that we can skip them
        _fillSpanned($$, newRows, i, k, rowspan, colspan)
        let cell = $$('table-cell', { id: c.id })
        cell.attr(attributes)
        cell.content = importer.annotatedText(c, cell.getPath())
        newRows[i].children[k] = cell
      }
    }
    node._childNodes = newRows.map(data => {
      let row = $$('table-row', { id: data.id }).append(data.children)
      return row.id
    })
  }

  export (node, el, exporter) {
    const $$ = exporter.$$
    let htmlTable = $$('table').attr('id', node.id)
    let tbody = $$('tbody')
    let rows = node.findAll('table-row')
    let matrix = node.getCellMatrix()
    for (let i = 0; i < rows.length; i++) {
      let row = rows[i]
      let cells = matrix[i]
      let tr = $$('tr').attr('id', row.id)
      for (let j = 0; j < cells.length; j++) {
        let cell = cells[j]
        if (cell.shadowed) continue
        let el = $$(cell.attr('heading') ? 'th' : 'td')
        let attributes = { id: cell.id }
        let rowspan = cell.attr('rowspan')
        if (rowspan) {
          rowspan = Math.max(1, parseInt(rowspan, 10))
          if (rowspan > 1) {
            attributes.rowspan = rowspan
          }
        }
        let colspan = cell.attr('colspan')
        if (colspan) {
          colspan = Math.max(1, parseInt(colspan, 10))
          if (colspan > 1) {
            attributes.colspan = colspan
          }
        }
        el.attr(attributes)
        el.setInnerXML(cell.getInnerXML())
        tr.append(el)
      }
      tbody.append(tr)
    }
    htmlTable.append(tbody)<commit_msg>Export annotations in table cells.<commit_after>export default class TableConverter {
  get tagName () { return 'table' }

  get type () { return 'table' }

  import (el, node, importer) {
    const doc = importer.state.doc
    const $$ = doc.createElement.bind(doc)
    let rows = el.findAll('tr')
    let newRows = rows.map(tr => {
      return {
        id: tr.id,
        children: []
      }
    })
    for (let i = 0; i < rows.length; i++) {
      let tr = rows[i]
      let newRow = newRows[i]
      let children = tr.getChildren()
      for (let j = 0, k = 0; j < children.length; j++, k++) {
        // skipping spanned cells which is necessary
        // because HTML tables have a sparse representation w.r.t. span
        while (newRow.children[k]) k++
        let c = children[j]
        let attributes = {}
        if (c.is('th')) attributes.heading = true
        let rowspan = c.attr('rowspan')
        if (rowspan) {
          rowspan = Math.max(1, parseInt(rowspan, 10))
          if (rowspan > 1) {
            attributes.rowspan = String(rowspan)
          }
        }
        let colspan = c.attr('colspan')
        if (colspan) {
          colspan = Math.max(1, parseInt(colspan, 10))
          if (colspan > 1) {
            attributes.colspan = String(colspan)
          }
        }
        // flag all spanned cells so that we can skip them
        _fillSpanned($$, newRows, i, k, rowspan, colspan)
        let cell = $$('table-cell', { id: c.id })
        cell.attr(attributes)
        cell.content = importer.annotatedText(c, cell.getPath())
        newRows[i].children[k] = cell
      }
    }
    node._childNodes = newRows.map(data => {
      let row = $$('table-row', { id: data.id }).append(data.children)
      return row.id
    })
  }

  export (node, el, exporter) {
    const $$ = exporter.$$
    let htmlTable = $$('table').attr('id', node.id)
    let tbody = $$('tbody')
    let rows = node.findAll('table-row')
    let matrix = node.getCellMatrix()
    for (let i = 0; i < rows.length; i++) {
      let row = rows[i]
      let cells = matrix[i]
      let tr = $$('tr').attr('id', row.id)
      for (let j = 0; j < cells.length; j++) {
        let cell = cells[j]
        if (cell.shadowed) continue
        let el = $$(cell.attr('heading') ? 'th' : 'td')
        let attributes = { id: cell.id }
        let rowspan = cell.attr('rowspan')
        if (rowspan) {
          rowspan = Math.max(1, parseInt(rowspan, 10))
          if (rowspan > 1) {
            attributes.rowspan = rowspan
          }
        }
        let colspan = cell.attr('colspan')
        if (colspan) {
          colspan = Math.max(1, parseInt(colspan, 10))
          if (colspan > 1) {
            attributes.colspan = colspan
          }
        }
        el.attr(attributes)
        el.append(exporter.annotatedText(cell.getPath()))
        tr.append(el)
      }
      tbody.append(tr)
    }
    htmlTable.append(tbody)
{'fraction_non_alphanumeric': 0.14584763212079616, 'fraction_numerical': 0.007206588881262869, 'mean_word_length': 1.858754291319274, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4322253', 'n_tokens_mistral': 1910, 'n_tokens_neox': 1851, 'n_words': 483}