<commit_before># frozen_string_literal: true

module ProtocolImporters
  class TablesBuilder
    def self.extract_tables_from_html_string(description_string, remove_first_column_row = false)
      tables = []

      doc = Nokogiri::HTML(description_string)
      tables_nodeset = doc.css('table')

      tables_nodeset.each do |table_node|
        rows_nodeset = table_node.css('tr')

        two_d_array = Array.new(rows_nodeset.count) { [] }

        rows_nodeset.each_with_index do |row, i|
          row.css('td').each_with_index do |cell, j|
            two_d_array[i][j] = cell.inner_html
          end
          two_d_array[i].shift if remove_first_column_row
        end
        two_d_array.shift if remove_first_column_row

        tables << Table.new(contents: { data: two_d_array }.to_json)
      end
      tables
    end

    def self.remove_tables_from_html(description_string)
      return '' if description_string.blank?

      doc = Nokogiri::HTML(description_string)
      doc.search('table').each do |t|
        t.swap('<br/><p><i>There was a table here, it was moved to tables section.</i></p>')
      end
      doc.css('body').first.inner_html
    end
  end
end<commit_msg>Add linebreak for 'There was a table' message<commit_after># frozen_string_literal: true

module ProtocolImporters
  class TablesBuilder
    def self.extract_tables_from_html_string(description_string, remove_first_column_row = false)
      tables = []

      doc = Nokogiri::HTML(description_string)
      tables_nodeset = doc.css('table')

      tables_nodeset.each do |table_node|
        rows_nodeset = table_node.css('tr')

        two_d_array = Array.new(rows_nodeset.count) { [] }

        rows_nodeset.each_with_index do |row, i|
          row.css('td').each_with_index do |cell, j|
            two_d_array[i][j] = cell.inner_html
          end
          two_d_array[i].shift if remove_first_column_row
        end
        two_d_array.shift if remove_first_column_row

        tables << Table.new(contents: { data: two_d_array }.to_json)
      end
      tables
    end

    def self.remove_tables_from_html(description_string)
      return '' if description_string.blank?

      doc = Nokogiri::HTML(description_string)
      doc.search('table').each do |t|
        t.swap('<br/><br/><p><i>There was a table here, it was moved to tables section.</i></p><br/><br/>')
      end
      doc.css('body').first.inner_html
    end
  end
end
{'fraction_non_alphanumeric': 0.10480887792848335, 'fraction_numerical': 0.0, 'mean_word_length': 2.8944, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7243204', 'n_tokens_mistral': 797, 'n_tokens_neox': 780, 'n_words': 156}