<commit_before>class Reporter::SdfExport
  def initialize
    @@sample_list = Array.new
  end

  def add_sample(sample)
    @@sample_list << sample
  end

  def generate_file(default_excluded_field, default_included_field, removed_field = [])
    return -1 if @@sample_list.empty? || @@sample_list.first == nil
    mdf_string = ""
    # We dont want to export Sample description
    default_excluded_field += ["description", "image", "molfile"]
    default_included_field = default_included_field.map! {|x|
      x.slice!("molecule.")
      x
    }
    included_field = @@sample_list.first.attribute_names
    # Exclude field
    included_field.delete_if { |x| default_excluded_field.include?(x) }
    included_field = included_field.reject { |x| x.empty? }
    included_field = included_field.uniq - removed_field

    @@sample_list.compact.each_with_index do |sample, index|
      sample_mdf = sample.molfile.split(/#|END/).first + "END\n"

      (default_included_field - removed_field).each do |field|
        value = sample.molecule.send(field).to_s || ""
        next if value.empty?

        value = validate_value(value)

        field_str = field.gsub(" ", "_").upcase

        field_mdf = ">  <#{field_str}> (#{sample.id})\n#{value}\n\n"

        sample_mdf = sample_mdf + field_mdf
      end
      included_field.each do |field|
        value = sample.send(field).to_s || ""
        next if value.empty?

        value = validate_value(value)

        field_str = field.gsub(" ", "_").upcase

        field_mdf = ">  <#{field_str}> (#{sample.id})\n#{value}\n\n"

        sample_mdf = sample_mdf + field_mdf
      end

      mdf_string = mdf_string + sample_mdf + "$$$$\n"
    end

    mdf_string
  end

  def validate_value value<commit_msg>rm trailing id from custom data header in sdfexp<commit_after>class Reporter::SdfExport
  def initialize
    @@sample_list = Array.new
  end

  def add_sample(sample)
    @@sample_list << sample
  end

  def generate_file(default_excluded_field, default_included_field, removed_field = [])
    return -1 if @@sample_list.empty? || @@sample_list.first == nil
    mdf_string = ""
    # We dont want to export Sample description
    default_excluded_field += ["description", "image", "molfile"]
    default_included_field = default_included_field.map! {|x|
      x.slice!("molecule.")
      x
    }
    included_field = @@sample_list.first.attribute_names
    # Exclude field
    included_field.delete_if { |x| default_excluded_field.include?(x) }
    included_field = included_field.reject { |x| x.empty? }
    included_field = included_field.uniq - removed_field

    @@sample_list.compact.each_with_index do |sample, index|
      sample_mdf = sample.molfile.split(/#|END/).first + "END\n"

      (default_included_field - removed_field).each do |field|
        value = sample.molecule.send(field).to_s || ""
        next if value.empty?

        value = validate_value(value)

        field_str = field.gsub(" ", "_").upcase

        field_mdf = ">  <#{field_str}>\n#{value}\n\n"

        sample_mdf = sample_mdf + field_mdf
      end
      included_field.each do |field|
        value = sample.send(field).to_s || ""
        next if value.empty?

        value = validate_value(value)

        field_str = field.gsub(" ", "_").upcase

        field_mdf = ">  <#{field_str}> (#{sample.id})\n#{value}\n\n"

        sample_mdf = sample_mdf + field_mdf
      end

      mdf_string = mdf_string + sample_mdf + "$$$$\n"
    end

    mdf_string
  end

  def validate_value value
{'fraction_non_alphanumeric': 0.12599318955732122, 'fraction_numerical': 0.0005675368898978433, 'mean_word_length': 2.9562289562289563, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6481014', 'n_tokens_mistral': 1206, 'n_tokens_neox': 1151, 'n_words': 228}