<commit_before>module Seek
  #A mixin for DataFiles to handle aspects of data file extraction
  module DataFileExtraction
    include SpreadsheetUtil
    include Seek::MimeTypes
    include SysMODB::SpreadsheetExtractor

    #returns an instance of Seek::Treatment, populated according to the contents of the spreadsheet if it matches a known template
    def treatments
      begin
        if is_extractable_spreadsheet?
	  xml = spreadsheet_xml
          Seek::Treatments.new xml
        else
          Seek::Treatments.new
        end
      rescue Exception=>e
        Rails.logger.error("Error reading spreadsheet #{e.message}")
        Seek::Treatments.new
      end
    end

    def bio_samples_population institution_id, to_populate=true
      begin
        if is_extractable_spreadsheet?

          bio_samples = nil

          ActiveRecord::Base.transaction do
           begin
               bio_samples = Seek::BioSamples.new self, spreadsheet_xml, to_populate, institution_id
           rescue  Exception => e
             bio_samples = Seek::BioSamples.new self
             bio_samples.errors = "Error parsing spreadsheet: #{e.message}"
             #Rails.logger.error bio_samples.errors
             raise ActiveRecord::Rollback

           end
          end
          bio_samples
        else
         Seek::BioSamples.new self
        end
      rescue Exception => e
        Rails.logger.error("Error parsing spreadsheet #{e.message}")
        bio_samples = Seek::BioSamples.new self
        bio_samples.errors = "Error parsing spreadsheet: #{e.backtrace.join('<br/>')}"
        bio_samples
      end

    end



    #returns an array of all cell content within the workbook.
    def spreadsheet_contents_for_search obj=self
      if obj.is_extractable_spreadsheet? && obj.content_blob.file_exists?
        content = Rails.cache.fetch("#{obj.content_blob.cache_key}-ss-content-for-search") do
          begin
            xml=obj.spreadsheet_xml
            unless xml.nil?
              content = extract_content(xml)
              content = filter_content(content)
              content = humanize_content(content)
              content
            else
              []
            end
          rescue Exception=>e
            Rails.logger.error("Error processing spreadsheet for content_blob #{obj.content_blob_id} #{e}")
            raise e unless Rails.env=="production"
            nil
          end
        end
      else
        Rails.logger.error("Unable to find file contents for #{obj.class.name} #{obj.id}")
      end
      content || []
    end

    private<commit_msg>changed content_blob_id to content_blob.id, minor bugfix<commit_after>module Seek
  #A mixin for DataFiles to handle aspects of data file extraction
  module DataFileExtraction
    include SpreadsheetUtil
    include Seek::MimeTypes
    include SysMODB::SpreadsheetExtractor

    #returns an instance of Seek::Treatment, populated according to the contents of the spreadsheet if it matches a known template
    def treatments
      begin
        if is_extractable_spreadsheet?
	  xml = spreadsheet_xml
          Seek::Treatments.new xml
        else
          Seek::Treatments.new
        end
      rescue Exception=>e
        Rails.logger.error("Error reading spreadsheet #{e.message}")
        Seek::Treatments.new
      end
    end

    def bio_samples_population institution_id, to_populate=true
      begin
        if is_extractable_spreadsheet?

          bio_samples = nil

          ActiveRecord::Base.transaction do
           begin
               bio_samples = Seek::BioSamples.new self, spreadsheet_xml, to_populate, institution_id
           rescue  Exception => e
             bio_samples = Seek::BioSamples.new self
             bio_samples.errors = "Error parsing spreadsheet: #{e.message}"
             #Rails.logger.error bio_samples.errors
             raise ActiveRecord::Rollback

           end
          end
          bio_samples
        else
         Seek::BioSamples.new self
        end
      rescue Exception => e
        Rails.logger.error("Error parsing spreadsheet #{e.message}")
        bio_samples = Seek::BioSamples.new self
        bio_samples.errors = "Error parsing spreadsheet: #{e.backtrace.join('<br/>')}"
        bio_samples
      end

    end



    #returns an array of all cell content within the workbook.
    def spreadsheet_contents_for_search obj=self
      if obj.is_extractable_spreadsheet? && obj.content_blob.file_exists?
        content = Rails.cache.fetch("#{obj.content_blob.cache_key}-ss-content-for-search") do
          begin
            xml=obj.spreadsheet_xml
            unless xml.nil?
              content = extract_content(xml)
              content = filter_content(content)
              content = humanize_content(content)
              content
            else
              []
            end
          rescue Exception=>e
            Rails.logger.error("Error processing spreadsheet for content_blob #{obj.content_blob.id} #{e}")
            raise e unless Rails.env=="production"
            nil
          end
        end
      else
        Rails.logger.error("Unable to find file contents for #{obj.class.name} #{obj.id}")
      end
      content || []
    end

    private
{'fraction_non_alphanumeric': 0.07149666348903717, 'fraction_numerical': 0.0, 'mean_word_length': 2.1041420118343197, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2209043', 'n_tokens_mistral': 1428, 'n_tokens_neox': 1349, 'n_words': 364}