<commit_before>
require 'simple-spreadsheet-extractor'

class DataFilesController < ApplicationController
  
  include IndexPager
  include SysMODB::SpreadsheetExtractor
  include MimeTypesHelper  
  include DotGenerator  
  include Seek::AssetsCommon

  #before_filter :login_required
  
  before_filter :find_assets, :only => [ :index ]
  before_filter :find_and_auth, :except => [ :index, :new, :upload_for_tool, :create, :request_resource, :preview, :test_asset_url, :update_tags_ajax]
  before_filter :find_display_data_file, :only=>[:show,:download]
    
  def new_version
    if (handle_data nil)          
      comments=params[:revision_comment]
      @data_file.content_blob = ContentBlob.new(:tmp_io_object => @tmp_io_object, :url=>@data_url)      
      @data_file.content_type = params[:data_file][:content_type]
      @data_file.original_filename=params[:data_file][:original_filename]
      factors = @data_file.studied_factors
      respond_to do |format|
        if @data_file.save_as_new_version(comments)
          #Duplicate studied factors
          factors.each do |f|
            new_f = f.clone
            new_f.data_file_version = @data_file.version
            new_f.save
          end
          flash[:notice]="New version uploaded - now on version #{@data_file.version}"
        else
          flash[:error]="Unable to save new version"          
        end
        format.html {redirect_to @data_file }
      end
    else
      flash[:error]=flash.now[:error]
      redirect_to @data_file
    end
  end
  
  # DELETE /models/1
  # DELETE /models/1.xml
  def destroy
    #FIXME: Double check auth is working for deletion. Also, maybe should only delete if not associated with any assays.
    @data_file.destroy
    
    respond_to do |format|
      format.html { redirect_to(data_files_path) }
      format.xml  { head :ok }
    end
  end
  
  def new
    @data_file = DataFile.new
    respond_to do |format|
      if current_user.person.member?
        format.html # new.html.erb
      else
        flash[:error] = "You are not authorized to upload new Data files. Only members of known projects, institutions or work groups are allowed to create new content."
        format.html { redirect_to data_files_path }
      end
    end
  end

  def upload_for_tool

    if handle_data

      _file = DataFile.new params[:data_file]
         b = ContentBlob.new :tmp_io_object => @tmp_io_object, :url=>@data_url
      Policy.new_for_upload_tool(@data_file, params[:recipient_id])

      if @data_file.save
        @data_file.creators = [current_user.person]

        #send email to the file uploader and receiver
        Mailer.deliver_file_uploaded(current_user,Person.find(params[:recipient_id]),@data_file,base_host)

        flash.now[:notice] ="Data file was successfully uploaded and saved." if flash.now[:notice].nil?
        render :text => flash.now[:notice]
      else
        errors = (@data_file.errors.map { |e| e.join(" ") }.join("\n"))
        render :text => errors, :status => 500
      end
    end
  end
  
  def create
    if handle_data
      
      @data_file = DataFile.new params[:data_file]
      @data_file.event_ids = params[:event_ids] || [] 
      @data_file.contributor=current_user
      @data_file.content_blob = ContentBlob.new :tmp_io_object => @tmp_io_object, :url=>@data_url


      update_tags @data_file

      assay_ids = params[:assay_ids] || []
      respond_to do |format|
        if @data_file.save
          # the Data file was saved successfully, now need to apply policy / permissions settings to it
          policy_err_msg = Policy.create_or_update_policy(@data_file, current_user, params)
          
          # update attributions
          Relationship.create_or_update_attributions(@data_file, params[:attributions])
          
          # update related publications
          Relationship.create_or_update_attributions(@data_file, params[:related_publication_ids].collect {|i| ["Publication", i.split(",").first]}.to_json, Relationship::RELATED_TO_PUBLICATION) unless params[:related_publication_ids].nil?
          
          #Add creators

          AssetsCreator.add_or_update_creator_list(@data_file, params[:creators])
          
          if policy_err_msg.blank?
            flash.now[:notice] = 'Data file was successfully uploaded and saved.' if flash.now[:notice].nil?
            format.html { redirect_to data_file_path(@data_file) }
          else
            flash[:notice] = "Data file was successfully created. However some problems occurred, please see these below.</br></br><span style='color: red;'>" + policy_err_msg + "</span>"
            format.html { redirect_to :controller => 'data_files', :id => @data_file, :action => "edit" }
          end

          assay_ids.each do |text|
            a_id, r_type = text.split(",")
            @assay = Assay.find(a_id)
            if @assay.can_edit? and AssayAsset.find_all_by_asset_id(@data_file.id, :conditions => ["assay_id = #{@assay.id}"]).empty?
              @assay.relate(@data_file, RelationshipType.find_by_title(r_type))
            end
          end
        else
          format.html {
            render :action => "new"
          }
        end
      end   
    end
  end
  
  def show
    # store timestamp of the previous last usage
    @last_used_before_now = @data_file.last_used_at
    
    # update timestamp in the current Data file record
    # (this will also trigger timestamp update in the corresponding Asset)
    @data_file.last_used_at = Time.now
    @data_file.save_without_timestamping
    
    respond_to do |format|
      format.html # show.html.erb
      format.xml
      format.svg { render :text=>to_svg(@data_file,params[:deep]=='true',@data_file)}
      format.dot { render :text=>to_dot(@data_file,params[:deep]=='true',@data_file)}
      format.png { render :text=>to_png(@data_file,params[:deep]=='true',@data_file)}
    end
  end
  
  def edit
    
  end
  
  def update
    # remove protected columns (including a "link" to content blob - actual data cannot be updated!)
    if params[:data_file]
      [:contributor_id, :contributor_type, :original_filename, :content_type, :content_blob_id, :created_at, :updated_at, :last_used_at].each do |column_name|
        params[:data_file].delete(column_name)
      end
      
      # update 'last_used_at' timestamp on the DataFile
      params[:data_file][:last_used_at] = Time.now
    end

    update_tags @data_file
    assay_ids = params[:assay_ids] || []
    respond_to do |format|
      data_file_params = params[:data_file]
      data_file_params[:event_ids] = params[:event_ids]
      if @data_file.update_attributes(data_file_params)
        # the Data file was updated successfully, now need to apply updated policy / permissions settings to it
        policy_err_msg = Policy.create_or_update_policy(@data_file, current_user, params)
        
        # update attributions
        Relationship.create_or_update_attributions(@data_file, params[:attributions])
        
        # update related publications        
        Relationship.create_or_update_attributions(@data_file, params[:related_publication_ids].collect {|i| ["Publication", i.split(",").first]}.to_json, Relationship::RELATED_TO_PUBLICATION) unless params[:related_publication_ids].nil?
        
        
        #update creators

        AssetsCreator.add_or_update_creator_list(@data_file, params[:creators])
        
        if policy_err_msg.blank?
          flash[:notice] = 'Data file metadata was successfully updated.'
          format.html { redirect_to data_file_path(@data_file) }
        else
          flash[:notice] = "Data file metadata was successfully updated. However some problems occurred, please see these below.</br></br><span style='color: red;'>" + policy_err_msg + "</span>"
          format.html { redirect_to :controller => 'data_files', :id => @data_file, :action => "edit" }
        end

        # Update new assay_asset
        a_ids = []
        assay_ids.each do |text|
          a_id, r_type = text.split(",")
          a_ids.push(a_id)
          @assay = Assay.find(a_id)
          if @assay.can_edit? and AssayAsset.find_all_by_asset_id(@data_file.id, :conditions => ["assay_id =  #{@assay.id}"]).empty?
            @assay.relate(@data_file, RelationshipType.find_by_title(r_type))
          end
        end

        #Destroy AssayAssets that aren't needed
        assay_assets = AssayAsset.find(:all, :conditions => ["asset_id = ? and asset_type = ?", @data_file.id, 'DataFile'])
        assay_assets.each do |assay_asset|
          if assay_asset.assay.can_edit? and !a_ids.include?(assay_asset.assay_id.to_s)<commit_msg>fixing accidental edit<commit_after>
require 'simple-spreadsheet-extractor'

class DataFilesController < ApplicationController
  
  include IndexPager
  include SysMODB::SpreadsheetExtractor
  include MimeTypesHelper  
  include DotGenerator  
  include Seek::AssetsCommon

  #before_filter :login_required
  
  before_filter :find_assets, :only => [ :index ]
  before_filter :find_and_auth, :except => [ :index, :new, :upload_for_tool, :create, :request_resource, :preview, :test_asset_url, :update_tags_ajax]
  before_filter :find_display_data_file, :only=>[:show,:download]
    
  def new_version
    if (handle_data nil)          
      comments=params[:revision_comment]
      @data_file.content_blob = ContentBlob.new(:tmp_io_object => @tmp_io_object, :url=>@data_url)      
      @data_file.content_type = params[:data_file][:content_type]
      @data_file.original_filename=params[:data_file][:original_filename]
      factors = @data_file.studied_factors
      respond_to do |format|
        if @data_file.save_as_new_version(comments)
          #Duplicate studied factors
          factors.each do |f|
            new_f = f.clone
            new_f.data_file_version = @data_file.version
            new_f.save
          end
          flash[:notice]="New version uploaded - now on version #{@data_file.version}"
        else
          flash[:error]="Unable to save new version"          
        end
        format.html {redirect_to @data_file }
      end
    else
      flash[:error]=flash.now[:error]
      redirect_to @data_file
    end
  end
  
  # DELETE /models/1
  # DELETE /models/1.xml
  def destroy
    #FIXME: Double check auth is working for deletion. Also, maybe should only delete if not associated with any assays.
    @data_file.destroy
    
    respond_to do |format|
      format.html { redirect_to(data_files_path) }
      format.xml  { head :ok }
    end
  end
  
  def new
    @data_file = DataFile.new
    respond_to do |format|
      if current_user.person.member?
        format.html # new.html.erb
      else
        flash[:error] = "You are not authorized to upload new Data files. Only members of known projects, institutions or work groups are allowed to create new content."
        format.html { redirect_to data_files_path }
      end
    end
  end

  def upload_for_tool

    if handle_data

      @data_file = DataFile.new params[:data_file]

      @data_file.contributor  = current_user
      @data_file.content_blob = ContentBlob.new :tmp_io_object => @tmp_io_object, :url=>@data_url
      Policy.new_for_upload_tool(@data_file, params[:recipient_id])

      if @data_file.save
        @data_file.creators = [current_user.person]

        #send email to the file uploader and receiver
        Mailer.deliver_file_uploaded(current_user,Person.find(params[:recipient_id]),@data_file,base_host)

        flash.now[:notice] ="Data file was successfully uploaded and saved." if flash.now[:notice].nil?
        render :text => flash.now[:notice]
      else
        errors = (@data_file.errors.map { |e| e.join(" ") }.join("\n"))
        render :text => errors, :status => 500
      end
    end
  end
  
  def create
    if handle_data
      
      @data_file = DataFile.new params[:data_file]
      @data_file.event_ids = params[:event_ids] || [] 
      @data_file.contributor=current_user
      @data_file.content_blob = ContentBlob.new :tmp_io_object => @tmp_io_object, :url=>@data_url

      update_tags @data_file

      assay_ids = params[:assay_ids] || []
      respond_to do |format|
        if @data_file.save
          # the Data file was saved successfully, now need to apply policy / permissions settings to it
          policy_err_msg = Policy.create_or_update_policy(@data_file, current_user, params)
          
          # update attributions
          Relationship.create_or_update_attributions(@data_file, params[:attributions])
          
          # update related publications
          Relationship.create_or_update_attributions(@data_file, params[:related_publication_ids].collect {|i| ["Publication", i.split(",").first]}.to_json, Relationship::RELATED_TO_PUBLICATION) unless params[:related_publication_ids].nil?
          
          #Add creators
          AssetsCreator.add_or_update_creator_list(@data_file, params[:creators])
          
          if policy_err_msg.blank?
            flash.now[:notice] = 'Data file was successfully uploaded and saved.' if flash.now[:notice].nil?
            format.html { redirect_to data_file_path(@data_file) }
          else
            flash[:notice] = "Data file was successfully created. However some problems occurred, please see these below.</br></br><span style='color: red;'>" + policy_err_msg + "</span>"
            format.html { redirect_to :controller => 'data_files', :id => @data_file, :action => "edit" }
          end

          assay_ids.each do |text|
            a_id, r_type = text.split(",")
            @assay = Assay.find(a_id)
            if @assay.can_edit? and AssayAsset.find_all_by_asset_id(@data_file.id, :conditions => ["assay_id = #{@assay.id}"]).empty?
              @assay.relate(@data_file, RelationshipType.find_by_title(r_type))
            end
          end
        else
          format.html {
            render :action => "new"
          }
        end
      end   
    end
  end
  
  def show
    # store timestamp of the previous last usage
    @last_used_before_now = @data_file.last_used_at
    
    # update timestamp in the current Data file record
    # (this will also trigger timestamp update in the corresponding Asset)
    @data_file.last_used_at = Time.now
    @data_file.save_without_timestamping
    
    respond_to do |format|
      format.html # show.html.erb
      format.xml
      format.svg { render :text=>to_svg(@data_file,params[:deep]=='true',@data_file)}
      format.dot { render :text=>to_dot(@data_file,params[:deep]=='true',@data_file)}
      format.png { render :text=>to_png(@data_file,params[:deep]=='true',@data_file)}
    end
  end
  
  def edit
    
  end
  
  def update
    # remove protected columns (including a "link" to content blob - actual data cannot be updated!)
    if params[:data_file]
      [:contributor_id, :contributor_type, :original_filename, :content_type, :content_blob_id, :created_at, :updated_at, :last_used_at].each do |column_name|
        params[:data_file].delete(column_name)
      end
      
      # update 'last_used_at' timestamp on the DataFile
      params[:data_file][:last_used_at] = Time.now
    end

    update_tags @data_file
    assay_ids = params[:assay_ids] || []
    respond_to do |format|
      data_file_params = params[:data_file]
      data_file_params[:event_ids] = params[:event_ids]
      if @data_file.update_attributes(data_file_params)
        # the Data file was updated successfully, now need to apply updated policy / permissions settings to it
        policy_err_msg = Policy.create_or_update_policy(@data_file, current_user, params)
        
        # update attributions
        Relationship.create_or_update_attributions(@data_file, params[:attributions])
        
        # update related publications        
        Relationship.create_or_update_attributions(@data_file, params[:related_publication_ids].collect {|i| ["Publication", i.split(",").first]}.to_json, Relationship::RELATED_TO_PUBLICATION) unless params[:related_publication_ids].nil?
        
        
        #update creators
        AssetsCreator.add_or_update_creator_list(@data_file, params[:creators])
        
        if policy_err_msg.blank?
          flash[:notice] = 'Data file metadata was successfully updated.'
          format.html { redirect_to data_file_path(@data_file) }
        else
          flash[:notice] = "Data file metadata was successfully updated. However some problems occurred, please see these below.</br></br><span style='color: red;'>" + policy_err_msg + "</span>"
          format.html { redirect_to :controller => 'data_files', :id => @data_file, :action => "edit" }
        end

        # Update new assay_asset
        a_ids = []
        assay_ids.each do |text|
          a_id, r_type = text.split(",")
          a_ids.push(a_id)
          @assay = Assay.find(a_id)
          if @assay.can_edit? and AssayAsset.find_all_by_asset_id(@data_file.id, :conditions => ["assay_id =  #{@assay.id}"]).empty?
            @assay.relate(@data_file, RelationshipType.find_by_title(r_type))
          end
        end

        #Destroy AssayAssets that aren't needed
        assay_assets = AssayAsset.find(:all, :conditions => ["asset_id = ? and asset_type = ?", @data_file.id, 'DataFile'])
        assay_assets.each do |assay_asset|
          if assay_asset.assay.can_edit? and !a_ids.include?(assay_asset.assay_id.to_s)
{'fraction_non_alphanumeric': 0.10317095588235294, 'fraction_numerical': 0.0005744485294117647, 'mean_word_length': 2.905114401076716, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3734358', 'n_tokens_mistral': 5387, 'n_tokens_neox': 5221, 'n_words': 1177}