<commit_before>require 'digest/md5'
require 'uuidtools'

module Jerm
  class Populator

    MESSAGES={:exists=>"Already exists.",
      :no_project=>"Unable to determine SEEK project.",
      :no_uri=>"Location of resource missing.",
      :no_author=>"Unable to determine the SEEK person for the author.",
      :no_default_policy=>"Unable to determine the default policy for this project.",
      :no_title=>"Unable to correctly determine the title",
      :success=>"Successfully added."
    }

    RESPONSE_CODES={:exists=>1,
      :no_project=>2,
      :no_uri=>3,
      :no_author=>4,
      :no_default_policy=>5,
      :no_title=>6,
      :success=>0
    }

    #calls populate on an enumeration of resources, and returns an enumeration of responses
    def populate_collection resources
      res=[]
      resources.each do |r|
        res << populate(r)
      end
      return res
    end
    
    #adds a resource to the central SEEK archive, referenced by the remote URI, or creates new version if already exists.
    #returns a report:
    # {:response=>:success|:fail|:skipped,:message=>"",:exception=>Exception|nil,:resource=>resource}
    def populate resource      
      begin
        if resource.uri.blank?
          response={:response=>:fail,:message=>"No URL to data file described"}
        elsif !exists?(resource)
          response=add_as_new(resource)
        else
          response={:response=>:skipped,:message=>MESSAGES[:exists],:response_code=>RESPONSE_CODES[:exists]}
        end
      rescue Exception => exception
        response={:response=>:fail,:message=>"Something went wrong",:exception=>exception}
      end
      response[:resource]=resource
      response[:uuid]=UUIDTools::UUID.random_create.to_s
      return response
    end

    #checks whether the resource already exists in the registry
    def exists? resource

      exists = false
      
      #Check the URI doesn't exist
      if find_by_uri(resource.uri).nil?
        #Check the file doesn't exist
        #FIXME: Checks project here and again in the embeddedpopulator later on.
        project = Project.find(:first,:conditions=>['name = ?',resource.project]) #get project
        if project.nil?
          return false
        end
        project.decrypt_credentials
        downloader = DownloaderFactory.create resource.project
        data_hash = downloader.get_remote_data(resource.uri,project.site_username,project.site_password)
        unless data_hash.nil?
          digest = Digest::MD5.new
          digest << data_hash[:data]
          md5sum = digest.hexdigest
          exists = !ContentBlob.find(:first,:conditions=>{:md5sum=>md5sum}).nil?
        else
          return false
        end
      else
        exists = true
      end
      
      return exists
    end

  end
end
<commit_msg>handle expection when failing to check is jerm item already exists<commit_after>require 'digest/md5'
require 'uuidtools'

module Jerm
  class Populator

    MESSAGES={:exists=>"Already exists.",
      :no_project=>"Unable to determine SEEK project.",
      :no_uri=>"Location of resource missing.",
      :no_author=>"Unable to determine the SEEK person for the author.",
      :no_default_policy=>"Unable to determine the default policy for this project.",
      :no_title=>"Unable to correctly determine the title",
      :success=>"Successfully added."
    }

    RESPONSE_CODES={:exists=>1,
      :no_project=>2,
      :no_uri=>3,
      :no_author=>4,
      :no_default_policy=>5,
      :no_title=>6,
      :success=>0
    }

    #calls populate on an enumeration of resources, and returns an enumeration of responses
    def populate_collection resources
      res=[]
      resources.each do |r|
        res << populate(r)
      end
      return res
    end
    
    #adds a resource to the central SEEK archive, referenced by the remote URI, or creates new version if already exists.
    #returns a report:
    # {:response=>:success|:fail|:skipped,:message=>"",:exception=>Exception|nil,:resource=>resource}
    def populate resource      
      begin
        if resource.uri.blank?
          response={:response=>:fail,:message=>"No URL to data file described"}
        elsif !exists?(resource)
          response=add_as_new(resource)
        else
          response={:response=>:skipped,:message=>MESSAGES[:exists],:response_code=>RESPONSE_CODES[:exists]}
        end
      rescue Exception => exception
        response={:response=>:fail,:message=>"Something went wrong",:exception=>exception}
      end
      response[:resource]=resource
      response[:uuid]=UUIDTools::UUID.random_create.to_s
      return response
    end

    #checks whether the resource already exists in the registry
    def exists? resource

      exists = false
      
      #Check the URI doesn't exist
      if find_by_uri(resource.uri).nil?
        #Check the file doesn't exist
        #FIXME: Checks project here and again in the embeddedpopulator later on.
        project = Project.find(:first,:conditions=>['name = ?',resource.project]) #get project
        if project.nil?
          return false
        end
        project.decrypt_credentials
        downloader = DownloaderFactory.create resource.project
        begin
          data_hash = downloader.get_remote_data(resource.uri,project.site_username,project.site_password)
          
        rescue Exception=>e
          puts "Error fetching from :#{resource.uri} - #{e.message}"
          return true
        end
      
        unless data_hash.nil?
          digest = Digest::MD5.new
          digest << data_hash[:data]
          md5sum = digest.hexdigest
          exists = !ContentBlob.find(:first,:conditions=>{:md5sum=>md5sum}).nil?
        else
          return false
        end
      else
        exists = true
      end
      
      return exists
    end

  end
end

{'fraction_non_alphanumeric': 0.10037684138403563, 'fraction_numerical': 0.0041109969167523125, 'mean_word_length': 2.6608150470219436, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '76276', 'n_tokens_mistral': 1649, 'n_tokens_neox': 1613, 'n_words': 438}