<commit_before>#module Task
#module PiplApi

def name
  "pipl_api"
end

def pretty_name
  "Search the Pipl database"
end

def description
  "Uses the Pipl API to search for information"
end

def allowed_types
  [ Entities::Username, 
    Entities::Person, 
    Entities::EmailAddress, 
    Entities::PhoneNumber ]
end

def setup(entity, options={})
  super(entity, options)
  @pipl_client = Client::Pipl::ApiClient.new
end

def run
  super

  if @entity.class == Entities::EmailAddress
    response = @pipl_client.search :email, @entity.name
  elsif @entity.class == Entities::PhoneNumber
    response = @pipl_client.search :phone, @entity.name
  elsif @entity.class == Entities::Username
    response = @pipl_client.search :username, @entity.name
  else 
    raise "Unknown Entity Type"
  end 

  # We need to make sure that we got a response
  # because pipl will just send us a false if we get
  # the key wrong (ie, the api key hasn't been configured)
  unless response
    @task_logger.error "Got no response from Pipl. Are you sure you've configured the key?"
    return 
  end

  if response['records']
    # Parse up the response records
    response['records'].each do |record|
      @task_logger.log "Record: #{record.to_s}\n"
      
      create_entity Entities::WebReference, { 
        :confidence => record['@query_person_match'],
        :uri => record['source']['url'],
        :domain => record['source']['domain'],
        :name => record['source']['name'],
        :category => record['source']['category'],
        :content => record['content'] ? record['content'].map{|x| x.to_s.join(" ")} : "" ,
        :source => "pipl record" }

      if record['usernames']
        record['usernames'].each do |username|
          create_entity Entities::Username, {:name => username['content'].downcase }
        end
      end
    end
  end

  if response['person']['sources']
    # Parse up the response sources
    response['person']['sources'].each do |source| 
      
      create_entity Entities::WebReference, { 
        :sponsored => source['@is_sponsored'],
        :uri => source['url'],
        :domain => source['domain'],
        :name => source['name'],
        :category => source['category'],
        :source => "pipl source" }

      @task_logger.log "Source: #{source}\n"
    end
  end

end

def cleanup
  super
end

#end
#end<commit_msg>add more entities to pipl_search<commit_after>#module Task
#module PiplApi

def name
  "pipl_api"
end

def pretty_name
  "Search the Pipl database"
end

def description
  "Uses the Pipl API to search for information"
end

def allowed_types
  [ Entities::Username, 
    Entities::Person, 
    Entities::FacebookAccount, 
    Entities::LinkedinAccount,
    Entities::TwitterAccount,
    Entities::FacebookAccount, 
    Entities::EmailAddress, 
    Entities::PhoneNumber ]
end

def setup(entity, options={})
  super(entity, options)
  @pipl_client = Client::Pipl::ApiClient.new
end

def run
  super

  if @entity.class == Entities::EmailAddress
    response = @pipl_client.search :email, @entity.name
  elsif @entity.class == Entities::PhoneNumber
    response = @pipl_client.search :phone, @entity.name
  elsif [ Entities::Username,
          Entities::FacebookAccount,
          Entities::TwitterAccount, 
          Entities::LinkedinAccount ].include? @entity.class
    response = @pipl_client.search :username, @entity.name
  else 
    raise "Unknown Entity Type"
  end 

  # We need to make sure that we got a response
  # because pipl will just send us a false if we get
  # the key wrong (ie, the api key hasn't been configured)
  unless response
    @task_logger.error "Got no response from Pipl. Are you sure you've configured the key?"
    return 
  end

  if response['records']
    # Parse up the response records
    response['records'].each do |record|
      @task_logger.log "Record: #{record.to_s}\n"
      
      create_entity Entities::WebReference, { 
        :confidence => record['@query_person_match'],
        :uri => record['source']['url'],
        :domain => record['source']['domain'],
        :name => record['source']['name'],
        :category => record['source']['category'],
        :content => record['content'] ? record['content'].map{|x| x.to_s.join(" ")} : "" ,
        :source => "pipl record" }

      if record['usernames']
        record['usernames'].each do |username|
          create_entity Entities::Username, {:name => username['content'].downcase }
        end
      end
    end
  end

  if response['person']['sources']
    # Parse up the response sources
    response['person']['sources'].each do |source| 
      
      create_entity Entities::WebReference, { 
        :sponsored => source['@is_sponsored'],
        :uri => source['url'],
        :domain => source['domain'],
        :name => source['name'],
        :category => source['category'],
        :source => "pipl source" }

      @task_logger.log "Source: #{source}\n"
    end
  end

end

def cleanup
  super
end

#end
#end
{'fraction_non_alphanumeric': 0.13153153153153152, 'fraction_numerical': 0.0, 'mean_word_length': 3.1529509559434747, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '219962', 'n_tokens_mistral': 1522, 'n_tokens_neox': 1463, 'n_words': 413}