<commit_before>require 'user'
require 'fastercsv'
require 'pp'
# require 'ripl'
require 'rforce'

module GoodData
  module UserHierarchies
    class UserHierarchy
    
      self::USER_ID = "ID"
      self::MANAGER_ID = "MANAGERID"
      self::DESCRIBE = 'USERNAME'
    
      def self.grab(options)
        sf_module = options[:module] || fail("Specify SFDC module")
        fields = options[:fields]
        binding = options[:sfdc_connection]
        output = options[:output]


          fields = fields.split(', ') if fields.kind_of? String
          values = fields.map {|v| v.to_sym}

          query = "SELECT #{values.join(', ')} from #{sf_module}"
          # puts query
          answer = binding.query({:queryString => query})

          # output << values
          # pp answer
          answer[:queryResponse][:result][:records].each do |row|
            output << row.values_at(*values)
          end

          more_locator = answer[:queryResponse][:result][:queryLocator]

          while more_locator do
            answer_more = binding.queryMore({:queryLocator => more_locator})
            answer_more[:queryMoreResponse][:result][:records].each do |row|
              output << row.values_at(*values)
            end
            more_locator = answer_more[:queryMoreResponse][:result][:queryLocator]
          end
      end
    
      def self::load_from_sf(user, password, options={})
      # puts "self::load_from_sf"
        additional_fields = options[:additional_fields] || []
          rforce_connection = RForce::Binding.new 'https://www.salesforce.com/services/Soap/u/21.0'
          rforce_connection.login(user, password)

          output = []
          fields = [:Email, :Id, :ManagerId] + additional_fields
          grab({
            :module => 'User',
            :output => output,
            :fields => fields,
            :sfdc_connection => rforce_connection
          })

          users_data = {}
          output.each do |record|
            row = {}
            (fields.zip record).each {|pairs| row[pairs[0].to_s] = pairs[1]}
            # puts record[1]
            users_data[record[1]] = row
          end
          
          h = self.build_hierarchy(users_data, {
            :user_id => "Id",
            :manager_id => "ManagerId"
          })
          block_given?() ? yield(h) : h
      end
    
      def self::read_from_csv(filename, options = {})
        user_id_key = options[:user_id] || self::USER_ID
<commit_msg>Using salesforce gem now, added downloading of roles from SF<commit_after>require 'user'
require 'fastercsv'
require 'pp'
# require 'ripl'
require 'salesforce'

module GoodData
  module UserHierarchies
    class UserHierarchy
    
      self::USER_ID = "ID"
      self::MANAGER_ID = "MANAGERID"
      self::DESCRIBE = 'USERNAME'
=begin    
      def self.grab(options)
        sf_module = options[:module] || fail("Specify SFDC module")
        fields = options[:fields]
        binding = options[:sfdc_connection]
        output = options[:output]


          fields = fields.split(', ') if fields.kind_of? String
          values = fields.map {|v| v.to_sym}

          query = "SELECT #{values.join(', ')} from #{sf_module}"
          # puts query
          answer = binding.query({:queryString => query})

          # output << values
          # pp answer
          answer[:queryResponse][:result][:records].each do |row|
            output << row.values_at(*values)
          end

          more_locator = answer[:queryResponse][:result][:queryLocator]

          while more_locator do
            answer_more = binding.queryMore({:queryLocator => more_locator})
            answer_more[:queryMoreResponse][:result][:records].each do |row|
              output << row.values_at(*values)
            end
            more_locator = answer_more[:queryMoreResponse][:result][:queryLocator]
          end
      end
=end    

      def self::load_roles_from_sf(user, password)
        fields = [:Id,:ParentRoleId]
        data = self.load_from_sf(user, password, options={
          :module => "UserRole",
          :fields => fields
        })
        
        users_data = self.get_mapped_data(data, fields, :Id)
        
        h = self.build_hierarchy(users_data, {
        :user_id => "Id",
        :manager_id => "ParentRoleId"
        })
        block_given?() ? yield(h) : h
        
      end
      
      def self::load_users_from_sf(user, password)
        fields = [:Email,:Id,:ManagerId]
        data = self.load_from_sf(user, password, options={
          :module => "User",
          :fields => fields
        })
        
        users_data = self.get_mapped_data(data, fields, :Id)
        
        
        h = self.build_hierarchy(users_data, {
        :user_id => "Id",
        :manager_id => "ManagerId"
        })
        block_given?() ? yield(h) : h

      end
      
      def self::get_mapped_data(data, fields, id_field)
        user_data = {}
        data.each do |row|
          hash = {}
          fields.each {|field| hash[field] = row[field]}
          user_data[row[id_field]] = hash 
        end
        user_data
      end
      
      
      def self::load_from_sf(user, password, options={})
          client = Salesforce::Client.new(user, password)
          output = []
          client.grab({
            :module => options[:module],
            :output => output,
            :fields => options[:fields],
            :as_hash => true
          })
          output
=begin          
          rforce_connection = RForce::Binding.new 'https://www.salesforce.com/services/Soap/u/21.0'
          rforce_connection.login(user, password)

          output = []
          fields = [:Email, :Id, :ManagerId] + additional_fields
          grab({
            :module => 'User',
            :output => output,
            :fields => fields,
            :sfdc_connection => rforce_connection
          })

          users_data = {}
          output.each do |record|
            row = {}
            (fields.zip record).each {|pairs| row[pairs[0].to_s] = pairs[1]}
            # puts record[1]
            users_data[record[1]] = row
          end
          
          h = self.build_hierarchy(users_data, {
            :user_id => "Id",
            :manager_id => "ManagerId"
          })
          block_given?() ? yield(h) : h
=end
      end
    
      def self::read_from_csv(filename, options = {})
        user_id_key = options[:user_id] || self::USER_ID

{'fraction_non_alphanumeric': 0.125, 'fraction_numerical': 0.002165841584158416, 'mean_word_length': 1.9134745380802163, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 31, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4191473', 'n_tokens_mistral': 1937, 'n_tokens_neox': 1904, 'n_words': 377}