<commit_before>#
# Cookbook Name:: sc-mongodb
# Definition:: mongodb
#
# Copyright 2011, edelight GmbH
# Authors:
#       <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

require 'json'

class Chef::ResourceDefinitionList::MongoDB
  # node['mongodb']['config']['mongod']['net']['port'] required for port
  # if node['mongodb']['use_ip_address']
  #   node['ipaddress'] is required
  # else
  #    node['fqdn'] is required
  # end
  #
  # if node['fqnd'] is a vagrant host, ignore it
  # node['mongodb']['replica_priority'] is required
  #
  def self.create_replicaset_member(node)
    return {} if node['fqdn'] =~ /\.vagrantup\.com$/

    port = node['mongodb']['config']['mongod']['net']['port']
    host = node['mongodb']['use_ip_address'] ? node['ipaddress'] : node['fqdn']
    address = "#{host}:#{port}"

    member = {}
    member['host'] = address
    member['arbiterOnly'] = true if node['mongodb']['replica_arbiter_only']
    member['buildIndexes'] = false unless node['mongodb']['replica_build_indexes']
    member['hidden'] = true if node['mongodb']['replica_hidden']
    slave_delay = node['mongodb']['replica_slave_delay']
    member['slaveDelay'] = slave_delay if slave_delay > 0

    priority = if member['buildIndexes'] == false || member['hidden'] || member['slaveDelay']
                 0
               else
                 node['mongodb']['replica_priority']
               end
    member['priority'] = priority unless priority == 1
    tags = node['mongodb']['replica_tags'].to_hash
    member['tags'] = tags unless tags.empty?
    votes = node['mongodb']['replica_votes']
    member['votes'] = votes unless votes == 1

    member.freeze
  end

  def self.configure_replicaset(node, name, members)
    # lazy require, to move loading this modules to runtime of the cookbook
    require 'rubygems'
    require 'mongo'

    if members.empty? && Chef::Config[:solo]
      Chef::Log.warn('Cannot search for member nodes with chef-solo, defaulting to single node replica set')
    end

    mongo_host = 'localhost'
    mongo_port = node['mongodb']['config']['net']['port']

    begin
      connection = nil
      rescue_connection_failure do
        connection = Mongo::Connection.new(mongo_host, mongo_port, op_timeout: 5, slave_ok: true)
        connection.database_names # check connection
      end
    rescue => e
      Chef::Log.warn("Could not connect to database: '#{mongo_host}:#{mongo_port}', reason: #{e}")
      return
    end

    # Want the node originating the connection to be included in the replicaset
    members << node unless members.any? { |m| m.name == node.name }
    members.sort! { |x, y| x.name <=> y.name }

    rs_members = members.each_with_index.map do |member, n|
      create_replicaset_member(member).merge('_id' => n)
    end

    Chef::Log.info(
      "Configuring replicaset with members #{members.map { |n| n['hostname'] }.join(', ')}"
    )

    Chef::Log.debug(
      "Configuring replicaset with config: #{rs_members}"
    )

    admin = connection['admin']
    cmd = BSON::OrderedHash.new
    cmd['replSetInitiate'] = {
      '_id' => name,
      'members' => rs_members,
    }

    begin
      result = admin.command(cmd, check_response: false)
    rescue Mongo::OperationTimeout
      Chef::Log.info('Started configuring the replicaset, this will take some time, another run should run smoothly')
      return
    end
    if result.fetch('ok', nil) == 1
      # everything is fine, do nothing
    elsif result.fetch('errmsg', nil) =~ /(\S+) is already initiated/ || \
          result.fetch('errmsg', nil) == 'already initialized' || \
          result.fetch('errmsg', nil) =~ /is not empty on the initiating member/
      mongo_configured_host, mongo_configured_port = \
        Regexp.last_match.nil? || Regexp.last_match.length < 2 ? [mongo_host, mongo_port] : Regexp.last_match[1].split(':')
      begin
        connection = Mongo::Connection.new(mongo_configured_host, mongo_configured_port, op_timeout: 5, slave_ok: true)
      rescue
        abort("Could not connect to database: '#{mongo_host}:#{mongo_port}'")
      end

      # check if both configs are the same
      config = connection['local']['system']['replset'].find_one('_id' => name)
      if config && config['members'] == rs_members
        # config is up-to-date, do nothing
        Chef::Log.info("Replicaset '#{name}' already configured")
      elsif config['_id'] == name && config['members'] == rs_member_ips
        # config is up-to-date, but ips are used instead of hostnames, change config to hostnames
        Chef::Log.info("Need to convert ips to hostnames for replicaset '#{name}'")
        old_members = config['members'].map { |m| m['host'] }
        mapping = {}
        rs_member_ips.each do |mem_h|
          members.each do |n|
            ip, prt = mem_h['host'].split(':')
            mapping["#{ip}:#{prt}"] = "#{n['fqdn']}:#{prt}" if ip == n['ipaddress']
          end
        end
        config['members'].map! do |m|
          host = mapping[m['host']]
          { '_id' => m['_id'], 'host' => host }.merge(rs_options[host])
        end
        config['version'] += 1

        rs_connection = nil
        rescue_connection_failure do
          rs_connection = Mongo::ReplSetConnection.new(old_members)
          rs_connection.database_names # check connection
        end

        admin = rs_connection['admin']
        cmd = BSON::OrderedHash.new
        cmd['replSetReconfig'] = config
        result = nil
        begin
          result = admin.command(cmd, check_response: false)
        rescue Mongo::ConnectionFailure
          # reconfiguring destroys existing connections, reconnect
          connection = Mongo::Connection.new('localhost', node['mongodb']['config']['port'], op_timeout: 5, slave_ok: true)
          config = connection['local']['system']['replset'].find_one('_id' => name)
          # Validate configuration change
          if config['members'] == rs_members
            Chef::Log.info("New config successfully applied: #{config.inspect}")
          else
            Chef::Log.error("Failed to apply new config. Current config: #{config.inspect} Target config #{rs_members}")
            return
          end
        end
        Chef::Log.error("configuring replicaset returned: #{result.inspect}") unless result.fetch('errmsg', nil).nil?
      else
        Chef::Log.info 'going to update the members of the replicaset'
        old_members = config['members'].dup
        new_members = rs_members.dup
        old_ids = old_members.map { |m| m['_id'] }

        old_members_by_host = old_members.group_by { |m| m['host'] }.map_values(&:first)
        new_members_by_host = new_members.group_by { |m| m['host'] }.map_values(&:first)

        ids = (0...256).to_a - old_ids

        new_members = new_members_by_host.map { |h, m| old_members_by_host.fetch(h, {}).merge(m) }
                                         .map_values { |m| m.merge('_id' => (m['_id'] || ids.shift)) }

        new_config = config.dup
        new_config['members'] = new_members
        new_config['version'] += 1

        Chef::Log.info "after updating the members, config = #{new_config}"<commit_msg>configure_replicaset: doc id selection<commit_after>#
# Cookbook Name:: sc-mongodb
# Definition:: mongodb
#
# Copyright 2011, edelight GmbH
# Authors:
#       <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

require 'json'

class Chef::ResourceDefinitionList::MongoDB
  # node['mongodb']['config']['mongod']['net']['port'] required for port
  # if node['mongodb']['use_ip_address']
  #   node['ipaddress'] is required
  # else
  #    node['fqdn'] is required
  # end
  #
  # if node['fqnd'] is a vagrant host, ignore it
  # node['mongodb']['replica_priority'] is required
  #
  def self.create_replicaset_member(node)
    return {} if node['fqdn'] =~ /\.vagrantup\.com$/

    port = node['mongodb']['config']['mongod']['net']['port']
    host = node['mongodb']['use_ip_address'] ? node['ipaddress'] : node['fqdn']
    address = "#{host}:#{port}"

    member = {}
    member['host'] = address
    member['arbiterOnly'] = true if node['mongodb']['replica_arbiter_only']
    member['buildIndexes'] = false unless node['mongodb']['replica_build_indexes']
    member['hidden'] = true if node['mongodb']['replica_hidden']
    slave_delay = node['mongodb']['replica_slave_delay']
    member['slaveDelay'] = slave_delay if slave_delay > 0

    priority = if member['buildIndexes'] == false || member['hidden'] || member['slaveDelay']
                 0
               else
                 node['mongodb']['replica_priority']
               end
    member['priority'] = priority unless priority == 1
    tags = node['mongodb']['replica_tags'].to_hash
    member['tags'] = tags unless tags.empty?
    votes = node['mongodb']['replica_votes']
    member['votes'] = votes unless votes == 1

    member.freeze
  end

  def self.configure_replicaset(node, name, members)
    # lazy require, to move loading this modules to runtime of the cookbook
    require 'rubygems'
    require 'mongo'

    if members.empty? && Chef::Config[:solo]
      Chef::Log.warn('Cannot search for member nodes with chef-solo, defaulting to single node replica set')
    end

    mongo_host = 'localhost'
    mongo_port = node['mongodb']['config']['net']['port']

    begin
      connection = nil
      rescue_connection_failure do
        connection = Mongo::Connection.new(mongo_host, mongo_port, op_timeout: 5, slave_ok: true)
        connection.database_names # check connection
      end
    rescue => e
      Chef::Log.warn("Could not connect to database: '#{mongo_host}:#{mongo_port}', reason: #{e}")
      return
    end

    # Want the node originating the connection to be included in the replicaset
    members << node unless members.any? { |m| m.name == node.name }
    members.sort! { |x, y| x.name <=> y.name }

    rs_members = members.each_with_index.map do |member, n|
      create_replicaset_member(member).merge('_id' => n)
    end

    Chef::Log.info(
      "Configuring replicaset with members #{members.map { |n| n['hostname'] }.join(', ')}"
    )

    Chef::Log.debug(
      "Configuring replicaset with config: #{rs_members}"
    )

    admin = connection['admin']
    cmd = BSON::OrderedHash.new
    cmd['replSetInitiate'] = {
      '_id' => name,
      'members' => rs_members,
    }

    begin
      result = admin.command(cmd, check_response: false)
    rescue Mongo::OperationTimeout
      Chef::Log.info('Started configuring the replicaset, this will take some time, another run should run smoothly')
      return
    end
    if result.fetch('ok', nil) == 1
      # everything is fine, do nothing
    elsif result.fetch('errmsg', nil) =~ /(\S+) is already initiated/ || \
          result.fetch('errmsg', nil) == 'already initialized' || \
          result.fetch('errmsg', nil) =~ /is not empty on the initiating member/
      mongo_configured_host, mongo_configured_port = \
        Regexp.last_match.nil? || Regexp.last_match.length < 2 ? [mongo_host, mongo_port] : Regexp.last_match[1].split(':')
      begin
        connection = Mongo::Connection.new(mongo_configured_host, mongo_configured_port, op_timeout: 5, slave_ok: true)
      rescue
        abort("Could not connect to database: '#{mongo_host}:#{mongo_port}'")
      end

      # check if both configs are the same
      config = connection['local']['system']['replset'].find_one('_id' => name)
      if config && config['members'] == rs_members
        # config is up-to-date, do nothing
        Chef::Log.info("Replicaset '#{name}' already configured")
      elsif config['_id'] == name && config['members'] == rs_member_ips
        # config is up-to-date, but ips are used instead of hostnames, change config to hostnames
        Chef::Log.info("Need to convert ips to hostnames for replicaset '#{name}'")
        old_members = config['members'].map { |m| m['host'] }
        mapping = {}
        rs_member_ips.each do |mem_h|
          members.each do |n|
            ip, prt = mem_h['host'].split(':')
            mapping["#{ip}:#{prt}"] = "#{n['fqdn']}:#{prt}" if ip == n['ipaddress']
          end
        end
        config['members'].map! do |m|
          host = mapping[m['host']]
          { '_id' => m['_id'], 'host' => host }.merge(rs_options[host])
        end
        config['version'] += 1

        rs_connection = nil
        rescue_connection_failure do
          rs_connection = Mongo::ReplSetConnection.new(old_members)
          rs_connection.database_names # check connection
        end

        admin = rs_connection['admin']
        cmd = BSON::OrderedHash.new
        cmd['replSetReconfig'] = config
        result = nil
        begin
          result = admin.command(cmd, check_response: false)
        rescue Mongo::ConnectionFailure
          # reconfiguring destroys existing connections, reconnect
          connection = Mongo::Connection.new('localhost', node['mongodb']['config']['port'], op_timeout: 5, slave_ok: true)
          config = connection['local']['system']['replset'].find_one('_id' => name)
          # Validate configuration change
          if config['members'] == rs_members
            Chef::Log.info("New config successfully applied: #{config.inspect}")
          else
            Chef::Log.error("Failed to apply new config. Current config: #{config.inspect} Target config #{rs_members}")
            return
          end
        end
        Chef::Log.error("configuring replicaset returned: #{result.inspect}") unless result.fetch('errmsg', nil).nil?
      else
        Chef::Log.info 'going to update the members of the replicaset'
        old_members = config['members'].dup
        new_members = rs_members.dup
        old_ids = old_members.map { |m| m['_id'] }

        old_members_by_host = old_members.group_by { |m| m['host'] }.map_values(&:first)
        new_members_by_host = new_members.group_by { |m| m['host'] }.map_values(&:first)

        ids = (0...256).to_a - old_ids

        new_members = new_members_by_host.map { |h, m| old_members_by_host.fetch(h, {}).merge(m) }
                                         # use the _id value when present, use a generated one from ids otherwise
                                         .map_values { |m| m.merge('_id' => (m['_id'] || ids.shift)) }

        new_config = config.dup
        new_config['members'] = new_members
        new_config['version'] += 1

        Chef::Log.info "after updating the members, config = #{new_config}"
{'fraction_non_alphanumeric': 0.13793326018201768, 'fraction_numerical': 0.0030981733686180858, 'mean_word_length': 3.0067235583139387, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4782949', 'n_tokens_mistral': 4722, 'n_tokens_neox': 4611, 'n_words': 1215}