<commit_before>

magic_shell_environment 'LD_LIBRARY_PATH' do
  value "#{node.hops.base_dir}/lib/native:$LD_LIBRARY_PATH"
end


case node.platform
when "ubuntu"
 if node.platform_version.to_f <= 14.04
   node.override.hops.systemd = "false"
 end
end


if node.hops.os_defaults == "true" then

  # http://blog.cloudera.com/blog/2015/01/how-to-deploy-apache-hadoop-clusters-like-a-boss/
  node.default.sysctl.allow_sysctl_conf = true
  node.default.sysctl.params.vm.swappiness = 1
  node.default.sysctl.params.vm.overcommit_memory = 1
  node.default.sysctl.params.vm.overcommit_ratio = 100
  node.default.sysctl.params.net.core.somaxconn= 1024
  include_recipe 'sysctl::apply'

  #
  # http://www.slideshare.net/vgogate/hadoop-configuration-performance-tuning
  #
  case node.platform_family
  when "debian"
    bash "configure_os" do
      user "root"
      code <<-EOF
   EOF
    end
  when "redhat"
    bash "configure_os" do
      user "root"
      code <<-EOF
      echo "never" > /sys/kernel/mm/redhat_transparent_hugepages/defrag
     EOF
    end
    
  end
  
end



include_recipe "java"


if "#{node.hops.group}" != "#{node.hops.hdfs.user}"
  group node.hops.group do
    action :create
    not_if "getent group #{node.hops.group}"
  end
end


user node.hops.hdfs.user do
  home "/home/#{node.hops.hdfs.user}"
  system true
  shell "/bin/bash"
  manage_home true
  action :create
  not_if "getent passwd #{node.hops.hdfs.user}"
end

user node.hops.yarn.user do
  home "/home/#{node.hops.yarn.user}"
#  system true
  shell "/bin/bash"
  manage_home true
  action :create
  not_if "getent passwd #{node.hops.yarn.user}"
end

user node.hops.mr.user do
  home "/home/#{node.hops.mr.user}"
  system true
  shell "/bin/bash"
  manage_home true
  action :create
  not_if "getent passwd #{node.hops.mr.user}"
end

group node.hops.group do
  action :modify
  members ["#{node.hops.hdfs.user}", "#{node.hops.yarn.user}", "#{node.hops.mr.user}"]
  append true
end

case node.platform_family
when "debian"
  package "openssh-server" do
    action :install
    options "--force-yes"
  end
<commit_msg>allow different user/group combo<commit_after>

magic_shell_environment 'LD_LIBRARY_PATH' do
  value "#{node.hops.base_dir}/lib/native:$LD_LIBRARY_PATH"
end


case node.platform
when "ubuntu"
 if node.platform_version.to_f <= 14.04
   node.override.hops.systemd = "false"
 end
end


if node.hops.os_defaults == "true" then

  # http://blog.cloudera.com/blog/2015/01/how-to-deploy-apache-hadoop-clusters-like-a-boss/
  node.default.sysctl.allow_sysctl_conf = true
  node.default.sysctl.params.vm.swappiness = 1
  node.default.sysctl.params.vm.overcommit_memory = 1
  node.default.sysctl.params.vm.overcommit_ratio = 100
  node.default.sysctl.params.net.core.somaxconn= 1024
  include_recipe 'sysctl::apply'

  #
  # http://www.slideshare.net/vgogate/hadoop-configuration-performance-tuning
  #
  case node.platform_family
  when "debian"
    bash "configure_os" do
      user "root"
      code <<-EOF
   EOF
    end
  when "redhat"
    bash "configure_os" do
      user "root"
      code <<-EOF
      echo "never" > /sys/kernel/mm/redhat_transparent_hugepages/defrag
     EOF
    end
    
  end
  
end



include_recipe "java"


group node.hops.group do
  action :create
  not_if "getent group #{node.hops.group}"
end

user node.hops.hdfs.user do
  home "/home/#{node.hops.hdfs.user}"
  gid node.hops.group
  system true
  shell "/bin/bash"
  manage_home true
  action :create
  not_if "getent passwd #{node.hops.hdfs.user}"
end

user node.hops.yarn.user do
  home "/home/#{node.hops.yarn.user}"
  gid node.hops.group
  system true
  shell "/bin/bash"
  manage_home true
  action :create
  not_if "getent passwd #{node.hops.yarn.user}"
end

user node.hops.mr.user do
  home "/home/#{node.hops.mr.user}"
  gid node.hops.group  
  system true
  shell "/bin/bash"
  manage_home true
  action :create
  not_if "getent passwd #{node.hops.mr.user}"
end

group node.hops.group do
  action :modify
  members ["#{node.hops.hdfs.user}", "#{node.hops.yarn.user}", "#{node.hops.mr.user}"]
  append true
end

case node.platform_family
when "debian"
  package "openssh-server" do
    action :install
    options "--force-yes"
  end

{'fraction_non_alphanumeric': 0.12983622122003324, 'fraction_numerical': 0.009019700925706147, 'mean_word_length': 3.291242362525458, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7276087', 'n_tokens_mistral': 1686, 'n_tokens_neox': 1559, 'n_words': 323}