<commit_before>require 'resolv'

ndb_connectstring()
my_ip = my_private_ip()

group node['hops']['secure_group'] do
  action :create
  not_if "getent group #{node['hops']['secure_group']}"
end

directory "#{node['hops']['dir']}/ndb-hops-#{node['hops']['version']}-#{node['ndb']['version']}" do
  owner node['hops']['hdfs']['user']
  group node['hops']['group']
  mode "750"
  action :create
end

link "#{node['hops']['dir']}/ndb-hops" do
  owner node['hops']['hdfs']['user']
  group node['hops']['group']
  to "#{node['hops']['dir']}/ndb-hops-#{node['hops']['version']}-#{node['ndb']['version']}"
end


package_url = node['dal']['download_url']
base_filename = File.basename(package_url)

remote_file "#{node['hops']['dir']}/ndb-hops/#{base_filename}" do
  source package_url
  owner node['hops']['hdfs']['user']
  group node['hops']['group']
  mode "0755"
  # TODO - checksum
  action :create_if_missing
end

hops_ndb "extract_ndb_hops" do
  base_filename base_filename
  action :install_ndb_hops
end

link "#{node['hops']['dir']}/ndb-hops/ndb-dal.jar" do
  owner node['hops']['hdfs']['user']
  group node['hops']['group']
  to "#{node['hops']['dir']}/ndb-hops/ndb-dal-#{node['hops']['version']}-#{node['ndb']['version']}.jar"
end

nvidia_url = node['nvidia']['download_url']
nvidia_jar = File.basename(nvidia_url)

remote_file "#{node['hops']['dir']}/ndb-hops/#{nvidia_jar}" do
  source nvidia_url
  owner node['hops']['hdfs']['user']
  group node['hops']['group']
  mode "0755"
  # TODO - checksum
  action :create_if_missing
end

link "#{node['hops']['dir']}/ndb-hops/nvidia-management.jar" do
  owner node['hops']['hdfs']['user']
  group node['hops']['group']
  to "#{node['hops']['dir']}/ndb-hops/nvidia-management-#{node['hops']['version']}-#{node['ndb']['version']}.jar"
end

mysql_ip = my_ip
if node['mysql']['localhost'] == "true"<commit_msg>nvidia-management.jar file is already bundled in hops, no need for separate download<commit_after>require 'resolv'

ndb_connectstring()
my_ip = my_private_ip()

group node['hops']['secure_group'] do
  action :create
  not_if "getent group #{node['hops']['secure_group']}"
end

directory "#{node['hops']['dir']}/ndb-hops-#{node['hops']['version']}-#{node['ndb']['version']}" do
  owner node['hops']['hdfs']['user']
  group node['hops']['group']
  mode "750"
  action :create
end

link "#{node['hops']['dir']}/ndb-hops" do
  owner node['hops']['hdfs']['user']
  group node['hops']['group']
  to "#{node['hops']['dir']}/ndb-hops-#{node['hops']['version']}-#{node['ndb']['version']}"
end


package_url = node['dal']['download_url']
base_filename = File.basename(package_url)

remote_file "#{node['hops']['dir']}/ndb-hops/#{base_filename}" do
  source package_url
  owner node['hops']['hdfs']['user']
  group node['hops']['group']
  mode "0755"
  # TODO - checksum
  action :create_if_missing
end

hops_ndb "extract_ndb_hops" do
  base_filename base_filename
  action :install_ndb_hops
end

link "#{node['hops']['dir']}/ndb-hops/ndb-dal.jar" do
  owner node['hops']['hdfs']['user']
  group node['hops']['group']
  to "#{node['hops']['dir']}/ndb-hops/ndb-dal-#{node['hops']['version']}-#{node['ndb']['version']}.jar"
end

# nvidia_url = node['nvidia']['download_url']
# nvidia_jar = File.basename(nvidia_url)

# remote_file "#{node['hops']['dir']}/ndb-hops/#{nvidia_jar}" do
#   source nvidia_url
#   owner node['hops']['hdfs']['user']
#   group node['hops']['group']
#   mode "0755"
#   # TODO - checksum
#   action :create_if_missing
# end

# link "#{node['hops']['dir']}/ndb-hops/nvidia-management.jar" do
#   owner node['hops']['hdfs']['user']
#   group node['hops']['group']
#   to "#{node['hops']['dir']}/ndb-hops/nvidia-management-#{node['hops']['version']}-#{node['ndb']['version']}.jar"
# end

mysql_ip = my_ip
if node['mysql']['localhost'] == "true"
{'fraction_non_alphanumeric': 0.2454664914586071, 'fraction_numerical': 0.005781865965834429, 'mean_word_length': 6.627254509018036, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5953998', 'n_tokens_mistral': 1472, 'n_tokens_neox': 1374, 'n_words': 230}