<commit_before>spark_pkg_version = node[:spark][:package][:version]
spark_bin_dir = node[:spark][:bin][:dir]

if node[:spark][:package][:install_meta] == true
  package 'spark' do
    action :upgrade
  end
else
  package "spark-#{spark_pkg_version}" do
    action :install
  end
end

template "#{spark_bin_dir}/conf/spark-env.sh" do
  source 'spark-env.sh.erb'
  mode 0o0755
  helper :config do
    node.bach_spark.environment.sort_by(&:first)
  end
  helpers(Spark::Configuration)
end

template "#{spark_bin_dir}/conf/spark-defaults.conf" do
  source 'spark-defaults.conf.erb'
  mode 0o0755
  helper :config do
    node.bach_spark.config.sort_by(&:first)
  end
  helpers(Spark::Configuration)
end

link "/#{spark_bin_dir}/yarn/spark-yarn-shuffle.jar" do
  to "#{spark_bin_dir}/yarn/spark-#{spark_pkg_version}-yarn-shuffle.jar"
end

link '/usr/spark/current' do
  to spark_bin_dir
end

# install fortran libs needed by some jobs
package 'libatlas3gf-base' do
  action :install
end

package 'libopenblas-base' do
  action :install
end<commit_msg>Update native library packages for Spark 2.x<commit_after>spark_pkg_version = node[:spark][:package][:version]
spark_bin_dir = node[:spark][:bin][:dir]

if node[:spark][:package][:install_meta] == true
  package 'spark' do
    action :upgrade
  end
else
  package "spark-#{spark_pkg_version}" do
    action :install
  end
end

template "#{spark_bin_dir}/conf/spark-env.sh" do
  source 'spark-env.sh.erb'
  mode 0o0755
  helper :config do
    node.bach_spark.environment.sort_by(&:first)
  end
  helpers(Spark::Configuration)
end

template "#{spark_bin_dir}/conf/spark-defaults.conf" do
  source 'spark-defaults.conf.erb'
  mode 0o0755
  helper :config do
    node.bach_spark.config.sort_by(&:first)
  end
  helpers(Spark::Configuration)
end

link "/#{spark_bin_dir}/yarn/spark-yarn-shuffle.jar" do
  to "#{spark_bin_dir}/yarn/spark-#{spark_pkg_version}-yarn-shuffle.jar"
end

link '/usr/spark/current' do
  to spark_bin_dir
end

# Spark 2.x expects to dynamically link LZO.
package 'liblzo2-2' do
  action :upgrade
end

# install fortran libs needed by some jobs
package 'libatlas3gf-base' do
  action :upgrade
end

package 'libopenblas-base' do
  action :upgrade
end
{'fraction_non_alphanumeric': 0.1256781193490054, 'fraction_numerical': 0.011754068716094032, 'mean_word_length': 4.703608247422681, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6945990', 'n_tokens_mistral': 880, 'n_tokens_neox': 816, 'n_words': 178}