<commit_before># install R for Cuneiform
package "r-base" do
  options "--force-yes"
end<commit_msg>build cuneiform on workers and move it to hiway lib dir<commit_after># install R for Cuneiform
package "r-base" do
  options "--force-yes"
end

if node[:hiway][:release] == "false"
  # install Git
  include_recipe "git"
  
  # install Maven
  package "maven" do
    options "--force-yes"
  end
  
  # git clone Cuneiform
  git "/tmp/cuneiform" do
    repository node[:hiway][:cuneiform][:github_url]
    reference "master"
    user node[:hiway][:user]
    group node[:hadoop][:group]
    action :sync
  end

  # maven build Cuneiform
  bash 'build-and-copy-cuneiform' do
    user node[:hiway][:user]
    group node[:hadoop][:group]
    code <<-EOH
    set -e && set -o pipefail
      mvn -f /tmp/cuneiform/pom.xml package
      version=$(grep -Po '(?<=^\t<version>)[^<]*(?=</version>)' /tmp/cuneiform/pom.xml)
      cp -r /tmp/cuneiform/cuneiform-dist/target/cuneiform-dist-$version/cuneiform-$version #{node[:hiway][:cuneiform][:home]}
      rm #{node[:hiway][:hiway][:home]}/lib/cuneiform-core-#{node[:hiway][:cuneiform][:release][:version]}.jar
      cp #{node[:hiway][:cuneiform][:home]}/cuneiform-core-$version.jar #{node[:hiway][:hiway][:home]}/lib/cuneiform-core-$version.jar
    EOH
    not_if { ::File.exist?("#{node[:hiway][:cuneiform][:home]}") }
  end
end
{'fraction_non_alphanumeric': 0.182882223847842, 'fraction_numerical': 0.0, 'mean_word_length': 3.903225806451613, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2105105', 'n_tokens_mistral': 515, 'n_tokens_neox': 508, 'n_words': 102}