<commit_before>require "ddr-extraction"
require "openssl"

DOWNLOAD_DIR = File.absolute_path("tmp")

namespace :tika do
  desc "Download Tika app"
  task :download => :download_dir do
    tika_app = File.join(DOWNLOAD_DIR, "tika-app.jar")
    Ddr::Extraction::Adapters::TikaAdapter.config do |tika|
      puts "Downloading Tika app ... "
      system "curl -L #{tika.download_url} -o #{tika_app}"
      puts "Verifiying checksum ... "
      digest = OpenSSL::Digest.const_get(tika.checksum_type).new
      digest << File.read(tika.path)
      if digest.to_s == tika.checksum
        FileUtils.mv(tika_app, tika.path)
      else
        puts "Checksums do not match!"
      end
    end
  end

  namespace :server do
    desc "Start the Tika server"
    task :start => :environment do
      
    end
  end
end

namespace :fits do
  desc "Download FITS tool"
  task :download => :download_dir do
    fits_tool = File.join(DOWNLOAD_DIR, "fits.zip")
    Ddr::Extraction::Adapters::FitsAdapter.config do |fits|
      puts "Downloading FITS tool ... "
      system "curl -L #{fits.download_url} -o #{fits_tool}"
      # Unzip options: convert text files, force overwrite, extra quiet
      system "unzip -a -o -qq -d bin #{fits_tool}"
      FileUtils.chmod(0755, fits.path)
    end
  end
end

task :download_dir do
  FileUtils.mkdir(DOWNLOAD_DIR) unless Dir.exists?(DOWNLOAD_DIR)
end
<commit_msg>Remove empty task<commit_after>require "ddr-extraction"
require "openssl"

DOWNLOAD_DIR = File.absolute_path("tmp")

namespace :tika do
  desc "Download Tika app"
  task :download => :download_dir do
    tika_app = File.join(DOWNLOAD_DIR, "tika-app.jar")
    Ddr::Extraction::Adapters::TikaAdapter.config do |tika|
      puts "Downloading Tika app ... "
      system "curl -L #{tika.download_url} -o #{tika_app}"
      puts "Verifiying checksum ... "
      digest = OpenSSL::Digest.const_get(tika.checksum_type).new
      digest << File.read(tika.path)
      if digest.to_s == tika.checksum
        FileUtils.mv(tika_app, tika.path)
      else
        puts "Checksums do not match!"
      end
    end
  end

  # namespace :server do
  #   desc "Start the Tika server"
  #   task :start => :environment do      
  #   end
  # end
end

namespace :fits do
  desc "Download FITS tool"
  task :download => :download_dir do
    fits_tool = File.join(DOWNLOAD_DIR, "fits.zip")
    Ddr::Extraction::Adapters::FitsAdapter.config do |fits|
      puts "Downloading FITS tool ... "
      system "curl -L #{fits.download_url} -o #{fits_tool}"
      # Unzip options: convert text files, force overwrite, extra quiet
      system "unzip -a -o -qq -d bin #{fits_tool}"
      FileUtils.chmod(0755, fits.path)
    end
  end
end

task :download_dir do
  FileUtils.mkdir(DOWNLOAD_DIR) unless Dir.exists?(DOWNLOAD_DIR)
end

{'fraction_non_alphanumeric': 0.11572912934432103, 'fraction_numerical': 0.002866356144750985, 'mean_word_length': 3.040520984081042, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1757100', 'n_tokens_mistral': 982, 'n_tokens_neox': 955, 'n_words': 252}