<commit_before>require 'java'
$CLASSPATH << File.expand_path(File.join('..', 'jar', 'tika-app-1.1.jar'))

class Jrtika

end
<commit_msg>adding tika text parsing with Autodetect parser<commit_after>require 'java'
$CLASSPATH << File.expand_path(File.join('..', 'jar', 'tika-app-1.1.jar'))

class Jrtika

  # Read text and metadata from a file and return a hash containing both.
  #
  #   Jrtika.read file_path

  def self.read(file_path)
    @RESOURCE_NAME_KEY = ""

        is = java.io.FileInputStream.new(file_path)
        parser = org.apache.tika.parser.AutoDetectParser.new()  #will work for all acceptable file types

        handler = org.apache.tika.sax.BodyContentHandler.new()  #this will contain the parsed output
        meta = org.apache.tika.metadata.Metadata.new()

        meta.set(@RESOURCE_NAME_KEY,file_path)  #setting file name for AutoDetectParser

        parser.parse(is, handler, meta)
        is.close()
        {:text => handler.toString(), :metadata => meta.toString()}

  end

end

{'fraction_non_alphanumeric': 0.12797619047619047, 'fraction_numerical': 0.003968253968253968, 'mean_word_length': 3.425438596491228, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2429340', 'n_tokens_mistral': 337, 'n_tokens_neox': 330, 'n_words': 78}