<commit_before>package applicant.etl

import org.apache.tika.metadata._
import org.apache.tika.parser._
import org.apache.tika.parser.pdf._
import org.apache.tika.sax.WriteOutContentHandler
import java.io._
import org.apache.commons.io.FilenameUtils
import org.apache.commons.codec.binary.Base64
import java.security.MessageDigest

import scala.collection.mutable.{ListBuffer, Map, LinkedHashMap}

/**
 * Apache Tika functions
 *
 */

object TextExtractor {

  /**
   * Uses Apache Tika library to parse out text from a PDF
   *
   *@param data A PortableDataStream from Spark of a PDF file
   */

  def extractAll(data: InputStream): (String, Map[String, String]) = {
    val handler : WriteOutContentHandler = new WriteOutContentHandler(-1)
    val metadata : Metadata = new Metadata()
    val context : ParseContext = new ParseContext()
    // Apache Tika parser object, auto detects file type
    val myparser : AutoDetectParser = new AutoDetectParser()
    // Input stream for parser, from PortableDataStream data
    val stream : InputStream = data

    try {
      // Parse text from file and store in hander object
      myparser.parse(stream, handler, metadata, context)
    }
    finally {
      stream.close
    }


    var metaDataMap = Map[String,String]()
    val metaDataNames = metadata.names()
    for (name <- metaDataNames) {
      metaDataMap += (name -> metadata.get(name))
    }

    return (handler.toString(), metaDataMap)
  }

}
<commit_msg>Added tesseract OCR capability to TextExtractor, Tika can now parse text from scanned PDFs and images<commit_after>package applicant.etl

import org.apache.tika.metadata._
import org.apache.tika.parser._
import org.apache.tika.parser.pdf._
import org.apache.tika.parser.pdf.PDFParserConfig
import org.apache.tika.parser.ocr.TesseractOCRParser
import org.apache.tika.parser.ocr.TesseractOCRConfig
import org.apache.tika.sax.BodyContentHandler
import java.io._
import org.apache.commons.io.FilenameUtils
import org.apache.commons.codec.binary.Base64
import java.security.MessageDigest

import scala.collection.mutable.{ListBuffer, Map, LinkedHashMap}

/**
 * Apache Tika functions
 *
 */

object TextExtractor {

  /**
   * Uses Apache Tika library to parse out text from a PDF
   *
   *@param data A PortableDataStream from Spark of a PDF file
   */

  def extractAll(data: InputStream): (String, Map[String, String]) = {
    //Needed if tesseract is not on system path
    val handler : BodyContentHandler = new BodyContentHandler()
    val metadata : Metadata = new Metadata()
    // Apache Tika parser object, auto detects file type
    val myparser : Parser = new AutoDetectParser()
    // Input stream for parser, from PortableDataStream data
    val stream : InputStream = data
    val config : TesseractOCRConfig = new TesseractOCRConfig()
    val pdfConfig : PDFParserConfig = new PDFParserConfig()
    val context : ParseContext = new ParseContext()
    //config.setTesseractPath("/usr/share/tesseract-ocr")
    pdfConfig.setExtractInlineImages(true)
    context.set(classOf[TesseractOCRConfig], config)
    context.set(classOf[PDFParserConfig], pdfConfig)
    context.set(classOf[Parser], myparser)

    try {
      // Parse text from file and store in hander object
      myparser.parse(stream, handler, metadata, context)
    }
    finally {
      stream.close
    }


    var metaDataMap = Map[String,String]()
    val metaDataNames = metadata.names()
    for (name <- metaDataNames) {
      metaDataMap += (name -> metadata.get(name))
    }

    return (handler.toString(), metaDataMap)
  }

}

{'fraction_non_alphanumeric': 0.0948782535684299, 'fraction_numerical': 0.001399384270920795, 'mean_word_length': 3.5997425997425996, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4145208', 'n_tokens_mistral': 1072, 'n_tokens_neox': 1064, 'n_words': 295}