<jupyter_start><jupyter_text>Example of parsing FoundationOne report using Spark OCR  Install spark-ocr python packgeNeed specify path to `spark-ocr-assembly-[version].jar` or `secret`<jupyter_code>secret = ""
license = ""
version = secret.split("-")[0]
spark_ocr_jar_path = "../../target/scala-2.11"
%%bash
if python -c 'import google.colab' &> /dev/null; then
    echo "Run on Google Colab!"
    echo "Install Open JDK"
    apt-get install -y openjdk-8-jdk-headless -qq > /dev/null
    java -version
fi
import sys
import os

if 'google.colab' in sys.modules:
  os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
  os.environ["PATH"] = os.environ["JAVA_HOME"] + "/bin:" + os.environ["PATH"]
# install from PYPI using secret
#%pip install spark-ocr==$version\.spark24 --extra-index-url=https://pypi.johnsnowlabs.com/$secret --upgrade
# or install from local path
#%pip install --user ../../python/dist/spark-ocr-1.9.0.spark24.tar.gz<jupyter_output><empty_output><jupyter_text>Initialization of spark session<jupyter_code>from pyspark.sql import SparkSession
from sparkocr import start

if license:
    os.environ['JSL_OCR_LICENSE'] = license

spark = start(secret=secret, jar_path=spark_ocr_jar_path)
spark<jupyter_output>Spark version: 2.4.4
Spark NLP version: 2.5.5
Spark OCR version: 1.9.0rc1<jupyter_text>Import OCR transformers<jupyter_code>from sparkocr.transformers import *
from sparkocr.enums import *
from pyspark.ml import PipelineModel<jupyter_output><empty_output><jupyter_text>Define OCR transformers and pipeline<jupyter_code>def pipeline():
    
    # Transforrm PDF document to images per page
    pdf_to_text = PdfToText()
    pdf_to_text.setOutputCol("text")
    pdf_to_text.setSplitPage(False)
    pdf_to_text.setSort(True)
    pdf_to_text.setTextStripper(TextStripperType.PDF_LAYOUT_TEXT_STRIPPER)

    genomic_parser = FoundationOneReportParser()
    genomic_parser.setInputCol("text")
    genomic_parser.setOutputCol("genomics")

    
    pipeline = PipelineModel(stages=[
        pdf_to_text,
        genomic_parser
    ])
    
    return pipeline<jupyter_output><empty_output><jupyter_text>Read PDF document as binary file<jupyter_code>import pkg_resources
pdf_example = pkg_resources.resource_filename('sparkocr', 'resources/ocr/pdfs/genomics/3.pdf')
pdf_example_df = spark.read.format("binaryFile").load(pdf_example).cache()<jupyter_output><empty_output><jupyter_text>Run OCR pipelines<jupyter_code>result = pipeline().transform(pdf_example_df).cache()<jupyter_output><empty_output><jupyter_text>Display results<jupyter_code>print(result.select("genomics").collect()[0].genomics)<jupyter_output>{
  "Appendix" : {
    "dna_gene_list" : [ "ABL1", "ACVR1B", "AKT1", "AKT2", "AKT3", "ALK", "ALOX12B", "AMER1", "(FAM123B)", "APC", "AR", "ARAF", "ARFRP1", "ARID1A", "ASXL1", "ATM", "ATR", "ATRX", "AURKA", "AURKB", "AXIN1", "AXL", "BAP1", "BARD1", "BCL2", "BCL2L1", "BCL2L2", "BCL6", "BCOR", "BCORL1", "BRAF", "BRCA1", "BRCA2", "BRD4", "BRIP1", "BTG1", "BTG2", "BTK", "C11orf30", "(EMSY)", "C17orf39", "(GID4)", "CALR", "CARD11", "CASP8", "CBFB", "CBL", "CCND1", "CCND2", "CCND3", "CCNE1", "CD22", "CD274", "(PD-L1)", "CD70", "CD79A", "CD79B", "CDC73", "CDH1", "CDK12", "CDK4", "CDK6", "CDK8", "CDKN1A", "CDKN1B", "CDKN2A", "CDKN2B", "CDKN2C", "CEBPA", "CHEK1", "CHEK2", "CIC", "CREBBP", "CRKL", "CSF1R", "CSF3R", "CTCF", "CTNNA1", "CTNNB1", "CUL3", "CUL4A", "CXCR4", "CYP17A1", "DAXX", "DDR1", "DDR2", "DIS3", "DNMT3A", "DOT1L", "EED", "EGFR", "EP300", "EPHA3", "EPHB1", "EPHB4", "ERBB2", "ERBB3", "ERBB4", "ERCC4", "ERG", "ERRFI1", "ESR1", "EZH2", "FAM46C", "FANCA", "FANCC", "FANCG", "FANCL", "FAS[...]<jupyter_text>Clear cache<jupyter_code>result.unpersist()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18281879194630873, 'fraction_numerical': 0.0348993288590604, 'mean_word_length': 5.990619136960601, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '567067', 'license': 'OML', 'n_tokens_mistral': 1599, 'n_tokens_neox': 1517, 'n_words': 276, 'path': 'jupyter/SparkOcrParseFoundationOneReport.ipynb', 'repo_name': 'galiph/spark-ocr-workshop', 'size': '3725'}