<jupyter_start><jupyter_text>Document Normalizer annotator notebook  Set up Colab environment<jupyter_code># This is only to setup PySpark and Spark NLP on Colab
!wget http://setup.johnsnowlabs.com/colab.sh -O - | bash<jupyter_output><empty_output><jupyter_text>Start Spark NLP session<jupyter_code># Import Spark NLP
from sparknlp.base import *
from sparknlp.annotator import *

def start():
    builder = SparkSession.builder \
        .appName("Spark NLP Licensed") \
        .master("local[4]") \
        .config("spark.driver.memory", "8G") \
        .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") \
        .config("spark.kryoserializer.buffer.max", "2000M") \
        .config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.11:2.7.0") \
        .config("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem") \
        .config("fs.AbstractFileSystem.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS")
    return builder.getOrCreate()

spark = start()<jupyter_output><empty_output><jupyter_text>Document Normalizer annotator overview<jupyter_code># The DocumentNormalizer is an annotator that can be used after the DocumentAssembler to narmalize documents once that they have been processed and indexed .
# It takes in input annotated documents of type Array[AnnotatorType](DOCUMENT) and gives as output annotated document of type AnnotatorType.DOCUMENT .
#
# Parameters are:
# - inputCol: input column name string which targets a column of type Array(AnnotatorType.DOCUMENT).
# - outputCol: output column name string which targets a column of type AnnotatorType.DOCUMENT.
# - action: action string to perform applying regex patterns, i.e. (clean | extract). Default is "clean".
# - cleanupPatterns: normalization regex patterns which match will be removed from document. Default is "<[^>]*>" (e.g., it removes all HTML tags).
# - replacement: replacement string to apply when regexes match. Default is " ".
# - lowercase: whether to convert strings to lowercase. Default is False.
# - removalPolicy: removalPolicy to remove patterns from text with a given policy. Valid policy values are: "all", "pretty_all", "first", "pretty_first". Defaults is "pretty_all".
# - encoding: file encoding to apply on normalized documents. Supported encodings are: UTF_8, UTF_16, US_ASCII, ISO-8859-1, UTF-16BE, UTF-16LE. Default is "UTF-8".


documentAssembler = DocumentAssembler() \
    .setInputCol('text') \
    .setOutputCol('document')

inpuColName = "document"
outputColName = "normalizedDocument"

action = "clean"
cleanUpPatterns = ["<[^>]*>"]
replacement = " "
removalPolicy = "pretty_all"
encoding = "UTF-8"

documentNormalizer = DocumentNormalizer() \
    .setInputCols(inpuColName) \
    .setOutputCol(outputColName) \
    .setAction(action) \
    .setPatterns(cleanUpPatterns) \
    .setReplacement(replacement) \
    .setPolicy(removalPolicy) \
    .setLowercase(True) \
    .setEncoding(encoding)<jupyter_output><empty_output><jupyter_text>Data loading<jupyter_code>path = "html-docs"

data = spark.sparkContext.wholeTextFiles("html-docs")
df = data.toDF(schema=["filename", "text"]).select("text")

df.show()<jupyter_output>+--------------------+
|                text|
+--------------------+
|<div class='w3-co...|
|<!DOCTYPE html>
<...|
|<span style="font...|
+--------------------+<jupyter_text>Example 1: remove all the tags from HTML text files<jupyter_code># Once data is loaded we can process the textual document applying a pipeline that normalizes the document right after the DocumentAssembler.
# For instance, let's imagine we are loading some HTML pages in our DataFrame and we want to remove all the tags in it:

documentAssembler = DocumentAssembler() \
    .setInputCol('text') \
    .setOutputCol('document')

cleanUpPatterns = ["<[^>]*>"]

documentNormalizer = DocumentNormalizer() \
    .setInputCols("document") \
    .setOutputCol("normalizedDocument") \
    .setAction("clean") \
    .setPatterns(cleanUpPatterns) \
    .setReplacement(" ") \
    .setPolicy("pretty_all") \
    .setLowercase(True)

sentenceDetector = SentenceDetector() \
      .setInputCols(["normalizedDocument"]) \
      .setOutputCol("sentence")

regexTokenizer = Tokenizer() \
      .setInputCols(["sentence"]) \
      .setOutputCol("token") \
      .fit(df)

docPatternRemoverPipeline = \
  Pipeline() \
    .setStages([
        documentAssembler,
        documentNormalizer,
        sentenceDetector,
        regexTokenizer])

ds = docPatternRemoverPipeline.fit(df).transform(df)

ds.select("normalizedDocument").show(10)<jupyter_output>+--------------------+
|  normalizedDocument|
+--------------------+
|[[document, 0, 67...|
|[[document, 0, 17...|
|[[document, 0, 31...|
+--------------------+<jupyter_text>Example 2: obfuscate PII such as emails in HTML content<jupyter_code>documentAssembler = DocumentAssembler() \
    .setInputCol('text') \
    .setOutputCol('document')

action = "clean"
patterns = ["([^.@\\s]+)(\\.[^.@\\s]+)*@([^.@\\s]+\\.)+([^.@\\s]+)"]
replacement = "***OBFUSCATED PII***"

documentNormalizer = DocumentNormalizer() \
    .setInputCols("document") \
    .setOutputCol("normalizedDocument") \
    .setAction("clean") \
    .setPatterns(cleanUpPatterns) \
    .setReplacement(replacement) \
    .setPolicy("pretty_all") \
    .setLowercase(True)

sentenceDetector = SentenceDetector() \
      .setInputCols(["normalizedDocument"]) \
      .setOutputCol("sentence")

regexTokenizer = Tokenizer() \
      .setInputCols(["sentence"]) \
      .setOutputCol("token") \
      .fit(df)

docPatternRemoverPipeline = \
  Pipeline() \
    .setStages([
        documentAssembler,
        documentNormalizer,
        sentenceDetector,
        regexTokenizer])

ds = docPatternRemoverPipeline.fit(df).transform(df)

ds.select("normalizedDocument").show(10, False)<jupyter_output>+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------[...]<jupyter_text>Example 3: obfuscate PII such as ages in HTML content<jupyter_code>documentAssembler = DocumentAssembler() \
    .setInputCol('text') \
    .setOutputCol('document')

action = "clean"
patterns = ["\\d+(?=[\\s]?year)", "(aged)[\\s]?\\d+"]
replacement = "***OBFUSCATED PII***"

documentNormalizer = DocumentNormalizer() \
    .setInputCols("document") \
    .setOutputCol("normalizedDocument") \
    .setAction(action) \
    .setPatterns(patterns) \
    .setReplacement(replacement) \
    .setPolicy("pretty_all") \
    .setLowercase(True)

sentenceDetector = SentenceDetector() \
      .setInputCols(["normalizedDocument"]) \
      .setOutputCol("sentence")

regexTokenizer = Tokenizer() \
      .setInputCols(["sentence"]) \
      .setOutputCol("token") \
      .fit(df)

docPatternRemoverPipeline = \
  Pipeline() \
    .setStages([
        documentAssembler,
        documentNormalizer,
        sentenceDetector,
        regexTokenizer])

ds = docPatternRemoverPipeline.fit(df).transform(df)

ds.select("normalizedDocument").show(10, False)<jupyter_output>+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------[...]<jupyter_text>Example 4: extract XML name tag contents<jupyter_code># data loading
data = spark.sparkContext.wholeTextFiles("xml-docs")
df = data.toDF(schema=["filename", "text"]).select("text")
df.show()
documentAssembler = DocumentAssembler() \
    .setInputCol('text') \
    .setOutputCol('document')

action = "extract"

tag = "name"
patterns = [tag]

documentNormalizer = DocumentNormalizer() \
    .setInputCols("document") \
    .setOutputCol("normalizedDocument") \
    .setAction(action) \
    .setPatterns(patterns) \
    .setReplacement("") \
    .setPolicy("pretty_all") \
    .setLowercase(True)

sentenceDetector = SentenceDetector() \
      .setInputCols(["normalizedDocument"]) \
      .setOutputCol("sentence")

regexTokenizer = Tokenizer() \
      .setInputCols(["sentence"]) \
      .setOutputCol("token") \
      .fit(df)

docPatternRemoverPipeline = \
  Pipeline() \
    .setStages([
        documentAssembler,
        documentNormalizer,
        sentenceDetector,
        regexTokenizer])

ds = docPatternRemoverPipeline.fit(df).transform(df)

ds.select("normalizedDocument").show(10, False)<jupyter_output>+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|normalizedDocument                                                                                                                                                                                                                                                                                                        [...]
{'fraction_non_alphanumeric': 0.35666546242774566, 'fraction_numerical': 0.004335260115606936, 'mean_word_length': 1.3579642248722317, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '137911', 'license': 'Apache-2.0', 'n_tokens_mistral': 2678, 'n_tokens_neox': 2567, 'n_words': 514, 'path': 'jupyter/annotation/english/document-normalizer/document_normalizer_notebook.ipynb', 'repo_name': 'fcivardi/spark-nlp-workshop', 'size': '11072'}