<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Description2&nbsp;&nbsp;Sparknlp setup3&nbsp;&nbsp;Load the libraries4&nbsp;&nbsp;Script (This works)  DescriptionBert using sparknlp  Sparknlp setup<jupyter_code>#=============== setup sparknlp
import os
import sys

sys.path.append("/Users/poudel/opt/miniconda3/envs/mysparknlp/lib/python3.7/site-packages")
os.environ["SPARK_HOME"] = "/Users/poudel/Softwares/Spark/spark-2.4.4-bin-hadoop2.7"
os.environ["PYSPARK_PYTHON"] = "/Users/poudel/opt/miniconda3/envs/mysparknlp/bin/python"
os.environ["PYSPARK_DRIVER_PYTHON"] = "jupyter"
os.environ["PYSPARK_DRIVER_PYTHON_OPTS"] = "notebook"
#================ setup sparknlp end<jupyter_output><empty_output><jupyter_text>Load the libraries<jupyter_code>import sparknlp

spark = sparknlp.start()
data = [
  ("New York is the greatest city in the world", 0),
  ("The beauty of Paris is vast", 1),
  ("The Centre Pompidou is in Paris", 1)
]

sdf = spark.createDataFrame(data, ["text","label"])
sdf.show()<jupyter_output>+--------------------+-----+
|                text|label|
+--------------------+-----+
|New York is the g...|    0|
|The beauty of Par...|    1|
|The Centre Pompid...|    1|
+--------------------+-----+<jupyter_text>Script (This works)Download the spark2.4.4 version : https://archive.apache.org/dist/spark/spark-2.4.4/<jupyter_code>%%writefile myexports.sh
export SPARK_HOME=/Users/poudel/Softwares/Spark/spark-2.4.4-bin-hadoop2.7
export PYSPARK_PYTHON=/Users/poudel/opt/miniconda3/envs/mysparknlp/bin/python
export PYSPARK_DRIVER_PYTHON=jupyter
export PYSPARK_DRIVER_PYTHON_OPTS=notebook

%%writefile test_sparknlp.py
import sys

sys.path.append("/Users/poudel/opt/miniconda3/envs/mysparknlp/lib/python3.7/site-packages")

import sparknlp


spark = sparknlp.start()
data = [
  ("New York is the greatest city in the world", 0),
  ("The beauty of Paris is vast", 1),
  ("The Centre Pompidou is in Paris", 1)
]

sdf = spark.createDataFrame(data, ["text","label"])
print(sdf.show())<jupyter_output>Overwriting test_sparknlp.py
{'fraction_non_alphanumeric': 0.19070904645476772, 'fraction_numerical': 0.018092909535452322, 'mean_word_length': 4.456, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '531965', 'license': 'Apache-2.0', 'n_tokens_mistral': 790, 'n_tokens_neox': 771, 'n_words': 136, 'path': 'a01_PySpark/m06_pyspark_sparknlp/example_sparknlp.ipynb', 'repo_name': 'mindis/Big_Data_Analysis', 'size': '2045'}