<jupyter_start><jupyter_text><jupyter_code># This is only to setup PySpark and Spark NLP on Colab
!wget http://setup.johnsnowlabs.com/colab.sh -O - | bash
!pip install gdown
!gdown --id 1AaaFcJ0Zzxr-fnlhlbp8mQePU6o8dFvk
!unzip -u "./BN_DATASET.zip" -d "./data"
!rm -f "./BN_DATASET.zip"
!cp -r ./data/BN-Bangla/. ./data && rm -rf ./data/BN-Bangla
import sparknlp
from sparknlp.annotator import *
from sparknlp.base import *
# if you have GPU
spark = sparknlp.start(gpu=True)
# spark = sparknlp.start()

print("Spark NLP version: ", sparknlp.version())
print("Apache Spark version: ", spark.version)
with open("data/bn_train.conll") as f:
    c=f.read()

print (c[:200])
from sparknlp.training import CoNLL

training_data = CoNLL().readDataset(spark, "./data/bn_train.conll")
training_data.show(3)
test_data = CoNLL().readDataset(spark, "./data/bn_dev.conll")
test_data.show(3)
training_data.count()
test_data.count()
# we use BERT Tiny
# bert_annotator = BertEmbeddings.pretrained('small_bert_L2_128', 'en') \
bert_annotator = XlmRoBertaEmbeddings.pretrained()\
 .setInputCols(["sentence",'token'])\
 .setOutputCol("bert")\
.setBatchSize(8)
test_data = bert_annotator.transform(test_data)
test_data.show(3)
# let's transform and save our test dataset for evaluation
test_data.write.parquet("data/test_withEmbeds.parquet")
test_data.select("bert.result","bert.embeddings",'label.result').show()
nerTagger = NerDLApproach()\
  .setInputCols(["sentence", "token", "bert"])\
  .setLabelColumn("label")\
  .setOutputCol("ner")\
  .setGraphFolder("/content/data")\
  .setMaxEpochs(5)\
  .setLr(0.001)\
  .setPo(0.005)\
  .setBatchSize(32)\
  .setEvaluationLogExtended(True) \
  .setEnableOutputLogs(True)\
  .setTestDataset("/content/data/test_withEmbeds.parquet")

pipeline = Pipeline(
    stages = [
    bert_annotator,
    nerTagger
  ])
ner_model = pipeline.fit(training_data)
!rm -f /root/annotator_logs/NerDLApproach_3f2872bcff8e.log
!ls -l /root/annotator_logs/
!cat /root/annotator_logs/NerDLApproach_3f2872bcff8e.log
ner_model.stages[1].write().overwrite().save('./NER_bert_20200219')
test_data.show()
predictions = ner_model.transform(test_data.select("sentence", "token", "label"))
predictions.show(3)
predictions.select('token.result','label.result','ner.result').show(truncate=40)
predictions.printSchema()
import pyspark.sql.functions as F

predictions.select(F.explode(F.arrays_zip('token.result','label.result','ner.result')).alias("cols")) \
.select(F.expr("cols['0']").alias("token"),
        F.expr("cols['1']").alias("ground_truth"),
        F.expr("cols['2']").alias("prediction")).show(truncate=False)




# !git clone https://github.com/JohnSnowLabs/spark-nlp
# %cd spark-nlp/python/tensorflow
# %cd /content
# %env PYTHONPATH="/env/python:/content/spark-nlp/python/tensorflow/lib/ner"
# %env PYTHONPATH="$/env/python:/content/spark-nlp/python/tensorflow/lib/ner"
# %set_env PYTHONPATH=/content/spark-nlp/python/tensorflow/lib/ner:/env/python ## working solution
# %set_env PYTHONPATH=/env/python
# !source /content/ner_venv/bin/activate; python ner/create_models.py 13 128 220 /content/data
# !pip install virtualenv
# !virtualenv ner_venv
# !virtualenv -p /usr/bin/python3.6 ner_venv ## working solution
# !source /content/ner_venv/bin/activate; pip install tensorflow==1.15.0<jupyter_output><empty_output><jupyter_text>Reference: [1](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/blogposts/3.NER_with_BERT.ipynb)<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16785612332286612, 'fraction_numerical': 0.01998287182415073, 'mean_word_length': 5.5251396648044695, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '341576', 'license': 'Apache-2.0', 'n_tokens_mistral': 1391, 'n_tokens_neox': 1327, 'n_words': 197, 'path': 'notebooks/spark_nlp_multiconer_000_0.ipynb', 'repo_name': 'bhuiyanmobasshir94/MultiCoNER', 'size': '3503'}