<jupyter_start><jupyter_text>**Resume Parser using spaCy and Spark-NLP** --- By <NAME>  Importing Required Liberaries<jupyter_code>import pandas as pd
import spacy
import pickle
import random
import re
import json
import random
import logging<jupyter_output><empty_output><jupyter_text>Reading data<jupyter_code>df = pd.read_json('Entity Recognition in Resumes.json', lines=True)<jupyter_output><empty_output><jupyter_text>Preview of Data<jupyter_code>df.head(5)<jupyter_output><empty_output><jupyter_text>Removing '/n' characters from data. (i.e. New Line Character)<jupyter_code>for i in range(len(df)):
    df["content"][i] = df["content"][i].replace("\n", " ")<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy<jupyter_text>Preview after removing new line characters<jupyter_code>df.head()<jupyter_output><empty_output><jupyter_text>Function for Converting DataFrame into required format. (i.e. Datatrucks to spaCy)<jupyter_code>def convert_to_spacy(JSON_FilePath):
    try:
        training_data = []
        lines=[]
        with open(JSON_FilePath,encoding="utf8") as f:
            lines = f.readlines()

        for line in lines:
            data = json.loads(line)
            text = data['content'].replace("\n", " ")
            entities = []
            data_annotations = data['annotation']
            if data_annotations is not None:
                for annotation in data_annotations:
                    #only a single point in text annotation.
                    point = annotation['points'][0]
                    labels = annotation['label']
                    # handle both list of labels or a single label.
                    if not isinstance(labels, list):
                        labels = [labels]

                    for label in labels:
                        point_start = point['start']
                        point_end = point['end']
                        point_text = point['text']
                        
                        lstrip_diff = len(point_text) - len(point_text.lstrip())
                        rstrip_diff = len(point_text) - len(point_text.rstrip())
                        if lstrip_diff != 0:
                            point_start = point_start + lstrip_diff
                        if rstrip_diff != 0:
                            point_end = point_end - rstrip_diff
                        entities.append((point_start, point_end + 1 , label))
            training_data.append((text, {"entities" : entities}))
        return training_data
    except Exception as e:
        logging.exception("Unable to process " + JSON_FilePath + "\n" + "error = " + str(e))
        return None<jupyter_output><empty_output><jupyter_text>Function for Removing leading & trailing white spaces from entity spans.> **Args: data (list)**: The data to be cleaned in spaCy JSON format.> **Returns: list**: The cleaned data and stored it in a List.<jupyter_code>def trim_entity_spans(data: list) -> list:
    invalid_span_tokens = re.compile(r'\s')

    cleaned_data = []
    for text, annotations in data:
        entities = annotations['entities']
        valid_entities = []
        for start, end, label in entities:
            valid_start = start
            valid_end = end
            while valid_start < len(text) and invalid_span_tokens.match(
                    text[valid_start]):
                valid_start += 1
            while valid_end > 1 and invalid_span_tokens.match(
                    text[valid_end - 1]):
                valid_end -= 1
            valid_entities.append([valid_start, valid_end, label])
        cleaned_data.append([text, {'entities': valid_entities}])
    return cleaned_data<jupyter_output><empty_output><jupyter_text>Converting our Data in Required Format<jupyter_code>conv_data=convert_to_spacy('Entity Recognition in Resumes.json')<jupyter_output><empty_output><jupyter_text>Preview of sample cleaned data in our required format<jupyter_code>data = trim_entity_spans(conv_data)
data[0]<jupyter_output><empty_output><jupyter_text>Creating NLP Blank Model with the help of spaCy<jupyter_code>nlp = spacy.blank('en')<jupyter_output><empty_output><jupyter_text>Function for training our model<jupyter_code>def train_model(train_data):
    if 'ner' not in nlp.pipe_names:
        ner = nlp.create_pipe('ner')
        nlp.add_pipe(ner, last = True)
    
        for _, annotation in train_data:
            for ent in annotation['entities']:
                ner.add_label(ent[2])
            
    
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']
    with nlp.disable_pipes(*other_pipes):  # only train NER
        optimizer = nlp.begin_training()
        for itn in range(50):
            print("Statring iteration " + str(itn))
            random.shuffle(train_data)
            losses = {}
            index = 0
            for text, annotations in train_data:
                try:
                    nlp.update(
                        [text],  # batch of texts
                        [annotations],  # batch of annotations
                        drop=0.2,  # dropout - make it harder to memorise data
                        sgd=optimizer,  # callable to update weights
                        losses=losses)
                except Exception as e:
                    pass
                
            print(losses)<jupyter_output><empty_output><jupyter_text>Model Training<jupyter_code>train_model(data)<jupyter_output>Statring iteration 0
{'ner': 16465.25536238796}
Statring iteration 1
{'ner': 15781.170626723966}
Statring iteration 2
{'ner': 11655.217217006186}
Statring iteration 3
{'ner': 10356.949478027476}
Statring iteration 4
{'ner': 9056.847333362526}
Statring iteration 5
{'ner': 8167.247733932967}
Statring iteration 6
{'ner': 7052.855012010317}
Statring iteration 7
{'ner': 7046.052151583607}
Statring iteration 8
{'ner': 7772.087388741171}
Statring iteration 9
{'ner': 5940.375102305628}
Statring iteration 10
{'ner': 6444.602800411875}
Statring iteration 11
{'ner': 6142.710391694642}
Statring iteration 12
{'ner': 5186.33132863332}
Statring iteration 13
{'ner': 5560.764955562939}
Statring iteration 14
{'ner': 6170.0963053789765}
Statring iteration 15
{'ner': 4828.416610520465}
Statring iteration 16
{'ner': 5565.6608713211435}
Statring iteration 17
{'ner': 5056.673536251201}
Statring iteration 18
{'ner': 5082.950300696691}
Statring iteration 19
{'ner': 4034.710897199544}
Statring iteration 20
{'ne[...]<jupyter_text>Saving our model into disk.<jupyter_code>nlp.to_disk('/content/my_nlp_model')<jupyter_output><empty_output><jupyter_text>Loading Saved Model with the help of spaCy<jupyter_code>nlp_model = spacy.load('my_nlp_model')<jupyter_output><empty_output><jupyter_text>Model Testing<jupyter_code>data[1][0]<jupyter_output><empty_output><jupyter_text>Function for cleaning text data<jupyter_code>def clean_data(df):
    clean_df = re.sub(r"[\(\[].*?[\)\]]", "", df)
    return (clean_df)
data1=clean_data(data[1][0])
doc = nlp_model(data1)<jupyter_output><empty_output><jupyter_text>Using Displacy<jupyter_code>from spacy import displacy
displacy.render(doc, style='ent', jupyter=True)<jupyter_output><empty_output><jupyter_text>Getting Entities<jupyter_code>for ent in doc.ents:
    print(f'{ent.label_.upper():{30}}- {ent.text}')<jupyter_output><empty_output><jupyter_text>**Spark-NLP**   Installing dependencies<jupyter_code>!pip install spark-nlp==3.0.0
!pip install pyspark<jupyter_output>Requirement already satisfied: spark-nlp==3.0.0 in /usr/local/lib/python3.7/dist-packages (3.0.0)
Requirement already satisfied: pyspark in /usr/local/lib/python3.7/dist-packages (3.1.1)
Requirement already satisfied: py4j==0.10.9 in /usr/local/lib/python3.7/dist-packages (from pyspark) (0.10.9)<jupyter_text>Importing Required Libraries<jupyter_code>import sparknlp
from sparknlp.base import *
from sparknlp.annotator import *<jupyter_output><empty_output><jupyter_text>Starting Sparknlp session<jupyter_code>spark = sparknlp.start()<jupyter_output><empty_output><jupyter_text>Preview of Sample Data<jupyter_code>data[1][0]<jupyter_output><empty_output><jupyter_text>Creating Spark DataFrame<jupyter_code>data2= spark.createDataFrame([[data[1][0]]]).toDF('text')
data2.show(truncate=False)
document = DocumentAssembler().setInputCol('text').setOutputCol('document').setCleanupMode('shrink')
sentence = SentenceDetector().setInputCols('document').setOutputCol('sentence')
sentence.setExplodeSentences(True)<jupyter_output><empty_output><jupyter_text>Tokenizing data<jupyter_code>tokenizer = Tokenizer().setInputCols('sentence').setOutputCol('token')
tokenizer.setExceptions(['e-mail'])
checker= NorvigSweetingModel.pretrained().setInputCols(['token']).setOutputCol('checked')<jupyter_output>spellcheck_norvig download started this may take some time.
Approximate size to download 4.2 MB
[OK!]<jupyter_text>Embedding Data by WordEmbeddings method<jupyter_code>embeddings=WordEmbeddingsModel.pretrained().setInputCols(['sentence','checked']).setOutputCol('embeddings')<jupyter_output>glove_100d download started this may take some time.
Approximate size to download 145.3 MB
[OK!]<jupyter_text>NER<jupyter_code>ner = NerDLModel.pretrained().setInputCols(['sentence','checked','embeddings']).setOutputCol('ner')
converter = NerConverter().setInputCols(['sentence','checked','ner']).setOutputCol('chunk')<jupyter_output><empty_output><jupyter_text>Importing Pipeline Pakage from pyspark.ml<jupyter_code>from pyspark.ml import Pipeline<jupyter_output><empty_output><jupyter_text>Creating Spark-NLP Pipeline<jupyter_code>pipeline = Pipeline().setStages([document, sentence, tokenizer, checker, embeddings, ner, converter])<jupyter_output><empty_output><jupyter_text>Model Building<jupyter_code>model = pipeline.fit(data2)
result = model.transform(data2)<jupyter_output><empty_output><jupyter_text>Outcome<jupyter_code>result.show()
result.select('sentence.result').show(truncate=False)
result.select('checked.result').show(truncate=False)
result.select('ner.result').show(truncate=False)
result.select('ner.begin', 'ner.end').show(truncate=False)
result.select('chunk.result','chunk.begin', 'chunk.end').show(truncate=False)<jupyter_output>+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------+---------------------------------------------------------+
|result                                                                                                                                                                          |begin                                                   |end                                                      |
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------+---------------------------------------------------------+
|[Nazish Alam Consultant, SAP ABAP Ghaziabad, Uttar Pradesh, Delhi, Delhi, Pradesh, Lucknow, Pradesh, SAP ABAP, Uttar [...]<jupyter_text>Sample testing<jupyter_code>Light=LightPipeline(model)
Light.annotate(data[2][0])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1592560553633218, 'fraction_numerical': 0.037197231833910036, 'mean_word_length': 2.844695710009977, 'pattern_counts': {'":': 0, '<': 121, '<?xml version=': 0, '>': 124, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '31', 'dir': 'jupyter-structured-clean-dedup', 'id': '241569', 'license': 'Apache-2.0', 'n_tokens_mistral': 3793, 'n_tokens_neox': 3428, 'n_words': 675, 'path': 'Resume_parser_spaCy_Spark_NLP.ipynb', 'repo_name': 'hrishikeshmalkar/Resume_parsing_with_spaCy_sparknlp', 'size': '11571'}