<jupyter_start><jupyter_text>Rule-Based Matching with spaCy<jupyter_code>import spacy
from spacy.matcher import Matcher

nlp = spacy.load("en_core_web_sm")

matcher = Matcher(nlp.vocab)
pattern = [{"TEXT": {"REGEX": "(?i)[0-9]+(?:[,.][0-9]+)?[ckwh]+"}}, 
           {'ORTH': '/'}, {"LOWER": {"REGEX": "(?i)^[ckwh]+$"}}]
matcher.add("Usage", None, pattern)

doc = nlp("Peak Usage 409 24.51c/kWh $100.25")

matches = matcher(doc)
for match_id, start, end in matches:
    string_id = nlp.vocab.strings[match_id]
    span = doc[start:end]
    print(match_id, string_id, start, end, span.text)
#Matcher.py
#import necessary modules and tools 
import spacy
from spacy.matcher import Matcher #import Matcher class from spacy
#import the Span class to extract the words from the document object
from spacy.tokens import Span 
#Language class with the English model 'en_core_web_sm' is loaded
nlp = spacy.load("en_core_web_sm")
#instantiate a new Matcher class object 
matcher = Matcher(nlp.vocab)
#define the pattern
pattern = [{'LOWER': 'computer', 'POS': 'NOUN'},
             {'POS':{'NOT_IN': ['VERB']}}]
#add the pattern to the previously created matcher object
matcher.add("Matching", None, pattern)
#The input text string is converted to a Document object
doc = nlp("""Computer programming is the process of writing instructions that get executed by computers. The instructions, also known as code, 
             are written in a programming language which the computer can understand and use to perform a task or solve a problem. 
             Basic computer programming involves the analysis of a problem and development of a logical sequence of instructions to solve it. 
             There can be numerous paths to a solution and the computer programmer seeks to design and code that which is most efficient. 
             Among the programmer’s tasks are understanding requirements, determining the right programming language to use, designing or architecting the solution, 
             coding, testing, debugging and writing documentation so that the solution can be easily understood by other programmers.Computer programming is at 
             the heart of computer science. It is the implementation portion of software development, application development and software engineering efforts, 
             transforming ideas and theories into actual, working solutions.""")
#call the matcher object the document object and it will return #match_id, start and stop indexes of the matched words
matches = matcher(doc)
#print the matched results and extract out the results
for match_id, start, end in matches:
    # Get the string representation 
    string_id = nlp.vocab.strings[match_id]  
    span = doc[start:end]  # The matched span
    print(match_id, string_id, start, end, span.text)
# PhraseMatcher.py
# import necessary modules
import spacy
from spacy.matcher import PhraseMatcher #import PhraseMatcher class
# Language class with the English model 'en_core_web_sm' is loaded
nlp = spacy.load('en_core_web_sm')
# create the PhraseMatcher object
matcher = PhraseMatcher(nlp.vocab, attr='LOWER')
# the list containing the pharses to be matched
terminology_list = ["Machine Learning", "Hidden Structure",              
                           "Unlabeled Data"]
# convert the phrases into document object using nlp.make_doc to #speed up.
patterns = [nlp.make_doc(text) for text in terminology_list]
# add the patterns to the matcher object without any callbacks
matcher.add("Phrase Matching", None, *patterns)
# the input text string is converted to a Document object
doc = nlp("""Supervised machine learning algorithms can apply what has been learned in the past to new data using labeled examples to predict future events. 
Starting from the analysis of a known training dataset, the learning algorithm produces an inferred function to make predictions about the output values. 
The system is able to provide targets for any new input after sufficient training. The learning algorithm can also compare its output with the correct, 
intended output and find errors in order to modify the model accordingly. In contrast, unsupervised machine learning algorithms are used when the information 
used to train is neither classified nor labeled. Unsupervised learning studies how systems can infer a function to describe a hidden structure from unlabeled data. 
The system doesn’t figure out the right output, but it explores the data and can draw inferences from datasets to describe hidden structures from unlabeled data. 
Semi-supervised machine learning algorithms fall somewhere in between supervised and unsupervised learning, since they use both labeled and unlabeled data 
for training – typically a small amount of labeled data and a large amount of unlabeled data. The systems that use this method are able to considerably improve 
learning accuracy. Usually, semi-supervised learning is chosen when the acquired labeled data requires skilled and relevant resources in order to 
train it / learn from it. Otherwise, acquiring unlabeled data generally doesn’t require additional resources.""")
#call the matcher object the document object and it will return #match_id, start and stop indexes of the matched words
matches = matcher(doc)
#print the matched results and extract out the results
for match_id, start, end in matches:
    # Get the string representation 
    string_id = nlp.vocab.strings[match_id]  
    span = doc[start:end]  # The matched span
    print(match_id, string_id, start, end, span.text)
# EntityRuler.py
# import necessary packages and tools 
import spacy
from spacy.pipeline import EntityRuler # import EntityRuler 
# load a blank English model from spacy
nlp = spacy.blank('en')
# convert the input sentence into the document object 
doc = nlp("The First Estate included the clergy (church leaders), the Second Estate included the nobles, and the Third Estate included the commoners. The Third Estate paid most of the taxes, while the nobility lived lives of luxury and got all the high-ranking jobs.")
# print the entity types of each entity in the above sentence
print([(ent.text, ent.label_) for ent in doc.ents])

# instantiate an object of EntityRuler class
ruler = EntityRuler(nlp)
# define the pattern
patterns = [{"label": "NOUN", "pattern": "church"}, 
             {"label": "ORG",              
             "pattern": [{"lower": "the"}, 
             {"lower": {"IN": ["first", "second", "third"]}},                          
             {"ORTH": "Estate"}]}]
# add the pattern to the matcher object
ruler.add_patterns(patterns)
# add the matcher object as a new pipe to the model
nlp.add_pipe(ruler)
# convert the input sentence into the document object using the newly added 'nlp'
doc = nlp("The First Estate included the clergy (church leaders), the Second Estate included the nobles, and the Third Estate included the commoners. The Third Estate paid most of the taxes, while the nobility lived lives of luxury and got all the high-ranking jobs.")
# print the entities in the sentenced after adding the EntityRuler matcher
print([(ent.text, ent.label_) for ent in doc.ents])<jupyter_output>[]
[('The First Estate', 'ORG'), ('church', 'NOUN'), ('the Second Estate', 'ORG'), ('the Third Estate', 'ORG'), ('The Third Estate', 'ORG')]<jupyter_text>Spacy Fr---<jupyter_code>from spacy.lang.fr import French

nlp = French()

# Traite le texte
doc = nlp(
    "En 1990, plus de 60 % de la population d'Asie orientale vivait dans une pauvreté extrême. "
    "Actuellement c'est moins de 4 %."
)

# Itère sur les tokens du doc
for token in doc:
    print(token)
    # Vérifie si le token ressemble à un nombre
    if token.like_num:
        # Obtiens le token suivant dans le document
        next_token = doc[token.i + 1]
        # Vérifie si le texte du token suivant est égal à "%"
        if next_token.text == "%":
            print("Pourcentage trouvé :", token.text)
!pip install fr_core_news_sm
from spacy.lang.fr import French

nlp = French()

# Traite le texte
doc = nlp(
    "En 1990, plus de 60 % de la population d'Asie orientale vivait dans une pauvreté extrême. "
    "Actuellement c'est moins de 4 %."
)

# Itère sur les tokens du doc
for token in doc:
    print(token)
    # Vérifie si le token ressemble à un nombre
    if token.like_num:
        # Obtiens le token suivant dans le document
        next_token = doc[token.i + 1]
        # Vérifie si le texte du token suivant est égal à "%"
        if next_token.text == "%":
            print("Pourcentage trouvé :", token.text)

print("*"*80)
doc = nlp("Importer un modèle LUIS et ajouter des intentions")
for token in doc:
    print(token.text, token.pos, token.dep)

for token in doc: 
  print(token, token.lemma_)     
import spacy
from spacy import displacy
from spacy.lang.fr import French

nlp = French()

doc = nlp("Demain je travaille à la maison.")
for token in doc:
    print("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}".format(
        token.text,
        token.idx,
        token.lemma_,
        token.is_punct,
        token.is_space,
        token.shape_,
        token.pos_,
        token.tag_,
        token.ent_type_
    ))

for ent in doc.ents:
    print(">", ent.text, ent.label_)

from spacy import displacy

doc = nlp("I live in Guwahati, Assam")
displacy.render(doc, style="dep")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08219324497648568, 'fraction_numerical': 0.004595981188542112, 'mean_word_length': 3.815748841996912, 'pattern_counts': {'":': 14, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '483198', 'license': 'MIT', 'n_tokens_mistral': 2661, 'n_tokens_neox': 2578, 'n_words': 1163, 'path': 'spacy_sept.ipynb', 'repo_name': 'Philippe-AD/essai2019', 'size': '9356'}