<jupyter_start><jupyter_text>1) Sentence Segmentation Basics<jupyter_code># Import spaCy
import spacy
# load the English language library
nlp = spacy.load(name='en_core_web_sm')
doc = nlp("This is a sentence. This is second sentence. This is last sentence.")
for sent in doc.sents:
    print(sent.text)

# note that doc.sents is a generator
doc.sents[0]
# if you try to grab individual sentences then you will get an error
# TypeError: 'generator' object is not subscriptable
# because this generates the sentences instead of holding them in memory
# grab a token
doc[0]
# we can grab individual tokens because spacy knows the language library,
# we can grab individual sentences with list
list(doc.sents)[0]
type(list(doc.sents)[0])
# these are the spacy's span objects not normal strings<jupyter_output><empty_output><jupyter_text>2) Adding Rules We can add our own rules for sentence segmentation, but they have to be added before the creation of doc objectbecauseas that is where parsing of segment start<jupyter_code>def set_custom_boundaries(doc):
    for token in doc[:-1]:   # we are going through every token except the last one
      if token.text == ';':    # here we are adding a new Segmentation rule
        doc[token.i+1].is_sent_start = True
    return doc

# after the semicolan, the next token is start of new sentence

# we are passing all the tokens through for loop except the last one because,
# we are taking "token.i+1" is the start of new sentence
# that is "token.i+1" includes last token in document(doc_2)
nlp.add_pipe(set_custom_boundaries, before='parser')
nlp.pipe_names
# new rule is added, that is 'set_custom_boundaries'
# tagger, parser and ner are already there
# new rule has to run before the document is parsed, hence define new document object witha same content 
doc_2 = nlp("This is a sentence; This is second sentence; This is last sentence.")
for sent in doc_2.sents:
    print(sent)

# new rule is working fine here<jupyter_output>This is a sentence;
This is second sentence;
This is last sentence.
{'fraction_non_alphanumeric': 0.06999510523739598, 'fraction_numerical': 0.006363191385217817, 'mean_word_length': 4.662049861495845, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '18040', 'license': 'MIT', 'n_tokens_mistral': 605, 'n_tokens_neox': 595, 'n_words': 275, 'path': '03SentenceSegmentation.ipynb', 'repo_name': 'MBadriNarayanan/NaturalLanguageProcessing', 'size': '2043'}