<jupyter_start><jupyter_text>Global Variables<jupyter_code>import print_metrics

file_path = "ITIL Books/ITIL 3/ITIL3 Service Operation chapter 4.pdf"
# extracted_text_file_path = "ITIL Books/ITIL 3/Continual service improvement chapter from notebook.txt"
# extracted_text_file_path = "ITIL Books/ITIL 3/Service operation chapter 4/Service operation chapter 4 - 4.txt"
# extracted_text_file_path = "ITIL Books\ITIL 3\Service operation chapter 4\Automated concepts extracted\\4.2\Service operation chapter 4 - 4.2 to 4.2.4 .txt"
output_file_path = "output/ITIL3 Continual Service Improvement.txt"

# Uncomment a single extracted_text_file_path and a single manual_concepts_file_path to run metric tests on that
# specific chapter subsection, e.g. 4.1 is uncommented now so metrics will be calculated for it within the last cell
# of the notebook

lemmatize = False
# lemmatize = True

### Chapter 4 start ###
# extracted_text_file_path = print_metrics.get_extracted_text_file_path(4) 
# manual_concepts_file_path = print_metrics.get_manual_concepts_file_path(4)

### Chapter 4 - 4.1 to 4.1.4 ###
extracted_text_file_path = print_metrics.get_extracted_text_file_path(4.1) 
manual_concepts_file_path = print_metrics.get_manual_concepts_file_path(4.1)

### Chapter 4 - 4.2 to 4.2.4 ###
# extracted_text_file_path = print_metrics.get_extracted_text_file_path(4.2) 
# manual_concepts_file_path = print_metrics.get_manual_concepts_file_path(4.2)

# ### Chapter 4 - 4.3 to 4.3.4 ###
# extracted_text_file_path = print_metrics.get_extracted_text_file_path(4.3) 
# manual_concepts_file_path = print_metrics.get_manual_concepts_file_path(4.3)

# ### Chapter 4 - 4.4 to 4.4.4 ###
# extracted_text_file_path = print_metrics.get_extracted_text_file_path(4.4) 
# manual_concepts_file_path = print_metrics.get_manual_concepts_file_path(4.4)

# ### Chapter 4 - 4.5 to 4.5.4 ###
# extracted_text_file_path = print_metrics.get_extracted_text_file_path(4.5) 
# manual_concepts_file_path = print_metrics.get_manual_concepts_file_path(4.5)

# ### Chapter 4 - 4.6 to 4.6.4 ###
# extracted_text_file_path = print_metrics.get_extracted_text_file_path(4.6) 
# manual_concepts_file_path = print_metrics.get_manual_concepts_file_path(4.6)<jupyter_output><empty_output><jupyter_text>Imports<jupyter_code>import pdfminer
import nltk
import re<jupyter_output><empty_output><jupyter_text>Global Functions<jupyter_code>with open(extracted_text_file_path, 'r') as file:
    extracted_text = file.read()
    
# extracted_text = "Particle dynamics involves the study of physics and chemistry"

# tokens = nltk.word_tokenize(extracted_text)
# print(tokens)

# ### Part of speech tagging ###
# part_of_speech_array = nltk.pos_tag(tokens)
# print(part_of_speech_array)<jupyter_output><empty_output><jupyter_text>Text sanitization and word tokenizing<jupyter_code>### Grab sections from text ###
# print(re.findall("^\d(\.|\d)*(\s|\w)*$", extracted_text))
# title_pattern = re.compile(r"^\d(\.|\d)*(\s|\w)*$", re.MULTILINE)
title_pattern = re.compile(r"^\d+.*$", re.MULTILINE)

sections = title_pattern.findall(extracted_text)
for counter, section in enumerate(sections):
    if not (section.find("%") == -1 and section.find(")") == -1):
        sections.remove(section)

### Sanitise extracted text ###
extracted_text_sanitised = extracted_text
extracted_text_sanitised = extracted_text.replace("¦", "")
extracted_text_sanitised = extracted_text_sanitised.replace("–", "")
        
### Tokenise extracted text ###
tokens = nltk.word_tokenize(extracted_text_sanitised)
# print(tokens)<jupyter_output><empty_output><jupyter_text>Part to speech tagging<jupyter_code>### Part of speech tagging ###
part_of_speech_array = nltk.pos_tag(tokens)
# print(part_of_speech_array)

from nltk.stem import WordNetLemmatizer 
lemmatizer = WordNetLemmatizer()

def lemmatize_part_of_speech_array(part_of_speech_array):
    part_of_speech_array_lemmatized = []
    
    for part_of_speech in part_of_speech_array:
        part_of_speech_array_lemmatized.append(
            (lemmatizer.lemmatize(part_of_speech[0]), part_of_speech[1])
        ) 
    return part_of_speech_array_lemmatized
    
# print(part_of_speech_array_lemmatized)

# Uncomment this if lemmatization is to be used
if lemmatize:
    part_of_speech_array = lemmatize_part_of_speech_array(part_of_speech_array)<jupyter_output><empty_output><jupyter_text>Term extraction<jupyter_code>### Term Extraction (NNP next to each other) ###
def extract_terms(part_of_speech_array_array, tags_to_use):
    terms_array = []
    term_phrase = []
    start_new_term = True
    for index, part in enumerate(part_of_speech_array_array):
        if(part[1] in tags_to_use):
            term_phrase.append(part[0])
            start_new_term = False if part_of_speech_array_array[index + 1][1] in tags_to_use else True

            if start_new_term == True:
                terms_array.append(" ".join(term_phrase))
                term_phrase = []
    return terms_array

### Term Extraction (NNP next to each other) ###
def extract_terms_with_adj(part_of_speech_array_array, tags_to_use):
    terms_array = []
    term_phrase = []
    start_new_term = True
    for index, part in enumerate(part_of_speech_array_array):
        if(part[1] in tags_to_use):
            term_phrase.append(part[0])
            start_new_term = False if part_of_speech_array_array[index + 1][1] in tags_to_use else True

            if start_new_term == True:
                terms_array.append(" ".join(term_phrase))
                term_phrase = []
    return terms_array

major_named_concepts = extract_terms(part_of_speech_array, {"NNP", "NNPS"})
other_concepts = extract_terms(part_of_speech_array, {"NN", "NNS"})
all_noun_phrases = extract_terms(part_of_speech_array, {"NNP", "NNPS", "NN", "NNS"})
all_noun_phrases_with_adj = extract_terms_with_adj(part_of_speech_array, {"NNP", "NNPS", "NN", "NNS", "JJ"})

print(f"major named concepts: \n{major_named_concepts}")
print(f"\nother concepts: \n{other_concepts}")
print(f"\nall noun phrases: \n{all_noun_phrases}")
print(f"\nall noun phrases with adj: \n{all_noun_phrases_with_adj}")<jupyter_output>major named concepts: 
['OPERATIONAL ACTIVITIES OF PROCESSES COVERED IN OTHER LIFECYCLE PHASES', 'Change Management Change Management', 'Service Transition', 'Change Management', 'Service Operation', 'Raising', 'RFCs', 'Service Operation', 'CAB', 'CAB/EC', 'Service Operation', 'ï¿½ Implementing', 'Change Management', 'Service Operation', 'Change Management', 'Service Operation', 'Helping', 'Service Operation', 'Service Operation', 'Change Management', 'Configuration Management Configuration Management', 'Service Transition', 'Configuration Management', 'Service Operation', 'Informing Configuration Management', 'CIs', 'CMS ï¿½ Making', 'Configuration Management', 'Service Operation', 'Responsibility', 'CMS', 'Configuration Management', 'Operations', 'Configuration Management', 'CIs', 'CIs', 'CMS', 'Operations', 'Release', 'Deployment Management Release', 'Deployment Management', 'Service Transition', 'Service Operation', 'Actual', 'Release', 'Deployment Management', 'Service Operation',[...]<jupyter_text>Major/common concept extraction<jupyter_code># concept_relationships = extract_terms(part_of_speech_array, {"VP"})
# print(concept_relationships)
# print(all_noun_phrases)

### Perform frequency analysis ###
### Concept Extraction Frequency analysis ###
major_named_concept_frequency_distribution = nltk.FreqDist(major_named_concepts)
other_concept_frequency_distribution = nltk.FreqDist(other_concepts)
all_noun_phrases_frequency_distribution = nltk.FreqDist(all_noun_phrases)

print(f"major named concepts: \n{major_named_concept_frequency_distribution.most_common(50)}")
print(f"\nother concepts: \n{other_concept_frequency_distribution.most_common(50)}")
print(f"\nall noun phrases: \n{all_noun_phrases_frequency_distribution.most_common(50)}")<jupyter_output>major named concepts: 
[('Service Operation', 12), ('Change Management', 4), ('Configuration Management', 4), ('CIs', 4), ('Service Transition', 3), ('Release', 3), ('Deployment Management', 3), ('CMS', 2), ('Operations', 2), ('OPERATIONAL ACTIVITIES OF PROCESSES COVERED IN OTHER LIFECYCLE PHASES', 1), ('Change Management Change Management', 1), ('Raising', 1), ('RFCs', 1), ('CAB', 1), ('CAB/EC', 1), ('ï¿½ Implementing', 1), ('Helping', 1), ('Configuration Management Configuration Management', 1), ('Informing Configuration Management', 1), ('CMS ï¿½ Making', 1), ('Responsibility', 1), ('Deployment Management Release', 1), ('Actual', 1), ('Participation', 1), ('DML', 1)]

other concepts: 
[('staff', 6), ('component', 5), ('service', 5), ('change', 4), ('publication', 3), ('aspect', 3), ('basis', 3), ('issue', 3), ('process', 2), ('discrepancy', 2), ('direction', 2), ('release', 2), ('meeting', 1), ('risk', 1), ('view', 1), ('account', 1), ('define', 1), ('maintain change model', 1), ('c[...]<jupyter_text>Concept relationship extraction<jupyter_code>def get_sentence_at_index(part_of_speech_array, index):
    sentence_starting_index = 0
    sentence_end_index = len(part_of_speech_array)
    
    ### Get sentence start index ###
    for i in range(0, index):
#         print(part_of_speech_array[index])
        if part_of_speech_array[index - i][1] == ".":
            sentence_starting_index = index - i
            break
    
    ### Get sentence end index ###
    for i in range(0, len(part_of_speech_array)):
        if part_of_speech_array[index + i][1] == ".":
            sentence_end_index = index + i
            break
            
    return (sentence_starting_index, sentence_end_index + 1)

### Term Extraction (NNP next to each other) ###
def extract_terms(part_of_speech_array_array, tags_to_use):
    part_of_speech_array_with_terms = []
    
    terms_array = []
    term_phrase = []
    start_new_term = True
    for index, part in enumerate(part_of_speech_array_array):
        if(part[1] in tags_to_use):
            term_phrase.append(part[0])
            start_new_term = False if part_of_speech_array_array[index + 1][1] in tags_to_use else True

            if start_new_term == True:
                if len(term_phrase) > 1:
#                     part_of_speech_array_with_terms.append((" ".join(term_phrase), f"NPhrase-{part[1]}"))
                    part_of_speech_array_with_terms.append((" ".join(term_phrase), "NPhrase"))
                else:
                    part_of_speech_array_with_terms.append((" ".join(term_phrase), part[1]))
                term_phrase = []
        else:
            part_of_speech_array_with_terms.append((part[0], part[1]))
    return part_of_speech_array_with_terms

part_of_speech_array_with_terms = extract_terms(part_of_speech_array, {"NNP", "NNPS", "NN", "NNS"})
# print(part_of_speech_array_with_terms)

sentences = []
temp_sentence = []
for word in part_of_speech_array_with_terms:
    if word[1] is ".":
        temp_sentence.append(word)
        sentences.append(temp_sentence)
        temp_sentence = []
    else:
        temp_sentence.append(word)
        
        
from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))

# for index, sentence in enumerate(sentences):
#     sentence_with_no_stop_words = [word_pos for word_pos in sentence if not word_pos[0] in stop_words]
#     
#     if index > 3:
#         break
#         
#     print(sentence)
#     print(sentence_with_no_stop_words)
#     print("-" * 100)

# from part_of_speech_regex import PartOfSpeechRegex

class PartOfSpeechRegex:
    def parseAndReturnPatterns(self, pattern_string, sentence):
        print(pattern_string)

pattern_string = "*<JJ><NNP><VBZ><NP>"

print(sentences[3])

part_of_speech_regex = PartOfSpeechRegex()
# part_of_speech_regex.parseAndReturnPatterns()

part_of_speech_regex.parseAndReturnPatterns(pattern_string, sentences[3])

most_common_major_concepts = major_named_concept_frequency_distribution.most_common(50)
# print(most_common_major_concepts)
# print(tokens.index(most_common_major_concepts[0][0]))
# print(part_of_speech_array[206])
## Get indices of all common concepts
indices = [i for i, x in enumerate(part_of_speech_array) if x[0] == most_common_major_concepts[0][0]]
# print(indices)
# print(most_common_major_concepts[1][0])

def get_sentence_at_index(part_of_speech_array, index):
    sentence_starting_index = 0
    sentence_end_index = len(part_of_speech_array)
    
    ### Get sentence start index ###
    for i in range(0, index):
        if part_of_speech_array[index - i][1] == ".":
            sentence_starting_index = index - i
            break
    
    ### Get sentence end index ###
    for i in range(0, index):
        if part_of_speech_array[index + i][1] == ".":
            sentence_end_index = index + i
            break
            
    return (sentence_starting_index + 1, sentence_end_index + 1)

def does_list_contain_verb_pos(part_of_speech_array):
    for word_pos in part_of_speech_array:
        if word_pos[1] in ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']:
            return True
    return False
    
# sentence_index = get_sentence_at_index(part_of_speech_array, indices[0])
# print(sentence_index)

concept_relations = []

i = 0
for index in indices:
#     if i < 3:
#         i = i + 1
#         continue
    sentence_pos_containing_concept = part_of_speech_array[
        get_sentence_at_index(part_of_speech_array, index)[0]:
        get_sentence_at_index(part_of_speech_array, index)[1]
    ]
    print(sentence_pos_containing_concept)
        
    last_concept = ()
    last_concept_index = -1
    # For word part_of_speech in sentence_part_of_speech_containing_concept
    for index, word_pos in enumerate(sentence_pos_containing_concept):
#         print(f"{word_pos[0]}: {word_pos[1]}")
#         print(word_pos[0] in all_noun_phrases)
        
#         if (word_pos[0] in all_noun_phrases):
        if (word_pos[0] in major_named_concepts):
            if last_concept_index != -1:# and does_list_contain_verb_pos(sentence_pos_containing_concept[last_concept_index + 1:index]):
                concept_relations.append(f"{last_concept}::{sentence_pos_containing_concept[last_concept_index + 1:index]}::{word_pos}")
            
            last_concept = word_pos
            last_concept_index = index
        
    # print(related_concepts)
    # print()
    
#     print('-----')
    i = i + 1
#     if i == 4:
#         break

for concept_relation in concept_relations:
    print(concept_relation)<jupyter_output><empty_output><jupyter_text>Metrics for term extraction chapter 4 first section<jupyter_code># This code has been moved into the print_metrics.py Python module to prevent code duplication

### The below cell code is now within the print_metrics python file so it can be used for iteration 1 and 2 without duplication
# # automatic_concepts_file_path = "ITIL Books/ITIL 3/Service operation chapter 4/Automated concepts extracted/4/Automated concepts extracted 4.txt"
# # manual_concepts_file_path = "ITIL Books/ITIL 3/Service operation chapter 4/Automated concepts extracted/4/Manual concepts extracted 4.txt"
# 
# # automatic_concepts_file_path = "ITIL Books/ITIL 3/Service operation chapter 4/Automated concepts extracted/4.2/Automated concepts extracted 4.2.txt"
# # manual_concepts_file_path = "ITIL Books/ITIL 3/Service operation chapter 4/Automated concepts extracted/4.2/Manual concepts extracted 4.2.txt"
# # 
# # with open(automatic_concepts_file_path, 'r') as file:
# #     automatic_concepts = file.read()
# 
# with open(manual_concepts_file_path, 'r') as file:
#     manual_concepts = file.read()
# 
# manual_concepts_list = manual_concepts.split('\n')
# manual_concepts_list = [x.lower() for x in manual_concepts_list]
# 
# # print("Manual concepts")
# # print(list(dict.fromkeys(manual_concepts_list)))
# # print()
# 
# # automatic_concepts_list = ['Service Operation', 'processes', 'paragraph', 'detail', 'chapter', 'reference', 'structure', 'processes', 'detail', 'chapter', 'Please note', 'roles', 'process', 'tools', 'process', 'Chapters', 'Management', 'process', 'monitors', 'events', 'IT infrastructure', 'operation', 'exception conditions', 'Incident Management', 'service', 'users', 'order', 'business impact', 'Problem Management', 'root-cause analysis', 'cause', 'events', 'incidents', 'activities', 'problems/incidents', 'Known Error subprocess', 'quicker diagnosis', 'resolution', 'incidents', 'NOTE', 'distinction', 'incidents', 'problems', 'Incident', 'Problem Records', 'danger', 'Incidents', 'support cycle', 'actions', 'recurrence', 'incidents', 'Incidents', 'root cause analysis', 'visibility', 'user ’ s service', 'SLA targets', 'service', 'users', 'expectations', 'results', 'number', 'incidents', '‘ purge ’', 'visibility', 'issues', 'Request Fulfilment', 'management', 'customer', 'user requests', 'incident', 'service delay', 'disruption', 'organizations', 'requests', 'category ’', 'incidents', 'information', 'Incident Management system', 'others', 'volumes', 'business priority', 'requests', 'provision', 'Request Fulfilment', 'Request Fulfilment process', 'practice', 'Request Fulfilment process', 'customer', 'user requests', 'types', 'requests', 'facilities', 'moves', 'supplies', 'IT services', 'requests', 'SLA measures', 'records', 'process flow', 'practice', 'organizations', 'Access Management', 'process', 'users', 'right', 'service', 'access', 'users', 'users', 'ability', 'access services', 'stages', 'resources', 'HR', 'lifecycle', 'Access Management', 'Identity', 'Rights Management', 'organizations', 'addition', 'processes', 'Service Operation', 'phases', 'Service Management Lifecycle', 'aspects', 'processes', 'part', 'chapter', 'include', 'Change Management', 'process', 'Configuration Management', 'Release Management', 'topics', 'Service Transition publication', 'Capacity', 'Availability Management', 'aspects', 'publication', 'detail', 'Service Design publication', 'Financial Management', 'Service Strategy publication', 'Knowledge Management', 'Service Transition publication', 'IT Service Continuity', 'Service Design publication', 'Service Reporting', 'Measurement', 'Continual Service Improvement publication']
# automatic_concepts_list = all_noun_phrases
# automatic_concepts_list = [x.lower() for x in automatic_concepts_list]
# 
# # print("all noun phrases")
# # print(list(dict.fromkeys(automatic_concepts_list)))
# 
# count = 0
# for concept in manual_concepts_list:
#     if concept in automatic_concepts_list:
#         count = count + 1
# 
# number_of_fully_correct_manual_concepts = count
# 
# number_of_manual_concepts = len(manual_concepts_list)
# 
# count = 0
# for concept in automatic_concepts_list:
#     if concept in manual_concepts_list:
#         count = count + 1
#     
# number_of_fully_correct_automatic_concepts = count
# 
# number_of_automatic_concepts = len(automatic_concepts_list)
# 
# print(f"number_of_manual_concepts: {number_of_manual_concepts}")
# print(f"number_of_automatic_concepts: {number_of_automatic_concepts}")
# print(f"number_of_fully_correct_manual_concepts: {number_of_fully_correct_manual_concepts}")
# print(f"number_of_fully_correct_automatic_concepts: {number_of_fully_correct_automatic_concepts}")
# 
# # Lists to words for partial matches
# automatic_concepts_list_single_words = [x.split() for x in automatic_concepts_list]
# # print(automatic_concepts_list_single_words)
# 
# manual_concepts_list_single_words = [x.split() for x in manual_concepts_list]
# # print(manual_concepts_list_single_words)
# 
# count = 0
# for concept in manual_concepts_list_single_words:
#     for word in concept:
#         if word in ' '.join(automatic_concepts_list).split():
#             count = count + 1
#             break
#         
# number_of_full_and_partial_correct_manual_concepts = count
# print(f"number_of_full_and_partial_correct_manual_concepts: {number_of_full_and_partial_correct_manual_concepts}")
# 
# count = 0
# for concept in automatic_concepts_list_single_words:
#     for word in concept:
#         if word in ' '.join(manual_concepts_list).split():
#             count = count + 1
#             break
#             
# number_of_full_and_partial_correct_automatic_concepts = count
# print(f"number_of_full_and_partial_correct_automatic_concepts: {number_of_full_and_partial_correct_automatic_concepts}")
import print_metrics

part_of_speech_array = nltk.pos_tag(tokens)
lemmatized_part_of_speech_array = lemmatize_part_of_speech_array(part_of_speech_array)

extract_terms(part_of_speech_array, {"NNP", "NNPS", "NN", "NNS"})   
extract_terms(lemmatized_part_of_speech_array, {"NNP", "NNPS", "NN", "NNS"})

print("With duplicates")
print_metrics.print_metrics(manual_concepts_file_path, all_noun_phrases, debug=True, remove_duplicates=False)

print()
print("Without duplicates")
print_metrics.print_metrics(manual_concepts_file_path, all_noun_phrases, debug=True, remove_duplicates=True)<jupyter_output>With duplicates
All noun phrases
['OPERATIONAL ACTIVITIES OF PROCESSES COVERED IN OTHER LIFECYCLE PHASES', 'Change Management Change Management', 'Service Transition publication', 'aspect', 'Change Management', 'Service Operation staff', 'basis', 'Raising', 'RFCs', 'Service Operation issue', 'CAB', 'CAB/EC meeting', 'Service Operation risk', 'issue', 'view', 'account ï¿½ Implementing change', 'Change Management', 'Service Operation component', 'service', 'change', 'Change Management', 'Service Operation component', 'service', 'Helping define', 'maintain change model', 'Service Operation component', 'service', 'change schedule', 'Service Operation staff', 'change', 'Change Management process', 'standard', 'change', 'Configuration Management Configuration Management', 'Service Transition publication', 'aspect', 'Configuration Management', 'Service Operation staff', 'basis', 'Informing Configuration Management', 'discrepancy', 'CIs', 'CMS ï¿½ Making', 'amendment', 'discrepancy', 'authorit[...]
{'fraction_non_alphanumeric': 0.12472058756443592, 'fraction_numerical': 0.010811550567948542, 'mean_word_length': 5.310305123776627, 'pattern_counts': {'":': 5, '<': 39, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '15538', 'license': 'Apache-2.0', 'n_tokens_mistral': 7498, 'n_tokens_neox': 6984, 'n_words': 1549, 'path': 'iteration_1_PDF data extractor.ipynb', 'repo_name': 'Jjk422/MSc-project', 'size': '21921'}