<jupyter_start><jupyter_text>Information Retrieval - Assigment 2Group 3: <NAME>, <NAME>, <NAME>  Import packages<jupyter_code>import nltk
import re
from numpy import cumsum
from pandas import DataFrame<jupyter_output><empty_output><jupyter_text>Import data<jupyter_code>#read textual documents from file
documents_path = 'AssociatedPress.txt'
with open(documents_path, 'r', encoding='utf-8') as doc_f:
    corpus_list = doc_f.readlines()<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def createTokens(collection):
    
    '''
    given a collection as input, it gives a list of all the tokens of that collection
    and a list lists where each list contains the tokens for that document of the collection
    
    '''
    
    #create create a list of lists for the tokens of each document
    tokenized_collection = []

    for doc in collection:
        tokenized_collection.append(nltk.tokenize.word_tokenize(doc))
        
        
    #create a list of the total tokens
    tokens = []

    for doc_tokens in tokenized_collection:
        for word in doc_tokens:
            tokens.append(word)
            
            
    return tokens, tokenized_collection;
def distinctTerms(tokenized_collection, tokens):
    
    '''
    given a tokenized collection and the total tokens as input, it gives a list of all 
    the distinct terms of that collection and a list lists where each list contains 
    the distinct terms for that document of the collection
    
    '''
    
    #create create a list of lists for the distinct terms of each document
    distinct_terms_collection = []

    for doc_tokens in tokenized_collection:
        distinct_terms_collection.append(sorted(set(doc_tokens)))
        
        
    #create a list of the total distinct terms
    distinct_terms = list(set(tokens))
            
            
    return distinct_terms, distinct_terms_collection;
def is_number(s):
    if "," in s:
        s = s.replace(",","")
    try:
        float(s)
        return True
    except ValueError:
        return False
def TermsNoNum(terms):
    
    '''
    given as input a list of terms, it gives a list of terms without numbers
    
    '''
    
    terms_no_number = []

    for term in terms:
        if not is_number(term):
            terms_no_number.append(term) 
            
    return terms_no_number;
       
def keepOnlyAlph(terms):
    
    '''
    given a list of strings, it checks if the strings are composed just by
    alphabetical values and returns a list containing only those strings 
    
    '''
    
    only_alpha_value_terms = []
    
    for term in terms:
        if term.isalpha() or ("-" in term and re.search("[a-zA-Z]", term.replace("-",""))):
            only_alpha_value_terms.append(term)
                   
    return only_alpha_value_terms
def caseFolding(terms):
    
    '''
    given as input a list of terms, it gives a list of terms without uppercases
    
    '''

    terms_no_uppercase = []
    
    for term in terms:
        if term.isalpha():
            terms_no_uppercase.append(term.lower()) 
        else:
            terms_no_uppercase.append(term)
            
    return terms_no_uppercase
def removeStopWords(terms, n):
    
    '''
    given as input a list of terms and the number of stop words to be removed, 
    it gives a list of terms removing those stop words stop words
    if the number of stop words is bigger than the total number of stop words we have
    it will remove the maximum number of stop words possible
    
    '''

    terms_no_stopwords = []
    stop_words = nltk.corpus.stopwords.words('english')
    
    if n >= len(stop_words):
        n = len(stop_words)
    
    for term in terms:
        if term not in stop_words[0:n]:
            terms_no_stopwords.append(term)
            
    return terms_no_stopwords
def stemming(terms):
    
    '''
    given as input a list of terms, it gives a list of terms after stemming
    we can have repetitions
    
    '''

    ps = nltk.stem.PorterStemmer()
    terms_stemmed = []
    
    for term in terms:
        terms_stemmed.append(ps.stem(term))

    return terms_stemmed

def frequency(terms):
    
    '''
    given as input a list of tokens, it computes the frequency of a token in that list
    
    '''    

    return nltk.FreqDist(terms)
def reductionPercentage(before_red, after_red):
    
    '''
    takes as input 2 ints that represent a value before a given reduction and a value
    after a reduction has been applied and compute the percentange of that reduction
    
    '''
    
    perc_red = int(((before_red - after_red)*100)/before_red)
    
    return perc_red
def allReductionPercentage(numbers):
    '''
    takes as input a list of numbers and compure the sorted percentage reduction
    between consecutive values as a list. The firts element will be always 0
    
    '''
    reduction = [0]
    
    for i in range(1,len(numbers)):
        j = i-1
        reduction.append(reductionPercentage(numbers[j], numbers[i]))
        
    return reduction;
    
def proportionWithUnfiltered(unfiltered_value, value_to_compare):
    
    '''
    it takes as input a value after reduction and a value before reduction and
    computes the proportion with the value of unfiltered terms 
    
    '''
    
    return int(((unfiltered_value-value_to_compare)/unfiltered_value)*100)<jupyter_output><empty_output><jupyter_text>Build the table of reductions<jupyter_code>Distinct_Terms = {"Type": ["Unfiltered", "No numbers", "Terms without symbols", "Case folding", "30 Stop Words", "150 Stop Words", "Stemming"]}<jupyter_output><empty_output><jupyter_text>Tokens<jupyter_code>tokenized_terms, tokenized_terms_per_doc = createTokens(corpus_list)<jupyter_output><empty_output><jupyter_text>Distinct terms<jupyter_code>distinct_terms, distinct_terms_per_doc = distinctTerms(tokenized_terms_per_doc, tokenized_terms)
unfiltered_value = len(distinct_terms)<jupyter_output><empty_output><jupyter_text>Distinct terms without numbers<jupyter_code>no_numbers = TermsNoNum(distinct_terms)
no_numbers_value = len(no_numbers)<jupyter_output><empty_output><jupyter_text>Distinct terms without symbols (except for combined words e.g. "first-degree")<jupyter_code>only_alpha_terms = keepOnlyAlph(no_numbers)
only_alpha_terms_value = len(only_alpha_terms)<jupyter_output><empty_output><jupyter_text>Distinct terms after case folding<jupyter_code>case_fold = list(set(caseFolding(no_numbers)))
case_fold_value = len(case_fold)<jupyter_output><empty_output><jupyter_text>Distinct terms after removing 30 stop words<jupyter_code>stop_words_30 = removeStopWords(case_fold, 30)
stop_words_30_value = len(stop_words_30)<jupyter_output><empty_output><jupyter_text>Distinct terms after removing 150 stop words<jupyter_code>stop_words_150 = removeStopWords(case_fold, 150)
stop_words_150_value = len(stop_words_150)<jupyter_output><empty_output><jupyter_text>Distinct terms after stemming<jupyter_code>stemmed = list(set(stemming(stop_words_150)))
stemmed_value = len(stemmed)<jupyter_output><empty_output><jupyter_text>Final results<jupyter_code>number_filtered = [unfiltered_value, no_numbers_value, only_alpha_terms_value ,case_fold_value, 
                   stop_words_30_value, stop_words_150_value, stemmed_value]


reduction_filtered = allReductionPercentage(number_filtered)

proportion_with_unfiltered = [0]
for value in number_filtered[1:]:
    proportion_with_unfiltered.append(proportionWithUnfiltered(unfiltered_value, value))

Distinct_Terms['Numbers'] = number_filtered
Distinct_Terms['Reduction %'] = reduction_filtered
Distinct_Terms['Proportion with unfiltered %'] = proportion_with_unfiltered

table_distinct_terms = DataFrame(Distinct_Terms)
table_distinct_terms<jupyter_output><empty_output><jupyter_text>Frequency<jupyter_code>frequency(caseFolding(tokenized_terms)).most_common(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.07783824832299709, 'fraction_numerical': 0.007214276673838754, 'mean_word_length': 3.66469893742621, 'pattern_counts': {'":': 1, '<': 60, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '121974', 'license': 'MIT', 'n_tokens_mistral': 2527, 'n_tokens_neox': 2374, 'n_words': 638, 'path': '03_information_retrieval/assignments/01_corpus/corpus_assignment.ipynb', 'repo_name': 'rachelfanti/data_science', 'size': '7927'}