<jupyter_start><jupyter_text>Scrape Wikipedia<jupyter_code>!pip install wikipedia
import wikipedia
import re
import pandas as pd
with open ("./data/misc/10000_articles_wikipedia_must_have.txt", encoding="utf-8") as f:
    wikipedia_urls = f.read().splitlines()

# remove possible duplicates
wikipedia_urls = list(dict.fromkeys(wikipedia_urls))
print(len(wikipedia_urls))
def articles_scrape(wikipedia_raw):
    article_content = []
    
    # remove the last sections with references etc     
    seperator = '== See also =='
    stripped_article = wikipedia_raw.split(seperator, 1)[0]

    # Delete all section titles      
    stripped_article = re.sub(r'==.*?==+', '', stripped_article)
    
    # remove content inside parentheses (usually unecessary information for our cause)     
    stripped_article = re.sub(r'\([^)]*\)', '', stripped_article)
    
    # remove urls     
    stripped_article = re.sub(r'^https?:\/\/.*[\r\n]*', '', stripped_article, flags=re.MULTILINE)  

    # remove '\n'
    stripped_article = stripped_article.replace('\n', ' ') 

    # remove big spaces
    stripped_article = re.sub('\s{2,}', " ", stripped_article)

    for line in stripped_article.splitlines():
        stripped_line = line.strip()        
        article_content.append(stripped_line)

    return '\n'.join(article_content) + '\n'
scraped_articles = []

# set the simple wikipedia version to be searched for
wikipedia.set_lang("simple")

# scrape wikipedia and retrieve the content of each article for each url
for url in wikipedia_urls:
    try:
        query = wikipedia.search(url)[0]
        page = wikipedia.page(query)
    except Exception:
        pass    
    scraped_pages = articles_scrape(page.content)
    scraped_articles.append(scraped_pages)<jupyter_output>/usr/local/lib/python3.7/dist-packages/wikipedia/wikipedia.py:389: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system ("lxml"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.

The code that caused this warning is on line 389 of the file /usr/local/lib/python3.7/dist-packages/wikipedia/wikipedia.py. To get rid of this warning, pass the additional argument 'features="lxml"' to the BeautifulSoup constructor.

  lis = BeautifulSoup(html).find_all('li')<jupyter_text>Preprocess the scraped articles<jupyter_code>import pickle
import statistics
import nltk
import string
import pandas as pd
from nltk.tokenize import RegexpTokenizer
nltk.download('punkt')
# this function is to avoid having mathematic-specific sentences, by controling on the number of allowed numbers that are contained
def delete_num_sentences(text):
    numbers = []
    tokenized_text = text.split()

    for word in tokenized_text:
        # some numbers might be in the form x.x or x,x
        if '.' in word or ',' in word:
            word = word.replace('.','').replace(',','')

        if word.isdigit():
            numbers.append(word)
  
    # accept until 2 numbers
    if len(numbers)<2:
        return text
    else:
        return False
# examine if words with non english chatacters exist
def check_all_english(text):
    english_texts = []
    tokenizer = RegexpTokenizer(r'\w+')
    tokenized_text = tokenizer.tokenize(text)

    result = all(word.encode().isalpha() for word in tokenized_text if not word.isdigit())

    if result:
        return text
    else:
        return False
def extra_preproces(sentences):
    cleaned_sentences = []

    for sentence in sentences:
        # find the sentences that start with a punctuation mark and skip them
        if (sentence.strip()[0] not in string.punctuation):

            # skip sentences containing digits
            sentence = delete_num_sentences(sentence)
            if (sentence):

                # skip sentences containing non-english words
                sentence = check_all_english(sentence)
                if (sentence):
          
                    # clean surrounding whitespace
                    sentence = sentence.strip()
                    # remove _
                    sentence = sentence.replace('_', ' ')
                    # remove `
                    sentence = sentence.replace('`', ' ')
                    # remove )
                    sentence = sentence.replace(')', ' ')
                    # remove [
                    sentence = sentence.replace('[', ' ')

                    # further process of some incosistencies
                    if (", ,") in sentence:
                        sentence = sentence.replace(", ,",",") 

                    # if a sentence starts with empty characters, delete them
                    if sentence[0] == ' ':
                        sentence = sentence[1:].strip()

                    cleaned_sentences.append(sentence)

  # remove empty elements of a list
  cleaned_sentences = list(filter(None, cleaned_sentences))
  
  return cleaned_sentences
# cleaning to retain only actual useful sentences
def filter_sentences(sentences):
    filtered_sentences = []
    length_sentences = []

    for sentence in sentences:

    # in some cases a punctuation mark would be classified as sentence 
    if sentence not in '".,;!-':
        token_sentence = nltk.word_tokenize(sentence)

        # clean rows indicating the ISBN (which was a recursive pattern in the corpus with no use)
        if ('ISBN') not in token_sentence:
            length_sentences.append(len(token_sentence))
            filtered_sentences.append(sentence)

    # returns a tuple of the filtered sentences and their lengths
    # so that we can have an insight on the average length of sentences  
    return (length_sentences, filtered_sentences)
corpus_list_tupled = []

# split the scraped articles in sentences
for wiki_article in scraped_articles:
    wikis_sentences = nltk.sent_tokenize(wiki_article)
    filter_wikis_sentences = filter_sentences(wikis_sentences)

    corpus_list_tupled.append(filter_wikis_sentences)
corpus_list_lengths = [elem[0] for elem in corpus_list_tupled]
lengths_flat_list = [item for sublist in corpus_list_lengths for item in sublist]

corpus_list_texts = [elem[1] for elem in corpus_list_tupled]  
texts_flat_list = [item for sublist in corpus_list_texts for item in sublist]
# calculate the average length
sentence_average_len = statistics.mean(lengths_flat_list)
print(sentence_average_len)
## remove possible duplicates
final_corpus = list(dict.fromkeys(texts_flat_list))
print(len(texts_flat_list))
print(len(final_corpus))
# expand abbreviations based on a predefined dictionary
abbr_dict={"what's":"what is", "what're":"what are", "who's":"who is", "who're":"who are", "where's":"where is", "where're":"where are", "when's":"when is", 
           "when're":"when are", "how's":"how is", "how're":"how are", "i'm":"i am", "we're":"we are", "you're":"you are", "they're":"they are", "it's":"it is",
           "he's":"he is", "she's":"she is", "that's":"that is", "there's":"there is", "there're":"there are", "i've":"i have", "we've":"we have", "you've":"you have",
    "they've":"they have", "who've":"who have", "would've":"would have", "not've":"not have", "i'll":"i will", "we'll":"we will", "you'll":"you will", "he'll":"he will",
    "she'll":"she will", "it'll":"it will", "they'll":"they will", "isn't":"is not", "wasn't":"was not", "aren't":"are not", "weren't":"were not", "can't":"can not",
    "couldn't":"could not", "don't":"do not", "didn't":"did not", "shouldn't":"should not", "wouldn't":"would not", "doesn't":"does not", "haven't":"have not",
    "hasn't":"has not", "hadn't":"had not", "won't":"will not", "what' s":"what is", "what' re":"what are", "who' s":"who is", "who' re":"who are", "where' s":"where is",
    "where' re":"where are", "when' s":"when is", "when' re":"when are", "how' s":"how is", "how' re":"how are", "i' m":"i am", "we' re":"we are", "you' re":"you are",
    "they' re":"they are", "it' s":"it is", "he' s":"he is", "she' s":"she is", "that' s":"that is", "there' s":"there is", "there' re":"there are", "i' ve":"i have",
    "we' ve":"we have", "you' ve":"you have", "they' ve":"they have", "who' ve":"who have", "would' ve":"would have", "not' ve":"not have", "i' ll":"i will", "we' ll":"we will",
    "you' ll":"you will", "he' ll":"he will", "she' ll":"she will", "it' ll":"it will", "they' ll":"they will", "isn' t":"is not", "wasn' t":"was not", "aren' t":"are not",
    "weren' t":"were not", "can' t":"can not", "couldn' t":"could not", "don' t":"do not", "didn' t":"did not", "shouldn' t":"should not", "wouldn' t":"would not",
    "doesn' t":"does not", "haven' t":"have not", "hasn' t":"has not", "hadn' t":"had not", "won' t":"will not"}

# this is just to be sure that the quotes will be those we have used in our abbreviation lexicon
quote_list = "‘’‛’❜'’`‘’"

abbr_corpus = []
for elem in final_corpus:

    # make all the single quotes, the one we have used in the abbreviation lexicon
    expand_abbr = ["'" if e in quote_list else e for e in elem]
    expand_abbr_string = ''.join(expand_abbr)
    abbr_corpus.append(expand_abbr_string)

final_corpus_df = pd.DataFrame(abbr_corpus, columns=['Sentences']) 
final_corpus_df.replace(abbr_dict,regex=True,inplace=True)
final_corpus = final_corpus_df.Sentences.tolist()
# further preprocessing of each sentence
final_clean_corpus = extra_preproces(final_corpus)
final_clean_corpus
# save the sentences of the corpus in a pickle format
with open('./data/misc/wikipedia_sentences_unparsed.pkl', 'wb') as f:
    pickle.dump(final_clean_corpus, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14086255041886442, 'fraction_numerical': 0.0027924294135898233, 'mean_word_length': 3.542038515735087, 'pattern_counts': {'":': 98, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '570393', 'license': 'MIT', 'n_tokens_mistral': 3101, 'n_tokens_neox': 2920, 'n_words': 887, 'path': 'notebooks/.ipynb_checkpoints/scrape_wikipedia-checkpoint.ipynb', 'repo_name': 'Valadis-Mastoras/SemiGramEx', 'size': '9669'}