<jupyter_start><jupyter_text>Unstructured Text Dataset Creation with Wikipedia > https://wikipedia.readthedocs.io/en/latest/code.htmlapi<jupyter_code>import wikipedia
import pandas as pd
import numpy as np
import spacy
nlp = spacy.load('en_core_web_lg')
nlp.add_pipe(nlp.create_pipe('sentencizer'))
wikipedia.WikipediaPage('Something').content
wikipedia.search('StarWars', suggestion=True)
wikipedia.summary('StarWars')
content = wikipedia.WikipediaPage('StarWars').content<jupyter_output><empty_output><jupyter_text>Get topics from Wikipedia content:<jupyter_code>topics = []
with open("./Ontology_Learning/dataset/starwars_topics.txt", "r") as file:
    for line in file:
        topics.append(line.strip())
len(topics)
total = 0
for i, topic in enumerate(topics):
    content = wikipedia.WikipediaPage(topic).content
    doc = nlp(content)
    sentences = [sent.string.strip() for sent in doc.sents]
    with open("./Ontology_Learning/dataset/starwars_text_dataset.txt", "a") as file:
        for sentence in sentences:
            file.write(sentence + "\n")
    total += len(sentences)
    print('Topic{}: {}, Sentences: {}, Total: {}'.format(i, topic, len(sentences), total))
dataset = pd.DataFrame(data=[['StarWars', content]], columns=['query_term','text'])
dataset
dataset = dataset.append(pd.DataFrame(data=[['<NAME>', wikipedia.WikipediaPage('Anakin Skywalker').content]], columns=['query_term','text']), ignore_index=True)
dataset = dataset.append(pd.DataFrame(data=[['<NAME>', wikipedia.WikipediaPage('Ge<NAME>as').content]], columns=['query_term','text']), ignore_index=True)
dataset = dataset.append(pd.DataFrame(data=[['Jedi', wikipedia.WikipediaPage('Jedi').content]], columns=['query_term','text']), ignore_index=True)
dataset = dataset.append(pd.DataFrame(data=[['Millennium Falcon', wikipedia.WikipediaPage('Millennium Falcon').content]], columns=['query_term','text']), ignore_index=True)
dataset
dataset.to_csv('starwars_data.csv')<jupyter_output><empty_output><jupyter_text>Text Cleaning<jupyter_code>import spacy
nlp = spacy.load("en_core_web_lg", disable=["tagger", "ner"])
nlp.pipeline
for index, row in dataset.iterrows():
    doc = nlp(row.text)
    sentences = [sent.string.strip() for sent in doc.sents]
    #print(sentences)
def set_custom_boundaries(doc):
    for token in doc[:-1]:
        if token.text in (":"):
            doc[token.i].is_sent_start = False
    return doc

nlp.add_pipe(set_custom_boundaries, before="parser")
nlp.pipeline
parsed = []
doc = nlp(dataset.iloc[0].text)
sentences = [sent.string.strip() for sent in doc.sents]
for sent in sentences:
    if len(sent) > 10 and '=' not in sent:
        parsed.append(sent)
nlp = spacy.load("en_core_web_lg")<jupyter_output><empty_output><jupyter_text>StarWarsAnakin Skywalker<NAME>Millennium Falcon<jupyter_code>doc = nlp(dataset.iloc[4].text)
results = pd.DataFrame(columns=['Text', 'Start', 'End', 'Label'])

for ent in doc.ents:  
    results = results.append({'Text':ent.text, 'Start':ent.start_char, 'End':ent.end_char, 'Label':ent.label_}, ignore_index=True)
results
results.to_csv('./entities/MillenniumFalcon_entities.csv')<jupyter_output><empty_output><jupyter_text>Read OL Dataset<jupyter_code>import spacy
nlp = spacy.load('en_core_web_lg')
dataset = pd.read_csv('./Ontology_Learning/dataset/starwars_OL_dataset.txt', delimiter='\n', header=None)
dataset
len(set(keywords))<jupyter_output><empty_output><jupyter_text>Automatic Dataset Creation<jupyter_code>start_topic = 'Military Weapons'
topics = wikipedia.search(start_topic, suggestion=False)
topics
curr_topic = 'Military Weapons'
topic_threshold = 100
all_topics = []

topic_count = 0
index = 0
while topic_count < topic_threshold:
    all_topics = list(set(all_topics))
    topics = wikipedia.search(curr_topic, suggestion=False)
    all_topics += topics
    curr_topic = all_topics[index]
    index += 1
    topic_count = len(all_topics)
    print('topic_count: {}'.format(topic_count))
dataset_name = 'military_weapons_dataset.txt'

total = 0
for i, topic in enumerate(all_topics):
    try:
        content = wikipedia.WikipediaPage(topic).content
        doc = nlp(content)
        sentences = [sent.string.strip() for sent in doc.sents]
        with open('./data/' + dataset_name, 'a') as file:
            for sentence in sentences:
                file.write(sentence + '\n')
        total += len(sentences)
        print('Topic{}: {}, Sentences: {}, Total: {}'.format(i, topic, len(sentences), total))
    except:
        continue<jupyter_output>Topic0: List of equipment of the Venezuelan Army, Sentences: 8, Total: 8
Topic1: United States Military Government in Cuba, Sentences: 46, Total: 54
Topic2: Military technology, Sentences: 250, Total: 304
Topic3: Russian Armed Forces, Sentences: 370, Total: 674
Topic4: List of states with nuclear weapons, Sentences: 223, Total: 897
Topic5: Brazilian Military Junta of 1969, Sentences: 54, Total: 951
Topic6: List of countries by military expenditures, Sentences: 33, Total: 984
Topic7: Mortar (weapon), Sentences: 229, Total: 1213
Topic8: Astra (weapon), Sentences: 30, Total: 1243
Topic9: List of active Russian Air Force aircraft, Sentences: 8, Total: 1251
Topic10: List of Russian flags, Sentences: 51, Total: 1302
Topic11: List of equipment of the Ukrainian Ground Forces, Sentences: 33, Total: 1335
Topic12: Outline of natural science, Sentences: 704, Total: 2039
Topic13: National Reorganization Process, Sentences: 209, Total: 2248
Topic14: Outline of formal science, Sentences: 345, Total: [...]
{'fraction_non_alphanumeric': 0.13724425131269238, 'fraction_numerical': 0.021365200072424408, 'mean_word_length': 5.607655502392345, 'pattern_counts': {'":': 1, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '306014', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1955, 'n_tokens_neox': 1805, 'n_words': 421, 'path': 'notebooks/Dataset_Creation.ipynb', 'repo_name': 'jupyrdf/ipykgb', 'size': '5551'}