<jupyter_start><jupyter_text>Imports<jupyter_code>import pickle
import gc<jupyter_output><empty_output><jupyter_text>Load articles<jupyter_code>wiki_file = '../DATA/eswiki-20200601-pages-articles-multistream.txt'

def load_data(wiki_text_file: str):
    
    '''This function loads the text file that contains all the Spanish Wikipedia Articles'''
    
    with open(wiki_text_file, 'r') as myfile:
        data = myfile.read()
        return data

data_raw = load_data(wiki_file)<jupyter_output><empty_output><jupyter_text>Filter articles<jupyter_code>data_wiki_articles = data_raw.split('<page>')
print(f'Total # of Spanish Wikipedia articles: {len(data_wiki_articles)}')

patterns = ['{{Ficha de artista musical', '|Discográfica', '|discográfica', '|voz', '|compañía discográfica', '|instrumento']
data_artists_articles = [pat for pat in data_wiki_articles if any(p in pat for p in patterns)]

patterns = ['{{Ficha de persona', '{{Ficha de artista musical']
data_artists_articles = [pat for pat in data_artists_articles if any(p in pat for p in patterns)]

patterns = ['{{Ficha de personaje']
data_artists_articles = [pat for pat in data_artists_articles if not any(p in pat for p in patterns)]
print(f'Total # of Spanish Wikipedia Musical Artists articles: {len(data_artists_articles)}')

del data_raw
del data_wiki_articles

gc.collect();<jupyter_output>Total # of Spanish Wikipedia articles: 3928922
Total # of Spanish Wikipedia Musical Artists articles: 26807<jupyter_text>Save the articles<jupyter_code>pickle.dump(data_artists_articles, open( "../DATA/artists_articles_raw.p", "wb" ) )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11083743842364532, 'fraction_numerical': 0.012315270935960592, 'mean_word_length': 6.629107981220657, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '607980', 'license': 'MIT', 'n_tokens_mistral': 577, 'n_tokens_neox': 549, 'n_words': 139, 'path': '1_extract_and_filter_articles.ipynb', 'repo_name': 'Hlaboa/nlg_wikimusica', 'size': '1624'}