<jupyter_start><jupyter_text>https://radimrehurek.com/gensim/scripts/segment_wiki.html https://towardsdatascience.com/wikipedia-data-science-working-with-the-worlds-largest-encyclopedia-c08efbac5f5c  https://github.com/tsjamm/WikiSearchMachine https://dumps.wikimedia.org/enwiki/latest/<jupyter_code>datadir = '/Users/klarnemann/Documents/Insight/Insight_project/data'
out_f = '%s/enwiki_titles.npy' % (datadir)
wiki_titles_f = '%s/enwiki-20190301-all-titles' % (datadir)
wiki_titles = []
for i, line in enumerate(open(wiki_titles_f, 'r')):
    wiki_titles.append(line[1:].strip())
    if i % 25 == 0:
        np.save(out_f, np.array(wiki_titles, dtype=str))
    if i % 100000 == 0:
        print(i)
wiki_titles = np.load(out_f)
wiki_f = '%s/enwiki-pages-articles.json.gz' % (datadir)
wiki_data = open(wiki_f)
wiki_data.
smart_open(wiki_f)
article.keys()
list(article['interlinks'].values())[:5]
article['section_titles']
article['section_texts']
i=0
# iterate over the plain text data we just created
for i, line in enumerate(open(wiki_f)):
    if i < 10:
        article = json.loads(line)
        print(article['title'])
        i += 1
    else:
        break<jupyter_output>Anarchism
Autism
Albedo
A
Alabama
Achilles
<NAME>
Aristotle
An American in Paris
Academy Award for Best Production Design<jupyter_text>Alternate<jupyter_code>base_url = 'https://dumps.wikimedia.org/enwiki/'
index = requests.get(base_url).text
soup_index = BeautifulSoup(index, 'html.parser')

# Find the links on the page
dumps = [a['href'] for a in soup_index.find_all('a') if 
         a.has_attr('href')]
dumps
dump_url = base_url + '20190520/'

# Retrieve the html
dump_html = requests.get(dump_url).text

# Convert to a soup
soup_dump = BeautifulSoup(dump_html, 'html.parser')

# Find list elements with the class file
soup_dump.find_all('li', {'class': 'file'})[:3]
saved_file_path = get_file(file, url)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14136671883150756, 'fraction_numerical': 0.019300991131977047, 'mean_word_length': 4.919753086419753, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '304438', 'license': 'MIT', 'n_tokens_mistral': 762, 'n_tokens_neox': 724, 'n_words': 135, 'path': 'notebooks/Wikipedia.ipynb', 'repo_name': 'klarnemann/pointsofinterest', 'size': '1926'}