<jupyter_start><jupyter_text>Downloading the pages meta history<jupyter_code>import requests
from bs4 import BeautifulSoup
import os
import sys
import keras
base_url = 'https://dumps.wikimedia.org/enwiki/'
index = requests.get(base_url).text
soup_index = BeautifulSoup(index, 'html.parser')

# Find the links that are dates of dumps
dumps = [a['href'] for a in soup_index.find_all('a') if 
         a.has_attr('href')]
dumps
dump_url = base_url + '20181001/'

# Retrieve the html
dump_html = requests.get(dump_url).text

# Convert to a soup
soup_dump = BeautifulSoup(dump_html, 'html.parser')

files = []

# Search through all files and store the relevant ones
for file in soup_dump.find_all('li', {'class': 'file'}):
    text = file.text
    if ('pages-meta-history15.' in text)  and '.bz2' in text:
        files.append((text.split()[0], text.split()[1:]))
print(files)

# Specify files to be downloaded
file_download = [file[0] for file in files][2:10]
file_download
from keras.utils import get_file

# Specify the download location
path = "/home/ec2-user/SageMaker/s3fs-fuse/bucket/wiki_trust/"

data_paths = []
file_info = []

# Iterate through each file
for file in file_download:
    # If file doesn't exist download it 
    if not os.path.exists(path + file):
        print(path+file)
        dump_url_mod = dump_url+file
        print(dump_url_mod)
        data_paths.append(get_file(file, dump_url_mod, cache_dir = path))<jupyter_output>/home/ec2-user/SageMaker/s3fs-fuse/bucket/wiki_trust/enwiki-20181001-pages-meta-history15.xml-p7933820p8024323.bz2
https://dumps.wikimedia.org/enwiki/20181001/enwiki-20181001-pages-meta-history15.xml-p7933820p8024323.bz2
Downloading data from https://dumps.wikimedia.org/enwiki/20181001/enwiki-20181001-pages-meta-history15.xml-p7933820p8024323.bz2
1533509632/1533509622 [==============================] - 737s 0us/step
/home/ec2-user/SageMaker/s3fs-fuse/bucket/wiki_trust/enwiki-20181001-pages-meta-history15.xml-p8024324p8123474.bz2
https://dumps.wikimedia.org/enwiki/20181001/enwiki-20181001-pages-meta-history15.xml-p8024324p8123474.bz2
Downloading data from https://dumps.wikimedia.org/enwiki/20181001/enwiki-20181001-pages-meta-history15.xml-p8024324p8123474.bz2
1088454656/1088449820 [==============================] - 530s 0us/step
/home/ec2-user/SageMaker/s3fs-fuse/bucket/wiki_trust/enwiki-20181001-pages-meta-history15.xml-p8123475p8220780.bz2
https://dumps.wikimedia.org/enwiki/20181001/e[...]
{'fraction_non_alphanumeric': 0.14763458401305057, 'fraction_numerical': 0.11786296900489396, 'mean_word_length': 4.854415274463007, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '437848', 'license': 'MIT', 'n_tokens_mistral': 1090, 'n_tokens_neox': 895, 'n_words': 153, 'path': 'Data_Acquisition/Wikimedia_corpus_downloader.ipynb', 'repo_name': 'Arnab777as3uj/Wikimedia-Trust-Safety', 'size': '2452'}