<jupyter_start><jupyter_text>Download montag.wtf<jupyter_code>articles = []
headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'}
def get_links(i):
    with requests.get(f'https://www.montag.wtf/page/{i}/', headers=headers, timeout=10) as f:
        soup = BeautifulSoup(f.text, features="html.parser")
        for article in soup.findAll(class_="homepage-post-info"):
            for x in article.findAll("a"):
                articles.append(f'https://www.montag.wtf{x["href"]}')
%%time
with ThreadPoolExecutor(27) as executor:
    results = executor.map(get_links, range(1,27))
len(articles)
def parse(url):
    with requests.get(url, headers=headers) as f:
        soup = BeautifulSoup(f.text, features='html.parser')
    article_text = soup.findAll(class_='content-text')
    words = "<|startoftext|>"
    for element in article_text:
        words += element.get_text()
    with open('montag.txt', 'a', encoding='utf-8') as f:
        f.write(words + "<|endoftext|>\n")
%%time

with ThreadPoolExecutor(100) as executor:
    results = executor.map(parse, articles[1:])<jupyter_output>Wall time: 1.63 s<jupyter_text>Download OneZero<jupyter_code>from selenium import webdriver
import shutil
import time
url = 'https://onezero.medium.com/trending'
driver = webdriver.Chrome()
driver.get(url)

for _ in range(1000):
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;")
#     time.sleep(.5)

articles = []
for date in driver.find_elements_by_tag_name('time'):
    link = date.find_element_by_xpath('..').get_attribute('href')
    articles.append(link)

driver.quit()
len(articles)
for i in range(0, 10):
    articles[i] = re.findall('(.*)\?', articles[i])[0]
for i in range(len(articles)):
    articles[i] += '?format=json'
with open('onezero_urls.txt', 'a') as f:
    for link in set(articles):
        f.write(link)
articles[-1]
with open('onezero_urls.txt', 'r') as f:
    articles = f.readlines()
# https://hackersandslackers.com/extract-data-from-complex-json-python/
def extract_values(obj, key):
    """Pull all values of specified key from nested JSON."""
    arr = []

    def extract(obj, arr, key):
        """Recursively search for values of key in JSON tree."""
        if isinstance(obj, dict):
            for k, v in obj.items():
                if isinstance(v, (dict, list)):
                    extract(v, arr, key)
                elif k == key:
                    arr.append(v)
        elif isinstance(obj, list):
            for item in obj:
                extract(item, arr, key)
        return arr

    results = extract(obj, arr, key)
    return results
headers = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'}
def parse(url):
    with requests.get(url, headers=headers) as response:
        soup = BeautifulSoup(response.text, features='html.parser')
#     article_text = soup.find(class_='section-inner')
#     with open("onezero.txt", "a") as f:
#         f.write('\n' + article_text.find('h1').get_text() + '\n')
#         f.write(article_text.find('h2').get_text() + '\n')
#         for element in article_text.findAll('p'):
#             words = element.get_text() + '\n'
#             f.write(words)
    image = soup.find(class_='progressiveMedia-image')
    image_url = image['data-src']
    response = requests.get(image_url, stream=True)
    image_name = re.findall('\*(.*)', image_url)[0]
    with open(f'D:/datasets/fake_news/hero_images/one_zero/{image_name}', 'wb') as out_file:
        shutil.copyfileobj(response.raw, out_file)
    del response
def parse_json(url):
    with requests.get(url) as response:
        article = json.loads(response.text[16:])
    text = extract_values(article, 'text')
    words = "<|startoftext|>"
    for paragraph in text:
            words += paragraph + '\n'

    with open('onezero.txt', 'a', encoding='utf-8') as f:
        f.write(words + "<|endoftext|>\n")
%%time

with ThreadPoolExecutor(100) as executor:
    results = executor.map(parse_json, articles)<jupyter_output>Wall time: 7.98 s<jupyter_text>Download Futura<jupyter_code>from selenium import webdriver
import shutil
import time
import json
import re
url = 'https://medium.com/short-sci-fi-stories/latest'
driver = webdriver.Chrome()
driver.get(url)

for _ in range(50):
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);var lenOfPage=document.body.scrollHeight;return lenOfPage;")
    time.sleep(.5)

articles = []
for date in driver.find_elements_by_tag_name('time'):
    link = date.find_element_by_xpath('..').get_attribute('href')
    articles.append(link)

driver.quit()
len(articles)
for i in range(0, 10):
    articles[i] = re.findall('(.*)\?', articles[i])[0]
for i in range(len(articles)):
    articles[i] += '?format=json'
with open('futura_urls.txt', 'a') as f:
    for link in set(articles):
        f.write(link + '\n')
# https://hackersandslackers.com/extract-data-from-complex-json-python/
def extract_values(obj, key):
    """Pull all values of specified key from nested JSON."""
    arr = []

    def extract(obj, arr, key):
        """Recursively search for values of key in JSON tree."""
        if isinstance(obj, dict):
            for k, v in obj.items():
                if isinstance(v, (dict, list)):
                    extract(v, arr, key)
                elif k == key:
                    arr.append(v)
        elif isinstance(obj, list):
            for item in obj:
                extract(item, arr, key)
        return arr

    results = extract(obj, arr, key)
    return results
def parse_json(url):
    with requests.get(url) as response:
        article = json.loads(response.text[16:])
    text = extract_values(article, 'text')
    words = "<|startoftext|>"
    for paragraph in text:
            words += paragraph + '\n'

    with open('futura.txt', 'a', encoding='utf-8') as f:
        f.write(words + "<|endoftext|>\n")
%%time

with ThreadPoolExecutor(80) as executor:
    results = executor.map(parse_json, articles)<jupyter_output>Wall time: 1.34 s
{'fraction_non_alphanumeric': 0.15187028415476, 'fraction_numerical': 0.017338256541981057, 'mean_word_length': 3.2352141400407888, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '337612', 'license': 'Unlicense', 'n_tokens_mistral': 2160, 'n_tokens_neox': 2059, 'n_words': 478, 'path': 'tests/Dataset Scraping.ipynb', 'repo_name': 'sirmammingtonham/futureNEWS', 'size': '6229'}