<jupyter_start><jupyter_text>TechCrunch<jupyter_code>#Get links from TechCrunch
time_delay = 0.1
nPages = 5
start_page = 1

url = "https://techcrunch.com/page/"

#scrape feed pages for article links
links = []
for i in range(start_page,start_page+nPages):
    response = requests.get(url + str(i))
    soup = BeautifulSoup(response.text, "lxml")
    data = soup.find('div',{"role": "main"}).find_all('h2', {'class': 'post-title'})
    time.sleep(time_delay)
    links.extend([data.find('a',href=True)['href'] for data in data])
    print('done with page', i)
    

#ditch links we don't care about
links = [l for l in links if '/video/' not in re.findall('/\w*/', l)]
links = [l for l in links if '/gallery/' not in re.findall('/\w*/', l)]

print('DONE')
subprocess.call(["C:\\WINDOWS\\system32\\WindowsPowerShell\\v1.0\\powershell.exe", "[console]::beep(500,1000)"])
#scrape data from TechCrunch (most recent first); stop once we've stopped seeing new articles
time_delay = 0.1

site = 'TechCrunch'
print(len(links), 'pages')
for i, url in enumerate(links):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "lxml")
    title = soup.find('h1', {'class': 'alpha tweet-title'}).text
    title = title.replace('\xa0', '')
    try:
        author = soup.find('a', {'rel': 'author'}).text
    except:
        author = None
    date = parser.parse(soup.find('div', {'class': 'byline'}).find('time', datetime=True)['datetime']).replace(hour=0,minute=0, second=0)
    regex = re.compile('.*article-entry text.*')
    body = ' '.join([i.text for i in soup.find('div', class_=regex).find_all('p')])
    body = body.replace('\xa0', '')
    
    tags = [a.find('a').text for a in soup.find_all('div', {'class': 'tag-item'})]
    observation = {'title': title, 'author': author, 'date': date, 'body': body, 'site': site, 'tags': tags}
    try:
        arts.insert_one(observation)
        print('success adding', i)
    except DuplicateKeyError:
        print('duplicate entry on', '"'+ title + '"')
        print('ending scrape')
        break
    time.sleep(time_delay)
    
print('DONE')
subprocess.call(["C:\\WINDOWS\\system32\\WindowsPowerShell\\v1.0\\powershell.exe", "[console]::beep(500,1000)"])<jupyter_output>100 pages
success adding 0
success adding 1
success adding 2
success adding 3
success adding 4
success adding 5
success adding 6
success adding 7
success adding 8
success adding 9
success adding 10
success adding 11
success adding 12
success adding 13
success adding 14
success adding 15
success adding 16
success adding 17
success adding 18
success adding 19
success adding 20
success adding 21
success adding 22
success adding 23
success adding 24
success adding 25
success adding 26
success adding 27
success adding 28
success adding 29
success adding 30
success adding 31
success adding 32
success adding 33
success adding 34
success adding 35
success adding 36
success adding 37
success adding 38
success adding 39
success adding 40
success adding 41
success adding 42
success adding 43
success adding 44
success adding 45
success adding 46
success adding 47
success adding 48
success adding 49
duplicate entry on "Roku cracks down on private channels"
ending scrape
DONE<jupyter_text>ArsTechnica<jupyter_code>#Get links from arstechnica
time_delay = 0.5
nPages = 4
start_page = 1

url = "https://arstechnica.com/page/"

links = []
for i in range(start_page,start_page+nPages):
    response = requests.get(url + str(i))
    soup = BeautifulSoup(response.text, "lxml")
    data = soup.find_all('article')
    time.sleep(time_delay)
    links.extend([data.find('a',href=True)['href'] for data in data])
    print('done with page', i)
    
print('DONE')
subprocess.call(["C:\\WINDOWS\\system32\\WindowsPowerShell\\v1.0\\powershell.exe", "[console]::beep(500,1000)"])
#scrape data from ArsTechnica links; stop once we've stopped adding new articles
time_delay = 0.5

site ='ArsTechnica'
print(len(links), 'pages')
for i, url in enumerate(links):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "lxml")
    
    title = soup.find('h1', {'itemprop': 'headline'}).text
    title = title.replace('\xa0', '')
    author = soup.find('p', {'class':'byline'}).find('span', {'itemprop': 'name'}).text
    date = parser.parse(soup.find('p', {'class':'byline'}).find('time').text).replace(hour=0,minute=0, second=0)
    body = ' '.join([a.text for a in soup.find('div', {'itemprop': 'articleBody'}).find_all('p')])
    body = body.replace('\xa0', '')
    
    tags = [i.split('/')[1] for i in re.findall('com/[a-zA-Z]+/', links[0])]
    
    observation = {'title': title, 'author': author, 'date': date, 'body': body, 'site': site, 'tags': tags}
    try:
        arts.insert_one(observation)
        print('success adding', i)
    except DuplicateKeyError:
        print('error: duplicate entry on', title)
        print('ending scrape')
        break
    time.sleep(time_delay)
    
print('DONE')
subprocess.call(["C:\\WINDOWS\\system32\\WindowsPowerShell\\v1.0\\powershell.exe", "[console]::beep(500,1000)"])
arts.count()<jupyter_output><empty_output><jupyter_text>Engadget<jupyter_code>#Get links from arstechnica
time_delay = 0.5
nPages = 5
start_page = 1

url = "https://www.engadget.com/all/page/"

links = []
for i in range(start_page,start_page+nPages):
    response = requests.get(url + str(i))
    soup = BeautifulSoup(response.text, "lxml")
    data = [x['href'] for x in soup.find('div', class_="pb-40@m- ").find_all('a',href=True, class_="o-hit__link")]
    time.sleep(time_delay)
    links.extend(data)
    print('done with page', i)
    
print('DONE')
subprocess.call(["C:\\WINDOWS\\system32\\WindowsPowerShell\\v1.0\\powershell.exe", "[console]::beep(500,1000)"])
#scrape data from ArsTechnica links; stop once we've stopped adding new articles
time_delay = 0.5

site ='Engadget'
print(len(links), 'pages')
for i, url in enumerate(links[10:]):
    url = 'https://www.engadget.com' + url
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "lxml")
    try:
        title = soup.find('h1', {'class': 't-h4@m- t-h1-b@tp t-h1@tl+ mt-20 mt-15@tp mt-0@m-'}).text
        title = title.replace('\xa0', '')
        author = soup.find('div', {'class':"t-meta-small@s t-meta@m+"}).find('a').text

        body = ' '.join([a.text for a in soup.find('div', class_="flush-top flush-bottom").find_all('p')])
        body = body.replace('\xa0', '')
        #print(body)

        tags = [soup.find('div', class_="grid@tl+__cell col-12-of-12@tp col-5-of-12@tl+ vm").find('div', class_="th-meta").find('a')['href']]
        tags = [i.split('/')[-2] for i in tags]

        date_string = re.findall('[0-9]+', url)
        date = '/'.join(date_string[:3])
        date = parser.parse(date)

        observation = {'title': title, 'author': author, 'date': date, 'body': body, 'site': site, 'tags': tags}
        try:
            arts.insert_one(observation)
            print('success adding', i)
        except DuplicateKeyError:
            print('error: duplicate entry on', title)
            print('ending scrape')
            break
    except:
        print('error on article', i)
    time.sleep(time_delay)
    
print('DONE')
subprocess.call(["C:\\WINDOWS\\system32\\WindowsPowerShell\\v1.0\\powershell.exe", "[console]::beep(500,1000)"])
arts.count()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1696367841110053, 'fraction_numerical': 0.029655829138892666, 'mean_word_length': 3.8656518861681004, 'pattern_counts': {'":': 1, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '45395', 'license': 'MIT', 'n_tokens_mistral': 2652, 'n_tokens_neox': 2478, 'n_words': 628, 'path': 'scraper.ipynb', 'repo_name': 'avikejriwal/tech_news_NLP', 'size': '7351'}