<jupyter_start><jupyter_text>Web scraping using BeautifulSoup and vkontakte.ru API<jupyter_code>import requests
from bs4 import BeautifulSoup
import time
import pandas as pd
import json<jupyter_output><empty_output><jupyter_text>Let's parse a page from a popular it website Habr. URL: habr.com/ru/all/<jupyter_code>#keywords we will be looking for in parsed data
KEYWORDS = ['python', 'парсинг']
#getting the page itself
habr = requests.get('https://habr.com/ru/all/')
habr
#parsing the page using BeautifulSoup library
soup = BeautifulSoup(habr.text, 'html.parser')
#looking for all articles
posts = soup.find_all('div', class_='tm-article-snippet')
posts
#extracting dates for all articles
posts_date = list(map(lambda x: str(x.find('span', class_='tm-article-snippet__datetime-published').time)[16:40], posts))
posts_date
#extracting headers for all articles
posts_title = list(map(lambda x: x.find('h2', class_='tm-article-snippet__title tm-article-snippet__title_h2').text, posts))
posts_title
#extracting links for all articles
posts_link = list(map(lambda x: 'https://habr.com' + x.find('a', class_='tm-article-snippet__title-link').get('href'), posts))
posts_link
#extracting text for all articles
posts_text = []
for link in posts_link:
    soup = BeautifulSoup(requests.get(link).text)
    time.sleep(0.3)
    text = soup.find(id='post-content-body').text
    posts_text.append(text)
posts_text
#creating a dataframe with the following columns date, title, link, text
#while checking if article's header or text contain the keywords defined earlier
habr_posts = pd.DataFrame()
for date, title, link, text in zip(posts_date, posts_title, posts_link, posts_text):
    if any(keyword.lower() in text.lower() for keyword in KEYWORDS) or any(keyword.lower() in title.lower() for keyword in KEYWORDS):
        row = {'date': date, 'title': title, 'link': link, 'text': text}
        habr_posts = pd.concat([habr_posts, pd.DataFrame([row])])
habr_posts.reset_index().drop(columns='index')<jupyter_output><empty_output><jupyter_text>Let's check if a random email has been hacked using Avast Hack Check<jupyter_code>EMAIL = ['<EMAIL>', '<EMAIL>']
#link to a hidden API which Avast uses to check email for hacks
hidden_api_url = 'https://identityprotection.avast.com/v1/web/query/site-breaches/unauthorized-data'
#mandatory headers
HEADERS = {
    'Vaar-Version': '0',
    'Vaar-Header-App-Product-Name': 'hackcheck-web-avast',
    'Vaar-Header-App-Build-Version': '1.0.0',
}
#extracting information regarding hacks in json format
email_check = requests.post(hidden_api_url, json={'emailAddresses': EMAIL}, headers=HEADERS)
email_check.text
#transforming json into python dictionary
email_check_dict = json.loads(email_check.text)
email_check_dict.keys()
email_check_pd = pd.DataFrame()
#creating dataframe with the following columns email, date, breach_source, and description
for breach_id, breach_source in zip(email_check_dict['breaches'], email_check_dict['data']):
    row = {
        'email': list(email_check_dict['data'][breach_source].keys())[0],
        'date': pd.to_datetime(email_check_dict['breaches'][breach_id]['publishDate'], format='%Y-%m-%d').strftime('%Y-%m-%d'),
        'breach_source': breach_source,
        'description': email_check_dict['breaches'][breach_id]['description']
    }
    email_check_pd = pd.concat([email_check_pd, pd.DataFrame([row])])
email_check_pd.reset_index().drop(columns='index')<jupyter_output><empty_output><jupyter_text>Let's write a script that will get 50 last posts from a given group in Vkontakte.ru, a popular social network within Russian speaking countries<jupyter_code>GROUP = 'netology'
#you would need a vk access token in order to work with website's API
with open('vk_access_token.txt', 'r') as token:
    TOKEN = token.read().strip()
#defining a method to get group's posts
GROUP_REQUEST = 'https://api.vk.com/method/wall.get'
VERSION = '5.58'
SLEEP = 0.33
#defining mandatory parameters
params = {
    'owner_id': -30159897,
    'domain': f'https://vk.com/{GROUP}',
    'access_token': TOKEN,
    'v': VERSION,
    'count': 50
}
#getting posts themselves
vk_posts = requests.get(GROUP_REQUEST, params)
vk_posts
vk_netology_posts = pd.DataFrame()
#creating a dataframe with the following columns date and text
for post in vk_posts.json()['response']['items']:
    row = {
        'date': pd.to_datetime(post['date'], unit='s'),
        'text': post['text']
    }
    vk_netology_posts = pd.concat([vk_netology_posts, pd.DataFrame([row])])
vk_netology_posts.reset_index().drop(columns='index')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13491717523975588, 'fraction_numerical': 0.007192676547515257, 'mean_word_length': 5.660377358490566, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '289017', 'license': 'FTL', 'n_tokens_mistral': 1550, 'n_tokens_neox': 1518, 'n_words': 382, 'path': 'web_scraping_beautifulsoup_api_vk.ipynb', 'repo_name': 'antonpavlovskyi/web_scraping_beautifulsoup_api_vk', 'size': '4595'}