<jupyter_start><jupyter_text>Step by step web-scraping of "Media Bias" data from https://www.allsides.com Tools used: python, requests, beautifulsoup<jupyter_code>import requests
from bs4 import BeautifulSoup<jupyter_output><empty_output><jupyter_text>Request the first page<jupyter_code>url = 'https://www.allsides.com/media-bias/media-bias-ratings'
r = requests.get(url)
print(r.content[:100])<jupyter_output>b'<!DOCTYPE html>\n<!--[if IEMobile 7]><html class="iem7"  lang="en" dir="ltr"><![endif]-->\n<!--[if lte'<jupyter_text>Create BeautifulSoup object<jupyter_code>soup = BeautifulSoup(r.content, 'html.parser')
soup<jupyter_output><empty_output><jupyter_text>Finding elements and data from Soup object<jupyter_code># To get each row of table, we just select all <tr> inside <tbody>
rows = soup.select('tbody tr')
rows<jupyter_output><empty_output><jupyter_text>New Source Name<jupyter_code>row = rows[0]
name = row.select_one('.source-title').text.strip()
print(name)<jupyter_output>ABC News<jupyter_text>News source page link<jupyter_code>allsides_page = row.select_one('.source-title a')['href']
allsides_page = 'https://www.allsides.com' + allsides_page
print(allsides_page)<jupyter_output>https://www.allsides.com/news-source/abc-news-media-bias<jupyter_text>Bias rating<jupyter_code>bias = row.select_one('.views-field-field-bias-image a')['href']
bias = bias.split('/')[-1]
print(bias)<jupyter_output>left-center<jupyter_text>Community Feedback Data<jupyter_code>agree = row.select_one('.agree').text
agree = int(agree)

disagree = row.select_one('.disagree').text
disagree = int(disagree)

agree_ratio = agree / disagree

print(f"Agree: {agree}, Disagree: {disagree}, Ratio {agree_ratio:.2f}")
def get_agreeance_text(ratio):
    if ratio > 3: return "absolutely agrees"
    elif 2 < ratio <= 3: return "strongly agrees"
    elif 1.5 < ratio <= 2: return "agrees"
    elif 1 < ratio <= 1.5: return "somewhat agrees"
    elif ratio == 1: return "neutral"
    elif 0.67 < ratio < 1: return "somewhat disagrees"
    elif 0.5 < ratio <= 0.67: return "disagrees"
    elif 0.33 < ratio <= 0.5: return "strongly disagrees"
    elif ratio <= 0.33: return "absolutely disagrees"
    else: return None
    
print(get_agreeance_text(2.5))<jupyter_output>strongly agrees<jupyter_text>So far we explored data extraction for single row, now let's loop and pull all data<jupyter_code>data= []

for row in rows:
    d = dict()
    
    d['name'] = row.select_one('.source-title').text.strip()
    d['allsides_page'] = 'https://www.allsides.com' + row.select_one('.source-title a')['href']
    d['bias'] = row.select_one('.views-field-field-bias-image a')['href'].split('/')[-1]
    d['agree'] = int(row.select_one('.agree').text)
    d['disagree'] = int(row.select_one('.disagree').text)
    d['agree_ratio'] = d['agree'] / d['disagree']
    d['agreeance_text'] = get_agreeance_text(d['agree_ratio'])
    
    data.append(d)
    
print(data[3])<jupyter_output>{'name': '<NAME>', 'allsides_page': 'https://www.allsides.com/news-source/american-spectator', 'bias': 'right', 'agree': 5324, 'disagree': 2255, 'agree_ratio': 2.3609756097560974, 'agreeance_text': 'strongly agrees'}<jupyter_text>Putting it all togetherRequesting and parsing multiple pages<jupyter_code>pages = [
    'https://www.allsides.com/media-bias/media-bias-ratings',
    'https://www.allsides.com/media-bias/media-bias-ratings?page=1',
    'https://www.allsides.com/media-bias/media-bias-ratings?page=2'
]<jupyter_output><empty_output><jupyter_text>Our loop will:* request a page* parse the page* wait ten seconds (according to AllSides' robots.txt we need to make sure we wait ten seconds before each request)* repeat for next page<jupyter_code>from time import sleep

data= []

for page in pages:
    r = requests.get(page)
    soup = BeautifulSoup(r.content, 'html.parser')
    
    rows = soup.select('tbody tr')

    for row in rows:
        d = dict()

        d['name'] = row.select_one('.source-title').text.strip()
        d['allsides_page'] = 'https://www.allsides.com' + row.select_one('.source-title a')['href']
        d['bias'] = row.select_one('.views-field-field-bias-image a')['href'].split('/')[-1]
        d['agree'] = int(row.select_one('.agree').text)
        d['disagree'] = int(row.select_one('.disagree').text)
        d['total_votes'] = int(d['agree'] + d['disagree'])
        d['agree_ratio'] = d['agree'] / d['disagree']
        d['agreeance_text'] = get_agreeance_text(d['agree_ratio'])

        data.append(d)
    
    sleep(10)
    
print("Total records fetched =", len(data))<jupyter_output>Total records featched = 73<jupyter_text>Fetch website name of each news source<jupyter_code>from tqdm import tqdm_notebook

for d in tqdm_notebook(data):
    r = requests.get(d['allsides_page'])
    soup = BeautifulSoup(r.content, 'html.parser')
    
    try:
        website = soup.select_one('.www')['href']
        d['website'] = website
    except TypeError:
        pass
    
    sleep(10)

print("Total records fetched =", len(data))<jupyter_output><empty_output><jupyter_text>Save the data<jupyter_code>import json

with open('allsides.json', 'w') as f:
    json.dump(data, f)
    

# Test if all records were daved
with open('allsides.json', 'r') as f:
    data = json.load(f)
print("Total records fetched =", len(data))<jupyter_output>Total records fetched = 73
{'fraction_non_alphanumeric': 0.16159731293151708, 'fraction_numerical': 0.014181750326553461, 'mean_word_length': 4.726495726495727, 'pattern_counts': {'":': 0, '<': 66, '<?xml version=': 0, '>': 53, 'https://': 10, 'lorem ipsum': 0, 'www.': 11, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '374066', 'license': 'MIT', 'n_tokens_mistral': 1996, 'n_tokens_neox': 1890, 'n_words': 376, 'path': 'web-scraping/allsides-media-bias-data/allsides-media-bias-data.ipynb', 'repo_name': 'SupratimH/learning-data-science', 'size': '5371'}