<jupyter_start><jupyter_text>Viewing Source code<jupyter_code>def show_html(url):
    body = requests.get(url)
    soup = bs(body.text, 'html.parser')
    for js in soup('script'):
        js.decompose()
    return soup.prettify()<jupyter_output><empty_output><jupyter_text>Extracting Hyperlinks from sourcecode<jupyter_code>body = requests.get(url)
soup = bs(body.text, 'html.parser')
links = soup.find_all('a')
for link in links:
    print(link.text)<jupyter_output>Skip to Content 

 Browse All

Web Development 

Programming 

Data 

Cloud & Networking 

Mobile 

Security 

IOT & Hardware  

Business 

Browse Categories 
 Web Development 
 Programming 
 Data 
 Cloud & Networking 
 Mobile 
 Security 
 IOT & Hardware 
 Business 

All Products 

All Books 

All Videos 

Best Sellers 

Latest Releases 

Browse By 
 All Products 
 All Books 
 All Videos 
 Best Sellers 
 Latest Releases 

Python 

C# 

C++ 

Javascript 

Java 

AWS 

React 

Azure 

Angular 

PHP 

Trending 
 Python 
 C# 
 C++ 
 Javascript 
 Java 
 AWS 
 React 
 Azure 
 Angular 
 PHP 
 Free Learning

Sign In 


Subscribe 

 Browse All

Web Development 

Programming 

Data 

Cloud & Networking 

Mobile 

Security 

IOT & Hardware 

Business 

Browse Categories 
 Web Development 
 Programming 
 Data 
 Cloud & Networking 
 Mobile 
 Security 
 IOT & Hardware 
 Business 

All Products 

All Books 

All Videos 

Best Sellers 

Latest Releases 

Browse By 
 All Products 
 All Books 
 All Vi[...]<jupyter_text>Get a list of most recommended books<jupyter_code>url = 'https://www.penguinrandomhouse.com/the-read-down/21-books-yoube-been-meaning-to-read'
print(show_html(url))
soup = bs(requests.get(url).text, 'html.parser')
books = soup.find('ol', {'class': 'awesome-list'})
books
soup.find('o', {'class': 'nav-item-text'})
soup
for a in soup.select('h2 a'):
    print(a.text)
url = 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies'
soup = bs(requests.get(url).text, 'lxml')
table = soup.find(id='constituents')
print(table.text)
file = open('sp500.txt', 'w')
type(table)
for row in table.find_all('tr'):
    row=row.text.split('\n')
    if len(row)==12:
        row.append("NA")
    if len(row) > 13:
        row.pop()
    for column in row[1:]:
        file.write(column+ '\t')
    file.write('\n')
#df = pd.read_csv('sp500.txt', sep='\t', encoding='utf-8')
import collections

Person = collections.namedtuple('Person', 'name age gender, sex')
Person
type(Person)
Person.age, Person.count, Person.gender, Person.index, Person.name
Person.__doc__<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11436377829820453, 'fraction_numerical': 0.0078064012490242, 'mean_word_length': 3.601436265709156, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '443449', 'license': 'Apache-2.0', 'n_tokens_mistral': 1004, 'n_tokens_neox': 889, 'n_words': 227, 'path': 'scrape_wikipedia.ipynb', 'repo_name': 'Pradeepgb10/web-scraping', 'size': '2562'}