<jupyter_start><jupyter_text>Pick a website and describe your objective- **Browse through different sites and pick one to scrape.**- **Identify the operation you'd like to scrape from the site. Select the format of the output CSV file.**-**Summarize your project idea and outline your strategy in a Jupyter Notebook.**  Project Outline-We're going to scrape https://github.com/topics- We'll get a list of topics. For each topic, we'll get a topic title, topic page URL and topic description- For each topic, we'll get the top 25 repositories in the topic from the topic page- For each repository, we'll grab the repo name, username, stars and repo URL- For each topic, we'll create a CSV file in the following format:```Repo Name,Username,Stars,Repo URLthree.js,mrdoob,69700,https://github.com/mrdoob/three.jslibgdx,libgdx,18300,https://github.com/libgdx/libgdx```  Use the requests library to download web pages<jupyter_code>!pip install requests --upgrade --quiet
import requests
topics_url = 'https://github.com/topics'
# creating a response object
response = requests.get(topics_url)
response.status_code
page_contents = response.text
page_contents[:200]
with open('webpage.html', 'w') as f:
  f.write(page_contents)<jupyter_output><empty_output><jupyter_text>Use BeautifulSoup to parse and extract information<jupyter_code>!pip install beautifulsoup4 --upgrade --quiet
from bs4 import BeautifulSoup

doc = BeautifulSoup(page_contents, 'html.parser')
type(doc)<jupyter_output><empty_output><jupyter_text>Doing queries on webpage<jupyter_code>p_tags = doc.find_all('p')
len(p_tags)
p_tags[:5]<jupyter_output><empty_output><jupyter_text>Specifying the attribute on the p tag and value on that tag.<jupyter_code>selection_class = 'f3 lh-condensed mb-0 mt-1 Link--primary'
topic_title_tags = doc.find_all('p', {'class': selection_class})
len(topic_title_tags)
topic_title_tags[:5]
desc_selector = 'f5 color-fg-muted mb-0 mt-1'
topic_desc_tags = doc.find_all('p', {'class': desc_selector})

topic_desc_tags[:4]
topic_title_tag0 = topic_title_tags[0]
topic_title_tag0.parent
topic_link_tags = doc.find_all('a', {'class': 'no-underline flex-1 d-flex flex-column'})
len(topic_link_tags)
topic_link_tags[0]['href']
topic0_url = 'https://github.com' + topic_link_tags[0]['href']
print(topic0_url)
topic_title_tags[0].text
topic_titles = []

for tag in topic_title_tags:
  topic_titles.append(tag.text)

print(topic_titles)
topic_descs = []

for tag in topic_desc_tags:
  topic_descs.append(tag.text.strip())

print(topic_descs)
topic_urls = []
base_url = 'https://github.com'
for tag in topic_link_tags:
    topic_urls.append(base_url + tag['href'])

print(topic_urls)
!pip install pandas --upgrade --quiet<jupyter_output><empty_output><jupyter_text>Create CSV files with the extracted information<jupyter_code>import pandas as pd
topic_df = pd.DataFrame({
    'topic title': topic_titles,
     'topic description': topic_descs,
      'topic_url': topic_urls
})
topic_df
topic_df.to_csv('topics.csv', index=None)<jupyter_output><empty_output><jupyter_text>Getting information out of a topic page<jupyter_code>topic_page_url = topic_urls[0]
topic_page_url
response = requests.get(topic_page_url)
response.status_code
len(response.text)
topic_doc = BeautifulSoup(response.text, 'html.parser')
h3_selection_class = 'f3 color-fg-muted text-normal lh-condensed'
repo_tags = topic_doc.find_all('h3', {'class': h3_selection_class})
len(repo_tags)<jupyter_output><empty_output><jupyter_text>There are exactly 30 repos on the page. So there are 30 h3 tags, with one tag for each repo.<jupyter_code>a_tags = repo_tags[0].find_all('a') # there are two a tags in first repo tag.
a_tags
a_tags[0].text.strip()
a_tags[1].text.strip()
base_url = 'https://github.com'
repo_url = base_url + a_tags[1]['href']
print(repo_url)
star_tags = topic_doc.find_all('span', {'class': 'Counter js-social-count'})
len(star_tags)
star_tags[0].text.strip()
def parse_star_count(stars_str):
  stars_str = stars_str.strip()
  if stars_str[-1] == 'k':
    return int(float(stars_str[:-1]) * 1000)
parse_star_count(star_tags[0].text.strip())
def get_repo_info(repo_tag, star_tag):
  # returns all the required info about a repo
  a_tags = repo_tag.find_all('a')
  username = a_tags[0].text.strip()
  repo_name = a_tags[1].text.strip()
  repo_url = base_url + a_tags[1]['href']
  stars = parse_star_count(star_tag.text.strip())
  return username, repo_name, stars, repo_url
get_repo_info(repo_tags[0], star_tags[0])
topic_repos_dict = {
    'username': [],
     'repo_name': [],
      'stars': [],
       'repo_url': []
}

for i in range(len(repo_tags)):
  repo_info = get_repo_info(repo_tags[i], star_tags[i])
  topic_repos_dict['username'].append(repo_info[0])
  topic_repos_dict['repo_name'].append(repo_info[1])
  topic_repos_dict['stars'].append(repo_info[2])
  topic_repos_dict['repo_url'].append(repo_info[3])
topic_repos_df = pd.DataFrame(topic_repos_dict)
topic_repos_df
import os
def get_topic_page(topic_url):
  # download the page
  response = requests.get(topic_url)
  # check if the response is successful
  if response.status_code != 200:
    raise Exception('Failed to load page {}'.format(topic_url))
  # parse using BeautifulSoup
  topic_doc = BeautifulSoup(response.text, 'html.parser')

  return topic_doc

def get_repo_info(repo_tag, star_tag):
  # returns all the required info about a repo
  a_tags = repo_tag.find_all('a')
  username = a_tags[0].text.strip()
  repo_name = a_tags[1].text.strip()
  repo_url = base_url + a_tags[1]['href']
  stars = parse_star_count(star_tag.text.strip())
  return username, repo_name, stars, repo_url


def get_topic_repos(topic_doc):
  # get the h3 tags containing repo title, repo url and username
  h3_selection_class = 'f3 color-fg-muted text-normal lh-condensed'
  repo_tags = topic_doc.find_all('h3', {'class': h3_selection_class})
  # get star tags
  star_tags = topic_doc.find_all('span', {'class': 'Counter js-social-count'})
  
  # topic repos dict
  topic_repos_dict = {
    'username': [],
     'repo_name': [],
      'stars': [],
       'repo_url': []
}


  # get repo info
  for i in range(len(repo_tags)):
    repo_info = get_repo_info(repo_tags[i], star_tags[i])
    topic_repos_dict['username'].append(repo_info[0])
    topic_repos_dict['repo_name'].append(repo_info[1])
    topic_repos_dict['stars'].append(repo_info[2])
    topic_repos_dict['repo_url'].append(repo_info[3])

  return pd.DataFrame(topic_repos_dict)

def scrape_topic(topic_url, path):
  if os.path.exists(path):
    print('The file {} already exists. Skipping...'.format(path))
    return
  topic_df = get_topic_repos(get_topic_page(topic_url))
  topic_df.to_csv(path, index=None)
url4 = topic_urls[4]
url4
topic4_doc = get_topic_page(url4)
topic4_repos = get_topic_repos(topic4_doc)
topic4_repos
get_topic_repos(get_topic_page(topic_urls[4])).to_csv('android.csv', index=None)<jupyter_output><empty_output><jupyter_text>Write a single function to:1. Get the list of topics from the topics page2. Get the list of top repos from the individual topic pages3. For each topic create a CSV file of the top repos for the topic<jupyter_code>def get_topic_titles(doc):
  selection_class = 'f3 lh-condensed mb-0 mt-1 Link--primary'
  topic_title_tags = doc.find_all('p', {'class': selection_class})
  topic_titles = []
  for tag in topic_title_tags:
    topic_titles.append(tag.text)
  return topic_titles

def get_topic_descs(doc):
  desc_selector = 'f5 color-fg-muted mb-0 mt-1'
  topic_desc_tags = doc.find_all('p', {'class': desc_selector})
  topic_descs = []
  for tag in topic_desc_tags:
    topic_descs.append(tag.text.strip())
  return topic_descs

def get_topic_urls(doc):
  topic_link_tags = doc.find_all('a', {'class':'no-underline flex-1 d-flex flex-column' }) 
  topic_urls = []
  base_url = 'https://github.com'
  for tag in topic_link_tags:
    topic_urls.append(base_url + tag['href'])
  return topic_urls


def scrape_topics():
  topics_url = 'https://github.com/topics'
  response = requests.get(topics_url)
  if response.status_code != 200:
    raise Exception('Failed to load page {}'.format(topics_url))
  doc = BeautifulSoup(response.text, 'html.parser')
  topics_dict = {
      'title': get_topic_titles(doc),
       'description': get_topic_descs(doc),
        'url': get_topic_urls(doc)
  }
  return pd.DataFrame(topics_dict)
def scrape_topics_repos():
  print('Scraping list of topics')
  topics_df = scrape_topics()

  os.makedirs('data', exist_ok=True)
  for index, row in topics_df.iterrows():
    print('scraping top repositories for "{}"'.format(row['title']))
    scrape_topic(row['url'], 'data/{}.csv'.format(row['title']))
  
topics_df = scrape_topics()
topics_df
scrape_topics_repos()
import os
from google.colab import drive
drive.mount('/content/drive')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12779480195210532, 'fraction_numerical': 0.011689933038247644, 'mean_word_length': 5.635542168674699, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 9, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '280046', 'license': 'MIT', 'n_tokens_mistral': 3200, 'n_tokens_neox': 3107, 'n_words': 692, 'path': 'web-scraping-python.ipynb', 'repo_name': 'vijay98-ops/Web-scraping-python', 'size': '8811'}