<jupyter_start><jupyter_text>Scraping the top repositories for Topic on GitHubTODO (Intro) :- Introduction about web scraping: Web scraping, web harvesting, or web data extraction is data scraping used for extracting data from websites. The web scraping software may directly access the World Wide Web using the Hypertext Transfer Protocol or a web browser. -  It is a form of copying in which specific data is gathered and copied from the web, typically into a central local database or spreadsheet, for later retrieval or analysis.- Introduction about GitHub :  GitHub, Inc. is a provider of Internet hosting for software development and version control using Git. It offers the distributed version control and source code management (SCM) functionality of Git, plus its own features. - Tools used (Python,requests,Beautiful Souo,Pandas) Scrap the list of Topics From GitHub- use requests to download the page- use BS4 to parse and extract information- convert to pandas DataFrame Lets write a function to download the page<jupyter_code>from bs4 import BeautifulSoup
import requests
import pandas as pd
import os
def get_topic_page(topic_urls):
   #download the page
  response=requests.get(topic_urls)
  #check sucessfull response
  if response.status_code!=200:
    raise Exception('failed to load page{}'.format(topic_urls))
  #parse using beautiful soup
  topic_doc=BeautifulSoup(response.text,'html.parser')
  return topic_doc
def parse_star_count(stars_str):
  stars_str=stars_str.strip()
  if stars_str[-1]=='k':
    return int(float(stars_str[:-1])*1000)
  return int(stars_str)
def get_repo_info(repo_tags,repo_stars):
  a_tags=repo_tags.find_all('a')
  username=a_tags[0].text.strip()
  repo_name=a_tags[1].text.strip()
  repo_url='https://github.com'+a_tags[1]['href']
  stars=parse_star_count(repo_stars.text.strip())
  return username, repo_name, stars, repo_url

def get_topic_repos(topic_doc):
  #get h3 tag
  h3_selection_class='f3 color-text-secondary text-normal lh-condensed'
  repo_tags=topic_doc.find_all('h3',{'class':h3_selection_class})
  #get star tags
  star="social-count float-none"
  repo_stars=topic_doc.find_all('a',{'class':star})
  repo_topics_dict={
    'username':[],
    'repo_name':[],
    'stars':[],
    'repo_url':[]
  }
  #get repo info
  for i in range(len(repo_tags)):
      repo_info=get_repo_info(repo_tags[i],repo_stars[i])
      repo_topics_dict['username'].append(repo_info[0])
      repo_topics_dict['repo_name'].append(repo_info[1])
      repo_topics_dict['stars'].append(repo_info[2])
      repo_topics_dict['repo_url'].append(repo_info[3])

  return pd.DataFrame(repo_topics_dict)
import os
def scrape_topic(topic_url,path):
  if os.path.exists(path):
    print('The file {} already exists. skipping  ...'.format(path))
    return
  topic_df=get_topic_repos(get_topic_page(topic_url))
  topic_df.to_csv(path,index=None)
def get_topic_titles(doc):
  selection_class='f3 lh-condensed mb-0 mt-1 Link--primary'
  topic_title_tags=doc.find_all('p',{'class':selection_class})
  topic_titles=[]
  for tag in topic_title_tags:
    y=tag.text
    topic_titles.append(y)
  return topic_titles

def get_topic_descs(doc):
  desc_selector='f5 color-text-secondary mb-0 mt-1'
  topic_desc_tags=doc.find_all('p',{'class':desc_selector})
  topic_descs=[]
  for tag in topic_desc_tags:
    y=tag.text.strip()
    topic_descs.append(y)
  return topic_descs

def get_topic_urls(doc):
  topic_links_selector='d-flex no-underline'
  topic_link_tags=doc.find_all('a',{'class':topic_links_selector})
  topic_urls=[]
  base_url='https://github.com'
  for tag in topic_link_tags:
    y=base_url+tag['href']
    topic_urls.append(y)
  return  topic_urls




def scrape_topics():
  topics_url='https://github.com/topics'
  response=requests.get(topics_url)
  if response.status_code!=200:
      raise Exception('failed to load page{}'.format(topic_urls))
  doc=BeautifulSoup(response.text,'html.parser')
  topics_dict={
      'title':get_topic_titles(doc),
      'description':get_topic_descs(doc),
      'url':get_topic_urls(doc)
  }
  return pd.DataFrame(topics_dict)
  
def scrape_topics_repos():
  print('Scraping Top Topics from github')
  topics_df=scrape_topics()
  #create folder
  os.makedirs('data',exist_ok=True)
  for index, row in topics_df.iterrows():
    print('Scraping Top Repositories for "{}"'.format(row['title']))
    scrape_topic(row['url'],'data/{}.csv'.format(row['title']))
scrape_topics_repos()<jupyter_output>Scraping Top Topics from github
Scraping Top Repositories for "3D"
The file data/3D.csv already exists. skipping  ...
Scraping Top Repositories for "Ajax"
The file data/Ajax.csv already exists. skipping  ...
Scraping Top Repositories for "Algorithm"
The file data/Algorithm.csv already exists. skipping  ...
Scraping Top Repositories for "Amp"
The file data/Amp.csv already exists. skipping  ...
Scraping Top Repositories for "Android"
The file data/Android.csv already exists. skipping  ...
Scraping Top Repositories for "Angular"
The file data/Angular.csv already exists. skipping  ...
Scraping Top Repositories for "Ansible"
The file data/Ansible.csv already exists. skipping  ...
Scraping Top Repositories for "API"
The file data/API.csv already exists. skipping  ...
Scraping Top Repositories for "Arduino"
The file data/Arduino.csv already exists. skipping  ...
Scraping Top Repositories for "ASP.NET"
The file data/ASP.NET.csv already exists. skipping  ...
Scraping Top Repositories for "Atom"[...]
{'fraction_non_alphanumeric': 0.11155815654718361, 'fraction_numerical': 0.006217995610826628, 'mean_word_length': 5.083426028921023, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '91762', 'license': 'MIT', 'n_tokens_mistral': 1859, 'n_tokens_neox': 1797, 'n_words': 466, 'path': 'scraping_github_topics_repositories.ipynb', 'repo_name': 'Ankitgupta2/Web-Scraping-Github-Website', 'size': '5468'}