<jupyter_start><jupyter_text>Scraping data relating to different breeds of dogs from American Kennel Club website<jupyter_code>from urllib.request import Request, urlopen
import requests
from bs4 import BeautifulSoup
import csv<jupyter_output><empty_output><jupyter_text>Getting url of individual pages for each breed<jupyter_code>mainsite = "http://www.akc.org/content/news/articles/most-popular-dog-breeds-full-ranking-list/?button"
hdr = {'User-Agent': 'Mozilla/5.0'}
page = requests.get(mainsite)
mainsoup = BeautifulSoup(page.text, "html.parser")
#print(mainsoup.prettify())
allurl = mainsoup.find_all('td')
#allurl
type(allurl)
allurl[0].a['href']
allurl[5].a['href']
len(allurl)
allurl[940].a['href']
940/5
for i in range(0, 5):
#for i in range(0, 189):
    print(allurl[5*i].text.strip())
for i in range(0, 5):
#for i in range(0, 189):
    print(allurl[5*i].a['href'])
name_db = []
url_db = []
for i in range(0, 189):
    name = allurl[5*i].text.strip()
    url = allurl[5*i].a['href']
    name_db.append(name)
    url_db.append(url)
name_db<jupyter_output><empty_output><jupyter_text>Scraping data of a particular breed<jupyter_code>site = "http://www.akc.org/dog-breeds/havanese/"
hdr = {'User-Agent': 'Mozilla/5.0'}
bookpage = requests.get(site)
soup = BeautifulSoup(bookpage.text, "html.parser")
#print(soup.prettify())
soup.find_all('span', class_ = "item")
soup.find_all('span', class_ = "item")[1].get_text()
soup.find('span', class_ = "title").get_text()
soup.find('span', class_ = "energy_levels").get_text().strip()
soup.find('span', class_ = "size").get_text().strip()
soup.find('div', class_ = "bigrank").get_text().strip()
j = soup.find('div', class_ = "breed-details__main")
j
j("li")[2].get_text().strip()
j("li")[3].get_text().strip()
j("li")[10].get_text().strip()
site = "http://www.akc.org/dog-breeds/labrador-retriever/"
hdr = {'User-Agent': 'Mozilla/5.0'}
bookpage = requests.get(site)
soup = BeautifulSoup(bookpage.text, "html.parser")
j = soup.find('div', class_ = "breed-details__main")
j
len(j)<jupyter_output><empty_output><jupyter_text>Putting all together<jupyter_code>import time
data = []
##testing ten pages
for count in range(0,11):
##comment the above and uncomment the below to run all pages
#for count in range(0,189):
    hdr = {'User-Agent': 'Mozilla/5.0'}
    page = requests.get(url_db[count])
    soup = BeautifulSoup(page.text, "html.parser")
    #print(soup.prettify())
    name = name_db[count]
    try:
        grp = soup.find('span', class_ = "title").get_text()
    except: 
        grp = "NA"
    try:
        trait1 = soup.find_all('span', class_ = "item")[0].get_text()
    except: 
        trait1 = "NA"
    try:
        trait2 = soup.find_all('span', class_ = "item")[1].get_text()
    except:
        trait2 = "NA"
    try:
        trait3 = soup.find_all('span', class_ = "item")[2].get_text()
    except:
        trait3 = "NA"
    try:
        energy = soup.find('span', class_ = "energy_levels").get_text().strip()
    except:
        energy = "NA"
    try:
        size = soup.find('span', class_ = "size").get_text().strip()
    except:
        size = "NA"
    try:
        rank = soup.find('div', class_ = "bigrank").get_text().strip()
    except:
        rank = "NA"
    j = soup.find('div', class_ = "breed-details__main")
    try: 
        children = j("li")[2].get_text().strip()
    except:
        children = "NA"
    try:
        otherdogs = j("li")[3].get_text().strip()
    except:
        otherdogs = "NA"
    try:
        shedding = j("li")[4].get_text().strip()
    except:
        shedding = "NA"
    try:
        grooming = j("li")[5].get_text().strip()
    except:
        grooming = "NA"
    try:
        trainability = j("li")[6].get_text().strip()
    except:
        trainability = "NA"
    try:
        height = j("li")[7].get_text().strip()
    except:
        height = "NA"
    try:
        weight = j("li")[8].get_text().strip()
    except:
        weight = "NA"
    try:
        life = j("li")[9].get_text().strip()
    except:
        life = "NA"
    try:
        barking = j("li")[10].get_text().strip()
    except:
        barking = "NA"
    if(barking == "NA"):
        j = str(j) 
    else:
        j = ""
    data.append((name,grp,trait1,trait2,trait3,energy,size,rank,children,otherdogs,shedding,grooming,trainability,
                height,weight,life,barking,j))    
    print(count)
    time.sleep(1)
#data
import pandas as pd
import numpy as np
df = pd.DataFrame(np.array(data))
df.columns = ['name','grp','trait1','trait2','trait3','energy','size','rank','children','otherdogs','shedding','grooming','trainability',
                'height','weight','life','barking','remarks']
df.head()<jupyter_output><empty_output><jupyter_text>Saving output as csv<jupyter_code>df.to_csv('akcdogs.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18502385397220494, 'fraction_numerical': 0.01576436423978428, 'mean_word_length': 3.5706161137440757, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '506918', 'license': 'MIT', 'n_tokens_mistral': 1834, 'n_tokens_neox': 1810, 'n_words': 323, 'path': 'notebooks/AKCDogs.ipynb', 'repo_name': 'alvations/datadoubleconfirm', 'size': '4821'}