<jupyter_start><jupyter_text>Final Project - Web Scraping<NAME>DAT 129 - Python 2 - Spring 2020Final ProjectThis program will scrape yarn.com to find different types of yarn and their prices. It will then rank the yarn by the average most expensive price. It will also output the most common yarn weight for sale. Finally, it will write the yarn dictionaryto a csv file.      Using urllib and BeautifulSoup, this project aims to scrape information from yarn.com to find out which yarn is on average, the most expensive.<jupyter_code>import csv
import numpy as np
import pandas as pd
import matplotlib as mtp
import urllib
from bs4 import BeautifulSoup
# getting html from search results at ravelery.com for both knitting and crochet patterns

def get_search_url(term):
    '''assembles a query against goodreads.com given a search term'''
    url = "https://www.yarn.com/search?&q=%s" % (str(term))
    return url

def getPageText(url):
    '''Given a URL, fetches the raw HTML''' 
    req = urllib.request.Request(url)
    with urllib.request.urlopen(req) as response:
        return response.read()<jupyter_output><empty_output><jupyter_text>A quick glance at yarn.com finds we can search for the below yarn types:* alpaca* bamboo* cashmere* cotton* luxury* polyester* silk* wool  Scrape for each different type of yarn below:<jupyter_code>'''The below code scrapes for polyester based yarns and returns the average price of those yarns'''
pageText = getPageText(get_search_url('polyester'))
'''Scrape for polyester yarn'''
soup = BeautifulSoup(pageText, 'html.parser')
#print(soup)
s = soup.find_all('div','product-prices__price')
poly_count = 0
total_price = 0

for item in s:
    price = item.find('span', None)
    try:
        if price:
            poly_count = poly_count +1 # increase counter
            price_variable = price.text
            price_num = float(price_variable.replace('$','')) # cast to float and replace the $ with an empty string
            total_price = total_price + price_num # add each price together
        else:
            continue
    except ValueError: # account for code scraping an item that cannot be cast to float 
        continue 

poly_average_price = float(total_price)/poly_count        
pretty_poly = round(poly_average_price, 2)
pageText = getPageText(get_search_url('wool'))
'''Scrape for wool yarn'''
soup = BeautifulSoup(pageText, 'html.parser')
s = soup.find_all('div','product-prices__price')
wool_count = 0
tot_price = 0

for item in s:
    price = item.find('span', None)
    try:
        if price:
            wool_count = wool_count +1 # increase counter
            price_variable = price.text
            price_num = float(price_variable.replace('$','')) # cast to float and replace the $ with an empty string
            tot_price = tot_price + price_num
        else:
            continue
    except ValueError: # account for code scraping an item that cannot be cast to float 
        continue

wool_average_price = float(tot_price)/wool_count
pretty_wool = round(wool_average_price, 2) # round to two decimal points 
pageText = getPageText(get_search_url('silk'))
'''Scrape for silk yarn'''
soup = BeautifulSoup(pageText, 'html.parser')
s = soup.find_all('div','product-prices__price')
silk_count = 0
tot_price = 0

for item in s:
    price = item.find('span', None)
    try:
        if price:
            silk_count = silk_count +1 # increase counter
            price_variable = price.text
            price_num = float(price_variable.replace('$','')) # cast to float and replace the $ with an empty string
            tot_price = tot_price + price_num
        else:
            continue
    except ValueError:
        continue

silk_average_price = float(tot_price)/silk_count
pretty_silk = round(silk_average_price, 2) # round to two decimal points 
pageText = getPageText(get_search_url('alpaca'))
'''Scrape for alpaca yarn'''
soup = BeautifulSoup(pageText, 'html.parser')
s = soup.find_all('div','product-prices__price')
alpaca_count = 0
tot_price = 0

for item in s:
    price = item.find('span', None)
    try:
        if price:
            alpaca_count = alpaca_count +1 # increase counter
            price_variable = price.text
            price_num = float(price_variable.replace('$','')) # cast to float and replace the $ with an empty string
            tot_price = tot_price + price_num
        else:
            continue
    except ValueError: # account for code scraping an item that cannot be cast to float 
        continue

alpaca_average_price = float(tot_price)/alpaca_count
pretty_alpaca = round(alpaca_average_price, 2) # round to two decimal points 
pageText = getPageText(get_search_url('bamboo'))
'''Scrape for bamboo yarn'''
soup = BeautifulSoup(pageText, 'html.parser')
#print(soup)
s = soup.find_all('div','product-prices__price')
bamboo_count = 0
total_price = 0

for item in s:
    price = item.find('span', None)
    try:
        if price:
            bamboo_count = bamboo_count +1 # increase counter
            price_variable = price.text
            price_num = float(price_variable.replace('$','')) # cast to float and replace the $ with an empty string
            total_price = total_price + price_num
        else:
            continue
    except ValueError: # account for code scraping an item that cannot be cast to float 
        continue 

bamboo_average_price = float(total_price)/bamboo_count        
pretty_bamboo = round(bamboo_average_price, 2) # round to two decimal points 
pageText = getPageText(get_search_url('cashmere'))
'''Scrape for cahsmere yarn'''
soup = BeautifulSoup(pageText, 'html.parser')
#print(soup)
s = soup.find_all('div','product-prices__price')
cash_count = 0
total_price = 0

for item in s:
    price = item.find('span', None)
    try:
        if price:
            cash_count = cash_count +1 # increase counter
            price_variable = price.text
            price_num = float(price_variable.replace('$','')) # cast to float and replace the $ with an empty string
            total_price = total_price + price_num
        else:
            continue
    except ValueError: # account for code scraping an item that cannot be cast to float 
        continue 
cash_average_price = float(total_price)/cash_count        
pretty_cash = round(cash_average_price, 2) # round to two decimal points 
pageText = getPageText(get_search_url('luxury'))
'''Scrape for luxury yarn'''
soup = BeautifulSoup(pageText, 'html.parser')
#print(soup)
s = soup.find_all('div','product-prices__price')
lux_count = 0
total_price = 0

for item in s:
    price = item.find('span', None)
    try: 
        if price:
            lux_count = lux_count +1 # increase counter
            price_variable = price.text
            price_num = float(price_variable.replace('$','')) # cast to float and replace the $ with an empty string
            total_price = total_price + price_num
        else:
            continue
    except ValueError: # account for code scraping an item that cannot be cast to float 
        continue 
lux_average_price = float(total_price)/lux_count        
pretty_lux = round(lux_average_price, 2)# round to two decimal points 
pageText = getPageText(get_search_url('cotton'))
'''Scrape for cotton yarn'''
soup = BeautifulSoup(pageText, 'html.parser')
s = soup.find_all('div','product-prices__price')
cotton_count = 0
total_price = 0

for item in s:
    price = item.find('span', None)
    try:
        if price:
            cotton_count = cotton_count +1 # increase counter
            price_variable = price.text
            price_num = float(price_variable.replace('$','')) # cast to float and replace the $ with an empty string
            total_price = total_price + price_num
        else:
            continue
    except ValueError: # account for code scraping an item that cannot be cast to float 
        continue 
cotton_average_price = float(total_price)/cotton_count        
pretty_cotton = round(cotton_average_price, 2) # round to two decimal points<jupyter_output><empty_output><jupyter_text>Next, find the most common weight.The different yarn weights are:* lace* fingering* sock * sport* dk* worsted* bulky* super bulkyScrape for the weights below and add to dictionary<jupyter_code>'''This code scrapes yarn.com for yarn types, counts each type, then adds the type to the dictionary'''
pageText = getPageText(get_search_url('yds'))
soup = BeautifulSoup(pageText, 'html.parser')
s = soup.find_all('div','product-summary__attributes')

# initialize counters
lace_count = 0
fingering_count = 0 
sock_count = 0
sport_count = 0
dk_count = 0
worsted_count = 0
bulky_count = 0 
superbulky_count = 0

# add up counts
for item in s:
    weight = item.find('div', None)
    if weight:
        weight_var = weight.text
        if weight_var == "Lace":
            lace_count = lace_count + 1 
        elif weight_var == "Fingering":
            fingering_count = fingering_count + 1
        elif weight_var == "Sock":
            sock_count = sock_count + 1
        elif weight_var == "Sport":
            sport_count = sport_count + 1 
        elif weight_var == "DK":
            dk_count = dk_count + 1 
        elif weight_var == "Worsted":
            worsted_count = worsted_count + 1
        elif weight_var == "Bulky":
            bulky_count = bulky_count + 1
        elif weight_var == "Super Bulky":
            superbulky_count = superbulky_count + 1
    else:
        continue

# put counts in dictionary         
weight_dict = {'Lace': lace_count,'Fingering': fingering_count, 'Sock': sock_count, 'sport': sport_count, 'DK': dk_count, 
              'Worsted': worsted_count, 'Bulky': bulky_count, 'Super Bulky': superbulky_count}<jupyter_output><empty_output><jupyter_text>Create function that will "sort" dictionaries so that the largest value is first<jupyter_code># this section of code taken from Python 1 final 

def sort_key_list(word_dictionary):
    '''This function takes a dictionary and moves the values to a list. From 
    there, it sorts the value list and compares it to the keys in the dictionary. 
    If the item in the list equals the key, it adds it to a new list. The 
    function returns the sorted list'''
    # intialize lists
    sorted_key_list = []
    value_list = []
    for value in word_dictionary:
        temp = word_dictionary[value] # pull out value
        value_list.append(temp) # add value to empty list
    # sort list
    value_list.sort()
    # sort list in descending order
    value_list_sort = value_list[::-1]

    for item in value_list_sort:
        for key in word_dictionary:
            # if key equals item in list and item is not already in sorted_key_list, add it to list
            if word_dictionary[key] == item and key not in sorted_key_list:
                sorted_key_list.append(key)
    return sorted_key_list<jupyter_output><empty_output><jupyter_text>Within main, create yarn dictionary, then sort weight and yarn dictionaries. Return first value in each dictionary.<jupyter_code>def main():
    yarn_dict = {'silk': pretty_silk, 'wool': pretty_wool, 'bamboo': pretty_bamboo, 'polyester': pretty_poly, 'cashmere': pretty_cash,
            'luxury':pretty_lux, 'cotton': pretty_cotton, 'alpaca': pretty_alpaca}
    
    sorted_weight_dict = sort_key_list(weight_dict) # sort the weight dictionary to find most common weight of yarn
    sorted_yarn_dict = sort_key_list(yarn_dict) # sort yarn dictionary to find most expensive yarn on average
    print("On average, which yarn is more expensive?")
    print('')
    print("Average price of silk yarn: $", pretty_silk)
    print("Average price of wool yarn: $", pretty_wool)
    print('Average price of polyester yarn: $', pretty_poly)
    print('Average price of bamboo yarn: $', pretty_bamboo)
    print('Average price of cashmere yarn: $', pretty_cash)
    print('Average price of cotton yarn: $', pretty_cotton)
    print('Average price of luxury yarn: $', pretty_lux)
    print('Average price of alpaca yarn: $', pretty_alpaca)
    print('')

    print('Which yarn is most expensive?')
    print(sorted_yarn_dict[0], 'yarn')
    print('')
    print("Which weight of yarn is most common?")
    print(sorted_weight_dict[0], 'weight')
    
    #for item in sorted_weight_dict:
     #   print(item, ':',weight_dict[item], sep='')
if __name__ == "__main__":
    main()<jupyter_output>On average, which yarn is more expensive?

Average price of silk yarn: $ 8.06
Average price of wool yarn: $ 9.0
Average price of polyester yarn: $ 7.8
Average price of bamboo yarn: $ 10.89
Average price of cashmere yarn: $ 12.51
Average price of cotton yarn: $ 9.21
Average price of luxury yarn: $ 12.08
Average price of alpaca yarn: $ 8.66

Which yarn is most expensive?
cashmere yarn

Which weight of yarn is most common?
DK weight<jupyter_text>Write yarn dictionary to csv file<jupyter_code>yarn_dict = {'silk': pretty_silk, 'wool': pretty_wool, 'bamboo': pretty_bamboo, 'polyester': pretty_poly, 'cashmere': pretty_cash,
            'luxury':pretty_lux, 'cotton': pretty_cotton, 'alpaca': pretty_alpaca}

'''Write yarn dictionary to csv file'''
with open('yarndict.csv', 'w') as y:
    fnames = ['silk', 'wool', 'polyester', 'bamboo', 'cashmere', 'cotton', 'luxury', 'alpaca'] # define header names
    writer = csv.DictWriter(y, fieldnames= fnames) 
    writer.writeheader() # write header row
    writer.writerow(yarn_dict) # pull in yarn dictionary for row. Keys should match header names<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09245381019514728, 'fraction_numerical': 0.006381242116197967, 'mean_word_length': 3.4146740910579756, 'pattern_counts': {'":': 9, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '156782', 'license': 'Apache-2.0', 'n_tokens_mistral': 4288, 'n_tokens_neox': 3993, 'n_words': 1345, 'path': 'FinalProject/FinalProject.ipynb', 'repo_name': 'kaylabracall/dat129_ccac', 'size': '13484'}