<jupyter_start><jupyter_text>Libraries<jupyter_code>import requests
import pandas as pd

from datetime import datetime

from bs4 import BeautifulSoup

from sklearn.preprocessing import OrdinalEncoder<jupyter_output><empty_output><jupyter_text>Step 01 - 1.0. Extracting Data<jupyter_code>#creating empty dataframe
df = pd.DataFrame()<jupyter_output><empty_output><jupyter_text>1.1. Classics<jupyter_code>#defining the url
url = 'https://books.toscrape.com/catalogue/category/books/classics_6/index.html'

#defining the header
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
page = requests.get(url, headers=headers)
#defining parser
soup = BeautifulSoup(page.text, 'html.parser')
#selecting start class
books = soup.find('ol', class_='row')<jupyter_output><empty_output><jupyter_text>1.1.1. Names<jupyter_code>#creating name_list
name_list = books.find_all('img', class_='thumbnail')

#books_name
books_name = [p.get('alt') for p in name_list]

#joining to df
df['titles'] = books_name<jupyter_output><empty_output><jupyter_text>1.1.2. Prices<jupyter_code>#creating price_list
price_list = books.find_all('p', class_='price_color')

#books_price
books_price = [p.get_text() for p in price_list]

#joining to df
df['prices'] = books_price<jupyter_output><empty_output><jupyter_text>1.1.3. Customers Avaliation<jupyter_code>products = soup.find('div', class_ = 'col-sm-8 col-md-9')

products_list = products.find_all('p', class_ = 'star-rating')

#Costumers_Avaliable
ca = [p.get('class') for p in products_list]

df['avaliation'] = pd.DataFrame(ca).drop(0, axis=1)<jupyter_output><empty_output><jupyter_text>1.1.4. Stock Availability<jupyter_code>#creating price_list
availability_list = products.find_all('p', class_='instock availability')

#books_price
books_availability = [list(filter(None, p.get_text().split('\n'))) for p in availability_list]

#joining to df
df['availability'] = (pd.DataFrame(books_availability)).drop([0, 2], axis=1)<jupyter_output><empty_output><jupyter_text>1.1.5. Classifying<jupyter_code>#generiting empty dataframe
df1 = pd.DataFrame(columns=['0'])

#copying and filling book_type column
df['book_type'] = df1.copy()
df = df.fillna('classic')<jupyter_output><empty_output><jupyter_text>1.2. Science Fiction<jupyter_code>#defining the url
url = 'https://books.toscrape.com/catalogue/category/books/science-fiction_16/index.html'

#defining the header
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
page = requests.get(url, headers=headers)
#defining parser
soup = BeautifulSoup(page.text, 'html.parser')
#selecting start class
books = soup.find('ol', class_='row')
#creating empty dataframe
df1 = pd.DataFrame()<jupyter_output><empty_output><jupyter_text>1.2.1. Names<jupyter_code>#creating name_list
name_list = books.find_all('img', class_='thumbnail')

#books_name
books_name = [p.get('alt') for p in name_list]

#joining to df
df1['titles'] = books_name<jupyter_output><empty_output><jupyter_text>1.2.2. Prices<jupyter_code>#creating price_list
price_list = books.find_all('p', class_='price_color')

#books_price
books_price = [p.get_text() for p in price_list]

#joining to df
df1['prices'] = books_price<jupyter_output><empty_output><jupyter_text>1.2.3. Customers Avaliation<jupyter_code>products = soup.find('div', class_ = 'col-sm-8 col-md-9')

products_list = products.find_all('p', class_ = 'star-rating')

#Costumers_Avaliable
ca = [p.get('class') for p in products_list]

df1['avaliation'] = pd.DataFrame(ca).drop(0, axis=1)<jupyter_output><empty_output><jupyter_text>1.2.4. Stock Availability<jupyter_code>#creating price_list
availability_list = products.find_all('p', class_='instock availability')

#books_price
books_availability = [list(filter(None, p.get_text().split('\n'))) for p in availability_list]

#joining to df
df1['availability'] = (pd.DataFrame(books_availability)).drop([0, 2], axis=1)<jupyter_output><empty_output><jupyter_text>1.2.5. Classifying<jupyter_code>#generiting empty dataframe
df2 = pd.DataFrame(columns=['book_type'])

#copying and filling book_type column
df1['book_type'] = df2.copy()
df1['book_type'] = df1['book_type'].fillna('science_fiction')

#merging with df
df_final = pd.concat([df, df1], axis=0).reset_index().drop('index', axis=1)<jupyter_output><empty_output><jupyter_text>1.3. Humor<jupyter_code>#defining the url
url = 'https://books.toscrape.com/catalogue/category/books/humor_30/index.html'

#defining the header
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
page = requests.get(url, headers=headers)
#defining parser
soup = BeautifulSoup(page.text, 'html.parser')
#selecting start class
books = soup.find('ol', class_='row')
#creating empty dataframe
df2 = pd.DataFrame()<jupyter_output><empty_output><jupyter_text>1.3.1. Names<jupyter_code>#creating name_list
name_list = books.find_all('img', class_='thumbnail')

#books_name
books_name = [p.get('alt') for p in name_list]

#joining to df
df2['titles'] = books_name<jupyter_output><empty_output><jupyter_text>1.3.2. Prices<jupyter_code>#creating price_list
price_list = books.find_all('p', class_='price_color')

#books_price
books_price = [p.get_text() for p in price_list]

#joining to df
df2['prices'] = books_price<jupyter_output><empty_output><jupyter_text>1.3.3. Customers Avaliation<jupyter_code>products = soup.find('div', class_ = 'col-sm-8 col-md-9')

products_list = products.find_all('p', class_ = 'star-rating')

#Costumers_Avaliable
ca = [p.get('class') for p in products_list]

df2['avaliation'] = pd.DataFrame(ca).drop(0, axis=1)<jupyter_output><empty_output><jupyter_text>1.3.4. Stock Availability<jupyter_code>#creating price_list
availability_list = products.find_all('p', class_='instock availability')

#books_price
books_availability = [list(filter(None, p.get_text().split('\n'))) for p in availability_list]

#joining to df
df2['availability'] = (pd.DataFrame(books_availability)).drop([0, 2], axis=1)<jupyter_output><empty_output><jupyter_text>1.3.5. Classifying<jupyter_code>#generiting empty dataframe
df3 = pd.DataFrame(columns=['book_type'])

#copying and filling book_type column
df2['book_type'] = df3.copy()
df2['book_type'] = df2['book_type'].fillna('humor')

#merging with df
df_final1 = pd.concat([df_final, df2], axis=0).reset_index().drop('index', axis=1)<jupyter_output><empty_output><jupyter_text>1.4. Business<jupyter_code>#defining the url
url = 'https://books.toscrape.com/catalogue/category/books/business_35/index.html'

#defining the header
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
page = requests.get(url, headers=headers)
#defining parser
soup = BeautifulSoup(page.text, 'html.parser')
#selecting start class
books = soup.find('ol', class_='row')
#creating empty dataframe
df3 = pd.DataFrame()<jupyter_output><empty_output><jupyter_text>1.4.1. Name<jupyter_code>#creating name_list
name_list = books.find_all('img', class_='thumbnail')

#books_name
books_name = [p.get('alt') for p in name_list]

#joining to df
df3['titles'] = books_name<jupyter_output><empty_output><jupyter_text>1.4.2. Prices<jupyter_code>#creating price_list
price_list = books.find_all('p', class_='price_color')

#books_price
books_price = [p.get_text() for p in price_list]

#joining to df
df3['prices'] = books_price<jupyter_output><empty_output><jupyter_text>1.4.3. Customers Avaliation<jupyter_code>products = soup.find('div', class_ = 'col-sm-8 col-md-9')

products_list = products.find_all('p', class_ = 'star-rating')

#Costumers_Avaliable
ca = [p.get('class') for p in products_list]

df3['avaliation'] = pd.DataFrame(ca).drop(0, axis=1)<jupyter_output><empty_output><jupyter_text>1.4.4. Stock Availability<jupyter_code>#creating price_list
availability_list = products.find_all('p', class_='instock availability')

#books_price
books_availability = [list(filter(None, p.get_text().split('\n'))) for p in availability_list]

#joining to df
df3['availability'] = (pd.DataFrame(books_availability)).drop([0, 2], axis=1)<jupyter_output><empty_output><jupyter_text>1.4.5. Classifying<jupyter_code>#generiting empty dataframe
df4 = pd.DataFrame(columns=['book_type'])

#copying and filling book_type column
df3['book_type'] = df4.copy()
df3['book_type'] = df3['book_type'].fillna('business')

#merging with df
df_final2 = pd.concat([df_final1, df3], axis=0).reset_index().drop('index', axis=1)<jupyter_output><empty_output><jupyter_text>Step 02 - 2.0. Delivarables  2.1. Links of Extractions These links was used to the data extractions:classics: https://books.toscrape.com/catalogue/category/books/classics_6/index.htmlScience Fiction: https://books.toscrape.com/catalogue/category/books/science-fiction_16/index.htmlHumor: https://books.toscrape.com/catalogue/category/books/humor_30/index.htmlBusiness: https://books.toscrape.com/catalogue/category/books/business_35/index.html  2.1. Saving to CSV<jupyter_code># saving to csv
df_final2.to_csv('df_books_extracted.csv', index=False)<jupyter_output><empty_output><jupyter_text>Step 3.0. Data Cleaning<jupyter_code>#loading data
df = pd.read_csv('df_books_extracted.csv')

#changing titles to snakecase
df['titles'] = df['titles'].apply(lambda x: x.replace(' ', '_').lower())

#changing availability to snakecase
df['availability'] = df['availability'].apply(lambda x: x.strip().replace(' ', '_').lower())

#changing avaliation strings to intergers
enc_dict = {'One':1, 'Two':2, 'Three':3, 'Four':4, 'Five':5}
df['avaliation'] = df['avaliation'].map(enc_dict)

#cleaning prices
df['prices'] = df['prices'].apply(lambda x: x.replace('Â£', ''))

#creating scrapy time column
df['scrapy_datetime'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
df['scrapy_datetime'] = pd.to_datetime(df['scrapy_datetime'])
df.to_csv('df_cleaned', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15644171779141106, 'fraction_numerical': 0.02810211755392836, 'mean_word_length': 7.609028960817717, 'pattern_counts': {'":': 0, '<': 113, '<?xml version=': 0, '>': 113, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '28', 'dir': 'jupyter-structured-clean-dedup', 'id': '566722', 'license': 'Apache-2.0', 'n_tokens_mistral': 3971, 'n_tokens_neox': 3818, 'n_words': 657, 'path': '.ipynb_checkpoints/exerc_02_datacleaning-checkpoint.ipynb', 'repo_name': 'Valcilio/star-jeans-engineer', 'size': '10106'}