<jupyter_start><jupyter_text>Insights BeautifulSoup<jupyter_code>#__version__:0.0.1
#__release__: ??
#UNESP-FEIS
#@Arthur_Chabole
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import requests
from urllib.request import urlopen
from bs4 import BeautifulSoup<jupyter_output><empty_output><jupyter_text>Abrindo a URL com request<jupyter_code># 1º modo
request = requests.get('https://lista.mercadolivre.com.br/carros#D[A:carros]')
request.encoding = 'utf-8'

#Criando o objeto soup
soup = BeautifulSoup(request.text, 'html.parser')
# 2º modo
#response = urlopen('https://lista.mercadolivre.com.br/carros#D[A:carros]')
#html = response.read()

html = urlopen('https://lista.mercadolivre.com.br/carros#D[A:carros]').read()

soup = BeautifulSoup(html, 'html.parser')
#print(soup.prettify())<jupyter_output><empty_output><jupyter_text>Pegando um elemento específico da página<jupyter_code>all_price = soup.find_all(class_='price-tag-fraction') #mostra os código html

#all_price
#convertendo em texto
prices = []
for price in all_price:
    prices.append(float(price.get_text()))
#Construindo um dataframe com 1 coluna
price_df = pd.DataFrame()

price_df['Preços'] = prices<jupyter_output><empty_output><jupyter_text>Construindo um DataFrame com a página<jupyter_code>#Pega todas as classes com esse nome
blocos = soup.find_all(class_='ui-search-layout ui-search-layout--grid')
tabela = []
for bloco in blocos:
    tabela.append({
        'Modelo':(bloco.find(class_="ui-search-item__title ui-search-item__group__element")).text,
        'Preços':float((bloco.find(class_="price-tag-fraction")).text),
        'Local':(bloco.find(class_="ui-search-item__group__element ui-search-item__location")).text,
        'Ano':float((bloco.find_all(class_="ui-search-card-attributes__attribute")[0]).text),
        'Km':((bloco.find_all(class_="ui-search-card-attributes__attribute")[1]).text) #Problema com isso
    })
df = pd.DataFrame(data=tabela)
df.head(10)
print(df['Preços'].mean(), df['Preços'].std())<jupyter_output>89.957375 61.65913283623657<jupyter_text>Carregando as informações de todas as páginas<jupyter_code>#Encontarndos todos os links 
links = soup.find(class_="ui-search-pagination andes-pagination")

#links
'''Testar funções lambda'''
#separando os links em texto pela tag['href']
all_links = []
for link in links:
    all_links.append(link.find(class_='andes-pagination__link ui-search-link')['href'])
    
all_links
tabela = []
for link in all_links:
    response = (urlopen(link))
    html = response.read()
    
    soup = BeautifulSoup(html, 'html.parser') #Inst objeto 
    
    blocos = soup.find_all(class_='ui-search-layout ui-search-layout--grid') #Pegando classes específicas
    
    for bloco in blocos:
        tabela.append({
            'Modelo':(bloco.find(class_="ui-search-item__title ui-search-item__group__element")).text,
            'Preços':float((bloco.find(class_="price-tag-fraction")).text),
            'Local':(bloco.find(class_="ui-search-item__group__element ui-search-item__location")).text,
            'Ano':float((bloco.find_all(class_="ui-search-card-attributes__attribute")[0]).text),
            'Km':((bloco.find_all(class_="ui-search-card-attributes__attribute")[1]).text) #Problema com isso
        })<jupyter_output><empty_output><jupyter_text>Criando o DataFrame com o dicionário<jupyter_code>pag_df = pd.DataFrame(data=tabela)
#Salvando o DataFrame em um arquivo excel
pag_df.to_excel("D:/Arquivos Perssoais/PythonProjects/Projetos de automação/Mercado_livre.xlsx",sheet_name='Tabela')<jupyter_output><empty_output><jupyter_text>'''Gravando em mais de uma planilha no mesmo arquivo'''df2 = df1.copy()with pd.ExcelWriter('output.xlsx') as writer:      df1.to_excel(writer, sheet_name='Sheet_name_1')    df2.to_excel(writer, sheet_name='Sheet_name_2')<jupyter_code>pag_df.tail()
pag_df.describe()
pag_df.info()
sns.barplot(x='Local', y='Preços', data=pag_df)
sns.pairplot(pag_df, diag_kind='kde')
sns.pairplot(pag_df, kind='kde', diag_kind='kde')
sns.pairplot(pag_df, kind='scatter', diag_kind='kde', hue='Local')<jupyter_output>C:\Users\arthu\Anaconda3\lib\site-packages\seaborn\distributions.py:305: UserWarning: Dataset has 0 variance; skipping density estimate.
  warnings.warn(msg, UserWarning)
C:\Users\arthu\Anaconda3\lib\site-packages\seaborn\distributions.py:305: UserWarning: Dataset has 0 variance; skipping density estimate.
  warnings.warn(msg, UserWarning)
C:\Users\arthu\Anaconda3\lib\site-packages\seaborn\distributions.py:305: UserWarning: Dataset has 0 variance; skipping density estimate.
  warnings.warn(msg, UserWarning)
C:\Users\arthu\Anaconda3\lib\site-packages\seaborn\distributions.py:305: UserWarning: Dataset has 0 variance; skipping density estimate.
  warnings.warn(msg, UserWarning)
{'fraction_non_alphanumeric': 0.1429165799875182, 'fraction_numerical': 0.013313917204077387, 'mean_word_length': 5.917985611510791, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '529950', 'license': 'MIT', 'n_tokens_mistral': 1824, 'n_tokens_neox': 1771, 'n_words': 283, 'path': 'Scraping_Websites/Scraping e DataFrame - Tutorial.ipynb', 'repo_name': 'Gabriel-Messia/Python', 'size': '4807'}