<jupyter_start><jupyter_text>Introdução- https://datageo.ambiente.sp.gov.br/navegar<jupyter_code>#!pip3 install traquitanas --upgrade
import os
import sys
import time
import math
import pandas as pd
from itertools import islice
from traquitanas import net
from selenium.webdriver.common.by import By
sys.path.append(os.path.join(os.getcwd(), '..', 'src'))
from paths import *<jupyter_output><empty_output><jupyter_text>Driver<jupyter_code>driver = net.scraping.create_driver(
    data_path,
    headless=True,
    adds_path=os.path.join('..', 'scrapy', 'adds'),
    log_path=os.path.join('..', 'scrapy', 'logs')    
)<jupyter_output><empty_output><jupyter_text>URL<jupyter_code>url = 'https://datageo.ambiente.sp.gov.br/geoportal/catalog/search/browse/browse.page?'
driver.get(url)<jupyter_output><empty_output><jupyter_text>Assuntos<jupyter_code>def get_subjects():
    # Abre Catálogo
    time.sleep(2)
    try: driver.find_element(by=By.XPATH, value='//*[contains(@class,"dijitTreeExpando dijitTreeExpandoClosed")]').click()
    except: pass

    # Abre Classificação de Metadados
    time.sleep(2)
    try: driver.find_element(by=By.XPATH, value='//*[contains(@class,"dijitTreeExpando dijitTreeExpandoClosed")]').click()
    except: pass

    dict_subjects = {}
    list_tree = driver.find_elements(by=By.XPATH, value='//*[@id="browse-toc"]//*[@class="dijitTreeLabel"]')
    for i in list_tree:
        subject = i.text
        if subject != '' and subject != 'Catálogo' and subject != 'Classificação do Metadado':
            dict_subjects[subject] = i
    return dict_subjects<jupyter_output><empty_output><jupyter_text>Pages<jupyter_code>def get_pages():
    time.sleep(2)
    start_index = int(driver.find_element(by=By.XPATH, value='//*[@id="browse-results-body"]//*[@id="startIndex"]').get_attribute('value'))
    items_per_page = int(driver.find_element(by=By.XPATH, value='//*[@id="browse-results-body"]//*[@id="itemsPerPage"]').get_attribute('value'))
    total_results = int(driver.find_element(by=By.XPATH, value='//*[@id="browse-results-body"]//*[@id="totalResults"]').get_attribute('value'))
    n_pages = math.ceil(total_results/items_per_page)
    dict_results = {
        'start_index': start_index,
        'items_per_page': items_per_page,
        'total_results': total_results,
        'n_pages': n_pages,
    }
    return dict_results<jupyter_output><empty_output><jupyter_text>Get Infos from Itens<jupyter_code>def get_infos_item(item, subject):
    # Título
    titulo = item.find_element(by=By.XPATH, value='.//*[@class="title"]').text

    # Código
    id_lyr = item.find_element(by=By.XPATH, value='.//*[@class="title"]').get_attribute('id')

    # Resumo
    resumo = item.find_element(by=By.XPATH, value='.//*[@class="abstract"]').text

    dict_links = {
            'Assunto': subject,
            'Titulo': titulo,
            'id_lyr': id_lyr,
            'Resumo': resumo,
            'Carregar no Visualizador': '',
            'Metadados': '',
            'XML': '',
    }

    # Links
    list_links = item.find_elements(by=By.XPATH, value='.//*[@class="links"]//*')
    for link in list_links:
        #
        link_text = link.text
        link_url = link.get_attribute('href')

        # Loop    
        if link_text == 'Metadados':        
            # Input dos caminhos para os metadados
            url_api = 'http://datageo.ambiente.sp.gov.br/geoportal/catalog/search/resource/details.page?uuid='
            link_url = '{}{}'.format(url_api, '%7B{}%7D'.format(id_lyr.replace('{', '').replace('}', '')))
        dict_links.update({link_text: link_url})
    return dict_links
#subject = 'Clima'
#get_infos_item(dict_subjects[subject], subject)<jupyter_output><empty_output><jupyter_text>Get All Itens<jupyter_code>def get_all_itens(n_pages, subject):
    # Vai para a página 1
    try: driver.find_element(by=By.XPATH, value='//*[@id="browse-results-header-pageControl"]//*[contains(text(),"Primeiro")]').click()
    except: pass
    try: driver.find_element(by=By.XPATH, value='//*[@id="browse-results-header-pageControl"]//*[contains(text(),"1")]').click()
    except: pass
    time.sleep(3)

    # Para cada página
    list_itens_infos = []
    for i in range(0, n_pages):
        # Clicar no número da página
        item_n = i + 1
        driver.find_element(by=By.XPATH, value='//*[@id="browse-results-header-pageControl"]//*[contains(text(),"{}")]'.format(item_n)).click()
        time.sleep(1)

        # Pega a lista de itens
        list_itens = driver.find_elements(by=By.XPATH, value='//*[@id="browse-results-body"]//*[@class="snippet"]')

        # Pega infos dos itens
        for item in list_itens:
            list_itens_infos.append(get_infos_item(item, subject))
    
    return list_itens_infos<jupyter_output><empty_output><jupyter_text>Run<jupyter_code>dict_subjects = get_subjects()
#dict_subjects = dict(islice(dict_subjects.items(), 2, 4))
dict_subjects
dfs = []
# Para cada Assunto
for subject in dict_subjects:
    # Vai para o Assunto
    dict_subjects[subject].click()

    # Pega informações da Página
    info_page = get_pages()
    n_pages = info_page['n_pages']
    print('Assunto: "{}" tem {} páginas de itens, totalizando {} itens'.format(subject, n_pages, info_page['total_results']))

    # Adiciona Dados
    temp = get_all_itens(n_pages, subject)
    dfs.append(pd.DataFrame(temp))

print('Fim')
# Results
df = pd.concat(dfs)
df.to_csv(os.path.join(output_path_tab, 'tab_itens.csv'), index=False)
df.head()
driver.quit()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16465065896371187, 'fraction_numerical': 0.002527532045495577, 'mean_word_length': 3.9730700179533214, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '346934', 'license': 'MIT', 'n_tokens_mistral': 2052, 'n_tokens_neox': 2013, 'n_words': 294, 'path': 'test/01_get_catalogo.ipynb', 'repo_name': 'michelmetran/geo_SP_DataGeo', 'size': '5539'}