<jupyter_start><jupyter_text>Crawledin - Parte 1:Após instalar as bibliotecas necessárias, siga executando uma célula por vez, após anterior finalizar<jupyter_code>import asyncio
from pyppeteer import launch
import pandas as pd
import requests
from json import loads
import time
import random
from datetime import datetime
from tqdm.notebook import tqdm


# !pip install tqdm<jupyter_output><empty_output><jupyter_text>Inicialização do browser para o usuário logarFaça login até chegar na página inicial do seu linkedin<jupyter_code>browser = await launch(
        headless=False,
        args=['--no-sandbox'],
        autoClose=False
    )
page = await browser.newPage()
await page.goto("https://www.linkedin.com/")

# digite se login e senha no browser e siga até estar na sua página inicial<jupyter_output><empty_output><jupyter_text>Vai até a página de seus seguidores para pegar headers da API<jupyter_code>await page.goto("https://www.linkedin.com/feed/followers/")
global headers
headers = {}
await page.setRequestInterception(True)
async def intercept(request):
    if ("https://www.linkedin.com/voyager/api/feed/richRecommendedEntities" in request.url ):
        global headers
        headers = request.headers
        print(len(headers))
    await request.continue_()
page.on('request', lambda req: asyncio.ensure_future(intercept(req)))
await page.evaluate( "() => { window.scrollBy(0, window.innerHeight); }");
time.sleep(5)
cookie1 = await page.cookies()
s = []
for e in cookie1:
    s.append(f"{e['name']}={e['value']}")
cookie1_str = "; ".join(s)
headers = {key: str(value) for key, value in headers.items()}
headers["cookie"] = cookie1_str<jupyter_output>8<jupyter_text>Funções para puxar dados do LinkedIn<jupyter_code>def get_data(countw, index_start, url, headers):
    try:
        resp = requests.get(url=url.format(countw=countw, index_start=index_start), headers=headers)
        if resp.status_code == 200:
            data = loads(resp.content.decode("utf-8"))
            return data
        else:
            raise Exception(f"not allowed. headers or coockies problems. Status: {resp.status_code}")
    except Exception as e:
        print(f"error: {e}")
        return []
    
def agrupa_entidades(elements, data):
    for element in data['data']['elements']:
        for inc in data['included']:
            ek = element.get("*recommendedEntity", "a").split(":member:")[-1]
            ik = inc.get('entityUrn',"b").split(":")[-1]
            if ek == ik:
                e = dict()
                e.update(element)
                e.update(inc)
                elements[ek] = elements.get(ek, {})
                elements[ek].update(e)
    return elements

def dict_element(e):
    return {
        "seguidores": e["followerCount"],
        "influenciador": e["influencer"],
        "nome": f"{e['firstName']} {e['lastName']}",
        "cargo": e["occupation"],
        "link": f'https://www.linkedin.com/in/{e["publicIdentifier"]}',
        "prova_social": e["socialProofTotalCount"],
        "publicacoes": f'https://www.linkedin.com/in/{e["publicIdentifier"]}/detail/recent-activity/shares/',
        "localizacao": e["locationName"]
    }

def dataFrame_element(e):
    return pd.DataFrame.from_dict([dict_element(e)])

                                  
def get_data_in_range(groups_qty, countw, first_index_start, url_inf_scroll, headers, wait_time, wait_random_time):
    df = pd.DataFrame([], columns=["seguidores", "influenciador", "nome", "cargo", "link", "prova_social", "publicacoes", "localizacao"])
    for i in tqdm(range(groups_qty)):
        data1 = get_data(countw, first_index_start + countw*i, url_inf_scroll, headers)
        elements1 = dict()
        agrupa_entidades(elements1, data1)
        time.sleep(wait_time + wait_random_time * random.random())
        for e in elements1.values():
            df_t = dataFrame_element(e)
            df = df.append(df_t, ignore_index=True)
    return df<jupyter_output><empty_output><jupyter_text>Essa etapa executa de fato a consultaNo primeiro parâmetro você passa seu número de seguidores divido por 20. Pode passar menos também, para trazer apenas uma amostra.Os dois ultimos parâmetros são de tempo para esperar entre uma requisição. Se quiser zerar ele fica mais rápido, mas não recomendo.<jupyter_code>countw = 20
index_start = 0
groups_qty = 2750 / 20 # seguidores dividos por 20
url_inf_scroll = "https://www.linkedin.com/voyager/api/feed/richRecommendedEntities?count={countw}&q=followerRecommendations&start={index_start}"

df = get_data_in_range(groups_qty, countw, index_start, url_inf_scroll, headers, .1, .5)
df<jupyter_output><empty_output><jupyter_text>Exibe o data frame ordenado por seguidores<jupyter_code>df.sort_values(by="seguidores", ascending=False).head()<jupyter_output><empty_output><jupyter_text>Salva o dataframe em csv com a data e hora<jupyter_code>df.sort_values(by="seguidores", ascending=False).to_csv(f'crawledin-{datetime.now().strftime("%Y-%m-%d_%H-%M")}.csv', index=False)<jupyter_output><empty_output><jupyter_text>Visita a página de seguidores com conteudo.<jupyter_code>for index, row in df.sample(20).iterrows():
    print(row['link'])
    await page.goto(row['publicacoes'])
    elements = (await page.querySelectorAll("h1"))
    tem_post = True
    for e in elements:
        if "Nada para ver por enquanto" in await page.evaluate('(element) => element.textContent', e):
            tem_post = False
    if tem_post:
        time.sleep(9 + 2 * random.random())
    else:
        time.sleep(1)<jupyter_output>https://www.linkedin.com/in/donizete-crisostomo-4b24a318b
https://www.linkedin.com/in/thais-matos-acacio
https://www.linkedin.com/in/phelipe-salustiano
{'fraction_non_alphanumeric': 0.1294282707821817, 'fraction_numerical': 0.008067344791301298, 'mean_word_length': 3.9419410745233967, 'pattern_counts': {'":': 10, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 9, 'lorem ipsum': 0, 'www.': 9, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '175896', 'license': 'MIT', 'n_tokens_mistral': 1959, 'n_tokens_neox': 1913, 'n_words': 413, 'path': 'CraledinRPA.ipynb', 'repo_name': 'GuiDuque/crawledin', 'size': '5702'}