<jupyter_start><jupyter_text>Ciclo de recolha de dados<jupyter_code>maindata = pd.DataFrame()

#urllist_test = urllist[23:25]

for i in tqdm(urllist):
    headers = requests.utils.default_headers()
    headers.update({'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0'})
    URL = i
    year = str(re.findall(r'(\d+)(?!.*\d)', URL))
    cal_week = str(re.findall(r'^\D*(\d+)', URL))
    page = requests.get(URL)
    
    soup = BeautifulSoup(page.content, "lxml")
    
    chart_table = [name.text for name in soup.find_all("td", attrs = {'class':'text'})] 
    chart_table  = chart_table[1:] #remove first useless thing
    
    def chunks(lst, n):
        for i in range(0, len(lst), n):
            yield lst[i:i + n]
    
    chart = list(chunks(chart_table, 7))
    #print(chart)
    chart_data = pd.DataFrame(chart)
    chart_data.reset_index(inplace=True)
    
    chart_data["calendar_week"] = cal_week
    chart_data["year"] = year
    maindata = pd.concat([maindata, chart_data])
    time.sleep(random.randint(1,2))
    
print("Já fomos lá buscar tudo")<jupyter_output><empty_output><jupyter_text>Limpar os dados para exportação<jupyter_code>maindata.dtypes
maindata.shape
maindata.rename(columns = {0: "rank", 1:"previous_week", 2: "empty", 3:"artist", 4: "album", 5:"label", 6:"weeks_on_chart"}, inplace=True)
maindata
maindata.drop(['empty', "index"], axis=1, inplace = True)
maindata['previous_week'] = maindata['previous_week'].str.strip("()")
maindata
maindata['calendar_week'] = maindata['calendar_week'].str.strip("['']")
maindata["year"] = maindata["year"].str.strip("['']")
maindata
maindata = maindata.astype({'rank': 'int64', 'weeks_on_chart': 'int64', "calendar_week" : 'int64', "year": 'int64'}).dtypes
maindata.to_csv("charts30-50.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.180931744312026, 'fraction_numerical': 0.0276273022751896, 'mean_word_length': 4.844936708860759, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '314480', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 734, 'n_tokens_neox': 697, 'n_words': 124, 'path': '01_pt-album-charts/recolha-de-dados.ipynb', 'repo_name': 'InterruptorPt/dados', 'size': '1846'}