<jupyter_start><jupyter_text>Getting names by genre<jupyter_code>import time
import requests
import pandas as pd
from bs4 import BeautifulSoup
from threading import Thread
base = 'csv/'
# url : url of a page
# returns: content of page requested (html)
def requestHTMLPage(url):
    r = requests.get(url)
    while r.status_code != 200:
        time.sleep(10)
        r = requests.get(url)
        print("Loop em potencial")
    return r.content
# urls where there are female names and male names
baseUrlFemale = "https://www.dicionariodenomesproprios.com.br/nomes-femininos/"
maxNumberFemale = 151 # total of pages with female names
baseUrlMale = "https://www.dicionariodenomesproprios.com.br/nomes-masculinos/"
maxNumberMale = 193 # total of pages with male names
# maxNumber : total of pages will be accessed to get names
# urlBase : where the names were found
# returns : list of string with names
def getNames(maxNumber, urlBase):
    namesByGenre = []
    for i in range(1, maxNumber + 1):
        c = requestHTMLPage(urlBase + str(i))
        soup = BeautifulSoup(c, "html.parser")
        names = soup.find_all('a', class_="lista-nome")
        for n in names:
            namesByGenre.append(n.getText().strip())
        print("Current page " + str(i))
    return namesByGenre
# saves in fineName the given names
def saveCSV(fileName, names):
    file = open(base + fileName, 'w')
    for n in names:
        file.write(n + '\n')
    file.close()
# get female names
female = getNames(maxNumberFemale, baseUrlFemale)
female = sorted(female)
saveCSV("femaleNames.csv", female)
# get male names
male = getNames(maxNumberMale, baseUrlMale)
male = sorted(male)
saveCSV("maleNames.csv", male)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10556844547563805, 'fraction_numerical': 0.008120649651972157, 'mean_word_length': 3.8050139275766015, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '206124', 'license': 'MIT', 'n_tokens_mistral': 570, 'n_tokens_neox': 540, 'n_words': 167, 'path': 'NamesScrapingByGender.ipynb', 'repo_name': 'pentalpha/ufrn-transparency-remunerations', 'size': '1724'}