<jupyter_start><jupyter_text>Get text for each species profile  Google Drive<jupyter_code>from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from pydrive.auth import ServiceAccountCredentials
credentialsFile = './credentials.json'
scope = ['https://www.googleapis.com/auth/drive']
gauth = GoogleAuth()
gauth.credentials = ServiceAccountCredentials.from_json_keyfile_name(credentialsFile,scope)
drive = GoogleDrive(gauth)
PROJECT_ID = '0B0M5IL0AEOXidHZmTjZzMHlLLWc'
PROFILES_ROOT_ID = drive.ListFile({'q': f"title contains 'Perfil' and '{PROJECT_ID}' in parents"}).GetList()[0]['id']
def getFamiliesFoldersIds():
    return {  f['title']:f['id'] 
              for f in drive.ListFile({'q': f"'{PROFILES_ROOT_ID}' in parents"}).GetList()
              if re.match('^[a-zA-Z]*ae$',f['title'])  }


def getFamilyDocuments(family, family_folder_id):
    return drive.ListFile({'q':f"'{family_folder_id}' in parents and mimeType contains 'document'"}).GetList() 

# Fetches all documents for each family

def fetchProfilesDocuments():
    
    os.makedirs('./tmp/especies')
    os.makedirs('./tmp/familias')
    for family,folder_id in getFamiliesFoldersIds().items():
        print(f"Fetching documents for family {family}")
        for document in getFamilyDocuments(family,folder_id):
            print(f"\tFetching document: {document['title']}")
            if document['title']=='familia':
                document.GetContentFile(f"./tmp/familias/{family.lower()}.md", mimetype="text/plain")
            else:
                document.GetContentFile(f"./tmp/especies/{document['title']}.md", mimetype="text/plain")
fetchProfilesDocuments()<jupyter_output>Fetching documents for family odontophrynidae
	Fetching document: familia
	Fetching document: proceratophrys-rotundipalpebra
	Fetching document: odontophrynus-salvatori
	Fetching document: odontophrynus-cultripes
	Fetching document: proceratophrys-goyana
Fetching documents for family microhylidae
	Fetching document: familia
	Fetching document: elachistocleis-cesarii
Fetching documents for family leptodactylidae
	Fetching document: leptodactylus-latrans
	Fetching document: leptodactylus-fuscus
	Fetching document: leptodactylus-furnarius
	Fetching document: leptodactylus-mystacinus
	Fetching document: leptodactylus-mystaceus
	Fetching document: familia
	Fetching document: pseudopaludicola-giarettai
	Fetching document: physalaemus-marmoratus
	Fetching document: leptodactylus-podicipinus
	Fetching document: leptodactylus-chaquensis
	Fetching document: adenomera-juikitam
	Fetching document: adenomera-cotuba
	Fetching document: pseudopaludicola-ternetzi
	Fetching document: pseudopaludicola-s[...]<jupyter_text>Species profile text<jupyter_code>def getSpeciesProfileStructure(txt):
    main_desc = re.findall('^([^#]*)',txt,flags=re.DOTALL)[0].strip()
    named_sections = [ (n.strip(),d.strip()) for n,d in re.findall('#+([\w\s]{1,30})\n([^#]+)',txt,flags=re.DOTALL)]
    structured_text = namedtuple('ProfileText',['main_desc','named_sections'])
    return structured_text(main_desc,named_sections)
    return namedtuple('ProfileText', ['main_desc','named_sections'],main_desc,named_sections)

def profileText_to_markdown(profile_txt_str):
    
    resText = ''
    resText += profile_txt_str.main_desc
    
    for name,content in profile_txt_str.named_sections:
        if name.lower()=='curiosidades':
            resText += f'''
<div class="col-lg-12">
  <div class="jumbotron">
    <h1 class="display-4">Você sabia?</h1>
    <hr class="my-4">
    <p>{markdown.markdown(content)}</p>
  </div>
</div>
'''
        
        else:
            resText += f'\n#### {name}\n{content}'
            
    return resText
import glob
for spfilePath in glob.glob('./tmp/especies/*'):
    species_slug = re.findall('([\w-]*).md$',spfilePath)[0]
    
    with open(spfilePath) as f:
        txt = f.read()
        profile_data = getSpeciesProfileStructure(txt)

    with open(f'../especies/{species_slug}.md','w') as f:
        f.write(profileText_to_markdown(profile_data))
    

species_slug = 'bokermannohyla-pseudopseudis'
text_file = f'../especies/{species_slug}.md'

with open(text_file) as f:
    txt = f.read()
    profile_data = getSpeciesProfileStructure(txt)

with open(f'../especies/{species_slug}-1.md','w') as f:
    f.write(profileText_to_markdown(profile_data))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1329532497149373, 'fraction_numerical': 0.003876852907639681, 'mean_word_length': 4.725848563968668, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '138145', 'license': 'MIT', 'n_tokens_mistral': 1522, 'n_tokens_neox': 1504, 'n_words': 262, 'path': 'build_species_dset/text_preprocessing.ipynb', 'repo_name': 'pedrosiracusa/guia_anuros_chapada', 'size': '4385'}