<jupyter_start><jupyter_text>PIPELINE<jupyter_code>- Imports
- Load Dataframes

- Analaysis

01: nº framing per month<jupyter_output><empty_output><jupyter_text>Imports<jupyter_code>import pandas as pd
import numpy as np
import dask
from dask import dataframe as dd
import os
from typing import Union<jupyter_output><empty_output><jupyter_text>Load Dataframes<jupyter_code>def load_emergency_aid(blocksize: Union[str, float] = 25e6) -> dask.dataframe.core.DataFrame:
    """Return a dask dataframe

    blocksize: num bytes of each parallel dataframe chunk. egg:'25MB' or 25e6 """

    path_data_09_csvs = os.path.join(os.getcwd(), '..', 'sprint_01_data_collection', 'data_09', '*.csv')

    return dd.read_csv(path_data_09_csvs, sep=";", encoding="ISO-8859-1", blocksize=blocksize, dtype={
        'CPF RESPONSÁVEL': 'str',
        'CÓDIGO MUNICÍPIO IBGE': 'str',
        'NOME MUNICÍPIO': 'str', 
        'UF': 'str', 
        'NIS BENEFICIÁRIO': 'str', 
        'CPF BENEFICIÁRIO': 'str', 
        'NOME BENEFICIÁRIO': 'str', 
        'NIS RESPONSÁVEL': 'str', 
        'CPF RESPONSÁVEL': 'str', 
        'NOME RESPONSÁVEL': 'str', 
        'ENQUADRAMENTO': 'str', 
        'PARCELA': 'str', 
        'OBSERVAÇÃO': 'str', 
        'VALOR BENEFÍCIO': 'str'})
df_emergencial_aid = load_emergency_aid()
df_emergencial_aid.head()
# df_emergencial_aid.tail()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1378066378066378, 'fraction_numerical': 0.016594516594516596, 'mean_word_length': 3.9892086330935252, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '343477', 'license': 'MIT', 'n_tokens_mistral': 580, 'n_tokens_neox': 556, 'n_words': 106, 'path': 'sprint_03_data_analysis/data_analysis_data_09_emergencial_aid.ipynb', 'repo_name': 'lgdc-ufpa/covid-19-and-malaria', 'size': '1386'}