<jupyter_start><jupyter_text>Election expensesRecently we found a congressperson who made an allegedly postal service expense in a company created for his own benefit, for his next candidacy in elections.We believe there are more cases like this one.<jupyter_code>import pandas as pd
import numpy as np

reimbursements = pd.read_csv('../data/2016-12-06-reimbursements.xz',
                             dtype={'cnpj_cpf': np.str},
                             low_memory=False)
companies = pd.read_csv('../data/2016-09-03-companies.xz', low_memory=False)
companies['cnpj'] = companies['cnpj'].str.replace(r'\D', '')
dataset = pd.merge(reimbursements, companies,
                   left_on='cnpj_cpf', right_on='cnpj')
is_election_company = \
    dataset['legal_entity'] == '409-0 - CANDIDATO A CARGO POLITICO ELETIVO'
suspect = dataset[is_election_company]
suspect.shape
suspect['total_net_value'].sum()
suspect['total_net_value'].describe()
suspect['name'].sample(10)
dataset['name'] = dataset['name'].astype(np.str)
contains_election_str = dataset['name'].str.lower().str.contains(r'(eleic)[(ao)(oes)]')
company_name_suspects = dataset[contains_election_str].index
np.array_equal(suspect.index, company_name_suspects)
import unicodedata

def normalize_string(string):
    if isinstance(string, str):
        nfkd_form = unicodedata.normalize('NFKD', string.lower())
        return nfkd_form.encode('ASCII', 'ignore').decode('utf-8')
suspect['congressperson_name'] = \
    suspect['congressperson_name'].apply(normalize_string)

suspect['name'] = suspect['name'].apply(normalize_string)
suspect[suspect.apply(lambda row: row['congressperson_name'] in row['name'], axis=1)]
suspect[['congressperson_name', 'name']]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14228341939185313, 'fraction_numerical': 0.013769363166953529, 'mean_word_length': 5.250896057347671, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '311457', 'license': 'MIT', 'n_tokens_mistral': 612, 'n_tokens_neox': 567, 'n_words': 104, 'path': 'research/develop/2016-12-07-irio-election-expenses.ipynb', 'repo_name': 'richardljs/serenata-de-amor', 'size': '1743'}