<commit_before>import os

import numpy as np
import pandas as pd

DATA_PATH = 'data'


class Suspicions:
    """
    Load suspicious reimbursements.
    """

    SOCIAL_ACCOUNTS_FILE = '2017-06-11-congresspeople-social-accounts.xz'

    def __init__(self, data_path=DATA_PATH):
        self.data_path = data_path

    def all(self):
        dataset = self.__reimbursements()
        dataset = dataset.merge(self.__companies(),
                                how='left',
                                left_on='cnpj_cpf',
                                right_on='cnpj')
        dataset = dataset.merge(self.__suspicions())
        dataset = dataset.merge(self.__twitter_profiles(),
                                on='congressperson_id')
        # rows = dataset.iloc[:, -6:].any(axis=1) \
        #     & dataset.congressperson_id.notnull()
        suspicious_cols = ['meal_price_outlier',
                           'suspicious_traveled_speed_day']
        rows = dataset.loc[:, suspicious_cols].any(axis=1) \
            & dataset.congressperson_id.notnull()
        return dataset[rows].sort_values('total_net_value', ascending=False)

    def __reimbursements(self):
        dataset = pd.read_csv(os.path.join(self.data_path, 'reimbursements.xz'),
                              dtype={'applicant_id': np.str,
                                     'cnpj_cpf': np.str,
                                     'congressperson_id': np.str,
                                     'subquota_number': np.str},
                              low_memory=False)
        dataset['issue_date'] = pd.to_datetime(
            dataset['issue_date'], errors='coerce')
        dataset = dataset.query('year >= 2016')
        # dataset = dataset.query('term == 2015')
        return dataset

    def __companies(self):
        path = os.path.join(self.data_path, '2016-09-03-companies.xz')
        dataset = pd.read_csv(path, dtype={'cnpj': np.str}, low_memory=False)
        dataset['cnpj'] = dataset['cnpj'].str.replace(r'\D', '')
        dataset['situation_date'] = pd.to_datetime(
            dataset['situation_date'], errors='coerce')
        return dataset

    def __suspicions(self):
        return pd.read_csv(os.path.join(self.data_path, 'suspicions.xz'),
                           dtype={'applicant_id': np.str})

    def __twitter_profiles(self):
        path = os.path.join(self.data_path, self.SOCIAL_ACCOUNTS_FILE)
        dataset = pd.read_csv(path, dtype={'congressperson_id': np.str})
        cols = ['twitter_profile', 'secondary_twitter_profile']
        return dataset[dataset[cols].any(axis=1)]<commit_msg>Create method to fetch dataset under Suspicions class<commit_after>import os

import numpy as np
import pandas as pd

from serenata_toolbox import datasets

DATA_PATH = 'data'


class Suspicions:
    """
    Load suspicious reimbursements.
    """

    COMPANIES_FILE = '2016-09-03-companies.xz'
    CONGRESSPEOPLE_FILE = '2017-05-29-deputies.xz'
    SOCIAL_ACCOUNTS_FILE = '2017-06-11-congresspeople-social-accounts.xz'

    def __init__(self, data_path=DATA_PATH):
        self.data_path = data_path

    def fetch(self):
        datasets.fetch(self.COMPANIES_FILE, self.data_path)
        datasets.fetch(self.CONGRESSPEOPLE_FILE, self.data_path)
        datasets.fetch(self.SOCIAL_ACCOUNTS_FILE, self.data_path)

    def all(self):
        dataset = self.__reimbursements()
        dataset = dataset.merge(self.__companies(),
                                how='left',
                                left_on='cnpj_cpf',
                                right_on='cnpj')
        dataset = dataset.merge(self.__suspicions())
        dataset = dataset.merge(self.__social_accounts(),
                                how='left',
                                on='congressperson_id')
        dataset = dataset.merge(self.__congresspeople(),
                                how='left',
                                on='congressperson_id')
        # rows = dataset.iloc[:, -6:].any(axis=1) \
        #     & dataset.congressperson_id.notnull()
        suspicious_cols = ['meal_price_outlier',
                           'suspicious_traveled_speed_day']
        rows = dataset.loc[:, suspicious_cols].any(axis=1) \
            & dataset.congressperson_id.notnull()
        return dataset[rows].sort_values('total_net_value', ascending=False)

    def __reimbursements(self):
        dataset = pd.read_csv(os.path.join(self.data_path, 'reimbursements.xz'),
                              dtype={'applicant_id': np.str,
                                     'cnpj_cpf': np.str,
                                     'congressperson_id': np.str,
                                     'subquota_number': np.str},
                              low_memory=False)
        dataset['issue_date'] = pd.to_datetime(
            dataset['issue_date'], errors='coerce')
        dataset = dataset.query('year >= 2016')
        # dataset = dataset.query('term == 2015')
        return dataset

    def __companies(self):
        path = os.path.join(self.data_path, self.COMPANIES_FILE)
        dataset = pd.read_csv(path, dtype={'cnpj': np.str}, low_memory=False)
        dataset['cnpj'] = dataset['cnpj'].str.replace(r'\D', '')
        dataset['situation_date'] = pd.to_datetime(
            dataset['situation_date'], errors='coerce')
        return dataset

    def __suspicions(self):
        return pd.read_csv(os.path.join(self.data_path, 'suspicions.xz'),
                           dtype={'applicant_id': np.str})

    def __social_accounts(self):
        path = os.path.join(self.data_path, self.SOCIAL_ACCOUNTS_FILE)
        return pd.read_csv(path, dtype={'congressperson_id': np.str})

    def __congresspeople(self):
        path = os.path.join(self.data_path, self.CONGRESSPEOPLE_FILE)
        return pd.read_csv(path, dtype={'congressperson_id': np.str})
{'fraction_non_alphanumeric': 0.1210984137813406, 'fraction_numerical': 0.010745352208766844, 'mean_word_length': 2.055758207399687, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6172742', 'n_tokens_mistral': 1862, 'n_tokens_neox': 1732, 'n_words': 265}